diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index e283e35b62..250503e471 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -91,9 +91,15 @@ explicit path or endpoint and do not need it. - ASAR or extracted JavaScript/Electron tree: `analyze_javascript_application`. -- Archive/package member inventory (ZIP/APK/IPA/MSIX/AppX or DMG): - `open_binary` with the supplied local path; use `inspect_artifact` when its - graph and findings help answer the question. +- Archive/package member inventory (ZIP/APK/IPA/MSIX/AppX, DMG, or a PyInstaller + executable): `open_binary` with the supplied local path; use `inspect_artifact` + when its graph and findings help answer the question. A PyInstaller CArchive + is the application layer; the bootloader is only the native prefix. Start with + `detail: "summary"` for large inventories, then use retained inventory views + for selected member fields/pages. A complete small inventory can cost less + than extra calls and schemas. Runtime options are metadata, not files. + Compact delivery still inventories and hashes members. See the + [Python packaged applications guide](https://github.com/morluto/rea/blob/main/docs/python-packaged-applications.md). - Apple app plist, Interface Builder, asset catalog, or keyed archive resources: `open_bundle_resources`, then the selected resource tool. This does not require `CFBundleExecutable` or a deep provider. Resource sessions have no @@ -201,6 +207,12 @@ explicit path or endpoint and do not need it. package order, dependency indices, and the section file range. A missing `.dep-v0` section is null metadata, not proof the binary has no dependencies. A rustc commit hash is an inference from a read-only `/rustc/<40-hex>/` path. + For selected crates, use `view: {collection: "packages", names: [...], + fields: ["name", "version", "source"]}` on the first call; explicitly include + `dependencies` for direct index references. Use `detail: "summary"` for counts + and ranges, or retained `inspect_analysis_view` Rust package/edge pages for + subsequent filters. Keep the complete parent Evidence accessible; CLI compact + delivery requires `--evidence_output`. See the [Rust binary guide](https://github.com/morluto/rea/blob/main/docs/rust-binaries.md). diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 235092e96f..a02b8463c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -409,7 +409,9 @@ jobs: with: save: ${{ github.event_name != 'pull_request' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} - run: npm run test:prepare - - name: Find available trusted full-suite timing history + # Fast and full shards measure different file costs. Download only the + # matching history so one tier cannot pack the other with its weights. + - name: Find available trusted timing history id: history continue-on-error: true env: @@ -417,13 +419,14 @@ jobs: CI_REPOSITORY: ${{ github.repository }} CI_REPOSITORY_ID: ${{ github.repository_id }} CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + CI_TIMING_TIER: ${{ needs.changes.outputs.tier }} run: node scripts/ci/test-history.mjs - name: Download previous timing report if: steps.history.outputs.run-id != '' continue-on-error: true uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: - name: test-timings + name: ${{ steps.history.outputs.artifact-name }} run-id: ${{ steps.history.outputs.run-id }} github-token: ${{ github.token }} path: .cache/ci-test-history @@ -555,11 +558,22 @@ jobs: include-hidden-files: true if-no-files-found: error retention-days: 30 + # Default-branch runs only. Pull-request artifacts share this name and + # the lookup reads one page, so publishing them would hide main. - name: Publish successful timing history - if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'full' && success() && needs.test-shard.result == 'success' + if: >- + success() && + needs.changes.outputs.code == 'true' && + needs.test-shard.result == 'success' && + ( + needs.changes.outputs.tier == 'full' || + (needs.changes.outputs.tier == 'fast' && needs.test-source.result == 'success') + ) && + github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && + (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: test-timings + name: test-timings-${{ needs.changes.outputs.tier }} path: .cache/ci-test-history/test-results.json include-hidden-files: true if-no-files-found: error diff --git a/README.md b/README.md index a4da4bb818..ff4777a55b 100644 --- a/README.md +++ b/README.md @@ -174,35 +174,36 @@ CLI commands use the same workflows. REA requires Node.js 22.x (>=22.19), 24.x (>=24.11), or 26+, plus npm. Additional tools and host support depend on the target: -| Target | What REA returns | Requirements and guide | -| ---------------------- | ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -| Desktop installations | Runtime candidates, supporting files, component markers and next probes | Built-in bounded directory/header inspection; [desktop applications](docs/desktop-applications.md) | -| Native binaries | Pseudocode, assembly, strings, symbols, calls and references | Hopper, Ghidra or IDA; [native analysis](docs/native-investigation.md) | -| Offline ELF layout | Sections, segments, original symbols/relocations and static mitigation candidates | Caller-supplied pwntools on Linux x64; [binary diagnostics](docs/binary-diagnostics.md) | -| Go binaries | Embedded compiler version, modules, replacements and build settings | Built-in static ELF/PE/thin Mach-O inspection; no Go installation required | -| Model artifacts | GGUF / safetensors metadata, tensor declarations and derived totals | Built-in header inspection; optional full-file hash; [model files](docs/model-artifacts.md) | -| IL2CPP metadata | Unity global-metadata.dat assembly images, type and method names and index ranges | Built-in static decoding of versions 24, 27, 29, 31 and 39; no Unity required | -| Godot PCK | Archive inventory and extraction, original header/directory facts and selected content identities | Built-in static formats 2, 3 and 4; no Godot required | -| Rust binaries | Embedded cargo-auditable crate names, versions, sources and dependency edges | Built-in static ELF/PE/thin Mach-O inspection; [Rust binaries](docs/rust-binaries.md) | -| WASM artifact | Validation, sections, exact import/export WAT forms and decoded WAT | Caller-supplied WABT 1.0.42; `inspect_wasm_artifact` | -| EVM bytecode | Dispatch selectors, byte offsets, inferred arguments and mutability | Local raw/hex carrier on Linux x64; [offline EVM guide](docs/evm-bytecode.md) | -| SQLite databases | Tables, columns, indexes, views, triggers and selected ordinary-table rows | Local database/WAL snapshot; Node.js 24.x >=24.15 or 26+ with native SQLite limits | -| Recorded Linux crashes | Raw notes, every recorded thread's registers/signals and optional mapping candidates | Caller-supplied pwntools on Linux x64; optional GDB/pwndbg; [recorded crashes](docs/recorded-crashes.md) | -| JavaScript / Electron | Modules, imports, source maps, routes, IPC and native add-on relationships | Node.js and npm; Linux: procps-compatible `ps`; [application analysis](docs/javascript-application-workflows.md) | -| Websites | Page structure, scripts, network observations and requested screenshots | A Chrome-family browser; [browser analysis](docs/browser-observation.md) | -| Saved network captures | Requests, responses, exposed payloads and source locations | HAR or Fiddler Everywhere SAZ; mitmdump on Linux for native mitmproxy; [capture guide](docs/web-network-captures.md) | -| .NET assemblies | Metadata, CIL instructions, declared native dependencies and build comparisons | None (static only); [managed-code guide](docs/managed-code-analysis.md) | -| Android APKs | Manifest declarations, classes, decompiled methods and references | Headless JADX and a full JDK on Linux/macOS/Windows x64; [Android guide](docs/android-analysis.md) | -| Android devices | Devices, packages, processes, logs, dumps, screen captures, and APK/file transfers | Caller-supplied adb (emulator or hardware); [ADB guide](docs/adb-device-analysis.md) | -| JEB projects | Engine-backed project units, artifact digests and type/method pseudo-code | A caller-started JEB client serving MCP, selected with REA_JEB_MCP_URL; [JEB guide](docs/jeb-analysis.md) | -| Android resources | Decoded manifest, string tables, locales and version facts | Caller-supplied Apktool launcher; [resource guide](docs/apktool-resource-analysis.md) | -| Flutter apps | Dart snapshot hash, per-ABI library digests and engine build provenance | None (static APK parsing); [Flutter guide](docs/flutter-analysis.md) | -| Firmware | Regions, extraction results and native-analysis handoffs | Binwalk / Unblob on Linux; [firmware guide](docs/firmware-analysis.md) | -| Recorded packets | Selected fields, repeated values, filters and decode-as rules | Linux: TShark 4.x + prlimit; [capture guide](docs/packet-captures.md) | -| Packet summaries | Endpoints, DNS, TLS SNI, protocol distribution and HTTP requests | Caller-supplied tshark and capinfos (Linux-verified); [pcap guide](docs/pcap-analysis.md) | -| Recorded logic signals | Decoder annotations, sample ranges and diagnostics | Linux: sigrok-cli 0.7.2 + prlimit; [signal recordings](docs/signal-recordings.md) | -| Packages and resources | File inventories, digests, plists, Apple bundle anatomy and extracted resources | [Artifact and JavaScript guide](docs/javascript-artifact-reconstruction.md), [Apple applications](docs/apple-application-analysis.md) | -| Process behavior | Terminal output, interactions, exit and filesystem observations, and run comparisons | Linux/macOS with a native PTY; [process capture](docs/process-capture.md) | +| Target | What REA returns | Requirements and guide | +| ----------------------- | ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| Desktop installations | Runtime candidates, supporting files, component markers and next probes | Built-in bounded directory/header inspection; [desktop applications](docs/desktop-applications.md) | +| Native binaries | Pseudocode, assembly, strings, symbols, calls and references | Hopper, Ghidra or IDA; [native analysis](docs/native-investigation.md) | +| Offline ELF layout | Sections, segments, original symbols/relocations and static mitigation candidates | Caller-supplied pwntools on Linux x64; [binary diagnostics](docs/binary-diagnostics.md) | +| Go binaries | Embedded compiler version, modules, replacements and build settings | Built-in static ELF/PE/thin Mach-O inspection; no Go installation required | +| Model artifacts | GGUF / safetensors metadata, tensor declarations and derived totals | Built-in header inspection; optional full-file hash; [model files](docs/model-artifacts.md) | +| IL2CPP metadata | Unity global-metadata.dat assembly images, type and method names and index ranges | Built-in static decoding of versions 24, 27, 29, 31 and 39; no Unity required | +| Godot PCK | Archive inventory and extraction, original header/directory facts and selected content identities | Built-in static formats 2, 3 and 4; no Godot required | +| Rust binaries | Embedded cargo-auditable crate names, versions, sources and dependency edges | Built-in static ELF/PE/thin Mach-O inspection; [Rust binaries](docs/rust-binaries.md) | +| WASM artifact | Validation, sections, exact import/export WAT forms and decoded WAT | Caller-supplied WABT 1.0.42; `inspect_wasm_artifact` | +| EVM bytecode | Dispatch selectors, byte offsets, inferred arguments and mutability | Local raw/hex carrier on Linux x64; [offline EVM guide](docs/evm-bytecode.md) | +| SQLite databases | Tables, columns, indexes, views, triggers and selected ordinary-table rows | Local database/WAL snapshot; Node.js 24.x >=24.15 or 26+ with native SQLite limits | +| Recorded Linux crashes | Raw notes, every recorded thread's registers/signals and optional mapping candidates | Caller-supplied pwntools on Linux x64; optional GDB/pwndbg; [recorded crashes](docs/recorded-crashes.md) | +| JavaScript / Electron | Modules, imports, source maps, routes, IPC and native add-on relationships | Node.js and npm; Linux: procps-compatible `ps`; [application analysis](docs/javascript-application-workflows.md) | +| Websites | Page structure, scripts, network observations and requested screenshots | A Chrome-family browser; [browser analysis](docs/browser-observation.md) | +| Saved network captures | Requests, responses, exposed payloads and source locations | HAR or Fiddler Everywhere SAZ; mitmdump on Linux for native mitmproxy; [capture guide](docs/web-network-captures.md) | +| .NET assemblies | Metadata, CIL instructions, declared native dependencies and build comparisons | None (static only); [managed-code guide](docs/managed-code-analysis.md) | +| Android APKs | Manifest declarations, classes, decompiled methods and references | Headless JADX and a full JDK on Linux/macOS/Windows x64; [Android guide](docs/android-analysis.md) | +| Android devices | Devices, packages, processes, logs, dumps, screen captures, and APK/file transfers | Caller-supplied adb (emulator or hardware); [ADB guide](docs/adb-device-analysis.md) | +| JEB projects | Engine-backed project units, artifact digests and type/method pseudo-code | A caller-started JEB client serving MCP, selected with REA_JEB_MCP_URL; [JEB guide](docs/jeb-analysis.md) | +| Android resources | Decoded manifest, string tables, locales and version facts | Caller-supplied Apktool launcher; [resource guide](docs/apktool-resource-analysis.md) | +| Flutter apps | Dart snapshot hash, per-ABI library digests and engine build provenance | None (static APK parsing); [Flutter guide](docs/flutter-analysis.md) | +| Firmware | Regions, extraction results and native-analysis handoffs | Binwalk / Unblob on Linux; [firmware guide](docs/firmware-analysis.md) | +| Recorded packets | Selected fields, repeated values, filters and decode-as rules | Linux: TShark 4.x + prlimit; [capture guide](docs/packet-captures.md) | +| Packet summaries | Endpoints, DNS, TLS SNI, protocol distribution and HTTP requests | Caller-supplied tshark and capinfos (Linux-verified); [pcap guide](docs/pcap-analysis.md) | +| Recorded logic signals | Decoder annotations, sample ranges and diagnostics | Linux: sigrok-cli 0.7.2 + prlimit; [signal recordings](docs/signal-recordings.md) | +| Packages and resources | File inventories, digests, plists, Apple bundle anatomy and extracted resources | [Artifact and JavaScript guide](docs/javascript-artifact-reconstruction.md), [Apple applications](docs/apple-application-analysis.md) | +| Process behavior | Terminal output, interactions, exit and filesystem observations, and run comparisons | Linux/macOS with a native PTY; [process capture](docs/process-capture.md) | +| PyInstaller executables | CArchive members, Python version, entry scripts and extracted bytecode | None (static only); [Python packaged applications](docs/python-packaged-applications.md) | Static JavaScript and .NET inspection read the supplied files without running the application. Runtime capture runs or interacts with the selected target diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 561b9f2213..86d825c13d 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -50,6 +50,10 @@ export default defineConfig({ { text: "Electron fuses", link: "/electron-fuses" }, { text: "Process capture", link: "/process-capture" }, { text: ".NET assemblies", link: "/managed-code-analysis" }, + { + text: "Python packaged applications", + link: "/python-packaged-applications", + }, { text: "Android applications", link: "/android-analysis" }, { text: "Apple applications", link: "/apple-application-analysis" }, { text: "Desktop installations", link: "/desktop-applications" }, diff --git a/docs/apple-application-analysis.md b/docs/apple-application-analysis.md index 926aae0d41..3f718d4cc7 100644 --- a/docs/apple-application-analysis.md +++ b/docs/apple-application-analysis.md @@ -213,20 +213,37 @@ Resolution follows dyld: Each edge lists every candidate path tried, with its outcome: `resolved`, `absent`, `not-mach-o`, `malformed`, `unsupported` (recognized Mach-O outside parser coverage, such as big-endian images), `architecture-missing`, -`outside-target`, `escapes-target`, or `undetermined`. Slices follow dyld's -graded architectures, so an `x86_64h` process also loads `x86_64`. The edge's `resolution` is one of: +`not-loadable`, `platform-mismatch`, `outside-target`, `escapes-target`, or +`undetermined`. `not-loadable` means a +matching-architecture Mach-O image was found but it is not a dylib and cannot +satisfy a dylib dependency. `platform-mismatch` means a matching-architecture +dylib was found but its observed platform is incompatible with the process. +These outcomes reject that candidate; dyld search may continue and resolve a +later candidate. Candidate outcomes describe individual search paths, while +`resolution.status` describes the final edge. Slices follow dyld's graded +architectures, so an `x86_64h` process also loads `x86_64`. The edge's +`resolution` is one of: -| Status | Meaning | -| -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `resolved` | An image inside the analyzed root loads, and no earlier candidate was undeterminable. | -| `conditional` | An image inside the root resolves, but an earlier candidate outside the root (for example `/usr/lib/swift`) would win if it exists on the running system. | -| `unresolved` | Every candidate is definitively absent or unusable inside the root. | -| `undetermined` | No candidate resolves inside the root, and some candidate lies outside it or depends on the environment. | +| Status | Meaning | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `resolved` | An image inside the analyzed root loads, and no earlier candidate was undeterminable. | +| `conditional` | An in-root image may load, but an earlier outside candidate, unmodeled environment input, or unknown platform compatibility can change which image dyld loads. | +| `unresolved` | Every candidate is definitively absent or unusable inside the root. | +| `undetermined` | No candidate resolves inside the root, and some candidate lies outside it or depends on the environment. | -A `conditional` image is still traced, because it is the image the analyzed -root supplies. Every edge below it has `loader_conditional: true`, and a -request that reuses it by install name is `conditional` too. Findings on those -edges say that they apply only if the conditional image loads. +A `conditional` image is still traced, because it is an in-root candidate +that may load. Every edge below it has `loader_conditional: true`, and a +request that reuses it by install name is `conditional` too. An in-root dylib +with no platform load command remains unknown unless the root is an executable +with observed macOS platform and the candidate’s CPU type is x86_64, i386, or arm64. In that macOS context, +[Apple's dyld](https://github.com/apple-oss-distributions/dyld/blob/fd8d0c4d52320ebf64db34f3cb280310d905c5ae/mach_o/UnsafeHeader.cpp#L1137-L1145) supplies an implicit macOS platform to legacy dylibs on those CPU +types. The candidate records `platform_basis: "macos-dyld-implicit"`; the +reported slice fields `platform: null` and `platforms: []` remain the parsed +facts. For other process contexts the platform remains unknown, coverage is +partial, and the candidate is not treated as incompatible. If a later known +candidate resolves, it remains the traced image with conditional resolution; +the trace does not claim the unknown candidate's dependency graph. Findings on +conditional edges say that they apply only if the conditional image loads. Absolute install names that name a path beneath the analyzed root are checked within that root, with symlinks and `..` resolved by the same bounded tree diff --git a/docs/cli.md b/docs/cli.md index 90bae07101..7984683a8a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -512,10 +512,12 @@ error and exit `1`, as does a listener that fails after binding. pair refusals, a non-empty or file-occupied directory, a missing package tarball or a name that is not a kebab-case label or is a Windows device name, writing nothing in each case. -A write that fails after the first file (a permission or disk-space failure -on a later path) propagates as the host's failure and leaves the partial -folder in place for inspection; remove it before retrying, since the next -attempt refuses the non-empty directory. +A write that fails after the first file (a permission, disk-space or +pathname-length failure on a later path) propagates as the host's failure, +naming its cause and, when it failed on one bundle file or the package +tarball, that file. A failure while syncing the finished folder names only its +cause. Either way the partial folder stays in place for inspection; remove it +before retrying, since the next attempt refuses the non-empty directory. A `cancelled` result does not by itself determine the exit status; it depends on the command's signal handling. Commands routed through the shared diff --git a/docs/ida-provider.md b/docs/ida-provider.md index f4a2c95dd6..ecaa2c66d8 100644 --- a/docs/ida-provider.md +++ b/docs/ida-provider.md @@ -41,6 +41,15 @@ rea function /absolute/path/to/program main --json rea decompile /absolute/path/to/program main --json ``` +If the registration cannot be read or validated, provider diagnostics retain its +selected `registration_path` and the configuration owner's setting/constraint +facts in `configuration_details.settings`, plus `configuration_remediation`. +`configured` means a path was selected, not that it is valid or an IDA connection +was established; failed registrations remain unavailable. Correct +`REA_IDA_MCP_CONFIG` in the next CLI invocation's environment or the MCP server's +launch environment. Restart/reconnect that server after changing its environment; +an export in another shell does not update a running server. + Open the **original input binary** in REA, not an `.idb` or `.i64` database. Its SHA-256 must match IDA's recorded input hash. REA checks the attached identity and image base around each analysis request. A mismatch or target switch fails with recovery guidance. The original input must remain readable by REA; WSL can use its mounted alias while the GUI runs on Windows. Attached profiles, including native mode, refuse universal Mach-O inputs at admission: every slice shares the recorded whole-file hash, so REA cannot verify which slice the GUI loaded. Open a thin file extracted with `lipo -thin` in both IDA and REA, or use headless mode, which analyzes the selected slice. REA's close operation releases its MCP connection or proxy. It never saves or closes the existing GUI database. External edits remain visible because results are live and are not replayed from snapshots. These checks do not lock the GUI or prove that the database is unpatched. diff --git a/docs/index.md b/docs/index.md index 74c72ef059..070b8aad51 100644 --- a/docs/index.md +++ b/docs/index.md @@ -47,19 +47,19 @@ For runtime instrumentation, see the [Frida guide](./frida-instrumentation.md). ## Choose a guide -| Task | Guides | -| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Identify a desktop installation | [Runtime candidates, evidence and follow-up files](desktop-applications.md) | -| Inspect native functions, references, or types | [Native binaries](native-investigation.md), [Go metadata](go-binaries.md), [Rust dependencies](rust-binaries.md), [offline ELF layout](binary-diagnostics.md), [recorded crashes](recorded-crashes.md) | -| Select a native provider or import format | [Ghidra semantics](ghidra-provider.md), [Windows](windows-ghidra-p0.md), [DOS](ghidra-dos.md), [raw bytes](raw-binary.md), [MIPS](ghidra-mips.md), [PSP](ghidra-psp.md), [NativeAOT](ghidra-nativeaot.md), [IDA](ida-provider.md), [Kuna](kuna-provider.md), [JEB](jeb-analysis.md) | -| Analyze JavaScript or Electron artifacts | [Static analysis](javascript-artifact-reconstruction.md), [tracing and comparisons](javascript-application-workflows.md), [application graph](javascript-application-graph.md), [source recovery](javascript-recovery.md) | -| Compare static JavaScript with runtime observations | [Runtime attribution](javascript-runtime-observation.md), [reconciliation](javascript-runtime-reconciliation.md), [Electron observation](electron-observation.md), [Electron fuses](electron-fuses.md) | -| Inspect assemblies, packages, or firmware | [Godot PCK](godot-pck.md), [.NET](managed-code-analysis.md), [Android](android-analysis.md), [Apktool resources](apktool-resource-analysis.md), [ADB acquisition](adb-device-analysis.md), [Apple](apple-application-analysis.md), [Harmony](harmony-analysis.md), [firmware](firmware-analysis.md), [WASM](wasm-artifacts.md), [EVM](evm-bytecode.md) | -| Observe or compare a browser scenario | [CDP observation](browser-observation.md), [scenarios](browser-scenario-contract.md), [runtime evidence](web-runtime.md), [historical captures](web-network-captures.md) | -| Serve recorded behavior as an agent environment | [Agent environments](agent-environments.md) | -| Trace captured website source | [Source locations](web-source-location.md), [module traces](website-module-trace.md), [script export](website-script-export.md) | -| Capture and compare a command | [Process capture](process-capture.md) | -| Use CLI/MCP and retain evidence | [CLI](cli.md), [MCP contracts](mcp-contracts.md), [agent prompts](mcp-prompts.md), [obligation ledgers](reconstruction-obligation-ledgers.md) | +| Task | Guides | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Identify a desktop installation | [Runtime candidates, evidence and follow-up files](desktop-applications.md) | +| Inspect native functions, references, or types | [Native binaries](native-investigation.md), [Go metadata](go-binaries.md), [Rust dependencies](rust-binaries.md), [offline ELF layout](binary-diagnostics.md), [recorded crashes](recorded-crashes.md) | +| Select a native provider or import format | [Ghidra semantics](ghidra-provider.md), [Windows](windows-ghidra-p0.md), [DOS](ghidra-dos.md), [raw bytes](raw-binary.md), [MIPS](ghidra-mips.md), [PSP](ghidra-psp.md), [NativeAOT](ghidra-nativeaot.md), [IDA](ida-provider.md), [Kuna](kuna-provider.md), [JEB](jeb-analysis.md) | +| Analyze JavaScript or Electron artifacts | [Static analysis](javascript-artifact-reconstruction.md), [tracing and comparisons](javascript-application-workflows.md), [application graph](javascript-application-graph.md), [source recovery](javascript-recovery.md) | +| Compare static JavaScript with runtime observations | [Runtime attribution](javascript-runtime-observation.md), [reconciliation](javascript-runtime-reconciliation.md), [Electron observation](electron-observation.md), [Electron fuses](electron-fuses.md) | +| Inspect assemblies, packages, or firmware | [Godot PCK](godot-pck.md), [.NET](managed-code-analysis.md), [Python packages](python-packaged-applications.md), [Android](android-analysis.md), [Apktool resources](apktool-resource-analysis.md), [ADB acquisition](adb-device-analysis.md), [Apple](apple-application-analysis.md), [Harmony](harmony-analysis.md), [firmware](firmware-analysis.md), [WASM](wasm-artifacts.md), [EVM](evm-bytecode.md) | +| Observe or compare a browser scenario | [CDP observation](browser-observation.md), [scenarios](browser-scenario-contract.md), [runtime evidence](web-runtime.md), [historical captures](web-network-captures.md) | +| Serve recorded behavior as an agent environment | [Agent environments](agent-environments.md) | +| Trace captured website source | [Source locations](web-source-location.md), [module traces](website-module-trace.md), [script export](website-script-export.md) | +| Capture and compare a command | [Process capture](process-capture.md) | +| Use CLI/MCP and retain evidence | [CLI](cli.md), [MCP contracts](mcp-contracts.md), [agent prompts](mcp-prompts.md), [obligation ledgers](reconstruction-obligation-ledgers.md) | For development, use [testing](testing.md), [client verification](client-verification.md), [tool design](tool-design.md), and [releasing](releasing.md). diff --git a/docs/model-artifacts.md b/docs/model-artifacts.md index 59b5da04f1..f59c2c7a32 100644 --- a/docs/model-artifacts.md +++ b/docs/model-artifacts.md @@ -5,6 +5,11 @@ GGUF v2/v3 or safetensors file. Detection uses content, including GGUF byte order, rather than its extension. REA does not load a model, interpret weights, run inference, import Python, open external-data references or use the network. +Unsupported GGUF versions remain refused. Their diagnostic preserves the four +version bytes and both UINT32 interpretations with byte order unknown, rather +than inferring big-endian from an unrecognized little-endian value. Little-endian +v2/v3 and big-endian v3 remain supported; big-endian v2 remains refused. + ```bash rea inspect-model-artifact /absolute/path/model.gguf --json rea inspect-model-artifact /absolute/path/model.safetensors --hash full --json diff --git a/docs/native-investigation.md b/docs/native-investigation.md index 60105b42c1..ebc5e646c4 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -298,13 +298,20 @@ markup, including UTF-8 and BOM-marked UTF-16 sources. Unidentified text and OpenStep plists, including a data-only value such as ``, report `unknown` even when plutil decodes their values through an XML conversion. +Binary plist inspection reads the original bytes through an owned file descriptor, +so conversion cannot erase control characters, fractional dates, negative zero, +or the high word of a 128-bit integer. It accepts at most 64 MiB of input and +reserves an estimated 64 MiB decode budget before parsing; this estimate is not +a hard process-memory bound. Binary-source provenance records the `file` +classification without claiming a `plutil` conversion. + A plist explicitly selected through the path input or as the active plist target reports an unreadable or malformed selection as `invalid_request`. A missing or unreadable default plist of another target reports `capability_unavailable`. Permission failures remain `access_denied`; output and cleanup failures retain their own error categories. -A plist decode refusal includes plutil's diagnostic from either output stream. +A conversion refusal includes plutil's diagnostic from either output stream. `details.captured_output` retains bounded stdout/stderr excerpts, observed byte totals and exit status. `truncated: true` means those excerpts are incomplete. @@ -444,6 +451,13 @@ complete observation. an unfixed zero in a nullable ABI field is treated as null. Incomplete or conflicting fixup streams keep affected pointers unresolved; coverage reports parsing and resource limits while retaining validated prefix facts. + Conformance records preserve the raw three-bit `type_reference_kind`. Kinds + 0 and 1 identify direct and indirect nominal descriptors; kind 2 preserves + the inline Objective-C class name and its source location. Kind 3 preserves + an imported class binding or the local class-object address when available, + but class metadata/name recovery requiring the Objective-C runtime remains + partial. Unknown kinds retain the raw kind and reference location without + being treated as absent or assigned a guessed type name. So an external superclass resolves from its `_OBJC_CLASS_$_` bind (for example `NSObject`). The `pointer_fixups` coverage facet names the mechanism. @@ -714,7 +728,13 @@ leap seconds) and out-of-range dates remain explicit unknowns. REA does not emulate CoreFoundation's normalization of overflowing calendar fields. Invalid dates returned by other decoders retain `{ "$plist_type": "date", "iso8601": null }` with a counted limitation. `inspect_plist` and Interface Builder decoding use -the same projection. +the same projection. Binary plist decoding refuses a reachable date whose marker +is not `0x33`; date and numeric payloads must end before the offset table. +Valid overlapping object records retain their independent source values. A valid `0x33` date is +reported with an ISO-8601 timestamp only when its float64 seconds are exactly +the canonical encoding of that millisecond instant. Any other finite binary +date uses `{ "$plist_type": "date", "iso8601": null }` and the counted +limitation; the archive bytes remain the source of the instant. A negative-zero real is reported as `{ "$plist_type": "real", "value": "-0" }` because a JSON number cannot carry the sign of zero. `inspect_plist` and diff --git a/docs/pcap-analysis.md b/docs/pcap-analysis.md index 3e13ec2251..58c5dcd037 100644 --- a/docs/pcap-analysis.md +++ b/docs/pcap-analysis.md @@ -67,7 +67,11 @@ Addresses count once per packet, including loopback and nested IP occurrences. DNS addresses are associated only with one question in the same response packet; additional records may be included, so this does not prove final resolution. Ambiguous addresses remain in a separate inventory. Protocol distribution counts -exact `frame.protocols` stack strings. TLS SNI is recorded handshake metadata, +exact nonempty `frame.protocols` stack strings. Frames for which TShark reports +an empty stack, including zero-length packets, still contribute to the packet +count without adding a protocol row. An all-empty stack inventory can have +complete examined coverage; it does not establish the absence of network traffic. +TLS SNI is recorded handshake metadata, not process identity or proof of the destination's authenticity. All examined inventories are retained within the resource budget; no fixed diff --git a/docs/process-capture.md b/docs/process-capture.md index 9096b494fa..67f81c085f 100644 --- a/docs/process-capture.md +++ b/docs/process-capture.md @@ -461,7 +461,33 @@ selected roots; it is not a per-file allowance. File contents are read in Ownership inspections bind run-token reads to native start identities and retain related generations through reparenting, group changes, omitted rows, and cleanup -retries. The captured run keeps one evidence owner through settlement and cleanup. +retries. A host without native identity reads still token-checks process-table +rows that appear only in a launcher-exit refresh before reporting that group +clean or publishing an empty lineage. A matching token authorizes a signal only +for a group whose leader carries the same token. A matching token whose leader +is absent, or an unreadable token for a still-live new row, stays unresolved. A +readable foreign token is not signaled. An unreadable row that has exited by +the confirmation read does not hide a new row in that confirmation table, and a +group proven while revalidating another owned group is signaled in the same +cleanup. A matched row that changes parent or group on the confirmation read +is token-checked again. A PID whose token was already read is read again +on the next confirmation even when its parent and group return to the +pre-exit values. A PID whose first token was foreign is read again +on that table, because the PID alone does not identify the process. A PID +from the pre-exit table is read again when its parent or group changes, and +an unchanged pre-existing row that was never token-read stays unread. Lineage +publishes one row for each PID. Confirmation reads are bounded; a table that keeps producing unreadable +replacements stays unresolved. That refresh does not retain a ledger +for a later attempt that never observes the launcher. Lineage reads a row again +when its parent, group, or launcher-tree membership changes, and it reads a +token again on the next refresh even when those fields are unchanged. The +launcher is read on every lineage refresh; a missing, unowned, or +relation-changed launcher PID is not published as a verified lineage. A +descendant absent from the latest refresh is omitted. An +ancestor move cannot leave a foreign descendant inside a verified result. A +group that reappears after cleanup already processed it is revalidated once; +another appearance stays unresolved. The captured run keeps +one evidence owner through settlement and cleanup. An absent or changed token leaves a previously related generation unresolved; current token proof is still required before signaling. Swept cleanup verifies the remaining family after its group signals. Coverage is bounded by the native diff --git a/docs/python-packaged-applications.md b/docs/python-packaged-applications.md new file mode 100644 index 0000000000..5785def1fb --- /dev/null +++ b/docs/python-packaged-applications.md @@ -0,0 +1,128 @@ +# Python packaged applications + +REA statically inventories PyInstaller executables with an appended CArchive. +`inspect_artifact` and `extract_artifact` share the same reader through CLI and +MCP. They read archive bytes without launching the application. Bytecode +source decompilation, encrypted PYZ recovery, zipapp and PEX analysis are outside +this capability. + +## Facts and focused queries + +The root occurrence's `format_metadata` records the CArchive layout (2.0 or +2.1), archive and cookie offsets, table extent, entry count, raw Python version, +normalized major/minor when known, library name and encryption marker. Member +metadata retains the original name, table index, type, compression flag, +relative offset and known directory record extent. Logical paths use archive +path rules; the source name retains its spelling, including backslashes. +`relative_offset` is relative to the CArchive start for CArchive rows, or to +the decoded PYZ image for PYZ rows. A CArchive directory record starts at +`archive_start + toc_offset + directory_record_offset`; its size is +`directory_record_size`. PYZ row directory extents remain unknown. + +Namespace packages remain directories. Their `relative_offset` and +`compressed_size` preserve any payload declared by older producers, while +`declared_size` is zero because no bytecode is materialized for a namespace. +Every declared span must end before the PYZ table of contents. +PYZ `compression_flag` is derived from storage: nonempty payloads are compressed, +including legacy namespaces; an empty namespace has no compressed payload. + +CArchive type `o` entries are runtime options. They are metadata occurrences, +with no content artifact or extractable file. Their query paths are +`__pyinstaller__/runtime-options/`; the option text is +`format_metadata.source_name`. Repeated declarations retain distinct occurrence +identities. These paths do not claim output filesystem names. + +Type `d` entries are multipackage dependencies. They are also metadata, under +`__pyinstaller__/dependencies/
`, and never materialize an empty file. +The reader preserves the raw reference and splits its first colon into +`dependency_archive` and `dependency_member`. `dependency_status` remains +`unresolved`, or `invalid-reference` with unknown parts for malformed references. +`external_dependency_count` summarizes these declarations. REA does not open +another executable or claim its contents are available. A real file with the +same logical path as metadata remains independently extractable. +Both declaration types require empty payloads and no compression. Their query +paths do not participate in filesystem collision checks, directory ancestry or +source-map target lookup; their parent identifies the declaring container. +Complete authenticated inventory page sets preserve producer identities in any +merge order, including metadata/file aliases. Malformed TOC diagnostics name the +table index; original names remain available in the supplied artifact. +Path failures use bounded escaped previews and report the original length; +accepted paths and retained source names are not shortened. + +Request `detail: "summary"` for a compact inspection, then query retained +Evidence with `inspect_analysis_view`. An inventory view can select +`collection: "occurrences"`, `entry_kinds: ["metadata"]`, exact logical paths or +occurrence IDs, requested `fields`, and `offset`/`limit`. Filters intersect; +identities, parent Evidence and coverage survive projection. Full inventory +Evidence remains available for export. A compact reply reduces returned context; +it still performs the complete member inventory and hashing work. +Focused views add calls and tool-schema costs. A complete small inventory can +cost less overall; prefer compact delivery when the full member graph would +overwhelm the answer. + +```bash +rea inspect-artifact /absolute/path/hello --detail summary --evidence_output /absolute/new/evidence.json --json +rea extract-artifact /absolute/path/hello --json +``` + +## Decoding and evidence limits + +Inventory hashes members individually through bounded streams. Stored PYZ +images use descriptor ranges; outer-compressed images stream to private files, +which the reader owns through cleanup and any cleanup retry. Extraction binds +the selected carrier to its inventoried digest and checks source identity and +state before and after reading. It does not retain expanded module bodies. +Opaque outer-compressed images reuse that verified private image when opened, +instead of inflating the carrier again. Direct source spans describe stored +CArchive bytes; decompressed content has no direct span even when its encoded +and decoded sizes coincide. Original offsets and compression remain metadata. + +Only the PYZ header and marshal directory are loaded for listing. PYZ tables +have a 16 MiB budget; CArchive tables have a 32 MiB budget. Parsed facts have a +32 MiB serialized budget. Projected entries share a 64 MiB retained budget, +including metadata, bookkeeping and conservative reservations for namespace +objects and possible case-collision annotations at each path component. These +reservations happen before normalization or trie allocation. +Outer-compressed PYZ images have a separate per-reader capacity of 128 private +files, bounding retained inodes and image identities. The complete image count +is admitted before any temporary file is created; excess images return a +capacity failure. This is a resource policy, not a PyInstaller format limit. +Only a zero-byte read establishes EOF; short successful descriptor reads fill +the bounded range. Compressed members must consume their complete declared +input span; trailing junk and concatenated streams are refused. +Plaintext expansion has an aggregate archive-derived budget. Exceeding a budget +returns a resource failure. + +Supported final Python bytecode magic values cover Python 3.7 through 3.14. +Supported marshal directory values are lists, tuples, strings, integers and +bytes. An unsupported magic or directory stays an opaque member with a +limitation and `pyz_status: "opaque"`; its module contents remain unknown. +Any opaque PYZ makes producer traversal coverage unknown, including retained +summaries and exhausted queries. Per-archive reasons remain on the opaque +occurrences; the coverage observation summarizes their count without repeating +every limitation. An empty result does not establish module absence. +Observed `pyz_python_magic` bytes remain available even when their Python version +is unknown. +The directory declares compressed module lengths, so uncompressed declared +sizes and source spans inside decoded data remain unknown. + +PYZ modules receive a derived 16-byte `.pyc` header from their own supported +magic. CArchive bytecode receives that header only when every sibling PYZ magic +and the cookie's Python version agree. Otherwise it retains its original bytes +and `bytecode_header: "unknown"`. `bytecode_magic` and graph derivation edges +identify synthesized headers; member digests cover the resulting derived files. +These headers do not establish successful execution or decompilation. + +The nonempty CArchive Python module `pyimod00_crypto_key` (`m`/`M`) is recorded +as an encryption marker; a same-named data file or script is not. Affected +PYZ blobs are reported without decoding; extraction refuses encrypted entries. +The raw PYZ container remains readable and hashable, including when its module +table is opaque. Its marker stays in metadata and opaque child coverage stays +unknown; extracting the container does not decrypt its payloads. +The marker also applies to nonempty legacy namespace payloads. Empty namespaces +have no payload to encrypt and retain a null encryption source. + +Detection searches the last 16 MiB for a cookie, allowing a trailing macOS code +signature within that bound. Member offsets are relative to the archive start: +cookie position plus cookie size minus package length. Native bootloader +analysis remains available through the native tools. diff --git a/docs/readme/README_ar.md b/docs/readme/README_ar.md index 99e7614769..86d7943131 100644 --- a/docs/readme/README_ar.md +++ b/docs/readme/README_ar.md @@ -152,6 +152,7 @@ npx rea-agents@latest setup | تسجيلات الشبكة المحفوظة | الطلبات، والاستجابات، ومحتويات الحمولة المتاحة، ومواضعها في المصدر | HAR؛ وmitmdump على Linux لتسجيلات mitmproxy بصيغتها الأصلية؛ [دليل تسجيلات الشبكة](../web-network-captures.md) | | التقاط الحزم | نقاط النهاية وDNS وTLS SNI وتوزيع البروتوكولات وطلبات HTTP | tshark وcapinfos يوفّرهما المُستدعي؛ [دليل pcap](../pcap-analysis.md) | | تجميعات .NET | البيانات الوصفية، وتعليمات CIL، واعتماديات الشيفرة الأصلية المعلنة، ومقارنات البناء | فحص ثابت؛ [دليل الشيفرة المُدارة](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | حزم Android APK | تصريحات ملف manifest، والأصناف، والدوال الناتجة عن فك الترجمة، والمراجع | JADX دون واجهة رسومية وJDK كامل على Linux/macOS/Windows x64؛ [دليل Android](../android-analysis.md) | | أجهزة Android | الأجهزة والحزم والعمليات والسجلات والدامبات ولقطات الشاشة ونقل APK/الملفات | adb يوفّره المُستدعي (محاكي أو جهاز)؛ [دليل ADB](../adb-device-analysis.md) | | مشاريع JEB | وحدات المشروع من المحرك وبصمات المصنّفات والكود الزائف للأنواع/الدوال | عميل JEB يبدأه المُستدعي ويقدّم MCP، ويُختار عبر REA_JEB_MCP_URL؛ [دليل JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_de.md b/docs/readme/README_de.md index 42cb9eaec1..13c5a56592 100644 --- a/docs/readme/README_de.md +++ b/docs/readme/README_de.md @@ -152,6 +152,7 @@ REA benötigt Node.js 22.x (>=22.19), 24.x (>=24.11) oder 26+ sowie npm. Zusätz | Gespeicherte Netzwerkaufzeichnungen | Anfragen, Antworten, zugängliche Nutzdaten und Quellpositionen | HAR; mitmdump unter Linux für Aufzeichnungen im nativen mitmproxy-Format; [Leitfaden zu Netzwerkaufzeichnungen](../web-network-captures.md) | | Paketerfassungen | Endpunkte, DNS, TLS SNI, Protokollverteilung und HTTP-Anfragen | Vom Aufrufer bereitgestellte tshark- und capinfos-Binaries; [pcap-Leitfaden](../pcap-analysis.md) | | .NET-Assemblies | Metadaten, CIL-Anweisungen, deklarierte native Abhängigkeiten und Build-Vergleiche | Statische Untersuchung; [Leitfaden für verwalteten Code](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | Android-APKs | Manifest-Deklarationen, Klassen, dekompilierte Methoden und Verweise | JADX ohne grafische Oberfläche und ein vollständiges JDK unter Linux/macOS/Windows x64; [Android-Leitfaden](../android-analysis.md) | | Android-Geräte | Geräte, Pakete, Prozesse, Logs, Dumps, Bildschirmaufnahmen sowie APK- und Dateiübertragungen | Vom Aufrufer bereitgestelltes adb (Emulator oder Gerät); [ADB-Leitfaden](../adb-device-analysis.md) | | JEB-Projekte | Engine-gestützte Projekteinheiten, Artefakt-Prüfsummen und Pseudokode von Typen/Methoden | Ein vom Aufrufer gestarteter JEB-Client mit MCP, ausgewählt über REA_JEB_MCP_URL; [JEB-Leitfaden](../jeb-analysis.md) | diff --git a/docs/readme/README_el.md b/docs/readme/README_el.md index 9912a8fea9..b7e9e19f7b 100644 --- a/docs/readme/README_el.md +++ b/docs/readme/README_el.md @@ -158,6 +158,7 @@ npx rea-agents@latest setup | Ιστότοποι | Δομή σελίδων, scripts, παρατηρήσεις δικτύου και ζητούμενα στιγμιότυπα οθόνης | Πρόγραμμα περιήγησης της οικογένειας Chrome· [ανάλυση μέσω προγράμματος περιήγησης](../browser-observation.md) | | Αποθηκευμένες καταγραφές δικτύου | Αιτήματα, αποκρίσεις, διαθέσιμα δεδομένα φορτίου και θέσεις στην πηγή | HAR ή SAZ του Fiddler Everywhere· mitmdump σε Linux για την εγγενή μορφή mitmproxy· [οδηγός καταγραφών](../web-network-captures.md) | | Assemblies .NET | Μεταδεδομένα, εντολές CIL, δηλωμένες εξαρτήσεις από κώδικα μηχανής και συγκρίσεις builds | Καμία πρόσθετη απαίτηση (μόνο στατική ανάλυση)· [οδηγός διαχειριζόμενου κώδικα](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | APK Android | Δηλώσεις manifest, κλάσεις, απομεταγλωττισμένες μέθοδοι και αναφορές | JADX χωρίς γραφικό περιβάλλον και πλήρες JDK σε Linux/macOS/Windows x64· [οδηγός Android](../android-analysis.md) | | Συσκευές Android | Συσκευές, πακέτα, διεργασίες, logs, dumps, στιγμιότυπα οθόνης και μεταφορές APK/αρχείων | adb που παρέχει ο χρήστης (εξομοιωτής ή φυσική συσκευή)· [οδηγός ADB](../adb-device-analysis.md) | | Έργα JEB | Μονάδες έργου από τη μηχανή ανάλυσης, ψηφιακά αποτυπώματα αρχείων και ψευδοκώδικα τύπων/μεθόδων | Client JEB που έχει ξεκινήσει ο χρήστης και παρέχει MCP, επιλεγμένος μέσω REA_JEB_MCP_URL· [οδηγός JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_es.md b/docs/readme/README_es.md index d7a79ade22..26f197ff62 100644 --- a/docs/readme/README_es.md +++ b/docs/readme/README_es.md @@ -152,6 +152,7 @@ REA requiere Node.js 22.x (>=22.19), 24.x (>=24.11) o 26+, además de npm. Las h | Capturas de red guardadas | Solicitudes, respuestas, contenido accesible de las cargas útiles y ubicaciones de origen | HAR; mitmdump en Linux para capturas en formato nativo de mitmproxy; [guía de capturas](../web-network-captures.md) | | Capturas de paquetes | Endpoints, DNS, TLS SNI, distribución de protocolos y solicitudes HTTP | tshark y capinfos suministrados por el autor de la llamada; [guía pcap](../pcap-analysis.md) | | Ensamblados .NET | Metadatos, instrucciones CIL, dependencias nativas declaradas y comparaciones de compilaciones | Inspección estática; [guía de código administrado](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | APK de Android | Declaraciones del manifiesto, clases, métodos descompilados y referencias | JADX sin interfaz gráfica y un JDK completo en Linux/macOS/Windows x64; [guía de Android](../android-analysis.md) | | Dispositivos Android | Dispositivos, paquetes, procesos, registros, volcados, capturas de pantalla y transferencias de APK/archivos | adb suministrado por el autor de la llamada (emulador o dispositivo); [guía de ADB](../adb-device-analysis.md) | | Proyectos JEB | Unidades de proyecto del motor, resúmenes de artefactos y pseudocódigo de tipos/métodos | Un cliente JEB iniciado por el autor de la llamada que sirve MCP, seleccionado con REA_JEB_MCP_URL; [guía de JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_fa.md b/docs/readme/README_fa.md index 8f88b2e4e1..fa23c15949 100644 --- a/docs/readme/README_fa.md +++ b/docs/readme/README_fa.md @@ -143,6 +143,7 @@ REA به Node.js 22.x (>=22.19)، 24.x (>=24.11) یا 26+، به‌همراه np | وب‌سایت‌ها | ساختار صفحه، اسکریپت‌ها، مشاهدات شبکه و اسکرین‌شات‌های درخواستی | مرورگر مبتنی بر Chrome؛ [تحلیل مرورگر](../browser-observation.md) | | داده‌های ضبط‌شدهٔ شبکه | درخواست‌ها، پاسخ‌ها، payloadهای آشکارشده و مکان آن‌ها در منبع | HAR؛ ابزار mitmdump در لینوکس برای ضبط‌های بومی mitmproxy؛ [راهنمای ضبط شبکه](../web-network-captures.md) | | اسمبلی‌های .NET | فراداده، دستورالعمل‌های CIL، وابستگی‌های بومی اعلام‌شده و مقایسهٔ buildها | بررسی ایستا؛ [راهنمای کد مدیریت‌شده](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | فایل‌های APK اندروید | تعاریف Manifest، کلاس‌ها، متدهای دیکامپایل‌شده و ارجاعات | JADX بدون رابط گرافیکی و JDK کامل در Linux/macOS/Windows x64؛ [راهنمای اندروید](../android-analysis.md) | | دستگاه‌های اندروید | دستگاه‌ها، بسته‌ها، فرایندها، لاگ‌ها، دامپ‌ها، اسکرین‌شات‌ها و انتقال APK/فایل | adb تأمین‌شده توسط فراخوان‌کننده (شبیه‌ساز یا دستگاه)؛ [راهنمای ADB](../adb-device-analysis.md) | | پروژه‌های JEB | واحدهای پروژه از موتور، هش‌های مصنوع و شبه‌کدِ نوع/متد | یک کلاینت JEB که توسط فراخوان‌کننده اجرا شده و MCP ارائه می‌دهد و با REA_JEB_MCP_URL انتخاب می‌شود؛ [راهنمای JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_fr.md b/docs/readme/README_fr.md index c912ffb02a..db2a7051bb 100644 --- a/docs/readme/README_fr.md +++ b/docs/readme/README_fr.md @@ -156,6 +156,7 @@ REA nécessite Node.js 22.x (>=22.19), 24.x (>=24.11) ou 26+, ainsi que npm. Les | Captures réseau enregistrées | Requêtes, réponses, payloads exposés et emplacements dans le code source | HAR ; mitmdump sous Linux pour les captures mitmproxy natives ; [guide des captures](../web-network-captures.md) | | Captures de paquets | Points de terminaison, DNS, TLS SNI, distribution des protocoles et requêtes HTTP | tshark et capinfos fournis par l'appelant ; [guide pcap](../pcap-analysis.md) | | Assemblys .NET | Métadonnées, instructions CIL, dépendances natives déclarées et comparaisons de builds | Inspection statique ; [guide du code managé](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | APK Android | Déclarations du manifeste, classes, méthodes décompilées et références | JADX headless et un JDK complet sous Linux/macOS/Windows x64 ; [guide Android](../android-analysis.md) | | Appareils Android | Appareils, paquets, processus, journaux, dumps, captures d'écran et transferts d'APK/fichiers | adb fourni par l'appelant (émulateur ou appareil) ; [guide ADB](../adb-device-analysis.md) | | Projets JEB | Unités de projet fournies par le moteur, condensés d'artefacts et pseudo-code de types/méthodes | Un client JEB démarré par l'appelant servant MCP, sélectionné via REA_JEB_MCP_URL ; [guide JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_id.md b/docs/readme/README_id.md index d4eafc7881..d824477757 100644 --- a/docs/readme/README_id.md +++ b/docs/readme/README_id.md @@ -159,6 +159,7 @@ Alat tambahan dan dukungan sistem host bergantung pada target yang dianalisis: | Rekaman jaringan tersimpan | Permintaan, respons, payload yang terekspos, dan lokasi sumber | HAR; mitmdump di Linux untuk rekaman native mitmproxy; [panduan rekaman](../web-network-captures.md) | | Capturan paket | Endpoint, DNS, TLS SNI, distribusi protokol, dan permintaan HTTP | tshark dan capinfos buatan pemanggil; [panduan pcap](../pcap-analysis.md) | | Assembly .NET | Metadata, instruksi CIL, dependensi native yang dideklarasikan, dan perbandingan build | Tidak ada (hanya analisis statis); [panduan kode terkelola](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | APK Android | Deklarasi manifest, kelas, metode hasil dekompilasi, dan referensi | JADX mode headless dan JDK lengkap di Linux/macOS/Windows x64; [panduan Android](../android-analysis.md) | | Perangkat Android | Perangkat, paket, proses, log, dump, tangkapan layar, serta transfer APK/berkas | adb buatan pemanggil (emulator atau perangkat); [panduan ADB](../adb-device-analysis.md) | | Proyek JEB | Unit proyek bertenaga mesin, hash artefak, dan pseudokode tipe/metode | Klien JEB yang dijalankan pemanggil dan menyajikan MCP, dipilih dengan REA_JEB_MCP_URL; [panduan JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_it.md b/docs/readme/README_it.md index 5678c813c1..7f951545a2 100644 --- a/docs/readme/README_it.md +++ b/docs/readme/README_it.md @@ -156,6 +156,7 @@ REA richiede Node.js 22.x (>=22.19), 24.x (>=24.11) o 26+, oltre a npm. Strument | Acquisizioni di rete salvate | Richieste, risposte, payload esposti e posizioni sorgente | HAR; mitmdump su Linux per acquisizioni mitmproxy native; [guida alle acquisizioni](../web-network-captures.md) | | Captured di pacchetti | Endpoint, DNS, TLS SNI, distribuzione dei protocolli e richieste HTTP | tshark e capinfos forniti dal chiamante; [guida pcap](../pcap-analysis.md) | | Assembly .NET | Metadati, istruzioni CIL, dipendenze native dichiarate e confronti di build | Nessuno (solo statico); [guida managed-code](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | APK Android | Dichiarazioni manifest, classi, metodi decompilati e riferimenti | JADX headless e un JDK completo su Linux/macOS/Windows x64; [guida Android](../android-analysis.md) | | Dispositivi Android | Dispositivi, pacchetti, processi, log, dump, screenshot e trasferimenti APK/file | adb fornito dal chiamante (emulatore o hardware); [guida ADB](../adb-device-analysis.md) | | Progetti JEB | Unità di progetto dal motore, digest degli artifact e pseudocodice di tipi/metodi | Client JEB avviato dal chiamante via MCP, selezionato con REA_JEB_MCP_URL; [guida JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_ja.md b/docs/readme/README_ja.md index 2022f088f0..f4595b6824 100644 --- a/docs/readme/README_ja.md +++ b/docs/readme/README_ja.md @@ -152,6 +152,7 @@ REA には Node.js 22.x（>=22.19）、24.x（>=24.11）、または 26+ と npm | 保存済みのネットワークキャプチャ | リクエスト、レスポンス、取得可能なペイロード、ソース内の位置 | HAR；mitmproxy ネイティブ形式のキャプチャには Linux 上の mitmdump；[キャプチャガイド](../web-network-captures.md) | | パケットキャプチャ | エンドポイント、DNS、TLS SNI、プロトコル分布、HTTPリクエスト | 呼び出し側が用意した tshark と capinfos；[pcapガイド](../pcap-analysis.md) | | .NET アセンブリ | メタデータ、CIL 命令、宣言されたネイティブ依存関係、ビルドの比較 | 静的な検査；[マネージドコードのガイド](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | Android APK | マニフェストの宣言、クラス、逆コンパイルされたメソッド、参照 | Linux/macOS/Windows x64 上のヘッドレス JADX と完全な JDK；[Android ガイド](../android-analysis.md) | | Android デバイス | デバイス、パッケージ、プロセス、ログ、ダンプ、スクリーンキャプチャ、APK/ファイル転送 | 呼び出し側が用意した adb（エミュレーターまたは実機）；[ADB ガイド](../adb-device-analysis.md) | | JEBプロジェクト | エンジンによるプロジェクトユニット、アーティファクトのダイジェスト、型/メソッドの擬似コード | 呼び出し側が起動し MCP を提供する JEB クライアント（REA_JEB_MCP_URL で選択）；[JEB ガイド](../jeb-analysis.md) | diff --git a/docs/readme/README_ko.md b/docs/readme/README_ko.md index 14ad6cbb96..3fa5f2569a 100644 --- a/docs/readme/README_ko.md +++ b/docs/readme/README_ko.md @@ -173,6 +173,7 @@ REA에는 Node.js 22.x(>=22.19), 24.x(>=24.11) 또는 26+와 npm이 필요합니 | 기록된 논리 신호 | 디코더 주석, 샘플 범위, 진단 정보 | Linux: sigrok-cli 0.7.2 + prlimit; [신호 기록 가이드](../signal-recordings.md) | | 패키지 및 리소스 | 파일 목록, 다이제스트, plist, Apple 번들 구조, 추출한 리소스 | [아티팩트 및 JavaScript 가이드](../javascript-artifact-reconstruction.md), [Apple 애플리케이션](../apple-application-analysis.md) | | 프로세스 동작 | 터미널 출력, 상호작용, 종료 및 파일 시스템 관찰 결과, 실행 비교 | 네이티브 PTY를 지원하는 Linux/macOS; [프로세스 캡처](../process-capture.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | 정적 JavaScript 및 .NET 검사는 제공된 파일을 읽으며 애플리케이션을 실행하지 않습니다. 런타임 캡처는 사용자의 권한으로 선택한 대상을 실행하거나 대상과 상호작용합니다. 각 런타임 가이드에서 그 영향을 설명합니다. diff --git a/docs/readme/README_ml.md b/docs/readme/README_ml.md index 1d715eb2b8..5e07f3663f 100644 --- a/docs/readme/README_ml.md +++ b/docs/readme/README_ml.md @@ -160,6 +160,7 @@ REA-യ്ക്ക് Node.js 22.x (>=22.19), 24.x (>=24.11), അല്ലെ | Websites | Page structure, scripts, network observations, ആവശ്യപ്പെട്ട screenshots | Chrome-family browser; [browser analysis](../browser-observation.md) | | Saved network captures | Requests, responses, പുറത്തുകാണുന്ന payload-ുകൾ, source locations | HAR അല്ലെങ്കിൽ Fiddler Everywhere SAZ; native mitmproxy-നായി Linux-ൽ mitmdump; [capture guide](../web-network-captures.md) | | .NET assemblies | Metadata, CIL instructions, പ്രഖ്യാപിച്ച native dependencies, build comparisons | ഒന്നും ആവശ്യമില്ല (static മാത്രം); [managed-code guide](../managed-code-analysis.md) | +| Python packaged apps | CArchive/PYZ entries, Python version, bytecode, resources and runtime options | Static PyInstaller parsing; [Python guide](../python-packaged-applications.md) | | Android APKs | Manifest declarations, classes, decompiled methods, references | Linux/macOS/Windows x64-ൽ Headless JADX-ഉം full JDK-ഉം; [Android guide](../android-analysis.md) | | Android devices | Devices, packages, processes, logs, dumps, screen captures, APK/file transfers | Caller നൽകുന്ന adb (emulator അല്ലെങ്കിൽ hardware); [ADB guide](../adb-device-analysis.md) | | JEB projects | Engine-backed project units, artifact digests, type/method pseudo-code | MCP സേവനം നൽകുന്ന caller ആരംഭിച്ച JEB client; REA_JEB_MCP_URL ഉപയോഗിച്ച് തിരഞ്ഞെടുക്കുക; [JEB guide](../jeb-analysis.md) | diff --git a/docs/readme/README_pl.md b/docs/readme/README_pl.md index e19413fd87..4c692d2364 100644 --- a/docs/readme/README_pl.md +++ b/docs/readme/README_pl.md @@ -152,6 +152,7 @@ REA wymaga Node.js 22.x (>=22.19), 24.x (>=24.11) lub 26+ oraz npm. Dodatkowe na | Zapisane przechwycenia ruchu sieciowego | Żądania, odpowiedzi, dostępne dane przesyłane i lokalizacje źródłowe | HAR; mitmdump w systemie Linux dla natywnych przechwyceń mitmproxy; [przewodnik przechwytywania](../web-network-captures.md) | | Przechwytywanie pakietów | Punkty końcowe, DNS, TLS SNI, dystrybucja protokołów i żądania HTTP | Dostarczone przez wywołującego tshark i capinfos; [przewodnik pcap](../pcap-analysis.md) | | Zestawy .NET | Metadane, instrukcje CIL, zadeklarowane zależności natywne i porównania kompilacji | Analiza statyczna; [przewodnik kodu zarządzanego](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | Pakiety APK dla Androida | Deklaracje manifestu, klasy, zdekompilowane metody i odwołania | JADX bez interfejsu graficznego i pełny JDK w systemach Linux/macOS/Windows x64; [przewodnik Androida](../android-analysis.md) | | Urządzenia z Androidem | Urządzenia, pakiety, procesy, logi, zrzuty, zrzuty ekranu oraz transfery APK/plików | adb dostarczony przez wywołującego (emulator lub urządzenie); [przewodnik ADB](../adb-device-analysis.md) | | Projekty JEB | Jednostki projektu obsługiwane przez silnik, skróty artefaktów i pseudokod typów/metod | Uruchomiony przez wywołującego klient JEB udostępniający MCP, wybrany przez REA_JEB_MCP_URL; [przewodnik JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_pt-BR.md b/docs/readme/README_pt-BR.md index efbd4e9b41..019bba1129 100644 --- a/docs/readme/README_pt-BR.md +++ b/docs/readme/README_pt-BR.md @@ -152,6 +152,7 @@ O REA requer Node.js 22.x (>=22.19), 24.x (>=24.11) ou 26+, além do npm. As fer | Capturas de rede salvas | Requisições, respostas, payloads acessíveis e localizações de origem | HAR; mitmdump no Linux para capturas no formato nativo do mitmproxy; [guia de capturas](../web-network-captures.md) | | Capturas de pacotes | Endpoints, DNS, TLS SNI, distribuição de protocolos e requisições HTTP | tshark e capinfos fornecidos pelo chamador; [guia pcap](../pcap-analysis.md) | | Assemblies .NET | Metadados, instruções CIL, dependências nativas declaradas e comparações de builds | Inspeção estática; [guia de código gerenciado](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | APKs Android | Declarações do manifesto, classes, métodos descompilados e referências | JADX sem interface gráfica e um JDK completo no Linux/macOS/Windows x64; [guia de Android](../android-analysis.md) | | Dispositivos Android | Dispositivos, pacotes, processos, logs, despejos, capturas de tela e transferências de APK/arquivos | adb fornecido pelo chamador (emulador ou aparelho); [guia de ADB](../adb-device-analysis.md) | | Projetos JEB | Unidades de projeto do motor, hashes de artefatos e pseudocódigo de tipos/métodos | Um cliente JEB iniciado pelo chamador servindo MCP, selecionado com REA_JEB_MCP_URL; [guia de JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_ru.md b/docs/readme/README_ru.md index 12f06df7bf..22ff426b8a 100644 --- a/docs/readme/README_ru.md +++ b/docs/readme/README_ru.md @@ -162,6 +162,7 @@ REA требует Node.js 22.x (>=22.19), 24.x (>=24.11) или 26+, а так | Записанные логические сигналы | Аннотации декодера, диапазоны отсчётов и диагностика | Linux: sigrok-cli 0.7.2 + prlimit; [руководство по сигналам](../signal-recordings.md) | | Пакеты и ресурсы | Списки файлов, хеши, plist, структура пакетов Apple и извлечённые ресурсы | [Руководство по артефактам и JavaScript](../javascript-artifact-reconstruction.md), [приложения Apple](../apple-application-analysis.md) | | Поведение процессов | Вывод терминала, взаимодействия, наблюдения за завершением и файловой системой, сравнения запусков | Linux/macOS с нативным PTY; [захват процессов](../process-capture.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | Статическое исследование JavaScript и .NET читает предоставленные файлы, не запуская приложение. Захват во время выполнения запускает выбранную цель или взаимодействует с ней с правами вашего пользователя; соответствующие руководства описывают последствия. diff --git a/docs/readme/README_th.md b/docs/readme/README_th.md index 304e0d5437..fb8c3ff604 100644 --- a/docs/readme/README_th.md +++ b/docs/readme/README_th.md @@ -152,6 +152,7 @@ REA ต้องใช้ Node.js 22.x (>=22.19), 24.x (>=24.11) หรือ 26 | ข้อมูลเครือข่ายที่บันทึกไว้ | คำขอ คำตอบ เพย์โหลดที่ปรากฏ และตำแหน่งต้นทาง | HAR; mitmdump บน Linux สำหรับข้อมูลที่บันทึกในรูปแบบเนทีฟของ mitmproxy; [คู่มือการบันทึกข้อมูล](../web-network-captures.md) | | การจับแพ็กเก็ต | จุดเชื่อมต่อ, DNS, TLS SNI, การกระจายโปรโตคอล และคำขอ HTTP | tshark และ capinfos ที่ผู้เรียกจัดเตรียม; [คู่มือ pcap](../pcap-analysis.md) | | แอสเซมบลี .NET | เมทาดาทา คำสั่ง CIL การพึ่งพาเนทีฟที่ประกาศไว้ และการเปรียบเทียบผลการบิลด์ | การตรวจสอบแบบสถิต; [คู่มือโค้ดแบบจัดการ](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | APK ของ Android | การประกาศใน manifest คลาส เมธอดที่ดีคอมไพล์ และการอ้างอิง | JADX แบบไม่มี GUI และ JDK แบบเต็มบน Linux/macOS/Windows x64; [คู่มือ Android](../android-analysis.md) | | อุปกรณ์ Android | อุปกรณ์ แพ็กเกจ โพรเซส บันทึก ดัมพ์ ภาพหน้าจอ และการถ่ายโอน APK/ไฟล์ | adb ที่ผู้เรียกจัดเตรียม (โปรแกรมจำลองหรืออุปกรณ์จริง); [คู่มือ ADB](../adb-device-analysis.md) | | โปรเจกต์ JEB | หน่วยโปรเจกต์จากเอนจิน, แฮชของอาร์ติแฟกต์ และซูโดโค้ดของไทป์/เมท็อด | ไคลเอนต์ JEB ที่ผู้เรียกเริ่มและให้บริการ MCP เลือกด้วย REA_JEB_MCP_URL; [คู่มือ JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_tr.md b/docs/readme/README_tr.md index b0ba8f343b..2cb89a5496 100644 --- a/docs/readme/README_tr.md +++ b/docs/readme/README_tr.md @@ -156,6 +156,7 @@ REA; Node.js 22.x (>=22.19), 24.x (>=24.11) veya 26+ ve npm gerektirir. Ek araç | Web siteleri | Sayfa yapısı, betikler, ağ gözlemleri ve istenen ekran görüntüleri | Chrome ailesinden bir tarayıcı; [tarayıcı analizi](../browser-observation.md) | | Kaydedilmiş ağ yakalamaları | İstekler, yanıtlar, erişilebilir yük içerikleri ve kaynak konumları | HAR veya Fiddler Everywhere SAZ; Linux'ta native mitmproxy için mitmdump; [ağ yakalama rehberi](../web-network-captures.md) | | .NET derlemeleri | Meta veriler, CIL talimatları, bildirilmiş yerel bağımlılıklar ve derleme karşılaştırmaları | Statik inceleme; [yönetilen kod rehberi](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | Android APK'leri | Manifest bildirimleri, sınıflar, tersine derlenmiş metotlar ve referanslar | Linux/macOS/Windows x64 üzerinde arayüzsüz JADX ve tam bir JDK; [Android rehberi](../android-analysis.md) | | Android cihazları | Cihazlar, paketler, süreçler, günlükler, dökümler, ekran görüntüleri ve APK/dosya aktarımları | Çağıranın sağladığı adb (emülatör veya cihaz); [ADB rehberi](../adb-device-analysis.md) | | JEB projeleri | Motor destekli proje birimleri, artefakt özetleri ve tür/metot sözdekodu | Çağıranın başlattığı, MCP sunan bir JEB istemcisi, REA_JEB_MCP_URL ile seçilir; [JEB rehberi](../jeb-analysis.md) | diff --git a/docs/readme/README_uk.md b/docs/readme/README_uk.md index 4a33ac96d4..eac65f273e 100644 --- a/docs/readme/README_uk.md +++ b/docs/readme/README_uk.md @@ -152,6 +152,7 @@ REA потребує Node.js 22.x (>=22.19), 24.x (>=24.11) або 26+, а та | Збережені мережеві захоплення | Запити, відповіді, доступні корисні дані та місця в джерелі | HAR; mitmdump у Linux для захоплень у власному форматі mitmproxy; [посібник мережевих захоплень](../web-network-captures.md) | | Захоплення пакетів | Кінцеві точки, DNS, TLS SNI, розподіл протоколів та HTTP-запити | Надані викликаючою стороною tshark та capinfos; [підручник pcap](../pcap-analysis.md) | | Збірки .NET | Метадані, інструкції CIL, оголошені нативні залежності й порівняння збірок | Статична перевірка; [посібник керованого коду](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | Android APK | Оголошення маніфесту, класи, декомпільовані методи й посилання | JADX без графічного інтерфейсу й повний JDK у Linux/macOS/Windows x64; [посібник Android](../android-analysis.md) | | Пристрої Android | Пристрої, пакети, процеси, журнали, дампи, знімки екрана та передавання APK/файлів | adb, наданий викликаючою стороною (емулятор або пристрій); [підручник ADB](../adb-device-analysis.md) | | Проєкти JEB | Єдиниці проєкту від рушія, дайджести артефактів і псевдокод типів/методів | Запущений викликаною стороною клієнт JEB з MCP, обирається через REA_JEB_MCP_URL; [посібник JEB](../jeb-analysis.md) | diff --git a/docs/readme/README_vi.md b/docs/readme/README_vi.md index 19f18c8ca2..cfba9f3c47 100644 --- a/docs/readme/README_vi.md +++ b/docs/readme/README_vi.md @@ -162,6 +162,7 @@ REA yêu cầu Node.js 22.x (>=22.19), 24.x (>=24.11) hoặc 26+, cùng với np | Tín hiệu logic đã ghi | Chú thích bộ giải mã, phạm vi mẫu và chẩn đoán | Linux: sigrok-cli 0.7.2 + prlimit; [hướng dẫn tín hiệu](../signal-recordings.md) | | Gói và tài nguyên | Danh sách tệp, giá trị băm, plist, cấu trúc bundle Apple và tài nguyên được trích xuất | [Hướng dẫn hiện vật phần mềm và JavaScript](../javascript-artifact-reconstruction.md), [ứng dụng Apple](../apple-application-analysis.md) | | Hành vi tiến trình | Đầu ra terminal, tương tác, quan sát khi thoát và hệ thống tệp, cùng so sánh các lần chạy | Linux/macOS có PTY gốc; [ghi lại tiến trình](../process-capture.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | Kiểm tra tĩnh JavaScript và .NET đọc các tệp được cung cấp mà không chạy ứng dụng. Ghi lại khi chạy sẽ khởi chạy hoặc tương tác với mục tiêu đã chọn bằng quyền người dùng của bạn; từng hướng dẫn mô tả tác động của nó. diff --git a/docs/readme/README_zh-TW.md b/docs/readme/README_zh-TW.md index 403cf3d2a4..635bcbc9af 100644 --- a/docs/readme/README_zh-TW.md +++ b/docs/readme/README_zh-TW.md @@ -156,6 +156,7 @@ REA 需要 Node.js 22.x（>=22.19）、24.x（>=24.11）或 26+，以及 npm。 | 已保存的網路擷取 | 請求、回應、可訪問的酬載和來源位置 | HAR；原生 mitmproxy 擷取需要 Linux 上的 mitmdump；[擷取指南](../web-network-captures.md) | | 封包擷取 | 端點、DNS、TLS SNI、通訊協定分佈和 HTTP 請求 | 呼叫端提供的 tshark 和 capinfos；[pcap 指南](../pcap-analysis.md) | | .NET 程式集 | 中繼資料、CIL 指令、聲明的原生依賴和建置對比 | 靜態檢查；[托管程式碼指南](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | Android APK | 清單聲明、類別、反編譯的方法和引用 | Linux/macOS/Windows x64 上的無界面 JADX 和完整 JDK；[Android 指南](../android-analysis.md) | | Android 裝置 | 裝置、套件、程序、日誌、傾印、螢幕擷取以及 APK/檔案傳輸 | 呼叫端提供的 adb（模擬器或實體裝置）；[ADB 指南](../adb-device-analysis.md) | | JEB 專案 | 引擎支援的專案單元、成品摘要和型別/方法的虛擬碼 | 由呼叫端啟動、提供 MCP 的 JEB 用戶端，透過 REA_JEB_MCP_URL 選擇；[JEB 指南](../jeb-analysis.md) | diff --git a/docs/readme/README_zh.md b/docs/readme/README_zh.md index 51275f4c08..313262ae53 100644 --- a/docs/readme/README_zh.md +++ b/docs/readme/README_zh.md @@ -156,6 +156,7 @@ REA 需要 Node.js 22.x（>=22.19）、24.x（>=24.11）或 26+，以及 npm。 | 已保存的网络捕获 | 请求、响应、可访问的载荷和来源位置 | HAR；原生 mitmproxy 捕获需要 Linux 上的 mitmdump；[捕获指南](../web-network-captures.md) | | 数据包捕获 | 端点、DNS、TLS SNI、协议分布和 HTTP 请求 | 调用方提供的 tshark 和 capinfos；[pcap 指南](../pcap-analysis.md) | | .NET 程序集 | 元数据、CIL 指令、声明的原生依赖和构建对比 | 静态检查；[托管代码指南](../managed-code-analysis.md) | +| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) | | Android APK | 清单声明、类、反编译的方法和引用 | Linux/macOS/Windows x64 上的无界面 JADX 和完整 JDK；[Android 指南](../android-analysis.md) | | Android 设备 | 设备、软件包、进程、日志、转储、屏幕截图以及 APK/文件传输 | 调用方提供的 adb（模拟器或实体设备）；[ADB 指南](../adb-device-analysis.md) | | JEB 项目 | 引擎支持的项目单元、工件摘要和类型/方法的伪代码 | 由调用方启动、提供 MCP 的 JEB 客户端，通过 REA_JEB_MCP_URL 选择；[JEB 指南](../jeb-analysis.md) | diff --git a/docs/rust-binaries.md b/docs/rust-binaries.md index e0f7a28f1c..a2cc0ff6fe 100644 --- a/docs/rust-binaries.md +++ b/docs/rust-binaries.md @@ -21,6 +21,67 @@ requires an absolute filesystem path and preserves that exact selection: ## Observations +Complete delivery remains the default. For a dependency question, select the +needed records on the first call instead of returning the complete graph: + +```json +{ + "name": "inspect_rust_binary", + "arguments": { + "path": "/artifacts/application", + "view": { + "collection": "packages", + "names": ["serde", "serde_derive"], + "fields": ["name", "version", "source", "dependencies"] + } + } +} +``` + +`detail: "summary"` returns section ranges, root index, package/edge/cycle counts +and compiler-hash count. A `view` selects compact delivery automatically; +combining it with explicit `detail: "complete"` is an input error. The complete +parent Evidence is retained in the MCP session and remains exportable. Subsequent +`inspect_analysis_view` calls accept that `parent_evidence_id`, or its portable +Evidence file, without reopening the binary: + +```json +{ + "source": { + "kind": "retained-evidence", + "evidence_id": "ev_" + }, + "view": { + "kind": "rust", + "collection": "edges", + "from_indices": [12], + "limit": 20 + } +} +``` + +Package filters accept exact `names`, `versions` and original `indices`; edge +filters accept original edge `indices`, `from_indices` and `to_indices`. +Filters intersect, and an empty filter array selects nothing. `offset` counts +matching records; `limit` defaults to 64. Pages retain original indices scoped +to the parent Evidence, selected fields, examined coverage and the next offset. +Default package fields omit dependency vectors; explicitly select +`dependencies` to obtain direct references. No dependency path is traversed. + +The CLI uses the same views and requires an exclusive output bundle for compact +delivery so the complete Evidence survives process exit: + +```bash +rea inspect-rust-binary /artifacts/application --json \ + --view '{"collection":"packages","names":["serde"],"fields":["name","version"]}' \ + --evidence_output /artifacts/rust-evidence.json +``` + +Compact results distinguish complete embedded-record coverage from unknown +runtime dependency coverage. A missing section remains unknown rather than an +empty dependency set. Retained views describe the captured artifact digest; +they do not assert that the current file or running application is unchanged. + The inline Evidence result contains the artifact path, SHA-256, byte length, container format, architecture, word size, and byte order. When a `.dep-v0` section is present, `auditable_metadata` contains: diff --git a/docs/specs/agent-environments/prefix-variance/plan.md b/docs/specs/agent-environments/prefix-variance/plan.md new file mode 100644 index 0000000000..8e13bfe856 --- /dev/null +++ b/docs/specs/agent-environments/prefix-variance/plan.md @@ -0,0 +1,18 @@ +# Plan + +1. Move tool naming (`parameterNames`, `pathTemplate`, `toolName`, + `compareTools`) from `environmentManifestRules.ts` to a new + `environmentManifestNames.ts`, so the rules module has room for the rule. +2. Add `prefix-variance` to `TEMPLATE_RULES`. A `prefix-variance` parameter + records the rule it shares (`sharedRule`), which names it and keys it. +3. Add the pass after `sibling-variance`: walk positions in order, index the + opt-in parameters at each position by their preceding parts, template + qualifying literals, then rebucket by naming key, keeping a tool's own rule + for each merged position. +4. Share the 32-segment split between the three comparing passes and list the + segment-limited rules once for the limitation. +5. Document the rule in the view schema description and in + `docs/web-network-captures.md`. +6. Tests: shared positions across methods and depths, the spelling condition, + merges that keep a tool's own rule, pairs, format-rule positions and the + segment limit. diff --git a/docs/specs/agent-environments/prefix-variance/spec.md b/docs/specs/agent-environments/prefix-variance/spec.md new file mode 100644 index 0000000000..a6baf4e5e3 --- /dev/null +++ b/docs/specs/agent-environments/prefix-variance/spec.md @@ -0,0 +1,59 @@ +# Share templated positions across tools + +Issue: https://github.com/morluto/rea/issues/4933 + +## Outcome + +A session usually edits one scratch entity: it creates it, then updates and +deletes it at a fixed path such as `/api/v1/collections/0RZ576QZQA5KG`. The +manifest templates a segment only when sibling routes of the same method vary +there, so those writes stay literal tools (`patch_collections_0rz576qzqa5kg`) +even though the reads of the same collection were templated as +`/api/v1/collections/{collection_id}`. A literal write tool addresses one +entity that may not exist in a later state, and nothing can model its effect. + +A new opt-in template rule, `prefix-variance`, templates such a literal with +the parameter another tool already uses at that position, so the write becomes +`patch_collections_by_collection_id` on the same template as the read. + +## Requirements + +- `prefix-variance` applies only when listed, after every other opt-in rule, so + it can share the parameters they produce. +- A literal at position `p` is templated when another tool on the same origin, + of any method and any segment count, has a parameter at `p` from one of the + other opt-in rules, after the same preceding parts. Parameters in the + preceding parts compare by the rule that names them, so a position templated + by `prefix-variance` matches the parameter it shares. +- The literal qualifies when it is one of the values such a parameter + observed at that position, when it carries a digit and has the length and + character classes (digits, lower case, upper case, each other character) of + one of them, or when every value the parameter observed, one or more, has one + length of eight or more characters, as generated identifiers often do, and + the literal has that length and uses only their character classes, so an + identifier that happens to hold no digit still qualifies. An empty literal, a + stop word or a `v1`-style version never qualifies. Each parameter's spelling + is summarised once, so a literal is checked without rescanning its values, + and a value longer than 128 characters is compared only exactly, so a huge + segment costs no classification. +- Positions templated by the four format rules are never shared, so + `/api/teams/{numeric}` still absorbs no literal. +- Values stay with the parameter that observed them: when parameters of + different rules share a position, the literal shares the one that observed + it, else the earliest one it is spelled like. +- The parameter takes the name of the parameter it shares (`collection_id`, + `uid`, a pair's `_namespace` and `_name`) and is labelled `prefix-variance`. +- Tools whose routes now match, by method and by the rule that names each + parameter, merge. A merged parameter keeps the rule of a tool that templated + that position itself, when one did. +- Like the sibling rules, it compares routes of at most 32 segments and says so + in a limitation when a longer route exists. + +## Non-goals + +- No change to the default rules or to any other rule: manifests that do not + list `prefix-variance` are byte-for-byte unchanged. +- No templating from response bodies, such as an id a create returned; the + manifest reads request lines only. +- No change to which groups are excluded: a write whose response has no body + or content type outside `/api/` is still excluded as `non-json`. diff --git a/docs/specs/agent-environments/prefix-variance/tasks.md b/docs/specs/agent-environments/prefix-variance/tasks.md new file mode 100644 index 0000000000..9708eff558 --- /dev/null +++ b/docs/specs/agent-environments/prefix-variance/tasks.md @@ -0,0 +1,10 @@ +# Tasks + +- [x] Record the outcome, requirements and non-goals. +- [x] Move tool naming into its own module. +- [x] Add the `prefix-variance` rule, its names, its merge and its limitation. +- [x] Document the rule in the schema and the capture guide. +- [x] Add rule tests. +- [x] Run static, formatting, duplication, documentation and test gates. +- [x] Project the 51 corpus captures with and without the rule and record the results. +- [x] Rebuild five randomly drawn environments end to end and compare them with their published versions and the live applications. diff --git a/docs/specs/agent-environments/prefix-variance/verification.md b/docs/specs/agent-environments/prefix-variance/verification.md new file mode 100644 index 0000000000..d337159d1a --- /dev/null +++ b/docs/specs/agent-environments/prefix-variance/verification.md @@ -0,0 +1,149 @@ +# Verification + +Verified base: 33b0643e on main 1629266e (the rule, the alias removal `knip` +asked for, and the fixed-length clause the end-to-end runs below led to). Executed on macOS with Node v22.23.1 (the repository pins +24.18.0 in `.nvmrc`, which was not installed). + +## Gates + +- `npm run build:cached`, `npm run check:fast` (typecheck and lint), + `npm run format:check`, `npx knip`, `npm run scan:todos`, + `npm run docs:check`, `npm run verify:test-discovery` and + `npm run mcp-catalog:check`: passed. +- `npm run jscpd`: 3.49% duplicated lines over 3561 files with 3067 clones; + main measures 3.49% with 3068, since the shared segment-limit split removes + one clone. +- oxlint reports no new finding on the changed files. The manifest accumulator + keeps main's three warnings (file length, and the complexity and parameter + count of its record append), one line longer. +- `npx vitest run --project domain --project services`: 7119 passed, 5 + pre-existing skips. +- `npm run test:prepare`, then `npm run test:focused --` the analysis-view MCP + and CLI boundaries, the compact input schema and tool schema validity + boundaries, and the environment serve and export boundaries: + 7 files, 76 passed. + +New tests in `environmentManifestView.test.ts` cover: + +- a literal shared across methods and depths, and the same capture without the rule; +- the spelling condition: same classes and length, case, missing digit, a colon word, a version; +- a digit-free member of a fixed-length identifier family, against a shorter word, a hyphenated + value and a family shorter than eight characters; +- merges that keep a tool's own rule, including a sibling-variance parameter under a shared one; +- both halves of a templated pair; +- format-rule positions and the segment limit. + +## The showcase captures + +The 50 showcase captures (45 HAR inspections and 5 `capture_browser_scenario` +captures) were projected with main's build and this branch's. + +- Without the rule, both builds give identical results for every capture: the + manifest with no rules, the manifest with `after-uid-literal`, + `opaque-token`, `sibling-variance` and `sibling-pair-variance`, and the + environment-state view. 150 of 150 outputs hash equal. +- With `prefix-variance` added to `after-uid-literal`, `opaque-token` and + `sibling-variance`, 69 literal tools in 10 captures are templated, 25 of them + writes: + - Komga: collection and readlist `PATCH` and `DELETE`, book pages. + - n8n: workflow `PATCH`, `DELETE` and archive, and the reads of a workflow + whose id holds no digit. + - Memos: memo `PATCH` and `DELETE`, user settings reads, and the reads of + two memos whose ids hold no digit. + - Navidrome: playlist `PUT` and `DELETE`. + - Mealie: recipe `PATCH` and `DELETE`. + - Grist: document `PATCH` and `DELETE`, and table record writes. + - Redpanda Console: subject and topic writes. + - Sentry: release `PUT` and `DELETE`. + - Shlink: short URL `PATCH`. + - Mattermost: `me` reads, which `sibling-variance` already folds into the + user parameter. + + Tools go from 1591 to 1578, because Redpanda Console's per-subject reads and + the per-memo and per-workflow reads merge. + +- Literals the rule leaves alone, as intended, include Espo CRM `layout`, + Federal Register `facets`, Ghost `slug`, Shlink `stats`, Lychee + `Metrics::photo` and Vault's path names. +- The environment-state view succeeds on every capture with the rule. Its types + move to the templated collections (531 become 526 as literal ones merge); the + one fewer item view is the Redpanda Console merge. + +With the write plans of #5600 (its branch with this commit and main merged in, +locally): + +- Served write tools rise from 33 to 39: Komga collection create, update and + delete; n8n workflow delete; Sentry release update and delete. +- Write tools with no state target fall from 102 to 87. +- The newly modelled tools that stay unserved are limited by the capture: + - no read after the write: Komga readlists, Mealie, Navidrome, Redpanda Console; + - n8n's item read shows an expanded `shared` relation its create response + never returned; + - Shlink returns `tags` sorted while the update echoes the request's order. + +## Five environments end to end + +Five of the ten environments the rule changes were drawn at random, redrawing +until the draw held one whose new writes are served: Komga, Mealie, n8n, +Sentry and Shlink. Each was rebuilt on its own Linux VM with `npm pack` of +#5600, main and this branch merged locally, from the same scoped HAR as its +published version: manifest with `prefix-variance`, exported bundle, smoke +task, the generated retrieval tasks for `gpt-oss-120b` and `GLM-5.3-Flash`, and +a dry-run dataset. Where writes are served, hand-written write tasks ask for a +change in plain language and are graded from the environment's state through +`data/get`. The first n8n run found that a digit-free workflow id stayed +literal; the fixed-length clause fixed it, and n8n was rebuilt. + +| Environment | Tools (published → rebuilt) | Served from state (reads / writes) | Retrieval gpt-oss | Retrieval GLM | Writes gpt-oss | Writes GLM | +| ----------- | --------------------------- | ---------------------------------- | ----------------- | ------------- | -------------- | ---------- | +| Komga | 42 → 46 | 26 / 3 | 12/12 (was 11/12) | 12/12 (11/12) | 3/4 | 4/4 | +| Mealie | 39 → 39 | 36 / 0 | 12/12 (12/12) | 12/12 (12/12) | – | – | +| n8n | 30 → 33 | 17 / 1 | 9/9 (9/9) | 9/9 (9/9) | 2/2 | 2/2 | +| Sentry | 34 → 34 | 26 / 3 | 12/12 (12/12) | 11/12 (12/12) | 2/4, rerun 0/4 | 4/4 | +| Shlink | 17 → 17 | 5 / 0 | 5/7, rerun 7/7 | 7/7 (7/7) | – | – | + +Every smoke task passed. The misses: + +- Sentry writes with gpt-oss: every failed episode passed `id: "sentry"` for + the `/api/0/` segment the tool schema describes as `observed: 0`, so its calls + addressed `/api/sentry/…`; every passing episode, and every GLM episode, passed + `0`. The served tools work; the format-rule parameter for a segment that only + ever held `0` invites the guess. +- Komga update with gpt-oss and Shlink's first gpt-oss run: the model provider + rejected tool calls the model generated (`null` and `undefined` for optional + query arguments) before the environment saw them. +- Sentry `qa-03` with GLM: the model never called the alert-rules tool. + +The same calls were also replayed against the live applications, seeded by +the same scripts (ids differ, so each side maps its ids to names): + +- Komga 1.29.1, 9 of 12 steps equal: create, update and delete of a + collection and the reads after them agree except for the generated id. The + collection list after a create omits the new collection, because that list + keeps its recorded membership, and the 404 for a deleted collection replays + the recorded body with its path and timestamp. +- n8n 1.123.85, 3 of 8 steps equal: n8n refuses to delete a workflow that is + not archived (400) and the environment deletes it; the workflow list replays + its recorded response, so a delete does not show in it; archiving is not + served. The read of the digit-free workflow failed in the first build (the + state held only its list fields) and agrees in the second, apart from ids the + seed generates. + +## After the review round + +The branch merged main at 52c84f6f, then kept each shared position's values +with the parameter that observed them (one donor per naming rule, the literal +taking the donor that observed it, else the earliest it is spelled like), +summarised each donor's spelling once, and let one observation form a +fixed-length family, as the requirement states. Two new tests, a mixed-rule +position and a single observed workflow id, fail on the previous head and pass +here. The gates above passed again (domain and services 7255 passed with 5 +pre-existing skips; the focused boundaries 76; `jscpd` 3.49%). Over the 50 +showcase captures the tools with and without `prefix-variance` are unchanged, +name for name: 69 literal tools templated, 25 of them writes. + +A second review round bounded the spelling summary: characters are classified +in one pass, and a value longer than 128 characters is compared only exactly. +Its test fails on the previous head, the gates passed again (domain and +services 7257, focused boundaries 76, `jscpd` 3.49%), and the census over the 50 +captures is again unchanged. diff --git a/docs/sqlite-databases.md b/docs/sqlite-databases.md index 087c5d27a7..4a9a43e9ca 100644 --- a/docs/sqlite-databases.md +++ b/docs/sqlite-databases.md @@ -102,8 +102,29 @@ bounds simultaneous input buffering and private snapshot/worker counts. The database and WAL together are limited to 256 MiB. SQLite values/records, SQL statements and the complete worker JSON reply each have a 16 MiB limit. -The owned worker has a 256 MiB JavaScript heap limit and a 30-second inspection -deadline. SQLite native allocations are separate from the JavaScript heap; +The owned worker has a 256 MiB JavaScript heap limit and defaults to a 30,000 ms +deadline. Select optional MCP `worker_timeout_ms`, or CLI +`--worker-timeout-ms `, to change the budget for that call. Values +must be integer milliseconds from 1 to 2,147,483,647 (the runtime timer range). +It covers worker startup, queries and reply writing, **not** DB/WAL snapshot +acquisition, final reply reads, cleanup or delivery. Allow corresponding headroom +in the MCP client's whole-call deadline; increasing only that deadline does not +change the worker budget. No server restart is needed. + +The result's `worker_timeout_ms` reports the actual parent-supervised budget, +including the default. Historical Evidence without this field has unknown +execution settings, not an inferred default. Explicit input selection is also +retained in Evidence parameters; omission stays omitted. Completed observations +retained under cancellation or cleanup failure keep the same budget facts. +Timeout errors name the selected milliseconds, exact CLI/MCP controls, and +cleanup-before-retry requirements. Incomplete cleanup remains a blocker even +when a larger budget is available. + +For example, `rea inspect-sqlite-database ./application.sqlite +--worker-timeout-ms 60000 --json` selects 60 seconds; equivalent MCP input is +`{"path":"/artifacts/application.sqlite","worker_timeout_ms":60000}`. + +SQLite native allocations are separate from the JavaScript heap; overall native heap and total resident memory are not capped. Row limits do not override these byte and resource limits. Resource-limit failures do not return an incomplete schema as successful complete evidence. diff --git a/docs/testing.md b/docs/testing.md index c6abc1411b..78bed167a6 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -156,7 +156,27 @@ rejection without a reason. That cleanup stays owned until a later close or call releases it. Another server does not open while the cleanup remains, and a cleanup that already succeeded is not run again. Pass `mcpCall` when a case needs its own server, environment, or lifecycle. CLI -cases still start a fresh process. +cases for that helper share one compiled CLI process per child environment. +The first case pays process startup; later cases reuse the loaded runtime. +The process restarts when the environment changes, after 32 commands, or when +the worker reports a resident set of at least 1 GiB. A fatal worker failure +or a broken session frame retires that process and waits for it to exit before +the next command starts. +`cli.run` still starts a fresh process, including stdin and explicit deadlines. +`runFileCli` uses the file session for commands that have neither. Go binary, +IL2CPP metadata, and dylib-resolution commands use that session. File teardown +stops the shared CLI process, including startup still in flight, and does not +open another after close. +`exportedFunctionBindingsCli.test.ts` keeps its own compiled CLI parity +process for the file and still compares that process's JSON command results +with the MCP session. A command that does not respond is rejected. That +process stays owned until it exits, and another process does not start while +it remains. A process that stays running after SIGKILL remains owned so a +later close can retry. Closing the session reports a failure when the process +does not exit after stdin closes, once the child has been stopped. +`applicationWorkflowCli.test.ts` keeps one compiled CLI process for its JSON +commands, including a non-zero status whose stdout is still a JSON document, +and closes that process after the file. The Apple artifact verifier step shares a job-temporary bytecode cache among its fresh CLI/MCP children via `REA_ARTIFACT_NODE_COMPILE_CACHE`. The artifact @@ -272,7 +292,23 @@ termination. Simulated Windows controls do not establish native Windows behavior Native process-ownership controls distinguish start identity, zombie lifecycle, current parent/group relation, and token-read evidence. Retain generation-bound candidates through smaller lineage observations, table omissions, reparenting, -group changes, and failed cleanup retries. A readable token with unavailable +group changes, and failed cleanup retries. Identityless exit-refresh controls +cover a replacement row that is not linked to the prior descendant set: a +matching token on its own group is signaled, a foreign or absent token is not, +an unreadable live token stays unresolved, a replacement that appears only in +the confirmation table is still token-checked, a group discovered during +revalidation is signaled before cleanup succeeds, a member that changes groups +during confirmation is signaled in its new group, a PID first seen with a +foreign token is read again on the confirmation table, a PID whose token was +already read is read again even when its parent and group return to the +pre-exit values, a pre-existing PID is read again when its parent or group +changes, an unchanged pre-existing row that was never token-read stays unread, +an unbounded confirmation +chain stays unresolved, lineage reads a row again when its launcher-tree +membership changes, a cached lineage token is read again on the next refresh, an initial descendant whose token already matched is read again on a later refresh, the launcher is read again on that refresh and a missing, unowned, or relation-changed launcher PID stays unavailable, a descendant absent from the latest refresh is omitted, +a group that reappears after it was processed is revalidated once, a further +reappearance stays unresolved, lineage publishes one row per PID, and lineage does not publish a +verified empty descendant list while that row remains unexamined. A readable token with unavailable identity stays unresolved. Native UID facts, rather than an earlier process-table row, determine whether a token denial concerns another user. Darwin tests compile the helper and exercise snapshot growth, absence, lifecycle, and UID observations; @@ -485,9 +521,15 @@ reports even when a shard fails, uploads available diagnostics, and writes the slowest files to the workflow summary. Partial reports are marked diagnostic only. Failed, cancelled, or missing selected shards still fail the required gate, and aggregate coverage thresholds remain enforced in full qualification. -Only successful complete full-tier merges publish the small `test-timings` artifact, with -checkout-relative paths; the build job downloads that artifact for the sequencer -instead of the coverage/JUnit bundle. Timing weights come from Vitest module +Only a successful complete merge on the default branch publishes shard history, +and only from a push, schedule, or manual run. Fast runs publish +`test-timings-fast`. Full runs publish `test-timings-full`. Pull requests do not +publish it: the lookup reads one unpaginated artifact page, and pull-request +artifacts would crowd out the default branch. The build job downloads the +artifact that matches its tier, so fast shards use durations measured without +coverage and full shards keep coverage-weighted durations. A tier with no +qualifying artifact leaves history unset. The download is that small +checkout-relative timing file, not the coverage/JUnit bundle. Timing weights come from Vitest module diagnostics and include tests and hooks, worker preparation, environment, imports, and setup. Import timings already include transform waits, which are not added again. The weights describe file costs under the measured worker and @@ -1354,11 +1396,22 @@ catalog, and managed-conformance metadata. Each shard still installs the locked dependencies and runs its assigned tests; full qualification enables coverage. It no longer regenerates the same metadata independently. -The build job also finds an available `test-timings` artifact from a successful -trusted default-branch scheduled or manual qualification and includes that same -report in each shard's snapshot. The bounded lookup verifies the artifact's -producer; newer schedules that reuse qualification without producing timings do -not hide the last full run's artifact. +The build job finds the newest trusted timing artifact for its tier +(`test-timings-fast` or `test-timings-full`) from a successful default-branch +push, schedule, or manual CI run, and includes that same report in each shard's +snapshot. The bounded lookup verifies the producer. Pull-request runs and other +branches are not producers. A tier with no qualifying artifact leaves history +unset, and Vitest uses its own assignment. A later schedule that reuses +qualification without uploading timings leaves the previous artifact in place. + +On 2026-10-21 the newest full history was run +`39545047294`, recorded before compiled-CLI session reuse. Fast run +`39552964873` packed its eight built shards from that history, and those shard +jobs executed for 319–591 seconds. Replaying that run's file durations through +the sequencer's two-worker schedule balances every shard at about 404 seconds +of file time. The same replay with the stale history weights leaves the slowest +shard near 547 seconds and the fastest near 293. That spread is file-schedule +time, not runner queue, which is why each tier keeps its own history. CI assigns discovered files by historical duration, with a deterministic fallback weight for new files, rather than dividing only by file count. Each file keeps its full duration on one worker. Extra workers overlap other diff --git a/docs/wasm-artifacts.md b/docs/wasm-artifacts.md index 7093df0a06..12bedbd379 100644 --- a/docs/wasm-artifacts.md +++ b/docs/wasm-artifacts.md @@ -26,6 +26,17 @@ execution validates tool files and version banners. Host process-ownership and private-workspace controls must also be available. The real-tool profile has been exercised on macOS arm64; other hosts require their own verification. +Configuration failures do not establish anything about the selected WASM bytes. +For a rejected version probe, `details.diagnostics` identifies the executable, +tool, expected version and configuration key; `details.captured_output` preserves +its actual stdout and stderr, including empty banners or warnings. Fix the +reported directory, executable access or release rather than changing the module +or repeating an unchanged call. All three tools must report 1.0.42 on stdout with +empty stderr. For CLI calls, set the environment before the next invocation. For +MCP, change the server's launch environment and restart/reconnect that server; +exporting a variable in a different shell does not update a running server. +REA never installs tools or relaxes version admission to recover. + ```bash REA_WABT_BIN_DIRECTORY=/absolute/wabt/bin rea inspect-wasm-artifact \ /artifacts/module.wasm --glue_paths /artifacts/glue.js \ @@ -64,7 +75,27 @@ The tool profile preserves upstream release/source identity, resolved executable paths, executable digests, exact banners and arguments. Executable digests are checked again after each command. Caller-supplied tools are trusted programs; version and digest recording are not publisher authentication or an OS sandbox. -Each subprocess has a 30-second deadline and 32 MiB combined output budget. +Each WABT version/analysis command defaults to a 30,000 ms deadline. Set optional +MCP `command_timeout_ms`, or CLI `--command-timeout-ms `, to select a +positive integer up to 2,147,483,647 ms (the runtime timer range) for that call. +The selected deadline is retained as `tool_profile.timeout_ms` and, when supplied, +in Evidence parameters. It applies separately to each command, not the whole +inspection, file admission or cleanup. The MCP client must allow time for all +commands, file checks, cleanup and delivery. A provider timeout reports the +actual selected deadline and these recovery controls; increasing only the client +deadline does not change the provider budget. No server restart is required to +change this per-call setting. Resolve incomplete cleanup before another attempt. + +For example, `rea inspect-wasm-artifact /artifacts/module.wasm +--command-timeout-ms 60000 --format json` uses a 60-second per-command deadline; +the equivalent MCP input is `{"path":"/artifacts/module.wasm", +"command_timeout_ms":60000,"detail":"summary"}`. Summary delivery retains the +complete Evidence but does not reduce WABT analysis work. + +The 8 MiB input allowance and 32 MiB combined command-output budget remain fixed; +increasing `command_timeout_ms`, host memory or the client deadline does not raise +them. Do not truncate required module bytes to fit. Select another suitable +complete artifact only if it answers the question, otherwise report the limit. Inspection output past the budget fails with `resource_constraint`; an oversized `--version` probe still reports `unreadable_output`. Both streams share the budget, so a stream that starts after the other has filled it retains nothing. diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md index ebcab44e3c..5df97ae319 100644 --- a/docs/web-network-captures.md +++ b/docs/web-network-captures.md @@ -449,12 +449,13 @@ template and `params` carry the four format rules with names `id`, `uuid`, only when listed, each at most once (a repeated rule is an `invalid_request`), in this order: -| Rule | Applies to | Parameter | -| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------- | -| `after-uid-literal` | The literal segment right after a literal `uid`, unless it is a stop word | `uid` | -| `opaque-token` | A lowercase alphanumeric segment of twelve or more characters mixing letters and digits that is not a `v1`-style version | `uid` | -| `sibling-pair-variance` | Groups equal in method, origin and segment count that differ in two adjacent literals right after a plural literal at index 2 or later and followed by a literal, with at least three distinct pairs and both positions varying (`/repos/{owner}/{repo}/issues`); stop words and `v1`-style versions never qualify | singular plural literal plus `_namespace`, then `_name` | -| `sibling-variance` | Groups equal in method, origin and segment count that differ in one literal at index 2 or later; the siblings whose literal is identifier-like (a digit or `:`, or three or more hyphenated names under a plural literal) merge, never a stop word, and the other literals stay their own tools | singular previous literal plus `_id`, else `id` | +| Rule | Applies to | Parameter | +| ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------- | +| `after-uid-literal` | The literal segment right after a literal `uid`, unless it is a stop word | `uid` | +| `opaque-token` | A lowercase alphanumeric segment of twelve or more characters mixing letters and digits that is not a `v1`-style version | `uid` | +| `sibling-pair-variance` | Groups equal in method, origin and segment count that differ in two adjacent literals right after a plural literal at index 2 or later and followed by a literal, with at least three distinct pairs and both positions varying (`/repos/{owner}/{repo}/issues`); stop words and `v1`-style versions never qualify | singular plural literal plus `_namespace`, then `_name` | +| `sibling-variance` | Groups equal in method, origin and segment count that differ in one literal at index 2 or later; the siblings whose literal is identifier-like (a digit or `:`, or three or more hyphenated names under a plural literal) merge, never a stop word, and the other literals stay their own tools | singular previous literal plus `_id`, else `id` | +| `prefix-variance` | A literal at a position that another tool, of any method or segment count, templates by one of the rules above after the same preceding segments, when the literal is one of that parameter's observed values, carries a digit and has the length and character classes (digits, lower case, upper case, each other character) of one, or has the single length of eight or more characters every observed value has and only their character classes; stop words and `v1`-style versions never qualify, and tools that now match merge | the name of the parameter it shares | With `sibling-variance` selected, literal siblings within its comparison scope fold into existing opt-in parameters as observed values unless they are stop words. @@ -471,7 +472,17 @@ cut to their first label, `_by_` appended when parameters exist, sanitized to `[a-z0-9_]`, at most 60 characters and made unique with numeric suffixes: `get_dashboards_by_uid`, `get_plugins_settings_by_plugin_id`. -`sibling-variance` and `sibling-pair-variance` compare routes of at most 32 segments; longer routes keep +`prefix-variance` runs last so it can share the parameters the other rules +produce: with `sibling-variance`, a `PATCH` or `DELETE` of the one entity a +session edited, `/api/v1/collections/0RZ576QZQA5KG`, becomes +`/api/v1/collections/{collection_id}` beside the reads that varied there, and +so do deeper routes such as `/api/v1/collections/{collection_id}/books`. Its +parameter is labelled `prefix-variance`, unless a merged tool templated the +position by its own rule. When parameters of different rules share a position, +a literal shares the one that observed it, else the earliest one it is spelled +like. + +`sibling-variance`, `sibling-pair-variance` and `prefix-variance` compare routes of at most 32 segments; longer routes keep their grouped templates and the manifest says so in a limitation. `body_shape` is derived for `POST`, `PUT` and `PATCH` from the tool's earliest record when it carries an `application/json` or `application/*+json` request body: top-level diff --git a/docs/web-runtime.md b/docs/web-runtime.md index 9514f66cf1..63b99d6968 100644 --- a/docs/web-runtime.md +++ b/docs/web-runtime.md @@ -32,7 +32,9 @@ A document replacement ends the window; final coverage then remains unavailable. Same-document navigation such as `history.pushState()` remains valid when the frame and known loader identity stay stable and the current origin is allowed. Unknown loader identity retains a conservative URL check; newly reported or replaced loaders are rejected. Malformed producer events end an armed window immediately and trigger cleanup, independently of the selected duration. -Target detachment ends the window even when the browser's root connection stays open. When the closed session cannot acknowledge cleanup, the operation returns a failure with the selected target, failed commands and original producer reason; it does not claim successful cleanup. +Target detachment ends the window with `session_detached`, even when the browser's root connection stays open. An `Inspector.targetCrashed` event ends it with `target_terminated`. A lost transport records `transport_disconnected` in browser observation; execution capture returns the typed transport failure. These outcomes retain the selected target and the observed lifecycle fact. + +When an ended session cannot acknowledge cleanup, the operation returns a failure with the failed commands and original typed producer reason. A coverage-stop failure also retains the observed window's end reason in its diagnostics; cleanup remains unconfirmed until the owning session can resolve it. Child frames, workers, service workers, WebAssembly, chronological execution order and framework-specific causal interpretation are outside this operation. Script ownership comes from frame/context metadata, never URL equality. Repeated URLs and synthetic names therefore retain separate identities. diff --git a/experiments/analysis-efficiency/README.md b/experiments/analysis-efficiency/README.md index 2f2f049544..154a09249f 100644 --- a/experiments/analysis-efficiency/README.md +++ b/experiments/analysis-efficiency/README.md @@ -1,5 +1,17 @@ # Analysis efficiency measurements +The [2026-10-21 coverage reconciliation](coverage-review-20261021.md) maps all +46 source directories to concrete workflow evidence, including original feature +PRs, historical no-benefit results and remaining verification boundaries. + +The [GGUF validation checkpoint](model-utf8-validation.md) removes decoding of +omitted strings while retaining full UTF-8 validation. It records real-model +phase gains, unchanged task token costs and the higher main-task memory peak. + +The [WAT framing checkpoint](wasm-string-framing.md) measures structural scanning +on real SQLite and VS Code components, with unchanged Evidence and token costs, +small whole-task timing gains and no memory improvement. + The [native inventory view checkpoint](native-inventory-views.md) compares complete delivery with summaries and selected rows on real `true` and jq inventories, including oversized-response recovery, portable old Evidence, diff --git a/experiments/analysis-efficiency/corpus.json b/experiments/analysis-efficiency/corpus.json index 03d4fa18c5..caba458c83 100644 --- a/experiments/analysis-efficiency/corpus.json +++ b/experiments/analysis-efficiency/corpus.json @@ -433,6 +433,38 @@ "members": 315, "expanded_bytes": 1301927 } + }, + { + "id": "alpine-ripgrep-rust", + "file": "ripgrep", + "bytes": 3416168, + "sha256": "78387bbe12b2110df8d2ce364390a00c190f4543a2f40b8279e49f6c5c85af9c", + "provenance": { + "project": "ripgrep", + "version": "14.1.1-r0", + "upstreamFile": "ripgrep-14.1.1-r0.apk", + "url": "https://dl-cdn.alpinelinux.org/alpine/v3.21/community/x86_64/ripgrep-14.1.1-r0.apk", + "transport_sha256": "213dd0b047163f6627cc2fdd6fbfe205009476eb4fc48c352a3f5b9ab647f6d4", + "transport_bytes": 1422830, + "aports_revision": "4e899d119e4c983a938436d6f397a509521225d1", + "derivation": "selected regular APK tar member usr/bin/rg; read-only static inspection; never executed" + } + }, + { + "id": "alpine-zellij-rust", + "file": "zellij", + "bytes": 17151864, + "sha256": "9a86e9ca47902a869fdf1886f4cf17714f4fa5655427128beadff498cfb392db", + "provenance": { + "project": "zellij", + "version": "0.40.0-r0", + "upstreamFile": "zellij-0.40.0-r0.apk", + "url": "https://dl-cdn.alpinelinux.org/alpine/v3.21/community/x86_64/zellij-0.40.0-r0.apk", + "transport_sha256": "575075945415b221857563e19c5f4c661af9f98bbba54f0994c7385c0e432432", + "transport_bytes": 6722129, + "aports_revision": "3be26a3a26e76e1bf3eff784bffe3956d29d77a0", + "derivation": "selected regular APK tar member usr/bin/zellij; read-only static inspection; never executed" + } } ] } diff --git a/experiments/analysis-efficiency/coverage-review-20261021.md b/experiments/analysis-efficiency/coverage-review-20261021.md new file mode 100644 index 0000000000..e3c6413004 --- /dev/null +++ b/experiments/analysis-efficiency/coverage-review-20261021.md @@ -0,0 +1,59 @@ +# Workflow coverage reconciliation, 2026-10-21 + +The checked snapshot is main `264862bf2`, with **46 first-level `src/` +directories and 212 production stdio MCP tools**. Directory counts and tool +counts are different inventories: one application workflow can cross several +directories, and one provider can expose many independently costly operations. +This is a source/history review, not a new runtime qualification of all tools. + +The search includes this experiment directory, owning contracts and source +history, original feature PRs and related open/closed issues. Missing experiment +files do not establish missing audits. In particular, the model and IL2CPP +feature PRs contain substantial real-task evidence absent from this directory. +Historical measurements retain their original revisions, platforms and limits. + +| Directories / workflow | Evidence already present | Remaining coverage boundary | +| ----------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `kuna`, `ghidra` | Earlier native entry work and [Ghidra projection](ghidra-evidence-projection.md) | Selected questions only; database reuse is a separate [research issue #3136](https://github.com/morluto/rea/issues/3136). | +| `ida` | [Windows native correctness and completed-task costs](ida-native-windows.md), including real `whoami.exe` | Warm selected-task gains do not establish cold analysis, all string formats, all operations or other IDA platforms. | +| `hopper`, `jeb` | [Hopper string records](hopper-string-records.md), [Linux runtime](hopper-linux-python-runtime.md), [real JEB CE](jeb-linux-native.md) | Preserve negative whole-task results; other editions, platforms and workflows are separate scopes. | +| `rizin`, `cutter` | Existing command/session adapters and lifecycle fixes | No comparable dedicated real completed-task performance record located in the searched records. Actual engine/GUI availability must be established before selecting a task. | +| `gdb`, `objdump` | [Real Bash command observations](command-observation-views.md) | Selected read-only disassembly is measured; other commands, large outputs and running-target observation are separate tasks. | +| `adb`, `frida`, `inspector` | [Frida native status/history experiment](frida-status-views.md) | Frida status does not qualify arbitrary scripts, ADB/device transfers, browser Inspector events or live target freshness. | +| `go`, `rust` | [Go build/dependency selection on real binaries](go-build-views.md); Rust cargo-auditable support [#2456](https://github.com/morluto/rea/pull/2456) | Go is already measured. No comparable dedicated Rust dependency-investigation cost record located in this review. | +| `javascript`, `browser`, `electron` | [Agent fact workflow](javascript-facts-agent-history.json), lifecycle experiments and [#4415](https://github.com/morluto/rea/issues/4415) | Real Joplin preparation remains exploratory. Individual JS features and browser/runtime workflows need their own evidence. | +| `android`, `apktool` | [Package summary](android-package-summary.md), [three-APK shared resource decoding](apktool-string-tables.md) | These do not qualify all JADX class/method/reference operations or physical-device workflows. #4777 is already fixed. | +| `dotnet`, `il2cpp` | [Managed summaries/pages](managed-artifact-summary.md), IL2CPP metadata [#3807](https://github.com/morluto/rea/pull/3807) and native links/seeds [#5327](https://github.com/morluto/rea/pull/5327) | IL2CPP already has real Windows/Android row comparisons, task costs and Ghidra import. Unsupported versions/generics/platforms remain separate correctness and coverage work. | +| `model` | GGUF/safetensors [#5603](https://github.com/morluto/rea/pull/5603): real models, independent readers and complete-task costs | Existing delivery is already audited. [#5774](https://github.com/morluto/rea/issues/5774) isolates omitted-string validation cost within that producer. | +| `native`, `artifacts` | [Native inventory](native-inventory-views.md), [entry views](native-entry-views.md); real NIB output reductions [#1460](https://github.com/morluto/rea/pull/1460), [#1477](https://github.com/morluto/rea/pull/1477) | Inventory does not qualify Apple UI/resource, signatures, dyld or runtime traces. NIB byte reductions alone do not establish token, time, CPU or memory savings for a completed question. | +| `firmware` | Region pilot plus real full OpenWrt extraction/publication [#4602](https://github.com/morluto/rea/pull/4602) | Extraction is already measured; noisy timings establish no general speedup. Other image layouts, extraction trees and publication filesystems remain unqualified. | +| `flutter`, `sqlite`, `wasm`, `evm`, `pcap`, `signals` | [Dart AOT](flutter-aot-summary.md), [SQLite controls](sqlite-existing-controls.md), [WAT framing](wasm-string-framing.md), [EVM](evm-interface-summary.md), [PCAP](pcap-small-captures.md), [signals](signal-selection-workflows.md) | Extend a specific question. Preserve cases where existing controls suffice and small tasks regress; these records do not qualify each format or tool exhaustively. | +| `application`, `domain`, `server`, `cli`, `contracts` | [Mixed retained investigations](retained-investigation-workflows.md), [immutable parent validation](immutable-parent-validation.md), [capture validation](retained-capture-validation.md) | Shared selected-query paths are already optimized. Reconstruction/environment composition and new fact families require their own end-to-end questions. | +| `browser` / recorded captures | [SAZ selection](saz-selection-workflows.md) and [real qlog producer/retained reads](retained-capture-validation.md) | [#3133](https://github.com/morluto/rea/issues/3133) still includes actual model/task questions; producer measurements do not close those requirements. | +| `filesystem`, `process`, `reference`, `providers`, `captures`, `windows`, `main`, `composition`, `config`, `evaluation` | Cross-workflow resource/cleanup checks, [process traces](process-trace-validation.md), [MCP profile](mcp-profile-identity.md) and Windows lifecycle checkpoints | Infrastructure coverage follows the exercised workflow. No blanket startup/resource claim; reference imports and `compare_source_to_bundle` need a dedicated completed source-tree question. | + +Every one of the 46 directories appears above; repeated `browser` highlights a +distinct workflow, not another directory. The table records located evidence, +not proof that every historical PR was inspected. Open issue state by itself +does not prove missing implementation or absent measurements. + +## Three bounded follow-up questions + +1. **Rust dependency identity:** inspect a public cargo-auditable binary, find + exact dependency/version edges and retain complete source Evidence. Compare + existing delivery first; independently decode its auditable section. Include + a small binary and account for all discovery, queries and serialization. +2. **Apple UI graph lookup:** on macOS, locate a specific control/outlet in a + real Transmission/XLD NIB graph using existing summary/document pages. + Compare equivalent complete questions and independently check archive + records. Historical byte reductions alone are not the completion criterion. +3. **Reference/source comparison:** import a pinned public source tree and + answer one source-to-bundle identity question through existing workflows. + Independently check file manifests, symlink/unknown semantics and selected + matches; measure repeated traversal and retained representation before + proposing an index or cache. + +Each lane stops with a measured benefit, an evidenced no-change decision, or an +explicit unavailable prerequisite. Rizin/Cutter and device/live workflows stay +visible as additional gaps; no new execution, injection or mutation authority is +implied by this review. No all-provider run or new API is justified by counts. diff --git a/experiments/analysis-efficiency/model-utf8-validation.json b/experiments/analysis-efficiency/model-utf8-validation.json new file mode 100644 index 0000000000..7e9fddbfeb --- /dev/null +++ b/experiments/analysis-efficiency/model-utf8-validation.json @@ -0,0 +1,435 @@ +{ + "baseline_source": "264862bf2e3612c9cb6fcfb861843cbba4e1893f", + "candidate_code": "c2550343173c5abd8f34e925eac1bcc12b7ee6a2", + "toolchain": { + "node": "24.18.0", + "npm": "11.16.0" + }, + "runtime_filetables_sha256": { + "baseline": "81c293ff8997017cfe9db2a65a8b70b188e3e6d20cef07902599c69ade5d9fac", + "candidate": "b4b654a7151f6affbb3ab8541bff74b643d9f78073d338b8feb2b095dd12b6b5" + }, + "lock_sha256": "699a14749e3825c62430faeb18224190fee1c9347760494e38e334e180bea6ec", + "gguf_source_sha256": { + "baseline": "9c387d5fba10db7830e74ec74a73f4b3d2e1347cca665e948b7891881b186007", + "candidate": "c43e5f8eebe04c0f82e0a970606f20ddf409c7a5fd0e298c88b09bcd0e7864a9" + }, + "driver_sha256": "9732f45b8b550eef47c1fc993b435129108934f3ec3501a34df1784e6ee37b44", + "resource_guard_sha256": { + "resource-run.py": "7a9642ab86ba34290564f636eb080cacf38b4185a254028680b238bff172993b", + "resource_payload.py": "6f089e645459eba80c5d73e39c21606c78173a44daf49fabc3b88c4631ce51c6" + }, + "oracle": { + "gguf": "0.19.0", + "safetensors": "0.8.0", + "receipts": [ + { + "file": "stories15M-q4_0.gguf", + "sha256": "66967fbece6dbe97886593fdbb73589584927e29119ec31f08090732d1861739", + "metadata": 20, + "tensors": 57, + "status": "matched" + }, + { + "file": "stories260K.gguf", + "sha256": "270cba1bd5109f42d03350f60406024560464db173c0e387d91f0426d3bd256d", + "metadata": 19, + "tensors": 48, + "status": "matched" + }, + { + "file": "tiny-bert.safetensors", + "sha256": "965f02b6a7e5520fc12f710e4e3b6132f697f1c8f648819553c5ade86752d2de", + "metadata": 2, + "tensors": 100, + "status": "matched" + } + ] + }, + "tokenizer": "tiktoken 0.12.0 cl100k_base; independent compact JSON chunks, catalog once + each named request/reply; not actual model usage or initialization/framing", + "cases": [ + { + "artifact": { + "repo": "ggml-org/models", + "revision": "499bc8821c6b12b4e53c5bffcb21ec206f212d81", + "member": "tinyllamas/stories15M-q4_0.gguf", + "file": "stories15M-q4_0.gguf", + "url": "https://huggingface.co/ggml-org/models/resolve/499bc8821c6b12b4e53c5bffcb21ec206f212d81/tinyllamas/stories15M-q4_0.gguf", + "bytes": 19077344 + }, + "normalized_sha256": "a894614da7ce9486b07eb2ede05ce11307d4726089722278bc4c95043feb6b66", + "answers_sha256": "64277f01f43440bc99a94ab0787a4ddde22866ed20140d87d3942b1723cbb067", + "catalog_sha256": "859b1433ef7c45ea23ba4073715d25ed7a76fd9df6d74285d20a5ea39e8e2898", + "mcp": [ + { + "pair": 1, + "side": "baseline", + "wall_seconds": 6.667, + "cpu_seconds": 6.480601, + "peak_family_mib": 769.34, + "catalog_plus_calls_tokens": 895583, + "calls_ms": [367.272, 1936.653, 162.868, 12.072, 19.236] + }, + { + "pair": 1, + "side": "candidate", + "wall_seconds": 6.867, + "cpu_seconds": 6.690645, + "peak_family_mib": 816.34, + "catalog_plus_calls_tokens": 895583, + "calls_ms": [324.806, 1822.885, 153.167, 11.988, 18.093] + }, + { + "pair": 2, + "side": "baseline", + "wall_seconds": 6.77, + "cpu_seconds": 6.555891, + "peak_family_mib": 798.98, + "catalog_plus_calls_tokens": 895583, + "calls_ms": [456.467, 1823.786, 160.038, 11.296, 18.607] + }, + { + "pair": 2, + "side": "candidate", + "wall_seconds": 6.565, + "cpu_seconds": 6.3467, + "peak_family_mib": 813.566, + "catalog_plus_calls_tokens": 895583, + "calls_ms": [329.862, 1767.086, 152.299, 10.948, 17.523] + }, + { + "pair": 3, + "side": "baseline", + "wall_seconds": 6.666, + "cpu_seconds": 6.49779, + "peak_family_mib": 771.02, + "catalog_plus_calls_tokens": 895583, + "calls_ms": [345.895, 1854.963, 156.895, 11.476, 15.852] + }, + { + "pair": 3, + "side": "candidate", + "wall_seconds": 6.567, + "cpu_seconds": 6.386103, + "peak_family_mib": 818.891, + "catalog_plus_calls_tokens": 895583, + "calls_ms": [342.734, 1827.835, 157.363, 10.122, 17.6] + } + ], + "phase": [ + { + "pair": 1, + "side": "baseline", + "first_ms": 154.027, + "repeat_median_ms": 56.14, + "wall_seconds": 1.028, + "cpu_seconds": 0.87953, + "peak_family_mib": 75.867 + }, + { + "pair": 1, + "side": "candidate", + "first_ms": 121.41, + "repeat_median_ms": 35.429, + "wall_seconds": 0.925, + "cpu_seconds": 0.717987, + "peak_family_mib": 77.945 + }, + { + "pair": 2, + "side": "baseline", + "first_ms": 159.766, + "repeat_median_ms": 53.575, + "wall_seconds": 1.027, + "cpu_seconds": 0.854739, + "peak_family_mib": 75.586 + }, + { + "pair": 2, + "side": "candidate", + "first_ms": 121.383, + "repeat_median_ms": 38.773, + "wall_seconds": 0.925, + "cpu_seconds": 0.733397, + "peak_family_mib": 75.125 + }, + { + "pair": 3, + "side": "baseline", + "first_ms": 148.447, + "repeat_median_ms": 53.58, + "wall_seconds": 1.026, + "cpu_seconds": 0.848173, + "peak_family_mib": 75.363 + }, + { + "pair": 3, + "side": "candidate", + "first_ms": 121.712, + "repeat_median_ms": 37.191, + "wall_seconds": 0.926, + "cpu_seconds": 0.728712, + "peak_family_mib": 77.887 + } + ] + }, + { + "artifact": { + "repo": "ggml-org/models", + "revision": "499bc8821c6b12b4e53c5bffcb21ec206f212d81", + "member": "tinyllamas/stories260K.gguf", + "file": "stories260K.gguf", + "url": "https://huggingface.co/ggml-org/models/resolve/499bc8821c6b12b4e53c5bffcb21ec206f212d81/tinyllamas/stories260K.gguf", + "bytes": 1185376 + }, + "normalized_sha256": "15f4dbdc30dd9bd936747057031c6391c38b4c1717d56ae33ed2f0cba4184405", + "answers_sha256": "99257428a01aa513b435c95ca7217c69f4641d9021372a6333e7cb9e2cb35d63", + "catalog_sha256": "859b1433ef7c45ea23ba4073715d25ed7a76fd9df6d74285d20a5ea39e8e2898", + "mcp": [ + { + "pair": 1, + "side": "baseline", + "wall_seconds": 6.568, + "cpu_seconds": 6.375324, + "peak_family_mib": 743.887, + "catalog_plus_calls_tokens": 895421, + "calls_ms": [223.11, 1938.73, 102.018, 10.759, 16.11] + }, + { + "pair": 1, + "side": "candidate", + "wall_seconds": 6.566, + "cpu_seconds": 6.338327, + "peak_family_mib": 744.852, + "catalog_plus_calls_tokens": 895421, + "calls_ms": [248.979, 1886.809, 105.801, 10.738, 15.653] + }, + { + "pair": 2, + "side": "baseline", + "wall_seconds": 6.365, + "cpu_seconds": 6.206216, + "peak_family_mib": 783.84, + "catalog_plus_calls_tokens": 895421, + "calls_ms": [248.196, 1845.737, 112.874, 10.724, 17.594] + }, + { + "pair": 2, + "side": "candidate", + "wall_seconds": 7.383, + "cpu_seconds": 7.211823, + "peak_family_mib": 742.281, + "catalog_plus_calls_tokens": 895421, + "calls_ms": [233.155, 1909.721, 104.803, 11.603, 19.357] + }, + { + "pair": 3, + "side": "baseline", + "wall_seconds": 6.565, + "cpu_seconds": 6.323702, + "peak_family_mib": 740.84, + "catalog_plus_calls_tokens": 895421, + "calls_ms": [233.664, 1918.301, 104.389, 14.561, 15.227] + }, + { + "pair": 3, + "side": "candidate", + "wall_seconds": 6.566, + "cpu_seconds": 6.315785, + "peak_family_mib": 778.133, + "catalog_plus_calls_tokens": 895421, + "calls_ms": [232.37, 1851.929, 78.318, 21.857, 16.438] + } + ], + "phase": [ + { + "pair": 1, + "side": "baseline", + "first_ms": 35.366, + "repeat_median_ms": 7.784, + "wall_seconds": 0.625, + "cpu_seconds": 0.429065, + "peak_family_mib": 62.922 + }, + { + "pair": 1, + "side": "candidate", + "first_ms": 30.009, + "repeat_median_ms": 7.934, + "wall_seconds": 0.624, + "cpu_seconds": 0.392848, + "peak_family_mib": 59.664 + }, + { + "pair": 2, + "side": "baseline", + "first_ms": 32.0, + "repeat_median_ms": 8.772, + "wall_seconds": 0.624, + "cpu_seconds": 0.412059, + "peak_family_mib": 60.422 + }, + { + "pair": 2, + "side": "candidate", + "first_ms": 32.032, + "repeat_median_ms": 9.36, + "wall_seconds": 0.624, + "cpu_seconds": 0.412412, + "peak_family_mib": 59.441 + }, + { + "pair": 3, + "side": "baseline", + "first_ms": 33.628, + "repeat_median_ms": 9.496, + "wall_seconds": 0.624, + "cpu_seconds": 0.396232, + "peak_family_mib": 60.41 + }, + { + "pair": 3, + "side": "candidate", + "first_ms": 30.506, + "repeat_median_ms": 8.172, + "wall_seconds": 0.622, + "cpu_seconds": 0.404398, + "peak_family_mib": 61.672 + } + ] + }, + { + "artifact": { + "repo": "hf-internal-testing/tiny-random-bert", + "revision": "f171d7baecaf37b5da5a3616d8833b9969753535", + "member": "model.safetensors", + "file": "tiny-bert.safetensors", + "url": "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/f171d7baecaf37b5da5a3616d8833b9969753535/model.safetensors", + "bytes": 520212 + }, + "normalized_sha256": "8658d90261cdbaaf1f5f8d2e9408ad301b40ad3e6b7aab7ac9c5d4be71ca36d5", + "answers_sha256": "82072404a0aa24e3b42979397421f455da2589c2ea649f28ba7279d43e6c3bc8", + "catalog_sha256": "859b1433ef7c45ea23ba4073715d25ed7a76fd9df6d74285d20a5ea39e8e2898", + "mcp": [ + { + "pair": 1, + "side": "baseline", + "wall_seconds": 6.667, + "cpu_seconds": 6.46146, + "peak_family_mib": 741.602, + "catalog_plus_calls_tokens": 893755, + "calls_ms": [251.201, 1965.785, 114.055, 10.837, 28.862] + }, + { + "pair": 1, + "side": "candidate", + "wall_seconds": 6.767, + "cpu_seconds": 6.519606, + "peak_family_mib": 736.367, + "catalog_plus_calls_tokens": 893755, + "calls_ms": [255.557, 1979.227, 86.726, 10.093, 24.531] + }, + { + "pair": 2, + "side": "baseline", + "wall_seconds": 6.565, + "cpu_seconds": 6.379249, + "peak_family_mib": 730.273, + "catalog_plus_calls_tokens": 893755, + "calls_ms": [245.517, 1900.802, 94.351, 10.0, 25.635] + }, + { + "pair": 2, + "side": "candidate", + "wall_seconds": 6.569, + "cpu_seconds": 6.388131, + "peak_family_mib": 785.746, + "catalog_plus_calls_tokens": 893755, + "calls_ms": [250.715, 1846.568, 103.314, 21.618, 27.261] + }, + { + "pair": 3, + "side": "baseline", + "wall_seconds": 6.771, + "cpu_seconds": 6.530512, + "peak_family_mib": 745.098, + "catalog_plus_calls_tokens": 893755, + "calls_ms": [255.441, 1949.189, 110.897, 10.679, 24.345] + }, + { + "pair": 3, + "side": "candidate", + "wall_seconds": 6.466, + "cpu_seconds": 6.296937, + "peak_family_mib": 741.348, + "catalog_plus_calls_tokens": 893755, + "calls_ms": [243.919, 1895.978, 113.995, 10.66, 27.269] + } + ], + "phase": [ + { + "pair": 1, + "side": "baseline", + "first_ms": 27.648, + "repeat_median_ms": 8.11, + "wall_seconds": 0.623, + "cpu_seconds": 0.394405, + "peak_family_mib": 57.281 + }, + { + "pair": 1, + "side": "candidate", + "first_ms": 27.003, + "repeat_median_ms": 8.082, + "wall_seconds": 0.523, + "cpu_seconds": 0.383678, + "peak_family_mib": 57.266 + }, + { + "pair": 2, + "side": "baseline", + "first_ms": 26.941, + "repeat_median_ms": 8.04, + "wall_seconds": 0.626, + "cpu_seconds": 0.401321, + "peak_family_mib": 59.504 + }, + { + "pair": 2, + "side": "candidate", + "first_ms": 28.012, + "repeat_median_ms": 8.006, + "wall_seconds": 0.625, + "cpu_seconds": 0.392541, + "peak_family_mib": 59.012 + }, + { + "pair": 3, + "side": "baseline", + "first_ms": 28.734, + "repeat_median_ms": 8.499, + "wall_seconds": 0.624, + "cpu_seconds": 0.411118, + "peak_family_mib": 59.23 + }, + { + "pair": 3, + "side": "candidate", + "first_ms": 29.961, + "repeat_median_ms": 8.632, + "wall_seconds": 0.625, + "cpu_seconds": 0.41875, + "peak_family_mib": 57.5 + } + ] + } + ], + "published_equivalent_source": "2f091ccc734d7bd945bd61432d41cf5ba68be12b", + "candidate_reproduction": "Use baseline_source plus the byte-identical src/model/GgufHeader.ts from published_equivalent_source; driver SHA-256 is pinned separately. Measured local candidate predates the integration rebase.", + "post_review_receipt_fix": { + "measured_driver_version": "2f091ccc734d7bd945bd61432d41cf5ba68be12b", + "driver_sha256": "449860fb973418f2e3e51183a81be1e3111ffa6453de8208a4b175d3d410322f", + "publication_owner_sha256": "901092ae6dc7fda1a8869d62362a0a9f560e03d5ef5667a5c6ed660db000b9a2", + "validation": "3 new fault-injection regressions reject original driver and pass current; 11 shared receipt regressions pass. Real stories15M MCP and seven-inspection phase reruns qualify; original paired samples retain their measured driver hash." + } +} diff --git a/experiments/analysis-efficiency/model-utf8-validation.md b/experiments/analysis-efficiency/model-utf8-validation.md new file mode 100644 index 0000000000..d0cf6594d7 --- /dev/null +++ b/experiments/analysis-efficiency/model-utf8-validation.md @@ -0,0 +1,147 @@ +# Omitted GGUF string validation + +[Issue #5774](https://github.com/morluto/rea/issues/5774) follows the existing +real-model and completed-task audit in [#5603](https://github.com/morluto/rea/pull/5603). +The [coverage reconciliation](coverage-review-20261021.md) explains why neither +model inspection nor IL2CPP, firmware extraction or IDA should be described as +wholly unaudited. This experiment changes only one model producer hot path. + +## Owner and preserved semantics + +GGUF arrays retain eight preview values. Previously, each omitted string still +constructed a fatal UTF-8 decoder and materialized discarded text. `GgufHeader` +now validates an omitted value of at most one existing 65,536-byte chunk with +Node's `isUtf8`. Longer values retain the streaming fatal decoder, including +multi-byte characters split across chunks. Retained values use their original +decoder, preserving BOMs and exact text. + +Every admitted byte is still read, hashed and validated. Header bounds, +representation admission, cancellation checks, array-loop fairness, source +offsets, partial-progress semantics, preview omission and complete retained +Evidence are unchanged. There is no cache, new contract, selector or default. +CLI and MCP share this producer through the existing application workflow. + +## Corpus and correctness + +The [curated receipts](model-utf8-validation.json) pin source commits, runtime +file tables, driver/resource sources, immutable public URLs, artifact digests, +complete report/answer fingerprints and all final samples. Baseline is main +`264862bf2e3612c9cb6fcfb861843cbba4e1893f`; measured candidate code is +`c2550343173c5abd8f34e925eac1bcc12b7ee6a2`. Later documentation/integration +commits do not relabel that cohort. That measured local commit predates the +rebase: its production file is byte-identical to published +`2f091ccc734d7bd945bd61432d41cf5ba68be12b`, with both source hashes recorded. + +| Artifact | Bytes | Metadata / tensors | Independent reader | +| ------------------------------------------------ | ---------: | -----------------: | ------------------ | +| ggml-org stories15M Q4_0 GGUF | 19,077,344 | 20 / 57 | gguf 0.19.0 | +| ggml-org stories260K GGUF | 1,185,376 | 19 / 48 | gguf 0.19.0 | +| hf-internal-testing tiny-random-bert safetensors | 520,212 | 2 / 100 | safetensors 0.8.0 | + +Official independent readers match all exposed metadata values/array previews +and counts, tensor names, types, shapes and parameter counts. GGUF reader +offsets/lengths match every tensor; safetensors reader geometry and independent +Python JSON offsets match its file ranges. Python byte hashing checks every +header digest. No weights are executed or loaded into a model framework. +The main GGUF has 32,000 vocabulary entries; the smaller GGUF and safetensors +are controls, not evidence for larger vocabularies or other GGUF versions. + +## Completed public tasks + +Linux x64, Node 24.18.0/npm 11.16.0. Three alternating baseline/candidate, +candidate/baseline, baseline/candidate pairs per artifact run serially. Each +fresh production stdio MCP task discovers all 212 tools, directly selects two +metadata records, retrieves two exact tensors from retained Evidence, repeats +both calls, exports complete Evidence, validates answers and closes the +transport. OS caches are unflushed; the controller reads inputs for hashing. +Fresh process does not mean cold disk or cold model-engine analysis. + +Medians of three whole tasks per side, including SDK/catalog/schema validation, +all five calls, source checks, export and teardown: + +| Artifact | Wall seconds, before → after | Family CPU seconds, before → after | Peak family MiB, before → after | +| ----------- | ---------------------------: | ---------------------------------: | ------------------------------: | +| stories15M | 6.667 → 6.567 | 6.498 → 6.386 | 771.0 → 816.3 | +| stories260K | 6.565 → 6.566 | 6.324 → 6.338 | 743.9 → 744.9 | +| tiny BERT | 6.667 → 6.569 | 6.461 → 6.388 | 741.6 → 741.3 | + +The primary task's observed wall reduction is about 1.5%, but the unchanged +safetensors control has similar variation. No stable whole-task speedup is +established. Main-model peak memory rises: ranges 769.3–799.0 → 813.6–818.9 +MiB; its cause is unmeasured. No peak-memory improvement is claimed. + +Separate fresh-process provider cohorts perform one first inspection and six +repeated inspections, comparing the complete report after every operation: + +| Artifact | First / repeat median ms, before | First / repeat median ms, after | +| ----------- | -------------------------------: | ------------------------------: | +| stories15M | 154.03 / 53.58 | 121.41 / 37.19 | +| stories260K | 33.63 / 8.77 | 30.51 / 8.17 | +| tiny BERT | 27.65 / 8.11 | 28.01 / 8.08 | + +For stories15M this is about 21% faster first inspection and 31% faster repeated +inspection. Median CPU for the seven-inspection process is 0.855 → 0.729 s; +peak family memory is 75.6 → 77.9 MiB. Phase gains do not include discovery +or substitute for the complete-task results above. + +One catalog plus all named request/reply JSON chunks estimates median +895,583 / 895,421 / 893,755 `cl100k_base` tokens for the three artifacts in +both implementations, using tiktoken 0.12.0. This includes serialized output +schemas, not merely named input schemas. It is an SDK serialization proxy; +actual model usage, repeated host injection, negotiation and framing are +unmeasured. No token saving is claimed. + +All 18 public tasks and 18 provider processes completed under one CPU, 4 GiB +hard / 3 GiB early-stop family memory, no swap and 192 tasks. Payload exit, +counters and owned cleanup are verified; no limit/OOM events occurred. Kernel +cgroup peak includes file cache. Terminal whole-scope counters after scope +removal remain unknown. Raw inputs, exports, profiles and transcripts stay +outside Git. + +## Reproduction and validation boundaries + +Build the baseline and the same baseline with the published GGUF source file +using the pinned toolchain; this reconstructs the measured local candidate. +Capture a complete +baseline normalized report from existing CLI/MCP Evidence using the same +absolute input path. Run `model-utf8-validation.mjs` with absolute `--runtime`, +`--artifact`, `--expected` and a new `--output` directory under `resource-run.py +--observe-payload-exit`. Add `--phase` for the seven-inspection provider probe. +The driver validates advertised named schemas through the actual SDK and +compares full exported parents, preview facts and tensor selections. A failing +receipt or cleanup invalidates a task; it is not silently retried as success. + +Five new producer regressions cover empty omitted strings, exact-chunk strings, +split multi-byte characters, both byte orders, retained BOMs and rejection of +short/long malformed omitted values. Existing malformed-header, admission, +cancellation, partial-progress and public CLI/MCP tests remain required. +An initial new-test assertion incorrectly hashed alignment padding; the +corrected oracle uses the actual declared header, matching existing semantics. +The first test launcher refused built projects; the exact built-project gate +was then used. Neither attempt is a passing check. The initial resource-wrapper +copy failed source-hash lookup before launching a build; corrected guarded +builds qualified before measurement. + +Integrated onto main `71b696582`: all 68 tests pass again, and three real public +CLI complete reports plus three production SDK tasks match the original +normalized reports. Typecheck, lint/module boundaries, formatting, metadata, +runtime build, docs checks (five tests) and separate architecture guards pass. +`npm run check` does **not** pass: Knip reports an unlisted `g++` invocation in +the Windows test added by main's #5772. An independent main source checkout, +with matching compiled worker artifacts and an uncached Knip run, reproduces +only that finding. It is outside this change and is left unchanged. Initial +control attempts lacked resolvable worker artifacts and are not clean controls. +Development builds/checks used separate 6/8 GiB hard guards; runtime experiments +retain their 4 GiB limit. No guard stop or OOM occurred. + +Greptile identified a receipt defect in the phase driver, not the model parser: +failed inspections/assertions were not recorded before cleanup, and a close +failure could suppress publication. Each attempted phase now records its cost +and primary failure before cleanup. Phase and MCP paths share one atomic receipt +publisher; cleanup failures are independent facts and cannot replace the primary +failure. Three new fault-injection tests fail on the original driver and pass +after the correction; all 11 existing shared receipt tests pass. These stand-ins +qualify failure accounting only. Real stories15M MCP and seven-inspection phase +reruns also pass with verified counters/cleanup. The original 36 paired samples +keep their measured driver hash; the receipt records the post-review driver and +publication-owner hashes separately. No timing cohort is relabeled. diff --git a/experiments/analysis-efficiency/model-utf8-validation.mjs b/experiments/analysis-efficiency/model-utf8-validation.mjs new file mode 100644 index 0000000000..fbaa134535 --- /dev/null +++ b/experiments/analysis-efficiency/model-utf8-validation.mjs @@ -0,0 +1,243 @@ +/** Equivalent model tasks; inputs, expected reports and raw receipts stay outside Git. */ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { createWriteStream } from "node:fs"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { finished } from "node:stream/promises"; +import { pathToFileURL } from "node:url"; +import { parseArgs } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { createRetainedQueryCaller } from "./retained-query-calls.mjs"; +import { + finalizeRetainedQueryReceipt, + publishRetainedQueryReceipt, +} from "./retained-query-finalization.mjs"; + +const { values } = parseArgs({ + options: { + ...Object.fromEntries( + ["runtime", "artifact", "expected", "output"].map((name) => [ + name, + { type: "string" }, + ]), + ), + phase: { type: "boolean", default: false }, + }, +}); +for (const name of ["runtime", "artifact", "expected", "output"]) + assert.ok(values[name] && values[name] === resolve(values[name]), name); +const hash = (value) => createHash("sha256").update(value).digest("hex"); +const expected = JSON.parse(await readFile(values.expected, "utf8")); +assert.equal(expected.artifact.path, values.artifact); +await mkdir(values.output, { mode: 0o700 }); +const report = { + artifact_sha256: hash(await readFile(values.artifact)), + normalized_sha256: hash(JSON.stringify(expected)), + calls: [], + answers: [], +}; + +if (values.phase) { + const { ModelArtifactProvider } = await import( + pathToFileURL(join(values.runtime, "dist/model/ModelArtifactProvider.js")) + ); + const provider = new ModelArtifactProvider(); + try { + for (let index = 0; index < 7; index++) { + const sample = { + phase: index === 0 ? "fresh" : "repeat", + is_error: true, + }; + const start = performance.now(); + try { + const result = await provider.inspect({ + path: values.artifact, + hash: "header", + }); + sample.elapsed_ms = performance.now() - start; + assert.ok(result.ok, JSON.stringify(result)); + assert.deepEqual(result.value, expected); + sample.is_error = false; + } catch (cause) { + sample.elapsed_ms ??= performance.now() - start; + sample.failure = String(cause?.stack ?? cause); + throw cause; + } finally { + report.calls.push(sample); + } + } + report.status = "passed"; + } catch (cause) { + report.status = "failed"; + report.failure = String(cause?.stack ?? cause); + process.exitCode = 1; + } finally { + report.cleanup_failures = []; + try { + await provider.close(); + } catch (cause) { + report.cleanup_failures.push(`provider: ${String(cause)}`); + report.status = "failed"; + } + if (!(await publishRetainedQueryReceipt({ report, output: values.output }))) + process.exitCode = 1; + } +} else { + await runMcpTask(); +} + +async function runMcpTask() { + const account = await mkdtemp(join(tmpdir(), "rea-model-utf8-")); + const client = new Client({ name: "model-utf8-validation", version: "1" }); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [join(values.runtime, "scripts/rea.mjs"), "mcp"], + env: { + HOME: account, + PATH: process.env.PATH, + NODE_OPTIONS: "--max-old-space-size=2048", + UV_THREADPOOL_SIZE: "1", + REA_LOG_LEVEL: "silent", + TMPDIR: account, + }, + stderr: "pipe", + }); + const serverLog = createWriteStream(join(values.output, "server.log")); + const serverLogFinished = finished(serverLog); + serverLogFinished.catch(() => {}); + transport.stderr?.pipe(serverLog); + const transcript = []; + const call = createRetainedQueryCaller(client, (receipt, event) => { + report.calls.push(receipt); + transcript.push(event); + }); + const observe = async (name, input, phase) => { + const reply = await call(name, input, phase); + assert.notEqual(reply.isError, true, JSON.stringify(reply)); + assert.ok(reply.structuredContent); + return reply.structuredContent; + }; + try { + await client.connect(transport); + const catalog = await client.listTools(); + assert.equal(catalog.nextCursor, undefined); + await writeFile( + join(values.output, "catalog.json"), + JSON.stringify(catalog), + ); + report.catalog_sha256 = hash(JSON.stringify(catalog)); + report.catalog_tools = catalog.tools.length; + const definitions = new Map(catalog.tools.map((tool) => [tool.name, tool])); + const originalCall = client.callTool.bind(client); + client.callTool = (input, options) => + originalCall(input, { + ...options, + toolDefinition: definitions.get(input.name), + }); + const keys = [ + ...new Set(expected.metadata.slice(0, 2).map((row) => row.key)), + ]; + const names = expected.tensors.slice(0, 2).map((row) => row.name); + const fields = [ + "key", + "type", + "value", + "element_count", + "preview_complete", + ]; + const tensorFields = [ + "name", + "type", + "shape", + "file_offset", + "byte_length", + ]; + const select = (rows, selectedFields) => + rows.map((row) => + Object.fromEntries( + ["index", ...selectedFields].map((key) => [key, row[key]]), + ), + ); + const observations = []; + for (const phase of ["fresh", "repeat"]) { + const metadata = await observe( + "inspect_model_artifact", + { + path: values.artifact, + view: { collection: "metadata", keys, fields }, + }, + phase, + ); + const parentId = metadata.normalized_result.parent_evidence_id; + assert.equal(typeof parentId, "string"); + assert.deepEqual( + metadata.normalized_result.items, + select( + expected.metadata.filter((row) => keys.includes(row.key)), + fields, + ), + ); + const tensors = await observe( + "inspect_analysis_view", + { + source: { kind: "retained-evidence", evidence_id: parentId }, + view: { + kind: "model-artifact", + collection: "tensors", + names, + fields: tensorFields, + }, + }, + `${phase}:tensors`, + ); + assert.equal(tensors.normalized_result.parent_evidence_id, parentId); + assert.deepEqual( + tensors.normalized_result.items, + select(expected.tensors.slice(0, 2), tensorFields), + ); + observations.push(parentId); + report.answers.push({ + parent_evidence_id: parentId, + metadata: metadata.normalized_result.items, + tensors: tensors.normalized_result.items, + }); + } + await observe( + "export_evidence_bundle", + { path: join(values.output, "evidence.json") }, + "export", + ); + const bundle = JSON.parse( + await readFile(join(values.output, "evidence.json"), "utf8"), + ); + for (const id of observations) { + const parent = bundle.records.find((record) => record.evidence_id === id); + assert.ok(parent); + assert.deepEqual(parent.normalized_result, expected); + } + assert.deepEqual(report.answers[0], report.answers[1]); + assert.equal(hash(await readFile(values.artifact)), report.artifact_sha256); + report.status = "passed"; + } catch (cause) { + report.status = "failed"; + report.failure = String(cause?.stack ?? cause); + process.exitCode = 1; + } finally { + if ( + !(await finalizeRetainedQueryReceipt({ + report, + client, + transport, + account, + serverLog, + serverLogFinished, + output: values.output, + transcript, + })) + ) + process.exitCode = 1; + } +} diff --git a/experiments/analysis-efficiency/retained-capture-validation.md b/experiments/analysis-efficiency/retained-capture-validation.md index 2323554d17..17631322e1 100644 --- a/experiments/analysis-efficiency/retained-capture-validation.md +++ b/experiments/analysis-efficiency/retained-capture-validation.md @@ -18,7 +18,7 @@ Current history and the existing evidence support these narrower conclusions: | Apktool | [#4777](https://github.com/morluto/rea/issues/4777) is closed. Shared decoding and optional primary-string omission have [three-APK evidence](apktool-string-tables.md). Do not implement another decode cache from the old issue list. | | Hopper | [#5017](https://github.com/morluto/rea/pull/5017) already reuses decoded string records. Its [checkpoint](hopper-string-records.md) preserves faster warm queries and the negative whole-task result. | | JEB | The [Linux CE audit](jeb-linux-native.md) verifies selected real CLI/MCP reads. It does not justify another wrapper cache; discovery and other platforms remain separate coverage gaps. | -| IDA | Existing [native Windows verification](ida-native-windows.md) qualifies selected correctness/authority paths. It does not establish a comparable full performance audit. | +| IDA | Existing [native Windows verification](ida-native-windows.md) includes selected real completed-task performance and correctness/authority checks. It does not qualify every operation or platform. | | JavaScript/browser/Electron | Prior experiments and retained pages already exist. [#4415](https://github.com/morluto/rea/issues/4415) remains exploratory; no safe general Joplin preparation speedup is established. | | Other static formats | Managed, Flutter, SQLite, WASM, EVM, PCAP, signals and SAZ already have checkpoints in this directory. Extend a specific measured question rather than create duplicate selectors or caches. | | Other providers, languages and infrastructure | Rizin/Cutter/GDB/objdump, Go/Rust, ADB/Frida/inspector and startup/resource owners still need bounded real-workflow questions. Historical local improvements do not prove complete coverage or establish another bottleneck. | diff --git a/experiments/analysis-efficiency/retained-query-finalization.mjs b/experiments/analysis-efficiency/retained-query-finalization.mjs index 2dd193e94d..205ef30223 100644 --- a/experiments/analysis-efficiency/retained-query-finalization.mjs +++ b/experiments/analysis-efficiency/retained-query-finalization.mjs @@ -58,6 +58,16 @@ export const finalizeRetainedQueryReceipt = async ({ ); if (report.cleanup_failures.length > 0) report.retained_workspace = account; + return publishRetainedQueryReceipt({ report, output, transcript, stdout }); +}; + +/** Share atomic publication and failure accounting with non-MCP phase probes. */ +export const publishRetainedQueryReceipt = async ({ + report, + output, + transcript, + stdout = process.stdout, +}) => { report.receipt_failures = []; if (transcript !== undefined) await attempt(report.receipt_failures, "transcript-file", () => diff --git a/experiments/analysis-efficiency/rust-dependency-costs.py b/experiments/analysis-efficiency/rust-dependency-costs.py new file mode 100644 index 0000000000..f8094a5ad8 --- /dev/null +++ b/experiments/analysis-efficiency/rust-dependency-costs.py @@ -0,0 +1,79 @@ +"""Count completed Rust questions; tokenizer estimates are not model usage.""" +import argparse +import hashlib +import json +from pathlib import Path + +import tiktoken + + +def compact(value): + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + + +def inspect(path, encoding): + report = json.loads((path / "report.json").read_text()) + catalog = json.loads((path / "catalog.json").read_text()) + transcript = json.loads((path / "transcript.json").read_text()) + log = path.with_suffix(".log") + scope = next(json.loads(line) for line in reversed(log.read_text().splitlines()) + if line.startswith('{"resource_scope":')) + observed = scope.get("payload_resource_observation") or {} + events = observed.get("resource_events", {}) + memory = events.get("memory.events", {}) + assert (report["status"] == "passed" and not report["cleanup_failures"] + and not report["receipt_failures"] and scope["abort"] is None + and scope.get("payload_resource_observation_verified") is True + and observed.get("payload_cleanup_verified") is True + and observed.get("payload_exit_code") == 0 + and all(memory.get(key) == 0 for key in ["high", "max", "oom", "oom_kill"]) + and events.get("pids.events", {}).get("max") == 0), path + assert [event["name"] for event in transcript] == [ + "inspect_rust_binary", "export_evidence_bundle"] + count = lambda value: len(encoding.encode(value, disallowed_special=())) + schema_fields = ["name", "description", "inputSchema"] + tool_tokens = lambda tools: count(compact([ + {key: tool[key] for key in schema_fields} for tool in tools])) + inputs = tool_tokens(catalog["tools"]) + selected = tool_tokens([tool for tool in catalog["tools"] if tool["name"] in [ + "inspect_rust_binary", "export_evidence_bundle"]]) + arguments = sum(count(compact({"name": event["name"], "arguments": event["input"]})) + for event in transcript) + replies = sum(count(block["text"]) for event in transcript + for block in event["reply"]["content"] if block["type"] == "text") + return {"case": path.name, "lane": report["lane"], + "answer": report["answer"], "parent_evidence_id": report["parent_evidence_id"], + "artifact_sha256": report["artifact_sha256"], + "complete_result_sha256": report["normalized_sha256"], + "catalog_sha256": report["catalog_sha256"], + "catalog_tools": report["catalog_tools"], "calls": report["calls"], + "wall_seconds": scope["elapsed_seconds"], + "cpu_seconds": observed["cpu_stat"]["usage_usec"] / 1e6, + "kernel_family_peak_bytes": observed["memory_peak_bytes"], + "events_at_payload_exit": events, + "final_whole_scope_events": scope.get("resource_events_final"), + "completed_with_verified_cleanup": True, + "token_proxy": {"input_catalog_once": inputs, "arguments": arguments, + "text_replies": replies, "combined": inputs + arguments + replies, + "only_two_selected_tools": selected + arguments + replies, + "complete_transport_catalog": count(compact(catalog))}, + "sha256": {file.name: hashlib.sha256(file.read_bytes()).hexdigest() + for file in [path / "report.json", path / "catalog.json", + path / "transcript.json", log]}} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("cases", type=Path, nargs="+") + args = parser.parse_args() + encoding = tiktoken.get_encoding("cl100k_base") + receipt = {"tiktoken": tiktoken.__version__, "encoding": encoding.name, + "actual_model_usage": None, + "projection": "All discovered tool descriptions/input schemas once, plus every call argument and text reply including export. Selected-two-tool and complete transport catalog costs are separate. No model, host injection, history replay or retry observed.", + "recordings": [inspect(path, encoding) for path in args.cases]} + args.output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/experiments/analysis-efficiency/rust-dependency-views.json b/experiments/analysis-efficiency/rust-dependency-views.json new file mode 100644 index 0000000000..ed2d0407c8 --- /dev/null +++ b/experiments/analysis-efficiency/rust-dependency-views.json @@ -0,0 +1,578 @@ +{ + "baseline_source": "d672cc23998b6cbb21c485f118595ebfb82fed33", + "candidate_code": "cae2571e7e64a7364b94bb92c2630cba80ec4478", + "runtime_filetables": { + "baseline": { + "sha256": "6497f70d4864489e948b6a6f9611cf1bd9e6d7f60722fcdd41c0984ab7c48d53", + "files": 1917 + }, + "candidate": { + "sha256": "078e6fd0f837ca252265d12d3a95e346f836762d2474117c51f4c9f7ae629b21", + "files": 1920 + } + }, + "lock_sha256": "699a14749e3825c62430faeb18224190fee1c9347760494e38e334e180bea6ec", + "source_sha256": { + ".agents/skills/reverse-engineer-anything/SKILL.md": "4baf850678f0781cbb989a7ac1452c4cbc5d98e4125bc059b5381c336cc8bc07", + "docs/rust-binaries.md": "3317a494376c3dde0c81a848ea678375db4af61fa85ba8d505b7d89b319c8371", + "experiments/analysis-efficiency/corpus.json": "81c223028c7c6308949e508413dea6aef79b691f6dc7fe02eb6ff81e6d52b0cc", + "experiments/analysis-efficiency/rust-dependency-costs.py": "c7eb7a9bc696e06c4cfba3b0aa730bf8cb9f2bcdec3a371eae3849a47c5f3135", + "experiments/analysis-efficiency/rust-dependency-views.mjs": "7c59302fe00c8a00e4cea06bca8f73974e40a618143ea4f1c7c5e275c9c07df5", + "src/application/rust/RustBinaryService.test.ts": "9b424569f52ed6c5e4629b3bda4fb8a8dc805adc0881e82f1bf39f38618c57a3", + "src/application/rust/RustBinaryService.ts": "caa089f62cdf24ee8a4ceb6eb7bdbf9df90fc724f921197f9e69b680366a98a7", + "src/cli/rustCommands.ts": "18c87736f20437f178d2094f0b46392d167ff510fa8a08a6b121610538fc257b", + "src/contracts/analysisViewToolContracts.ts": "149cd98c2ed22444ac9e59fa9144b6840c3fd13bd1613ddf8da23310766829f2", + "src/contracts/rust/rustToolContracts.ts": "5433f15f581618b3184147c04ccd21e04bc4a3a880b7ea0db6dbc38e5825efab", + "src/domain/analysisView/analysisView.ts": "8122d62bf795f3abae4bd903a641e336f0c2650cab98836051839bfaec4e1d65", + "src/domain/analysisView/goBinaryView.ts": "c434f7d7c198481d2fab2c2f7d5672f0db34256e9c8700a52c65c55def05df0b", + "src/domain/analysisView/rustBinaryView.test.ts": "3c53131b5643ed874214c0ba90e66960d2acdf9d22a881d283ef44aba671dfe4", + "src/domain/analysisView/rustBinaryView.ts": "8c330cf31b3ccb37e3b12a40e8157c50a9d43ac42cf6477dcca969398e0f79ad", + "src/domain/analysisView/selectViewRows.ts": "6eb486166e991045bf0a74cf874c80d2542862147b7ba095e45a61ec26afd1e5", + "src/domain/rust/rustBinary.ts": "90fc6fc20faa0f663dc14e08e8735804ae77abaa7fdb2e918162b358efb42462", + "src/domain/rust/rustBinaryQuery.ts": "ecb2ad8a97e3f3d232f94f5969baaef62143392362d18b4d46ab6babcfe96da2", + "src/server/createServer.ts": "b005d3d71edc98b9eb3422d83b8ddeddcc7db0b1982a1e93ce4578d588dbd8e6", + "src/server/registerRustTools.ts": "e57723a66916f3e2d9208e7202ac50b28536234874fbc3752e9259e0145299ae", + "tests/acceptance/rustBinaryCli.test.ts": "85962f740bd1d2f1fc99601337ab7e7ca3f4e659553d41297aa216e8928c0af4", + "tests/acceptance/rustDependencyViews.test.ts": "b1505497e9dbfb1ce855725a8af501f1ebc82bb2c528b8992e9b33cf8628ef7f", + "tests/boundary/mcp/rustBinaryMcp.test.ts": "4d2c5f172ebcd3407265b195adc7c5bc5dd590d97a443560a6d8c6afda60d08a" + }, + "resource_guard_sha256": { + "resource-run.py": "7a9642ab86ba34290564f636eb080cacf38b4185a254028680b238bff172993b", + "resource_payload.py": "6f089e645459eba80c5d73e39c21606c78173a44daf49fabc3b88c4631ce51c6" + }, + "host": { + "platform": "Linux-6.1.0-54-cloud-amd64-x86_64-with-glibc2.36", + "machine": "x86_64", + "affinity": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + "node": "v24.18.0", + "npm": "11.16.0", + "objcopy": "GNU objcopy (GNU Binutils for Debian) 2.40", + "readelf": "GNU readelf (GNU Binutils for Debian) 2.40", + "python": "3.11.2" + }, + "artifacts": [ + { + "package": "ripgrep-14.1.1-r0.apk", + "url": "https://dl-cdn.alpinelinux.org/alpine/v3.21/community/x86_64/ripgrep-14.1.1-r0.apk", + "package_sha256": "213dd0b047163f6627cc2fdd6fbfe205009476eb4fc48c352a3f5b9ab647f6d4", + "package_bytes": 1422830, + "member": "usr/bin/rg", + "bytes": 3416168, + "sha256": "78387bbe12b2110df8d2ce364390a00c190f4543a2f40b8279e49f6c5c85af9c", + "aports_revision": "4e899d119e4c983a938436d6f397a509521225d1" + }, + { + "package": "zellij-0.40.0-r0.apk", + "url": "https://dl-cdn.alpinelinux.org/alpine/v3.21/community/x86_64/zellij-0.40.0-r0.apk", + "package_sha256": "575075945415b221857563e19c5f4c661af9f98bbba54f0994c7385c0e432432", + "package_bytes": 6722129, + "member": "usr/bin/zellij", + "bytes": 17151864, + "sha256": "9a86e9ca47902a869fdf1886f4cf17714f4fa5655427128beadff498cfb392db", + "aports_revision": "3be26a3a26e76e1bf3eff784bffe3956d29d77a0" + } + ], + "oracle": [ + { + "file": "ripgrep", + "packages": 42, + "edges": 88, + "reply_bytes": 10605, + "root": 33, + "upstream_json_sha256": "ea478aa80f91aa8f5159aba648ddfdda642fa2c98759e54e65623a19065634be", + "evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "matched": true + }, + { + "file": "zellij", + "packages": 422, + "edges": 1063, + "reply_bytes": 86805, + "root": 417, + "upstream_json_sha256": "9bcf8d694440a7e064189616aa40b69046e32ee1c70683ef2e0778150b9163ef", + "evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "matched": true + } + ], + "root_cycle_oracle": [ + { + "file": "ripgrep", + "root_index_independently_matched": 33, + "independent_topological_visit_count": 42, + "acyclic": true + }, + { + "file": "zellij", + "root_index_independently_matched": 417, + "independent_topological_visit_count": 422, + "acyclic": true + } + ], + "tokenizer": { + "tiktoken": "0.12.0", + "encoding": "cl100k_base", + "actual_model_usage": null, + "projection": "All discovered tool descriptions/input schemas once, plus every call argument and text reply including export. Selected-two-tool and complete transport catalog costs are separate. No model, host injection, history replay or retry observed." + }, + "samples": [ + { + "case": "r1-zellij-before", + "wall_seconds": 5.77, + "cpu_seconds": 5.626476, + "kernel_family_peak_bytes": 644210688, + "completed_with_verified_cleanup": true, + "sha256": { + "report.json": "f2bd86aedd299c4fe16cca470eeaf135b6553c1bfe345018def503424c5661a6", + "catalog.json": "411381fb2417945f8a29eb8db330f82b44d8790d3f1e88c7f766b532c528008f", + "transcript.json": "bca9b5474a2124a99535c4aca0c38d9c77146d508822904aae4adc458b05b10e", + "r1-zellij-before.log": "79acdc6c966345d59ba88aafd22b5fb11cdd759c62611e720a47aef308ef070c" + }, + "calls_ms": [397.233, 103.9], + "reply_bytes": [185833, 310], + "combined_tokens": 223755 + }, + { + "case": "r2-zellij-before", + "wall_seconds": 4.957, + "cpu_seconds": 4.781552, + "kernel_family_peak_bytes": 636317696, + "completed_with_verified_cleanup": true, + "sha256": { + "report.json": "f3f58187dc63116f02586b2b3403848a8a88f6f72c5e322d0be00c9fcb2b7710", + "catalog.json": "411381fb2417945f8a29eb8db330f82b44d8790d3f1e88c7f766b532c528008f", + "transcript.json": "718aa02bf3fd74099b44418ac220e639719495b3e718357fb4e34748e5499e41", + "r2-zellij-before.log": "3e36a14c0b7e8d4d6e6f8b4b024c090133773f2699a8737d17dc5352b5f490ac" + }, + "calls_ms": [380.788, 40.185], + "reply_bytes": [185833, 310], + "combined_tokens": 223755 + }, + { + "case": "r3-zellij-before", + "wall_seconds": 4.854, + "cpu_seconds": 4.693446, + "kernel_family_peak_bytes": 639963136, + "completed_with_verified_cleanup": true, + "sha256": { + "report.json": "75d54eb72935201b8946d83a3bef98d93dc1219022bf735935d42934069b7dea", + "catalog.json": "411381fb2417945f8a29eb8db330f82b44d8790d3f1e88c7f766b532c528008f", + "transcript.json": "940eb7899a07a1115402fbc7b89010b463723b26e129f62a51263b9c9e598b1f", + "r3-zellij-before.log": "0f266f87cf5f2984ac10964f3255ebb76a706695256683c7f80c071b84d1f3a6" + }, + "calls_ms": [340.588, 30.985], + "reply_bytes": [185833, 310], + "combined_tokens": 223755 + }, + { + "case": "r1-zellij-after", + "wall_seconds": 6.277, + "cpu_seconds": 6.056786, + "kernel_family_peak_bytes": 637652992, + "completed_with_verified_cleanup": true, + "sha256": { + "report.json": "8d34bbdba7cfedc58e281efc372cf6969def10bc99b022e4bc786d447daf4709", + "catalog.json": "ba99f10270e8ab53d765f124856921f59156482d603fa883240e1b4068eab921", + "transcript.json": "e09cc44b0ce78a74be6cc9d7c90a78e2173514e5a4cc1dd9f02e5a728c880600", + "r1-zellij-after.log": "a90915676741794e52568dd30604a4e306ebfec77df37982bef6ea3d4411e405" + }, + "calls_ms": [517.341, 43.422], + "reply_bytes": [9261, 308], + "combined_tokens": 199584 + }, + { + "case": "r2-zellij-after", + "wall_seconds": 5.671, + "cpu_seconds": 5.441365, + "kernel_family_peak_bytes": 633688064, + "completed_with_verified_cleanup": true, + "sha256": { + "report.json": "425d70965ad49048d01a88473c1fb4790a7ef6f4ceb1c5374a652fc239fa96e5", + "catalog.json": "ba99f10270e8ab53d765f124856921f59156482d603fa883240e1b4068eab921", + "transcript.json": "96d3bc1798136135beed8e7f2b4eac626124655fb01ca1cdfe1040786ddf9efa", + "r2-zellij-after.log": "74fed3f25bd859ce4f6d4f4e7a6d3aaf31c9f079f730427fffa489a003adcfaa" + }, + "calls_ms": [473.788, 28.945], + "reply_bytes": [9261, 308], + "combined_tokens": 199584 + }, + { + "case": "r3-zellij-after", + "wall_seconds": 5.054, + "cpu_seconds": 4.816254, + "kernel_family_peak_bytes": 622559232, + "completed_with_verified_cleanup": true, + "sha256": { + "report.json": "349360e9b1c19faaa796106a70a26fd5711b804af807fc1b9c98a6e012530f9e", + "catalog.json": "ba99f10270e8ab53d765f124856921f59156482d603fa883240e1b4068eab921", + "transcript.json": "12fc240871df42621b8e80efdb6451136ea317ae364f7c6741aed348a12a73a9", + "r3-zellij-after.log": "18d9edcae6ac46c4034b958ba2e607a524aaf80dfbc671a5fd654f6390ee5c3f" + }, + "calls_ms": [359.933, 34.275], + "reply_bytes": [9261, 308], + "combined_tokens": 199584 + }, + { + "case": "r1-ripgrep-before", + "wall_seconds": 4.859, + "cpu_seconds": 4.643315, + "kernel_family_peak_bytes": 599961600, + "completed_with_verified_cleanup": true, + "sha256": { + "report.json": "2918f88999120cfba046ad5dc615d12bcc98c034c767a00edb3631a2b8bd622a", + "catalog.json": "411381fb2417945f8a29eb8db330f82b44d8790d3f1e88c7f766b532c528008f", + "transcript.json": "aa98de6fb6cbd24672e3a5b8aaf22ff1f977276d00c14ebd32e90501bd9eb4aa", + "r1-ripgrep-before.log": "a3963e991e1286e84a03f69f2d7e8a7e313ea262123269777599d93b7e529153" + }, + "calls_ms": [181.636, 30.005], + "reply_bytes": [22643, 312], + "combined_tokens": 200603 + }, + { + "case": "r2-ripgrep-before", + "wall_seconds": 5.057, + "cpu_seconds": 4.853134, + "kernel_family_peak_bytes": 596680704, + "completed_with_verified_cleanup": true, + "sha256": { + "report.json": "35f8de985a6201efc40f629195a9a801d2f333d605a46220bd4ae313c35f2abd", + "catalog.json": "411381fb2417945f8a29eb8db330f82b44d8790d3f1e88c7f766b532c528008f", + "transcript.json": "bed19b9d031b3416bc79870fb8d94ed0a2376004dfb9b69b78c56bd4a38912ff", + "r2-ripgrep-before.log": "ec0a5b2933307131af7435d737dde4a72ad3c92f3923e46aaaae7b0b7b7b1fd3" + }, + "calls_ms": [189.36, 29.044], + "reply_bytes": [22643, 312], + "combined_tokens": 200603 + }, + { + "case": "r3-ripgrep-before", + "wall_seconds": 4.753, + "cpu_seconds": 4.542499, + "kernel_family_peak_bytes": 589565952, + "completed_with_verified_cleanup": true, + "sha256": { + "report.json": "daaf848260488ef347afc4c600fe89c2902d6a68fa65085c0fcae7a20d7c6d17", + "catalog.json": "411381fb2417945f8a29eb8db330f82b44d8790d3f1e88c7f766b532c528008f", + "transcript.json": "43e570e7421f14aaaf7dc92b8c4a69d12a221851c7e813587eda8ca73e0b1d7f", + "r3-ripgrep-before.log": "6f88e1abbe8ebb8a68cf3a5d678c227516c8506029b0114094b1a42bfd056971" + }, + "calls_ms": [201.092, 30.174], + "reply_bytes": [22643, 312], + "combined_tokens": 200603 + }, + { + "case": "r1-ripgrep-after", + "wall_seconds": 5.159, + "cpu_seconds": 4.945401, + "kernel_family_peak_bytes": 609128448, + "completed_with_verified_cleanup": true, + "sha256": { + "report.json": "14294e20d39e6413e1dac9836a01dd498b7a669c4fa0a9708fd131452a0b6881", + "catalog.json": "ba99f10270e8ab53d765f124856921f59156482d603fa883240e1b4068eab921", + "transcript.json": "dd32475dc2205d3574a0496730a9c60751464c2fcc4a955051e889f1b0547c0d", + "r1-ripgrep-after.log": "dd22e6f0e9eac449e0485a67ae79839958109e5331f4872aa9a527a51aae1766" + }, + "calls_ms": [244.153, 30.492], + "reply_bytes": [9207, 310], + "combined_tokens": 199584 + }, + { + "case": "r2-ripgrep-after", + "wall_seconds": 4.757, + "cpu_seconds": 4.528198, + "kernel_family_peak_bytes": 615854080, + "completed_with_verified_cleanup": true, + "sha256": { + "report.json": "d45dc890ea69f370b47ccefc2b454dc1d0c3baf2beafd0fbe36ffb91c890e8e3", + "catalog.json": "ba99f10270e8ab53d765f124856921f59156482d603fa883240e1b4068eab921", + "transcript.json": "2b060caea34407a0c5c18fac3cad43a4dd048eee54bb760dd56ca95665b6a492", + "r2-ripgrep-after.log": "36cdb1176e1fdeef6e00f360c1a36ddb97506fcd31b426ee9be746da9ece7535" + }, + "calls_ms": [249.866, 28.919], + "reply_bytes": [9207, 310], + "combined_tokens": 199584 + }, + { + "case": "r3-ripgrep-after", + "wall_seconds": 4.758, + "cpu_seconds": 4.512774, + "kernel_family_peak_bytes": 615874560, + "completed_with_verified_cleanup": true, + "sha256": { + "report.json": "61ec33fe49c943a71177562212d255780ee6cf4cce273c1733a9ae6c138c6167", + "catalog.json": "ba99f10270e8ab53d765f124856921f59156482d603fa883240e1b4068eab921", + "transcript.json": "eb48ed154e646e3aa1ea9cb1aae99e363bb68ac122777ba4f3ac63d84547db37", + "r3-ripgrep-after.log": "34fd034def0ab7e0c22e9aa89aaaef5f5cc58a9125606400d8e866023578432b" + }, + "calls_ms": [224.772, 27.335], + "reply_bytes": [9207, 310], + "combined_tokens": 199584 + } + ], + "resource_limits": { + "cpu_quota_percent": 100, + "affinity": [15], + "hard_memory_gib": 4, + "early_stop_gib": 3, + "swap_bytes": 0, + "tasks": 192, + "host_preflight_gib": 8, + "host_stop_gib": 4 + }, + "cache_state": "Fresh process/session every task; unflushed OS caches, controller hashes inputs first; not cold disk.", + "failed_attempts": [ + { + "attempt": "baseline runtime build and cached retry", + "outcome": "Commands exited 0; cleanup counters unqualified due extra observed child; no qualified build resource claim. Actual CLI and SDK tasks separately qualify." + }, + { + "attempt": "prototype source typechecks", + "outcome": "Failed on new optional subject and expanded result-union typings; fixed and passed final check." + }, + { + "attempt": "prototype static gate", + "outcome": "New acceptance function exceeded line limit; refactored and passed." + }, + { + "attempt": "first one-CPU acceptance cohort", + "outcome": "Existing cross-Go/Rust acceptance test timed out at its unchanged 30s limit. All nine CLI acceptance tests passed with a bounded two-CPU/4-GiB cohort; no timeout change." + } + ], + "rejected_prototype": { + "change": "Replace native JSON.stringify report sizing with shared JS byte counter", + "provider_first_repeat_ms": { + "provider-ripgrep-candidate": { + "first_ms": 76.985, + "repeat_median_ms": 20.225 + }, + "provider-zellij-baseline": { + "first_ms": 181.069, + "repeat_median_ms": 92.804 + }, + "provider-ripgrep-baseline": { + "first_ms": 60.221, + "repeat_median_ms": 18.323 + }, + "provider-zellij-candidate": { + "first_ms": 183.659, + "repeat_median_ms": 76.214 + } + }, + "decision": "No consistent CPU or memory advantage established across both real artifacts; reverted. Final provider file is byte-identical to baseline." + }, + "diagnostic_phase_controls": { + "driver_sha256": "eeca9644b4b7199e45e49447bf83f7b33ac60c9ce1c38908b06f59d72e3b7485", + "method": "Same task driver plus performance.now around client.connect and listTools; baseline complete, candidate complete and candidate selected in forward/reverse order. Private instrumented driver, not the primary cohort.", + "samples": [ + { + "case": "control1-baseline", + "connect_ms": 2515.583, + "catalog_ms": 1676.58, + "calls_ms": [431.356, 57.797], + "wall_seconds": 5.46, + "cpu_seconds": 5.243091, + "kernel_family_peak_bytes": 644329472, + "qualified": true, + "report_sha256": "badff5397d47a240272feb4bd2348e3122d5482eea116933b29eddd740768955", + "log_sha256": "9b0eb1bfe51e932a4e8978d165fc26f213b003ee4452f704da1f9ce813830850" + }, + { + "case": "control1-candidate-complete", + "connect_ms": 3086.124, + "catalog_ms": 1726.06, + "calls_ms": [485.865, 30.696], + "wall_seconds": 6.164, + "cpu_seconds": 5.94611, + "kernel_family_peak_bytes": 634658816, + "qualified": true, + "report_sha256": "ac65641dcf7dd556017348860484b2a0485628803ae2f9e619cee40007fe8aff", + "log_sha256": "457a2dee6f38d2f57b4616c5b1138abc26c18235844fa4589e89b4f15ca4abf8" + }, + { + "case": "control1-candidate-selected", + "connect_ms": 3057.187, + "catalog_ms": 1677.082, + "calls_ms": [431.301, 32.986], + "wall_seconds": 6.063, + "cpu_seconds": 5.798686, + "kernel_family_peak_bytes": 639565824, + "qualified": true, + "report_sha256": "ef8e46c78e81f13c84f1bf0a8a8284078252d5af16302a716144149c136f89f5", + "log_sha256": "8d72501364d0a95fb250e2d0486b9b34076688309748b8dc054a75681e8543e0" + }, + { + "case": "control2-baseline", + "connect_ms": 2678.85, + "catalog_ms": 1467.674, + "calls_ms": [413.185, 36.964], + "wall_seconds": 5.364, + "cpu_seconds": 5.183688, + "kernel_family_peak_bytes": 643944448, + "qualified": true, + "report_sha256": "d7e3318331528e1201fd1efc7cdc4edb0624a32e6c6db501504cc77f49374157", + "log_sha256": "d61f437e5c88b2eb994e1f716c05915f0955d5877b1d34207fd39e3d36f85970" + }, + { + "case": "control2-candidate-complete", + "connect_ms": 2527.714, + "catalog_ms": 1545.887, + "calls_ms": [436.068, 30.776], + "wall_seconds": 5.36, + "cpu_seconds": 5.159006, + "kernel_family_peak_bytes": 635162624, + "qualified": true, + "report_sha256": "fb432b0f5eeedc74e66144c136954e580f5efb5b86a921ace3bf29477e90f0aa", + "log_sha256": "4e7485b8b2a5bb0619f0e84df96b68e02827e2afee11533e0f12532c12fd84f7" + }, + { + "case": "control2-candidate-selected", + "connect_ms": 2642.283, + "catalog_ms": 1465.879, + "calls_ms": [424.082, 32.29], + "wall_seconds": 5.362, + "cpu_seconds": 5.110599, + "kernel_family_peak_bytes": 637669376, + "qualified": true, + "report_sha256": "2d4b794902c9fd76b11a2f7ae87bb06be3cc9effadfb161f6f4debb9044ebcfd", + "log_sha256": "e98cc393c717ae19bb0c880d609a05e2f2d4249701f245c19b2f030b8be305fb" + } + ] + }, + "cohort_facts": { + "zellij-complete": { + "answer": { + "packages": [ + { + "index": 294, + "name": "serde", + "version": "1.0.137", + "source": "crates.io", + "dependencies": [296] + }, + { + "index": 296, + "name": "serde_derive", + "version": "1.0.137", + "source": "crates.io", + "dependencies": [256, 264, 331] + } + ], + "direct_edges": [ + { + "from_index": 294, + "to_index": 296 + } + ] + }, + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "artifact_sha256": "9a86e9ca47902a869fdf1886f4cf17714f4fa5655427128beadff498cfb392db", + "complete_result_sha256": "a9ce92f47aaa92fb0edc1c288d06c298ae623d731499fd4cedb86ab9c2bbb483", + "catalog_sha256": "411381fb2417945f8a29eb8db330f82b44d8790d3f1e88c7f766b532c528008f", + "catalog_tools": 212, + "token_proxy": { + "input_catalog_once": 197359, + "arguments": 58, + "text_replies": 26338, + "combined": 223755, + "only_two_selected_tools": 26850, + "complete_transport_catalog": 886882 + } + }, + "zellij-selected": { + "answer": { + "packages": [ + { + "name": "serde", + "version": "1.0.137", + "source": "crates.io", + "dependencies": [296], + "index": 294 + }, + { + "name": "serde_derive", + "version": "1.0.137", + "source": "crates.io", + "dependencies": [256, 264, 331], + "index": 296 + } + ], + "direct_edges": [ + { + "from_index": 294, + "to_index": 296 + } + ] + }, + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "artifact_sha256": "9a86e9ca47902a869fdf1886f4cf17714f4fa5655427128beadff498cfb392db", + "complete_result_sha256": "a9ce92f47aaa92fb0edc1c288d06c298ae623d731499fd4cedb86ab9c2bbb483", + "catalog_sha256": "ba99f10270e8ab53d765f124856921f59156482d603fa883240e1b4068eab921", + "catalog_tools": 212, + "token_proxy": { + "input_catalog_once": 198278, + "arguments": 83, + "text_replies": 1223, + "combined": 199584, + "only_two_selected_tools": 2256, + "complete_transport_catalog": 891294 + } + }, + "ripgrep-complete": { + "answer": { + "packages": [ + { + "index": 36, + "name": "serde", + "version": "1.0.210", + "source": "crates.io", + "dependencies": [] + } + ], + "direct_edges": [] + }, + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "artifact_sha256": "78387bbe12b2110df8d2ce364390a00c190f4543a2f40b8279e49f6c5c85af9c", + "complete_result_sha256": "9eb75332f423bcb94f627694cde96fb3f39ce7d47313468e027e8ccf03c38f72", + "catalog_sha256": "411381fb2417945f8a29eb8db330f82b44d8790d3f1e88c7f766b532c528008f", + "catalog_tools": 212, + "token_proxy": { + "input_catalog_once": 197359, + "arguments": 62, + "text_replies": 3182, + "combined": 200603, + "only_two_selected_tools": 3698, + "complete_transport_catalog": 886882 + } + }, + "ripgrep-selected": { + "answer": { + "packages": [ + { + "name": "serde", + "version": "1.0.210", + "source": "crates.io", + "dependencies": [], + "index": 36 + } + ], + "direct_edges": [] + }, + "parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4", + "artifact_sha256": "78387bbe12b2110df8d2ce364390a00c190f4543a2f40b8279e49f6c5c85af9c", + "complete_result_sha256": "9eb75332f423bcb94f627694cde96fb3f39ce7d47313468e027e8ccf03c38f72", + "catalog_sha256": "ba99f10270e8ab53d765f124856921f59156482d603fa883240e1b4068eab921", + "catalog_tools": 212, + "token_proxy": { + "input_catalog_once": 198278, + "arguments": 87, + "text_replies": 1219, + "combined": 199584, + "only_two_selected_tools": 2256, + "complete_transport_catalog": 891294 + } + } + }, + "accounting": "Kernel cgroup CPU/peak memory at payload exit while an observer remains alive, with verified exclusive observer and cleanup. Final whole-scope counters unavailable; observer overhead is included. All qualified samples have zero high/max/OOM/pid events.", + "published_equivalent_source": "cc83fd1ee3827861ea94d0e8f325e4521323bc89", + "integration_base": "a5f982de8dbe4b092cc20ab89d492b7e7028b758", + "candidate_reproduction": "Primary paired cohort uses candidate_code on baseline_source. Integration rebase production files are byte-identical to published_equivalent_source; original measured commit was pushed before rebase. Driver/counter/resource hashes and runtime tables are pinned separately." +} diff --git a/experiments/analysis-efficiency/rust-dependency-views.md b/experiments/analysis-efficiency/rust-dependency-views.md new file mode 100644 index 0000000000..f556dd931c --- /dev/null +++ b/experiments/analysis-efficiency/rust-dependency-views.md @@ -0,0 +1,176 @@ +# Rust dependency fact views + +Issue [#5802](https://github.com/morluto/rea/issues/5802) asks for exact embedded +crate versions, sources and selected direct relationships without receiving the +entire dependency report. The existing path-only operation returned every +package, dependency vector and edge. This work adds opt-in first-call selection, +summary delivery and retained package/edge queries to the existing CLI/MCP +contracts. Complete delivery remains the default. + +The producer still reads and validates the complete admitted file and report. +This is a context optimization; it does not establish faster binary analysis or +lower peak memory. It adds no tool, path cache, dependency walker or graph index. +Only successful interpretation of an authenticated immutable Evidence result is +reused, keyed by that exact object in a WeakMap for its lifetime. Mutable imports +are revalidated, and every view checks the artifact subject again. + +## Pinned artifacts and independent checks + +[Curated receipts](rust-dependency-views.json) pin source commits, runtime file +tables, source/driver/resource hashes, package URLs and all measured samples. +Baseline is `d672cc23998b6cbb21c485f118595ebfb82fed33`; measured candidate is +`cae2571e7e64a7364b94bb92c2630cba80ec4478`, pushed before the integration rebase. +The production files are byte-identical to published +`cc83fd1ee` after rebasing onto `a5f982de8`. Later integration measurements do +not relabel the original cohort. + +Official Alpine v3.21/community/x86_64 APKs, not locally rebuilt test applications: + +| Artifact | File bytes | Packages / edges | Complete CLI reply bytes | +| ---------------------------------- | ---------: | ---------------: | -----------------------: | +| ripgrep 14.1.1-r0, `usr/bin/rg` | 3,416,168 | 42 / 88 | 10,605 | +| Zellij 0.40.0-r0, `usr/bin/zellij` | 17,151,864 | 422 / 1,063 | 86,805 | + +The [corpus manifest](corpus.json) records APK and extracted-file SHA-256, +transport size and aports revision. Acquisition selected only the named regular +tar member, with bounded download/extraction. Neither binary was executed. +HTTPS provenance and recorded digests are not an APK-signature verification. + +GNU objcopy/readelf 2.40 independently extracted `.dep-v0` into private output +files without overwriting the input. Python 3.11.2 zlib/JSON parsing, following +the [upstream parsing format](https://github.com/rust-secure-code/cargo-auditable/blob/master/PARSING.md), +matched every exposed package field and dependency edge, omitted revision, +root index, compressed/inflated length, exact section range and full artifact +digest. An independent topological pass visited all 42/422 packages and matched +the empty cyclic sets. This qualifies these two ELF artifacts; PE/Mach-O view +coverage below uses constructed format fixtures, not real shipped applications. + +## Equivalent completed SDK tasks + +Linux x64, Node 24.18.0/npm 11.16.0. Three alternating before/after, +after/before, before/after pairs per artifact run serially. Each fresh production +stdio SDK session discovers all 212 tools and uses their named output schemas. +OS caches are unflushed; the controller hashes inputs first. These are fresh +process/session runs, not cold-disk runs. + +The fixed question is: which versions and sources of `serde` and `serde_derive` +are embedded, and is there a direct edge between those selected packages? +Both lanes make exactly two RPCs: inspect once, then export complete Evidence. +Baseline derives the answer from the complete reply. Candidate selects both +names and `name/version/source/dependencies` in the initial inspection. There +are no extra lookup calls. All runs compare the answer and exported complete +parent against the independently checked report, including identical parent +Evidence IDs and source SHA-256. Zellij has the direct edge 294 → 296; +ripgrep has only `serde`, with no such edge within the embedded records. + +Medians of three whole tasks, including startup, discovery, SDK validation, +both calls, answer/export checks and teardown: + +| Artifact | Wall seconds, before → after | Family CPU seconds, before → after | Peak family MiB, before → after | +| -------- | ---------------------------: | ---------------------------------: | ------------------------------: | +| Zellij | 4.957 → 5.671 | 4.782 → 5.441 | 610.3 → 604.3 | +| ripgrep | 4.859 → 4.758 | 4.643 → 4.528 | 569.0 → 587.3 | + +The main cohort's Zellij elapsed/CPU medians increase about 14%; do not read the +context improvement as a whole-task speedup. Two further forward/reverse +diagnostic controls added clocks around connect and discovery, comparing +baseline complete, candidate complete and candidate selected. In the first +control these took 5.460/6.164/6.063 seconds, with connect times +2.516/3.086/3.057 seconds. In the reverse control all took 5.360–5.364 seconds, +with connect times 2.528–2.679 seconds. Candidate complete also varied, so those +controls do not isolate a persistent regression to compact projection. They +do not erase the original slower cohort. No stable speedup, peak-memory +improvement or exact cause of startup variation is established. + +Initial inspector RPC medians were 380.8 → 473.8 ms for Zellij and +189.4 → 244.2 ms for ripgrep. These include first-use named SDK schema validation, +retention, projection and serialization; they are not isolated parser timings. +Extra retained reads need no source reopen, as the CLI/MCP regressions demonstrate; +their warm latency was not benchmarked. + +## Context accounting and limits + +Offline tiktoken 0.12.0, `cl100k_base`, counts all discovered tool descriptions +and input schemas once, every call argument and every text reply, including +export. It counts the whole scripted task rather than only the selected reply: + +| Artifact | All 212 input definitions + calls, before → after | Only the two used definitions + calls, before → after | +| -------- | ------------------------------------------------: | ----------------------------------------------------: | +| Zellij | 223,755 → 199,584 (−10.8%) | 26,850 → 2,256 (−91.6%) | +| ripgrep | 200,603 → 199,584 (−0.5%) | 3,698 → 2,256 (−39.0%) | + +The Zellij inspect response, including both MCP text and structured content, +shrinks from 185,833 to 9,261 bytes; ripgrep shrinks from 22,643 to 9,207. +Those byte counts alone are not token measurements. Full transport catalogs, +including all output schemas, grow from 886,882 to 891,294 estimated tokens. +If all of that transport metadata is charged once along with calls, totals are +913,278 → 892,600 for Zellij and 890,126 → 892,600 for ripgrep: the small case +then costs more. Schema expansion has a cost even when Rust is not called. + +No AI inference, actual model usage/billing, host schema injection, prompt-history +replays or analyst answer quality was measured. The two-definition column is a +discovery scenario, not an observed routed host. Do not extrapolate these +proxies to a host that repeats schemas or truncates full replies. Opt-in +delivery preserves caller choice and avoids forcing the small-case tradeoff. + +## Correctness, resource bounds and rejected changes + +Original-index facts, all matching versions, exact intersecting batch filters, +empty selections, selected fields and matching-row pagination share one domain +projector through the existing application layer. Package defaults omit +dependency vectors; callers explicitly request them for direct references. +Edges support both endpoint filters. Complete parent Evidence stays available +for export, and portable/retained queries still work after deleting the source. +Missing metadata remains unknown, cycles are counted without walking paths, +compiler hashes remain inferred, and embedded coverage never implies complete +runtime dependencies. Defaults preserve complete Evidence identity. + +Executed tests cover real CLI/stdio SDK schemas, ELF/PE/thin-Mach-O fixtures, +duplicate package names/versions, retained and portable imports, malformed +edge imports, subject mismatch, unknown metadata, contradictory delivery, +retention requirements, bounded warnings and immutable-cache trust. Existing +Go, malformed Rust, cancellation, capacity and cleanup coverage is retained. +Full typecheck/lint/format/knip/module/architecture checks passed before the +integration rebase; final integration checks are recorded in the PR. + +Runtime tasks use one pinned CPU, 4-GiB hard/3-GiB early-stop family limits, +no swap, 192 tasks and host-memory preflight/stop guards. All 12 primary and +six diagnostic samples have verified payload exit/cleanup and zero high/max/ +OOM/pid events. CPU and peak memory use cgroup counters at payload exit while +the observer remains alive; final whole-scope counters are unavailable. +Build/check guards use 6/8-GiB hard bounds respectively. An existing 30-second +mixed Go/Rust test timed out in the initial single-CPU cohort; all nine CLI +acceptance tests passed with a bounded two-CPU/4-GiB rerun, without changing +the test's timeout. Other corrected prototype failures remain in the receipts. + +A proposed JS JSON-size counter replacement had inconsistent provider timing +and no demonstrated memory benefit on these artifacts. It was reverted; the +final Rust provider is unchanged. Full-file input allocation, complete report +construction, ledger retention and schema duplication remain costs. This issue +does not qualify every Rust workflow, larger graphs, real PE/Mach-O applications, +WASM support or current runtime dependency state. + +## Reproduction + +Acquire the exact public APK URLs from the manifest and extract only the named +regular binary. Verify both SHA-256 values before inspection. Use objcopy +`--dump-section .dep-v0=SECTION INPUT PRIVATE_COPY`, then Python zlib/JSON to +compare the normalized package/edge records and section bytes. Save the verified +complete CLI `normalized_result` as `EXPECTED`. + +Build isolated baseline/candidate runtimes with the pinned toolchain. Keep raw +outputs outside Git. For each artifact, alternate three serial pairs: + +```bash +python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \ + node experiments/analysis-efficiency/rust-dependency-views.mjs \ + --runtime /absolute/runtime --artifact /absolute/binary \ + --expected /absolute/expected-report.json --output /absolute/new-case \ + --lane complete +# Candidate uses --lane selected. Both lanes include complete Evidence export. +``` + +With private `tiktoken==0.12.0`, run +`rust-dependency-costs.py --output FILE CASE...`; each `CASE.log` must contain +qualified guard observations. The counter rejects failed tasks, missing +cleanup and resource events. Guard-stopped attempts are not completed samples. diff --git a/experiments/analysis-efficiency/rust-dependency-views.mjs b/experiments/analysis-efficiency/rust-dependency-views.mjs new file mode 100644 index 0000000000..1306ddb1d8 --- /dev/null +++ b/experiments/analysis-efficiency/rust-dependency-views.mjs @@ -0,0 +1,172 @@ +/** Equivalent selected dependency questions; raw artifacts/transcripts stay outside Git. */ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { createWriteStream } from "node:fs"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { finished } from "node:stream/promises"; +import { parseArgs } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { createRetainedQueryCaller } from "./retained-query-calls.mjs"; +import { finalizeRetainedQueryReceipt } from "./retained-query-finalization.mjs"; + +const { values } = parseArgs({ + options: { + ...Object.fromEntries( + ["runtime", "artifact", "expected", "output"].map((name) => [ + name, + { type: "string" }, + ]), + ), + lane: { type: "string" }, + }, +}); +for (const name of ["runtime", "artifact", "expected", "output"]) + assert.ok(values[name] && values[name] === resolve(values[name]), name); +assert.ok(["complete", "selected"].includes(values.lane)); +const hash = (value) => createHash("sha256").update(value).digest("hex"); +const expected = JSON.parse(await readFile(values.expected, "utf8")); +assert.equal(expected.artifact.path, values.artifact); +const names = ["serde", "serde_derive"]; +const fields = ["name", "version", "source", "dependencies"]; +const choose = (report) => + (report.auditable_metadata?.packages ?? []).flatMap((row, index) => + names.includes(row.name) + ? [ + { + index, + ...Object.fromEntries(fields.map((field) => [field, row[field]])), + }, + ] + : [], + ); +const answer = (rows) => { + const targets = new Set( + rows.filter((row) => row.name === "serde_derive").map((row) => row.index), + ); + return { + packages: rows, + direct_edges: rows + .filter((row) => row.name === "serde") + .flatMap((row) => + row.dependencies + .filter((index) => targets.has(index)) + .map((to_index) => ({ from_index: row.index, to_index })), + ), + }; +}; +const expectedRows = choose(expected); +await mkdir(values.output, { mode: 0o700 }); +const account = await mkdtemp(join(tmpdir(), "rea-rust-views-")); +const client = new Client({ name: "rust-dependency-views", version: "1" }); +const transport = new StdioClientTransport({ + command: process.execPath, + args: [join(values.runtime, "scripts/rea.mjs"), "mcp"], + env: { + HOME: account, + PATH: process.env.PATH, + NODE_OPTIONS: "--max-old-space-size=2048", + UV_THREADPOOL_SIZE: "1", + REA_LOG_LEVEL: "silent", + TMPDIR: account, + }, + stderr: "pipe", +}); +const serverLog = createWriteStream(join(values.output, "server.log")); +const serverLogFinished = finished(serverLog); +serverLogFinished.catch(() => {}); +transport.stderr?.pipe(serverLog); +const report = { + lane: values.lane, + artifact_sha256: hash(await readFile(values.artifact)), + normalized_sha256: hash(JSON.stringify(expected)), + calls: [], +}; +const transcript = []; +const call = createRetainedQueryCaller(client, (receipt, event) => { + report.calls.push(receipt); + transcript.push(event); +}); +const observe = async (name, input, phase) => { + const reply = await call(name, input, phase); + assert.notEqual(reply.isError, true, JSON.stringify(reply)); + assert.ok(reply.structuredContent); + return reply.structuredContent; +}; +try { + await client.connect(transport); + const catalog = await client.listTools(); + assert.equal(catalog.nextCursor, undefined); + report.catalog_tools = catalog.tools.length; + report.catalog_sha256 = hash(JSON.stringify(catalog)); + await writeFile(join(values.output, "catalog.json"), JSON.stringify(catalog)); + const definitions = new Map(catalog.tools.map((tool) => [tool.name, tool])); + const original = client.callTool.bind(client); + client.callTool = (input, options) => + original(input, { + ...options, + toolDefinition: definitions.get(input.name), + }); + const selected = values.lane === "selected"; + const reply = await observe( + "inspect_rust_binary", + { + path: values.artifact, + ...(selected ? { view: { collection: "packages", names, fields } } : {}), + }, + "question", + ); + const parentId = selected + ? reply.normalized_result.parent_evidence_id + : reply.evidence_id; + const rows = selected + ? reply.normalized_result.page.rows + : choose(reply.normalized_result); + assert.deepEqual(rows, expectedRows); + report.answer = answer(rows); + assert.deepEqual(report.answer, answer(expectedRows)); + report.parent_evidence_id = parentId; + await observe( + "export_evidence_bundle", + { path: join(values.output, "evidence.json") }, + "export", + ); + const bundle = JSON.parse( + await readFile(join(values.output, "evidence.json"), "utf8"), + ); + const parent = bundle.records.find( + (record) => record.evidence_id === parentId, + ); + assert.ok(parent); + assert.deepEqual(parent.normalized_result, expected); + assert.equal(parent.subject.digest.sha256, report.artifact_sha256); + if (selected) + assert.deepEqual(reply.normalized_result.source_coverage, { + auditable_metadata_available: expected.auditable_metadata !== null, + embedded_records: + expected.auditable_metadata === null ? "unknown" : "complete", + runtime_dependencies: "unknown", + }); + assert.equal(hash(await readFile(values.artifact)), report.artifact_sha256); + report.status = "passed"; +} catch (cause) { + report.status = "failed"; + report.failure = String(cause?.stack ?? cause); + process.exitCode = 1; +} finally { + if ( + !(await finalizeRetainedQueryReceipt({ + report, + client, + transport, + account, + serverLog, + serverLogFinished, + output: values.output, + transcript, + })) + ) + process.exitCode = 1; +} diff --git a/experiments/analysis-efficiency/test_model_utf8_validation.py b/experiments/analysis-efficiency/test_model_utf8_validation.py new file mode 100644 index 0000000000..f6d0266d44 --- /dev/null +++ b/experiments/analysis-efficiency/test_model_utf8_validation.py @@ -0,0 +1,68 @@ +"""Phase receipt fault injection only; these stand-ins do not qualify a model provider.""" +import json +from pathlib import Path +import subprocess +import tempfile +import unittest + + +class ModelPhaseReceipts(unittest.TestCase): + def run_phase(self, inspect_body, close_body): + driver = Path(__file__).with_name('model-utf8-validation.mjs').resolve() + with tempfile.TemporaryDirectory(prefix='rea-model-phase-test-') as directory: + root = Path(directory) + artifact = root / 'input' + artifact.write_bytes(b'receipt fixture, not a model') + expected = {'artifact': {'path': str(artifact)}} + (root / 'expected.json').write_text(json.dumps(expected)) + (root / 'package.json').write_text('{"type":"module"}') + module = root / 'dist' / 'model' + module.mkdir(parents=True) + (module / 'ModelArtifactProvider.js').write_text( + 'export class ModelArtifactProvider {\n' + 'async inspect() {' + inspect_body.replace('EXPECTED', json.dumps(expected)) + '}\n' + 'async close() {' + close_body + '}\n}\n') + result = subprocess.run([ + 'node', str(driver), '--phase', '--runtime', str(root), + '--artifact', str(artifact), '--expected', str(root / 'expected.json'), + '--output', str(root / 'output'), + ], capture_output=True, text=True, timeout=15) + self.assertEqual(result.returncode, 1, result.stderr) + report = json.loads((root / 'output' / 'report.json').read_text()) + self.assertEqual(json.loads(result.stdout), report) + self.assertEqual(report['status'], 'failed') + self.assertEqual(report['receipt_failures'], []) + return report + + def test_rejected_inspection_keeps_cost_and_primary_failure_after_close_failure(self): + report = self.run_phase( + 'throw new Error("inspection failed");', + 'throw new Error("cleanup failed");') + self.assertEqual(len(report['calls']), 1) + sample = report['calls'][0] + self.assertEqual(sample['phase'], 'fresh') + self.assertTrue(sample['is_error']) + self.assertGreaterEqual(sample['elapsed_ms'], 0) + self.assertIn('inspection failed', sample['failure']) + self.assertIn('inspection failed', report['failure']) + self.assertEqual(report['cleanup_failures'], ['provider: Error: cleanup failed']) + + def test_mismatch_keeps_inspection_time_and_failure(self): + report = self.run_phase('return {ok:true,value:{wrong:true}};', '') + self.assertEqual(len(report['calls']), 1) + self.assertTrue(report['calls'][0]['is_error']) + self.assertGreaterEqual(report['calls'][0]['elapsed_ms'], 0) + self.assertIn('AssertionError', report['failure']) + self.assertEqual(report['cleanup_failures'], []) + + def test_close_failure_keeps_all_successful_samples_and_publishes_failed_receipt(self): + report = self.run_phase('return {ok:true,value:EXPECTED};', + 'throw new Error("cleanup failed");') + self.assertEqual(len(report['calls']), 7) + self.assertTrue(all(not sample['is_error'] for sample in report['calls'])) + self.assertNotIn('failure', report) + self.assertEqual(report['cleanup_failures'], ['provider: Error: cleanup failed']) + + +if __name__ == '__main__': + unittest.main() diff --git a/experiments/analysis-efficiency/wasm-string-framing.json b/experiments/analysis-efficiency/wasm-string-framing.json new file mode 100644 index 0000000000..d27ced7efa --- /dev/null +++ b/experiments/analysis-efficiency/wasm-string-framing.json @@ -0,0 +1,561 @@ +{ + "schema_version": 1, + "issue": 5760, + "baseline_source": "d0067ba34a39ea47dd7e11bb2e6169bc0938cc3f", + "candidate_code_source": "9776d78f6c799cba8d8cf5c9b3139a86449eaebd", + "integration_base": "335c042d166b2c8b2c0600935cf77607ae11edbc", + "toolchain": { + "node": "24.18.0", + "npm": "11.16.0", + "wabt": "1.0.42", + "tokenizer": "0.12.0", + "encoding": "cl100k_base" + }, + "catalog_sha256": "4586abd317171cbb307295cc687c24f7f45e603959538643325895b546a38242", + "catalog_tools": 212, + "package_lock_sha256": "699a14749e3825c62430faeb18224190fee1c9347760494e38e334e180bea6ec", + "runtime_mcp_sha256": { + "baseline": "fa6831f05d97b6b28642b709b5497e351d2ee632dd1abc0d2f87a648057afe87", + "candidate": "e4af82d847fe3d9816f4f7ae1f584ef129ba151bc5a6f95080a7519fd08be2cf" + }, + "sources_sha256": { + "WabtString.ts": "398f3cd98c087740a553768d10c21912d28f33e13bdab589eec760da4ed0b9fd", + "WabtWat.ts": "70fb9acfbe747be4fa5d8be235012a88f94485bc0baf0fbba1fe22f18b87f44b", + "WabtInterface.ts": "1a63c834a46268bbaf97e6eb39a99b3ccd3f2893527749d65ee6f5026bc2782a" + }, + "native_tools_sha256": { + "wasm-validate": "6b722ed0df68204cfc7a8effdf1fc2ddeac2d184bd211cdf7fdfa5f9d49d3eb5", + "wasm-objdump": "dad264def1a3b7755859c5bf48227f01259a95fa82b7c38358b12b9e70b27a41", + "wasm2wat": "4322d652f650c1ee117b549c9722f42fc93eb4676ff046d60ba033b5639697a5" + }, + "artifacts": { + "small": { + "sha256": "f399acbd52258f950f24dd5ca4a82674f1f26cea60e7be7c5daaade568768fda", + "bytes": 80, + "wat_sha256": "918ecd8a14daf20b58707d0879efda8b38df37f219279688b3aa3a812ae5646a", + "wat_bytes": 270, + "answer_normalized_sha256": "8eb8aedcf17bac4a98d34f5a004b5541f671350f94918575e2fc4eee9fd2db29", + "parent_evidence_id": "ev_88b35d9f926672b9e42fc4e1514090eaf2f99bb918439f1270dd53f7fe684e74" + }, + "sqlite": { + "sha256": "2ee8f3dab694532afc8840e07703127287662d08b74e6ff50491ce63f00d5752", + "bytes": 868907, + "wat_sha256": "7b1f8cd4fcbf310f4203ce64e15b4eb068ecdf29838551137245efecb7283f81", + "wat_bytes": 15382128, + "answer_normalized_sha256": "330b84e01541f70a964c826754da3e5f49a1bb1bd535ae1d1f00fb44a1157ff6", + "parent_evidence_id": "ev_899ecb126cd0a283169641085fa10e46460504bf8bc8eba69580fa6d641e431e" + }, + "css": { + "sha256": "8a23977fe271357cce6f254ef88c9bebf3854602d8046605aef6a45c02135c59", + "bytes": 128668, + "wat_sha256": "c53273cb7da0867ac52490d083c7952fcc538b3848c6d0d1f2e2621e2ded23c0", + "wat_bytes": 13599679, + "answer_normalized_sha256": "3d301feaa48a646c4d9f2a026353544519cc4694c9840b1c764d81ec9ba7fb65", + "parent_evidence_id": "ev_27108db83ae4d0a9289c23dace9c0e9e6d573f72f1e982b490c7f45394ba702c" + } + }, + "samples": [ + { + "case": "small", + "lane": "baseline", + "round": 1, + "wall_seconds": 12.417, + "cpu_seconds": 12.114979, + "memory_peak_bytes": 823410688, + "transport_proxy_tokens": 894936, + "fresh_inspect_ms": 1781.0228269999998, + "repeat_inspect_ms": 1793.3802260000011, + "reply_bytes": 36180, + "raw_guard_log_sha256": "f4ab763c21c4dd22f07ab978690966fd1efc6961216e41e195316e7330c269d6" + }, + { + "case": "small", + "lane": "candidate", + "round": 1, + "wall_seconds": 10.191, + "cpu_seconds": 9.868799, + "memory_peak_bytes": 829722624, + "transport_proxy_tokens": 894936, + "fresh_inspect_ms": 1298.6234909999998, + "repeat_inspect_ms": 1917.7533390000008, + "reply_bytes": 36182, + "raw_guard_log_sha256": "5cfd93018a2cabbe5b9fa4b9062fbed8e6da638ad5672fd566f5cf76182fecd5" + }, + { + "case": "sqlite", + "lane": "baseline", + "round": 1, + "wall_seconds": 14.317, + "cpu_seconds": 13.845753, + "memory_peak_bytes": 1032982528, + "transport_proxy_tokens": 895044, + "fresh_inspect_ms": 3405.006531, + "repeat_inspect_ms": 3345.840587999999, + "reply_bytes": 37248, + "raw_guard_log_sha256": "46b1b2e9f02e41be7a7cbae8a311b799ab52148a150ec5c0c683e0745c44ad03" + }, + { + "case": "sqlite", + "lane": "candidate", + "round": 1, + "wall_seconds": 13.311, + "cpu_seconds": 12.956772, + "memory_peak_bytes": 1029627904, + "transport_proxy_tokens": 895044, + "fresh_inspect_ms": 2838.048675, + "repeat_inspect_ms": 3233.226218, + "reply_bytes": 37250, + "raw_guard_log_sha256": "8933144b3f6fd100de72b1c1f22a791b7c4bef74c355bb9211be4c9e9383fe4b" + }, + { + "case": "css", + "lane": "baseline", + "round": 1, + "wall_seconds": 13.211, + "cpu_seconds": 12.79919, + "memory_peak_bytes": 1099599872, + "transport_proxy_tokens": 895430, + "fresh_inspect_ms": 2823.246057, + "repeat_inspect_ms": 3302.541784000001, + "reply_bytes": 37426, + "raw_guard_log_sha256": "9f4b7dfcd97faf8c85463a4c17f0612b6c37b7e29843f5e52b43df27aaae0ac6" + }, + { + "case": "css", + "lane": "candidate", + "round": 1, + "wall_seconds": 12.912, + "cpu_seconds": 12.46982, + "memory_peak_bytes": 1110261760, + "transport_proxy_tokens": 895430, + "fresh_inspect_ms": 2839.796418, + "repeat_inspect_ms": 3002.9903570000006, + "reply_bytes": 37428, + "raw_guard_log_sha256": "dc2ed426cd1191ca779a4e736bceb3d0001a5b8344dbba01544ecd91a6bb73b6" + }, + { + "case": "small", + "lane": "candidate", + "round": 2, + "wall_seconds": 10.295, + "cpu_seconds": 9.986392, + "memory_peak_bytes": 807264256, + "transport_proxy_tokens": 894936, + "fresh_inspect_ms": 1687.949067, + "repeat_inspect_ms": 1907.0628659999993, + "reply_bytes": 36182, + "raw_guard_log_sha256": "376199e896af92b6351d2dde7c765b5aa4b1feac0e9664da934e24732eaa22ec" + }, + { + "case": "small", + "lane": "baseline", + "round": 2, + "wall_seconds": 10.192, + "cpu_seconds": 9.86319, + "memory_peak_bytes": 826806272, + "transport_proxy_tokens": 894936, + "fresh_inspect_ms": 1621.7653269999996, + "repeat_inspect_ms": 1937.1286520000012, + "reply_bytes": 36180, + "raw_guard_log_sha256": "2d40d7309203eefcb7cbb5a5e81640e728839eb30f29cad017584738ea50616f" + }, + { + "case": "sqlite", + "lane": "candidate", + "round": 2, + "wall_seconds": 13.52, + "cpu_seconds": 13.133355, + "memory_peak_bytes": 1211944960, + "transport_proxy_tokens": 895044, + "fresh_inspect_ms": 3167.13338, + "repeat_inspect_ms": 3425.1366179999986, + "reply_bytes": 37250, + "raw_guard_log_sha256": "a962200a36a2530b044548b475b28d7234f73b875ffbc3fe16ef4b36e61ae49a" + }, + { + "case": "sqlite", + "lane": "baseline", + "round": 2, + "wall_seconds": 13.515, + "cpu_seconds": 13.170183, + "memory_peak_bytes": 1017561088, + "transport_proxy_tokens": 895044, + "fresh_inspect_ms": 3224.859665, + "repeat_inspect_ms": 3230.52117, + "reply_bytes": 37248, + "raw_guard_log_sha256": "0452dae8c26a023bd6220bca03dfd7a58425ef579f8e7964f04b4cb944723f84" + }, + { + "case": "css", + "lane": "candidate", + "round": 2, + "wall_seconds": 12.112, + "cpu_seconds": 11.727334, + "memory_peak_bytes": 1186643968, + "transport_proxy_tokens": 895430, + "fresh_inspect_ms": 2662.82945, + "repeat_inspect_ms": 2489.199032999999, + "reply_bytes": 37428, + "raw_guard_log_sha256": "9d27bfa9720f4291324aa6fc47e44ffdee8b6fbc63ea29a2c82d297d0ba1f0c6" + }, + { + "case": "css", + "lane": "baseline", + "round": 2, + "wall_seconds": 12.409, + "cpu_seconds": 12.038676, + "memory_peak_bytes": 1079783424, + "transport_proxy_tokens": 895430, + "fresh_inspect_ms": 2793.9973870000003, + "repeat_inspect_ms": 2541.021634999999, + "reply_bytes": 37426, + "raw_guard_log_sha256": "ddc43e4517a2ed634d5a969104b9051ac4e654485d3552f19d4c141a5834b205" + }, + { + "case": "small", + "lane": "baseline", + "round": 3, + "wall_seconds": 9.797, + "cpu_seconds": 9.449418, + "memory_peak_bytes": 822263808, + "transport_proxy_tokens": 894936, + "fresh_inspect_ms": 1612.1414539999996, + "repeat_inspect_ms": 1833.922163000001, + "reply_bytes": 36180, + "raw_guard_log_sha256": "92aa46fe57caeae60cb3105b2ac370a8f8463afed4a7f666f46be885cad0ca4d" + }, + { + "case": "small", + "lane": "candidate", + "round": 3, + "wall_seconds": 10.192, + "cpu_seconds": 9.868713, + "memory_peak_bytes": 829575168, + "transport_proxy_tokens": 894936, + "fresh_inspect_ms": 1629.0064629999997, + "repeat_inspect_ms": 1962.5034050000004, + "reply_bytes": 36182, + "raw_guard_log_sha256": "0451dacd244ced0ac56ceb0f73b39e1b86fa1517a0ec88391a3ab37d45b9bd20" + }, + { + "case": "sqlite", + "lane": "baseline", + "round": 3, + "wall_seconds": 13.714, + "cpu_seconds": 13.319143, + "memory_peak_bytes": 1196531712, + "transport_proxy_tokens": 895044, + "fresh_inspect_ms": 3228.7134460000007, + "repeat_inspect_ms": 3397.271211000001, + "reply_bytes": 37248, + "raw_guard_log_sha256": "d3f7e1e572a4c3251ca479a3c018d2f024b80103e443a8daeb125bd896165d2a" + }, + { + "case": "sqlite", + "lane": "candidate", + "round": 3, + "wall_seconds": 12.709, + "cpu_seconds": 12.349818, + "memory_peak_bytes": 1205280768, + "transport_proxy_tokens": 895044, + "fresh_inspect_ms": 3071.7333470000003, + "repeat_inspect_ms": 3024.3819120000007, + "reply_bytes": 37250, + "raw_guard_log_sha256": "57b405e4a4e8a9acbb4d9e4bb4663d931ad7ffec52a2b6a2969a7362287db892" + }, + { + "case": "css", + "lane": "baseline", + "round": 3, + "wall_seconds": 12.315, + "cpu_seconds": 11.921752, + "memory_peak_bytes": 1081831424, + "transport_proxy_tokens": 895430, + "fresh_inspect_ms": 2798.227023, + "repeat_inspect_ms": 2563.7297770000005, + "reply_bytes": 37426, + "raw_guard_log_sha256": "2f6fca011f6718111b455d8104121ff6880131f58e0a82df069efef9445f5c07" + }, + { + "case": "css", + "lane": "candidate", + "round": 3, + "wall_seconds": 11.901, + "cpu_seconds": 11.54646, + "memory_peak_bytes": 1123659776, + "transport_proxy_tokens": 895430, + "fresh_inspect_ms": 2600.1125399999996, + "repeat_inspect_ms": 2398.082381, + "reply_bytes": 37428, + "raw_guard_log_sha256": "180726dbb90af48c442109f75ccc50bf243ebcc8392908f9322eb91100a5f179" + } + ], + "parser_samples": [ + { + "case": "small.decoded", + "lane": "baseline", + "round": 1, + "elapsed_ms": [ + 2.9900399999999934, 1.9184900000000056, 0.3164599999999922, + 0.622399999999999, 0.6818110000000104, 1.7410099999999886 + ], + "result_sha256": "beb5c7e4e8da08e3c104753b768c8ca40559d517dba80c483253991721b39c7f", + "wall_seconds": 0.422, + "cpu_seconds": 0.169929, + "memory_peak_bytes": 28078080 + }, + { + "case": "small.decoded", + "lane": "candidate", + "round": 1, + "elapsed_ms": [ + 3.8333299999999895, 1.5887500000000045, 0.4467700000000008, + 0.3875499999999903, 0.3238100000000088, 0.21590999999999383 + ], + "result_sha256": "beb5c7e4e8da08e3c104753b768c8ca40559d517dba80c483253991721b39c7f", + "wall_seconds": 0.323, + "cpu_seconds": 0.154998, + "memory_peak_bytes": 27508736 + }, + { + "case": "sqlite", + "lane": "baseline", + "round": 1, + "elapsed_ms": [ + 248.049268, 224.29203599999994, 176.49019599999997, 155.392424, + 162.08057399999996, 172.70000600000003 + ], + "result_sha256": "74eb7108ddd459dfa7bfb7a1fe653c8862eb43acd89d91ae033e7ac9158587ce", + "wall_seconds": 1.532, + "cpu_seconds": 1.326985, + "memory_peak_bytes": 71315456 + }, + { + "case": "sqlite", + "lane": "candidate", + "round": 1, + "elapsed_ms": [ + 103.56140300000001, 103.344674, 56.359200999999985, 43.188242, + 44.79720200000003, 44.11639200000002 + ], + "result_sha256": "74eb7108ddd459dfa7bfb7a1fe653c8862eb43acd89d91ae033e7ac9158587ce", + "wall_seconds": 0.827, + "cpu_seconds": 0.598442, + "memory_peak_bytes": 71557120 + }, + { + "case": "tree-sitter-css", + "lane": "baseline", + "round": 1, + "elapsed_ms": [ + 133.568665, 122.39741400000003, 110.63244400000002, 111.27637299999998, + 111.761034, 111.34842400000002 + ], + "result_sha256": "b59e5bdc32939c215443da6f3ce48e61fe2bfd86ff75b7a85d3c10a93b6fae7e", + "wall_seconds": 1.127, + "cpu_seconds": 0.882723, + "memory_peak_bytes": 55656448 + }, + { + "case": "tree-sitter-css", + "lane": "candidate", + "round": 1, + "elapsed_ms": [ + 15.454230999999993, 15.1781, 17.94840099999999, 15.499510999999984, + 15.028279999999995, 2.9674699999999916 + ], + "result_sha256": "b59e5bdc32939c215443da6f3ce48e61fe2bfd86ff75b7a85d3c10a93b6fae7e", + "wall_seconds": 0.424, + "cpu_seconds": 0.255557, + "memory_peak_bytes": 54427648 + }, + { + "case": "small.decoded", + "lane": "candidate", + "round": 2, + "elapsed_ms": [ + 2.492229000000009, 1.6617090000000019, 0.4688189999999963, + 0.40376000000000545, 0.18850899999999626, 0.19684000000000879 + ], + "result_sha256": "beb5c7e4e8da08e3c104753b768c8ca40559d517dba80c483253991721b39c7f", + "wall_seconds": 0.324, + "cpu_seconds": 0.14932, + "memory_peak_bytes": 27750400 + }, + { + "case": "small.decoded", + "lane": "baseline", + "round": 2, + "elapsed_ms": [ + 3.0103999999999758, 2.4378910000000076, 0.30259000000000924, + 0.5248499999999865, 0.6185799999999801, 1.3529399999999896 + ], + "result_sha256": "beb5c7e4e8da08e3c104753b768c8ca40559d517dba80c483253991721b39c7f", + "wall_seconds": 0.428, + "cpu_seconds": 0.219561, + "memory_peak_bytes": 30007296 + }, + { + "case": "sqlite", + "lane": "candidate", + "round": 2, + "elapsed_ms": [ + 150.102906, 136.73758599999996, 85.33841300000006, 42.61153200000001, + 45.540752, 43.07188099999996 + ], + "result_sha256": "74eb7108ddd459dfa7bfb7a1fe653c8862eb43acd89d91ae033e7ac9158587ce", + "wall_seconds": 0.959, + "cpu_seconds": 0.738843, + "memory_peak_bytes": 68399104 + }, + { + "case": "sqlite", + "lane": "baseline", + "round": 2, + "elapsed_ms": [ + 236.412329, 219.61998900000003, 163.37286700000004, 152.177506, + 153.764226, 153.57296699999984 + ], + "result_sha256": "74eb7108ddd459dfa7bfb7a1fe653c8862eb43acd89d91ae033e7ac9158587ce", + "wall_seconds": 1.532, + "cpu_seconds": 1.317278, + "memory_peak_bytes": 70189056 + }, + { + "case": "tree-sitter-css", + "lane": "candidate", + "round": 2, + "elapsed_ms": [ + 15.074011000000013, 22.501490000000018, 23.890242, 9.095690999999988, + 25.85492100000002, 16.470331000000016 + ], + "result_sha256": "b59e5bdc32939c215443da6f3ce48e61fe2bfd86ff75b7a85d3c10a93b6fae7e", + "wall_seconds": 0.524, + "cpu_seconds": 0.297941, + "memory_peak_bytes": 53497856 + }, + { + "case": "tree-sitter-css", + "lane": "baseline", + "round": 2, + "elapsed_ms": [ + 135.69754600000002, 128.91107599999998, 115.180316, 115.23454600000002, + 117.18869600000005, 103.74374499999999 + ], + "result_sha256": "b59e5bdc32939c215443da6f3ce48e61fe2bfd86ff75b7a85d3c10a93b6fae7e", + "wall_seconds": 1.13, + "cpu_seconds": 0.916205, + "memory_peak_bytes": 57880576 + }, + { + "case": "small.decoded", + "lane": "baseline", + "round": 3, + "elapsed_ms": [ + 2.7721499999999963, 2.0805500000000023, 0.2946500000000043, + 0.4885200000000083, 0.37317999999999074, 1.052209999999988 + ], + "result_sha256": "beb5c7e4e8da08e3c104753b768c8ca40559d517dba80c483253991721b39c7f", + "wall_seconds": 0.321, + "cpu_seconds": 0.161694, + "memory_peak_bytes": 28037120 + }, + { + "case": "small.decoded", + "lane": "candidate", + "round": 3, + "elapsed_ms": [ + 2.4845909999999947, 1.5559899999999942, 1.9349599999999896, + 0.6082500000000124, 0.23478000000000065, 0.22989100000000917 + ], + "result_sha256": "beb5c7e4e8da08e3c104753b768c8ca40559d517dba80c483253991721b39c7f", + "wall_seconds": 0.321, + "cpu_seconds": 0.154815, + "memory_peak_bytes": 27426816 + }, + { + "case": "sqlite", + "lane": "baseline", + "round": 3, + "elapsed_ms": [ + 219.92179099999998, 204.47974999999997, 159.57980799999996, + 152.3218579999999, 150.58105799999998, 149.3406970000001 + ], + "result_sha256": "74eb7108ddd459dfa7bfb7a1fe653c8862eb43acd89d91ae033e7ac9158587ce", + "wall_seconds": 1.428, + "cpu_seconds": 1.22534, + "memory_peak_bytes": 71987200 + }, + { + "case": "sqlite", + "lane": "candidate", + "round": 3, + "elapsed_ms": [ + 103.150995, 103.714395, 58.592934000000014, 47.94820199999998, + 47.04763299999996, 47.239012 + ], + "result_sha256": "74eb7108ddd459dfa7bfb7a1fe653c8862eb43acd89d91ae033e7ac9158587ce", + "wall_seconds": 0.828, + "cpu_seconds": 0.60423, + "memory_peak_bytes": 72187904 + }, + { + "case": "tree-sitter-css", + "lane": "baseline", + "round": 3, + "elapsed_ms": [ + 126.407417, 121.22317700000002, 109.654766, 108.04800599999999, + 109.459836, 100.50628600000005 + ], + "result_sha256": "b59e5bdc32939c215443da6f3ce48e61fe2bfd86ff75b7a85d3c10a93b6fae7e", + "wall_seconds": 1.029, + "cpu_seconds": 0.856826, + "memory_peak_bytes": 58101760 + }, + { + "case": "tree-sitter-css", + "lane": "candidate", + "round": 3, + "elapsed_ms": [ + 14.889070999999987, 15.635390999999998, 17.013621, 15.000391000000008, + 17.56236100000001, 2.893010000000004 + ], + "result_sha256": "b59e5bdc32939c215443da6f3ce48e61fe2bfd86ff75b7a85d3c10a93b6fae7e", + "wall_seconds": 0.422, + "cpu_seconds": 0.269288, + "memory_peak_bytes": 57966592 + } + ], + "limits": { + "cpu_affinity": [15], + "cpu_quota_percent": 100, + "hard_memory_bytes": 4294967296, + "early_stop_bytes": 3221225472, + "swap_bytes": 0, + "task_ceiling": 192 + }, + "limitations": [ + "Fresh server sessions; OS file caches warm/unflushed. Parser samples use one fresh and five repeated in-memory parses per process.", + "SDK transport proxy counts one complete catalog plus five named request/reply envelopes. Initialization/framing and actual model usage are unmeasured. No token saving is claimed.", + "Memory is kernel process-family memory.peak including file cache; final counters after transient scope removal remain unknown.", + "First string-only 18-task development cohort, one invalid-source-kind harness attempt, and the cache-hit build with unverified payload cleanup are preserved outside Git and excluded from final benefit claims.", + "Independent native WebAssembly.Module import/export oracle compiles without instantiating or executing target code. It does not prove every typed signature or unknown future proposal." + ], + "resource_qualification": { + "completed_mcp_tasks": 18, + "completed_parser_processes": 18, + "all_payload_exit_codes_zero": true, + "all_payload_cleanup_verified": true, + "all_payload_counters_verified": true, + "all_memory_high_hard_oom_and_task_limit_events_zero": true, + "final_whole_scope_counters": "unknown" + }, + "call_sequence": [ + "inspect_wasm_artifact(summary):fresh", + "inspect_analysis_view(exports,first4)", + "inspect_wasm_artifact(summary):repeat", + "inspect_analysis_view(exports,first4)", + "export_evidence_bundle" + ], + "transport_proxy_scope": "One complete SDK catalog and all five named request/reply envelopes, compact UTF-8 JSON encoded with cl100k_base; not actual model input." +} diff --git a/experiments/analysis-efficiency/wasm-string-framing.md b/experiments/analysis-efficiency/wasm-string-framing.md new file mode 100644 index 0000000000..e4d3d4522e --- /dev/null +++ b/experiments/analysis-efficiency/wasm-string-framing.md @@ -0,0 +1,107 @@ +# WAT framing costs on shipped WASM + +[Issue #5760](https://github.com/morluto/rea/issues/5760) follows the current +WASM summary workflow; no new tool, selector, cache or default is needed. +The [curated measurements](wasm-string-framing.json) pin source, runtime, +artifact, producer and answer fingerprints, including every final sample. + +## Owner and preserved behavior + +WABT's WAT framing scanned instruction text character by character. Interface +tokenization also decoded quoted bytes merely to obtain an end position. +`WabtString` now owns a shared quoted-string boundary operation, preserving +escaped-quote/backslash parity. The top-level framer searches for structural +ASCII delimiters and skips string payloads. Name and custom-payload consumers +still decode their bytes; unsupported typed signatures remain unknown. +Nested/line comments, exact source forms and full retained WAT remain available. +Data forms were already excluded from interface parsing; this does not add a +new data filter or reinterpret raw WASM outside WABT. + +Baseline source is `d0067ba34a39`; candidate code is +`9776d78f6c799cba8d8cf5c9b3139a86449eaebd`, based on latest main +`335c042d166b` (the intervening main change is testing documentation). +Linux x64, Node 24.18.0/npm 11.16.0, native WABT 1.0.42. Three alternating +baseline/candidate, candidate/baseline, baseline/candidate pairs ran serially. +Each task starts a fresh production stdio MCP server, discovers all 212 tools, +inspects the selected artifact twice with existing summary delivery, reads the +first four exact exports after each inspection, exports complete Evidence and +closes. OS file caches are warm/unflushed; no cold-disk claim is made. + +The public SQLite Wasm 3.53.4-build1 and VS Code tree-sitter-wasm 0.3.1 CSS +components match the pinned [corpus](corpus.json) archive and member digests. +An 80-byte source-assembled module supplies the small control, with an escaped +import name. Native `WebAssembly.Module` import/export metadata independently +checks all returned names/kinds without instantiating or executing target code. +A separate native `wasm2wat --enable-all` output checks exact full WAT and its +SHA-256. Every before/after original Evidence ID, normalized report digest, +source byte digest, selected export and catalog digest agrees. This does not +independently prove every typed signature or future WASM proposal. + +## Completed-task costs + +Medians of three complete tasks per side; kernel process-family memory includes +file cache. Wall time includes startup, discovery, all five investigation RPCs, +client validation, independent checks, export and transport cleanup. + +| Artifact | Wall seconds, before → after | CPU seconds, before → after | Peak MiB, before → after | +| ------------- | ---------------------------: | --------------------------: | -----------------------: | +| Small control | 10.192 → 10.192 | 9.863 → 9.869 | 785.3 → 791.1 | +| SQLite | 13.714 → 13.311 | 13.319 → 12.957 | 985.1 → 1149.4 | +| CSS grammar | 12.409 → 12.112 | 12.039 → 11.727 | 1031.7 → 1071.6 | + +Whole-task median wall reductions are about 2.9%/2.4% for SQLite/CSS; small-task +time is unchanged. Three pairs are descriptive observations, not statistical +or cross-provider guarantees. Memory does not improve: SQLite's median is +higher, with overlapping 970–1141 versus 982–1156 MiB ranges. No heap-retention +cause or memory-saving claim is established. + +Separate fresh-process parser cohorts use one first parse and five repeated +in-memory parses, checking the complete parsed-result digest after each parse. +Median first/warm parse milliseconds: SQLite 236.41/153.76 → 103.56/45.54; +CSS 133.57/111.35 → 15.07/15.64; small 2.99/0.62 → 2.49/0.40. These phase +improvements do not include native WABT production or complete task costs. + +The unchanged catalog and all five named request/reply envelopes cost median +894,936 / 895,044 / 895,430 `cl100k_base` tokens for small/SQLite/CSS in both +implementations. This is a single serialized SDK transport proxy using +tiktoken 0.12.0; actual model input/usage, repeated model namespaces and protocol +initialization/framing are unmeasured. No token saving is claimed. + +All 18 final MCP tasks and 18 parser processes completed under one CPU, +4 GiB hard/3 GiB early-stop memory, no swap and 192 tasks, with payload-exit +counters and cleanup verified and no limit/OOM events. Final whole-scope counters +after transient scope removal remain unknown. Raw binaries, exports, +transcripts and logs stay outside Git. The earlier string-only 18-task +cohort, an invalid source-kind harness attempt and a cache-hit build with +unverified payload cleanup are retained and excluded from final benefit claims. + +## Reproduction + +Build/freeze each runtime with the pinned toolchain; keep the selected artifact, +WABT directory and independently produced WAT paths identical across lanes: + +```sh +python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \ + node experiments/analysis-efficiency/wasm-string-framing.mjs \ + --runtime /absolute/runtime --artifact /absolute/input.wasm \ + --wat /absolute/native-output.wat --tools /absolute/wabt/bin \ + --output /absolute/new-output-directory +``` + +For parser-only cohorts, import `dist/wasm/WabtWat.js` from each frozen runtime, +load the same native WAT once, and call `parseWabtWat(text)` six times. Measure +each call with `performance.now()`; outside that timer, hash +`JSON.stringify(result)` and require the same digest for every call and lane. +Keep each artifact in a separate fresh guarded process. + +Source verification passed 76 tests across nine files, including malformed +strings, slash parity, comments, invalid names, capacity/partial facts, +cancellation, cleanup retry and shared application behavior. `npm run check` +passed typecheck, lint, formatting, Knip, metadata and architecture guards. +Four compiled CLI/MCP/acceptance files passed another 14 tests. The native WABT +artifact verifier passed 51 CLI/MCP cases; the native interface verifier passed +nine cases, including all admitted kinds, escaped/BOM names, padded LEB128, +signature changes, duplicate imports and unsupported reference semantics. +The build used a separate 6 GiB hard/5 GiB early-stop guard; the full check used +8/7 GiB, peaking at 5.37 GiB without resource events. These are build/check +budgets; runtime tasks retain the smaller bounds above. diff --git a/experiments/analysis-efficiency/wasm-string-framing.mjs b/experiments/analysis-efficiency/wasm-string-framing.mjs new file mode 100644 index 0000000000..67cf0c634e --- /dev/null +++ b/experiments/analysis-efficiency/wasm-string-framing.mjs @@ -0,0 +1,184 @@ +/** Compare unchanged public WASM tasks; raw inputs, exports and transcripts stay outside Git. */ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { createWriteStream } from "node:fs"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { tmpdir } from "node:os"; +import { finished } from "node:stream/promises"; +import { parseArgs } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { createRetainedQueryCaller } from "./retained-query-calls.mjs"; +import { finalizeRetainedQueryReceipt } from "./retained-query-finalization.mjs"; + +const { values } = parseArgs({ + options: Object.fromEntries( + ["runtime", "artifact", "wat", "tools", "output"].map((name) => [ + name, + { type: "string" }, + ]), + ), +}); +for (const name of ["runtime", "artifact", "wat", "tools", "output"]) + assert.ok( + values[name] && values[name] === resolve(values[name]), + `${name} must be absolute`, + ); +const hash = (value) => createHash("sha256").update(value).digest("hex"); +const bytes = await readFile(values.artifact); +const nativeWat = await readFile(values.wat, "utf8"); +// Native compilation validates the independent name inventory; no instance or target code runs. +const module = new WebAssembly.Module(bytes); +const nativeImports = WebAssembly.Module.imports(module); +const nativeExports = WebAssembly.Module.exports(module); +await mkdir(values.output, { mode: 0o700 }); +const account = await mkdtemp(join(tmpdir(), "rea-wat-framing-")); +const client = new Client({ name: "wasm-string-framing", version: "1" }); +const transport = new StdioClientTransport({ + command: process.execPath, + args: [join(values.runtime, "scripts/rea.mjs"), "mcp"], + env: { + HOME: account, + PATH: process.env.PATH, + NODE_OPTIONS: "--max-old-space-size=2048", + UV_THREADPOOL_SIZE: "1", + REA_LOG_LEVEL: "silent", + REA_WABT_BIN_DIRECTORY: values.tools, + TMPDIR: account, + }, + stderr: "pipe", +}); +const serverLog = createWriteStream(join(values.output, "server.log")); +const serverLogFinished = finished(serverLog); +serverLogFinished.catch(() => {}); +transport.stderr?.pipe(serverLog); +const report = { + artifact_sha256: hash(bytes), + artifact_bytes: bytes.length, + calls: [], + answers: [], +}; +const transcript = []; +const call = createRetainedQueryCaller(client, (receipt, event) => { + report.calls.push(receipt); + transcript.push(event); +}); +const observe = async (name, input, phase) => { + const reply = await call(name, input, phase); + assert.notEqual(reply.isError, true, JSON.stringify(reply)); + assert.ok(reply.structuredContent); + return reply.structuredContent; +}; +const started = performance.now(); +try { + await client.connect(transport); + const catalog = await client.listTools(); + assert.equal(catalog.nextCursor, undefined); + await writeFile(join(values.output, "catalog.json"), JSON.stringify(catalog)); + report.catalog_sha256 = hash(JSON.stringify(catalog)); + report.catalog_tools = catalog.tools.length; + const definitions = new Map( + catalog.tools.map((definition) => [definition.name, definition]), + ); + const originalCall = client.callTool.bind(client); + client.callTool = (input, options) => + originalCall(input, { + ...options, + toolDefinition: definitions.get(input.name), + }); + const observations = []; + for (const phase of ["fresh", "repeat"]) { + const summary = await observe( + "inspect_wasm_artifact", + { path: values.artifact, detail: "summary" }, + phase, + ); + const parentId = summary.normalized_result.parent_evidence_id; + const exports = await observe( + "inspect_analysis_view", + { + source: { kind: "retained-evidence", evidence_id: parentId }, + view: { kind: "wasm", facet: "exports", offset: 0, limit: 4 }, + }, + `${phase}:exports`, + ); + assert.equal(exports.normalized_result.parent_evidence_id, parentId); + observations.push({ summary, exports, parentId }); + } + await observe( + "export_evidence_bundle", + { path: join(values.output, "evidence.json") }, + "export", + ); + const bundle = JSON.parse( + await readFile(join(values.output, "evidence.json"), "utf8"), + ); + const kind = (value) => (value === "func" ? "function" : value); + for (const { summary, exports, parentId } of observations) { + const parent = bundle.records.find( + (record) => record.evidence_id === parentId, + ); + assert.ok(parent); + assert.equal(parent.subject.digest.sha256, report.artifact_sha256); + const facts = parent.normalized_result; + assert.equal(facts.validation, "valid"); + assert.equal(facts.wat.text, nativeWat); + assert.equal(facts.wat.sha256, hash(nativeWat)); + assert.equal(facts.interface.state, "complete"); + assert.deepEqual( + facts.interface.entries + .filter((entry) => entry.direction === "import") + .map((entry) => ({ + module: entry.module, + name: entry.name, + kind: kind(entry.kind), + })), + nativeImports, + ); + assert.deepEqual( + facts.interface.entries + .filter((entry) => entry.direction === "export") + .map((entry) => ({ name: entry.name, kind: kind(entry.kind) })), + nativeExports, + ); + assert.deepEqual(exports.normalized_result.item, facts.exports.slice(0, 4)); + assert.equal( + summary.normalized_result.summary.available_counts.imports, + nativeImports.length, + ); + assert.equal( + summary.normalized_result.summary.available_counts.exports, + nativeExports.length, + ); + assert.equal(facts.runtime_execution, "not-performed"); + assert.equal(facts.network_fetch, "not-performed"); + report.answers.push({ + parent_evidence_id: parentId, + normalized_sha256: hash(JSON.stringify(facts)), + exports: exports.normalized_result.item, + }); + } + assert.deepEqual(report.answers[0], report.answers[1]); + assert.equal(hash(await readFile(values.artifact)), report.artifact_sha256); + report.status = "passed"; +} catch (cause) { + report.status = "failed"; + report.failure = String(cause?.stack ?? cause); + process.exitCode = 1; +} finally { + report.elapsed_ms_before_cleanup = performance.now() - started; + if ( + !(await finalizeRetainedQueryReceipt({ + report, + client, + transport, + account, + serverLog, + serverLogFinished, + output: values.output, + transcript, + })) + ) + process.exitCode = 1; +} diff --git a/knip.json b/knip.json index c97d15a675..550b351717 100644 --- a/knip.json +++ b/knip.json @@ -18,7 +18,7 @@ "vitepress": { "entry": ["docs/.vitepress/config.ts"] }, - "ignoreBinaries": ["mkfifo"], + "ignoreBinaries": ["^g\\+\\+$", "mkfifo"], "ignoreDependencies": [ "@stryker-mutator/command-runner", "electron", diff --git a/native/windows/README.md b/native/windows/README.md index 28f0f9344c..0ddf7a2366 100644 --- a/native/windows/README.md +++ b/native/windows/README.md @@ -111,8 +111,21 @@ automatically. Alternatively, build it using the Linux/MinGW lane above. - Cleanup walks locked directory objects, removes files by handle, and unlinks reparse entries without traversing their targets. Failed removal retains the root and ancestor leases for retry; a partially removed runtime accepts only - cleanup, and its registry entry remains until removal succeeds. The runtime - owner retries only native close failures with Win32 sharing/lock codes 32/33, + cleanup, and its registry entry remains until removal succeeds. `CloseHandle` + is checked. A value that is still open, or whose liveness cannot be read, + stays with its runtime, process, or file owner and is not passed to + `CloseHandle` again. A process pipe left in that state is treated as ended + and is not read or written again. An open liveness probe does not prove that the object + still belongs to that owner. A value that is no longer in the process handle table is + dropped and is not closed again. An incomplete snapshot whose writer cannot + be closed stays on the runtime instead of being forgotten; source handles + that remain open after a published snapshot are closed with that runtime. An + admitted file whose close fails stays on the private runtime and is closed + again before that runtime's tree walk. After the root directory handle is + released, a later close resumes with the remaining parent handles and does + not walk from that released root. A process whose job close succeeded and + whose later handle close failed is retried with process close alone. The + runtime owner retries only native close failures with Win32 sharing/lock codes 32/33, for at most ten attempts with a 100 ms delay between attempts. Other native errors are returned immediately; exhausting the retry budget preserves the runtime owner for a later explicit close. These cleanup retries do not use an diff --git a/native/windows/src/authority.hpp b/native/windows/src/authority.hpp index 0c10dd113e..b4b270bd34 100644 --- a/native/windows/src/authority.hpp +++ b/native/windows/src/authority.hpp @@ -7,11 +7,14 @@ #include #include +#include "handle_close.hpp" + #include #include #include #include #include +#include #include #include #include @@ -54,24 +57,134 @@ inline void require(bool condition, const char* constraint, const std::wstring& } class Handle { - HANDLE value_ = INVALID_HANDLE_VALUE; -public: - Handle() = default; - explicit Handle(HANDLE value) : value_(value) {} - ~Handle() { reset(); } + OwnedHandle owned_; + + public: + Handle() noexcept : owned_(INVALID_HANDLE_VALUE) {} + explicit Handle(HANDLE value) noexcept : owned_(INVALID_HANDLE_VALUE) { adopt(value, false); } + ~Handle() noexcept; Handle(const Handle&) = delete; Handle& operator=(const Handle&) = delete; - Handle(Handle&& other) noexcept : value_(other.release()) {} - Handle& operator=(Handle&& other) noexcept { reset(other.release()); return *this; } - HANDLE get() const { return value_; } - bool valid() const { return value_ != INVALID_HANDLE_VALUE && value_ != nullptr; } - HANDLE release() { return std::exchange(value_, INVALID_HANDLE_VALUE); } - void reset(HANDLE next = INVALID_HANDLE_VALUE) { - if (valid()) CloseHandle(value_); - value_ = next; - } + Handle(Handle&& other) noexcept : owned_(std::move(other.owned_)) {} + Handle& operator=(Handle&& other) noexcept; + HANDLE get() const noexcept { return owned_.get(); } + bool valid() const noexcept { return owned_.valid(); } + bool fullyReleased() const noexcept { return owned_.fullyReleased(); } + // A failed close leaves the numeric value blocked. It must not be used for + // I/O or closed again; only a not-in-table probe may drop it. + bool closeBlocked() const noexcept { return owned_.closeBlocked(); } + void adopt(HANDLE value, bool blockClose) noexcept; + HandleCloseDecision release() noexcept; + void reset(HANDLE next = INVALID_HANDLE_VALUE) noexcept; }; +struct RetainedNativeHandle { + HANDLE value; + bool blockClose; +}; + +struct RetainedNativeHandles { + std::mutex mutex; + std::vector handles; +}; + +// Leaked for the process lifetime. Handle destructors can run during exit, +// after an ordinary function-local static would already have been destroyed. +inline RetainedNativeHandles& retainedNativeHandles() { + static auto* const retained = new RetainedNativeHandles(); + return *retained; +} + +inline HANDLE normalizedNativeHandle(HANDLE value) noexcept { + return value == nullptr ? INVALID_HANDLE_VALUE : value; +} + +inline CloseAttempt closeNativeHandle(HANDLE value) noexcept { + if (CloseHandle(value) != FALSE) return {true, 0}; + return {false, GetLastError()}; +} + +inline LivenessProbe probeNativeHandle(HANDLE value) noexcept { + DWORD flags = 0; + if (GetHandleInformation(value, &flags) != FALSE) return {HandleLiveness::Open, 0}; + const DWORD error = GetLastError(); + if (error == ERROR_INVALID_HANDLE) return {HandleLiveness::NotInTable, error}; + return {HandleLiveness::Unavailable, error}; +} + +// Last owner for a value whose C++ handle is dying. A failed record keeps the +// kernel object until process exit instead of closing an unconfirmed number. +inline void retainNativeHandle(HANDLE value, bool blockClose) noexcept { + value = normalizedNativeHandle(value); + if (value == INVALID_HANDLE_VALUE) return; + try { + auto& retained = retainedNativeHandles(); + const std::lock_guard lock(retained.mutex); + retained.handles.push_back(RetainedNativeHandle{value, blockClose}); + } catch (...) { + // The number stays open until process exit. Closing it here could reuse it. + } +} + +inline void Handle::adopt(HANDLE value, bool blockClose) noexcept { + value = normalizedNativeHandle(value); + if (value == INVALID_HANDLE_VALUE) return; + if (!owned_.park(value, blockClose)) retainNativeHandle(value, blockClose); +} + +inline HandleCloseDecision Handle::release() noexcept { + return owned_.release(&closeNativeHandle, &probeNativeHandle); +} + +inline void Handle::reset(HANDLE next) noexcept { + owned_.reset(normalizedNativeHandle(next), &closeNativeHandle, &probeNativeHandle, + &retainNativeHandle); +} + +inline Handle::~Handle() noexcept { + owned_.abandon(&closeNativeHandle, &probeNativeHandle, &retainNativeHandle); +} + +inline Handle& Handle::operator=(Handle&& other) noexcept { + if (this == &other) return *this; + const auto incoming = other.owned_.take(); + owned_.release(&closeNativeHandle, &probeNativeHandle); + const auto keep = [&](HANDLE value, bool blockClose) noexcept { + if (value == INVALID_HANDLE_VALUE || value == nullptr) return; + if (!owned_.park(value, blockClose)) retainNativeHandle(value, blockClose); + }; + keep(incoming.value, incoming.blockValue); + keep(incoming.extra, incoming.blockExtra); + return *this; +} + +inline void requireReleased(Handle& handle, const char* constraint, + const std::wstring& path = L"") { + const HandleCloseDecision decision = handle.release(); + if (handle.fullyReleased()) return; + const DWORD code = decision.closeError != 0 ? decision.closeError + : decision.evidenceError != 0 ? decision.evidenceError + : ERROR_GEN_FAILURE; + throw Failure(constraint, path, code); +} + +// Closes values detached by a dying owner. Survivors stay in the retainer. +inline void retryRetainedNativeHandles() noexcept { + std::vector pending; + try { + auto& retained = retainedNativeHandles(); + const std::lock_guard lock(retained.mutex); + pending.swap(retained.handles); + } catch (...) { + return; + } + for (const RetainedNativeHandle& item : pending) { + // The destructor releases the value once and parks it again on failure. + Handle handle; + handle.adopt(item.value, item.blockClose); + } +} + enum class Kind { File, Runtime, Process }; struct Resource { const Kind kind; @@ -91,7 +204,13 @@ struct Runtime : Resource { std::vector parents; Handle directory; std::vector immutableFiles; + // Handles whose close stayed unconfirmed after snapshot or owner teardown. + // runtime close retries these before deleting the private tree. + std::vector unreleased; bool closing = false; + // The private tree and root directory handle have been released. Later + // closes resume with the parent handles and do not duplicate that root. + bool treeReleased = false; // Main-thread admission is single-flight; only the accepted copy owns the // atomic cancellation flag consumed by its async worker. ERROR_BUSY rejects // another copy before it can reset that flag or replace pending ownership. diff --git a/native/windows/src/filesystem.cc b/native/windows/src/filesystem.cc index f8b88aabc1..7690e22e30 100644 --- a/native/windows/src/filesystem.cc +++ b/native/windows/src/filesystem.cc @@ -482,14 +482,31 @@ static void dispose(HANDLE handle, const std::wstring& path) { require(SetFileInformationByHandle(handle, FileDispositionInfo, &deletion, sizeof(deletion)), "Delete owned object by handle failed", path); } +static void parkUnresolved(Runtime& root, Handle& handle) noexcept { + if (handle.fullyReleased()) return; + handle.release(); + if (handle.fullyReleased()) return; + try { + root.unreleased.push_back(std::move(handle)); + } catch (...) { + // The owning Handle remains live. Its destructor retains the same value. + } +} + +static void parkUnresolved(Runtime& root, std::vector& handles) noexcept { + for (auto& handle : handles) parkUnresolved(root, handle); + handles.clear(); +} + static void disposeSnapshot(Runtime& root, Handle& file, const std::wstring& path) { // Snapshot readers do not share DELETE access. Release the writer only for // failed-copy cleanup, with its parent pinned and its identity recorded. + // A failed close keeps this writer; opening a second handle would drop it. auto parent = runtimeDirectory(root, path.substr(0, path.find_last_of(L'\\'))); FILE_ID_INFO expected{}; require(GetFileInformationByHandleEx(file.get(), FileIdInfo, &expected, sizeof(expected)), "Read incomplete snapshot identity failed", path); - file.reset(); + requireReleased(file, "Close incomplete snapshot writer failed", path); auto cleanup = openRelative(parent->get(), path.substr(path.find_last_of(L'\\') + 1), DELETE | GENERIC_READ | FILE_WRITE_ATTRIBUTES, FILE_SHARE_READ, FILE_NON_DIRECTORY_FILE, path); @@ -505,12 +522,24 @@ static void disposeSnapshot(Runtime& root, Handle& file, const std::wstring& pat } void closeRuntime(Runtime& root) { + retryRetainedNativeHandles(); if (root.closed) return; root.closing = true; // Windows keeps the bearer descriptor and snapshot leases until this owner // cleanup. POSIX removes its consumed descriptor earlier. Abrupt owner death // closes process jobs but cannot run this walk, so private files can remain. + for (auto& handle : root.unreleased) + requireReleased(handle, "Close retained runtime handle failed", root.path); + root.unreleased.clear(); + // Writers share only FILE_SHARE_READ, which blocks a later delete open. + // Release them before the walk. A failed close stays on this owner and does + // not open a second delete handle. + for (auto& handle : root.immutableFiles) + requireReleased(handle, "Close retained runtime file failed", root.path); root.immutableFiles.clear(); + // Once the root directory handle is released, another walk would duplicate + // an invalid handle and never reach surviving parents. + if (!root.treeReleased) { // Enumerate and open children through the retained directory handles. Paths // below are diagnostic labels only: renames and ancestor junction replacement // cannot redirect cleanup into an unrelated tree. @@ -567,6 +596,7 @@ void closeRuntime(Runtime& root) { std::wstring name; if (!frame.next(name)) { dispose(frame.directory.get(), frame.path); + requireReleased(frame.directory, "Close runtime directory handle failed", frame.path); frames.pop_back(); continue; } @@ -578,10 +608,17 @@ void closeRuntime(Runtime& root) { require(GetFileInformationByHandleEx(child.get(), FileAttributeTagInfo, &info, sizeof(info)), "Read cleanup object tag failed", path); if ((info.FileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0 && (info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) == 0) frames.push_back(std::make_unique(std::move(child), path)); - else dispose(child.get(), path); + else { + dispose(child.get(), path); + requireReleased(child, "Close runtime child handle failed", path); + } } // Failed walks retain the root's identity leases for a later cleanup retry. - root.directory.reset(); + requireReleased(root.directory, "Close runtime root handle failed", root.path); + root.treeReleased = true; + } + for (auto& parent : root.parents) + requireReleased(parent, "Close runtime parent handle failed", root.path); root.parents.clear(); root.closed = true; } @@ -657,12 +694,17 @@ static void completeSnapshot(napi_env env, napi_status status, void* pointer) { set(env, result, "source", identity(env, work->source->get(), work->source->requestedPath)); set(env, result, "snapshot", identity(env, work->destination.get(), work->path)); work->root->immutableFiles.push_back(std::move(work->destination)); + // The digest is the byte outcome. Source handles that fail close stay on + // the runtime and are retried by runtime close; they do not erase the copy. + if (work->source != nullptr) parkUnresolved(*work->root, work->source->handles); check(napi_resolve_deferred(env, work->deferred, result)); } catch (const Failure& failure) { reject(failure); } catch (const std::exception& failure) { reject(Failure(failure.what(), work->path, ERROR_GEN_FAILURE)); } + if (work->source != nullptr) parkUnresolved(*work->root, work->source->handles); + parkUnresolved(*work->root, work->destination); napi_delete_reference(env, work->rootReference); napi_delete_async_work(env, work->work); } @@ -707,8 +749,14 @@ napi_value filesystemCall(napi_env env, const std::wstring& operation, return result; } if (operation == L"close") { + retryRetainedNativeHandles(); auto& file = static_cast(resource(env, args[0], Kind::File)); - file.handles.clear(); file.closed = true; + // Leave a failed handle on this File. The private runtime keeps that + // wrapper and retries this close before runtime_close. + for (auto& handle : file.handles) + requireReleased(handle, "Close admitted file handle failed", file.path); + file.handles.clear(); + file.closed = true; return null(env); } if (operation == L"read") { diff --git a/native/windows/src/handle_close.hpp b/native/windows/src/handle_close.hpp new file mode 100644 index 0000000000..d944c37683 --- /dev/null +++ b/native/windows/src/handle_close.hpp @@ -0,0 +1,206 @@ +#pragma once + +#include +#include + +namespace rea { + +// A failed CloseHandle and a later liveness probe are not one atomic fact. +// Another thread can reuse the numeric value before GetHandleInformation +// runs, so an Open result does not prove the probed object is still ours. +// After a failed close, drop the value only when the probe says it is gone. +// Otherwise keep it and do not call CloseHandle again. + +enum class HandleLiveness { Open, NotInTable, Unavailable }; + +enum class HandleCloseDisposition { + Released, + TerminalInvalid, + Unconfirmed, +}; + +struct CloseAttempt { + bool succeeded = false; + std::uint32_t error = 0; +}; + +struct LivenessProbe { + HandleLiveness liveness = HandleLiveness::Unavailable; + std::uint32_t error = 0; +}; + +struct HandleCloseDecision { + HandleCloseDisposition disposition = HandleCloseDisposition::Released; + bool clearValue = false; + bool retryClose = false; + std::uint32_t closeError = 0; + std::uint32_t evidenceError = 0; +}; + +inline HandleCloseDecision decideOwnedHandleClose(bool closeSucceeded, + std::uint32_t closeError, + LivenessProbe evidence) { + if (closeSucceeded) + return {HandleCloseDisposition::Released, true, false, 0, 0}; + switch (evidence.liveness) { + case HandleLiveness::Open: + case HandleLiveness::Unavailable: + return {HandleCloseDisposition::Unconfirmed, false, false, closeError, + evidence.error}; + case HandleLiveness::NotInTable: + return {HandleCloseDisposition::TerminalInvalid, true, false, closeError, + evidence.error}; + } + return {HandleCloseDisposition::Unconfirmed, false, false, closeError, + evidence.error}; +} + +// One current value plus at most one parked replacement. Close and liveness +// are injected so this gate can be tested without a kernel handle table. +template +class OwnedHandle { + public: + explicit OwnedHandle(T invalid) noexcept + : invalid_(invalid), value_(invalid), extra_(invalid) {} + + OwnedHandle(const OwnedHandle&) = delete; + OwnedHandle& operator=(const OwnedHandle&) = delete; + + OwnedHandle(OwnedHandle&& other) noexcept + : invalid_(other.invalid_), + value_(std::exchange(other.value_, other.invalid_)), + extra_(std::exchange(other.extra_, other.invalid_)), + blockValue_(std::exchange(other.blockValue_, false)), + blockExtra_(std::exchange(other.blockExtra_, false)) {} + + OwnedHandle& operator=(OwnedHandle&&) = delete; + + T get() const noexcept { return value_; } + T invalid() const noexcept { return invalid_; } + bool valid() const noexcept { return value_ != invalid_; } + bool fullyReleased() const noexcept { + return value_ == invalid_ && extra_ == invalid_; + } + bool closeBlocked() const noexcept { return blockValue_; } + + // Install a value into an empty slot. A live owner uses reset. + bool park(T value, bool blockClose) noexcept { + if (value == invalid_) return true; + if (value_ == invalid_) { + value_ = value; + blockValue_ = blockClose; + return true; + } + if (extra_ == invalid_) { + extra_ = value; + blockExtra_ = blockClose; + return true; + } + return false; + } + + struct Slots { + T value; + T extra; + bool blockValue; + bool blockExtra; + }; + + Slots take() noexcept { + Slots slots{value_, extra_, blockValue_, blockExtra_}; + value_ = invalid_; + extra_ = invalid_; + blockValue_ = false; + blockExtra_ = false; + return slots; + } + + template + HandleCloseDecision release(Close&& close, Probe&& probe) noexcept { + HandleCloseDecision last{HandleCloseDisposition::Released, true, false, 0, 0}; + for (;;) { + if (value_ == invalid_) { + if (extra_ == invalid_) return last; + value_ = extra_; + blockValue_ = blockExtra_; + extra_ = invalid_; + blockExtra_ = false; + } + last = finalizeSlot(value_, blockValue_, close, probe); + if (value_ != invalid_) return last; + } + } + + // Close the current owner. On an unconfirmed close, keep it and park `next` + // instead of overwriting the numeric value. `sink` receives a value that + // does not fit in the parked slot; it must retain that exact value. + template + void reset(T next, Close&& close, Probe&& probe, Sink&& sink) noexcept { + release(close, probe); + if (!fullyReleased()) { + if (next != invalid_ && next != value_ && next != extra_ && !park(next, false)) + sink(next, false); + return; + } + value_ = next; + blockValue_ = false; + } + + // Last-resort detach. Release once, then hand any surviving values to `sink` + // without a second close of a value whose liveness is not Open. + template + void abandon(Close&& close, Probe&& probe, Sink&& sink) noexcept { + release(close, probe); + if (value_ != invalid_) sink(value_, blockValue_); + if (extra_ != invalid_) sink(extra_, blockExtra_); + value_ = invalid_; + extra_ = invalid_; + blockValue_ = false; + blockExtra_ = false; + } + + private: + T invalid_; + T value_; + T extra_; + bool blockValue_ = false; + bool blockExtra_ = false; + + template + HandleCloseDecision finalizeSlot(T& slot, bool& block, Close&& close, + Probe&& probe) noexcept { + if (slot == invalid_) + return {HandleCloseDisposition::Released, true, false, 0, 0}; + if (block) { + const LivenessProbe evidence = probe(slot); + if (evidence.liveness == HandleLiveness::NotInTable) { + slot = invalid_; + block = false; + return {HandleCloseDisposition::TerminalInvalid, true, false, 0, + evidence.error}; + } + // Open or unreadable: the number stays close-blocked. Closing it could + // release an object that reused the value. + return {HandleCloseDisposition::Unconfirmed, false, false, 0, + evidence.error}; + } + const CloseAttempt attempt = close(slot); + if (attempt.succeeded) { + slot = invalid_; + block = false; + return {HandleCloseDisposition::Released, true, false, 0, 0}; + } + const LivenessProbe evidence = probe(slot); + const HandleCloseDecision decision = + decideOwnedHandleClose(false, attempt.error, evidence); + if (decision.clearValue) { + slot = invalid_; + block = false; + } else { + block = !decision.retryClose; + } + return decision; + } +}; + +} // namespace rea diff --git a/native/windows/src/handle_close_test.cc b/native/windows/src/handle_close_test.cc new file mode 100644 index 0000000000..72aa7db4f8 --- /dev/null +++ b/native/windows/src/handle_close_test.cc @@ -0,0 +1,254 @@ +#include "handle_close.hpp" + +#include +#include +#include +#include +#include + +namespace { + +int failures = 0; + +void expect(bool condition, const char* description) { + if (condition) return; + ++failures; + std::cerr << "failed: " << description << '\n'; +} + +rea::CloseAttempt fail(std::uint32_t error) { + return {false, error}; +} + +rea::LivenessProbe live(rea::HandleLiveness liveness, std::uint32_t error = 0) { + return {liveness, error}; +} + +void testDecisionMatrix() { + const auto released = rea::decideOwnedHandleClose(true, 5, live(rea::HandleLiveness::Open, 9)); + expect(released.disposition == rea::HandleCloseDisposition::Released && released.clearValue && + !released.retryClose && released.closeError == 0, + "a successful close clears the value and does not retry"); + + const auto still = rea::decideOwnedHandleClose(false, 6, live(rea::HandleLiveness::Open)); + expect(still.disposition == rea::HandleCloseDisposition::Unconfirmed && !still.clearValue && + !still.retryClose && still.closeError == 6, + "an open probe after a failed close does not authorize another close"); + + const auto gone = rea::decideOwnedHandleClose(false, 6, live(rea::HandleLiveness::NotInTable, 6)); + expect(gone.disposition == rea::HandleCloseDisposition::TerminalInvalid && gone.clearValue && + !gone.retryClose && gone.evidenceError == 6, + "a value missing from the handle table is dropped and never closed again"); + + const auto unknown = + rea::decideOwnedHandleClose(false, 5, live(rea::HandleLiveness::Unavailable, 12)); + expect(unknown.disposition == rea::HandleCloseDisposition::Unconfirmed && !unknown.clearValue && + !unknown.retryClose && unknown.closeError == 5 && unknown.evidenceError == 12, + "unavailable liveness keeps the value and blocks another close"); +} + +void testReleaseRetainsAndRetriesOnlyLiveValues() { + rea::OwnedHandle handle(-1); + expect(handle.park(4, false), "adopt the first value"); + int closes = 0; + int probes = 0; + const auto decision = handle.release( + [&](int) { + ++closes; + return fail(6); + }, + [&](int) { + ++probes; + return live(rea::HandleLiveness::Open); + }); + expect(decision.disposition == rea::HandleCloseDisposition::Unconfirmed && handle.get() == 4 && + closes == 1 && probes == 1 && handle.closeBlocked(), + "a failed close of a value that still looks open keeps that value blocked"); + + const auto retried = handle.release( + [&](int) { + ++closes; + return rea::CloseAttempt{true, 0}; + }, + [&](int) { + ++probes; + return live(rea::HandleLiveness::Open); + }); + expect(retried.disposition == rea::HandleCloseDisposition::Unconfirmed && handle.get() == 4 && + handle.closeBlocked() && closes == 1 && probes == 2, + "a later open probe does not close a value whose previous close failed"); +} + +void testUnconfirmedCloseDoesNotCallCloseAgain() { + rea::OwnedHandle handle(-1); + expect(handle.park(8, false), "adopt the unconfirmed value"); + int closes = 0; + int probes = 0; + handle.release( + [&](int) { + ++closes; + return fail(5); + }, + [&](int) { + ++probes; + return live(rea::HandleLiveness::Unavailable, 12); + }); + expect(handle.get() == 8 && handle.closeBlocked() && closes == 1 && probes == 1, + "the first failed close records unconfirmed liveness"); + + handle.release( + [&](int) { + ++closes; + return fail(5); + }, + [&](int) { + ++probes; + return live(rea::HandleLiveness::Unavailable, 12); + }); + expect(closes == 1 && probes == 2 && handle.get() == 8 && handle.closeBlocked(), + "unconfirmed liveness does not call close again"); + + handle.release( + [&](int) { + ++closes; + return rea::CloseAttempt{true, 0}; + }, + [&](int) { + ++probes; + return live(rea::HandleLiveness::Open); + }); + expect(handle.get() == 8 && handle.closeBlocked() && closes == 1 && probes == 3, + "a later open probe still does not close the unconfirmed value"); +} + +void testTerminalInvalidityDropsTheValue() { + rea::OwnedHandle handle(-1); + expect(handle.park(3, false), "adopt the invalid value"); + int closes = 0; + const auto decision = handle.release( + [&](int) { + ++closes; + return fail(6); + }, + [&](int) { return live(rea::HandleLiveness::NotInTable, 6); }); + expect(decision.disposition == rea::HandleCloseDisposition::TerminalInvalid && + handle.fullyReleased() && closes == 1, + "terminal invalidity clears the owner"); + handle.release( + [&](int) { + ++closes; + return fail(6); + }, + [&](int) { return live(rea::HandleLiveness::Open); }); + expect(closes == 1 && handle.fullyReleased(), + "a dropped value is not passed to close again"); +} + +void testBlockedProbeCanDiscoverTerminalInvalidityWithoutClosing() { + rea::OwnedHandle handle(-1); + expect(handle.park(9, true), "install a close-blocked value"); + int closes = 0; + const auto decision = handle.release( + [&](int) { + ++closes; + return fail(1); + }, + [&](int) { return live(rea::HandleLiveness::NotInTable, 6); }); + expect(decision.disposition == rea::HandleCloseDisposition::TerminalInvalid && + handle.fullyReleased() && closes == 0, + "a blocked value that is no longer in the table is not closed"); +} + +void testResetParksTheReplacementAndOverflowsTheThirdValue() { + rea::OwnedHandle handle(-1); + expect(handle.park(1, false), "adopt the current value"); + std::vector> sunk; + handle.reset( + 2, + [&](int) { return fail(6); }, + [&](int) { return live(rea::HandleLiveness::Open); }, + [&](int value, bool block) { sunk.emplace_back(value, block); }); + expect(handle.get() == 1 && sunk.empty(), "a failed close does not overwrite the live value"); + + int closes = 0; + handle.reset( + 3, + [&](int value) { + ++closes; + return value == 1 ? fail(6) : rea::CloseAttempt{true, 0}; + }, + [&](int) { return live(rea::HandleLiveness::Open); }, + [&](int value, bool block) { sunk.emplace_back(value, block); }); + expect(handle.get() == 1 && handle.closeBlocked() && sunk.size() == 1 && sunk[0].first == 3 && + !sunk[0].second && closes == 0, + "a blocked value is not closed again and a third value is retained by the sink"); +} + +void testReleaseClosesTheParkedReplacementAfterTheCurrentValue() { + rea::OwnedHandle handle(-1); + expect(handle.park(1, false), "adopt the current value"); + expect(handle.park(2, false), "park the replacement"); + std::vector order; + const auto decision = handle.release( + [&](int value) { + order.push_back(value); + return rea::CloseAttempt{true, 0}; + }, + [&](int) { return live(rea::HandleLiveness::Unavailable, 1); }); + expect(decision.disposition == rea::HandleCloseDisposition::Released && handle.fullyReleased() && + order.size() == 2 && order[0] == 1 && order[1] == 2, + "release closes the current value and then the parked replacement"); +} + +void testAbandonHandsSurvivingValuesToTheSinkOnce() { + rea::OwnedHandle handle(-1); + expect(handle.park(7, false), "adopt the abandoned value"); + expect(handle.park(8, false), "park a second abandoned value"); + int closes = 0; + std::vector> sunk; + handle.abandon( + [&](int) { + ++closes; + return fail(5); + }, + [&](int) { return live(rea::HandleLiveness::Unavailable, 11); }, + [&](int value, bool block) { sunk.emplace_back(value, block); }); + expect(handle.fullyReleased() && closes == 1 && sunk.size() == 2 && sunk[0] == std::make_pair(7, true) && + sunk[1] == std::make_pair(8, false), + "abandon retains every surviving value and does not close the unconfirmed one twice"); +} + +void testMovePreservesTheCloseGate() { + rea::OwnedHandle handle(-1); + expect(handle.park(4, true), "adopt a blocked value"); + rea::OwnedHandle moved(std::move(handle)); + expect(handle.fullyReleased() && moved.get() == 4 && moved.closeBlocked(), + "move transfers the value and its close gate"); + int closes = 0; + moved.release([&](int) { + ++closes; + return fail(1); + }, + [&](int) { return live(rea::HandleLiveness::Unavailable, 2); }); + expect(closes == 0 && moved.get() == 4 && moved.closeBlocked(), + "a moved blocked value is probed and not closed while liveness is unknown"); +} + +} // namespace + +int main() { + testDecisionMatrix(); + testReleaseRetainsAndRetriesOnlyLiveValues(); + testUnconfirmedCloseDoesNotCallCloseAgain(); + testTerminalInvalidityDropsTheValue(); + testBlockedProbeCanDiscoverTerminalInvalidityWithoutClosing(); + testResetParksTheReplacementAndOverflowsTheThirdValue(); + testReleaseClosesTheParkedReplacementAfterTheCurrentValue(); + testAbandonHandsSurvivingValuesToTheSinkOnce(); + testMovePreservesTheCloseGate(); + if (failures != 0) { + std::cerr << failures << " assertion(s) failed\n"; + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} diff --git a/native/windows/src/process.cc b/native/windows/src/process.cc index bfdbbcdb6e..1ce422212e 100644 --- a/native/windows/src/process.cc +++ b/native/windows/src/process.cc @@ -217,14 +217,19 @@ static napi_value readOutput(napi_env env, Handle& read, bool& ended) { constexpr size_t kMaximumReadPerPoll = 1024 * 1024; std::array chunk; std::vector bytes; - if (!read.valid()) ended = true; + // A close-blocked pipe is EOF. Its number may have been reused, so this + // poll must not pass it to PeekNamedPipe or ReadFile. process_close still + // owns that value. + if (!read.valid() || read.closeBlocked()) ended = true; else { while (bytes.size() < kMaximumReadPerPoll) { DWORD available = 0; if (!PeekNamedPipe(read.get(), nullptr, 0, nullptr, &available, nullptr)) { require(GetLastError() == ERROR_BROKEN_PIPE, "Observe owned process output failed"); - read.reset(); + // Broken pipe is EOF for the bytes already queued. A failed close keeps + // this handle for process_close instead of dropping those bytes. + read.release(); ended = true; break; } @@ -350,7 +355,8 @@ static void completeInput(napi_env env, napi_status status, void* pointer) { } static napi_value inputWrite(napi_env env, napi_value owner, Process& process, napi_value input) { - require(process.stdinWrite.valid(), "Owned process has no writable input", L"", ERROR_BROKEN_PIPE); + require(process.stdinWrite.valid() && !process.stdinWrite.closeBlocked(), + "Owned process has no writable input", L"", ERROR_BROKEN_PIPE); require(!process.inputPending, "Owned process input already has a pending write", L"", ERROR_BUSY); auto work = std::make_unique(process); void* bytes = nullptr; @@ -411,7 +417,7 @@ napi_value processCall(napi_env env, const std::wstring& operation, const std::v require(args.size() == 1, "Wrong native process handle argument count", L"", ERROR_INVALID_PARAMETER); auto& process = static_cast(resource(env, args[0], Kind::Process)); if (operation == L"process_stdin_close") { - process.stdinWrite.reset(); + requireReleased(process.stdinWrite, "Close owned process stdin failed"); return null(env); } if (operation == L"process_poll") { @@ -446,8 +452,17 @@ napi_value processCall(napi_env env, const std::wstring& operation, const std::v } if (operation == L"process_close") { // Closing the job is the crash-safe authority; no PID enumeration is used. - process.job.reset(); process.process.reset(); process.stdoutRead.reset(); process.stderrRead.reset(); process.stdinWrite.reset(); - process.notifications.reset(); + // A failed close leaves that handle on this process so a later close can + // retry it. Handles that already closed stay released, so the retry + // resumes at the first handle still owned. Callers that have observed + // settlement must retry this close alone; kill and poll still need the job. + retryRetainedNativeHandles(); + requireReleased(process.job, "Close owned Job Object failed"); + requireReleased(process.process, "Close owned process failed"); + requireReleased(process.stdoutRead, "Close owned process stdout failed"); + requireReleased(process.stderrRead, "Close owned process stderr failed"); + requireReleased(process.stdinWrite, "Close owned process stdin failed"); + requireReleased(process.notifications, "Close owned process notifications failed"); process.closed = true; return null(env); } diff --git a/scripts/ci/test-history.mjs b/scripts/ci/test-history.mjs index 14dbb582b4..f14f73873c 100644 --- a/scripts/ci/test-history.mjs +++ b/scripts/ci/test-history.mjs @@ -4,21 +4,38 @@ import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { normalizeWorkflowRun } from "./workflow-run.mjs"; -const ARTIFACT_NAME = "test-timings"; +const TIER_EVENTS = ["push", "schedule", "workflow_dispatch"]; +const TIER_PROFILES = { + fast: { artifactName: "test-timings-fast", events: TIER_EVENTS }, + full: { artifactName: "test-timings-full", events: TIER_EVENTS }, +}; const MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; const MAX_CANDIDATES = 5; +const EMPTY_SELECTION = { runId: "", artifactName: "" }; const positiveId = (value) => (typeof value === "string" || typeof value === "number") && /^[1-9]\d*$/.test(String(value)) && Number.isSafeInteger(Number(value)); -function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) { +function findTimingRun({ + repository, + branch, + repositoryId, + tier, + now = Date.now(), +}) { if ( !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository ?? "") || !branch || !positiveId(repositoryId) ) - return ""; + return EMPTY_SELECTION; + const profile = Object.hasOwn(TIER_PROFILES, tier) + ? TIER_PROFILES[tier] + : null; + // Reused and other tiers do not publish shard history. Missing history uses + // Vitest's own assignment rather than another tier's artifact. + if (!profile) return EMPTY_SELECTION; const deadline = Date.now() + 20_000; const api = (...arguments_) => { const remaining = deadline - Date.now(); @@ -33,83 +50,93 @@ function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) { }), ); }; - - const { artifacts } = api( - `repos/${repository}/actions/artifacts`, - "-f", - `name=${ARTIFACT_NAME}`, - "-F", - "per_page=100", - ); - if (!Array.isArray(artifacts)) - throw new Error("Missing timing artifact inventory"); - const candidates = artifacts - .filter((artifact) => { - const lineage = artifact?.workflow_run; - const age = now - Date.parse(artifact?.created_at); - return ( - artifact?.name === ARTIFACT_NAME && - artifact?.expired === false && - typeof artifact?.created_at === "string" && - Number.isFinite(age) && - age >= 0 && - age < MAX_AGE_MS && - positiveId(lineage?.id) && - positiveId(lineage?.repository_id) && - positiveId(lineage?.head_repository_id) && - lineage.repository_id === lineage.head_repository_id && - lineage.repository_id === Number(repositoryId) && - lineage.head_branch === branch - ); - }) - .sort( - (left, right) => - Date.parse(right.created_at) - Date.parse(left.created_at), - ) - .slice(0, MAX_CANDIDATES); - - for (const artifact of candidates) { - const run = normalizeWorkflowRun( - api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`), + const search = ({ artifactName, events }) => { + const { artifacts } = api( + `repos/${repository}/actions/artifacts`, + "-f", + `name=${artifactName}`, + "-F", + "per_page=100", ); - const runAge = now - Date.parse(run?.created_at); - if ( - typeof run?.created_at === "string" && - Number.isFinite(runAge) && - runAge >= 0 && - runAge < MAX_AGE_MS && - run.id === artifact.workflow_run.id && - run.status === "completed" && - run.conclusion === "success" && - run.workflowPath?.file === ".github/workflows/ci.yml" && - ["schedule", "workflow_dispatch"].includes(run.event) && - run.head_branch === branch && - run.head_repository?.full_name === repository - ) - return String(run.id); - } - return ""; + if (!Array.isArray(artifacts)) + throw new Error("Missing timing artifact inventory"); + const candidates = artifacts + .filter((artifact) => { + const lineage = artifact?.workflow_run; + const age = now - Date.parse(artifact?.created_at); + return ( + artifact?.name === artifactName && + artifact?.expired === false && + typeof artifact?.created_at === "string" && + Number.isFinite(age) && + age >= 0 && + age < MAX_AGE_MS && + positiveId(lineage?.id) && + positiveId(lineage?.repository_id) && + positiveId(lineage?.head_repository_id) && + lineage.repository_id === lineage.head_repository_id && + lineage.repository_id === Number(repositoryId) && + lineage.head_branch === branch + ); + }) + .sort( + (left, right) => + Date.parse(right.created_at) - Date.parse(left.created_at), + ) + .slice(0, MAX_CANDIDATES); + + for (const artifact of candidates) { + const run = normalizeWorkflowRun( + api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`), + ); + const runAge = now - Date.parse(run?.created_at); + if ( + typeof run?.created_at === "string" && + Number.isFinite(runAge) && + runAge >= 0 && + runAge < MAX_AGE_MS && + run.id === artifact.workflow_run.id && + run.status === "completed" && + run.conclusion === "success" && + run.workflowPath?.file === ".github/workflows/ci.yml" && + events.includes(run.event) && + run.head_branch === branch && + run.head_repository?.full_name === repository + ) + return String(run.id); + } + return ""; + }; + + const runId = search(profile); + return runId + ? { runId, artifactName: profile.artifactName } + : EMPTY_SELECTION; } -function write(runId) { +function write({ runId, artifactName }) { if (process.env.GITHUB_OUTPUT) - appendFileSync(process.env.GITHUB_OUTPUT, `run-id=${runId}\n`); - console.log(JSON.stringify({ run_id: runId })); + appendFileSync( + process.env.GITHUB_OUTPUT, + `run-id=${runId}\nartifact-name=${artifactName}\n`, + ); + console.log(JSON.stringify({ run_id: runId, artifact_name: artifactName })); } if ( process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1]) ) { - let runId = ""; + let selection = EMPTY_SELECTION; try { - runId = findTimingRun({ + selection = findTimingRun({ repository: process.env.CI_REPOSITORY, branch: process.env.CI_DEFAULT_BRANCH, repositoryId: process.env.CI_REPOSITORY_ID, + tier: process.env.CI_TIMING_TIER, }); } catch { // History improves shard ordering but must never block CI. } - write(runId); + write(selection); } diff --git a/scripts/lib/browser-runtime-termination-e2e.mjs b/scripts/lib/browser-runtime-termination-e2e.mjs index 52dd90d5ac..a57d0716f7 100644 --- a/scripts/lib/browser-runtime-termination-e2e.mjs +++ b/scripts/lib/browser-runtime-termination-e2e.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { CdpConnection } from "../../dist/browser/CdpConnection.js"; -import { mcpTextValue } from "./mcp-verifier-results.mjs"; +import { mcpTextValue, requireMcpToolError } from "./mcp-verifier-results.mjs"; import { closeVerifierResources } from "./verifier-cleanup.mjs"; /** Close a separately owned fixture page after actual MCP arming and inspect native cleanup diagnostics. */ @@ -60,17 +60,26 @@ export async function verifyRuntimeTargetTermination(client, input, origin) { "Detached instrumentation cannot claim confirmed cleanup", ); const failure = mcpTextValue(response); + const projected = requireMcpToolError(response); assert.match(failure, /cleanup.*confirmed/u); assert.ok( failure.includes(targetId), "Failure must retain the selected target identity", ); assert.match(failure, /Profiler\.stopPreciseCoverage/u); + assert.equal(projected.code, "cleanup_incomplete"); + assert.equal( + projected.details?.diagnostics?.primary_error?.details?.diagnostics + ?.observed_window?.end_reason, + "session_detached", + "Target.closeTarget detaches this CDP session; it does not prove the target crashed", + ); return { target_id: targetId, page_closed_after_arming: true, failure_inline: true, cleanup: "unconfirmed_reported", + observed_window_end_reason: "session_detached", }; } catch (cause) { primaryError = cause; diff --git a/scripts/verify-real-browser.mjs b/scripts/verify-real-browser.mjs index 54c1eaa112..4ba8d1e964 100644 --- a/scripts/verify-real-browser.mjs +++ b/scripts/verify-real-browser.mjs @@ -546,13 +546,14 @@ async function verifyPageScopedTransport(provider, proxy, origin) { const disconnected = await disconnectedPromise; if ( !disconnected.ok || - disconnected.value.window.end_reason !== "target_terminated" || + disconnected.value.window.end_reason !== "transport_disconnected" || !disconnected.value.timeline.some( - ({ type }) => type === "target_terminated", - ) + ({ type }) => type === "transport_disconnected", + ) || + disconnected.value.timeline.some(({ type }) => type === "target_terminated") ) throw new Error( - "Page-scoped CDP disconnect was not reported as target_terminated", + "Page-scoped CDP disconnect must report transport loss without claiming a target crash", ); } diff --git a/scripts/verify-wasm-artifact.mjs b/scripts/verify-wasm-artifact.mjs index 7b9c08071b..38ee229bd4 100644 --- a/scripts/verify-wasm-artifact.mjs +++ b/scripts/verify-wasm-artifact.mjs @@ -118,6 +118,9 @@ try { entrypoint, "inspect-wasm-artifact", input.path, + ...(input.command_timeout_ms === undefined + ? [] + : ["--command-timeout-ms", String(input.command_timeout_ms)]), ...input.glue_paths.flatMap((path) => ["--glue_paths", path]), ...input.candidate_paths.flatMap((path) => [ "--candidate_paths", @@ -238,7 +241,12 @@ try { ); for (const [index, gluePath] of gluePaths.entries()) await writeFile(gluePath, glueSources[index]); - const input = { path, glue_paths: gluePaths, candidate_paths: [candidate] }; + const input = { + path, + glue_paths: gluePaths, + candidate_paths: [candidate], + ...(index === 0 ? { command_timeout_ms: 60_000 } : {}), + }; assert.ok(ajv.validate(advertised.inputSchema, input)); const cliValue = await inspectCli(input); assert.equal(cliValue.category, undefined, JSON.stringify(cliValue)); @@ -256,6 +264,11 @@ try { assert.equal(cli.evidence_id, mcp.evidence_id); assert.deepEqual(cli, mcp); const report = mcp.normalized_result; + assert.equal( + report.tool_profile.timeout_ms, + input.command_timeout_ms ?? 30_000, + ); + assert.equal(mcp.parameters.command_timeout_ms, input.command_timeout_ms); assert.equal(report.artifact.sha256, sha256(original)); assert.equal(report.artifact.bytes, original.length); assert.equal(report.wat.text, wat); @@ -359,6 +372,7 @@ try { path: invalidPath, glue_paths: [], candidate_paths: [], + ...(index === 0 ? { command_timeout_ms: 42_000 } : {}), }; const cliError = await inspectCli(invalidInput); assert.equal( @@ -391,6 +405,14 @@ try { ), ); assert.equal(failed.subject.digest.sha256, sha256(invalidBytes)); + assert.equal( + failed.normalized_result.tool_profile.timeout_ms, + invalidInput.command_timeout_ms ?? 30_000, + ); + assert.equal( + failed.normalized_result.tool_profile.output_budget_bytes, + 32 * 1024 * 1024, + ); assert.equal( failed.normalized_result.validation, "rejected-by-selected-profile", diff --git a/src/application/AppBundleExecutable.ts b/src/application/AppBundleExecutable.ts index eb160ead28..ca7dfba21e 100644 --- a/src/application/AppBundleExecutable.ts +++ b/src/application/AppBundleExecutable.ts @@ -4,7 +4,7 @@ import { join } from "node:path"; import { parseBinary } from "plist"; import { z } from "zod"; -import { preflightPropertyListDecode } from "../artifacts/apple/InterfaceBuilderDecodeBudget.js"; +import { preflightPropertyListDecode } from "../artifacts/apple/PropertyListDecodeBudget.js"; import { BinaryTargetError } from "../domain/configurationErrors.js"; import { isPathWithinRoot } from "../domain/localPath.js"; import { diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.ts index b2b6834e14..7b7f58afb0 100644 --- a/src/application/BinaryTargetResolver.ts +++ b/src/application/BinaryTargetResolver.ts @@ -90,11 +90,15 @@ export const parseBinaryTarget = async ( const hostArchitecture = options.hostArchitecture ?? process.arch; const { targetKind, formatHint, signal, architecture, entryPoint } = options; const candidate = absoluteCallerPath(input, { cwd }); + // Failures before canonicalization concern the selected path itself; later + // failures concern the canonical path opened with symbolic links rejected. + let stage: TargetFilesystemStage = "resolve"; try { throwIfTargetResolutionCancelled(signal); await access(candidate, constants.R_OK); throwIfTargetResolutionCancelled(signal); const canonical = await realpath(candidate); + stage = "read"; // Windows native admission must inspect the caller-selected reparse components. const sourcePath = process.platform === "win32" ? candidate : canonical; throwIfTargetResolutionCancelled(signal); @@ -135,6 +139,7 @@ export const parseBinaryTarget = async ( return ok({ path, sourcePath, + ...bundle, sha256: await sha256Handle(handle, path, metadata, signal), kind: "executable", format: "raw-binary", @@ -174,6 +179,7 @@ export const parseBinaryTarget = async ( return ok({ path, sourcePath, + ...bundle, sha256: await sha256Handle(handle, path, metadata, signal), kind: "executable", format: "dos-com", @@ -245,24 +251,39 @@ export const parseBinaryTarget = async ( ); if (cause instanceof BinaryTargetError) return err(cause); if (!(cause instanceof Error)) throw cause; - const failure = classifyBinaryTargetFilesystemFailure(candidate, cause); + const failure = classifyTargetFilesystemFailure(candidate, cause, stage); if (failure === undefined) throw cause; return err(failure); } }; +/** + * `resolve` covers the caller-selected path before canonicalization; `read` + * covers a canonical path opened with symbolic links rejected. + */ +type TargetFilesystemStage = "resolve" | "read"; + /** Classify a filesystem failure while reading a previously selected target. */ export const classifyBinaryTargetFilesystemFailure = ( path: string, cause: unknown, +): BinaryTargetError | undefined => + classifyTargetFilesystemFailure(path, cause, "read"); + +const classifyTargetFilesystemFailure = ( + path: string, + cause: unknown, + stage: TargetFilesystemStage, ): BinaryTargetError | undefined => { if (!(cause instanceof Error)) return undefined; const code = filesystemErrorCode(cause); return code === undefined ? undefined - : new BinaryTargetError(path, targetFailureReason(code, cause.message), { - cause, - }); + : new BinaryTargetError( + path, + targetFailureReason(code, cause.message, stage), + { cause }, + ); }; const filesystemErrorCode = (cause: Error): string | undefined => { @@ -270,7 +291,11 @@ const filesystemErrorCode = (cause: Error): string | undefined => { return typeof code === "string" ? code : undefined; }; -const targetFailureReason = (code: string, detail: string): string => { +const targetFailureReason = ( + code: string, + detail: string, + stage: TargetFilesystemStage, +): string => { switch (code) { case "ENOENT": case "ENOTDIR": @@ -283,7 +308,12 @@ const targetFailureReason = (code: string, detail: string): string => { case "ENOTFILE": return "target is not a regular file"; case "ELOOP": - return `target changed while being opened: ${detail}`; + // A canonical path has no symbolic links left, so a link rejected while + // reading it appeared after resolution. Before that, ELOOP is a loop or + // an over-deep link chain in the selected path itself. + return stage === "resolve" + ? `target path could not be resolved: too many symbolic links or a symbolic link loop: ${detail}` + : `target changed while being opened: ${detail}`; default: return `target could not be read (${code}): ${detail}`; } diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index 0949773277..87e2dc711a 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -685,6 +685,7 @@ const targetAvailability = ({ targetKind, targetFormat, targetOpen, + descriptors, }: AvailabilityContext): Availability | null => { if (FRIDA_OPERATION_NAMES.has(name)) return null; if (!targetOpen && ACTIVE_TARGET_KINDS.has(kind)) @@ -692,6 +693,14 @@ const targetAvailability = ({ reason: "target_required", remediation: "Call open_binary with a supported local target.", }; + const descriptor = descriptors.get(name); + if ( + targetOpen && + descriptor?.available === false && + (descriptor.availability_code === "target_kind_unsupported" || + descriptor.availability_code === "target_format_unsupported") + ) + return { reason: "target_unsupported", remediation: descriptor.reason }; if ( targetKind !== undefined && targetKind !== "executable" && diff --git a/src/application/binary/BinarySessionStatus.test.ts b/src/application/binary/BinarySessionStatus.test.ts index ba1c451441..12cac2defb 100644 --- a/src/application/binary/BinarySessionStatus.test.ts +++ b/src/application/binary/BinarySessionStatus.test.ts @@ -1,3 +1,4 @@ +import type { BinaryTarget } from "../../domain/binaryTargetTypes.js"; import { describe, expect, it } from "vitest"; import type { @@ -124,6 +125,7 @@ const project = async ( string, { readonly reason: string } > = new Map(), + target?: BinaryTarget, ) => { const route: SessionProviderRoute = { identity: providerA, @@ -142,7 +144,7 @@ const project = async ( }; return projectedStatusSchema.parse( await binarySessionStatus({ - target: undefined, + target, route, router, runtimeUnavailability, @@ -412,3 +414,33 @@ describe("binary session provider capability projection", () => { ]); }); }); + +it.each(["elf", "pe"] as const)( + "reports format rejection for %s dylib discovery", + async (format) => { + const target: BinaryTarget = { + kind: "executable", + format, + path: "/fixture", + sha256: "a".repeat(64), + architecture: "x86_64", + availableArchitectures: ["x86_64"], + ...(format === "pe" + ? { executableRole: "application" as const, managed: false } + : {}), + } as BinaryTarget; + const status = await project( + [descriptor(providerA, "trace_dylib_resolution", [])], + [], + new Map(), + target, + ); + expect(expandCanonicalCapabilities(status.capabilities)).toMatchObject([ + { + operation: "trace_dylib_resolution", + available: false, + availability_code: "target_format_unsupported", + }, + ]); + }, +); diff --git a/src/application/binary/BinarySessionStatus.ts b/src/application/binary/BinarySessionStatus.ts index 104d7e1011..7cc45e1f18 100644 --- a/src/application/binary/BinarySessionStatus.ts +++ b/src/application/binary/BinarySessionStatus.ts @@ -1,3 +1,4 @@ +import { artifactTargetRefusal } from "../../domain/apple/artifactTargetApplicability.js"; import type { BinaryTarget } from "../../domain/binaryTargetTypes.js"; import type { ResolvedAppBundleResourceRoot } from "../AppBundleExecutable.js"; import { binaryTargetSummary } from "./BinaryTargetSummary.js"; @@ -62,6 +63,17 @@ export const binarySessionStatus = async ({ compareUnicodeCodePoints(left.operation, right.operation), ) .map((descriptor) => { + const targetRefusal = + target === undefined + ? undefined + : artifactTargetRefusal(descriptor.operation, target); + if (targetRefusal !== undefined && descriptor.available) + return { + ...descriptor, + available: false as const, + availabilityCode: targetRefusal.code, + reason: targetRefusal.reason, + }; const unavailable = runtimeUnavailability.get(descriptor.operation); return unavailable === undefined ? descriptor diff --git a/src/application/environment/environmentBundleWrite.test.ts b/src/application/environment/environmentBundleWrite.test.ts index 6bbac53e89..1e1ec5dd5e 100644 --- a/src/application/environment/environmentBundleWrite.test.ts +++ b/src/application/environment/environmentBundleWrite.test.ts @@ -19,6 +19,10 @@ import { } from "../../../tests/fixtures/environmentServe.js"; import { SafeOutputTree } from "../../artifacts/SafeOutputTree.js"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; +import { + AnalysisAccessDeniedError, + AnalysisOutputError, +} from "../../domain/analysisErrorCore.js"; import { exportEnvironmentBundle, readEnvironmentExportInputs, @@ -186,6 +190,7 @@ it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( error: { _tag: "AnalysisAccessDeniedError", operation: "environment-export", + diagnostics: { output_directory: directory }, }, }); expect(await readdir(directory)).toEqual([]); @@ -195,6 +200,111 @@ it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( }, ); +it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "names the nested path when write permission is lost after earlier files", + async () => { + const root = await createTestTempDirectory("rea-bundle-later-denied-"); + const files = await writeEnvironmentServeFiles(root); + const inputs = await readEnvironmentExportInputs( + files.manifestPath, + files.capturePath, + ); + if (!inputs.ok) throw inputs.error; + const directory = join(root, "bundle"); + const original = SafeOutputTree.prototype.write; + let writes = 0; + // The real host refuses the second file once the directory is read-only. + const writer = vi + .spyOn(SafeOutputTree.prototype, "write") + .mockImplementation(async function (this: SafeOutputTree, ...args) { + writes += 1; + if (writes === 2) await chmod(directory, 0o500); + return original.apply(this, args); + }); + try { + const result = await exportEnvironmentBundle({ + ...inputs.value, + source: environmentServeSource(), + directory, + }); + if (result.ok) throw new Error("expected a permission refusal"); + expect(result.error).toMatchObject({ + _tag: "AnalysisAccessDeniedError", + diagnostics: { output_directory: directory }, + }); + if (!(result.error instanceof AnalysisAccessDeniedError)) + throw result.error; + expect(result.error.path.startsWith(`${directory}/`)).toBe(true); + expect(await readdir(directory)).not.toEqual([]); + } finally { + writer.mockRestore(); + await chmod(directory, 0o700); + } + }, +); + +it.each([ + { + failure: "pathless descriptor failure", + // A disk-full `FileHandle.write()` reports a code and syscall, no path. + error: () => + Object.assign(new Error("ENOSPC: no space left on device, write"), { + code: "ENOSPC", + syscall: "write", + }), + }, + { + failure: "integrity refusal", + error: () => + new ArtifactReaderFailure( + "integrity", + "Extracted content disagrees with inventory", + ), + }, +] as const)( + "reports a later $failure with its own recovery", + async ({ failure, error }) => { + const root = await createTestTempDirectory("rea-bundle-later-failure-"); + const files = await writeEnvironmentServeFiles(root); + const inputs = await readEnvironmentExportInputs( + files.manifestPath, + files.capturePath, + ); + if (!inputs.ok) throw inputs.error; + const directory = join(root, "bundle"); + const original = SafeOutputTree.prototype.write; + const targets: string[] = []; + const writer = vi + .spyOn(SafeOutputTree.prototype, "write") + .mockImplementation(async function (this: SafeOutputTree, ...args) { + targets.push(args[0]); + if (targets.length === 2) throw error(); + return original.apply(this, args); + }); + try { + const result = await exportEnvironmentBundle({ + ...inputs.value, + source: environmentServeSource(), + directory, + }); + if (result.ok || !(result.error instanceof AnalysisOutputError)) + throw new Error("expected an output failure"); + expect(result.error.diagnostics).toEqual({ + output_directory: directory, + }); + if (failure === "pathless descriptor failure") { + expect(result.error.reason).toContain( + join(directory, targets[1] ?? ""), + ); + expect(result.error.remediationAction).toContain("free disk space"); + } else expect(result.error.remediationAction).toBeUndefined(); + expect(await readdir(directory)).not.toEqual([]); + } finally { + writer.mockRestore(); + } + }, +); + it .skipIf(process.platform === "win32") .each(["replace-root", "foreign-entry", "symlink-parent"] as const)( diff --git a/src/application/environment/environmentBundleWrite.ts b/src/application/environment/environmentBundleWrite.ts index d3ec3c55dc..47f225e8a0 100644 --- a/src/application/environment/environmentBundleWrite.ts +++ b/src/application/environment/environmentBundleWrite.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; import { mkdir, type FileHandle } from "node:fs/promises"; -import { dirname } from "node:path"; +import { dirname, join } from "node:path"; import { Readable } from "node:stream"; import { @@ -12,6 +12,7 @@ import { ArtifactFileCleanupFailure } from "../../artifacts/ArtifactFileOperatio import { RegularFileCleanupFailure } from "../../filesystem/OwnedRegularFile.js"; import { readFileHandleChunks } from "../../filesystem/readFileHandleChunks.js"; import { streamChunkToBuffer } from "../../artifacts/StreamBytes.js"; +import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; import { SafeOutputTree } from "../../artifacts/SafeOutputTree.js"; import { SafeOutputTreeCreationFailure } from "../../artifacts/SafeOutputTreeCreationFailure.js"; @@ -26,15 +27,69 @@ import type { WrittenBundleFile, } from "./environmentExport.js"; +/** + * A host failure on one file the bundle writer was reading or writing. + * Descriptor operations such as `read`, `write` and `fsync` fail without a + * path, so this names the file and the direction of the failed operation. + */ +export class BundleFileHostFailure extends Error { + constructor( + readonly path: string, + readonly operation: "reading" | "writing", + cause: Error, + ) { + super(`${cause.message}, ${operation} ${path}`, { cause }); + } +} + +/** An untyped host system failure, which carries an errno-style code. */ +const isHostSystemFailure = (cause: unknown): cause is Error => + cause instanceof Error && + !(cause instanceof ArtifactReaderFailure) && + typeof Reflect.get(cause, "code") === "string"; + +/** Attribute a pathless host failure to the bundle file being written. */ +const writingBundleFile = async ( + directory: string, + relative: string, + write: () => Promise, +): Promise => { + try { + return await write(); + } catch (cause: unknown) { + if (isHostSystemFailure(cause) && Reflect.get(cause, "path") === undefined) + throw new BundleFileHostFailure( + join(directory, relative), + "writing", + cause, + ); + throw cause; + } +}; + +/** Read the admitted tarball, attributing host failures to the source. */ +async function* tarballChunks(source: BundleTarball): AsyncIterable { + try { + yield* readFileHandleChunks(source.handle); + } catch (cause: unknown) { + if (isHostSystemFailure(cause)) + throw new BundleFileHostFailure(source.path, "reading", cause); + throw cause; + } +} + const writeBytes = async ( tree: SafeOutputTree, + directory: string, relative: string, bytes: Buffer, ): Promise => { - const written = await tree.write(relative, Readable.from([bytes]), { - bytes: bytes.length, - sha256: createHash("sha256").update(bytes).digest("hex"), - }); + const written = await writingBundleFile(directory, relative, () => + tree.write(relative, Readable.from([bytes]), { + bytes: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + }), + ); return { path: written.relativePath, bytes: written.bytesWritten }; }; @@ -47,29 +102,50 @@ interface BundleTarball { /** Hash and copy only the instance retained before destination admission. */ const writeTarball = async ( tree: SafeOutputTree, + directory: string, relative: string, source: BundleTarball, ): Promise => { const hash = createHash("sha256"); let observed = 0; - for await (const chunk of readFileHandleChunks(source.handle)) { + for await (const chunk of tarballChunks(source)) { const bytes = streamChunkToBuffer(chunk); observed += bytes.length; if (observed > source.size) throw new RegularFileChangedError(source.path); hash.update(bytes); } if (observed !== source.size) throw new RegularFileChangedError(source.path); - const written = await tree.write( - relative, - readFileHandleChunks(source.handle), - { + const sha256 = hash.digest("hex"); + const written = await writingBundleFile(directory, relative, () => + tree.write(relative, Readable.from(tarballChunks(source)), { bytes: source.size, - sha256: hash.digest("hex"), - }, + sha256, + }), ); return { path: written.relativePath, bytes: written.bytesWritten }; }; +/** + * Only destination admission can blame the selected directory. Path failures + * after admission are host write failures on bundle files, not argument refusals. + */ +const directoryRefusal = (cause: unknown): unknown => + cause instanceof ArtifactReaderFailure && + cause.reason === "path" && + cause.cleanup === undefined + ? new AnalysisInputError( + ENVIRONMENT_EXPORT_OPERATION, + { cause, userMessage: cause.message }, + [ + { + path: ["directory"], + reason: "invalid_value", + message: cause.message, + }, + ], + ) + : cause; + /** The output owner anchors every create and verifies the complete tree before publication. */ const writeBundleFiles = async ( input: EnvironmentExportInput, @@ -79,6 +155,7 @@ const writeBundleFiles = async ( const resources = new ArtifactResourceScope(); let tree: SafeOutputTree | undefined; let written: WrittenBundleFile[] | undefined; + let admitted = false; let outcome: | { readonly kind: "completed"; @@ -95,19 +172,37 @@ const writeBundleFiles = async ( if (cause instanceof SafeOutputTreeCreationFailure) tree = cause.tree; throw cause; } + admitted = true; written = []; for (const file of plan.files) written.push( - await writeBytes(tree, file.path, Buffer.from(file.content)), + await writeBytes( + tree, + input.directory, + file.path, + Buffer.from(file.content), + ), ); written.push( - await writeBytes(tree, plan.copies.manifest, input.manifest.bytes), + await writeBytes( + tree, + input.directory, + plan.copies.manifest, + input.manifest.bytes, + ), ); written.push( - await writeBytes(tree, plan.copies.evidence, input.evidence.bytes), + await writeBytes( + tree, + input.directory, + plan.copies.evidence, + input.evidence.bytes, + ), ); if (plan.copies.tarball !== null && tarball !== undefined) - written.push(await writeTarball(tree, plan.copies.tarball, tarball)); + written.push( + await writeTarball(tree, input.directory, plan.copies.tarball, tarball), + ); await tree.commit(); outcome = { kind: "completed", files: written }; } catch (cause: unknown) { @@ -125,7 +220,7 @@ const writeBundleFiles = async ( } if (outcome.kind === "failed") { if (owner?.published === true && written !== undefined) return written; - throw outcome.cause; + throw admitted ? outcome.cause : directoryRefusal(outcome.cause); } return outcome.files; }; diff --git a/src/application/environment/environmentExport.test.ts b/src/application/environment/environmentExport.test.ts index 1ec7fb1af5..e55a847c02 100644 --- a/src/application/environment/environmentExport.test.ts +++ b/src/application/environment/environmentExport.test.ts @@ -1,8 +1,7 @@ import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises"; import { join, sep } from "node:path"; -import * as v8 from "node:v8"; -import { describe, expect, it, vi } from "vitest"; +import { describe, expect, it } from "vitest"; import { environmentServeFixture, @@ -12,17 +11,11 @@ import { import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; import { PACKAGE_METADATA } from "../../generatedPackageMetadata.js"; -import { RegularFileChangedError } from "../../filesystem/RegularFile.js"; -import { nativeJsonParseRequiredBytes } from "../JsonInputFile.js"; -import * as regularFileRead from "../RegularFileRead.js"; import { exportEnvironmentBundle, readEnvironmentExportInputs, } from "./environmentExport.js"; -vi.mock("node:v8", { spy: true }); -vi.mock("../RegularFileRead.js", { spy: true }); - const listFiles = async (root: string): Promise => (await readdir(root, { recursive: true, withFileTypes: true })) .filter((entry) => entry.isFile()) @@ -52,41 +45,6 @@ const prepare = async (prefix: string) => { }; describe("readEnvironmentExportInputs", () => { - it.each(["manifest", "evidence"] as const)( - "returns a tagged input error when the %s snapshot changes", - async (role) => { - const root = await createTestTempDirectory( - "rea-environment-input-changed-", - ); - const files = await writeEnvironmentServeFiles(root); - const changedPath = - role === "manifest" ? files.manifestPath : files.capturePath; - const actual = await vi.importActual( - "../RegularFileRead.js", - ); - const reader = vi.mocked(regularFileRead.readRegularFile); - reader.mockImplementation(async (path, options) => { - if (path === changedPath) throw new RegularFileChangedError(path); - return actual.readRegularFile(path, options); - }); - try { - const result = await readEnvironmentExportInputs( - files.manifestPath, - files.capturePath, - ); - if (result.ok) throw new Error("Expected changed input refusal"); - expect(projectAnalysisError(result.error)).toMatchObject({ - code: "invalid_request", - details: { - operation: "environment-export", - issues: [{ path: [role], reason: "invalid_value" }], - }, - }); - } finally { - reader.mockImplementation(actual.readRegularFile); - } - }, - ); it("reads each file once as the bytes on disk and the value they parse to", async () => { const root = await createTestTempDirectory("rea-environment-inputs-"); const fixture = environmentServeFixture(); @@ -107,32 +65,6 @@ describe("readEnvironmentExportInputs", () => { expect(inputs.value.evidence.value).toEqual(fixture.capture); }); - it("refuses an input whose whole read and parse would not fit live heap headroom, before reading it", async () => { - const root = await createTestTempDirectory("rea-environment-export-heap-"); - const files = await writeEnvironmentServeFiles( - root, - environmentServeFixture(), - ); - vi.mocked(v8.getHeapStatistics).mockReturnValueOnce({ - ...v8.getHeapStatistics(), - total_available_size: 1024, - }); - const refused = await readEnvironmentExportInputs( - files.manifestPath, - files.capturePath, - ); - expect(refused.ok).toBe(false); - if (refused.ok) return; - expect(refused.error).toMatchObject({ - _tag: "AnalysisResourceConstraintError", - resource: "memory", - reportedLimits: { - input_path: files.manifestPath, - available_heap_bytes: 1024, - }, - }); - }); - it("refuses inline JSON, a directory and a file that is not JSON by role", async () => { const root = await createTestTempDirectory("rea-environment-inputs-bad-"); const files = await writeEnvironmentServeFiles(root); @@ -316,34 +248,3 @@ describe("exportEnvironmentBundle file modes", () => { }, ); }); - -describe("readEnvironmentExportInputs heap admission", () => { - it("reserves headroom for snapshot sealing after the native parse", async () => { - const root = await createTestTempDirectory( - "rea-environment-export-copies-", - ); - const files = await writeEnvironmentServeFiles( - root, - environmentServeFixture(), - ); - const { size } = await stat(files.manifestPath); - // Room for parsing, but not for the owned snapshot traversal that follows it. - vi.mocked(v8.getHeapStatistics).mockReturnValueOnce({ - ...v8.getHeapStatistics(), - total_available_size: nativeJsonParseRequiredBytes(size) * 1.5, - }); - const refused = await readEnvironmentExportInputs( - files.manifestPath, - files.capturePath, - ); - expect(refused.ok).toBe(false); - if (refused.ok) return; - expect(refused.error).toMatchObject({ - _tag: "AnalysisResourceConstraintError", - reportedLimits: { - input_path: files.manifestPath, - required_heap_bytes: nativeJsonParseRequiredBytes(size, true), - }, - }); - }); -}); diff --git a/src/application/environment/environmentExport.ts b/src/application/environment/environmentExport.ts index 8660248602..336011db9d 100644 --- a/src/application/environment/environmentExport.ts +++ b/src/application/environment/environmentExport.ts @@ -1,3 +1,4 @@ +import { isAbsolute } from "node:path"; import { getHeapStatistics } from "node:v8"; import { AnalysisError } from "../../domain/analysisErrorBase.js"; @@ -21,7 +22,10 @@ import { AnalysisCleanupLease } from "../../artifacts/AnalysisCleanupLease.js"; import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; import { RegularFileCleanupFailure } from "../../filesystem/OwnedRegularFile.js"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; -import { writeEnvironmentBundle } from "./environmentBundleWrite.js"; +import { + BundleFileHostFailure, + writeEnvironmentBundle, +} from "./environmentBundleWrite.js"; import { sealOwnedJsonInput } from "../OwnedJsonInput.js"; import { nativeJsonParseRequiredBytes } from "../JsonInputFile.js"; import { readRegularFile } from "../RegularFileRead.js"; @@ -94,9 +98,36 @@ const systemCode = (cause: unknown): string | undefined => ? cause.code : undefined; +/** Recovery for a host write failure that may leave partial output behind. */ +const hostWriteFailure = (directory: string) => ({ + userMessage: `Writing the environment bundle to ${directory} failed on the host; the details name the cause and, when it failed on one bundle file, that file.`, + remediationAction: `Resolve the reported host failure (for example, choose a shorter destination path or free disk space). Inspect ${directory} and remove any partial output this export wrote before retrying, since a retry refuses a non-empty directory.`, + diagnostics: { output_directory: directory }, +}); + +/** Recovery for a host failure reading the package tarball during the copy. */ +const hostReadFailure = (source: string, directory: string) => ({ + userMessage: `Reading ${source} failed on the host while writing the environment bundle to ${directory}; the details name the cause.`, + remediationAction: `Resolve the reported host failure reading ${source}. Inspect ${directory} and remove any partial output this export wrote before retrying, since a retry refuses a non-empty directory.`, + diagnostics: { output_directory: directory }, +}); + +/** The host failure beneath a wrapper, whose message names its code and absolute path. */ +const hostSystemError = (cause: Error): Error | undefined => { + const seen = new Set([cause]); + let current = cause.cause; + while (current instanceof Error && !seen.has(current)) { + if (systemCode(current) !== undefined) return current; + seen.add(current); + current = current.cause; + } + return undefined; +}; + const accessDeniedError = ( cause: unknown, path: string, + diagnostics?: Readonly>, ): AnalysisAccessDeniedError | undefined => { const seen = new Set(); let current = cause; @@ -110,6 +141,7 @@ const accessDeniedError = ( code, { cause, + ...(diagnostics === undefined ? {} : { diagnostics }), ...(cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined ? { cleanup: cause.cleanup } @@ -124,6 +156,21 @@ const accessDeniedError = ( return undefined; }; +/** The absolute path a host failure beneath wrappers names, if it names one. */ +const hostFailurePath = (cause: unknown): string | undefined => { + const seen = new Set(); + let current = cause; + while (current instanceof Error && !seen.has(current)) { + seen.add(current); + if (current instanceof BundleFileHostFailure) return current.path; + const path = Reflect.get(current, "path"); + if (systemCode(current) !== undefined && typeof path === "string") + return isAbsolute(path) ? path : undefined; + current = current.cause; + } + return undefined; +}; + /** Host permission failures retain their typed cause; unexpected failures propagate. */ const rethrowUnlessDenied = (cause: unknown, path: string): never => { throw accessDeniedError(cause, path) ?? cause; @@ -301,7 +348,13 @@ export const exportEnvironmentBundle = async ( files = await writeEnvironmentBundle(input, plan.value); } catch (cause: unknown) { if (cause instanceof AnalysisError) return err(cause); - const denied = accessDeniedError(cause, input.directory); + // A refused bundle write names the path the host refused, which may be a + // nested file after earlier files were written into the directory. + const denied = accessDeniedError( + cause, + hostFailurePath(cause) ?? input.directory, + { output_directory: input.directory }, + ); if (denied !== undefined) return err(denied); if (cause instanceof RegularFileChangedError) return err( @@ -313,32 +366,37 @@ export const exportEnvironmentBundle = async ( ), ); if (cause instanceof ArtifactReaderFailure) { - if (cause.reason === "path" && cause.cleanup === undefined) - return err( - inputError( - { - path: ["directory"], - reason: "invalid_value", - message: cause.message, - }, - cause.message, - ), - ); + const host = hostSystemError(cause); const error = new AnalysisOutputError( ENVIRONMENT_EXPORT_OPERATION, - cause.message, + host === undefined + ? cause.message + : `${cause.message}: ${host.message}`, { cause, - ...(cause.cleanup === undefined ? {} : { cleanup: cause.cleanup }), + // Only a host failure gets host recovery; an integrity or other + // refusal keeps its own message without disk-space advice. + ...(cause.cleanup !== undefined + ? { cleanup: cause.cleanup } + : host === undefined + ? { diagnostics: { output_directory: input.directory } } + : hostWriteFailure(input.directory)), }, ); bindArtifactFileCleanup(cause, error); return err(error); } - if (cause instanceof Error && systemCode(cause) !== undefined) + if ( + cause instanceof BundleFileHostFailure || + (cause instanceof Error && systemCode(cause) !== undefined) + ) return err( new AnalysisOutputError(ENVIRONMENT_EXPORT_OPERATION, cause.message, { cause, + ...(cause instanceof BundleFileHostFailure && + cause.operation === "reading" + ? hostReadFailure(cause.path, input.directory) + : hostWriteFailure(input.directory)), }), ); throw cause; diff --git a/src/application/rust/RustBinaryService.test.ts b/src/application/rust/RustBinaryService.test.ts index 69260975ef..0f85cf424b 100644 --- a/src/application/rust/RustBinaryService.test.ts +++ b/src/application/rust/RustBinaryService.test.ts @@ -4,7 +4,7 @@ import { AnalysisInputError, AnalysisOutputError, } from "../../domain/analysisErrorCore.js"; -import { parseEvidence } from "../../domain/evidence.js"; +import { parseEvidence, type Evidence } from "../../domain/evidence.js"; import type { RustBinary } from "../../domain/rust/rustBinary.js"; import { ok } from "../../domain/result.js"; import { RustBinaryService } from "./RustBinaryService.js"; @@ -59,7 +59,7 @@ it("rejects a relative path and extra arguments before reading", async () => { const relative = await service.inspect({ path: "program" }); const extra = await service.inspect({ path: report.artifact.path, - detail: "complete", + unrecognized: true, }); expect(relative.ok).toBe(false); expect(extra.ok).toBe(false); @@ -221,3 +221,70 @@ it.each([ expect(reason.cleanupIncomplete).toBe(true); }, ); + +it("requires retention and rejects contradictory detail before invoking the reader", async () => { + let calls = 0; + const service = new RustBinaryService({ + identity, + inspect: async () => { + calls++; + return ok(report); + }, + }); + const missing = await service.inspect({ + path: report.artifact.path, + detail: "summary", + }); + expect(missing.ok).toBe(false); + const conflict = await service.inspect({ + path: report.artifact.path, + detail: "complete", + view: { collection: "packages" }, + }); + if (conflict.ok) throw new Error("Contradictory delivery was accepted"); + expect(conflict.error).toBeInstanceOf(AnalysisInputError); + expect(calls).toBe(0); +}); + +it("retains the unchanged parent and keeps delivery selectors out of provider inputs", async () => { + const inputs: unknown[] = []; + const service = new RustBinaryService({ + identity, + inspect: async (input) => { + inputs.push(input); + return ok(report); + }, + }); + const first = await service.inspect({ path: report.artifact.path }); + const explicit = await service.inspect({ + path: report.artifact.path, + detail: "complete", + }); + if (!first.ok || !explicit.ok) throw new Error("Complete delivery failed"); + expect(explicit.value).toEqual(first.value); + const records = new Map(); + const selected = await service.inspect( + { + path: report.artifact.path, + view: { collection: "packages", names: ["sample"], fields: ["name"] }, + }, + undefined, + { + recordEvidence: (evidence) => { + records.set(evidence.evidence_id, evidence); + return ok("added" as const); + }, + evidenceById: (id) => records.get(id), + }, + ); + if (!selected.ok) throw selected.error; + expect(selected.value.normalized_result).toMatchObject({ + kind: "rust", + parent_evidence_id: first.value.evidence_id, + page: { collection: "packages", rows: [{ index: 0, name: "sample" }] }, + }); + expect(records.get(first.value.evidence_id)).toEqual(first.value); + expect(inputs).toEqual( + Array.from({ length: 3 }, () => ({ path: report.artifact.path })), + ); +}); diff --git a/src/application/rust/RustBinaryService.ts b/src/application/rust/RustBinaryService.ts index 0c999958c2..61e5c2d4aa 100644 --- a/src/application/rust/RustBinaryService.ts +++ b/src/application/rust/RustBinaryService.ts @@ -8,30 +8,22 @@ import { AnalysisOutputError, } from "../../domain/analysisErrorCore.js"; import { createEvidence, type Evidence } from "../../domain/evidence.js"; -import { BINARY_ARCHITECTURES } from "../../domain/binaryTargetTypes.js"; import { - inspectRustBinaryInputSchema, + rustArchitectureForSubject, rustBinarySchema, } from "../../domain/rust/rustBinary.js"; +import { inspectRustBinaryRequestSchema } from "../../domain/rust/rustBinaryQuery.js"; import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; import { err, ok, type Result } from "../../domain/result.js"; import { completeProviderEvidence } from "../ProviderOperationExecution.js"; +import { + requireAnalysisViewRetention, + projectRetainedAnalysis, + type AnalysisViewRetention, +} from "../analysisView/AnalysisViewService.js"; const OPERATION = "inspect_rust_binary"; -const architectureForSubject = (architecture: string) => { - switch (architecture) { - case "386": - return "x86"; - case "amd64": - return "x86_64"; - default: - return BINARY_ARCHITECTURES.find( - (supported) => supported === architecture, - ); - } -}; - /** Shared exact input validation and artifact-bound Evidence for CLI and MCP. */ export class RustBinaryService { constructor(readonly provider: RustBinaryPort) {} @@ -44,12 +36,16 @@ export class RustBinaryService { async inspect( rawInput: unknown, options?: ExecutionOptions, + retention: AnalysisViewRetention = { + recordEvidence: undefined, + evidenceById: undefined, + }, ): Promise> { if (options?.signal?.aborted) return err( new AnalysisCancelledError(OPERATION, { cause: options.signal.reason }), ); - const input = inspectRustBinaryInputSchema.safeParse(rawInput); + const input = inspectRustBinaryRequestSchema.safeParse(rawInput); if (!input.success) return err( analysisInputErrorFromIssues(OPERATION, input.error.issues, rawInput, { @@ -66,7 +62,13 @@ export class RustBinaryService { }, ]), ); - const inspected = await this.provider.inspect(input.data, options); + const { detail, view, ...parameters } = input.data; + const compact = detail === "summary" || view !== undefined; + if (compact) { + const ready = requireAnalysisViewRetention(OPERATION, retention); + if (!ready.ok) return ready; + } + const inspected = await this.provider.inspect(parameters, options); if (!inspected.ok) return inspected; const report = rustBinarySchema.safeParse(inspected.value); if (!report.success || report.data.artifact.path !== input.data.path) @@ -76,10 +78,17 @@ export class RustBinaryService { "Rust dependency reader returned malformed metadata or changed the selected artifact identity.", ), ); - return completeProviderEvidence( - evidenceFor(this.provider.identity, input.data, report.data), + const completed = completeProviderEvidence( + evidenceFor(this.provider.identity, parameters, report.data), options?.signal, ); + return compact && completed.ok + ? projectRetainedAnalysis( + completed.value, + retention, + view === undefined ? { kind: "summary" } : { kind: "rust", ...view }, + ) + : completed; } } @@ -88,7 +97,7 @@ const evidenceFor = ( parameters: { readonly path: string }, value: ReturnType, ): Evidence => { - const architecture = architectureForSubject(value.architecture); + const architecture = rustArchitectureForSubject(value.architecture); const section = value.auditable_metadata; return createEvidence( { diff --git a/src/application/sqlite/SqliteDatabaseService.test.ts b/src/application/sqlite/SqliteDatabaseService.test.ts index 4fbd6e5172..7732cacaeb 100644 --- a/src/application/sqlite/SqliteDatabaseService.test.ts +++ b/src/application/sqlite/SqliteDatabaseService.test.ts @@ -4,11 +4,13 @@ import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCo import { err } from "../../domain/result.js"; import { ok } from "../../domain/result.js"; import { createAnalysisExecution } from "../AnalysisProvider.js"; +import { sqliteDatabaseSchema } from "../../domain/sqlite/sqliteDatabase.js"; const fixture = () => ({ artifact: { path: "/selected.db", sha256: "a".repeat(64), bytes: 4096 }, wal: null, engine: { name: "SQLite", version: "3.50.2" }, + worker_timeout_ms: 30_000, database_encoding: "UTF-8", schema: { scope: "inventory", @@ -43,6 +45,10 @@ it.each([ table: "data", row_limit: Number.MAX_SAFE_INTEGER + 1, }, + ...[0, -1, 1.5, 2_147_483_648, "1000", null].map((worker_timeout_ms) => ({ + path: "/selected.db", + worker_timeout_ms, + })), ])( "rejects invalid SQLite selection before provider effects: %j", async (input) => { @@ -58,6 +64,70 @@ it.each([ }, ); +it("preserves the selected worker budget in provider input and Evidence", async () => { + const report = { ...fixture(), worker_timeout_ms: 1234 }; + const service = new SqliteDatabaseService({ + inspect: async (input) => { + expect(input.worker_timeout_ms).toBe(1234); + return ok(execution(report)); + }, + }); + const result = await service.inspect({ + path: "/selected.db", + worker_timeout_ms: 1234, + }); + if (!result.ok) throw result.error; + expect(result.value.parameters).toMatchObject({ worker_timeout_ms: 1234 }); + expect(result.value.normalized_result).toMatchObject({ + worker_timeout_ms: 1234, + }); +}); + +it.each([ + { selected: undefined, reported: undefined }, + { selected: undefined, reported: 1234 }, + { selected: 1234, reported: undefined }, + { selected: 1234, reported: 30_000 }, +])( + "rejects a missing or changed fresh producer budget: %j", + async ({ selected, reported }) => { + const { worker_timeout_ms: _default, ...report } = fixture(); + const service = new SqliteDatabaseService({ + inspect: async () => + ok( + execution({ + ...report, + ...(reported === undefined ? {} : { worker_timeout_ms: reported }), + }), + ), + }); + expect( + await service.inspect({ + path: "/selected.db", + ...(selected === undefined ? {} : { worker_timeout_ms: selected }), + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); + }, +); + +it("retains the actual default without inventing an explicit caller selection", async () => { + const service = new SqliteDatabaseService({ + inspect: async () => ok(execution(fixture())), + }); + const result = await service.inspect({ path: "/selected.db" }); + if (!result.ok) throw result.error; + expect(result.value.normalized_result).toMatchObject({ + worker_timeout_ms: 30_000, + }); + expect(result.value.parameters).not.toHaveProperty("worker_timeout_ms"); +}); + +it("parses historical report omissions as unknown without supplying a default", () => { + const { worker_timeout_ms: _default, ...historical } = fixture(); + const parsed = sqliteDatabaseSchema.parse(historical); + expect(parsed).not.toHaveProperty("worker_timeout_ms"); +}); + it("preserves producer failures and rejects pre-cancelled requests before effects", async () => { const failure = new AnalysisCapabilityUnavailableError( "rea-sqlite", diff --git a/src/application/sqlite/SqliteDatabaseService.ts b/src/application/sqlite/SqliteDatabaseService.ts index 99c31b4bec..1798fb172e 100644 --- a/src/application/sqlite/SqliteDatabaseService.ts +++ b/src/application/sqlite/SqliteDatabaseService.ts @@ -18,6 +18,7 @@ import { type InspectSqliteDatabaseInput, type SqliteDatabase, SQLITE_ROW_LIMIT_DEFAULT, + DEFAULT_SQLITE_WORKER_TIMEOUT_MS, sqliteSchemaScope, sqliteInspectionParameters, } from "../../domain/sqlite/sqliteDatabase.js"; @@ -36,6 +37,9 @@ const matchesSelectedDatabase = ( subject.path === input.path && subject.sha256 === report.artifact.sha256 && subject.format === "file" && + // Fresh executions must report known settings; only historical parsing permits omission. + report.worker_timeout_ms === + (input.worker_timeout_ms ?? DEFAULT_SQLITE_WORKER_TIMEOUT_MS) && (report.wal === null || report.wal.path === `${input.path}-wal`) && report.schema.scope === sqliteSchemaScope(input) && (report.schema.scope !== "selected_table" || @@ -105,7 +109,7 @@ export class SqliteDatabaseService { return err( new AnalysisOutputError( OPERATION, - "SQLite adapter returned malformed data or changed the selected database identity/table/row limit.", + "SQLite adapter returned malformed data or changed the selected database identity/table/row limit/worker budget.", ), ); return ok( diff --git a/src/application/wasm/WasmArtifactService.test.ts b/src/application/wasm/WasmArtifactService.test.ts index afb1e80101..4933fa9168 100644 --- a/src/application/wasm/WasmArtifactService.test.ts +++ b/src/application/wasm/WasmArtifactService.test.ts @@ -10,6 +10,10 @@ it.each([ { path: "/module", glue_paths: ["relative"] }, { path: "/module", candidate_paths: ["relative"] }, { path: "/module", fetch: true }, + ...[0, -1, 1.5, 2_147_483_648, "1000", null].map((command_timeout_ms) => ({ + path: "/module", + command_timeout_ms, + })), ])("rejects invalid selection before effects: %j", async (input) => { const service = new WasmArtifactService({ close: () => Promise.resolve(), @@ -22,6 +26,39 @@ it.each([ error: { _tag: "AnalysisInputError" }, }); }); + +it("preserves the selected command budget in provider input and Evidence", async () => { + const value = wasmArtifactFixture(); + value.tool_profile.timeout_ms = 42_000; + const service = new WasmArtifactService({ + close: () => Promise.resolve(), + inspect: async (input) => { + expect(input).toMatchObject({ command_timeout_ms: 42_000 }); + return ok(wasmArtifactExecution(value)); + }, + }); + const result = await service.inspect({ + path: value.artifact.path, + command_timeout_ms: 42_000, + }); + if (!result.ok) throw result.error; + expect(result.value.parameters).toMatchObject({ command_timeout_ms: 42_000 }); + expect(result.value.normalized_result).toEqual(value); +}); + +it("rejects a provider report that silently changes the selected command budget", async () => { + const value = wasmArtifactFixture(); + const service = new WasmArtifactService({ + close: () => Promise.resolve(), + inspect: async () => ok(wasmArtifactExecution(value)), + }); + expect( + await service.inspect({ + path: value.artifact.path, + command_timeout_ms: 42_000, + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); +}); it.each(["path", "digest", "wat", "candidates", "glue"])( "rejects mismatched upstream observation: %s", async (problem) => { diff --git a/src/application/wasm/WasmArtifactService.ts b/src/application/wasm/WasmArtifactService.ts index 07aed7bd0d..10927acf0e 100644 --- a/src/application/wasm/WasmArtifactService.ts +++ b/src/application/wasm/WasmArtifactService.ts @@ -13,6 +13,7 @@ import { inspectWasmArtifactInputSchema, wasmArtifactSchema, wasmArtifactMatchesSubject, + DEFAULT_WASM_COMMAND_TIMEOUT_MS, } from "../../domain/wasm/wasmArtifact.js"; import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; import { err, type Result } from "../../domain/result.js"; @@ -73,6 +74,8 @@ export class WasmArtifactService { if ( !report.success || report.data.artifact.path !== input.data.path || + report.data.tool_profile.timeout_ms !== + (input.data.command_timeout_ms ?? DEFAULT_WASM_COMMAND_TIMEOUT_MS) || !wasmArtifactMatchesSubject(report.data.artifact, execution.subject) || JSON.stringify(report.data.glue.map((item) => item.artifact.path)) !== JSON.stringify(input.data.glue_paths) || diff --git a/src/artifacts/ArtifactOperationFailure.ts b/src/artifacts/ArtifactOperationFailure.ts index f6630ba9a0..13a7adda0e 100644 --- a/src/artifacts/ArtifactOperationFailure.ts +++ b/src/artifacts/ArtifactOperationFailure.ts @@ -5,6 +5,7 @@ import { AnalysisUnsupportedTargetError, } from "../domain/analysisErrorCore.js"; import { ArtifactOperationError } from "../domain/artifactOperationError.js"; +import { BinaryTargetError } from "../domain/configurationErrors.js"; import { ArtifactReaderFailure } from "./ArtifactReader.js"; import { artifactFileCancellationCause, @@ -36,12 +37,13 @@ export const translateArtifactFailure = ( bindArtifactFileCleanup(cause, error); return error; } - // Caller-selection and unsupported-target failures are already typed; keep - // their correction details instead of reducing them to an I/O failure. + // Caller-selection, target and unsupported-target failures are already typed; + // keep their correction details instead of reducing them to an I/O failure. if ( cause instanceof AnalysisInputError || cause instanceof AnalysisCancelledError || - cause instanceof AnalysisUnsupportedTargetError + cause instanceof AnalysisUnsupportedTargetError || + cause instanceof BinaryTargetError ) return cause; return new ArtifactOperationError(operation, "io"); diff --git a/src/artifacts/ArtifactPaths.test.ts b/src/artifacts/ArtifactPaths.test.ts index 5957bb825e..b57a37b3f1 100644 --- a/src/artifacts/ArtifactPaths.test.ts +++ b/src/artifacts/ArtifactPaths.test.ts @@ -4,13 +4,44 @@ import { ArtifactPathRegistry, annotateEnglishUnicodeCaseCollisions, ENGLISH_UNICODE_CASE_INVENTORY_LIMITATION, + normalizeArtifactPath, } from "./ArtifactPaths.js"; const NOTE = "a case-insensitive destination cannot store both. This comparison is not a filesystem observation."; +it.each(["drive", "escaped", "normalization", "prefix", "collision"])( + "bounds path failure diagnostics at the owning boundary: %s", + (kind) => { + const name = "x".repeat(1024 * 1024); + const registry = new ArtifactPathRegistry(); + const fail = () => { + if (kind === "drive") normalizeArtifactPath("C:" + name); + else if (kind === "escaped") + normalizeArtifactPath("\0".repeat(name.length)); + else if (kind === "normalization") + normalizeArtifactPath(name + "/../child"); + else { + registry.add(name, "file"); + registry.add(kind === "prefix" ? name + "/child" : name, "directory"); + } + }; + const failure = (() => { + try { + fail(); + } catch (cause: unknown) { + return cause; + } + return undefined; + })(); + expect(failure).toMatchObject({ reason: "path" }); + expect((failure as Error).message.length).toBeLessThan(1024); + }, +); + const annotate = (paths: readonly string[]) => { const occurrences = paths.map((logical_path) => ({ + entry_kind: "file" as const, logical_path, limitations: [] as string[], })); diff --git a/src/artifacts/ArtifactPaths.ts b/src/artifacts/ArtifactPaths.ts index 2bc22f7642..f29c1faa9f 100644 --- a/src/artifacts/ArtifactPaths.ts +++ b/src/artifacts/ArtifactPaths.ts @@ -2,10 +2,25 @@ import { posix } from "node:path"; import { logicalPathNfc } from "../domain/artifactIdentity.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; -import { ArtifactReaderFailure } from "./ArtifactReader.js"; +import { ArtifactReaderFailure, type ArtifactEntry } from "./ArtifactReader.js"; const DRIVE_PREFIX = /^[A-Za-z]:/u; +// Bound diagnostic JSON expansion to 768 characters before formatting. This is +// a presentation budget, never a limit on accepted paths or retained facts. +const PATH_DIAGNOSTIC_CODE_UNITS = 128; +const pathDiagnostic = (path: string): string => + path.length <= PATH_DIAGNOSTIC_CODE_UNITS + ? JSON.stringify(path) + : `${JSON.stringify(path.slice(0, PATH_DIAGNOSTIC_CODE_UNITS))}… (${String(path.length)} UTF-16 code units)`; + +type DestinationEntryKind = Exclude; + +/** Declarations have query identities, but never claim a destination path. */ +export const hasDestinationPath = ( + kind: ArtifactEntry["kind"], +): kind is DestinationEntryKind => kind !== "metadata"; + /** * English Unicode case folding used only to describe logical names a * case-insensitive destination might collapse. It is not a filesystem case table. @@ -25,7 +40,7 @@ export const normalizeArtifactPath = (input: string): string => { if (input.includes("\0") || input.startsWith("/") || DRIVE_PREFIX.test(input)) throw new ArtifactReaderFailure( "path", - `Artifact path is absolute or unsafe: ${JSON.stringify(input)}`, + `Artifact path is absolute or unsafe: ${pathDiagnostic(input)}`, ); const normalized = logicalPathNfc(input).replace(/\/+$/u, ""); const parts = normalized.split("/"); @@ -36,7 +51,7 @@ export const normalizeArtifactPath = (input: string): string => { ) throw new ArtifactReaderFailure( "path", - `Artifact path is not normalized: ${JSON.stringify(input)}`, + `Artifact path is not normalized: ${pathDiagnostic(input)}`, ); return normalized; }; @@ -45,14 +60,15 @@ export const normalizeArtifactPath = (input: string): string => { export class ArtifactPathRegistry { readonly #root: PathTrieNode = { kind: undefined, children: new Map() }; - add(path: string, kind: "file" | "directory" | "symlink" | "slice"): void { + add(path: string, kind: ArtifactEntry["kind"]): void { + if (!hasDestinationPath(kind)) return; const parts = path.split("/"); let node = this.#root; for (const [index, part] of parts.entries()) { if (node.kind !== undefined && node.kind !== "directory") throw new ArtifactReaderFailure( "path", - `Artifact prefix conflict: ${path}`, + `Artifact prefix conflict: ${pathDiagnostic(path)}`, ); let child = node.children.get(part); if (child === undefined) { @@ -64,12 +80,12 @@ export class ArtifactPathRegistry { if (node.kind !== undefined && node.kind !== kind) throw new ArtifactReaderFailure( "path", - `Artifact path collision: ${path}`, + `Artifact path collision: ${pathDiagnostic(path)}`, ); if (kind !== "directory" && node.children.size > 0) throw new ArtifactReaderFailure( "path", - `Artifact prefix conflict: ${path}`, + `Artifact prefix conflict: ${pathDiagnostic(path)}`, ); } } @@ -78,7 +94,7 @@ export class ArtifactPathRegistry { } interface PathTrieNode { - kind: "file" | "directory" | "symlink" | "slice" | undefined; + kind: DestinationEntryKind | undefined; readonly children: Map; } @@ -103,15 +119,19 @@ const englishCaseFold = (part: string): string => */ export const annotateEnglishUnicodeCaseCollisions = ( occurrences: readonly { + readonly entry_kind: ArtifactEntry["kind"]; readonly logical_path: string; limitations: string[]; }[], ): string | undefined => { + const destinations = occurrences.filter(({ entry_kind }) => + hasDestinationPath(entry_kind), + ); const collisions = englishUnicodeCaseCollisions( - occurrences.map(({ logical_path }) => logical_path), + destinations.map(({ logical_path }) => logical_path), ); if (collisions.size === 0) return undefined; - for (const occurrence of occurrences) + for (const occurrence of destinations) for (const collision of collisions.get(occurrence.logical_path) ?? []) occurrence.limitations.push( caseCollisionLimitation(occurrence.logical_path, collision), diff --git a/src/artifacts/ArtifactProvider.ts b/src/artifacts/ArtifactProvider.ts index 2a115286bc..bba244b4c5 100644 --- a/src/artifacts/ArtifactProvider.ts +++ b/src/artifacts/ArtifactProvider.ts @@ -1,3 +1,4 @@ +import { artifactTargetRefusal } from "../domain/apple/artifactTargetApplicability.js"; import { bindArtifactFileCleanup } from "./ArtifactFileOperation.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import { translateArtifactFailure } from "./ArtifactOperationFailure.js"; @@ -24,10 +25,7 @@ import { artifactExtractionExecutionSchema, type ArtifactAnalysisOperation, } from "../contracts/artifactToolContracts.js"; -import { - isMachOFormat, - type BinaryTarget, -} from "../domain/binaryTargetTypes.js"; +import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { AnalysisCapabilityUnavailableError, AnalysisInputError, @@ -215,23 +213,17 @@ class ArtifactClient implements AnalysisClient { ), ); try { + const targetRefusal = artifactTargetRefusal(operation, this.target); + if (targetRefusal !== undefined) + throw this.unsupportedTarget(operation, targetRefusal.reason); if (operation === "inspect_artifact") { const inspected = await this.inspectArtifact(parameters, options); return inspected; } if (operation === "decode_interface_builder") { - if ( - this.target.kind !== "executable" || - this.target.sourcePath === undefined || - !this.target.sourcePath.toLowerCase().endsWith(".app") - ) - throw this.unsupportedTarget( - operation, - "decode_interface_builder requires an active .app bundle target", - ); const limits = interfaceBuilderLimitsSchema.parse(parameters); const result = await analyzeInterfaceBuilderBundle({ - bundlePath: this.target.sourcePath, + bundlePath: this.target.sourcePath ?? this.target.path, targetSha256: this.target.sha256, limits, ...(options?.signal === undefined ? {} : { signal: options.signal }), @@ -266,15 +258,7 @@ class ArtifactClient implements AnalysisClient { ]); const bundlePath = standalone ? dirname(this.target.path) - : this.target.sourcePath; - if ( - bundlePath === undefined || - (!standalone && !bundlePath.toLowerCase().endsWith(".app")) - ) - throw this.unsupportedTarget( - operation, - "inspect_keyed_archive requires an active plist or .app bundle", - ); + : (this.target.sourcePath ?? this.target.path); const result = await inspectBundleKeyedArchive({ bundlePath, targetSha256: this.target.sha256, @@ -386,16 +370,7 @@ class ArtifactClient implements AnalysisClient { parameters: Readonly>, options?: ExecutionOptions, ) { - const bundlePath = this.target.sourcePath; - if ( - this.target.kind !== "executable" || - bundlePath === undefined || - !bundlePath.toLowerCase().endsWith(".app") - ) - throw this.unsupportedTarget( - "inspect_asset_catalog", - "inspect_asset_catalog requires an active .app bundle target", - ); + const bundlePath = this.target.sourcePath ?? this.target.path; const result = await analyzeAppleAssetCatalogs({ environment: this.environment, bundlePath, @@ -418,11 +393,6 @@ class ArtifactClient implements AnalysisClient { parameters: Readonly>, options?: ExecutionOptions, ) { - if (this.target.kind !== "executable" || !isMachOFormat(this.target.format)) - throw this.unsupportedTarget( - "trace_dylib_resolution", - "trace_dylib_resolution requires an active Mach-O or .app bundle target", - ); // Only a target opened from an app bundle directory carries its Info.plist; // a regular file whose name ends in .app is a standalone image. const bundle = diff --git a/src/artifacts/ArtifactReader.ts b/src/artifacts/ArtifactReader.ts index 1a2a5a6147..33cb771ea2 100644 --- a/src/artifacts/ArtifactReader.ts +++ b/src/artifacts/ArtifactReader.ts @@ -33,7 +33,7 @@ export interface ArtifactReaderFailureOptions extends ErrorOptions { /** Archive-neutral entry metadata. Reader adapters never choose output paths. */ export interface ArtifactEntry { readonly path: string; - readonly kind: "file" | "directory" | "symlink" | "slice"; + readonly kind: "file" | "directory" | "symlink" | "slice" | "metadata"; readonly declaredSize: number | null; readonly compressedSize: number | null; readonly executable: boolean; @@ -167,6 +167,7 @@ export interface ArtifactReader { | ZipPackageFormat | "asar" | "godot-pck" + | "pyinstaller" | "file"; entries( signal?: AbortSignal, diff --git a/src/artifacts/DesktopApplicationReader.ts b/src/artifacts/DesktopApplicationReader.ts index c9f7ee879e..02d8fcbfa5 100644 --- a/src/artifacts/DesktopApplicationReader.ts +++ b/src/artifacts/DesktopApplicationReader.ts @@ -2,6 +2,7 @@ import { desktopPathConventions } from "../domain/nativeConvention.js"; import { isAbsolute } from "node:path"; import { DirectoryArtifactReader, + DirectoryArtifactRootFailure, DirectoryArtifactTraversalLimit, type DirectoryArtifactEntry, } from "./DirectoryArtifactReader.js"; @@ -33,7 +34,15 @@ import { digestCanonicalValue } from "../domain/canonicalDigest.js"; import { err, ok } from "../domain/result.js"; import { createDesktopApplicationEvidence } from "../domain/desktopApplicationEvidence.js"; import { ARTIFACT_GRAPH_PROVIDER } from "../application/InvestigationProviders.js"; -import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; +import { + AnalysisResourceConstraintError, + AnalysisUnsupportedTargetError, +} from "../domain/analysisErrorCore.js"; +import type { + AnalysisError, + AnalysisPartialObservation, +} from "../domain/analysisErrorBase.js"; +import { BinaryTargetError } from "../domain/configurationErrors.js"; const RETAINED_METADATA_BYTES = 32 * 1024 * 1024; @@ -141,6 +150,8 @@ export class DesktopApplicationReader implements DesktopApplicationPort { throw this.#resources.cleanupFailure(outcome, [closed], partial); if (outcome.kind === "failed") { const cause = outcome.cause; + if (cause instanceof DirectoryArtifactRootFailure) + throw selectedRootFailure(input.path, cause, partial); if (!(cause instanceof ArtifactReaderFailure) || partial === undefined) throw cause; throw new ArtifactReaderFailure( @@ -167,6 +178,33 @@ export class DesktopApplicationReader implements DesktopApplicationPort { } } +/** + * The root is the caller's selection: a non-directory is an unsupported + * target, and host failures keep the shared target-unavailable/access-denied + * classification instead of becoming artifact I/O faults. + */ +const selectedRootFailure = ( + path: string, + failure: DirectoryArtifactRootFailure, + partialObservation: AnalysisPartialObservation | undefined, +): AnalysisError => + failure.reason === "path" + ? new AnalysisUnsupportedTargetError( + "inspect_desktop_application", + path, + failure.message, + { + cause: failure, + partialObservation, + remediationAction: + "Select the application's local installation directory, such as a macOS .app bundle or the directory containing its executable.", + }, + ) + : new BinaryTargetError(path, failure.message, { + cause: failure.cause, + partialObservation, + }); + const scanDirectory = async ( reader: DirectoryArtifactReader, input: InspectDesktopApplicationInput, diff --git a/src/artifacts/DirectoryArtifactReader.ts b/src/artifacts/DirectoryArtifactReader.ts index 6a95213956..764915923a 100644 --- a/src/artifacts/DirectoryArtifactReader.ts +++ b/src/artifacts/DirectoryArtifactReader.ts @@ -60,6 +60,12 @@ export class DirectoryArtifactTraversalLimit extends ArtifactReaderFailure { } } +/** + * The selected root could not be admitted or opened. The consumer that owns + * the caller's selection classifies it; `cause` retains the host failure. + */ +export class DirectoryArtifactRootFailure extends ArtifactReaderFailure {} + /** Non-symlink-following directory reader. */ export class DirectoryArtifactReader implements ArtifactReader { readonly format = "directory" as const; @@ -133,7 +139,7 @@ export class DirectoryArtifactReader implements ArtifactReader { "Artifact root changed during traversal", ); if (!metadata.isDirectory()) - throw new ArtifactReaderFailure( + throw new DirectoryArtifactRootFailure( "path", "Artifact root must be a directory", ); @@ -143,7 +149,12 @@ export class DirectoryArtifactReader implements ArtifactReader { }); })().catch((cause: unknown) => { if (cause instanceof ArtifactReaderFailure) throw cause; - throw directoryIoFailure("resolve artifact root", this.#rootPath, cause); + throw directoryIoFailure( + "resolve artifact root", + this.#rootPath, + cause, + DirectoryArtifactRootFailure, + ); })); } @@ -158,7 +169,14 @@ export class DirectoryArtifactReader implements ArtifactReader { const root = admittedRoot.canonicalPath; const handle = new OwnedDirectoryHandle( await opendir(directory).catch((cause: unknown) => { - throw directoryIoFailure("open directory", directory, cause); + throw directoryIoFailure( + "open directory", + directory, + cause, + directory === root + ? DirectoryArtifactRootFailure + : ArtifactReaderFailure, + ); }), ); this.#directories.add(handle); @@ -484,6 +502,7 @@ const directoryIoFailure = ( operation: string, path: string, cause: unknown, + Failure: typeof ArtifactReaderFailure = ArtifactReaderFailure, ): ArtifactReaderFailure => { if (cause instanceof ArtifactReaderFailure) return cause; const code = @@ -494,7 +513,7 @@ const directoryIoFailure = ( ? ` (${cause.code})` : ""; const detail = failureMessage(cause); - return new ArtifactReaderFailure( + return new Failure( "io", `Could not ${operation} at ${path}${code}: ${detail}`, { cause }, diff --git a/src/artifacts/PyInstallerArtifactReader.ts b/src/artifacts/PyInstallerArtifactReader.ts new file mode 100644 index 0000000000..9f79d5f513 --- /dev/null +++ b/src/artifacts/PyInstallerArtifactReader.ts @@ -0,0 +1,608 @@ +import type { Readable } from "node:stream"; +import { finished } from "node:stream/promises"; +import { setImmediate } from "node:timers/promises"; +import { jsonValueByteLength } from "../domain/jsonValueByteLength.js"; +import type { JsonValue } from "../domain/jsonValue.js"; +import { normalizeArtifactPath } from "./ArtifactPaths.js"; +import { lstat, type FileHandle } from "node:fs/promises"; + +import type { PyInstallerMetadata } from "../domain/artifactFormatMetadata.js"; +import { + cookieMetadata, + scriptPythonMagic, +} from "./pyinstaller/PyInstallerMetadata.js"; +import { artifactExpansionBudgetBytes } from "../domain/artifactExpansionBudget.js"; +import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import { + RegularFileAdmissionFailure, + openRegularFile, + sameRegularFileState, + type StableRegularFileDescriptor, +} from "../filesystem/RegularFile.js"; +import { + ArtifactReaderFailure, + sameArtifactEntry, + type ArtifactEntry, + type ArtifactReader, + type ArtifactReaderTraversal, + type ArtifactTraversalOptions, +} from "./ArtifactReader.js"; +import { abortIfNeeded, hashReadable } from "./ArtifactHash.js"; +import { readArtifactFileChunks } from "./readArtifactFileChunks.js"; +import { ArtifactResourceScope } from "./ArtifactResourceScope.js"; +import { artifactIntegrityDetails } from "../domain/artifactIntegrity.js"; +import { PyInstallerPyzStore } from "./pyinstaller/PyInstallerPyzStore.js"; +import { pyInstallerMemberStream } from "./pyinstaller/PyInstallerMemberStream.js"; +import { + findPyInstallerCookie, + readCArchiveToc, + type CArchiveEntry, + type PyInstallerCookie, +} from "./pyinstaller/CArchiveLayout.js"; +import { + hasEncryptionMarker, + projectPyzContainer, + projectPyzMember, + projectStoredMember, + sealMember, + type PyInstallerMember, + type PyzImage, +} from "./pyinstaller/CArchiveMembers.js"; +import { readPyzArchive } from "./pyinstaller/PyzArchive.js"; +import type { PyInstallerFailure } from "./pyinstaller/PyInstallerFailure.js"; +import { pyInstallerFileAdmissionFailure } from "./pyinstaller/PyInstallerFileAdmission.js"; + +const MAX_PARSED_FACT_BYTES = 32 * 1024 * 1024; +const MAX_RETAINED_ENTRY_BYTES = 64 * 1024 * 1024; +const RECORD_OWNERSHIP_RESERVATION = 256; +// Conservative allowance for registry/case trie nodes, maps and traversal state. +// Repeated prefixes are charged again rather than underestimating retained work. +const PATH_COMPONENT_RESERVATION = 1024; + +interface LoadedArchive { + readonly members: readonly PyInstallerMember[]; + readonly traversal: ArtifactReaderTraversal; +} + +/** + * Read-only inventory of a PyInstaller CArchive appended to a native executable. + * Encrypted PYZ entries are reported and not decoded. Code members are not decompiled. + */ +export class PyInstallerArtifactReader implements ArtifactReader { + readonly format = "pyinstaller" as const; + #source: StableRegularFileDescriptor | undefined; + #opening: Promise | undefined; + #closed = false; + readonly #scope = new ArtifactResourceScope(); + readonly #cleanup = new ArtifactResourceScope(); + readonly #lifecycle = new AbortController(); + #boundSha256: string | undefined; + #loaded: Promise | undefined; + #budget = 0; + #charged = 0; + #parsedFactBytes = 0; + #retainedEntryBytes = 0; + #metadata: PyInstallerMetadata | undefined; + readonly #pyzStore = new PyInstallerPyzStore(); + readonly #members = new Map(); + readonly #openedPyzSizes = new Map(); + + constructor( + private readonly path: string, + source?: StableRegularFileDescriptor, + ) { + this.#source = source; + this.#cleanup.retain({ + kind: "lease", + resource: "PyInstaller private PYZ images", + lease: { release: () => this.#pyzStore.close() }, + }); + } + + formatMetadata(): PyInstallerMetadata | undefined { + return this.#metadata === undefined ? undefined : { ...this.#metadata }; + } + + async prepareContainer( + expectedSha256: string, + signal?: AbortSignal, + ): Promise { + const active = this.#signal(signal); + await this.#scope.run(async () => { + const handle = await this.#openHandle(active); + await this.#verifySource(); + const observed = + this.#boundSha256 ?? + ( + await hashReadable( + readArtifactFileChunks(handle, { signal: active }), + active, + ) + ).sha256; + await this.#verifySource(); + abortIfNeeded(active); + if (observed !== expectedSha256) + throw new ArtifactReaderFailure( + "integrity", + "PyInstaller carrier changed after inventory", + undefined, + artifactIntegrityDetails( + ".", + { algorithm: "sha256", digest: expectedSha256 }, + observed, + false, + ), + ); + this.#boundSha256 = observed; + }); + } + + async *entries( + signal?: AbortSignal, + options?: ArtifactTraversalOptions, + ): AsyncIterable { + const active = this.#signal(signal); + const loaded = await this.#scope.run(async () => { + const value = await this.#load(active); + await this.#verifySource(); + this.#members.clear(); + return value; + }); + options?.onCoverage?.(loaded.traversal); + for (const member of loaded.members) { + abortIfNeeded(active); + const sealed = sealMember(member); + this.#members.set(sealed.entry.adapterKey, sealed); + yield sealed.entry; + } + } + + async open(entry: ArtifactEntry, signal?: AbortSignal): Promise { + const active = this.#signal(signal); + return this.#scope.run(async () => { + const stored = this.#members.get(entry.adapterKey); + if (stored === undefined || !sameArtifactEntry(entry, stored.entry)) + throw new ArtifactReaderFailure( + "integrity", + `PyInstaller entry was not produced by this reader: ${entry.path}`, + ); + if (entry.kind !== "file") + throw new ArtifactReaderFailure( + "format", + `PyInstaller entry is not a regular file: ${entry.path}`, + ); + if (entry.encrypted) + throw new ArtifactReaderFailure( + "unavailable", + `PYZ entry is encrypted and was not decoded: ${entry.path}`, + ); + const read = stored.read; + if (read === undefined) + throw new ArtifactReaderFailure( + "format", + `PyInstaller entry has no readable bytes: ${entry.path}`, + ); + await this.#verifySource(); + const knownSize = + read.kind === "pyz" + ? (read.uncompressedSize ?? + this.#openedPyzSizes.get(entry.adapterKey)) + : undefined; + if (read.kind === "pyz" && knownSize === undefined) + this.#charge(read.header?.length ?? 0); + const parts = await this.#pyzStore.memberSource( + read, + this.#source?.handle, + active, + ); + if (active.aborted) { + parts.source.destroy(); + await finished(parts.source, { cleanup: true }).catch(() => undefined); + abortIfNeeded(active); + } + // Keep one close attempt in this read. Failed owners stay in the store + // for reader.close() to retry, without losing the original failure. + let closeAttempt: Promise | undefined; + const closeSource = (primary?: unknown) => + (closeAttempt ??= parts.close?.(primary)); + const stream = pyInstallerMemberStream(parts.source, { + sourceBytes: + read.kind === "carchive" ? read.source.compressedSize : read.length, + compressed: + read.kind === "pyz" + ? read.compressed + : read.source.compressionFlag === 1, + header: read.header, + expectedBytes: + read.kind === "carchive" ? read.source.uncompressedSize : knownSize, + maxBytes: + read.kind === "carchive" + ? read.source.uncompressedSize + : (knownSize ?? this.#budget - this.#charged), + signal: active, + ...(read.kind === "pyz" && knownSize === undefined + ? { charge: (bytes: number) => this.#charge(bytes) } + : {}), + complete: async (bytes) => { + try { + await parts.verify?.(); + await this.#verifySource(); + abortIfNeeded(active); + } catch (cause: unknown) { + await closeSource(cause); + throw cause; + } + await closeSource(); + abortIfNeeded(active); + if (read.kind === "pyz" && knownSize === undefined) + this.#openedPyzSizes.set(entry.adapterKey, bytes); + }, + }); + void this.#scope + .run(async () => { + let primary: unknown; + try { + await stream.completion; + } catch (cause: unknown) { + primary = cause; + } + await closeSource(primary); + if (primary !== undefined) throw primary; + }) + .catch((cause: unknown) => + stream.output.destroy( + cause instanceof Error + ? cause + : new Error("PyInstaller stream failed", { cause }), + ), + ); + return stream.output; + }); + } + + provenance(): readonly [] { + return []; + } + + async close(): Promise { + this.#closed = true; + const closing = new ArtifactReaderFailure( + "unavailable", + "PyInstaller reader closed", + ); + this.#lifecycle.abort(closing); + await this.#scope.close(); + await this.#cleanup.close(); + this.#source = undefined; + this.#closed = true; + this.#loaded = undefined; + this.#members.clear(); + this.#openedPyzSizes.clear(); + } + + #signal(signal?: AbortSignal): AbortSignal { + abortIfNeeded(signal); + if (this.#closed) + throw new ArtifactReaderFailure( + "unavailable", + "PyInstaller archive is closed", + ); + return signal === undefined + ? this.#lifecycle.signal + : AbortSignal.any([signal, this.#lifecycle.signal]); + } + + async #load(signal?: AbortSignal): Promise { + this.#loaded ??= this.#parse(signal); + return this.#loaded; + } + + async #parse(signal?: AbortSignal): Promise { + const handle = await this.#openHandle(signal); + await this.#verifySource(); + const size = (await handle.stat()).size; + const cookie = await this.#cookie(handle, size, signal); + this.#budget = artifactExpansionBudgetBytes(cookie.packageLength); + const toc = await this.#toc(handle, cookie, signal); + for (const entry of toc) { + this.#charge(entry.uncompressedSize); + this.#reserveMetadata({ ...entry }, false); + } + const images = await this.#pyzImages(handle, cookie, toc, signal); + const magic = scriptPythonMagic( + cookie, + [...images.values()].map((image) => image.archive), + ); + const encrypted = hasEncryptionMarker(toc); + const members = await this.#project( + toc, + cookie, + images, + magic, + encrypted, + signal, + ); + for (const member of members) + if (member.read?.kind === "carchive") + this.#charge(member.read.header?.length ?? 0); + await this.#verifySource(); + this.#metadata = cookieMetadata(cookie, toc, encrypted); + let opaqueImages = 0; + for (const image of images.values()) + if (image.archive.kind === "opaque") opaqueImages++; + return { + members, + traversal: { + status: opaqueImages === 0 ? "complete" : "unknown", + limitations: + opaqueImages === 0 + ? [] + : [ + `${String(opaqueImages)} PYZ archives remain opaque; module traversal is unknown. See opaque PYZ occurrences for source magic and per-archive limitations.`, + ], + }, + }; + } + + async #openHandle(signal?: AbortSignal): Promise { + abortIfNeeded(signal); + if (this.#closed) + throw new ArtifactReaderFailure( + "unavailable", + "PyInstaller archive is closed", + ); + if (this.#source !== undefined) return this.#source.handle; + // Admission belongs to the reader. A departing caller cannot cancel a peer + // or replace a late descriptor owner; every caller joins the same admission. + this.#opening ??= this.#admit(); + let handle: FileHandle; + try { + handle = await this.#opening; + } catch (cause: unknown) { + if ( + cause instanceof ArtifactReaderFailure && + cause.reason === "cancelled" && + signal?.aborted + ) + throw new ArtifactReaderFailure("cancelled", cause.message, { + cause: signal.reason, + cleanup: cause.cleanup, + cleanupLeases: cause.cleanupLeases, + }); + throw cause; + } + abortIfNeeded(signal); + return handle; + } + + async #admit(): Promise { + try { + const handle = await openRegularFile(this.path, { + symlinks: "reject", + signal: this.#lifecycle.signal, + }); + this.#retainCarrier(new OwnedFileHandle(handle)); + this.#source = { handle, initial: await handle.stat() }; + return handle; + } catch (cause: unknown) { + if (cause instanceof RegularFileAdmissionFailure) + this.#retainCarrier(cause.owner); + throw pyInstallerFileAdmissionFailure(this.path, cause); + } + } + + #retainCarrier(handle: OwnedFileHandle): void { + this.#cleanup.retain({ kind: "file-handle", resource: this.path, handle }); + } + + async #cookie( + handle: FileHandle, + size: number, + signal?: AbortSignal, + ): Promise { + const found = await findPyInstallerCookie(handle, size, signal); + if (found === undefined) + throw new ArtifactReaderFailure( + "format", + "PyInstaller CArchive cookie was not found", + ); + if (!found.ok) throw failure(found.error); + return found.value; + } + + async #toc( + handle: FileHandle, + cookie: PyInstallerCookie, + signal?: AbortSignal, + ): Promise { + const toc = await readCArchiveToc(handle, cookie, signal); + if (!toc.ok) throw failure(toc.error); + return toc.value; + } + + async #pyzImages( + handle: FileHandle, + cookie: PyInstallerCookie, + toc: readonly CArchiveEntry[], + signal: AbortSignal | undefined, + ): Promise> { + const images = new Map(); + // Admit the complete filesystem footprint before the first private write. + this.#pyzStore.reserveImages( + toc.reduce( + (count, entry) => + count + + (entry.typeCode === "z" && entry.compressionFlag === 1 ? 1 : 0), + 0, + ), + ); + for (const entry of toc) { + if (entry.typeCode !== "z") continue; + abortIfNeeded(signal); + const spooled = + entry.compressionFlag === 0 + ? undefined + : await this.#pyzStore.spool( + handle, + cookie.archiveStart, + entry, + signal, + ); + const source = spooled ?? { + handle, + start: cookie.archiveStart + entry.offset, + }; + const parsed = await readPyzArchive( + source.handle, + source.start, + entry.uncompressedSize, + signal, + ); + if (!parsed.ok) throw failure(parsed.error); + if (parsed.value.kind === "entries") + for (const item of parsed.value.entries) + this.#reserveMetadata({ ...item, container: entry.name }, false); + if (spooled !== undefined) await this.#pyzStore.release(spooled.handle); + images.set(entry, { + archive: parsed.value, + handle: source.handle, + start: source.start, + }); + } + return images; + } + + async #project( + toc: readonly CArchiveEntry[], + cookie: PyInstallerCookie, + images: ReadonlyMap, + magic: Buffer | undefined, + encrypted: boolean, + signal: AbortSignal | undefined, + ): Promise { + const members: PyInstallerMember[] = []; + const append = (member: PyInstallerMember): void => { + this.#reservePathComponents(member.entry.path); + const entry = { + ...member.entry, + path: normalizeArtifactPath(member.entry.path), + }; + this.#reserveMetadata( + { + path: entry.path, + adapter_key: entry.adapterKey, + format_metadata: entry.formatMetadata ?? null, + limitations: [...entry.limitations], + }, + true, + ); + members.push({ ...member, entry }); + }; + for (const entry of toc) { + if (members.length % 256 === 0) await setImmediate(); + abortIfNeeded(signal); + const image = entry.typeCode === "z" ? images.get(entry) : undefined; + if (image === undefined) { + append(projectStoredMember(entry, cookie.archiveStart, magic)); + continue; + } + append(projectPyzContainer(entry, cookie.archiveStart, image, encrypted)); + if (image.archive.kind !== "entries") continue; + for (const item of image.archive.entries) { + if (members.length % 256 === 0) await setImmediate(); + abortIfNeeded(signal); + append( + projectPyzMember( + entry, + item, + encrypted, + image.archive.pythonMagic, + item.namespace || encrypted + ? undefined + : { + handle: image.handle, + offset: image.start + item.offset, + length: item.length, + }, + ), + ); + } + } + return members; + } + + async #verifySource(): Promise { + const source = this.#source; + if (source === undefined) + throw new ArtifactReaderFailure( + "unavailable", + "PyInstaller archive is closed", + ); + const [opened, currentPath] = await Promise.all([ + source.handle.stat(), + lstat(this.path), + ]); + if ( + !sameRegularFileState(source.initial, opened) || + !sameRegularFileState(source.initial, currentPath) + ) + throw new ArtifactReaderFailure( + "integrity", + `PyInstaller source changed during reading: ${this.path}`, + ); + } + + #reservePathComponents(path: string): void { + const remaining = MAX_RETAINED_ENTRY_BYTES - this.#retainedEntryBytes; + // Case annotations can carry the path and two prefixes at every component. + // Reserve escaped text growth as well as registry/case trie nodes and maps. + const componentCost = + PATH_COMPONENT_RESERVATION + 6 * jsonValueByteLength(path); + let reservation = componentCost; + for ( + let index = 0; + index < path.length && reservation <= remaining; + index++ + ) + if (path.charCodeAt(index) === 47) reservation += componentCost; + if (reservation > remaining) + throw new ArtifactReaderFailure( + "limit", + "PyInstaller paths exceed their retained namespace budget", + { boundary: "memory" }, + ); + this.#retainedEntryBytes += reservation; + } + + #reserveMetadata(value: JsonValue, projected: boolean): void { + const used = projected ? this.#retainedEntryBytes : this.#parsedFactBytes; + const limit = projected ? MAX_RETAINED_ENTRY_BYTES : MAX_PARSED_FACT_BYTES; + const remaining = limit - used - RECORD_OWNERSHIP_RESERVATION; + const bytes = remaining < 0 ? 0 : jsonValueByteLength(value, remaining); + if (remaining < 0 || bytes > remaining) + throw new ArtifactReaderFailure( + "limit", + "PyInstaller directory facts exceed their retained metadata budget", + { boundary: "memory" }, + ); + if (projected) + this.#retainedEntryBytes += bytes + RECORD_OWNERSHIP_RESERVATION; + else this.#parsedFactBytes += bytes + RECORD_OWNERSHIP_RESERVATION; + } + + #charge(bytes: number): void { + if ( + !Number.isSafeInteger(bytes) || + bytes < 0 || + this.#charged + bytes > this.#budget + ) + throw new ArtifactReaderFailure( + "limit", + `PyInstaller members declare ${String(this.#charged + Math.max(0, bytes))} uncompressed bytes, exceeding the decompression budget`, + ); + this.#charged += bytes; + } +} + +const failure = (error: PyInstallerFailure): ArtifactReaderFailure => + new ArtifactReaderFailure(error.reason, error.message); diff --git a/src/artifacts/apple/InterfaceBuilderAnalysis.ts b/src/artifacts/apple/InterfaceBuilderAnalysis.ts index 26efffab26..bcf6820146 100644 --- a/src/artifacts/apple/InterfaceBuilderAnalysis.ts +++ b/src/artifacts/apple/InterfaceBuilderAnalysis.ts @@ -10,6 +10,7 @@ import { import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js"; import { projectPlistValue, + binaryDatePrecisionOverlapLimitation, unknownDateLimitation, unknownScalarLimitation, } from "../../domain/apple/plistValue.js"; @@ -18,8 +19,9 @@ import { parseXmlPropertyList, } from "../../domain/propertyListKeys.js"; import { decodeNibArchive, type NibArchiveDocument } from "./NibArchive.js"; +import { preflightPropertyListDecode } from "./PropertyListDecodeBudget.js"; import { - preflightPropertyListDecode, + interfaceBuilderDecodeBudgetFailure, InterfaceBuilderDecodeBudget, InterfaceBuilderDecodeBudgetExceeded, } from "./InterfaceBuilderDecodeBudget.js"; @@ -39,9 +41,10 @@ import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js" import { decodeXmlPlistText } from "../../domain/propertyListXmlText.js"; import { - decodeBinaryPlistNumbers, - scanBinaryPlistNumberLiterals, -} from "./PlistNumberLiterals.js"; + decodeBinaryPlistScalars, + scanBinaryPlistScalars, + type BinaryPlistScalarDecodePlan, +} from "./PlistScalars.js"; import type { PlistNumberLiterals } from "../../domain/apple/plistNumbers.js"; import { hasBinaryPlistSignature, @@ -70,6 +73,7 @@ export const analyzeInterfaceBuilderBundle = async (input: { const unresolvedConnections = new Map(); const prototypeKeyOmissions = new Map(); const unknownDates = new Map(); + const datePrecisionOverlaps = new Map(); const unknownScalars = new Map(); let omitted = 0; let attempted = 0; @@ -118,6 +122,14 @@ export const analyzeInterfaceBuilderBundle = async (input: { let nib: ReturnType | null = null; let xmlText: string | undefined; let binaryPrototypeKeyObserved = false; + let scalarPlan: BinaryPlistScalarDecodePlan = { + reachableDateOffsets: [], + overlappingDateOffsets: [], + reachableWideIntegerOffsets: [], + overlappingWideIntegerOffsets: [], + reachableNumericOffsets: [], + }; + let datePrecisionOverlapCount = 0; if (isNibArchive) { const documentBudget = new InterfaceBuilderDecodeBudget( decodeBudget.remainingBytes, @@ -138,7 +150,11 @@ export const analyzeInterfaceBuilderBundle = async (input: { bytes, decodeBudget.remainingBytes, xmlText, - { numberProjection: true }, + { + numberProjection: true, + budget: "the aggregate Interface Builder decode budget", + fail: interfaceBuilderDecodeBudgetFailure, + }, ); decodeBudget.reserve( decodedReservation.estimatedBytes, @@ -147,6 +163,9 @@ export const analyzeInterfaceBuilderBundle = async (input: { plistReservation = decodedReservation.estimatedBytes; binaryPrototypeKeyObserved = decodedReservation.binaryPrototypeKeyObserved; + scalarPlan = decodedReservation; + datePrecisionOverlapCount = + decodedReservation.datePrecisionOverlapCount; } const { value: raw, @@ -155,7 +174,7 @@ export const analyzeInterfaceBuilderBundle = async (input: { unknownScalarCount, numberLiterals, } = nib === null - ? decodePlist(bytes, xmlText, binaryPrototypeKeyObserved) + ? decodePlist(bytes, xmlText, binaryPrototypeKeyObserved, scalarPlan) : { value: nib.raw, omittedPrototypeKeys: 0, @@ -171,6 +190,8 @@ export const analyzeInterfaceBuilderBundle = async (input: { prototypeKeyOmissions.set(entry.path, omittedPrototypeKeys); if (unknownDateCount !== 0) unknownDates.set(entry.path, unknownDateCount); + if (datePrecisionOverlapCount > 0) + datePrecisionOverlaps.set(entry.path, datePrecisionOverlapCount); if (unknownScalarCount !== 0) unknownScalars.set(entry.path, unknownScalarCount); const documentHash = createHash("sha256").update(bytes).digest("hex"); @@ -218,6 +239,7 @@ export const analyzeInterfaceBuilderBundle = async (input: { unresolvedConnections, prototypeKeyOmissions, unknownDates, + datePrecisionOverlaps, unknownScalars, invalid, omittedArchives, @@ -288,6 +310,7 @@ const finalizeHierarchyCoverage = ( unresolvedConnections: ReadonlyMap; prototypeKeyOmissions: ReadonlyMap; unknownDates: ReadonlyMap; + datePrecisionOverlaps: ReadonlyMap; unknownScalars: ReadonlyMap; invalid: readonly string[]; omittedArchives: readonly { path: string; reason: string }[]; @@ -301,6 +324,7 @@ const finalizeHierarchyCoverage = ( unresolvedConnections, prototypeKeyOmissions, unknownDates, + datePrecisionOverlaps, unknownScalars, invalid, omittedArchives, @@ -390,6 +414,9 @@ const finalizeHierarchyCoverage = ( ...[...unknownDates].map( ([path, count]) => `${path}: ${unknownDateLimitation(count)}`, ), + ...[...datePrecisionOverlaps].map(([path, count]) => { + return `${path}: ${binaryDatePrecisionOverlapLimitation(count)}`; + }), ...[...unknownScalars].map( ([path, count]) => `${path}: ${unknownScalarLimitation(count)}`, ), @@ -759,6 +786,7 @@ const decodePlist = ( bytes: Buffer, xmlText: string | undefined, binaryPrototypeKeyObserved: boolean, + scalarPlan: BinaryPlistScalarDecodePlan, ): { readonly value: JsonValue; readonly omittedPrototypeKeys: number | null; @@ -767,16 +795,16 @@ const decodePlist = ( readonly numberLiterals: PlistNumberLiterals; } => { const binary = hasBinaryPlistSignature(bytes); - const binaryNumberMetadata = binary - ? scanBinaryPlistNumberLiterals(bytes) + const binaryScalarMetadata = binary + ? scanBinaryPlistScalars(bytes, scalarPlan) : undefined; const { value, omittedPrototypeKeys, numberLiterals } = binary ? { value: parseBinary( - decodeBinaryPlistNumbers(bytes, binaryNumberMetadata!), + decodeBinaryPlistScalars(bytes, binaryScalarMetadata!, scalarPlan), ), omittedPrototypeKeys: binaryPrototypeKeyObserved ? null : 0, - numberLiterals: binaryNumberMetadata!.literals, + numberLiterals: binaryScalarMetadata!.literals, } : parseXmlWithinDecodeBudget(xmlText ?? decodeXmlPlistText(bytes)); const projected = projectPlistValue(value); diff --git a/src/artifacts/apple/InterfaceBuilderDecodeBudget.test.ts b/src/artifacts/apple/InterfaceBuilderDecodeBudget.test.ts index 61492d69d3..dd87e23353 100644 --- a/src/artifacts/apple/InterfaceBuilderDecodeBudget.test.ts +++ b/src/artifacts/apple/InterfaceBuilderDecodeBudget.test.ts @@ -6,9 +6,14 @@ import { buildBinary } from "plist"; import { describe, expect, it } from "vitest"; import { analyzeInterfaceBuilderBundle } from "./InterfaceBuilderAnalysis.js"; -import { preflightPropertyListDecode } from "./InterfaceBuilderDecodeBudget.js"; +import { interfaceBuilderDecodeBudgetFailure } from "./InterfaceBuilderDecodeBudget.js"; +import { preflightPropertyListDecode } from "./PropertyListDecodeBudget.js"; +import { unknownDateLimitation } from "../../domain/apple/plistValue.js"; import { parseXmlPropertyList } from "../../domain/propertyListKeys.js"; import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import { scanBinaryPlistScalars } from "./PlistScalars.js"; +import { binaryPlistWithOverlappingDateAndData } from "../../../tests/fixtures/binaryPlistObjectTable.js"; +import { embedSubmillisecondDateInWideIntegerPayload } from "../../../tests/fixtures/binaryPlistScalarAlias.js"; const example = ""; @@ -33,8 +38,12 @@ it.each([ undefined, { numberProjection: true }, ).estimatedBytes; - if (_format === "binary") - expect(projected).toBeGreaterThanOrEqual(ordinary + bytes.length); + if (_format === "binary") { + const objectCount = Number(bytes.readBigUInt64BE(bytes.length - 24)); + expect(projected).toBeGreaterThanOrEqual( + ordinary + bytes.length + objectCount * 96, + ); + } expect(() => preflightPropertyListDecode(bytes, ordinary, undefined, { numberProjection: true, @@ -112,7 +121,15 @@ describe("inert XML declaration examples in Interface Builder archives", () => { ']> - preflightPropertyListDecode(Buffer.from(xml), 16 * 1024 * 1024), + preflightPropertyListDecode( + Buffer.from(xml), + 16 * 1024 * 1024, + undefined, + { + budget: "the aggregate Interface Builder decode budget", + fail: interfaceBuilderDecodeBudgetFailure, + }, + ), ).toThrow( "XML plist internal entity expansion exceeds the aggregate Interface Builder decode budget", ); @@ -141,3 +158,107 @@ describe("inert XML declaration examples in Interface Builder archives", () => { ).toMatchObject({ binaryPrototypeKeyObserved: false }); }); }); + +it("retains explicit date uncertainty for an overlapping binary plist object", async () => { + const root = await createTestTempDirectory("rea-ib-binary-date-overlap-"); + const bundle = join(root, "Example.app"); + await mkdir(bundle); + const bytes = binaryPlistWithOverlappingDateAndData(); + await writeFile(join(bundle, "Panel.nib"), bytes); + + const result = await analyzeInterfaceBuilderBundle({ + bundlePath: bundle, + targetSha256: "a".repeat(64), + }); + + expect(result.limitations).toContainEqual( + expect.stringContaining( + "Panel.nib: 1 date value(s) are reported with unknown timestamps because their sub-millisecond source records overlap", + ), + ); +}); + +it("preserves aliased wide-integer precision and marks the date unknown in bundle output", async () => { + const root = await createTestTempDirectory("rea-ib-binary-scalar-alias-"); + const bundle = join(root, "Example.app"); + await mkdir(bundle); + const source = Buffer.from( + buildBinary({ + "com.apple.ibtool.document.objects": { + button: { + customClass: "UIButton", + wide: Buffer.alloc(16), + date: new Date(0), + }, + }, + "com.apple.ibtool.document.connections": {}, + "com.apple.ibtool.document.hierarchy": [], + "com.apple.ibtool.document.classes": {}, + }), + ); + const { bytes, integerDecimal } = + embedSubmillisecondDateInWideIntegerPayload(source); + await writeFile(join(bundle, "Panel.nib"), bytes); + + const result = await analyzeInterfaceBuilderBundle({ + bundlePath: bundle, + targetSha256: "a".repeat(64), + }); + + const button = result.graph.nodes.find( + ({ attributes }) => attributes.interface_builder_object_id === "button", + ); + expect(button?.attributes).toMatchObject({ + date: { $plist_type: "date", iso8601: null }, + wide: { $plist_type: "integer", decimal: integerDecimal }, + }); +}); + +describe("binary Interface Builder plist dates", () => { + const control = Buffer.from( + "62706c6973743030d4010203040506090c5924617263686976657258246f626a656374735424746f70582476657273696f6e5f100f4e534b657965644172636869766572a2070855246e756c6c333fb999999999999ad10a0b54726f6f74800112000186a008111b24293244474d56595e600000000000000101000000000000000d00000000000000000000000000000065", + "hex", + ); + const marker = Buffer.alloc(9); + marker[0] = 0x33; + marker.writeDoubleBE(0.1, 1); + + const nib = async (bytes: Buffer, name: string) => { + const root = await createTestTempDirectory(`rea-ib-date-${name}-`); + const bundle = join(root, "Example.app"); + await mkdir(bundle); + await writeFile(join(bundle, "Panel.nib"), bytes); + return analyzeInterfaceBuilderBundle({ + bundlePath: bundle, + targetSha256: "a".repeat(64), + }); + }; + + it("refuses an invalid reachable date marker without projecting a timestamp", async () => { + const bytes = Buffer.from(control); + const offset = bytes.indexOf(marker); + bytes[offset] = 0x30; + const result = await nib(bytes, "marker"); + expect(JSON.stringify(result)).not.toContain("2001-01-01T00:00:00.100Z"); + expect(result.limitations.join("\n")).toContain( + "binary plist date marker 0x30 is invalid", + ); + }); + + it("counts a sub-millisecond date as unknown and keeps the canonical control known", async () => { + const submillisecond = Buffer.from(control); + const offset = submillisecond.indexOf(marker); + submillisecond.writeDoubleBE(0.1005, offset + 1); + const unknown = await nib(submillisecond, "subms"); + expect(unknown.limitations).toContain( + `Panel.nib: ${unknownDateLimitation(1)}`, + ); + expect(JSON.stringify(unknown)).not.toContain("2001-01-01T00:00:00.100Z"); + + const known = await nib(Buffer.from(control), "known"); + expect(known.limitations.join("\n")).not.toContain("date value(s)"); + expect(known.limitations.join("\n")).not.toContain( + "binary plist date marker", + ); + }); +}); diff --git a/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts b/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts index fe813e8a98..11aad7169c 100644 --- a/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts +++ b/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts @@ -1,6 +1,3 @@ -import { decodeXmlPlistText } from "../../domain/propertyListXmlText.js"; -import { hasBinaryPlistSignature } from "../../domain/apple/appleArchiveFormat.js"; - /** Reason reported when an archive would exceed the shared decoding budget. */ export type InterfaceBuilderBudgetReason = "aggregate_decode_budget_exhausted"; @@ -15,6 +12,16 @@ export class InterfaceBuilderDecodeBudgetExceeded extends Error { } } +/** Map shared plist preflight limits to Interface Builder's public reason. */ +export const interfaceBuilderDecodeBudgetFailure = ( + _kind: "cycle" | "depth" | "expansion", + message: string, +): Error => + new InterfaceBuilderDecodeBudgetExceeded( + "aggregate_decode_budget_exhausted", + message, + ); + /** Shared byte budget for decoder records and projected archive data. */ export class InterfaceBuilderDecodeBudget { readonly #initialBytes: number; @@ -96,311 +103,3 @@ export class InterfaceBuilderDecodeBudget { return true; } } - -/** Preflight the representation created by the pinned plist decoder. */ -export const preflightPropertyListDecode = ( - bytes: Buffer, - maxBytes: number, - xmlText?: string, - options: Pick = {}, -): PlistDecodePreflight => { - const { numberProjection = false } = options; - const expansion = hasBinaryPlistSignature(bytes) - ? estimateBinaryPlistExpansion(bytes, maxBytes, { numberProjection }) - : { - estimatedBytes: estimateXmlPlistExpansion( - bytes, - maxBytes, - xmlText, - numberProjection, - ), - binaryPrototypeKeyObserved: false, - }; - if (expansion.estimatedBytes > maxBytes) - throw new InterfaceBuilderDecodeBudgetExceeded( - "aggregate_decode_budget_exhausted", - "plist representation exceeds the aggregate Interface Builder decode budget", - ); - return expansion; -}; - -const estimateXmlPlistExpansion = ( - bytes: Buffer, - maxBytes: number, - xmlText?: string, - numberProjection = false, -): number => { - const xml = xmlText ?? decodeXmlPlistText(bytes); - const decodedSourceBytes = xml.length * 2; - let elementCount = 0; - // Reserve room for numeric source metadata and the projected attribute copy. - const elementBytes = numberProjection ? 256 : 128; - // Value nesting is bounded by the shared plist parser, which counts plist - // value depth; raw element depth would also count the envelope and keys. - for (const token of xmlPreflightTokens(xml)) { - if (token.doctype) { - // Only a declaration outside opaque text can introduce an internal - // subset. Brackets in quoted external identifiers are literal text. - if (token.internalSubset) - throw new InterfaceBuilderDecodeBudgetExceeded( - "aggregate_decode_budget_exhausted", - "XML plist internal entity expansion exceeds the aggregate Interface Builder decode budget", - ); - continue; - } - elementCount += 1; - if (decodedSourceBytes * 2 + elementCount * elementBytes > maxBytes) - return maxBytes + 1; - } - return decodedSourceBytes * 2 + elementCount * elementBytes; -}; - -/** Consume each XML span once; malformed tokens cannot restart suffix scans. */ -function* xmlPreflightTokens(xml: string) { - let cursor = 0; - while (cursor < xml.length) { - const start = xml.indexOf("<", cursor); - if (start < 0) return; - const opaqueEnd = xml.startsWith("" - : xml.startsWith("" - : xml.startsWith("" - : undefined; - if (opaqueEnd !== undefined) { - const end = xml.indexOf(opaqueEnd, start + 2); - if (end < 0) return; - cursor = end + opaqueEnd.length; - continue; - } - const doctype = /^") break; - else if (doctype && character === "[") internalSubset = true; - } - if (end === xml.length) return; - cursor = end + 1; - if (doctype || element) yield { doctype, internalSubset }; - } -} - -/** Resource a binary plist preflight found over its budget, or a malformed reference cycle. */ -export type BinaryPlistExpansionFailure = "cycle" | "depth" | "expansion"; - -const interfaceBuilderBudgetFailure = ( - _kind: BinaryPlistExpansionFailure, - message: string, -): Error => - new InterfaceBuilderDecodeBudgetExceeded( - "aggregate_decode_budget_exhausted", - message, - ); - -/** How a caller names and reports binary plist expansion failures. */ -export interface BinaryPlistExpansionOptions { - /** Include original number metadata and exact-integer attribute projection. */ - readonly numberProjection?: boolean; - /** Budget named in failure messages. */ - readonly budget?: string; - /** Error for budget, depth, and reference-cycle failures. */ - readonly fail?: (kind: BinaryPlistExpansionFailure, message: string) => Error; - /** - * Structural bounds: "strict" keeps objects and the offset table before the - * trailer; "decoder" follows the decoder's object-table boundary policy - * while still rejecting structures whose coercion would lose plist facts. - */ - readonly bounds?: "strict" | "decoder"; -} - -/** Preflight size and binary-key observations; XML keys are inspected by decode. */ -export interface PlistDecodePreflight { - readonly estimatedBytes: number; - readonly binaryPrototypeKeyObserved: boolean; -} - -const PROTOTYPE_KEY = "__proto__"; - -/** - * Count every expanded binary-plist reference before calling plist.parseBinary, - * which recurses on each reference and copies shared containers. Malformed - * structure throws TypeError; budget, depth, and cycle failures use `fail`. - */ -export const estimateBinaryPlistExpansion = ( - bytes: Buffer, - maxBytes: number, - { - budget = "the aggregate Interface Builder decode budget", - fail = interfaceBuilderBudgetFailure, - bounds = "strict", - numberProjection = false, - }: BinaryPlistExpansionOptions = {}, -): PlistDecodePreflight => { - if (bytes.length < 40) - throw new TypeError("binary plist trailer is truncated"); - const trailer = bytes.length - 32; - const offsetSize = bytes[trailer + 6] ?? 0; - const referenceSize = bytes[trailer + 7] ?? 0; - if ( - ![1, 2, 4, 8].includes(offsetSize) || - ![1, 2, 4, 8].includes(referenceSize) - ) - throw new TypeError("binary plist trailer integer sizes are invalid"); - const readInteger = (offset: number, size: number): number => { - if (offset < 0 || size < 1 || offset + size > bytes.length) return -1; - let value = 0; - for (let index = 0; index < size; index += 1) - value = value * 256 + (bytes[offset + index] ?? 0); - return Number.isSafeInteger(value) ? value : -1; - }; - const objectCount = readInteger(trailer + 8, 8); - const topObject = readInteger(trailer + 16, 8); - const offsetTable = readInteger(trailer + 24, 8); - const tableEnd = bounds === "strict" ? trailer : bytes.length; - const objectsEnd = bounds === "strict" ? offsetTable : bytes.length; - if ( - objectCount < 1 || - topObject < 0 || - topObject >= objectCount || - offsetTable < 8 || - offsetTable + objectCount * offsetSize > tableEnd - ) - throw new TypeError("binary plist trailer or object table is invalid"); - // Metadata covers the full object table; projected values may repeat through references. - const objectBytes = numberProjection ? 144 : 16; - const expandedNodeBytes = numberProjection ? 224 : 96; - // Number projection may require a same-sized parser-compatible input copy - // for wide integers. Reserve for it before the caller allocates that copy. - let estimate = - bytes.length * (numberProjection ? 3 : 2) + objectCount * objectBytes; - let binaryPrototypeKeyObserved = false; - if (estimate > maxBytes) - throw fail("expansion", `binary plist object table exceeds ${budget}`); - - const active = new Set(); - const pending: Array<{ - object: number; - depth: number; - exit: boolean; - dictionaryKey: boolean; - }> = [{ object: topObject, depth: 0, exit: false, dictionaryKey: false }]; - let pendingObjectCount = 1; - while (pending.length > 0) { - const entry = pending.pop(); - if (entry === undefined) break; - if (entry.exit) { - active.delete(entry.object); - continue; - } - pendingObjectCount -= 1; - if (entry.depth > 128) - throw fail("depth", `binary plist nesting exceeds ${budget}`); - if (active.has(entry.object)) - throw fail("cycle", `binary plist reference cycle exceeds ${budget}`); - estimate += expandedNodeBytes; - if (estimate > maxBytes) - throw fail( - "expansion", - `binary plist reference expansion exceeds ${budget}`, - ); - const offset = readInteger( - offsetTable + entry.object * offsetSize, - offsetSize, - ); - if (offset < 8 || offset >= objectsEnd) - throw new TypeError("binary plist object offset is invalid"); - const marker = bytes[offset] ?? 0; - const type = marker >> 4; - if (entry.dictionaryKey && type !== 5 && type !== 6) - throw new TypeError("binary plist dictionary key is not a string"); - let size = marker & 0x0f; - let cursor = offset + 1; - if (size === 0x0f && type !== 0 && type !== 8) { - const extMarker = bytes[cursor] ?? 0; - if (extMarker >> 4 !== 1 || (extMarker & 0x0f) > 3) - throw new TypeError("binary plist extended object size is invalid"); - cursor += 1; - const integerSize = 1 << (extMarker & 0x0f); - size = readInteger(cursor, integerSize); - cursor += integerSize; - if (size < 0 || cursor > objectsEnd) - throw new TypeError("binary plist extended object size is invalid"); - } - if (type === 4 || type === 5 || type === 6) { - const byteLength = type === 6 ? size * 2 : size; - if (cursor + byteLength > objectsEnd) - throw new TypeError("binary plist scalar object is truncated"); - if ( - !binaryPrototypeKeyObserved && - entry.dictionaryKey && - (type === 5 || type === 6) && - size === PROTOTYPE_KEY.length - ) { - let matches = true; - for (let index = 0; index < PROTOTYPE_KEY.length; index += 1) { - const codeUnit = - type === 5 - ? (bytes[cursor + index] ?? 0) - : ((bytes[cursor + index * 2] ?? 0) << 8) | - (bytes[cursor + index * 2 + 1] ?? 0); - if (codeUnit !== PROTOTYPE_KEY.charCodeAt(index)) { - matches = false; - break; - } - } - if (matches) binaryPrototypeKeyObserved = true; - } - estimate += type === 4 ? size + 4 * Math.ceil(size / 3) : byteLength * 2; - if (estimate > maxBytes) - throw fail( - "expansion", - `binary plist scalar expansion exceeds ${budget}`, - ); - continue; - } - if (type !== 10 && type !== 13) continue; - const childCount = type === 13 ? size * 2 : size; - const refsEnd = cursor + childCount * referenceSize; - if (!Number.isSafeInteger(refsEnd) || refsEnd > objectsEnd) - throw new TypeError("binary plist reference table is truncated"); - const availableNodes = Math.floor( - (maxBytes - estimate) / expandedNodeBytes, - ); - if (childCount > availableNodes - pendingObjectCount) - throw fail( - "expansion", - `binary plist reference expansion exceeds ${budget}`, - ); - active.add(entry.object); - pending.push({ - object: entry.object, - depth: entry.depth, - exit: true, - dictionaryKey: false, - }); - for (let index = childCount - 1; index >= 0; index -= 1) { - const reference = readInteger( - cursor + index * referenceSize, - referenceSize, - ); - if (reference < 0 || reference >= objectCount) - throw new TypeError("binary plist reference is invalid"); - pending.push({ - object: reference, - depth: entry.depth + 1, - exit: false, - dictionaryKey: type === 13 && index < size, - }); - } - pendingObjectCount += childCount; - } - return { estimatedBytes: estimate, binaryPrototypeKeyObserved }; -}; diff --git a/src/artifacts/apple/KeyedArchiveReader.test.ts b/src/artifacts/apple/KeyedArchiveReader.test.ts index 71f62c8136..c4a1325ad6 100644 --- a/src/artifacts/apple/KeyedArchiveReader.test.ts +++ b/src/artifacts/apple/KeyedArchiveReader.test.ts @@ -1,7 +1,24 @@ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + import { describe, expect, it } from "vitest"; import { build, buildBinary, type PlistValue } from "plist"; -import { decodeKeyedArchiveBytes } from "./KeyedArchiveReader.js"; +import { estimateBinaryPlistExpansion } from "./PropertyListDecodeBudget.js"; +import { + decodeKeyedArchiveBytes, + inspectBundleKeyedArchive, +} from "./KeyedArchiveReader.js"; +import { unknownDateLimitation } from "../../domain/apple/plistValue.js"; import { deeplyNestedXmlPlist } from "../../../tests/fixtures/keyedArchiveXmlNesting.js"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import { scanBinaryPlistScalars } from "./PlistScalars.js"; +import { embedSubmillisecondDateInWideIntegerPayload } from "../../../tests/fixtures/binaryPlistScalarAlias.js"; +import { + readBinaryPlistObjectOffset, + readBinaryPlistObjectTableLayout, + writeBinaryPlistObjectOffset, +} from "../../../tests/fixtures/binaryPlistObjectTable.js"; const archive = { $archiver: "NSKeyedArchiver", @@ -22,6 +39,146 @@ const archive = { ], $top: { root: { UID: 1 }, other: { UID: 2 } }, }; + +const binaryArchiveWithDate = (): Buffer => + Buffer.from( + buildBinary({ + ...archive, + $objects: ["$null", new Date("2001-01-01T00:00:00.100Z")], + $top: { root: { UID: 1 } }, + }), + ); + +const binaryArchiveWithOverlappingDate = (): Buffer => { + const bytes = Buffer.from( + buildBinary({ + ...archive, + $objects: [ + "$null", + new Date("2001-01-01T00:00:00.000Z"), + Buffer.alloc(0), + ], + $top: { root: { UID: 1 } }, + }), + ); + const date = scanBinaryPlistScalars(bytes).dates[0]; + if (date === undefined) + throw new Error("binary archive fixture lost its date"); + bytes.writeDoubleBE(0.0005, date.payloadOffset); + const objectTable = readBinaryPlistObjectTableLayout(bytes); + for (let objectId = 0; objectId < objectTable.objectCount; objectId += 1) { + const offset = readBinaryPlistObjectOffset(bytes, objectTable, objectId); + if (offset !== date.offset && bytes[offset] === 0x40) { + writeBinaryPlistObjectOffset( + bytes, + objectTable, + objectId, + date.payloadOffset + 1, + ); + return bytes; + } + } + throw new Error("binary archive fixture lost its empty data object"); +}; + +const binaryArchiveDateCrossingOffsetTable = (): Buffer => { + const bytes = Buffer.from(binaryArchiveWithDate()); + const date = scanBinaryPlistScalars(bytes).dates[0]; + if (date === undefined) + throw new Error("binary archive fixture lost its date"); + const objectTable = readBinaryPlistObjectTableLayout(bytes); + let dateObjectId = -1; + for (let objectId = 0; objectId < objectTable.objectCount; objectId += 1) { + if ( + readBinaryPlistObjectOffset(bytes, objectTable, objectId) === date.offset + ) + dateObjectId = objectId; + } + if (dateObjectId < 0) + throw new Error("binary archive fixture lost its date object-table entry"); + + const crossingOffset = objectTable.tableOffset - 1; + bytes[crossingOffset] = 0x33; + writeBinaryPlistObjectOffset( + bytes, + objectTable, + dateObjectId, + crossingOffset, + ); + return bytes; +}; + +describe("keyed archive scalar projection", () => { + it("preserves the exact integer and unknown date through archive projection", () => { + const source = Buffer.from( + buildBinary({ + ...archive, + $objects: ["$null", Buffer.alloc(16), new Date(0)], + $top: { root: { UID: 1 } }, + }), + ); + const { bytes, integerDecimal } = + embedSubmillisecondDateInWideIntegerPayload(source); + + const graph = decodeKeyedArchiveBytes(bytes, { + root: "root", + offset: 1, + limit: 2, + }); + + expect(graph.objects).toContainEqual( + expect.objectContaining({ + id: 1, + value: { $plist_type: "integer", decimal: integerDecimal }, + }), + ); + expect(graph.objects).toContainEqual( + expect.objectContaining({ + id: 2, + value: { $plist_type: "date", iso8601: null }, + }), + ); + }); + + it("rejects a reachable date whose payload enters the offset table", () => { + expect(() => + decodeKeyedArchiveBytes(binaryArchiveDateCrossingOffsetTable(), { + root: "root", + offset: 0, + limit: 2, + }), + ).toThrow( + expect.objectContaining({ + message: "binary plist date object is truncated", + }), + ); + }); + + it("keeps overlapping date precision unknown without changing its sibling object", () => { + const graph = decodeKeyedArchiveBytes(binaryArchiveWithOverlappingDate(), { + root: "root", + offset: 0, + limit: 4, + }); + + expect(graph.objects).toContainEqual( + expect.objectContaining({ + id: 1, + value: { $plist_type: "date", iso8601: null }, + }), + ); + expect(graph.objects).toContainEqual( + expect.objectContaining({ + id: 2, + value: { $plist_type: "data", base64: "" }, + }), + ); + expect(graph.limitations).toContainEqual( + expect.stringContaining("sub-millisecond source records overlap"), + ); + }); +}); + describe("inert keyed archive decoding", () => { it.each(["bplist00", "NIBArchive"])( "rejects high-bit mutations of %s as malformed bytes rather than another archive kind", @@ -375,6 +532,223 @@ describe("binary plist reference preflight", () => { }); }); +/** + * Python `plistlib` binary NSKeyedArchiver with `$objects = ["$null", date]` + * and `$top.root` as UID 1. The date is marker `0x33` and float64 `0.1` + * seconds after the 2001 epoch (`2001-01-01T00:00:00.100Z`). + */ +const pythonBinaryDateControl = Buffer.from( + "62706c6973743030d4010203040506090c5924617263686976657258246f626a656374735424746f70582476657273696f6e5f100f4e534b657965644172636869766572a2070855246e756c6c333fb999999999999ad10a0b54726f6f74800112000186a008111b24293244474d56595e600000000000000101000000000000000d00000000000000000000000000000065", + "hex", +); +const canonicalDateMarker = Buffer.alloc(9); +canonicalDateMarker[0] = 0x33; +canonicalDateMarker.writeDoubleBE(0.1, 1); + +/** Copy the control and mutate its single canonical date marker. */ +const mutateCanonicalDate = ( + mutate: (markerOffset: number, bytes: Buffer) => void, +): Buffer => { + const bytes = Buffer.from(pythonBinaryDateControl); + const offset = bytes.indexOf(canonicalDateMarker); + if (offset < 0 || bytes.indexOf(canonicalDateMarker, offset + 1) !== -1) + throw new Error("expected one canonical binary date in the control"); + mutate(offset, bytes); + return bytes; +}; + +const dateScalar = (iso8601: string | null) => ({ + id: 1, + kind: "scalar" as const, + status: "observed" as const, + value: { $plist_type: "date", iso8601 }, +}); + +describe("binary keyed archive dates", () => { + const selection = { offset: 0, limit: 10 }; + + it("keeps the canonical millisecond control as an observed timestamp", async () => { + const bytes = Buffer.from(pythonBinaryDateControl); + const graph = decodeKeyedArchiveBytes(bytes, selection); + expect(graph.objects[1]).toMatchObject( + dateScalar("2001-01-01T00:00:00.100Z"), + ); + expect(graph.limitations).not.toContain(unknownDateLimitation(1)); + expect(bytes).toEqual(pythonBinaryDateControl); + expect(createHash("sha256").update(bytes).digest("hex")).toBe( + "39f08db8341bd74220352dfd559485515cce1e6035e08f27a59ab8f856374a30", + ); + const root = await createTestTempDirectory("rea-keyed-date-control-"); + await writeFile(join(root, "archive.plist"), bytes); + const inspected = await inspectBundleKeyedArchive({ + bundlePath: root, + targetSha256: createHash("sha256").update(bytes).digest("hex"), + parameters: { path: "archive.plist" }, + }); + expect(inspected.objects[1]).toMatchObject( + dateScalar("2001-01-01T00:00:00.100Z"), + ); + expect(inspected.limitations).not.toContain(unknownDateLimitation(1)); + expect(await readFile(join(root, "archive.plist"))).toEqual(bytes); + }); + + it.each( + [0x30, 0x31, 0x32, 0x34].map( + (marker) => [marker.toString(16), marker] as const, + ), + )( + "refuses reachable binary date marker 0x%s instead of projecting a timestamp", + async (_hex, marker) => { + const bytes = mutateCanonicalDate((offset, copy) => { + copy[offset] = marker; + }); + const original = Buffer.from(bytes); + expect(() => decodeKeyedArchiveBytes(bytes, selection)).toThrow( + new TypeError( + `binary plist date marker 0x${marker.toString(16)} is invalid`, + ), + ); + expect(bytes).toEqual(original); + if (marker === 0x30) + expect(createHash("sha256").update(bytes).digest("hex")).toBe( + "ac5696b866c312feb53c2e82edd8fea79dbeb572eb1b07e4291603faa41e8c0e", + ); + const root = await createTestTempDirectory("rea-keyed-date-marker-"); + const path = join(root, "archive.plist"); + await writeFile(path, bytes); + await expect( + inspectBundleKeyedArchive({ + bundlePath: root, + targetSha256: null, + parameters: { path: "archive.plist" }, + }), + ).rejects.toMatchObject({ + name: "ArtifactReaderFailure", + reason: "format", + message: `Cannot decode selected Foundation keyed archive: binary plist date marker 0x${marker.toString(16)} is invalid`, + }); + expect(await readFile(path)).toEqual(bytes); + }, + ); + + it("keeps a sub-millisecond instant unknown without changing the canonical sibling", async () => { + const submillisecond = mutateCanonicalDate((offset, copy) => { + copy.writeDoubleBE(0.1005, offset + 1); + }); + const original = Buffer.from(submillisecond); + const graph = decodeKeyedArchiveBytes(submillisecond, selection); + expect(graph.objects[1]).toMatchObject(dateScalar(null)); + expect(JSON.stringify(graph.objects[1]?.value)).not.toContain( + "2001-01-01T00:00:00.100Z", + ); + expect(graph.limitations).toContain(unknownDateLimitation(1)); + expect(graph.limitations.join("\n")).not.toContain( + "number metadata could not be completely observed", + ); + expect(submillisecond).toEqual(original); + expect(createHash("sha256").update(submillisecond).digest("hex")).toBe( + "ab69ba6e8034c6f60b828ee323a5a3e81bae3e7657477e0ebfbfdb93d53ed2ce", + ); + const root = await createTestTempDirectory("rea-keyed-date-subms-"); + await writeFile(join(root, "archive.plist"), submillisecond); + const inspected = await inspectBundleKeyedArchive({ + bundlePath: root, + targetSha256: createHash("sha256").update(submillisecond).digest("hex"), + parameters: { path: "archive.plist" }, + }); + expect(inspected.objects[1]).toMatchObject(dateScalar(null)); + expect(inspected.limitations).toContain(unknownDateLimitation(1)); + expect(await readFile(join(root, "archive.plist"))).toEqual(submillisecond); + + const mixed = Buffer.from( + buildBinary({ + $archiver: "NSKeyedArchiver", + $version: 100000, + $objects: [ + "$null", + new Date("2001-01-01T00:00:00.100Z"), + new Date("2001-01-01T00:00:00.100Z"), + new Date("2001-01-01T00:00:00.100Z"), + ], + $top: { root: { UID: 1 } }, + }), + ); + const pattern = Buffer.from(canonicalDateMarker); + const offsets: number[] = []; + for ( + let offset = mixed.indexOf(pattern); + offset !== -1; + offset = mixed.indexOf(pattern, offset + pattern.length) + ) + offsets.push(offset); + expect(offsets).toHaveLength(3); + // 0.1004 and 0.1005 both round to the same millisecond as the control. + mixed.writeDoubleBE(0.1004, (offsets[1] ?? 0) + 1); + mixed.writeDoubleBE(0.1005, (offsets[2] ?? 0) + 1); + const mixedOriginal = Buffer.from(mixed); + const mixedGraph = decodeKeyedArchiveBytes(mixed, selection); + expect(mixedGraph.objects.slice(1).map(({ value }) => value)).toEqual([ + { $plist_type: "date", iso8601: "2001-01-01T00:00:00.100Z" }, + { $plist_type: "date", iso8601: null }, + { $plist_type: "date", iso8601: null }, + ]); + expect(mixedGraph.limitations).toContain(unknownDateLimitation(2)); + expect(mixed).toEqual(mixedOriginal); + }); +}); + +describe("binary plist date object region", () => { + it("refuses a binary date that extends past the object region", () => { + // Strict bounds end objects at the offset table. Four payload bytes + // remain before that table, short of the required eight. + const bytes = Buffer.alloc(8 + 5 + 1 + 32); + bytes.write("bplist00"); + bytes[8] = 0x33; + const offsetTable = 13; + bytes[offsetTable] = 8; + const trailer = bytes.length - 32; + bytes[trailer + 6] = 1; + bytes[trailer + 7] = 1; + bytes.writeBigUInt64BE(1n, trailer + 8); + bytes.writeBigUInt64BE(0n, trailer + 16); + bytes.writeBigUInt64BE(BigInt(offsetTable), trailer + 24); + expect(() => + estimateBinaryPlistExpansion(bytes, 1024 * 1024, { bounds: "strict" }), + ).toThrow(new TypeError("binary plist date object is truncated")); + // Decoder bounds extend other objects to the end of the file. The date + // payload still has to end at the offset table. The keyed-archive reader + // uses those bounds and must refuse the same payload. + expect(() => + estimateBinaryPlistExpansion(bytes, 1024 * 1024, { bounds: "decoder" }), + ).toThrow(new TypeError("binary plist date object is truncated")); + expect(() => + decodeKeyedArchiveBytes(bytes, { offset: 0, limit: 1 }), + ).toThrow(new TypeError("binary plist date object is truncated")); + }); + + it("refuses a decoder-bounds date whose marker sits past the offset table", () => { + // The one-byte offset table at byte 8 names an object at byte 9. That + // 0x33 marker and its eight payload bytes fit in the file, which decoder + // bounds would otherwise accept, but they are outside the object region. + const bytes = Buffer.alloc(64); + bytes.write("bplist00"); + bytes[8] = 9; + bytes[9] = 0x33; + const trailer = bytes.length - 32; + bytes[trailer + 6] = 1; + bytes[trailer + 7] = 1; + bytes.writeBigUInt64BE(1n, trailer + 8); + bytes.writeBigUInt64BE(0n, trailer + 16); + bytes.writeBigUInt64BE(8n, trailer + 24); + expect(() => + estimateBinaryPlistExpansion(bytes, 1024 * 1024, { bounds: "decoder" }), + ).toThrow(new TypeError("binary plist date object is truncated")); + expect(() => + decodeKeyedArchiveBytes(bytes, { offset: 0, limit: 1 }), + ).toThrow(new TypeError("binary plist date object is truncated")); + }); +}); + describe("XML plist nesting preflight", () => { const selection = { offset: 0, limit: 1 }; diff --git a/src/artifacts/apple/KeyedArchiveReader.ts b/src/artifacts/apple/KeyedArchiveReader.ts index a15b8cfbe5..0857cbd183 100644 --- a/src/artifacts/apple/KeyedArchiveReader.ts +++ b/src/artifacts/apple/KeyedArchiveReader.ts @@ -7,6 +7,7 @@ import { AnalysisUnsupportedTargetError, } from "../../domain/analysisErrorCore.js"; import { + binaryDatePrecisionOverlapLimitation, negativeZeroRealLimitation, nonFiniteRealLimitation, unknownDateLimitation, @@ -14,9 +15,9 @@ import { } from "../../domain/apple/plistValue.js"; import { createPlistNumberProjection } from "../../domain/apple/plistNumbers.js"; import { - decodeBinaryPlistNumbers, - scanBinaryPlistNumberLiterals, -} from "./PlistNumberLiterals.js"; + decodeBinaryPlistScalars, + scanBinaryPlistScalars, +} from "./PlistScalars.js"; import { omittedPrototypeKeysLimitation, parseXmlPropertyList, @@ -36,7 +37,7 @@ import { hasBinaryPlistSignature, hasNibArchiveSignature, } from "../../domain/apple/appleArchiveFormat.js"; -import { estimateBinaryPlistExpansion } from "./InterfaceBuilderDecodeBudget.js"; +import { estimateBinaryPlistExpansion } from "./PropertyListDecodeBudget.js"; const MAX_BYTES = 64 * 1024 * 1024; /** Decoded representation budget, the same 8:1 ratio Interface Builder uses. */ @@ -72,16 +73,20 @@ export const decodeKeyedArchiveBytes = ( numberProjection: true, }) : undefined; - const binaryNumberMetadata = binary - ? scanBinaryPlistNumberLiterals(bytes) + const binaryScalarMetadata = binary + ? scanBinaryPlistScalars(bytes, binaryExpansion) : undefined; const parsed = binaryExpansion !== undefined ? { value: parseBinary( - decodeBinaryPlistNumbers(bytes, binaryNumberMetadata!), + decodeBinaryPlistScalars( + bytes, + binaryScalarMetadata!, + binaryExpansion!, + ), ), - numberLiterals: binaryNumberMetadata!.literals, + numberLiterals: binaryScalarMetadata!.literals, omittedPrototypeKeys: binaryExpansion.binaryPrototypeKeyObserved ? null : 0, @@ -127,6 +132,14 @@ export const decodeKeyedArchiveBytes = ( ...(unknownDateCount === 0 ? [] : [unknownDateLimitation(unknownDateCount)]), + ...(binaryExpansion === undefined || + binaryExpansion.datePrecisionOverlapCount === 0 + ? [] + : [ + binaryDatePrecisionOverlapLimitation( + binaryExpansion.datePrecisionOverlapCount, + ), + ]), ...(unknownScalarCount === 0 ? [] : [unknownScalarLimitation(unknownScalarCount)]), diff --git a/src/artifacts/apple/MachoLoadCommandReader.test.ts b/src/artifacts/apple/MachoLoadCommandReader.test.ts index 94dd4891c1..c2008e599c 100644 --- a/src/artifacts/apple/MachoLoadCommandReader.test.ts +++ b/src/artifacts/apple/MachoLoadCommandReader.test.ts @@ -142,6 +142,112 @@ it("uses legacy simulator and device platform facts when resolving dylibs", asyn ); }); +it.each([false, true] as const)( + "applies macOS dyld's implicit platform to a platform-less dylib and traces its required child present=%s", + async (childPresent) => { + const root = await createTestTempDirectory("rea-dylib-platform-unknown-"); + const executablePath = "Contents/MacOS/probe"; + const dylibPath = "Contents/Frameworks/libA.dylib"; + const fallbackPath = "Contents/Fallbacks/libA.dylib"; + const childPath = "Contents/Frameworks/libMissing.dylib"; + const fallbackChildPath = "Contents/Fallbacks/libFallbackOnly.dylib"; + const executable = machoImage({ + cpu: CPU.arm64, + commands: [ + buildVersionCommand(1), + rpathCommand("@executable_path/../Frameworks"), + rpathCommand("@executable_path/../Fallbacks"), + dylibCommand(LC.LOAD_DYLIB, "@rpath/libA.dylib"), + ], + }); + const dylib = machoImage({ + cpu: CPU.arm64, + fileType: FILE_TYPE.dylib, + commands: [ + rpathCommand("@loader_path"), + dylibCommand(LC.LOAD_DYLIB, "@rpath/libMissing.dylib"), + ], + }); + const child = machoImage({ + cpu: CPU.arm64, + fileType: FILE_TYPE.dylib, + commands: [buildVersionCommand(1)], + }); + const fallback = machoImage({ + cpu: CPU.arm64, + fileType: FILE_TYPE.dylib, + commands: [ + rpathCommand("@loader_path"), + dylibCommand(LC.LOAD_DYLIB, "@rpath/libFallbackOnly.dylib"), + ], + }); + const fallbackChild = machoImage({ + cpu: CPU.arm64, + fileType: FILE_TYPE.dylib, + commands: [buildVersionCommand(1)], + }); + await mkdir(join(root, "Contents/MacOS"), { recursive: true }); + await mkdir(join(root, "Contents/Frameworks"), { recursive: true }); + await mkdir(join(root, "Contents/Fallbacks"), { recursive: true }); + await writeFile(join(root, executablePath), executable); + await writeFile(join(root, dylibPath), dylib); + await writeFile(join(root, fallbackPath), fallback); + await writeFile(join(root, fallbackChildPath), fallbackChild); + if (childPresent) await writeFile(join(root, childPath), child); + + const trace = await traceDylibResolution({ + rootPath: root, + targetPath: join(root, executablePath), + targetSha256: createHash("sha256").update(executable).digest("hex"), + enumerateRoots: false, + parameters: { roots: [executablePath] }, + }); + + expect( + trace.images.find(({ path }) => path === executablePath)?.slices[0], + ).toMatchObject({ platform: 1, platforms: [1] }); + expect( + trace.images.find(({ path }) => path === dylibPath)?.slices[0], + ).toMatchObject({ platform: null, platforms: [] }); + expect(trace.edges[0]).toMatchObject({ + loader: executablePath, + install_name: "@rpath/libA.dylib", + candidates: [ + { + outcome: "resolved", + resolved_path: dylibPath, + platform_basis: "macos-dyld-implicit", + }, + ], + resolution: { status: "resolved", image: dylibPath }, + }); + expect(trace.edges[1]).toMatchObject({ + loader: dylibPath, + install_name: "@rpath/libMissing.dylib", + resolution: childPresent + ? { status: "resolved", image: childPath } + : { status: "unresolved", image: null }, + loader_conditional: false, + }); + expect(trace.images.map(({ path }) => path)).toEqual( + childPresent + ? [executablePath, childPath, dylibPath].sort() + : [executablePath, dylibPath].sort(), + ); + expect(trace.images.map(({ path }) => path)).not.toContain(fallbackPath); + expect(trace.images.map(({ path }) => path)).not.toContain( + fallbackChildPath, + ); + expect(trace.coverage.status).toBe("complete"); + expect(trace.limitations).toContainEqual( + expect.stringContaining(dylibPath), + ); + expect(trace.findings.map(({ kind }) => kind)).toEqual( + childPresent ? [] : ["required-load-unresolved"], + ); + }, +); + describe("absolute dyld candidate paths at the filesystem reader boundary", () => { it.each(["install-name", "rpath"] as const)( "resolves an absolute in-root %s while preserving its source path", diff --git a/src/artifacts/apple/PlistNumberLiterals.test.ts b/src/artifacts/apple/PlistNumberLiterals.test.ts deleted file mode 100644 index 103f28bed6..0000000000 --- a/src/artifacts/apple/PlistNumberLiterals.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { - decodeBinaryPlistNumbers, - scanBinaryPlistNumberLiterals, -} from "./PlistNumberLiterals.js"; - -const dataObjectWithUnusedOverlappingInteger = (): Buffer => { - const bytes = Buffer.alloc(66); - Buffer.from("bplist00", "ascii").copy(bytes); - bytes[8] = 0xa1; // One-element array referencing data object 1. - bytes[9] = 1; - bytes[10] = 0x4f; // Data object with an extended length of 18 bytes. - bytes[11] = 0x10; - bytes[12] = 18; - bytes[13] = 0x14; // An unused object-table entry points into this data. - bytes.fill(0xff, 14, 30); - bytes[30] = 0; - bytes.set([8, 10, 13], 31); // object offsets, including the overlapping entry - bytes[40] = 1; // offset width - bytes[41] = 1; // reference width - bytes.writeBigUInt64BE(3n, 42); - bytes.writeBigUInt64BE(0n, 50); - bytes.writeBigUInt64BE(31n, 58); - return bytes; -}; - -describe("binary plist number rewriting", () => { - it("does not rewrite an integer marker inside another object's data", () => { - const bytes = dataObjectWithUnusedOverlappingInteger(); - const metadata = scanBinaryPlistNumberLiterals(bytes); - - expect(metadata.literals.incomplete).toBe(true); - expect(metadata.wideIntegers).toEqual([]); - expect(decodeBinaryPlistNumbers(bytes, metadata)).toBe(bytes); - expect(bytes.subarray(13, 30)).toEqual( - Buffer.from([0x14, ...Array(16).fill(0xff)]), - ); - }); -}); diff --git a/src/artifacts/apple/PlistNumberLiterals.ts b/src/artifacts/apple/PlistNumberLiterals.ts deleted file mode 100644 index 12b55ecb9b..0000000000 --- a/src/artifacts/apple/PlistNumberLiterals.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { - createPlistNumberCollector, - type PlistNumberLiterals, -} from "../../domain/apple/plistNumbers.js"; -import { hasBinaryPlistSignature } from "../../domain/apple/appleArchiveFormat.js"; - -export interface BinaryPlistNumberMetadata { - readonly literals: PlistNumberLiterals; - /** 16-byte integers the pinned plist decoder cannot decode correctly. */ - readonly wideIntegers: readonly { offset: number; value: bigint }[]; -} - -const binaryObjectEnd = ( - bytes: Buffer, - offset: number, - objectEnd: number, - referenceWidth: number, -): number | undefined => { - const marker = bytes[offset]; - if (marker === undefined) return undefined; - const type = marker >> 4; - let length = marker & 0x0f; - let cursor = offset + 1; - if (length === 0x0f && type !== 0 && type !== 8) { - const extendedMarker = bytes[cursor]; - if (extendedMarker === undefined || extendedMarker >> 4 !== 1) - return undefined; - const width = 1 << (extendedMarker & 0x0f); - if (width > 8 || cursor + 1 + width > objectEnd) return undefined; - cursor += 1; - let extendedLength = 0n; - for (let index = 0; index < width; index += 1) - extendedLength = - (extendedLength << 8n) | BigInt(bytes[cursor + index] ?? 0); - if (extendedLength > BigInt(Number.MAX_SAFE_INTEGER)) return undefined; - length = Number(extendedLength); - cursor += width; - } - let payloadLength: number; - if (type === 0) payloadLength = 0; - else if (type === 1 || type === 2) { - if ((marker & 0x0f) > 4) return undefined; - payloadLength = 1 << (marker & 0x0f); - } else if (type === 3) { - if (marker !== 0x33) return undefined; - payloadLength = 8; - } else if (type === 4 || type === 5 || type === 7) { - payloadLength = length; - } else if (type === 6) { - payloadLength = length * 2; - } else if (type === 8) { - payloadLength = (marker & 0x0f) + 1; - } else if (type === 10 || type === 11 || type === 12) { - payloadLength = length * referenceWidth; - } else if (type === 13) { - payloadLength = length * 2 * referenceWidth; - } else return undefined; - const end = cursor + payloadLength; - return Number.isSafeInteger(end) && end <= objectEnd ? end : undefined; -}; - -/** Observe bounded binary number markers without changing primary decode acceptance. */ -export const scanBinaryPlistNumberLiterals = ( - bytes: Buffer, -): BinaryPlistNumberMetadata => { - const output = createPlistNumberCollector(); - const wideIntegers: { offset: number; value: bigint }[] = []; - if (bytes.length < 40 || !hasBinaryPlistSignature(bytes)) - return { literals: output.result(true), wideIntegers }; - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); - const trailer = bytes.length - 32; - const offsetWidth = bytes[trailer + 6] ?? 0; - if (![1, 2, 4, 8].includes(offsetWidth)) - return { literals: output.result(true), wideIntegers }; - const countRaw = view.getBigUint64(trailer + 8); - const tableRaw = view.getBigUint64(trailer + 24); - const safeMaximum = BigInt(Number.MAX_SAFE_INTEGER); - if (countRaw === 0n || countRaw > safeMaximum || tableRaw > safeMaximum) - return { literals: output.result(true), wideIntegers }; - if (tableRaw < 8n || tableRaw >= BigInt(trailer)) - return { literals: output.result(true), wideIntegers }; - const offsetTable = Number(tableRaw); - const capacity = Math.floor((trailer - offsetTable) / offsetWidth); - if (countRaw > BigInt(capacity)) - return { literals: output.result(true), wideIntegers }; - const count = Number(countRaw); - const referenceWidth = bytes[trailer + 7] ?? 0; - if (![1, 2, 4, 8].includes(referenceWidth)) - return { literals: output.result(true), wideIntegers }; - const unsigned = (offset: number): number | undefined => { - if (offset < offsetTable || offset + offsetWidth > trailer) - return undefined; - if (offsetWidth === 1) return view.getUint8(offset); - if (offsetWidth === 2) return view.getUint16(offset); - if (offsetWidth === 4) return view.getUint32(offset); - const value = view.getBigUint64(offset); - return value > safeMaximum ? undefined : Number(value); - }; - const objectOffsets: number[] = []; - for (let index = 0; index < count; index += 1) { - const offset = unsigned(offsetTable + index * offsetWidth); - if (offset === undefined || offset < 8 || offset >= offsetTable) - return { literals: output.result(true), wideIntegers }; - objectOffsets.push(offset); - } - const sortedOffsets = [...new Set(objectOffsets)].sort((a, b) => a - b); - for (let index = 0; index < sortedOffsets.length; index += 1) { - const offset = sortedOffsets[index]; - if (offset === undefined) continue; - const nextOffset = sortedOffsets[index + 1] ?? offsetTable; - if ( - binaryObjectEnd(bytes, offset, nextOffset, referenceWidth) === undefined - ) - return { literals: output.result(true), wideIntegers: [] }; - } - for (const offset of objectOffsets) { - const marker = bytes[offset] ?? 0; - const type = marker >> 4; - if (type !== 1 && type !== 2) continue; - const width = 2 ** (marker & 0xf); - if (offset + 1 + width > offsetTable) - return { literals: output.result(true), wideIntegers }; - if (type === 1 && width === 8) output.integer(view.getBigInt64(offset + 1)); - else if (type === 1 && width === 16) { - const value = BigInt.asIntN( - 128, - (view.getBigUint64(offset + 1) << 64n) | view.getBigUint64(offset + 9), - ); - output.integer(value); - wideIntegers.push({ offset, value }); - } else if (type === 1 && width > 4) - return { literals: output.result(true), wideIntegers }; - else if (type === 2 && width === 4) - output.real(view.getFloat32(offset + 1)); - else if (type === 2 && width === 8) - output.real(view.getFloat64(offset + 1)); - else if (type === 2) return { literals: output.result(true), wideIntegers }; - } - return { literals: output.result(), wideIntegers }; -}; - -/** Preserve 16-byte integers through plist@5's Number-only parser. */ -export const decodeBinaryPlistNumbers = ( - bytes: Buffer, - metadata: BinaryPlistNumberMetadata, -): Buffer => { - if (metadata.wideIntegers.length === 0) return bytes; - const compatible = Buffer.from(bytes); - for (const { offset, value } of metadata.wideIntegers) { - // Convert only the decoder's input copy to a double. Exact integer metadata - // from the original bytes restores decimal text after the structural parse. - compatible[offset] = 0x23; - compatible.writeDoubleBE(Number(value), offset + 1); - } - return compatible; -}; diff --git a/src/artifacts/apple/PlistScalars.test.ts b/src/artifacts/apple/PlistScalars.test.ts new file mode 100644 index 0000000000..75f0ee78a6 --- /dev/null +++ b/src/artifacts/apple/PlistScalars.test.ts @@ -0,0 +1,507 @@ +import { binaryPlistObjectTable } from "../../../tests/fixtures/binaryPlistObjectTable.js"; +import { describe, expect, it } from "vitest"; +import { parseBinary } from "plist"; + +import { createPlistNumberProjection } from "../../domain/apple/plistNumbers.js"; +import { projectPlistValue } from "../../domain/apple/plistValue.js"; +import { + binaryDateRequiresExactSeconds, + decodeBinaryPlistScalars, + scanBinaryPlistScalars, +} from "./PlistScalars.js"; +import { estimateBinaryPlistExpansion } from "./PropertyListDecodeBudget.js"; + +const dataObjectWithUnusedOverlappingInteger = (): Buffer => { + const bytes = Buffer.alloc(66); + bytes[8] = 0xa1; // One-element array referencing data object 1. + bytes[9] = 1; + bytes[10] = 0x4f; // Data object with an extended length of 18 bytes. + bytes[11] = 0x10; + bytes[12] = 18; + bytes[13] = 0x14; // An unused object-table entry points into this data. + bytes.fill(0xff, 14, 30); + bytes[30] = 0; + return binaryPlistObjectTable(bytes, [8, 10, 13], 31); +}; + +const integerPayloadContainingDate = (): Buffer => { + const bytes = Buffer.alloc(63); + bytes.set([0xa2, 1, 2, 0x14], 8); + bytes.fill(0, 12, 28); + bytes[18] = 0x33; // Reachable date marker aliases the 128-bit integer payload. + bytes.writeDoubleBE(0.1005, 19); + return binaryPlistObjectTable(bytes, [8, 11, 18], 28); +}; + +const integerPayloadContainingReal = (): Buffer => { + const bytes = Buffer.alloc(57); + Buffer.from("bplist00", "ascii").copy(bytes); + bytes.set([0xa2, 1, 2, 0x13], 8); + bytes[12] = 0x23; // Reachable real marker aliases the int64 payload. + bytes.writeDoubleBE(1, 13); + bytes.set([8, 11, 12], 22); + const trailer = bytes.length - 32; + bytes[trailer + 6] = 1; + bytes[trailer + 7] = 1; + bytes.writeBigUInt64BE(3n, trailer + 8); + bytes.writeBigUInt64BE(0n, trailer + 16); + bytes.writeBigUInt64BE(22n, trailer + 24); + return bytes; +}; + +const roundedIntegerAndReal = (): Buffer => { + const bytes = Buffer.alloc(64); + Buffer.from("bplist00", "ascii").copy(bytes); + bytes.set([0xa2, 1, 2, 0x13], 8); + bytes.writeBigInt64BE(9007199254740993n, 12); + bytes[20] = 0x23; + bytes.writeDoubleBE(9007199254740992, 21); + bytes.set([8, 11, 20], 29); + const trailer = bytes.length - 32; + bytes[trailer + 6] = 1; + bytes[trailer + 7] = 1; + bytes.writeBigUInt64BE(3n, trailer + 8); + bytes.writeBigUInt64BE(0n, trailer + 16); + bytes.writeBigUInt64BE(29n, trailer + 24); + return bytes; +}; + +const unreachableOverlappingIntegerAndReal = (): Buffer => { + const bytes = Buffer.alloc(78); + Buffer.from("bplist00", "ascii").copy(bytes); + bytes.set([0xa2, 1, 3, 0x4f, 0x10, 18, 0x13], 8); + bytes.writeBigInt64BE(9007199254740993n, 15); + bytes[32] = 0x23; + bytes.writeDoubleBE(9007199254740992, 33); + bytes.set([8, 11, 14, 32], 42); + const trailer = bytes.length - 32; + bytes[trailer + 6] = 1; + bytes[trailer + 7] = 1; + bytes.writeBigUInt64BE(4n, trailer + 8); + bytes.writeBigUInt64BE(0n, trailer + 16); + bytes.writeBigUInt64BE(42n, trailer + 24); + return bytes; +}; + +const dateInsideDataWithLargeOffsetTable = (): Buffer => { + const bytes = Buffer.alloc(295); + bytes.set([0xa2, 1, 2, 0x4a, 0], 8); + bytes[13] = 0x33; + bytes.writeDoubleBE(0.1005, 14); + return binaryPlistObjectTable(bytes, [8, 11, 13], 260); +}; + +const scalarCrossingOffsetTable = (marker: number, payload: Buffer): Buffer => { + const offsetTable = 14; + const trailer = offsetTable + 1; + const bytes = Buffer.alloc(trailer + 32); + Buffer.from("bplist00", "ascii").copy(bytes); + bytes[8] = marker; + payload.copy(bytes, 9); + bytes[offsetTable] = 8; + bytes[trailer + 6] = 1; + bytes[trailer + 7] = 1; + bytes.writeBigUInt64BE(1n, trailer + 8); + bytes.writeBigUInt64BE(0n, trailer + 16); + bytes.writeBigUInt64BE(BigInt(offsetTable), trailer + 24); + return bytes; +}; + +const scalarStartingAtBoundary = ( + marker: number, + boundary: "offset table" | "trailer", +): Buffer => { + const scalarOffset = marker; + const offsetTable = + boundary === "offset table" ? scalarOffset : scalarOffset - 1; + const trailer = offsetTable + 1; + const bytes = Buffer.alloc(trailer + 32); + Buffer.from("bplist00", "ascii").copy(bytes); + bytes[offsetTable] = scalarOffset; + bytes[scalarOffset] = marker; + bytes[trailer + 6] = 1; + bytes[trailer + 7] = 1; + bytes.writeBigUInt64BE(1n, trailer + 8); + bytes.writeBigUInt64BE(0n, trailer + 16); + bytes.writeBigUInt64BE(BigInt(offsetTable), trailer + 24); + return bytes; +}; + +describe("binary plist date precision", () => { + it("distinguishes canonical milliseconds from nearby source instants", () => { + const next = Buffer.alloc(8); + next.writeDoubleBE(0.1); + next.writeBigUInt64BE(next.readBigUInt64BE(0) + 1n); + for (const seconds of [0, -0, 0.001, 0.1]) + expect(binaryDateRequiresExactSeconds(seconds)).toBe(false); + for (const seconds of [0.1004, 0.1005, next.readDoubleBE(0)]) + expect(binaryDateRequiresExactSeconds(seconds)).toBe(true); + }); + + it.each([ + Number.NaN, + Number.POSITIVE_INFINITY, + Number.NEGATIVE_INFINITY, + 1e20, + ])( + "keeps an invalid or out-of-range date (%s) explicitly unknown", + (seconds) => { + const bytes = Buffer.alloc(50); + bytes[8] = 0x33; + bytes.writeDoubleBE(seconds, 9); + binaryPlistObjectTable(bytes, [8], 17); + const plan = estimateBinaryPlistExpansion(bytes, 1024 * 1024, { + numberProjection: true, + }); + const metadata = scanBinaryPlistScalars(bytes, plan); + expect(binaryDateRequiresExactSeconds(seconds)).toBe(false); + const decoded = parseBinary( + decodeBinaryPlistScalars(bytes, metadata, plan), + ); + expect(projectPlistValue(decoded).value).toEqual({ + $plist_type: "date", + iso8601: null, + }); + }, + ); +}); + +describe("binary plist scalar bounds", () => { + it("validates without retaining scalar plans when projection is not requested", () => { + const bytes = integerPayloadContainingDate(); + const plan = estimateBinaryPlistExpansion(bytes, 1024 * 1024); + expect(plan).toMatchObject({ + datePrecisionOverlapCount: 0, + reachableDateOffsets: [], + overlappingDateOffsets: [], + reachableWideIntegerOffsets: [], + overlappingWideIntegerOffsets: [], + reachableNumericOffsets: [], + }); + expect( + estimateBinaryPlistExpansion(bytes, plan.estimatedBytes).estimatedBytes, + ).toBe(plan.estimatedBytes); + const malformed = Buffer.from(bytes); + malformed[18] = 0x30; + expect(() => estimateBinaryPlistExpansion(malformed, 1024 * 1024)).toThrow( + /date marker/u, + ); + }); + + it.each([ + ["date", 0x33, "3f00000000080000"], + ["integer", 0x13, "0020000000080000"], + ["real", 0x23, "3ff0000000080000"], + ])( + "rejects %s payloads that cross the offset table", + (_kind, marker, hex) => { + const bytes = scalarCrossingOffsetTable(marker, Buffer.from(hex, "hex")); + + expect(() => + estimateBinaryPlistExpansion(bytes, 1024 * 1024, { + bounds: "decoder", + numberProjection: true, + }), + ).toThrow(/truncated/u); + }, + ); + + it.each([ + ["date", 0x33], + ["integer", 0x13], + ["real", 0x23], + ])( + "rejects %s records starting in the offset table or trailer", + (_kind, marker) => { + for (const boundary of ["offset table", "trailer"] as const) { + const bytes = scalarStartingAtBoundary(marker, boundary); + + expect(() => + estimateBinaryPlistExpansion(bytes, 1024 * 1024, { + bounds: "decoder", + numberProjection: true, + }), + ).toThrow(/truncated/u); + } + }, + ); +}); + +describe("binary plist number rewriting", () => { + it("does not rewrite an integer marker inside another object's data", () => { + const bytes = dataObjectWithUnusedOverlappingInteger(); + const metadata = scanBinaryPlistScalars(bytes); + + expect(metadata.literals.incomplete).toBe(false); + expect(metadata.wideIntegers).toEqual([{ offset: 13, value: -1n }]); + expect( + decodeBinaryPlistScalars(bytes, metadata, { + reachableDateOffsets: [], + overlappingDateOffsets: [], + reachableWideIntegerOffsets: [], + overlappingWideIntegerOffsets: [], + reachableNumericOffsets: [], + }), + ).toBe(bytes); + expect(bytes.subarray(13, 30)).toEqual( + Buffer.from([0x14, ...Array(16).fill(0xff)]), + ); + }); + + it("does not rewrite a wide integer when a date aliases its payload", () => { + const bytes = integerPayloadContainingDate(); + const original = Buffer.from(bytes); + const plan = estimateBinaryPlistExpansion(bytes, 1024 * 1024, { + numberProjection: true, + }); + const metadata = scanBinaryPlistScalars(bytes, plan); + + expect(metadata.literals.incomplete).toBe(false); + expect(metadata.wideIntegers).toEqual([ + { offset: 11, value: 242016218615821083930112n }, + ]); + expect(metadata.dates).toEqual([ + { offset: 18, payloadOffset: 19, secondsSince2001: 0.1005 }, + ]); + expect(plan.reachableWideIntegerOffsets).toEqual([11]); + expect(plan.overlappingWideIntegerOffsets).toEqual([11]); + expect(plan.overlappingDateOffsets).toEqual([18]); + expect(() => + estimateBinaryPlistExpansion(bytes, plan.estimatedBytes - 1, { + numberProjection: true, + }), + ).toThrow(/budget/u); + const decoded = decodeBinaryPlistScalars(bytes, metadata, plan); + expect(decoded).not.toBe(bytes); + expect(decoded.readDoubleBE(29)).toBeNaN(); + const parsed = projectPlistValue(parseBinary(decoded)); + expect( + createPlistNumberProjection(metadata.literals).project(parsed.value), + ).toEqual([ + { + $plist_type: "integer", + decimal: "242016218615821083930112", + }, + { $plist_type: "date", iso8601: null }, + ]); + expect(bytes).toEqual(original); + expect(bytes[11]).toBe(0x14); + expect(bytes[18]).toBe(0x33); + }); +}); + +/** + * Three-element array whose last two entries share one scalar record, beside + * an unrelated Int64 sibling. Objects: array, shared scalar, duplicate entry, + * sibling. + */ +const sharedScalarEntries = (scalar: Buffer): Buffer => { + const sibling = Buffer.alloc(9); + sibling[0] = 0x13; + sibling.writeBigInt64BE(1152921504606846979n, 1); + const scalarOffset = 12; + const siblingOffset = scalarOffset + scalar.length; + const tableOffset = siblingOffset + sibling.length; + const bytes = Buffer.alloc(tableOffset + 4 + 32); + bytes.set([0xa3, 1, 2, 3], 8); + scalar.copy(bytes, scalarOffset); + sibling.copy(bytes, siblingOffset); + return binaryPlistObjectTable( + bytes, + [8, scalarOffset, scalarOffset, siblingOffset], + tableOffset, + ); +}; + +const int64 = (value: bigint): Buffer => { + const bytes = Buffer.alloc(9); + bytes[0] = 0x13; + bytes.writeBigInt64BE(value, 1); + return bytes; +}; + +describe("binary plist shared object-table entries", () => { + const plan = (bytes: Buffer) => + estimateBinaryPlistExpansion(bytes, 1024 * 1024, { + numberProjection: true, + }); + + it("keeps every exact integer when two entries share one Int64 record", () => { + const bytes = sharedScalarEntries(int64(9007199254740993n)); + const original = Buffer.from(bytes); + const scalarPlan = plan(bytes); + const metadata = scanBinaryPlistScalars(bytes, scalarPlan); + + expect(metadata.literals.incomplete).toBe(false); + // The shared record is one source literal, not two. + expect(metadata.literals.unsafeIntegerLiterals).toBe(2); + const decoded = decodeBinaryPlistScalars(bytes, metadata, scalarPlan); + const parsed = projectPlistValue(parseBinary(decoded)); + const numbers = createPlistNumberProjection(metadata.literals); + expect(numbers.project(parsed.value)).toEqual([ + { $plist_type: "integer", decimal: "9007199254740993" }, + { $plist_type: "integer", decimal: "9007199254740993" }, + { $plist_type: "integer", decimal: "1152921504606846979" }, + ]); + expect(numbers.limitations().join("\n")).not.toContain( + "could not be completely observed", + ); + expect(bytes).toEqual(original); + }); + + it("rewrites a shared 16-byte integer once for every entry", () => { + const wide = Buffer.alloc(17); + wide[0] = 0x14; + wide.writeBigUInt64BE(1n, 1); + wide.writeBigUInt64BE(2n, 9); + const bytes = sharedScalarEntries(wide); + const original = Buffer.from(bytes); + const scalarPlan = plan(bytes); + const metadata = scanBinaryPlistScalars(bytes, scalarPlan); + + expect(metadata.literals.incomplete).toBe(false); + expect(metadata.wideIntegers).toEqual([ + { offset: 12, value: (1n << 64n) | 2n }, + ]); + const decoded = decodeBinaryPlistScalars(bytes, metadata, scalarPlan); + const parsed = projectPlistValue(parseBinary(decoded)); + expect( + createPlistNumberProjection(metadata.literals).project(parsed.value), + ).toEqual([ + { $plist_type: "integer", decimal: "18446744073709551618" }, + { $plist_type: "integer", decimal: "18446744073709551618" }, + { $plist_type: "integer", decimal: "1152921504606846979" }, + ]); + expect(bytes).toEqual(original); + }); + + it("records a shared date once and keeps its sibling exact", () => { + const date = Buffer.alloc(9); + date[0] = 0x33; + date.writeDoubleBE(0.1005, 1); + const bytes = sharedScalarEntries(date); + const scalarPlan = plan(bytes); + const metadata = scanBinaryPlistScalars(bytes, scalarPlan); + + expect(metadata.literals.incomplete).toBe(false); + expect(metadata.dates).toEqual([ + { offset: 12, payloadOffset: 13, secondsSince2001: 0.1005 }, + ]); + const decoded = decodeBinaryPlistScalars(bytes, metadata, scalarPlan); + const [first, second, sibling] = parseBinary(decoded) as [ + Date, + Date, + number, + ]; + expect(Number.isNaN(first.getTime())).toBe(true); + expect(Number.isNaN(second.getTime())).toBe(true); + expect( + createPlistNumberProjection(metadata.literals).project(sibling), + ).toEqual({ $plist_type: "integer", decimal: "1152921504606846979" }); + }); + + it("still reports metadata incomplete for an out-of-range entry", () => { + const bytes = sharedScalarEntries(int64(9007199254740993n)); + expect(scanBinaryPlistScalars(bytes).literals.incomplete).toBe(false); + // Repoint the duplicate into the header, before the object region. + const table = Number(bytes.readBigUInt64BE(bytes.length - 8)); + bytes[table + 2] = 7; + expect(bytes.subarray(0, 8).toString("ascii")).toBe("bplist00"); + expect(scanBinaryPlistScalars(bytes).literals.incomplete).toBe(true); + }); +}); + +describe("binary plist numeric association", () => { + it("keeps an overlapping int64 exact without rewriting either object", () => { + const bytes = integerPayloadContainingReal(); + const plan = estimateBinaryPlistExpansion(bytes, 1024 * 1024, { + numberProjection: true, + }); + const metadata = scanBinaryPlistScalars(bytes, plan); + + expect(metadata.literals.incomplete).toBe(false); + expect(metadata.literals.integers.get(2540012597650915300)).toEqual( + new Set(["2540012597650915328"]), + ); + const decoded = decodeBinaryPlistScalars(bytes, metadata, plan); + expect(decoded).toBe(bytes); + const parsed = projectPlistValue(parseBinary(decoded)); + expect( + createPlistNumberProjection(metadata.literals).project(parsed.value), + ).toEqual([{ $plist_type: "integer", decimal: "2540012597650915328" }, 1]); + }); + + it("keeps an integral real collision ambiguous after overlap-safe scanning", () => { + const bytes = roundedIntegerAndReal(); + const metadata = scanBinaryPlistScalars(bytes); + expect(metadata.literals.incomplete).toBe(false); + + const parsed = projectPlistValue(parseBinary(bytes)); + const numbers = createPlistNumberProjection(metadata.literals); + expect(numbers.project(parsed.value)).toEqual([ + 9007199254740992, 9007199254740992, + ]); + expect(numbers.limitations()).toEqual([ + '1 observed integer literal(s) in the original archive exceed the exact range of a JSON number. Decoded numbers may lose integer precision unless represented as { "$plist_type": "integer", "decimal": "" }; the original archive bytes retain those literals.', + "2 number(s) beyond the exact integer range remain as decoded because their source integer/real type or reference marker cannot be associated unambiguously; integer precision loss remains possible.", + ]); + }); + + it("does not associate unreachable overlapping integers with a reachable real", () => { + const bytes = unreachableOverlappingIntegerAndReal(); + const plan = estimateBinaryPlistExpansion(bytes, 1024 * 1024, { + numberProjection: true, + }); + const sourceMetadata = scanBinaryPlistScalars(bytes); + const metadata = scanBinaryPlistScalars(bytes, plan); + + expect(sourceMetadata.literals.integers.get(9007199254740992)).toEqual( + new Set(["9007199254740993"]), + ); + expect(metadata.literals.integers.has(9007199254740992)).toBe(false); + expect(metadata.literals.unsafeIntegerLiterals).toBe(0); + const decoded = decodeBinaryPlistScalars(bytes, metadata, plan); + expect(decoded).toBe(bytes); + const parsed = projectPlistValue(parseBinary(decoded)); + expect( + createPlistNumberProjection(metadata.literals).project(parsed.value), + ).toEqual([ + { + $plist_type: "data", + base64: "EwAgAAAAAAABAAAAAAAAAAAA", + }, + 9007199254740992, + ]); + }); +}); + +describe("binary plist scalar clone budgeting", () => { + it("widens offset entries and budgets isolated date clones before allocation", () => { + const bytes = dateInsideDataWithLargeOffsetTable(); + const plan = estimateBinaryPlistExpansion(bytes, 1024 * 1024, { + numberProjection: true, + }); + expect(plan.overlappingDateOffsets).toEqual([13]); + expect(plan.datePrecisionOverlapCount).toBe(1); + expect(() => + estimateBinaryPlistExpansion(bytes, plan.estimatedBytes - 1, { + numberProjection: true, + }), + ).toThrow(/budget/u); + expect( + estimateBinaryPlistExpansion(bytes, plan.estimatedBytes, { + numberProjection: true, + }).estimatedBytes, + ).toBe(plan.estimatedBytes); + + const metadata = scanBinaryPlistScalars(bytes); + const compatible = decodeBinaryPlistScalars(bytes, metadata, plan); + expect(compatible.length).toBeGreaterThan(bytes.length); + expect(compatible[compatible.length - 26]).toBe(2); + expect(compatible.readBigUInt64BE(compatible.length - 8)).toBeGreaterThan( + 255n, + ); + expect(bytes[13]).toBe(0x33); + expect(bytes.readDoubleBE(14)).toBe(0.1005); + }); +}); diff --git a/src/artifacts/apple/PlistScalars.ts b/src/artifacts/apple/PlistScalars.ts new file mode 100644 index 0000000000..1566608f4c --- /dev/null +++ b/src/artifacts/apple/PlistScalars.ts @@ -0,0 +1,381 @@ +import { + createPlistNumberCollector, + type PlistNumberLiterals, +} from "../../domain/apple/plistNumbers.js"; +import { hasBinaryPlistSignature } from "../../domain/apple/appleArchiveFormat.js"; + +export interface BinaryPlistScalarMetadata { + readonly literals: PlistNumberLiterals; + /** 16-byte integers the pinned plist decoder cannot decode correctly. */ + readonly wideIntegers: readonly { offset: number; value: bigint }[]; + /** Valid date objects, retaining their exact source seconds before Date conversion. */ + readonly dates: readonly { + readonly offset: number; + readonly payloadOffset: number; + readonly secondsSince2001: number; + }[]; +} + +const containsSortedOffset = ( + offsets: readonly number[], + target: number, +): boolean => { + let low = 0; + let high = offsets.length; + while (low < high) { + const middle = low + Math.floor((high - low) / 2); + const value = offsets[middle]; + if (value === target) return true; + if (value === undefined || value < target) low = middle + 1; + else high = middle; + } + return false; +}; + +export interface BinaryPlistScalarDecodePlan { + readonly reachableDateOffsets: readonly number[]; + readonly overlappingDateOffsets: readonly number[]; + readonly reachableWideIntegerOffsets: readonly number[]; + readonly overlappingWideIntegerOffsets: readonly number[]; + readonly reachableNumericOffsets: readonly number[]; +} + +const APPLE_EPOCH_MILLISECONDS = 978_307_200_000; + +/** Whether Date's millisecond projection would lose finite, in-range source seconds. */ +export const binaryDateRequiresExactSeconds = ( + secondsSince2001: number, +): boolean => { + if (!Number.isFinite(secondsSince2001)) return false; + const date = new Date(secondsSince2001 * 1000 + APPLE_EPOCH_MILLISECONDS); + return ( + Number.isFinite(date.getTime()) && + (date.getTime() - APPLE_EPOCH_MILLISECONDS) / 1000 !== secondsSince2001 + ); +}; + +/** Return the encoded end of one bounded binary-plist object, if recognized. */ +export const binaryPlistObjectEnd = ( + bytes: Buffer, + offset: number, + objectEnd: number, + referenceWidth: number, +): number | undefined => { + const marker = bytes[offset]; + if (marker === undefined) return undefined; + const type = marker >> 4; + let length = marker & 0x0f; + let cursor = offset + 1; + if (length === 0x0f && type !== 0 && type !== 8) { + const extendedMarker = bytes[cursor]; + if (extendedMarker === undefined || extendedMarker >> 4 !== 1) + return undefined; + const width = 1 << (extendedMarker & 0x0f); + if (width > 8 || cursor + 1 + width > objectEnd) return undefined; + cursor += 1; + let extendedLength = 0n; + for (let index = 0; index < width; index += 1) + extendedLength = + (extendedLength << 8n) | BigInt(bytes[cursor + index] ?? 0); + if (extendedLength > BigInt(Number.MAX_SAFE_INTEGER)) return undefined; + length = Number(extendedLength); + cursor += width; + } + let payloadLength: number; + if (type === 0) payloadLength = 0; + else if (type === 1 || type === 2) { + if ((marker & 0x0f) > 4) return undefined; + payloadLength = 1 << (marker & 0x0f); + } else if (type === 3) { + // Apple's date scalar is exactly type 3, info 3, followed by Float64. + if (marker !== 0x33) return undefined; + payloadLength = 8; + } else if (type === 4 || type === 5 || type === 7) { + payloadLength = length; + } else if (type === 6) { + payloadLength = length * 2; + } else if (type === 8) { + payloadLength = (marker & 0x0f) + 1; + } else if (type === 10 || type === 11 || type === 12) { + payloadLength = length * referenceWidth; + } else if (type === 13) { + payloadLength = length * 2 * referenceWidth; + } else return undefined; + const end = cursor + payloadLength; + return Number.isSafeInteger(end) && end <= objectEnd ? end : undefined; +}; + +/** Observe bounded binary scalar markers before parser conversion loses source facts. */ +export const scanBinaryPlistScalars = ( + bytes: Buffer, + scalarPlan?: BinaryPlistScalarDecodePlan, +): BinaryPlistScalarMetadata => { + const output = createPlistNumberCollector(); + const wideIntegers: { offset: number; value: bigint }[] = []; + const dates: { + offset: number; + payloadOffset: number; + secondsSince2001: number; + }[] = []; + const result = (incomplete: boolean): BinaryPlistScalarMetadata => ({ + literals: output.result(incomplete), + wideIntegers, + dates, + }); + if (bytes.length < 40 || !hasBinaryPlistSignature(bytes)) return result(true); + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const trailer = bytes.length - 32; + const offsetWidth = bytes[trailer + 6] ?? 0; + if (![1, 2, 4, 8].includes(offsetWidth)) return result(true); + const countRaw = view.getBigUint64(trailer + 8); + const tableRaw = view.getBigUint64(trailer + 24); + const safeMaximum = BigInt(Number.MAX_SAFE_INTEGER); + if (countRaw === 0n || countRaw > safeMaximum || tableRaw > safeMaximum) + return result(true); + if (tableRaw < 8n || tableRaw >= BigInt(trailer)) return result(true); + const offsetTable = Number(tableRaw); + const capacity = Math.floor((trailer - offsetTable) / offsetWidth); + if (countRaw > BigInt(capacity)) return result(true); + const count = Number(countRaw); + const referenceWidth = bytes[trailer + 7] ?? 0; + if (![1, 2, 4, 8].includes(referenceWidth)) return result(true); + const unsigned = (offset: number): number | undefined => { + if (offset < offsetTable || offset + offsetWidth > trailer) + return undefined; + if (offsetWidth === 1) return view.getUint8(offset); + if (offsetWidth === 2) return view.getUint16(offset); + if (offsetWidth === 4) return view.getUint32(offset); + const value = view.getBigUint64(offset); + return value > safeMaximum ? undefined : Number(value); + }; + const objectOffsets: number[] = []; + let incomplete = false; + for (let index = 0; index < count; index += 1) { + const offset = unsigned(offsetTable + index * offsetWidth); + if (offset === undefined || offset < 8 || offset >= offsetTable) { + incomplete = true; + continue; + } + objectOffsets.push(offset); + } + const sortedOffsets = [...new Set(objectOffsets)].sort((a, b) => a - b); + const validOffsets = new Set(); + for (const offset of sortedOffsets) { + const encodedEnd = binaryPlistObjectEnd( + bytes, + offset, + trailer, + referenceWidth, + ); + if (encodedEnd === undefined) { + incomplete = true; + continue; + } + validOffsets.add(offset); + } + const visitedOffsets = new Set(); + for (const offset of objectOffsets) { + if (!visitedOffsets.has(offset)) { + const marker = bytes[offset] ?? 0; + if ( + marker === 0x33 && + offset + 9 <= offsetTable && + (scalarPlan === undefined || + containsSortedOffset(scalarPlan.reachableDateOffsets, offset)) + ) { + const payloadOffset = offset + 1; + dates.push({ + offset, + payloadOffset, + secondsSince2001: view.getFloat64(payloadOffset), + }); + } + } + if (!validOffsets.has(offset)) { + incomplete = true; + continue; + } + // Several table entries may share one record. Its source facts are + // identical for every entry, so observe them once. + if (visitedOffsets.has(offset)) continue; + visitedOffsets.add(offset); + const marker = bytes[offset] ?? 0; + const type = marker >> 4; + if (type === 3 && marker === 0x33) { + continue; + } + if (type !== 1 && type !== 2) continue; + const width = 2 ** (marker & 0xf); + if (offset + 1 + width > offsetTable) incomplete = true; + if (offset + 1 + width > offsetTable) continue; + const isReachable = + scalarPlan === undefined || + containsSortedOffset(scalarPlan.reachableNumericOffsets, offset); + if (type === 1 && width === 8) { + const value = view.getBigInt64(offset + 1); + if (isReachable) output.integer(value); + } else if (type === 1 && width === 16) { + const value = BigInt.asIntN( + 128, + (view.getBigUint64(offset + 1) << 64n) | view.getBigUint64(offset + 9), + ); + if (isReachable) { + output.integer(value); + if ( + scalarPlan === undefined || + containsSortedOffset(scalarPlan.reachableWideIntegerOffsets, offset) + ) + wideIntegers.push({ offset, value }); + } + } else if (type === 1 && width > 4) { + incomplete = true; + } else if (type === 2 && width === 4) { + const value = view.getFloat32(offset + 1); + if (isReachable) output.real(value); + } else if (type === 2 && width === 8) { + const value = view.getFloat64(offset + 1); + if (isReachable) output.real(value); + } else if (type === 2) { + incomplete = true; + } + } + return result(incomplete); +}; + +/** Preserve source scalars through plist@5's lossy Number and Date parser. */ +export const decodeBinaryPlistScalars = ( + bytes: Buffer, + metadata: BinaryPlistScalarMetadata, + scalarPlan: BinaryPlistScalarDecodePlan, +): Buffer => { + const reachableDateOffsets = new Set(scalarPlan.reachableDateOffsets); + const overlappingDateOffsets = new Set(scalarPlan.overlappingDateOffsets); + const reachableWideIntegerOffsets = new Set( + scalarPlan.reachableWideIntegerOffsets, + ); + const overlappingWideIntegerOffsets = new Set( + scalarPlan.overlappingWideIntegerOffsets, + ); + const unrepresentableDates = metadata.dates.filter( + ({ offset, secondsSince2001 }) => + reachableDateOffsets.has(offset) && + binaryDateRequiresExactSeconds(secondsSince2001), + ); + const reachableWideIntegers = metadata.wideIntegers.filter(({ offset }) => + reachableWideIntegerOffsets.has(offset), + ); + const cloneOffsets = new Set([ + ...unrepresentableDates + .filter(({ offset }) => overlappingDateOffsets.has(offset)) + .map(({ offset }) => offset), + ...reachableWideIntegers + .filter(({ offset }) => overlappingWideIntegerOffsets.has(offset)) + .map(({ offset }) => offset), + ]); + if (reachableWideIntegers.length === 0 && unrepresentableDates.length === 0) + return bytes; + const compatible = + cloneOffsets.size === 0 + ? Buffer.from(bytes) + : cloneOverlappingScalars(bytes, cloneOffsets, metadata); + for (const { offset, value } of reachableWideIntegers) { + if (overlappingWideIntegerOffsets.has(offset)) continue; + // Convert only the decoder's input copy to a double. Exact integer metadata + // from the original bytes restores decimal text after the structural parse. + compatible[offset] = 0x23; + compatible.writeDoubleBE(Number(value), offset + 1); + } + // NaN becomes the existing explicit unknown-date projection. Only the private + // decode copy changes; scanner metadata retains the original Float64 seconds. + for (const { offset, payloadOffset } of unrepresentableDates) { + if (!overlappingDateOffsets.has(offset)) + compatible.writeDoubleBE(Number.NaN, payloadOffset); + } + return compatible; +}; + +const cloneOverlappingScalars = ( + bytes: Buffer, + scalarOffsets: ReadonlySet, + metadata: BinaryPlistScalarMetadata, +): Buffer => { + const trailerOffset = bytes.length - 32; + const offsetSize = bytes[trailerOffset + 6] ?? 0; + const objectCount = Number(bytes.readBigUInt64BE(trailerOffset + 8)); + const oldTableOffset = Number(bytes.readBigUInt64BE(trailerOffset + 24)); + const readOffset = (offset: number): bigint => { + let value = 0n; + for (let index = 0; index < offsetSize; index += 1) + value = value * 256n + BigInt(bytes[offset + index] ?? 0); + return value; + }; + const offsets = Array.from({ length: objectCount }, (_, index) => + readOffset(oldTableOffset + index * offsetSize), + ); + const cloneEntries = [...scalarOffsets]; + const wideIntegersByOffset = new Map( + metadata.wideIntegers.map((entry) => [entry.offset, entry.value]), + ); + const newOffsetSize = [1, 2, 4, 8].find((width) => { + if (width < offsetSize) return false; + const maximum = + width === 8 ? Number.MAX_SAFE_INTEGER : 2 ** (width * 8) - 1; + return cloneEntries.every( + (_, index) => oldTableOffset + index * 9 <= maximum, + ); + }); + if (newOffsetSize === undefined) + throw new RangeError( + "binary plist cloned scalar offset exceeds supported width", + ); + const cloneStart = oldTableOffset; + const cloneBytes = cloneEntries.length * 9; + const newTableOffset = cloneStart + cloneBytes; + const newLength = + bytes.length + cloneBytes + objectCount * (newOffsetSize - offsetSize); + if (!Number.isSafeInteger(newLength)) + throw new RangeError( + "binary plist cloned scalar representation is too large", + ); + const compatible = Buffer.alloc(newLength); + bytes.copy(compatible, 0, 0, oldTableOffset); + const cloneOffsetBySource = new Map(); + cloneEntries.forEach((sourceOffset, index) => { + const cloneOffset = cloneStart + index * 9; + cloneOffsetBySource.set(sourceOffset, BigInt(cloneOffset)); + const wideInteger = wideIntegersByOffset.get(sourceOffset); + if (wideInteger !== undefined) { + compatible[cloneOffset] = 0x23; + compatible.writeDoubleBE(Number(wideInteger), cloneOffset + 1); + } else { + bytes.copy(compatible, cloneOffset, sourceOffset, sourceOffset + 9); + compatible.writeDoubleBE(Number.NaN, cloneOffset + 1); + } + }); + const tableStart = newTableOffset; + const tableEnd = tableStart + objectCount * newOffsetSize; + const oldTableEnd = oldTableOffset + objectCount * offsetSize; + const padding = bytes.subarray(oldTableEnd, trailerOffset); + offsets.forEach((sourceOffset, index) => { + const sourceNumber = + sourceOffset <= BigInt(Number.MAX_SAFE_INTEGER) + ? Number(sourceOffset) + : undefined; + let value = + sourceNumber === undefined + ? sourceOffset + : (cloneOffsetBySource.get(sourceNumber) ?? sourceOffset); + for (let byte = newOffsetSize - 1; byte >= 0; byte -= 1) { + compatible[tableStart + index * newOffsetSize + byte] = Number( + value & 0xffn, + ); + value >>= 8n; + } + }); + const trailer = Buffer.from(bytes.subarray(trailerOffset)); + trailer[6] = newOffsetSize; + trailer.writeBigUInt64BE(BigInt(newTableOffset), 24); + padding.copy(compatible, tableEnd); + trailer.copy(compatible, tableEnd + padding.length); + return compatible; +}; diff --git a/src/artifacts/apple/PropertyListDecodeBudget.ts b/src/artifacts/apple/PropertyListDecodeBudget.ts new file mode 100644 index 0000000000..45ca4cbd0a --- /dev/null +++ b/src/artifacts/apple/PropertyListDecodeBudget.ts @@ -0,0 +1,522 @@ +import { decodeXmlPlistText } from "../../domain/propertyListXmlText.js"; +import { hasBinaryPlistSignature } from "../../domain/apple/appleArchiveFormat.js"; +import { + binaryDateRequiresExactSeconds, + binaryPlistObjectEnd, +} from "./PlistScalars.js"; + +/** Preflight the representation created by the pinned plist decoder. */ +export const preflightPropertyListDecode = ( + bytes: Buffer, + maxBytes: number, + xmlText?: string, + options: Pick< + BinaryPlistExpansionOptions, + "numberProjection" | "budget" | "fail" + > = {}, +): PlistDecodePreflight => { + const { + numberProjection = false, + budget = "the property list decode budget", + fail = propertyListDecodeBudgetFailure, + } = options; + const expansion = hasBinaryPlistSignature(bytes) + ? estimateBinaryPlistExpansion(bytes, maxBytes, { + numberProjection, + budget, + fail, + }) + : { + estimatedBytes: estimateXmlPlistExpansion( + bytes, + maxBytes, + xmlText, + numberProjection, + budget, + fail, + ), + binaryPrototypeKeyObserved: false, + datePrecisionOverlapCount: 0, + reachableDateOffsets: [], + overlappingDateOffsets: [], + reachableWideIntegerOffsets: [], + overlappingWideIntegerOffsets: [], + reachableNumericOffsets: [], + }; + if (expansion.estimatedBytes > maxBytes) + throw fail("expansion", `plist representation exceeds ${budget}`); + return expansion; +}; + +const estimateXmlPlistExpansion = ( + bytes: Buffer, + maxBytes: number, + xmlText?: string, + numberProjection = false, + budget = "the property list decode budget", + fail: ( + kind: BinaryPlistExpansionFailure, + message: string, + ) => Error = propertyListDecodeBudgetFailure, +): number => { + const xml = xmlText ?? decodeXmlPlistText(bytes); + const decodedSourceBytes = xml.length * 2; + let elementCount = 0; + // Reserve room for numeric source metadata and the projected attribute copy. + const elementBytes = numberProjection ? 256 : 128; + // Value nesting is bounded by the shared plist parser, which counts plist + // value depth; raw element depth would also count the envelope and keys. + for (const token of xmlPreflightTokens(xml)) { + if (token.doctype) { + // Only a declaration outside opaque text can introduce an internal + // subset. Brackets in quoted external identifiers are literal text. + if (token.internalSubset) + throw fail( + "expansion", + `XML plist internal entity expansion exceeds ${budget}`, + ); + continue; + } + elementCount += 1; + if (decodedSourceBytes * 2 + elementCount * elementBytes > maxBytes) + return maxBytes + 1; + } + return decodedSourceBytes * 2 + elementCount * elementBytes; +}; + +/** Consume each XML span once; malformed tokens cannot restart suffix scans. */ +function* xmlPreflightTokens(xml: string) { + let cursor = 0; + while (cursor < xml.length) { + const start = xml.indexOf("<", cursor); + if (start < 0) return; + const opaqueEnd = xml.startsWith("" + : xml.startsWith("" + : xml.startsWith("" + : undefined; + if (opaqueEnd !== undefined) { + const end = xml.indexOf(opaqueEnd, start + 2); + if (end < 0) return; + cursor = end + opaqueEnd.length; + continue; + } + const doctype = /^") break; + else if (doctype && character === "[") internalSubset = true; + } + if (end === xml.length) return; + cursor = end + 1; + if (doctype || element) yield { doctype, internalSubset }; + } +} + +/** Resource or structural failure found while preflighting a plist. */ +export type BinaryPlistExpansionFailure = "cycle" | "depth" | "expansion"; + +/** Typed failure for shared plist preflight resource limits. */ +export class PropertyListDecodeBudgetFailure extends Error { + constructor( + readonly kind: BinaryPlistExpansionFailure, + message: string, + ) { + super(message); + this.name = "PropertyListDecodeBudgetFailure"; + } +} + +const propertyListDecodeBudgetFailure = ( + kind: BinaryPlistExpansionFailure, + message: string, +): Error => new PropertyListDecodeBudgetFailure(kind, message); + +/** How a caller names and reports binary plist expansion failures. */ +export interface BinaryPlistExpansionOptions { + /** Include original number metadata and exact-integer attribute projection. */ + readonly numberProjection?: boolean; + /** Budget named in failure messages. */ + readonly budget?: string; + /** Error for budget, depth, and reference-cycle failures. */ + readonly fail?: (kind: BinaryPlistExpansionFailure, message: string) => Error; + /** + * Structural bounds: "strict" keeps objects and the offset table before the + * trailer; "decoder" follows the decoder's object-table boundary policy + * while still rejecting structures whose coercion would lose plist facts. + */ + readonly bounds?: "strict" | "decoder"; +} + +/** Preflight size and binary-key observations; XML keys are inspected by decode. */ +export interface PlistDecodePreflight { + readonly estimatedBytes: number; + readonly binaryPrototypeKeyObserved: boolean; + /** Number of expanded references to precision-unknown overlapping dates. */ + readonly datePrecisionOverlapCount: number; + /** Unique reachable date records in the selected value graph. */ + readonly reachableDateOffsets: readonly number[]; + /** Precision-unknown date offsets requiring decoder-copy isolation. */ + readonly overlappingDateOffsets: readonly number[]; + /** Unique reachable 16-byte integer records in the selected value graph. */ + readonly reachableWideIntegerOffsets: readonly number[]; + /** Source offsets requiring isolated decoder-compatible scalar records. */ + readonly overlappingWideIntegerOffsets: readonly number[]; + /** Reachable integer and real source records for exact number association. */ + readonly reachableNumericOffsets: readonly number[]; +} + +const PROTOTYPE_KEY = "__proto__"; + +type BinaryObjectSpan = { readonly offset: number; readonly end: number }; + +const findOverlappingScalarOffsets = ( + spans: readonly BinaryObjectSpan[], + scalarOffsets: ReadonlySet, +): Set => { + let maxPriorEnd = -1; + const overlappingOffsets = new Set(); + const activeScalars = new Map(); + for (const span of spans) { + for (const [offset, end] of activeScalars) + if (end <= span.offset) activeScalars.delete(offset); + if (scalarOffsets.has(span.offset)) { + if (maxPriorEnd > span.offset) { + overlappingOffsets.add(span.offset); + for (const activeOffset of activeScalars.keys()) + overlappingOffsets.add(activeOffset); + activeScalars.clear(); + } else activeScalars.set(span.offset, span.end); + } else { + for (const activeOffset of activeScalars.keys()) + overlappingOffsets.add(activeOffset); + activeScalars.clear(); + } + maxPriorEnd = Math.max(maxPriorEnd, span.end); + } + return overlappingOffsets; +}; + +const findExactScalarCloneOffsetWidth = ( + currentWidth: number, + firstCloneOffset: number, + cloneCount: number, +): number | undefined => { + const lastCloneOffset = firstCloneOffset + cloneCount * 9 - 9; + return [currentWidth, 2, 4, 8] + .filter((width) => width >= currentWidth) + .find((width) => + width === 8 + ? lastCloneOffset <= Number.MAX_SAFE_INTEGER + : lastCloneOffset < 2 ** (width * 8), + ); +}; + +/** + * Count every expanded binary-plist reference before calling plist.parseBinary, + * which recurses on each reference and copies shared containers. Malformed + * structure throws TypeError; resource and cycle failures use `fail`. + */ +export const estimateBinaryPlistExpansion = ( + bytes: Buffer, + maxBytes: number, + { + budget = "the property list decode budget", + fail = propertyListDecodeBudgetFailure, + bounds = "strict", + numberProjection = false, + }: BinaryPlistExpansionOptions = {}, +): PlistDecodePreflight => { + if (bytes.length < 40) + throw new TypeError("binary plist trailer is truncated"); + const trailer = bytes.length - 32; + const offsetSize = bytes[trailer + 6] ?? 0; + const referenceSize = bytes[trailer + 7] ?? 0; + if ( + ![1, 2, 4, 8].includes(offsetSize) || + ![1, 2, 4, 8].includes(referenceSize) + ) + throw new TypeError("binary plist trailer integer sizes are invalid"); + const readInteger = (offset: number, size: number): number => { + if (offset < 0 || size < 1 || offset + size > bytes.length) return -1; + let value = 0; + for (let index = 0; index < size; index += 1) + value = value * 256 + (bytes[offset + index] ?? 0); + return Number.isSafeInteger(value) ? value : -1; + }; + const objectCount = readInteger(trailer + 8, 8); + const topObject = readInteger(trailer + 16, 8); + const offsetTable = readInteger(trailer + 24, 8); + const tableEnd = bounds === "strict" ? trailer : bytes.length; + const objectsEnd = bounds === "strict" ? offsetTable : bytes.length; + if ( + objectCount < 1 || + topObject < 0 || + topObject >= objectCount || + offsetTable < 8 || + offsetTable + objectCount * offsetSize > tableEnd + ) + throw new TypeError("binary plist trailer or object table is invalid"); + // Reserve source metadata and reachable-offset indexes for scalar projection. + // Without projection, the traversal retains only structural validation state. + const objectBytes = numberProjection ? 320 : 16; + const expandedNodeBytes = numberProjection ? 224 : 96; + // Scalar projection may require a same-sized parser-compatible input copy + // for wide integers and sub-millisecond dates. Reserve before allocation. + let estimate = + bytes.length * (numberProjection ? 3 : 2) + objectCount * objectBytes; + let binaryPrototypeKeyObserved = false; + if (estimate > maxBytes) + throw fail("expansion", `binary plist object table exceeds ${budget}`); + + const active = new Set(); + const pending: Array<{ + object: number; + depth: number; + exit: boolean; + dictionaryKey: boolean; + }> = [{ object: topObject, depth: 0, exit: false, dictionaryKey: false }]; + let pendingObjectCount = 1; + const reachableObjectOffsets = new Set(); + const reachableDateOffsets = new Set(); + const reachableWideIntegerOffsets = new Set(); + const reachableNumericOffsets = new Set(); + const reachableDateOccurrences: number[] = []; + while (pending.length > 0) { + const entry = pending.pop(); + if (entry === undefined) break; + if (entry.exit) { + active.delete(entry.object); + continue; + } + pendingObjectCount -= 1; + if (entry.depth > 128) + throw fail("depth", `binary plist nesting exceeds ${budget}`); + if (active.has(entry.object)) + throw fail("cycle", `binary plist reference cycle exceeds ${budget}`); + estimate += expandedNodeBytes; + if (estimate > maxBytes) + throw fail( + "expansion", + `binary plist reference expansion exceeds ${budget}`, + ); + const offset = readInteger( + offsetTable + entry.object * offsetSize, + offsetSize, + ); + if (offset < 8 || offset >= objectsEnd) + throw new TypeError("binary plist object offset is invalid"); + if (numberProjection) reachableObjectOffsets.add(offset); + const marker = bytes[offset] ?? 0; + const type = marker >> 4; + if (entry.dictionaryKey && type !== 5 && type !== 6) + throw new TypeError("binary plist dictionary key is not a string"); + if (type === 3) { + if (marker !== 0x33) + throw new TypeError( + `binary plist date marker 0x${marker.toString(16)} is invalid`, + ); + // The offset table and trailer are not part of an encoded object. Keep + // this scalar boundary even when compatibility mode follows decoder + // bounds for other object and reference checks. + if (offset + 9 > offsetTable) + throw new TypeError("binary plist date object is truncated"); + if (numberProjection) { + reachableDateOffsets.add(offset); + reachableDateOccurrences.push(offset); + } + continue; + } + if (type === 1 || type === 2) { + const info = marker & 0x0f; + if (type === 1 && info > 4) + throw new TypeError("binary plist integer width is invalid"); + if (type === 2 && info !== 2 && info !== 3) + throw new TypeError("binary plist real width is invalid"); + const width = 1 << info; + if (offset + 1 + width > offsetTable) + throw new TypeError("binary plist numeric scalar is truncated"); + if (numberProjection) { + reachableNumericOffsets.add(offset); + if (type === 1 && width === 16) reachableWideIntegerOffsets.add(offset); + } + continue; + } + let size = marker & 0x0f; + let cursor = offset + 1; + if (size === 0x0f && type !== 0 && type !== 8) { + const extMarker = bytes[cursor] ?? 0; + if (extMarker >> 4 !== 1 || (extMarker & 0x0f) > 3) + throw new TypeError("binary plist extended object size is invalid"); + cursor += 1; + const integerSize = 1 << (extMarker & 0x0f); + size = readInteger(cursor, integerSize); + cursor += integerSize; + if (size < 0 || cursor > objectsEnd) + throw new TypeError("binary plist extended object size is invalid"); + } + if (type === 4 || type === 5 || type === 6) { + const byteLength = type === 6 ? size * 2 : size; + if (cursor + byteLength > objectsEnd) + throw new TypeError("binary plist scalar object is truncated"); + if ( + !binaryPrototypeKeyObserved && + entry.dictionaryKey && + (type === 5 || type === 6) && + size === PROTOTYPE_KEY.length + ) { + let matches = true; + for (let index = 0; index < PROTOTYPE_KEY.length; index += 1) { + const codeUnit = + type === 5 + ? (bytes[cursor + index] ?? 0) + : ((bytes[cursor + index * 2] ?? 0) << 8) | + (bytes[cursor + index * 2 + 1] ?? 0); + if (codeUnit !== PROTOTYPE_KEY.charCodeAt(index)) { + matches = false; + break; + } + } + if (matches) binaryPrototypeKeyObserved = true; + } + estimate += type === 4 ? size + 4 * Math.ceil(size / 3) : byteLength * 2; + if (estimate > maxBytes) + throw fail( + "expansion", + `binary plist scalar expansion exceeds ${budget}`, + ); + continue; + } + if (type !== 10 && type !== 13) continue; + const childCount = type === 13 ? size * 2 : size; + const refsEnd = cursor + childCount * referenceSize; + if (!Number.isSafeInteger(refsEnd) || refsEnd > objectsEnd) + throw new TypeError("binary plist reference table is truncated"); + const availableNodes = Math.floor( + (maxBytes - estimate) / expandedNodeBytes, + ); + if (childCount > availableNodes - pendingObjectCount) + throw fail( + "expansion", + `binary plist reference expansion exceeds ${budget}`, + ); + active.add(entry.object); + pending.push({ + object: entry.object, + depth: entry.depth, + exit: true, + dictionaryKey: false, + }); + for (let index = childCount - 1; index >= 0; index -= 1) { + const reference = readInteger( + cursor + index * referenceSize, + referenceSize, + ); + if (reference < 0 || reference >= objectCount) + throw new TypeError("binary plist reference is invalid"); + pending.push({ + object: reference, + depth: entry.depth + 1, + exit: false, + dictionaryKey: type === 13 && index < size, + }); + } + pendingObjectCount += childCount; + } + if (!numberProjection) + return { + estimatedBytes: estimate, + binaryPrototypeKeyObserved, + datePrecisionOverlapCount: 0, + reachableDateOffsets: [], + overlappingDateOffsets: [], + reachableWideIntegerOffsets: [], + overlappingWideIntegerOffsets: [], + reachableNumericOffsets: [], + }; + + // Distinct objects can legally overlap in binary plists. Rewriting a date or + // wide integer in place could change another reachable value, so isolate + // affected scalar records in the decoder copy. Track both overlap directions + // using the same bounded span parser as scalar collection. + const reachableSpans = [...reachableObjectOffsets] + .map((offset) => ({ + offset, + end: binaryPlistObjectEnd(bytes, offset, objectsEnd, referenceSize), + })) + .filter( + (span): span is { offset: number; end: number } => span.end !== undefined, + ) + .sort((left, right) => left.offset - right.offset); + const overlappingDateOffsets = findOverlappingScalarOffsets( + reachableSpans, + reachableDateOffsets, + ); + const overlappingWideIntegerOffsets = findOverlappingScalarOffsets( + reachableSpans, + reachableWideIntegerOffsets, + ); + const precisionUnknownDateOffsets = [...overlappingDateOffsets] + .filter((offset) => { + const seconds = new DataView( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength, + ).getFloat64(offset + 1); + return binaryDateRequiresExactSeconds(seconds); + }) + .sort((left, right) => left - right); + const cloneOffsets = new Set([ + ...precisionUnknownDateOffsets, + ...overlappingWideIntegerOffsets, + ]); + if (cloneOffsets.size > 0) { + const cloneBytes = cloneOffsets.size * 9; + const newOffsetSize = findExactScalarCloneOffsetWidth( + offsetSize, + offsetTable, + cloneOffsets.size, + ); + if (newOffsetSize === undefined) + throw new TypeError( + "binary plist cloned scalar offset exceeds supported width", + ); + estimate += cloneBytes + objectCount * (newOffsetSize - offsetSize); + if (estimate > maxBytes) + throw fail( + "expansion", + `binary plist scalar-copy expansion exceeds ${budget}`, + ); + } + const precisionUnknownOffsets = new Set(precisionUnknownDateOffsets); + const datePrecisionOverlapCount = reachableDateOccurrences.filter((offset) => + precisionUnknownOffsets.has(offset), + ).length; + return { + estimatedBytes: estimate, + binaryPrototypeKeyObserved, + datePrecisionOverlapCount, + reachableDateOffsets: [...reachableDateOffsets].sort( + (left, right) => left - right, + ), + overlappingDateOffsets: precisionUnknownDateOffsets, + reachableWideIntegerOffsets: [...reachableWideIntegerOffsets].sort( + (left, right) => left - right, + ), + overlappingWideIntegerOffsets: [...overlappingWideIntegerOffsets].sort( + (left, right) => left - right, + ), + reachableNumericOffsets: [...reachableNumericOffsets].sort( + (left, right) => left - right, + ), + }; +}; diff --git a/src/artifacts/extraction/ArtifactExtraction.test.ts b/src/artifacts/extraction/ArtifactExtraction.test.ts new file mode 100644 index 0000000000..d22855d6e6 --- /dev/null +++ b/src/artifacts/extraction/ArtifactExtraction.test.ts @@ -0,0 +1,54 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it, vi } from "vitest"; + +import { ArtifactReaderFailure } from "../ArtifactReader.js"; +import { ArtifactResourceScope } from "../ArtifactResourceScope.js"; +import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import { extractArtifact } from "./ArtifactExtraction.js"; + +vi.mock("../pyinstaller/CArchiveLayout.js", async (importOriginal) => ({ + ...(await importOriginal()), + containsPyInstallerArchive: () => + Promise.reject(new ArtifactReaderFailure("format", "detection failed")), +})); + +it("keeps the detection error primary and reports a failed detection close", async () => { + const directory = await createTestTempDirectory("rea-extraction-detect-"); + const path = join(directory, "target.elf"); + await writeFile(path, Buffer.from("not an archive")); + const resourceScope = new ArtifactResourceScope(); + const originalClose = OwnedFileHandle.prototype.close; + const closeFailure = new Error("detection close failed"); + let attempts = 0; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + attempts += 1; + if (attempts === 1) throw closeFailure; + await originalClose.call(this); + }); + try { + await expect( + extractArtifact({ + inputPath: path, + inputFormat: "elf", + outputRoot: join(directory, "out"), + environment: process.env, + integrityPolicy: "fail", + resourceScope, + }), + ).rejects.toMatchObject({ + reason: "format", + message: "detection failed", + cleanup: { reason: closeFailure.message }, + }); + await resourceScope.close(); + expect(attempts).toBe(2); + } finally { + close.mockRestore(); + await resourceScope.close().catch(() => undefined); + } +}); diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts index 8a3a4cb0bc..2b5b591f1b 100644 --- a/src/artifacts/extraction/ArtifactExtraction.ts +++ b/src/artifacts/extraction/ArtifactExtraction.ts @@ -23,6 +23,8 @@ import { SafeOutputTreeCreationFailure } from "../SafeOutputTreeCreationFailure. import { ZipArtifactReader } from "../ZipArtifactReader.js"; import { isZipPackageFormat } from "../../domain/zipPackageFormat.js"; import { MachOSliceArtifactReader } from "../MachOSliceArtifactReader.js"; +import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js"; +import { containsPyInstallerArchive } from "../pyinstaller/CArchiveLayout.js"; import { artifactExtractionResultSchema, type ArtifactExtractionResult, @@ -244,6 +246,7 @@ const materializeSelection = async ({ sourcePath, input, inventory.manifest.root_format, + signal, ); const readerOwner: ArtifactResourceOwner = { kind: "reader" as const, @@ -314,14 +317,15 @@ const writeSelectedEntries = async ({ for await (const entry of reader.entries(signal)) { const path = normalizeArtifactPath(entry.path); registry.add(path, entry.kind); + // Logical declarations may share a query path with a real file. They do + // not consume that file's selection or participate in materialization. + if (entry.kind !== "file" && entry.kind !== "slice") continue; const selectedItem = byPath.get(path); if (selectedItem === undefined) { - if (entry.kind === "file" || entry.kind === "slice") - throw new ArtifactReaderFailure( - "integrity", - `Regular artifact entry is missing from inventory: ${path}`, - ); - continue; + throw new ArtifactReaderFailure( + "integrity", + `Regular artifact entry is missing from inventory: ${path}`, + ); } preflight(entry); const stream = await reader.open(entry, signal); @@ -444,10 +448,17 @@ const requireExtractableFormat = async ( path, input.resourceScope, signal, - async (handle) => - locateGodotPck(handle, (await handle.stat()).size, signal), + async (handle) => { + const size = (await handle.stat()).size; + if (await locateGodotPck(handle, size, signal)) return true; + if (format !== "elf" && format !== "pe") return false; + abortIfNeeded(signal); + const packaged = await containsPyInstallerArchive(handle, size, signal); + abortIfNeeded(signal); + return packaged; + }, ); - if (pack === undefined) + if (!pack) throw new AnalysisUnsupportedTargetError( "extract_artifact", input.inputPath, @@ -464,10 +475,13 @@ const createReader = async ( "inputPath" | "inputFormat" | "environment" | "resourceScope" >, containerFormat: ArtifactGraphManifest["root_format"], + signal?: AbortSignal, ): Promise => { const format = input.inputFormat; if (containerFormat === "godot-pck") return new GodotPckArtifactReader(path); - if (await requireExtractableFormat(path, input)) + if (containerFormat === "pyinstaller") + return new PyInstallerArtifactReader(path); + if (await requireExtractableFormat(path, input, signal)) return new DirectoryArtifactReader(path); if (format === "asar") return new AsarArtifactReader(path); if (isZipPackageFormat(format)) return new ZipArtifactReader(path, format); diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts index 7432e7751c..1c5fd3edc8 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts @@ -10,7 +10,7 @@ import { createArtifactEdges, classifyArtifactContent, createOccurrence, - indexOccurrencesByPath, + indexDestinationOccurrencesByPath, resolveOccurrenceParents, } from "./ArtifactGraphConstruction.js"; import { createArtifactClassificationProbe } from "../ArtifactClassificationProbe.js"; @@ -42,7 +42,7 @@ describe("occurrence parent ownership", () => { resolveOccurrenceParents( occurrences, - indexOccurrencesByPath(occurrences), + indexDestinationOccurrencesByPath(occurrences), new Set(), ); @@ -57,7 +57,7 @@ describe("occurrence parent ownership", () => { const resolution = resolveOccurrenceParents( occurrences, - indexOccurrencesByPath(occurrences), + indexDestinationOccurrencesByPath(occurrences), new Set(), ); diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.ts b/src/artifacts/inventory/ArtifactGraphConstruction.ts index ad828bf97d..3cf2c9caee 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.ts @@ -1,6 +1,7 @@ import { hasGodotPckHeader } from "../../domain/godotPckFormat.js"; import type { ArtifactClassificationEvidence } from "../ArtifactClassificationProbe.js"; import { ArtifactReaderFailure } from "../ArtifactReader.js"; +import { hasDestinationPath } from "../ArtifactPaths.js"; import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; import { isJavaScriptSourcePath } from "../../domain/javascript/javascriptArtifactFiles.js"; import { digestCanonicalValue } from "../../domain/canonicalDigest.js"; @@ -56,12 +57,13 @@ export interface MutableOccurrence { limitations: string[]; } -/** Index every occurrence by canonical logical path without choosing an owner. */ -export const indexOccurrencesByPath = ( +/** Index destination paths without mixing in declaration query names or choosing an owner. */ +export const indexDestinationOccurrencesByPath = ( occurrences: readonly MutableOccurrence[], ): ReadonlyMap => { const byPath = new Map(); for (const occurrence of occurrences) { + if (!hasDestinationPath(occurrence.entry_kind)) continue; const matches = byPath.get(occurrence.logical_path); if (matches === undefined) byPath.set(occurrence.logical_path, [occurrence]); @@ -82,16 +84,18 @@ export const createOccurrence = ( duplicateOrdinal = 0, ): MutableOccurrence => { const role = - entry.kind === "directory" - ? { - kind: path.toLowerCase().endsWith(".framework") - ? ("framework" as const) - : ("container" as const), - format: "directory" as const, - } - : containerSuffixFamily(path) === undefined - ? classifyArtifactPath(path) - : ({ kind: "unknown", format: "unknown" } as const); + entry.kind === "metadata" + ? { kind: "package-metadata" as const, format: "unknown" as const } + : entry.kind === "directory" + ? { + kind: path.toLowerCase().endsWith(".framework") + ? ("framework" as const) + : ("container" as const), + format: "directory" as const, + } + : containerSuffixFamily(path) === undefined + ? classifyArtifactPath(path) + : ({ kind: "unknown", format: "unknown" } as const); return { occurrence_id: `occ_${digestCanonicalValue( duplicateOrdinal === 0 @@ -263,7 +267,7 @@ export const rootOccurrenceFor = ( artifact_kind: metadata.format === "directory" || isZipPackageFormat(metadata.format) || - ["asar", "godot-pck", "dmg", "pkg"].includes(metadata.format) + ["asar", "godot-pck", "dmg", "pkg", "pyinstaller"].includes(metadata.format) ? "container" : artifactRoleForFormat(metadata.path, metadata.format).kind, artifact_format: metadata.format, @@ -316,7 +320,7 @@ export const createArtifactEdges = ( producer?: ArtifactCommand, ): { readonly edges: ArtifactEdge[]; readonly limitations: string[] } => { const byId = new Map(occurrences.map((item) => [item.occurrence_id, item])); - const byPath = indexOccurrencesByPath(occurrences); + const byPath = indexDestinationOccurrencesByPath(occurrences); const edges: ArtifactEdge[] = []; const limitations: string[] = []; for (const occurrence of occurrences) { @@ -364,11 +368,14 @@ export const createArtifactEdges = ( relation: occurrence.entry_kind === "slice" ? ("slice-of" as const) - : isSourceMap - ? mappedSource === undefined - ? ("contains" as const) - : ("maps-source" as const) - : relationFor(occurrence.logical_path), + : occurrence.format_metadata?.kind === "pyinstaller-entry" && + occurrence.format_metadata.bytecode_header === "derived" + ? ("derived-from" as const) + : isSourceMap + ? mappedSource === undefined + ? ("contains" as const) + : ("maps-source" as const) + : relationFor(occurrence.logical_path), occurrence_id: occurrence.occurrence_id, }; edges.push({ @@ -428,6 +435,9 @@ export const resolveOccurrenceParents = ( const limitations = new Set(); const unknownParentOccurrences = new Set(); for (const occurrence of occurrences) { + // Query paths do not imply filesystem containment. The scanner records + // the declaring container for metadata instead. + if (occurrence.entry_kind === "metadata") continue; const path = occurrence.logical_path; let branch = root; let parent: MutableOccurrence | undefined; diff --git a/src/artifacts/inventory/classify.ts b/src/artifacts/inventory/classify.ts index 06e75b3899..c36d6e43df 100644 --- a/src/artifacts/inventory/classify.ts +++ b/src/artifacts/inventory/classify.ts @@ -5,6 +5,7 @@ import type { Stats } from "node:fs"; import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js"; import type { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js"; +import { containsPyInstallerArchive } from "../pyinstaller/CArchiveLayout.js"; import { classifyArtifactContent } from "./ArtifactGraphConstruction.js"; import { abortIfNeeded } from "../ArtifactHash.js"; import { createArtifactClassificationProbe } from "../ArtifactClassificationProbe.js"; @@ -114,7 +115,8 @@ export const classifyAndHashRootForInventory = async ( const retainSource = isZipPackageFormat(format) || format === "mach-o-universal" || - format === "godot-pck"; + format === "godot-pck" || + format === "pyinstaller"; outcome = { kind: "completed", value: { @@ -178,7 +180,7 @@ const classifyRootFormat = async ( evidence.fatSliceHeaders, signal, ); - return classifyArtifactContent( + const format = classifyArtifactContent( path, { ...evidence, @@ -186,6 +188,16 @@ const classifyRootFormat = async ( }, archiveBytes, ).format; + if (["elf", "pe", "mach-o", "mach-o-universal"].includes(format)) { + const packaged = await containsPyInstallerArchive( + handle, + archiveBytes, + signal, + ); + abortIfNeeded(signal); + if (packaged) return "pyinstaller"; + } + return format; }; /** Read exact FAT header windows missed by the bounded prefix probe. */ diff --git a/src/artifacts/inventory/reader.ts b/src/artifacts/inventory/reader.ts index 03b91077f6..8b9956a89e 100644 --- a/src/artifacts/inventory/reader.ts +++ b/src/artifacts/inventory/reader.ts @@ -7,6 +7,7 @@ import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js"; import { GodotPckArtifactReader } from "../GodotPckArtifactReader.js"; import { MachOSliceArtifactReader } from "../MachOSliceArtifactReader.js"; import { NativeDmgArtifactReader } from "../NativeDmgArtifactReader.js"; +import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js"; import { ZipArtifactReader } from "../ZipArtifactReader.js"; import type { ArtifactInventoryCoverage, @@ -31,6 +32,8 @@ export const createReader = ( return new DirectoryArtifactReader(path); case "asar": return new AsarArtifactReader(path); + case "pyinstaller": + return new PyInstallerArtifactReader(path, rootSource); case "godot-pck": return new GodotPckArtifactReader(path, rootSource); case "mach-o-universal": diff --git a/src/artifacts/inventory/scanReader.ts b/src/artifacts/inventory/scanReader.ts index 616eafc0a0..9d0be9d0b8 100644 --- a/src/artifacts/inventory/scanReader.ts +++ b/src/artifacts/inventory/scanReader.ts @@ -26,7 +26,7 @@ import { createArtifactNode, createOccurrence, noteUnreadContainerSuffix, - indexOccurrencesByPath, + indexDestinationOccurrencesByPath, resolveOccurrenceParents, type MutableOccurrence, } from "./ArtifactGraphConstruction.js"; @@ -107,7 +107,7 @@ export const scanReader = async ( await visitArtifactEntries(context, reader, ""); // Archive directories may appear after their children. Resolve containment // against the complete index before directory identities are materialized. - const byPath = indexOccurrencesByPath(occurrences); + const byPath = indexDestinationOccurrencesByPath(occurrences); const parentResolution = resolveOccurrenceParents( occurrences, byPath, @@ -167,11 +167,13 @@ const visitArtifactEntries = async ( const stack: Array<{ readonly reader: ArtifactReader; readonly prefix: string; + readonly containerId: string | null; readonly iterator: AsyncIterator; }> = [ { reader: currentReader, prefix, + containerId: null, iterator: inventoryEntries(context, currentReader, prefix), }, ]; @@ -202,7 +204,7 @@ const visitArtifactEntries = async ( const occurrence = createOccurrence( entry, logicalPath, - null, + entry.kind === "metadata" ? frame.containerId : null, duplicateOrdinal, ); let digested: @@ -254,6 +256,7 @@ const visitArtifactEntries = async ( stack.push({ reader: nested, prefix: logicalPath, + containerId: occurrence.occurrence_id, iterator: inventoryEntries(context, nested, logicalPath), }); } diff --git a/src/artifacts/pyinstaller/CArchive.fixture.ts b/src/artifacts/pyinstaller/CArchive.fixture.ts new file mode 100644 index 0000000000..de4691ac18 --- /dev/null +++ b/src/artifacts/pyinstaller/CArchive.fixture.ts @@ -0,0 +1,211 @@ +import { deflateSync } from "node:zlib"; + +import { PYINSTALLER_COOKIE_MAGIC } from "./CArchiveLayout.js"; + +/** Final CPython 3.9 bytecode magic, little-endian with the CR LF trailer. */ +export const PYTHON_39_MAGIC = Buffer.from("610d0d0a", "hex"); + +const ELF_PREFIX = (() => { + const bytes = Buffer.alloc(64); + bytes.writeUInt32LE(0x464c457f, 0); + bytes[4] = 2; + bytes[5] = 1; + bytes[6] = 1; + bytes.writeUInt16LE(2, 16); + bytes.writeUInt16LE(62, 18); + bytes.writeUInt32LE(1, 20); + bytes.writeUInt16LE(64, 52); + return bytes; +})(); + +/** One CArchive member supplied as uncompressed bytes. */ +export interface CArchiveFixtureMember { + readonly name: string; + readonly typeCode: string; + readonly payload: Buffer; + readonly compress: boolean; + readonly compressedSuffix?: Buffer; + readonly declaredUncompressedSize?: number; +} + +/** Build a tiny ELF with an appended CArchive, optionally followed by a trailer. */ +export const cArchiveExecutable = (input: { + readonly layout: "2.0" | "2.1"; + readonly pythonVersion: number; + readonly pythonLibrary?: string; + readonly members: readonly CArchiveFixtureMember[]; + readonly trailer?: Buffer; + readonly tocBytes?: Buffer; +}): Buffer => { + const payloads: Buffer[] = []; + const entries: Buffer[] = []; + let dataLength = 0; + for (const member of input.members) { + const stored = member.compress + ? Buffer.concat([ + deflateSync(member.payload), + member.compressedSuffix ?? Buffer.alloc(0), + ]) + : member.payload; + entries.push( + tocEntry({ + name: member.name, + typeCode: member.typeCode, + offset: dataLength, + compressedSize: stored.length, + uncompressedSize: + member.declaredUncompressedSize ?? member.payload.length, + flag: member.compress ? 1 : 0, + }), + ); + payloads.push(stored); + dataLength += stored.length; + } + const toc = input.tocBytes ?? Buffer.concat(entries); + const cookie = cookieBytes({ + layout: input.layout, + pythonVersion: input.pythonVersion, + pythonLibrary: input.pythonLibrary, + tocOffset: dataLength, + tocLength: toc.length, + }); + return Buffer.concat([ + ELF_PREFIX, + ...payloads, + toc, + cookie, + input.trailer ?? Buffer.alloc(0), + ]); +}; + +/** Cookie whose package range starts before the file. */ +export const cookieRangeOutsideFile = (): Buffer => { + const cookie = Buffer.alloc(88); + PYINSTALLER_COOKIE_MAGIC.copy(cookie, 0); + cookie.writeUInt32BE(200, 8); + cookie.writeUInt32BE(80, 12); + cookie.writeUInt32BE(32, 16); + cookie.writeUInt32BE(314, 20); + return Buffer.concat([ELF_PREFIX, cookie]); +}; + +/** PYZ image. Module bodies are zlib-compressed. Namespace entries have no body. */ +export const pyzArchive = (input: { + readonly magic: Buffer; + readonly modules: readonly PyzFixtureModule[]; + readonly toc?: Buffer; +}): Buffer => { + const blobs: Buffer[] = []; + const entries: Buffer[] = []; + let offset = 12; + for (const module of input.modules) { + const compressed = + module.typeCode === 3 && module.legacyNamespacePayload !== true + ? Buffer.alloc(0) + : Buffer.concat([ + deflateSync(module.body), + module.compressedSuffix ?? Buffer.alloc(0), + ]); + entries.push( + marshalPair(module.name, module.typeCode, offset, compressed.length), + ); + if (compressed.length > 0) { + blobs.push(compressed); + offset += compressed.length; + } + } + const toc = input.toc ?? marshalList(entries); + const header = Buffer.alloc(12); + Buffer.from("PYZ\0").copy(header, 0); + input.magic.copy(header, 4, 0, 4); + header.writeUInt32BE( + 12 + blobs.reduce((sum, blob) => sum + blob.length, 0), + 8, + ); + return Buffer.concat([header, ...blobs, toc]); +}; + +/** Type 3 normally stores no body; older producers can write a namespace payload. */ +export interface PyzFixtureModule { + readonly legacyNamespacePayload?: boolean; + readonly compressedSuffix?: Buffer; + readonly name: string; + readonly typeCode: number; + readonly body: Buffer; +} + +const tocEntry = (input: { + readonly name: string; + readonly typeCode: string; + readonly offset: number; + readonly compressedSize: number; + readonly uncompressedSize: number; + readonly flag: number; +}): Buffer => { + const name = Buffer.from(input.name, "utf8"); + const length = Math.ceil((18 + name.length + 1) / 16) * 16; + const bytes = Buffer.alloc(length); + bytes.writeUInt32BE(length, 0); + bytes.writeUInt32BE(input.offset, 4); + bytes.writeUInt32BE(input.compressedSize, 8); + bytes.writeUInt32BE(input.uncompressedSize, 12); + bytes[16] = input.flag; + bytes[17] = input.typeCode.charCodeAt(0); + name.copy(bytes, 18); + return bytes; +}; + +const cookieBytes = (input: { + readonly layout: "2.0" | "2.1"; + readonly pythonVersion: number; + readonly pythonLibrary: string | undefined; + readonly tocOffset: number; + readonly tocLength: number; +}): Buffer => { + const size = input.layout === "2.1" ? 88 : 24; + const bytes = Buffer.alloc(size); + PYINSTALLER_COOKIE_MAGIC.copy(bytes, 0); + bytes.writeUInt32BE(input.tocOffset + input.tocLength + size, 8); + bytes.writeUInt32BE(input.tocOffset, 12); + bytes.writeUInt32BE(input.tocLength, 16); + bytes.writeUInt32BE(input.pythonVersion, 20); + if (input.pythonLibrary !== undefined) + Buffer.from(input.pythonLibrary, "utf8").copy(bytes, 24); + return bytes; +}; + +const marshalList = (items: readonly Buffer[]): Buffer => { + const count = Buffer.alloc(4); + count.writeInt32LE(items.length, 0); + return Buffer.concat([Buffer.from([0x5b]), count, ...items]); +}; + +const marshalPair = ( + name: string, + typeCode: number, + offset: number, + length: number, +): Buffer => + smallTuple([ + marshalString(name), + smallTuple([int32(typeCode), int32(offset), int32(length)]), + ]); + +const smallTuple = (items: readonly Buffer[]): Buffer => + Buffer.concat([Buffer.from([0x29, items.length]), ...items]); + +const marshalString = (text: string): Buffer => { + const body = Buffer.from(text, "utf8"); + if (body.length <= 255 && body.every((byte) => byte < 0x80)) + return Buffer.concat([Buffer.from([0x7a, body.length]), body]); + const length = Buffer.alloc(4); + length.writeUInt32LE(body.length); + return Buffer.concat([Buffer.from([0x75]), length, body]); +}; + +const int32 = (value: number): Buffer => { + const bytes = Buffer.alloc(5); + bytes[0] = 0x69; + bytes.writeInt32LE(value, 1); + return bytes; +}; diff --git a/src/artifacts/pyinstaller/CArchiveLayout.ts b/src/artifacts/pyinstaller/CArchiveLayout.ts new file mode 100644 index 0000000000..b5bc1f4375 --- /dev/null +++ b/src/artifacts/pyinstaller/CArchiveLayout.ts @@ -0,0 +1,389 @@ +import type { FileHandle } from "node:fs/promises"; +import { readPyInstallerBytes } from "./PyInstallerBytes.js"; + +import { err, ok, type Result } from "../../domain/result.js"; +import { + pyInstallerFailure, + type PyInstallerFailure, +} from "./PyInstallerFailure.js"; + +/** Magic at the start of a CArchive cookie. */ +export const PYINSTALLER_COOKIE_MAGIC = Buffer.from("MEI\x0c\x0b\x0a\x0b\x0e"); + +/** PyInstaller 2.1+ cookie size, including the 64-byte library name. */ +export const PYINSTALLER_COOKIE_V21_BYTES = 88; + +/** PyInstaller 2.0 cookie size, without the library name. */ +export const PYINSTALLER_COOKIE_V20_BYTES = 24; + +/** + * How far before EOF the cookie may sit. macOS code signatures are appended + * after the cookie, and a large binary's signature hashes are larger than the + * bootloader's own 8 KiB scan chunk, so detection uses one bounded tail. + */ +export const PYINSTALLER_COOKIE_SEARCH_BYTES = 16 * 1024 * 1024; + +const MAX_TOC_BYTES = 32 * 1024 * 1024; +const MAX_TOC_ENTRIES = 100_000; +const TOC_HEADER_BYTES = 18; + +/** Observed CArchive cookie. Offsets are relative to the archive start. */ +export interface PyInstallerCookie { + readonly layout: "2.0" | "2.1"; + readonly archiveStart: number; + readonly packageLength: number; + readonly tocOffset: number; + readonly tocLength: number; + readonly pythonVersion: number; + readonly pythonLibrary: string | null; + readonly cookieOffset: number; +} + +/** One CArchive table entry. */ +export interface CArchiveEntry { + readonly index: number; + readonly tocRecordOffset: number; + readonly tocRecordLength: number; + readonly name: string; + readonly typeCode: string; + readonly offset: number; + readonly compressedSize: number; + readonly uncompressedSize: number; + readonly compressionFlag: number; +} + +/** Runtime options and external dependencies are declarations, never local blobs. */ +export const isCArchiveDeclaration = ( + entry: CArchiveEntry, +): entry is CArchiveEntry & { readonly typeCode: "o" | "d" } => + entry.typeCode === "o" || entry.typeCode === "d"; + +/** True when a native executable has a structurally valid CArchive cookie. */ +export const containsPyInstallerArchive = async ( + handle: FileHandle, + fileSize: number, + signal?: AbortSignal, +): Promise => { + const found = await findPyInstallerCookie(handle, fileSize, signal); + return found?.ok === true; +}; + +/** Locate a structurally valid cookie. Invalid magic copies are skipped. */ +export const findPyInstallerCookie = async ( + handle: FileHandle, + fileSize: number, + signal?: AbortSignal, +): Promise | undefined> => { + if ( + !Number.isSafeInteger(fileSize) || + fileSize < PYINSTALLER_COOKIE_V20_BYTES + ) + return undefined; + const tailSize = Math.min(fileSize, PYINSTALLER_COOKIE_SEARCH_BYTES); + const read = await readPyInstallerBytes(handle, { + position: fileSize - tailSize, + length: tailSize, + label: "CArchive tail", + signal, + }); + if (!read.ok) return read; + const observed = read.value; + let failure: PyInstallerFailure | undefined; + let from = observed.length; + while (from >= PYINSTALLER_COOKIE_MAGIC.length) { + const relative = observed + .subarray(0, from) + .lastIndexOf(PYINSTALLER_COOKIE_MAGIC); + if (relative < 0) break; + const cookieOffset = fileSize - tailSize + relative; + const parsed = parseCookieAt(observed, relative, cookieOffset, fileSize); + if (parsed.ok) return parsed; + failure = parsed.error; + from = relative; + } + return failure === undefined ? undefined : err(failure); +}; + +/** Read and parse the TOC that a validated cookie names. */ +export const readCArchiveToc = async ( + handle: FileHandle, + cookie: PyInstallerCookie, + signal?: AbortSignal, +): Promise> => { + if (cookie.tocLength > MAX_TOC_BYTES) + return err( + pyInstallerFailure( + "limit", + "CArchive table of contents exceeds its bound", + ), + ); + const position = cookie.archiveStart + cookie.tocOffset; + const read = await readPyInstallerBytes(handle, { + position, + length: cookie.tocLength, + label: "CArchive table of contents", + signal, + }); + if (!read.ok) return read; + const parsed = parseCArchiveToc(read.value); + if (!parsed.ok) return parsed; + const valid = validateCArchiveMembers(parsed.value, cookie.tocOffset); + return valid.ok ? parsed : valid; +}; + +/** Parse TOC bytes. Each entry is a big-endian header plus a NUL-padded name. */ +export const parseCArchiveToc = ( + bytes: Buffer, +): Result => { + const entries: CArchiveEntry[] = []; + let offset = 0; + while (offset < bytes.length) { + if (entries.length >= MAX_TOC_ENTRIES) + return err( + pyInstallerFailure( + "limit", + "CArchive table of contents exceeds its entry bound", + ), + ); + if (offset + TOC_HEADER_BYTES > bytes.length) + return err( + pyInstallerFailure( + "format", + "CArchive table of contents entry is truncated", + ), + ); + const entryLength = bytes.readUInt32BE(offset); + if (entryLength < TOC_HEADER_BYTES || offset + entryLength > bytes.length) + return err( + pyInstallerFailure( + "format", + "CArchive table of contents entry extends outside the table", + ), + ); + const nameBytes = bytes.subarray( + offset + TOC_HEADER_BYTES, + offset + entryLength, + ); + const name = tocName(nameBytes); + if (!name.ok) return name; + const compressionFlag = bytes[offset + 16] ?? 0; + if (compressionFlag !== 0 && compressionFlag !== 1) + return err( + pyInstallerFailure( + "format", + `CArchive compression flag ${String(compressionFlag)} is not stored or zlib`, + ), + ); + const typeCode = bytes[offset + 17] ?? 0; + if (typeCode < 0x20 || typeCode > 0x7e) + return err( + pyInstallerFailure( + "format", + "CArchive type code is not a printable character", + ), + ); + entries.push({ + index: entries.length, + tocRecordOffset: offset, + tocRecordLength: entryLength, + name: name.value, + typeCode: String.fromCharCode(typeCode), + offset: bytes.readUInt32BE(offset + 4), + compressedSize: bytes.readUInt32BE(offset + 8), + uncompressedSize: bytes.readUInt32BE(offset + 12), + compressionFlag, + }); + offset += entryLength; + } + return ok(entries); +}; + +const parseCookieAt = ( + tail: Buffer, + relative: number, + cookieOffset: number, + fileSize: number, +): Result => { + const modern = parseCookie(tail, relative, cookieOffset, fileSize, "2.1"); + if (modern.ok) return modern; + const legacy = parseCookie(tail, relative, cookieOffset, fileSize, "2.0"); + return legacy.ok ? legacy : modern; +}; + +const parseCookie = ( + tail: Buffer, + relative: number, + cookieOffset: number, + fileSize: number, + layout: "2.0" | "2.1", +): Result => { + const cookieBytes = + layout === "2.1" + ? PYINSTALLER_COOKIE_V21_BYTES + : PYINSTALLER_COOKIE_V20_BYTES; + if (relative + cookieBytes > tail.length) + return err( + pyInstallerFailure( + "format", + "CArchive cookie extends past the end of the file", + ), + ); + const view = tail.subarray(relative, relative + cookieBytes); + const packageLength = view.readUInt32BE(8); + const tocOffset = view.readUInt32BE(12); + const tocLength = view.readUInt32BE(16); + const pythonVersion = view.readUInt32BE(20); + if ( + !safeSum(tocOffset, tocLength, cookieBytes) || + packageLength !== tocOffset + tocLength + cookieBytes + ) + return err( + pyInstallerFailure( + "format", + "CArchive cookie does not describe a table of contents inside the package", + ), + ); + const archiveStart = cookieOffset + cookieBytes - packageLength; + if (archiveStart < 0 || archiveStart + packageLength > fileSize) + return err( + pyInstallerFailure("format", "CArchive cookie range is outside the file"), + ); + const library = + layout === "2.1" ? libraryName(view.subarray(24, 88)) : ok(null); + if (!library.ok) return library; + return ok({ + layout, + archiveStart, + packageLength, + tocOffset, + tocLength, + pythonVersion, + pythonLibrary: library.value, + cookieOffset, + }); +}; + +const libraryName = ( + bytes: Buffer, +): Result => { + const end = bytes.indexOf(0); + const text = end < 0 ? bytes : bytes.subarray(0, end); + if (end >= 0 && bytes.subarray(end).some((byte) => byte !== 0)) + return err( + pyInstallerFailure( + "format", + "CArchive Python library name is not NUL-padded", + ), + ); + if (text.length === 0) return ok(null); + try { + return ok( + new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(text), + ); + } catch { + return err( + pyInstallerFailure( + "format", + "CArchive Python library name is not valid UTF-8", + ), + ); + } +}; + +const tocName = (bytes: Buffer): Result => { + const end = bytes.indexOf(0); + const text = end < 0 ? bytes : bytes.subarray(0, end); + if (text.length === 0) + return err(pyInstallerFailure("format", "CArchive entry name is empty")); + try { + return ok( + new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(text), + ); + } catch { + return err( + pyInstallerFailure("format", "CArchive entry name is not valid UTF-8"), + ); + } +}; + +const safeSum = (left: number, middle: number, right: number): boolean => + Number.isSafeInteger(left) && + Number.isSafeInteger(middle) && + Number.isSafeInteger(right) && + left >= 0 && + middle >= 0 && + right >= 0 && + left + middle + right <= Number.MAX_SAFE_INTEGER; + +/** Source directory semantics are validated once before any consumer projects paths. */ +const validateCArchiveMembers = ( + entries: readonly CArchiveEntry[], + tocOffset: number, +): Result => { + const names = new Set(); + for (const entry of entries) { + if (isCArchiveDeclaration(entry)) { + if ( + entry.compressedSize !== 0 || + entry.uncompressedSize !== 0 || + entry.compressionFlag !== 0 + ) + return err( + pyInstallerFailure( + "integrity", + `CArchive declaration at table index ${String(entry.index)} has local payload or compression`, + ), + ); + } else { + if (names.has(entry.name)) + return err( + pyInstallerFailure( + "integrity", + `Duplicate CArchive member name at table index ${String(entry.index)}`, + ), + ); + names.add(entry.name); + } + if ( + entry.compressionFlag === 0 && + entry.compressedSize !== entry.uncompressedSize + ) + return err( + pyInstallerFailure( + "integrity", + `Stored CArchive member size disagrees with its table entry at table index ${String(entry.index)}`, + ), + ); + if (entry.offset > tocOffset) + return err( + pyInstallerFailure( + "format", + `CArchive member offset is outside the archive at table index ${String(entry.index)}`, + ), + ); + } + const ranges = entries + .filter((entry) => entry.compressedSize > 0) + .map((entry) => ({ + index: entry.index, + start: entry.offset, + end: entry.offset + entry.compressedSize, + })) + .sort((left, right) => left.start - right.start); + let occupied = 0; + for (const range of ranges) { + if ( + !Number.isSafeInteger(range.end) || + range.end > tocOffset || + range.start < occupied + ) + return err( + pyInstallerFailure( + "integrity", + `CArchive member extends outside the archive or overlaps another member at table index ${String(range.index)}`, + ), + ); + occupied = range.end; + } + return ok(null); +}; diff --git a/src/artifacts/pyinstaller/CArchiveMembers.ts b/src/artifacts/pyinstaller/CArchiveMembers.ts new file mode 100644 index 0000000000..2047f6e006 --- /dev/null +++ b/src/artifacts/pyinstaller/CArchiveMembers.ts @@ -0,0 +1,340 @@ +import type { FileHandle } from "node:fs/promises"; +import { copyArtifactEntry, type ArtifactEntry } from "../ArtifactReader.js"; +import { isCArchiveDeclaration, type CArchiveEntry } from "./CArchiveLayout.js"; +import type { PyzArchive, PyzEntry } from "./PyzArchive.js"; +import { + RECONSTRUCTED_PYTHON_HEADER_LIMITATION, + reconstructedPycHeader, +} from "./PythonBytecode.js"; +import { + cArchiveEntryMetadata, + pyzEntryMetadata, + pyzEntryEncrypted, +} from "./PyInstallerMetadata.js"; +import type { PyInstallerEntryMetadata } from "../../domain/artifactFormatMetadata.js"; + +const CRYPTO_KEY_NAME = "pyimod00_crypto_key"; + +/** Parsed facts and the descriptor identity used to locate a retained image. */ +export interface PyzImage { + readonly archive: PyzArchive; + readonly handle: FileHandle; + readonly start: number; +} + +/** Stable entry descriptions and bounded read plans; expanded member bytes are never retained. */ +export interface PyInstallerMember { + readonly entry: ArtifactEntry; + readonly read: + | { + readonly kind: "carchive"; + readonly archiveStart: number; + readonly source: CArchiveEntry; + readonly header: Buffer | undefined; + } + | { + readonly kind: "pyz"; + readonly handle: FileHandle; + readonly offset: number; + readonly length: number; + readonly compressed: boolean; + readonly uncompressedSize: number | undefined; + readonly header: Buffer | undefined; + } + | undefined; +} + +/** The producer's key is an importable, nonempty Python module, not a data filename. */ +export const hasEncryptionMarker = ( + entries: readonly CArchiveEntry[], +): boolean => + entries.some( + (entry) => + entry.name === CRYPTO_KEY_NAME && + (entry.typeCode === "m" || entry.typeCode === "M") && + entry.uncompressedSize > 0, + ); + +export const projectStoredMember = ( + entry: CArchiveEntry, + archiveStart: number, + pythonMagic: Buffer | undefined, +): PyInstallerMember => { + if (isCArchiveDeclaration(entry)) + return { + entry: fileEntry({ + path: `__pyinstaller__/${entry.typeCode === "o" ? "runtime-options" : "dependencies"}/${String(entry.index)}`, + kind: "metadata", + declaredSize: entry.uncompressedSize, + compressedSize: entry.compressedSize, + byteOffset: null, + limitations: + entry.typeCode === "d" + ? [ + "Multipackage dependency bytes belong to another executable and remain unresolved.", + ] + : [], + adapterKey: `carchive:${String(entry.index)}:${entry.name}`, + formatMetadata: cArchiveEntryMetadata(entry), + }), + read: undefined, + }; + const header = + pythonMagic !== undefined && isBytecodeType(entry.typeCode) + ? reconstructedPycHeader(pythonMagic) + : undefined; + const path = canonicalMemberPath(entry.name); + const symlink = entry.typeCode === "n"; + const limitations: string[] = []; + if (symlink) limitations.push("PyInstaller symlink target was not read."); + if (header !== undefined) + limitations.push(RECONSTRUCTED_PYTHON_HEADER_LIMITATION); + const declaredSize = entry.uncompressedSize; + return { + entry: fileEntry({ + path, + kind: symlink ? "symlink" : "file", + declaredSize, + compressedSize: entry.compressedSize, + byteOffset: sourceByteOffset(entry, archiveStart), + limitations, + adapterKey: `carchive:${String(entry.index)}:${entry.name}`, + formatMetadata: cArchiveEntryMetadata(entry, header), + }), + read: { + kind: "carchive", + archiveStart, + source: entry, + header, + }, + }; +}; + +export const projectPyzContainer = ( + entry: CArchiveEntry, + archiveStart: number, + image: PyzImage, + encrypted: boolean, +): PyInstallerMember => { + const { archive } = image; + const path = canonicalMemberPath(entry.name); + const byteOffset = sourceByteOffset(entry, archiveStart); + const limitations: string[] = []; + if (archive.kind === "opaque") + return { + entry: fileEntry({ + path, + declaredSize: entry.uncompressedSize, + compressedSize: entry.compressedSize, + byteOffset, + // Legacy encryption applies to inner payloads, never the PYZ image. + encrypted: false, + limitations: [ + ...limitations, + archive.limitation, + ...(encrypted + ? ["PYZ entries are encrypted and were not decoded."] + : []), + ], + adapterKey: `carchive:${String(entry.index)}:${entry.name}`, + formatMetadata: cArchiveEntryMetadata( + entry, + undefined, + archive, + encrypted, + ), + }), + read: + entry.compressionFlag === 1 + ? { + kind: "pyz", + handle: image.handle, + offset: image.start, + length: entry.uncompressedSize, + compressed: false, + uncompressedSize: entry.uncompressedSize, + header: undefined, + } + : { + kind: "carchive", + archiveStart, + source: entry, + header: undefined, + }, + }; + return { + entry: { + path, + kind: "directory", + declaredSize: entry.uncompressedSize, + compressedSize: entry.compressedSize, + executable: false, + encrypted: false, + byteOffset, + declaredIntegrity: null, + unpacked: false, + limitations: [ + ...limitations, + `PyInstaller PYZ sha256 ${archive.sha256}.`, + ...(encrypted + ? ["PYZ entries are encrypted and were not decoded."] + : []), + ], + adapterKey: `carchive:${String(entry.index)}:${entry.name}`, + formatMetadata: cArchiveEntryMetadata( + entry, + undefined, + archive, + encrypted, + ), + }, + read: undefined, + }; +}; + +export const projectPyzMember = ( + container: CArchiveEntry, + item: PyzEntry, + encryptionMarker: boolean, + pythonMagic: Buffer | undefined, + range: + | { + readonly handle: FileHandle; + readonly offset: number; + readonly length: number; + } + | undefined, +): PyInstallerMember => { + const path = pyzMemberPath(canonicalMemberPath(container.name), item); + const encrypted = pyzEntryEncrypted(item, encryptionMarker); + if (item.namespace) + return { + entry: { + path: path, + kind: "directory", + declaredSize: 0, + compressedSize: item.length, + executable: false, + encrypted, + byteOffset: null, + declaredIntegrity: null, + unpacked: false, + limitations: [ + "Namespace package has no importable bytecode; its declared payload is not materialized.", + ...(encrypted + ? [ + "PYZ namespace payload is declared encrypted and is not materialized.", + ] + : []), + ], + adapterKey: `pyz:${String(container.index)}:${String(item.index)}:${path}`, + formatMetadata: pyzEntryMetadata( + container, + item, + undefined, + encryptionMarker, + ), + }, + read: undefined, + }; + const header = + !encrypted && + pythonMagic !== undefined && + (item.typeCode === 0 || item.typeCode === 1) + ? reconstructedPycHeader(pythonMagic) + : undefined; + const limitations: string[] = []; + if (encrypted) + limitations.push("PYZ entry is encrypted and was not decoded."); + if (header !== undefined) + limitations.push(RECONSTRUCTED_PYTHON_HEADER_LIMITATION); + limitations.push( + "PYZ tables declare compressed lengths; the uncompressed size is unknown until decoding.", + ); + return { + entry: fileEntry({ + path: path, + declaredSize: null, + compressedSize: item.length, + byteOffset: null, + encrypted, + limitations, + adapterKey: `pyz:${String(container.index)}:${String(item.index)}:${path}`, + formatMetadata: pyzEntryMetadata( + container, + item, + header, + encryptionMarker, + ), + }), + read: + range === undefined + ? undefined + : { + kind: "pyz", + ...range, + compressed: true, + uncompressedSize: undefined, + header, + }, + }; +}; + +/** + * CArchive and PYZ data names treat `\` as a separator. The logical path is + * canonical; the stored spelling is kept when that conversion changes it. + * `source_location` is an offset plus declared size, so a compressed member + * publishes a file span only when that size is the physical extent. Otherwise + * the span stays unknown, as ZIP does for compressed entries. + */ +const canonicalMemberPath = (storedName: string): string => + storedName.replaceAll("\\", "/"); + +const sourceByteOffset = ( + entry: CArchiveEntry, + archiveStart: number, +): number | null => + entry.compressionFlag === 0 ? archiveStart + entry.offset : null; + +const pyzMemberPath = (container: string, item: PyzEntry): string => { + if (item.typeCode === 2) + return `${container}/${canonicalMemberPath(item.name)}`; + const dotted = item.name.replaceAll(".", "/"); + if (item.namespace) return `${container}/${dotted}`; + if (item.package) return `${container}/${dotted}/__init__.pyc`; + return `${container}/${dotted}.pyc`; +}; + +const isBytecodeType = (typeCode: string): boolean => + typeCode === "s" || typeCode === "m" || typeCode === "M"; + +const fileEntry = (input: { + readonly path: string; + readonly declaredSize: number | null; + readonly compressedSize: number; + readonly byteOffset: number | null; + readonly limitations: readonly string[]; + readonly adapterKey: string; + readonly formatMetadata: PyInstallerEntryMetadata; + readonly encrypted?: boolean; + readonly kind?: "file" | "symlink" | "metadata"; +}): ArtifactEntry => ({ + path: input.path, + kind: input.kind ?? "file", + declaredSize: input.declaredSize, + compressedSize: input.compressedSize, + executable: false, + encrypted: input.encrypted === true, + byteOffset: input.byteOffset, + declaredIntegrity: null, + unpacked: false, + limitations: input.limitations, + adapterKey: input.adapterKey, + formatMetadata: input.formatMetadata, +}); + +/** Stable entry stored for a later open(). */ +export const sealMember = (member: PyInstallerMember): PyInstallerMember => ({ + ...member, + entry: copyArtifactEntry(member.entry), +}); diff --git a/src/artifacts/pyinstaller/MarshalSubset.test.ts b/src/artifacts/pyinstaller/MarshalSubset.test.ts new file mode 100644 index 0000000000..c4dee6781e --- /dev/null +++ b/src/artifacts/pyinstaller/MarshalSubset.test.ts @@ -0,0 +1,62 @@ +import { expect, it } from "vitest"; + +import { readMarshalSubset } from "./MarshalSubset.js"; + +const bytes = (hex: string): Buffer => Buffer.from(hex, "hex"); + +it("reads a Python 3.9 reference-flagged PYZ table", () => { + const parsed = readMarshalSubset( + bytes( + "db04000000a902da0568656c6c6fa903e900000000e90a000000e914000000a902da03706b67a903e901000000e91e000000e928000000a902fa0c6a736f6e2e6465636f646572a9037204000000e964000000e932000000a902da026e73a903e90300000072040000007204000000", + ), + ); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.value).toEqual([ + ["hello", [0, 10, 20]], + ["pkg", [1, 30, 40]], + ["json.decoder", [0, 100, 50]], + ["ns", [3, 0, 0]], + ]); +}); + +it("resolves a shared string through a reference reserved before the tuple", () => { + const parsed = readMarshalSubset( + bytes( + "db020000002902da06736861726564a903e900000000e901000000e90200000029027201000000a9037204000000e903000000e904000000", + ), + ); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.value).toEqual([ + ["shared", [0, 1, 2]], + ["shared", [1, 3, 4]], + ]); +}); + +it("reads a long integer that fits in a safe integer", () => { + const parsed = readMarshalSubset( + bytes( + "5b01000000a902da03626967a903e900000000ec03000000000000000004e905000000", + ), + ); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.value).toEqual([["big", [0, 2 ** 40, 5]]]); +}); + +it("reads a negative 32-bit integer", () => { + const parsed = readMarshalSubset( + bytes("5b01000000a902da016ea903e900000000e9ffffffffe901000000"), + ); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.value).toEqual([["n", [0, -1, 1]]]); +}); + +it("rejects a marshal type outside the subset", () => { + const parsed = readMarshalSubset(Buffer.from([0x4e])); + expect(parsed.ok).toBe(false); + if (parsed.ok) return; + expect(parsed.error.reason).toBe("format"); +}); diff --git a/src/artifacts/pyinstaller/MarshalSubset.ts b/src/artifacts/pyinstaller/MarshalSubset.ts new file mode 100644 index 0000000000..665d203c70 --- /dev/null +++ b/src/artifacts/pyinstaller/MarshalSubset.ts @@ -0,0 +1,282 @@ +import { err, ok, type Result } from "../../domain/result.js"; +import { + pyInstallerFailure, + type PyInstallerFailure, +} from "./PyInstallerFailure.js"; + +const MAX_DEPTH = 16; +const MAX_NODES = 100_000; +const MAX_SEQUENCE = 100_000; +const MAX_STRING_BYTES = 65_536; +const FLAG_REF = 0x80; + +/** Values the PYZ table of contents is allowed to contain. */ +export type MarshalValue = + | number + | string + | Uint8Array + | readonly MarshalValue[]; + +interface MarshalCursor { + readonly bytes: Buffer; + offset: number; + readonly refs: Array; + depth: number; + nodes: number; +} + +/** + * Read one marshal value using only list, tuple, str, int, and bytes. + * Container slots are reserved before their elements so later references match + * CPython. Any other type fails the whole value. + */ +export const readMarshalSubset = ( + bytes: Buffer, +): Result => { + const cursor: MarshalCursor = { + bytes, + offset: 0, + refs: [], + depth: 0, + nodes: 0, + }; + const value = readObject(cursor); + if (!value.ok) return value; + if (cursor.offset !== bytes.length) + return err( + pyInstallerFailure( + "format", + "PYZ marshal value did not consume its table of contents", + ), + ); + return value; +}; + +const readObject = ( + cursor: MarshalCursor, +): Result => { + if (cursor.depth > MAX_DEPTH || cursor.nodes >= MAX_NODES) + return err( + pyInstallerFailure( + "limit", + "PYZ marshal value exceeds the depth or object bound", + ), + ); + cursor.nodes += 1; + const code = readByte(cursor); + if (!code.ok) return code; + const flag = (code.value & FLAG_REF) !== 0; + const type = code.value & ~FLAG_REF; + if (type === 0x69) return storeScalar(cursor, flag, readInt32(cursor)); + if (type === 0x6c) return storeScalar(cursor, flag, readLong(cursor)); + if (type === 0x7a || type === 0x5a) + return storeScalar(cursor, flag, readShortAscii(cursor)); + if (type === 0x61 || type === 0x41) + return storeScalar(cursor, flag, readCountedAscii(cursor)); + if (type === 0x75) return storeScalar(cursor, flag, readUtf8(cursor)); + if (type === 0x73) return storeScalar(cursor, flag, readByteString(cursor)); + if (type === 0x28) return readSequence(cursor, flag, readInt32(cursor)); + if (type === 0x29) return readSequence(cursor, flag, readByte(cursor)); + if (type === 0x5b) return readSequence(cursor, flag, readInt32(cursor)); + if (type === 0x72) return readReference(cursor, flag); + return err( + pyInstallerFailure( + "format", + "PYZ marshal value uses a type outside the supported subset", + ), + ); +}; + +const storeScalar = ( + cursor: MarshalCursor, + flag: boolean, + value: Result, +): Result => { + if (!value.ok) return value; + if (flag) cursor.refs.push(value.value); + return value; +}; + +const readSequence = ( + cursor: MarshalCursor, + flag: boolean, + count: Result, +): Result => { + if (!count.ok) return count; + if (count.value < 0 || count.value > MAX_SEQUENCE) + return err( + pyInstallerFailure( + "limit", + "PYZ marshal sequence exceeds the supported count", + ), + ); + const slot = flag ? cursor.refs.length : undefined; + if (slot !== undefined) cursor.refs.push(undefined); + const items: MarshalValue[] = []; + cursor.depth += 1; + for (let index = 0; index < count.value; index += 1) { + const item = readObject(cursor); + if (!item.ok) return item; + items.push(item.value); + } + cursor.depth -= 1; + if (slot !== undefined) cursor.refs[slot] = items; + return ok(items); +}; + +const readReference = ( + cursor: MarshalCursor, + flag: boolean, +): Result => { + const index = readInt32(cursor); + if (!index.ok) return index; + const value = cursor.refs[index.value]; + if (value === undefined) + return err( + pyInstallerFailure( + "format", + "PYZ marshal reference does not point at a completed value", + ), + ); + if (flag) cursor.refs.push(value); + return ok(value); +}; + +const readShortAscii = ( + cursor: MarshalCursor, +): Result => { + const length = readByte(cursor); + if (!length.ok) return length; + return readAsciiBytes(cursor, length.value); +}; + +const readCountedAscii = ( + cursor: MarshalCursor, +): Result => { + const length = readInt32(cursor); + if (!length.ok) return length; + return readAsciiBytes(cursor, length.value); +}; + +const readAsciiBytes = ( + cursor: MarshalCursor, + length: number, +): Result => { + const bytes = readBytes(cursor, length); + if (!bytes.ok) return bytes; + if (bytes.value.some((byte) => byte > 0x7f)) + return err( + pyInstallerFailure( + "format", + "PYZ marshal ASCII string contains a non-ASCII byte", + ), + ); + return ok(bytes.value.toString("ascii")); +}; + +const readUtf8 = ( + cursor: MarshalCursor, +): Result => { + const length = readInt32(cursor); + if (!length.ok) return length; + const bytes = readBytes(cursor, length.value); + if (!bytes.ok) return bytes; + try { + return ok( + new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( + bytes.value, + ), + ); + } catch { + return err( + pyInstallerFailure("format", "PYZ marshal string is not valid UTF-8"), + ); + } +}; + +const readByteString = ( + cursor: MarshalCursor, +): Result => { + const length = readInt32(cursor); + if (!length.ok) return length; + const bytes = readBytes(cursor, length.value); + if (!bytes.ok) return bytes; + return ok(bytes.value); +}; + +const readLong = ( + cursor: MarshalCursor, +): Result => { + const digits = readInt32(cursor); + if (!digits.ok) return digits; + const count = Math.abs(digits.value); + if (count > 8) + return err( + pyInstallerFailure("limit", "PYZ marshal integer exceeds the safe range"), + ); + let magnitude = 0n; + for (let index = 0; index < count; index += 1) { + const digit = readUint16(cursor); + if (!digit.ok) return digit; + if (digit.value > 0x7fff) + return err( + pyInstallerFailure("format", "PYZ marshal long digit is out of range"), + ); + magnitude += BigInt(digit.value) << BigInt(15 * index); + } + if (magnitude > BigInt(Number.MAX_SAFE_INTEGER)) + return err( + pyInstallerFailure("limit", "PYZ marshal integer exceeds the safe range"), + ); + const signed = digits.value < 0 ? -Number(magnitude) : Number(magnitude); + return ok(signed); +}; + +const readBytes = ( + cursor: MarshalCursor, + length: number, +): Result => { + if (!Number.isSafeInteger(length) || length < 0 || length > MAX_STRING_BYTES) + return err( + pyInstallerFailure( + "limit", + "PYZ marshal string exceeds the supported length", + ), + ); + const end = cursor.offset + length; + if (end > cursor.bytes.length) + return err(pyInstallerFailure("format", "PYZ marshal value is truncated")); + const bytes = cursor.bytes.subarray(cursor.offset, end); + cursor.offset = end; + return ok(bytes); +}; + +const readByte = ( + cursor: MarshalCursor, +): Result => { + if (cursor.offset >= cursor.bytes.length) + return err(pyInstallerFailure("format", "PYZ marshal value is truncated")); + const value = cursor.bytes[cursor.offset] ?? 0; + cursor.offset += 1; + return ok(value); +}; + +const readInt32 = ( + cursor: MarshalCursor, +): Result => { + if (cursor.offset + 4 > cursor.bytes.length) + return err(pyInstallerFailure("format", "PYZ marshal value is truncated")); + const value = cursor.bytes.readInt32LE(cursor.offset); + cursor.offset += 4; + return ok(value); +}; + +const readUint16 = ( + cursor: MarshalCursor, +): Result => { + if (cursor.offset + 2 > cursor.bytes.length) + return err(pyInstallerFailure("format", "PYZ marshal value is truncated")); + const value = cursor.bytes.readUInt16LE(cursor.offset); + cursor.offset += 2; + return ok(value); +}; diff --git a/src/artifacts/pyinstaller/PyInstallerAdmissionBoundaries.test.ts b/src/artifacts/pyinstaller/PyInstallerAdmissionBoundaries.test.ts new file mode 100644 index 0000000000..4242b390f8 --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerAdmissionBoundaries.test.ts @@ -0,0 +1,139 @@ +import type { FileHandle } from "node:fs/promises"; +import { expect, it, vi } from "vitest"; +import { withPyInstallerReader } from "../../../tests/fixtures/pyinstallerReader.js"; +import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js"; +import type { ArtifactEntry } from "../ArtifactReader.js"; +import { PyInstallerPyzStore } from "./PyInstallerPyzStore.js"; +import { + cArchiveExecutable, + pyzArchive, + PYTHON_39_MAGIC, +} from "./CArchive.fixture.js"; + +const compressedImages = (count: number, modules = false) => + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: Array.from({ length: count }, (_, index) => ({ + name: `PYZ-${index}.pyz`, + typeCode: "z", + compress: true, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: modules + ? [{ name: "app", typeCode: 0, body: Buffer.from("body") }] + : [], + }), + })), + }); + +it("refuses excess private images before creating any temporary file", async () => { + const spool = vi + .spyOn(PyInstallerPyzStore.prototype, "spool") + .mockRejectedValue(new Error("unexpected private file creation")); + try { + await withPyInstallerReader(compressedImages(129), async (reader) => { + await expect( + reader.entries()[Symbol.asyncIterator]().next(), + ).rejects.toMatchObject({ + reason: "limit", + boundary: "capacity", + }); + expect(spool).not.toHaveBeenCalled(); + }); + } finally { + spool.mockRestore(); + } +}); + +it("accepts the private-image capacity and retires all image owners", async () => { + const spool = vi.spyOn(PyInstallerPyzStore.prototype, "spool"); + try { + await withPyInstallerReader(compressedImages(128), async (reader, root) => { + const listed: ArtifactEntry[] = []; + for await (const entry of reader.entries()) listed.push(entry); + expect(listed).toHaveLength(128); + expect(spool).toHaveBeenCalledTimes(128); + await reader.close(); + expect(root.fd).toBeGreaterThanOrEqual(0); + }); + } finally { + spool.mockRestore(); + } +}); + +it.each([false, true])( + "preserves private-image admission cancellation (failed cleanup: %s)", + async (failClose) => { + await withPyInstallerReader( + compressedImages(1, true), + async (reader, root) => { + let selected: ArtifactEntry | undefined; + for await (const entry of reader.entries()) + if (entry.path.endsWith("app.pyc")) selected = entry; + if (selected === undefined) throw new Error("Missing child"); + let entered!: () => void, release!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + const gate = new Promise((resolve) => { + release = resolve; + }); + const prototype = Object.getPrototypeOf(root) as FileHandle; + const nativeStat = prototype.stat, + nativeClose = OwnedFileHandle.prototype.close; + let admitted: FileHandle | undefined, + retained: OwnedFileHandle | undefined, + attempts = 0; + const cancelled = new Error("cancel private-image admission"), + cleanup = new Error("admission cleanup EIO"); + const signal = new AbortController(); + const stat = vi + .spyOn(prototype, "stat") + .mockImplementation(async function ( + this: FileHandle, + ...args: Parameters + ) { + if (this !== root && admitted === undefined) { + admitted = this; + entered(); + await gate; + } + return Reflect.apply(nativeStat, this, args); + }); + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + if (this.handle === admitted) { + retained = this; + attempts++; + if (failClose && attempts === 1) throw cleanup; + } + await nativeClose.call(this); + }); + const opening = reader.open(selected, signal.signal); + void opening.catch(() => undefined); + try { + await started; + signal.abort(cancelled); + release(); + await expect(opening).rejects.toMatchObject({ + reason: "cancelled", + cancellation: { reason: cancelled }, + ...(failClose ? { cleanup: { reason: cleanup.message } } : {}), + }); + if (failClose) expect(retained?.handle.fd).toBeGreaterThanOrEqual(0); + await reader.close(); + expect(attempts).toBe(failClose ? 2 : 1); + expect(admitted?.fd).toBe(-1); + expect(root.fd).toBeGreaterThanOrEqual(0); + } finally { + release(); + await opening.catch(() => undefined); + stat.mockRestore(); + close.mockRestore(); + } + }, + ); + }, +); diff --git a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts new file mode 100644 index 0000000000..3220bbb4e0 --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts @@ -0,0 +1,1178 @@ +import { createHash } from "node:crypto"; +import { open, readFile, stat, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { buffer } from "node:stream/consumers"; + +import { expect, it, vi } from "vitest"; + +import { ArtifactReaderFailure } from "../ArtifactReader.js"; +import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js"; +import { materializeArtifactInventory } from "../../../tests/fixtures/artifactInventory.js"; +import { scanCanonicalArtifactInventory } from "../../../tests/fixtures/artifactInventory.js"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js"; +import { reconstructedPycHeader } from "./PythonBytecode.js"; +import { + PYTHON_39_MAGIC, + cArchiveExecutable, + cookieRangeOutsideFile, + pyzArchive, + type CArchiveFixtureMember, +} from "./CArchive.fixture.js"; + +const sha256 = (bytes: Buffer): string => + createHash("sha256").update(bytes).digest("hex"); + +const derived = (payload: Buffer): Buffer => + Buffer.concat([reconstructedPycHeader(PYTHON_39_MAGIC), payload]); + +const writeArchive = async (name: string, bytes: Buffer): Promise => { + const directory = await createTestTempDirectory("rea-pyinstaller-"); + const path = join(directory, name); + await writeFile(path, bytes); + return path; +}; + +const sampleMembers = ( + script: Buffer, + stored: Buffer, + pyz: Buffer, +): CArchiveFixtureMember[] => [ + { name: "hello", typeCode: "s", payload: script, compress: true }, + { name: "data.bin", typeCode: "x", payload: stored, compress: false }, + { name: "PYZ.pyz", typeCode: "z", payload: pyz, compress: false }, + { + name: "pyi-contents-directory _internal", + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }, +]; + +it("inventories a 2.1 archive, nested PYZ modules, and derived bytecode", async () => { + const script = Buffer.from("print('hi')"); + const stored = Buffer.from("stored-data"); + const moduleBody = Buffer.from("module-body"); + const path = await writeArchive( + "hello", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 314, + pythonLibrary: "libpython3.14.dylib", + trailer: Buffer.from("signature"), + members: sampleMembers( + script, + stored, + pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "json.decoder", typeCode: 0, body: moduleBody }, + { name: "pkg", typeCode: 1, body: moduleBody }, + { name: "ns", typeCode: 3, body: Buffer.alloc(0) }, + ], + }), + ), + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + expect(snapshot.manifest.root_format).toBe("pyinstaller"); + expect(snapshot.coverage.status).toBe("complete"); + const root = snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === ".", + ); + expect(root).toMatchObject({ artifact_kind: "container" }); + expect(root?.format_metadata).toMatchObject({ + kind: "pyinstaller", + layout_version: "2.1", + python_version_raw: 314, + python_major: 3, + python_minor: 14, + python_library: "libpython3.14.dylib", + runtime_option_count: 1, + }); + const hello = snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "hello", + ); + expect(hello?.format_metadata).toMatchObject({ + kind: "pyinstaller-entry", + container: "carchive", + type_code: "s", + compression_flag: 1, + bytecode_header: "unknown", + bytecode_magic: null, + }); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(script)), + ).toMatchObject({ size: script.length }); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(stored)), + ).toBeDefined(); + expect( + snapshot.occurrences.map((occurrence) => occurrence.logical_path), + ).toEqual( + expect.arrayContaining([ + "hello", + "data.bin", + "PYZ.pyz", + "PYZ.pyz/json/decoder.pyc", + "PYZ.pyz/pkg/__init__.pyc", + "PYZ.pyz/ns", + "__pyinstaller__/runtime-options/3", + ]), + ); + expect( + snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "PYZ.pyz", + )?.entry_kind, + ).toBe("directory"); + const scriptEdge = snapshot.edges.find( + (edge) => edge.occurrence_id === hello?.occurrence_id, + ); + expect(scriptEdge?.relation).toBe("contains"); +}); + +it("records a 2.0 cookie without a Python library name", async () => { + const path = await writeArchive( + "legacy", + cArchiveExecutable({ + layout: "2.0", + pythonVersion: 307, + members: [ + { + name: "hello", + typeCode: "s", + payload: Buffer.from("print"), + compress: false, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + const root = snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === ".", + ); + expect(root?.format_metadata).toMatchObject({ + kind: "pyinstaller", + layout_version: "2.0", + python_version_raw: 307, + python_major: 3, + python_minor: 7, + python_library: null, + }); +}); + +it("extracts derived members into the safe output tree", async () => { + const script = Buffer.from("print('hi')"); + const path = await writeArchive( + "hello", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "libpython3.9.so", + members: sampleMembers( + script, + Buffer.from("stored"), + pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "app", typeCode: 0, body: Buffer.from("app-body") }, + ], + }), + ), + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + const directory = await createTestTempDirectory("rea-pyinstaller-out-"); + const output = join(directory, "extracted"); + const extracted = await materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot: output, + environment: process.env, + integrityPolicy: "fail", + }, + path, + snapshot, + ); + expect(await readFile(join(output, "hello"))).toEqual(derived(script)); + expect( + extracted.artifacts.find((artifact) => artifact.relative_path === "hello") + ?.sha256, + ).toBe(sha256(derived(script))); + expect(await readFile(join(output, "PYZ.pyz", "app.pyc"))).toEqual( + derived(Buffer.from("app-body")), + ); +}); + +it("reports encrypted PYZ entries and refuses to extract them", async () => { + const path = await writeArchive( + "encrypted", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "libpython3.9.so", + members: [ + { + name: "pyimod00_crypto_key", + typeCode: "m", + payload: Buffer.from("key"), + compress: false, + }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "secret", typeCode: 0, body: Buffer.from("hidden") }, + ], + }), + compress: false, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + expect( + snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "PYZ.pyz/secret.pyc", + ), + ).toMatchObject({ + encrypted: true, + hash_status: "unavailable", + }); + const directory = await createTestTempDirectory("rea-pyinstaller-out-"); + await expect( + materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot: join(directory, "extracted"), + environment: process.env, + integrityPolicy: "fail", + }, + path, + snapshot, + ), + ).rejects.toThrow(/encrypted/u); +}); + +it.each([false, true])( + "leaves an unsupported PYZ magic as one opaque member (marker: %s)", + async (marker) => { + const payload = pyzArchive({ + magic: Buffer.alloc(4), + modules: [{ name: "app", typeCode: 0, body: Buffer.from("app") }], + }); + const path = await writeArchive( + "opaque", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 314, + pythonLibrary: "libpython3.14.dylib", + members: [ + ...(marker + ? [ + { + name: "pyimod00_crypto_key", + typeCode: "m", + payload: Buffer.from("key"), + compress: false, + }, + ] + : []), + { + name: "PYZ.pyz", + typeCode: "z", + payload, + compress: false, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + const pyz = snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "PYZ.pyz", + ); + expect(pyz).toMatchObject({ + entry_kind: "file", + encrypted: false, + hash_status: "verified", + }); + expect(snapshot.coverage.status).toBe("unknown"); + expect( + snapshot.nodes.find((node) => node.artifact_id === pyz?.artifact_id), + ).toMatchObject({ sha256: sha256(payload), size: payload.length }); + expect(pyz?.format_metadata).toMatchObject({ + pyz_python_magic: "00000000", + pyz_python_version: null, + pyz_status: "opaque", + encryption_source: marker ? "carchive-marker" : null, + }); + expect(pyz?.limitations.join(" ")).toContain("left opaque"); + expect( + snapshot.occurrences.some((occurrence) => + occurrence.logical_path.endsWith(".pyc"), + ), + ).toBe(false); + const outputRoot = join( + await createTestTempDirectory("rea-opaque-out-"), + "extracted", + ); + await materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot, + environment: process.env, + integrityPolicy: "fail", + }, + path, + snapshot, + ); + expect(await readFile(join(outputRoot, "PYZ.pyz"))).toEqual(payload); + }, +); + +it("reads children of a zlib-compressed PYZ", async () => { + const body = Buffer.from("compressed-child"); + const path = await writeArchive( + "compressed-pyz", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "libpython3.9.so", + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "child", typeCode: 0, body }], + }), + compress: true, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(derived(body))), + ).toMatchObject({ size: derived(body).length }); +}); + +it("rejects a truncated table of contents", async () => { + const path = await writeArchive( + "truncated", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 314, + pythonLibrary: "libpython3.14.dylib", + members: [], + tocBytes: Buffer.concat([Buffer.from([0, 0, 0, 64]), Buffer.alloc(16)]), + }), + ); + await expect(scanCanonicalArtifactInventory(path)).rejects.toBeInstanceOf( + ArtifactReaderFailure, + ); + await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({ + reason: "format", + }); +}); + +it("rejects a cookie whose package range is outside the file", async () => { + const path = await writeArchive("outside", cookieRangeOutsideFile()); + const reader = new PyInstallerArtifactReader(path); + await expect(async () => { + for await (const entry of reader.entries()) void entry; + }).rejects.toMatchObject({ + reason: "format", + message: expect.stringContaining("outside the file"), + }); + await reader.close(); +}); + +it("rejects a member whose declared size exceeds the decompression budget", async () => { + const path = await writeArchive( + "bomb", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 314, + pythonLibrary: "libpython3.14.dylib", + members: [ + { + name: "bomb.bin", + typeCode: "b", + payload: Buffer.alloc(0), + compress: true, + declaredUncompressedSize: 300 * 1024 * 1024, + }, + ], + }), + ); + await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({ + reason: "limit", + }); +}); + +it("rejects a zlib member that expands past its declared size", async () => { + const path = await writeArchive( + "expand", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 314, + pythonLibrary: "libpython3.14.dylib", + members: [ + { + name: "expand.bin", + typeCode: "b", + payload: Buffer.from("hello world hello world"), + compress: true, + declaredUncompressedSize: 4, + }, + ], + }), + ); + await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({ + reason: "limit", + }); +}); + +it("reads a compressed member whose uncompressed size is zero", async () => { + const path = await writeArchive( + "empty-compressed", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "libpython3.9.so", + members: [ + { + name: "empty.bin", + typeCode: "x", + payload: Buffer.alloc(0), + compress: true, + }, + { + name: "empty.py", + typeCode: "s", + payload: Buffer.alloc(0), + compress: true, + }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "app", typeCode: 0, body: Buffer.from("app") }], + }), + compress: false, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(Buffer.alloc(0))), + ).toMatchObject({ size: 0 }); + expect( + snapshot.nodes.find( + (node) => node.sha256 === sha256(derived(Buffer.alloc(0))), + ), + ).toMatchObject({ size: derived(Buffer.alloc(0)).length }); + + const expanded = await writeArchive( + "zero-declared", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "libpython3.9.so", + members: [ + { + name: "expand.bin", + typeCode: "x", + payload: Buffer.from("x"), + compress: true, + declaredUncompressedSize: 0, + }, + ], + }), + ); + await expect(scanCanonicalArtifactInventory(expanded)).rejects.toMatchObject({ + reason: "limit", + }); +}); + +it("preserves an uppercase Z ZIP member as a readable file", async () => { + const zip = Buffer.from("PK\u0003\u0004zip-member"); + const path = await writeArchive( + "zip-member", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "libpython3.9.so", + members: [ + { + name: "payload.zip", + typeCode: "Z", + payload: zip, + compress: false, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + expect( + snapshot.occurrences.map((occurrence) => occurrence.logical_path).sort(), + ).toEqual([".", "payload.zip"]); + expect( + snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "payload.zip", + ), + ).toMatchObject({ + entry_kind: "file", + format_metadata: { kind: "pyinstaller-entry", type_code: "Z" }, + }); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(zip)), + ).toMatchObject({ size: zip.length }); +}); + +it("keeps a PYZ data entry resource path", async () => { + const body = Buffer.from('{"enabled":true}'); + const path = await writeArchive( + "pyz-data", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "libpython3.9.so", + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "pkg/config.json", typeCode: 2, body }], + }), + compress: false, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + expect( + snapshot.occurrences.map((occurrence) => occurrence.logical_path).sort(), + ).toEqual([".", "PYZ.pyz", "PYZ.pyz/pkg/config.json"]); + expect( + snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "PYZ.pyz/pkg/config.json", + ), + ).toMatchObject({ entry_kind: "file" }); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(body)), + ).toMatchObject({ size: body.length }); +}); + +it("projects CArchive type n entries as symlinks", async () => { + const target = Buffer.from("libfoo.so.1\0"); + const payload = Buffer.from("stored-data"); + const path = await writeArchive( + "symlink", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "libpython3.9.so", + members: [ + { + name: "libfoo.so", + typeCode: "n", + payload: target, + compress: false, + }, + { name: "data.bin", typeCode: "x", payload, compress: false }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + expect( + snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "libfoo.so", + ), + ).toMatchObject({ + entry_kind: "symlink", + artifact_id: null, + hash_status: "not-hashed", + format_metadata: { + kind: "pyinstaller-entry", + type_code: "n", + compression_flag: 0, + }, + limitations: expect.arrayContaining([ + "PyInstaller symlink target was not read.", + ]), + }); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(target)), + ).toBeUndefined(); + const directory = await createTestTempDirectory("rea-pyinstaller-out-"); + const output = join(directory, "extracted"); + await materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot: output, + environment: process.env, + integrityPolicy: "fail", + }, + path, + snapshot, + ); + await expect(readFile(join(output, "libfoo.so"))).rejects.toMatchObject({ + code: "ENOENT", + }); + expect(await readFile(join(output, "data.bin"))).toEqual(payload); +}); + +it.each([false, true])( + "retains the file owner after a failed close (temporary PYZ: %s)", + async (temporaryPyz) => { + const path = await writeArchive( + "close-owner", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "libpython3.9.so", + members: [ + { + name: temporaryPyz ? "PYZ.pyz" : "data.bin", + typeCode: temporaryPyz ? "z" : "x", + payload: temporaryPyz + ? pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }) + : Buffer.from("data"), + compress: temporaryPyz, + }, + ], + }), + ); + const reader = new PyInstallerArtifactReader(path); + for await (const entry of reader.entries()) void entry; + const originalClose = OwnedFileHandle.prototype.close; + const owners: OwnedFileHandle[] = []; + const closeFailure = new Error("close failed"); + let attempts = 0; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + owners.push(this); + attempts += 1; + if (attempts === 1) throw closeFailure; + await originalClose.call(this); + }); + try { + await expect(reader.close()).rejects.toMatchObject({ + cleanup: { reason: closeFailure.message, resources: [path] }, + }); + await reader.close(); + // Spooled PYZ images close after listing, so only the source owner remains. + expect(attempts).toBe(2); + expect(owners[0]).toBe(owners[1]); + } finally { + close.mockRestore(); + await reader.close(); + } + }, +); + +it("retains the descriptor owner when admission cleanup fails", async () => { + const directory = await createTestTempDirectory("rea-pyinstaller-admission-"); + const reader = new PyInstallerArtifactReader(directory); + const originalClose = OwnedFileHandle.prototype.close; + const owners: OwnedFileHandle[] = []; + const closeFailure = new Error("admission close failed"); + let attempts = 0; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + owners.push(this); + attempts += 1; + if (attempts === 1) throw closeFailure; + await originalClose.call(this); + }); + try { + await expect( + reader.entries()[Symbol.asyncIterator]().next(), + ).rejects.toMatchObject({ + reason: "format", + message: `PyInstaller source is not a regular file: ${directory}`, + cleanup: { + reason: closeFailure.message, + resources: [directory], + }, + }); + expect(owners).toHaveLength(1); + await reader.close(); + expect(attempts).toBe(2); + expect(owners[0]).toBe(owners[1]); + } finally { + close.mockRestore(); + await reader.close().catch(() => undefined); + await owners[0]?.close().catch(() => undefined); + } +}); + +it("inventories and extracts a Windows member name under its canonical path", async () => { + const storedName = String.raw`pkg\data.bin`; + const pyzName = String.raw`res\icon.bin`; + const payload = Buffer.from("nested-bytes"); + const icon = Buffer.from("icon-bytes"); + const path = await writeArchive( + "windows", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "python39.dll", + members: [ + { name: storedName, typeCode: "x", payload, compress: false }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: pyzName, typeCode: 2, body: icon }], + }), + compress: false, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + const member = snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "pkg/data.bin", + ); + const resource = snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === "PYZ.pyz/res/icon.bin", + ); + expect(member?.format_metadata).toMatchObject({ source_name: storedName }); + expect(resource?.format_metadata).toMatchObject({ source_name: pyzName }); + const directory = await createTestTempDirectory("rea-pyinstaller-out-"); + const output = join(directory, "extracted"); + await materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot: output, + environment: process.env, + integrityPolicy: "fail", + }, + path, + snapshot, + ); + expect(await readFile(join(output, "pkg", "data.bin"))).toEqual(payload); + expect(await readFile(join(output, "PYZ.pyz", "res", "icon.bin"))).toEqual( + icon, + ); +}); + +it("rejects a stored member whose declared size disagrees with its bytes", async () => { + const path = await writeArchive( + "stored-size", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "python39.dll", + members: [ + { + name: "data.bin", + typeCode: "x", + payload: Buffer.from("abcd"), + compress: false, + declaredUncompressedSize: 0, + }, + ], + }), + ); + await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({ + reason: "integrity", + message: expect.stringContaining( + "Stored CArchive member size disagrees with its table entry", + ), + }); +}); + +it("keeps compressed member spans on the physical extent or unknown", async () => { + const payload = Buffer.alloc(10_000, 0x61); + const path = await writeArchive( + "compressed-span", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "python39.dll", + members: [ + { name: "payload.bin", typeCode: "x", payload, compress: true }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "app", typeCode: 0, body: payload }], + }), + compress: false, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + const fileSize = (await stat(path)).size; + expect(fileSize).toBeLessThan(payload.length); + for (const logicalPath of ["payload.bin", "PYZ.pyz/app.pyc"]) { + const occurrence = snapshot.occurrences.find( + (candidate) => candidate.logical_path === logicalPath, + ); + if (logicalPath === "payload.bin") { + expect(occurrence?.declared_size).toBeGreaterThan(fileSize); + expect(occurrence?.compressed_size ?? fileSize).toBeLessThan( + occurrence?.declared_size ?? 0, + ); + } else { + expect(occurrence?.declared_size).toBeNull(); + expect( + snapshot.nodes.find( + (node) => node.artifact_id === occurrence?.artifact_id, + )?.size, + ).toBeGreaterThan(fileSize); + } + const span = occurrence?.source_location; + if (span === null || span === undefined) { + expect(span).toBeNull(); + continue; + } + expect(span.length).toBe(occurrence?.compressed_size); + expect(span.offset + span.length).toBeLessThanOrEqual(fileSize); + } + for (const occurrence of snapshot.occurrences) { + const span = occurrence.source_location; + if (span === null) continue; + expect(span.offset + span.length).toBeLessThanOrEqual(fileSize); + } +}); + +it("reconstructs each PYZ image's bytecode with that image's own magic", async () => { + const python314Magic = Buffer.from("2b0e0d0a", "hex"); + const first = Buffer.from("first-module"); + const second = Buffer.from("second-module"); + const path = await writeArchive( + "two-pyz", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 314, + pythonLibrary: "libpython3.14.dylib", + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + compress: false, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "first", typeCode: 0, body: first }], + }), + }, + { + name: "second.pyz", + typeCode: "z", + compress: false, + payload: pyzArchive({ + magic: python314Magic, + modules: [{ name: "second", typeCode: 0, body: second }], + }), + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + const withHeader = (magic: Buffer, body: Buffer): Buffer => + Buffer.concat([reconstructedPycHeader(magic), body]); + const firstBytes = withHeader(PYTHON_39_MAGIC, first); + const secondBytes = withHeader(python314Magic, second); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(firstBytes)), + ).toMatchObject({ size: firstBytes.length }); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(secondBytes)), + ).toMatchObject({ size: secondBytes.length }); + expect( + snapshot.nodes.find( + (node) => node.sha256 === sha256(withHeader(PYTHON_39_MAGIC, second)), + ), + ).toBeUndefined(); +}); + +it("opens selected PYZ modules lazily and permits repeat reads without retaining their expanded bytes", async () => { + const good = Buffer.from("selected module"); + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "damaged", typeCode: 0, body: Buffer.from("damaged module") }, + { name: "selected", typeCode: 0, body: good }, + ], + }); + image[12] = 0; // Damage the first zlib header without changing the valid PYZ table. + const path = await writeArchive( + "lazy-pyz", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { name: "PYZ.pyz", typeCode: "z", payload: image, compress: false }, + ], + }), + ); + const reader = new PyInstallerArtifactReader(path); + try { + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + const selected = entries.find( + (entry) => entry.path === "PYZ.pyz/selected.pyc", + ); + const damaged = entries.find( + (entry) => entry.path === "PYZ.pyz/damaged.pyc", + ); + if (selected === undefined || damaged === undefined) + throw new Error("Missing PYZ entries"); + expect(selected.declaredSize).toBeNull(); + for (let attempt = 0; attempt < 2; attempt += 1) + expect(await buffer(await reader.open(selected))).toEqual(derived(good)); + await expect(buffer(await reader.open(damaged))).rejects.toMatchObject({ + reason: "format", + }); + } finally { + await reader.close(); + } +}); + +it.each([false, true])( + "lists a large file-backed PYZ and reads a selected module (outer compression: %s)", + async (compress) => { + const original = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "selected", typeCode: 0, body: Buffer.from("selected module") }, + ], + }); + const toc = original.readUInt32BE(8); + // Unreferenced image padding is legal and must not become retained module bytes. + const image = Buffer.concat([ + original.subarray(0, toc), + Buffer.alloc(64 * 1024 * 1024), + original.subarray(toc), + ]); + image.writeUInt32BE(toc + 64 * 1024 * 1024, 8); + const path = await writeArchive( + "large-pyz", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [{ name: "PYZ.pyz", typeCode: "z", payload: image, compress }], + }), + ); + const reader = new PyInstallerArtifactReader(path); + try { + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + const selected = entries.find( + (entry) => entry.path === "PYZ.pyz/selected.pyc", + ); + if (selected === undefined) throw new Error("Missing selected module"); + expect(await buffer(await reader.open(selected))).toEqual( + derived(Buffer.from("selected module")), + ); + } finally { + await reader.close(); + } + }, +); + +it("uses known PYZ magic for separate CArchive scripts when the marshal table stays opaque", async () => { + const script = Buffer.from("entry script"); + const path = await writeArchive( + "opaque-table", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { name: "hello", typeCode: "s", payload: script, compress: true }, + { + name: "PYZ.pyz", + typeCode: "z", + compress: false, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [], + toc: Buffer.from([0x4e]), + }), + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + const opaque = snapshot.occurrences.find( + (entry) => entry.logical_path === "PYZ.pyz", + ); + expect(opaque).toMatchObject({ entry_kind: "file" }); + expect(opaque?.limitations.join(" ")).toContain("left opaque"); + expect( + snapshot.nodes.find((node) => node.sha256 === sha256(derived(script))), + ).toBeDefined(); + const output = join( + await createTestTempDirectory("rea-opaque-script-"), + "extracted", + ); + await materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot: output, + environment: process.env, + integrityPolicy: "fail", + }, + path, + snapshot, + ); + expect(await readFile(join(output, "hello"))).toEqual(derived(script)); +}); + +it("rejects duplicate CArchive member names before projecting PYZ children", async () => { + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "child", typeCode: 0, body: Buffer.from("child") }], + }); + const path = await writeArchive( + "duplicate-names", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { name: "PYZ.pyz", typeCode: "z", payload: image, compress: false }, + { + name: "PYZ.pyz", + typeCode: "x", + payload: Buffer.alloc(0), + compress: false, + }, + ], + }), + ); + await expect(scanCanonicalArtifactInventory(path)).rejects.toMatchObject({ + reason: "integrity", + message: "Duplicate CArchive member name at table index 1", + }); +}); + +it("closes each spooled PYZ image after listing and reopens it for a requested module", async () => { + const bodies = [0, 1, 2].map((index) => Buffer.from(`module ${index}`)); + const path = await writeArchive( + "spooled-pyz", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: bodies.map((body, index) => ({ + name: `PYZ-${index}.pyz`, + typeCode: "z" as const, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "child", typeCode: 0, body }], + }), + compress: true, + })), + }), + ); + const closed = new Set(); + const originalClose = OwnedFileHandle.prototype.close; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + await originalClose.call(this); + closed.add(this); + }); + const rootHandle = await open(path, "r"); + const reader = new PyInstallerArtifactReader(path, { + handle: rootHandle, + initial: await rootHandle.stat(), + }); + try { + const iterator = reader.entries()[Symbol.asyncIterator](); + const first = await iterator.next(); + // Every spooled image is closed once its table is read, before any entry is yielded. + expect(closed.size).toBeGreaterThanOrEqual(bodies.length); + for (const owner of closed) expect(owner.handle.fd).toBe(-1); + const listedHandles = closed.size; + const entries = first.done === true ? [] : [first.value]; + for ( + let next = await iterator.next(); + next.done !== true; + next = await iterator.next() + ) + entries.push(next.value); + const selected = entries.find( + (entry) => entry.path === "PYZ-1.pyz/child.pyc", + ); + if (selected === undefined) throw new Error("Missing PYZ child"); + expect(await buffer(await reader.open(selected))).toEqual( + derived(bodies[1] as Buffer), + ); + await reader.close(); + expect(closed.size).toBe(listedHandles + 1); + for (const owner of closed) expect(owner.handle.fd).toBe(-1); + expect(rootHandle.fd).toBeGreaterThanOrEqual(0); + } finally { + try { + await reader.close(); + } finally { + close.mockRestore(); + await rootHandle.close(); + } + } +}); + +it.each([1, 2])( + "retains a private PYZ owner whose close fails at stage %s", + async (failedStage) => { + const path = await writeArchive( + "spool-close-owner", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }), + compress: true, + }, + ], + }), + ); + const originalClose = OwnedFileHandle.prototype.close; + const owners: OwnedFileHandle[] = []; + const closeFailure = new Error("spool close failed"); + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + owners.push(this); + if (owners.length === failedStage) throw closeFailure; + await originalClose.call(this); + }); + const reader = new PyInstallerArtifactReader(path); + try { + await expect( + reader.entries()[Symbol.asyncIterator]().next(), + ).rejects.toBe(closeFailure); + await reader.close(); + expect( + owners.filter((owner) => owner === owners[failedStage - 1]), + ).toHaveLength(2); + } finally { + close.mockRestore(); + await reader.close().catch(() => undefined); + } + }, +); diff --git a/src/artifacts/pyinstaller/PyInstallerBoundaries.test.ts b/src/artifacts/pyinstaller/PyInstallerBoundaries.test.ts new file mode 100644 index 0000000000..ae8b84204e --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerBoundaries.test.ts @@ -0,0 +1,673 @@ +import { withPyInstallerReader } from "../../../tests/fixtures/pyinstallerReader.js"; +import { join } from "node:path"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import { createHash } from "node:crypto"; +import { writeFile, type FileHandle } from "node:fs/promises"; +import { buffer } from "node:stream/consumers"; +import { deflateSync } from "node:zlib"; +import { setImmediate } from "node:timers/promises"; +import { expect, it, vi } from "vitest"; +import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js"; +import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js"; +import { hashReadable } from "../ArtifactHash.js"; +import { + ArtifactReaderFailure, + type ArtifactEntry, +} from "../ArtifactReader.js"; +import { + cArchiveExecutable, + pyzArchive, + PYTHON_39_MAGIC, +} from "./CArchive.fixture.js"; + +const hash = (bytes: Buffer): string => + createHash("sha256").update(bytes).digest("hex"); +const archive = (body: Buffer) => + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [{ name: "data", typeCode: "x", payload: body, compress: false }], + }); +const members = async (reader: PyInstallerArtifactReader) => { + const entries: ArtifactEntry[] = []; + for await (const entry of reader.entries()) entries.push(entry); + return entries; +}; +const deferred = () => { + let resolve: () => void = () => undefined; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +}; + +it.each([false, true])( + "settles independent private and carrier closes and retries only failed owners (carrier failure: %s)", + async (failCarrier) => { + const path = join(await createTestTempDirectory("rea-pyi-cleanup-"), "app"); + await writeFile( + path, + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: ["one", "two"].map((name) => ({ + name: `${name}.pyz`, + typeCode: "z", + compress: true, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "app", typeCode: 0, body: Buffer.from(name) }], + }), + })), + }), + ); + const reader = new PyInstallerArtifactReader(path); + const entries = (await members(reader)).filter((entry) => + entry.path.endsWith("app.pyc"), + ); + expect(entries).toHaveLength(2); + const nativeClose = OwnedFileHandle.prototype.close; + const privateOwners = new Set(); + const attempts: OwnedFileHandle[] = []; + let collecting = true; + let failing = true; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + attempts.push(this); + if (collecting) privateOwners.add(this); + if (failing && (privateOwners.has(this) || failCarrier)) + throw new Error( + privateOwners.has(this) ? "private close EIO" : "carrier close EIO", + ); + await nativeClose.call(this); + }); + try { + for (const entry of entries) + await expect(buffer(await reader.open(entry))).rejects.toMatchObject({ + cleanup: { reason: "private close EIO" }, + }); + expect(privateOwners.size).toBe(2); + collecting = false; + attempts.length = 0; + const failure = await reader.close().then( + () => undefined, + (cause: unknown) => cause, + ); + expect(attempts).toHaveLength(3); + expect(new Set(attempts).size).toBe(3); + expect(failure).toMatchObject({ + cleanup: { + resources: expect.arrayContaining(failCarrier ? [path] : []), + }, + }); + expect( + (failure as ArtifactReaderFailure).cleanup?.resources, + ).toHaveLength(failCarrier ? 3 : 2); + expect((failure as ArtifactReaderFailure).cleanup?.reason).toContain( + "private close EIO", + ); + if (failCarrier) + expect((failure as ArtifactReaderFailure).cleanup?.reason).toContain( + "carrier close EIO", + ); + const carrier = attempts.find((owner) => !privateOwners.has(owner))!; + expect(carrier.handle.fd >= 0).toBe(failCarrier); + for (const owner of privateOwners) + expect(owner.handle.fd).toBeGreaterThanOrEqual(0); + failing = false; + attempts.length = 0; + await reader.close(); + expect(attempts).toHaveLength(failCarrier ? 3 : 2); + for (const owner of [...privateOwners, carrier]) + expect(owner.handle.fd).toBe(-1); + attempts.length = 0; + await reader.close(); + expect(attempts).toEqual([]); + } finally { + close.mockRestore(); + await reader.close(); + } + }, +); + +it("binds materialization to the inventoried digest and checks cached carrier freshness", async () => { + const bytes = archive(Buffer.from("first")); + await withPyInstallerReader(bytes, async (reader, _handle, path) => { + await expect(reader.prepareContainer("0".repeat(64))).rejects.toMatchObject( + { reason: "integrity" }, + ); + await reader.prepareContainer(hash(bytes)); + await reader.prepareContainer(hash(bytes)); + await writeFile(path, archive(Buffer.from("other"))); + await expect(reader.prepareContainer(hash(bytes))).rejects.toMatchObject({ + reason: "integrity", + }); + }); +}); + +it("bounds descriptor reads while hashing a selected stored member", async () => { + const payload = Buffer.alloc(2 * 1024 * 1024, 0x61); + await withPyInstallerReader(archive(payload), async (reader, handle) => { + const entry = (await members(reader))[0]!; + const read = handle.read.bind(handle); + let maximum = 0; + const spy = vi + .spyOn(handle, "read") + .mockImplementation((...args: unknown[]) => { + if (typeof args[2] === "number") maximum = Math.max(maximum, args[2]); + return Reflect.apply(read, handle, args); + }); + try { + const observed = await hashReadable(await reader.open(entry)); + expect(observed.sha256).toBe(hash(payload)); + expect(observed.bytes).toBe(payload.length); + expect(maximum).toBeGreaterThan(0); + expect(maximum).toBeLessThanOrEqual(64 * 1024); + } finally { + spy.mockRestore(); + } + }); +}); + +it.each([false, true])( + "joins pending native reads during cancellation/close (I/O failure: %s)", + async (fail) => { + await withPyInstallerReader( + archive(Buffer.alloc(256 * 1024, 0x61)), + async (reader, handle) => { + const entry = (await members(reader))[0]!; + const entered = deferred(), + release = deferred(); + const read = handle.read.bind(handle); + const io = Object.assign(new Error("native read failed"), { + code: "EIO", + }); + const caller = new Error("caller stopped reading"); + const controller = new AbortController(); + let held = false; + const spy = vi + .spyOn(handle, "read") + .mockImplementation(async (...args: unknown[]) => { + if (!held) { + held = true; + entered.resolve(); + await release.promise; + if (fail) throw io; + } + return Reflect.apply(read, handle, args); + }); + const consumed = reader + .open(entry, controller.signal) + .then((stream) => buffer(stream)); + void consumed.catch(() => undefined); + let closing: Promise | undefined; + try { + await entered.promise; + controller.abort(caller); + let closed = false; + closing = reader.close().then(() => { + closed = true; + }); + await setImmediate(); + expect(closed).toBe(false); + expect(handle.fd).toBeGreaterThanOrEqual(0); + release.resolve(); + await expect(consumed).rejects.toMatchObject({ + reason: fail ? "io" : "cancelled", + cause: fail ? io : caller, + }); + await closing; + expect(handle.fd).toBeGreaterThanOrEqual(0); + } finally { + release.resolve(); + await Promise.allSettled([ + consumed, + ...(closing === undefined ? [] : [closing]), + ]); + spy.mockRestore(); + } + }, + ); + }, +); + +it("preserves BOM characters in length-delimited UTF-8 fields", async () => { + const bytes = cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "\ufefflibpython.so", + members: [ + { + name: "\ufeffasset", + typeCode: "x", + payload: Buffer.from("data"), + compress: false, + }, + { + name: "PYZ.pyz", + typeCode: "z", + compress: false, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "\ufeffmodule", typeCode: 0, body: Buffer.from("module") }, + ], + }), + }, + ], + }); + await withPyInstallerReader(bytes, async (reader) => { + const entries = await members(reader); + expect(reader.formatMetadata()).toMatchObject({ + python_library: "\ufefflibpython.so", + }); + expect(entries.map((entry) => entry.path)).toContain("\ufeffasset"); + expect( + entries.find((entry) => entry.path === "\ufeffasset")?.formatMetadata, + ).toMatchObject({ source_name: "\ufeffasset" }); + expect( + entries.find((entry) => entry.path === "PYZ.pyz/\ufeffmodule.pyc") + ?.formatMetadata, + ).toMatchObject({ source_name: "\ufeffmodule" }); + }); +}); + +it("keeps runtime options as declarations with literal names and distinct source indices", async () => { + const option = "pyi-runtime-tmpdir /tmp/../chosen"; + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: option, + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }, + { + name: option, + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }, + { + name: "data", + typeCode: "x", + payload: Buffer.from("data"), + compress: false, + }, + ], + }), + async (reader) => { + const entries = await members(reader); + const options = entries.filter((entry) => entry.kind === "metadata"); + expect(options.map((entry) => entry.path)).toEqual([ + "__pyinstaller__/runtime-options/0", + "__pyinstaller__/runtime-options/1", + ]); + expect(options.map((entry) => entry.formatMetadata)).toMatchObject([ + { source_name: option, index: 0 }, + { source_name: option, index: 1 }, + ]); + expect(reader.formatMetadata()).toMatchObject({ + runtime_option_count: 2, + entry_count: 3, + }); + await expect(reader.open(options[0]!)).rejects.toMatchObject({ + reason: "format", + }); + }, + ); +}); + +it("rejects metadata expansion before retaining every repeated long container path", async () => { + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: Array.from({ length: 3000 }, (_, index) => ({ + name: `module${String(index)}`, + typeCode: 0, + body: Buffer.from("x"), + })), + }); + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "a".repeat(40000), + typeCode: "z", + payload: image, + compress: false, + }, + ], + }), + async (reader) => { + let failure: unknown; + try { + for await (const entry of reader.entries()) { + void entry; + } + } catch (cause: unknown) { + failure = cause; + } + expect(failure).toBeInstanceOf(ArtifactReaderFailure); + expect(failure).toMatchObject({ reason: "limit", boundary: "memory" }); + }, + ); +}); + +it.each([false, true])( + "validates compressed member bounds through the stream (truncated: %s)", + async (truncated) => { + const payload = Buffer.from("complete module payload"); + const encoded = deflateSync(payload); + const bytes = cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "data", + typeCode: "x", + payload, + compress: true, + declaredUncompressedSize: truncated ? payload.length : 4, + }, + ], + }); + if (truncated) + bytes.writeUInt32BE(encoded.length - 2, 64 + encoded.length + 8); + await withPyInstallerReader(bytes, async (reader) => { + const entry = (await members(reader))[0]!; + await expect(buffer(await reader.open(entry))).rejects.toMatchObject({ + reason: truncated ? "format" : "limit", + }); + }); + }, +); + +it("joins a late private-file close and retries its failed owner after cancellation", async () => { + const bytes = cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }), + compress: true, + }, + ], + }); + await withPyInstallerReader(bytes, async (reader, borrowed) => { + const entered = deferred(), + release = deferred(); + const nativeClose = OwnedFileHandle.prototype.close; + const failure = new Error("private final close failed"); + let owner: OwnedFileHandle | undefined; + let attempts = 0; + const spy = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + if (owner === undefined) { + owner = this; + entered.resolve(); + await release.promise; + throw failure; + } + if (this === owner) attempts += 1; + await nativeClose.call(this); + }); + const signal = new AbortController(); + const listing = reader + .entries(signal.signal) + [Symbol.asyncIterator]() + .next() + .catch((cause: unknown) => cause); + try { + await entered.promise; + let settled = false; + const closing = reader.close().then(() => { + settled = true; + }); + signal.abort(new Error("cancel during final cleanup")); + await setImmediate(); + expect(settled).toBe(false); + expect(owner?.handle.fd).toBeGreaterThanOrEqual(0); + release.resolve(); + expect(await listing).toBe(failure); + await closing; + expect(attempts).toBe(1); + expect(owner?.handle.fd).toBe(-1); + expect(borrowed.fd).toBeGreaterThanOrEqual(0); + } finally { + release.resolve(); + spy.mockRestore(); + await listing; + await reader.close(); + } + }); +}); + +for (const cancel of [false, true]) { + it.each([false, true])( + `awaits private member close before completion (cancel: ${cancel}, close failure: %s)`, + async (failClose) => { + const bytes = cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + compress: true, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "app", typeCode: 0, body: Buffer.from("body") }, + ], + }), + }, + ], + }); + await withPyInstallerReader(bytes, async (reader, borrowed) => { + const entry = (await members(reader)).find((item) => + item.path.endsWith("app.pyc"), + )!; + const entered = deferred(), + release = deferred(); + const signal = new AbortController(); + const failure = new Error("private member final close failed"); + const nativeClose = OwnedFileHandle.prototype.close; + let owner: OwnedFileHandle | undefined; + let attempts = 0; + const spy = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + if (owner === undefined) { + owner = this; + attempts += 1; + entered.resolve(); + await release.promise; + if (failClose) throw failure; + } else if (this === owner) attempts += 1; + await nativeClose.call(this); + }); + const output = await reader.open(entry, signal.signal); + const reading = buffer(output).then( + (value) => ({ ok: true as const, value }), + (cause: unknown) => ({ ok: false as const, cause }), + ); + try { + await entered.promise; + expect(output.readableEnded).toBe(false); + expect(owner?.handle.fd).toBeGreaterThanOrEqual(0); + if (cancel) { + signal.abort(new Error("cancel during member cleanup")); + await setImmediate(); + expect(output.destroyed).toBe(false); + } + release.resolve(); + const result = await reading; + if (failClose || cancel) { + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected read failure"); + expect(result.cause).toMatchObject( + failClose + ? { reason: "io", message: failure.message } + : { reason: "cancelled" }, + ); + if (failClose) expect(owner?.handle.fd).toBeGreaterThanOrEqual(0); + } else { + expect(result.ok).toBe(true); + if (!result.ok) throw result.cause; + expect(result.value.subarray(16)).toEqual(Buffer.from("body")); + } + await reader.close(); + expect(attempts).toBe(failClose ? 2 : 1); + expect(owner?.handle.fd).toBe(-1); + expect(borrowed.fd).toBeGreaterThanOrEqual(0); + } finally { + release.resolve(); + await reading; + spy.mockRestore(); + await reader.close(); + } + }); + }, + ); +} + +it("preserves a private-image read failure when its descriptor close also fails", async () => { + const bytes = cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "app", typeCode: 0, body: Buffer.from("body") }], + }), + compress: true, + }, + ], + }); + await withPyInstallerReader(bytes, async (reader, borrowed) => { + const entry = (await members(reader)).find((item) => + item.path.endsWith("app.pyc"), + )!; + const nativeClose = OwnedFileHandle.prototype.close; + const nativeRead = borrowed.read; + const io = Object.assign(new Error("private read EIO"), { code: "EIO" }); + let failedOwner: OwnedFileHandle | undefined; + const read = vi + .spyOn(Object.getPrototypeOf(borrowed), "read") + .mockImplementation(function (this: FileHandle, ...args: unknown[]) { + return this === borrowed + ? Reflect.apply(nativeRead, this, args) + : Promise.reject(io); + }); + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + if (this.handle !== borrowed) { + failedOwner = this; + throw new Error("private close EIO"); + } + await nativeClose.call(this); + }); + try { + await expect(buffer(await reader.open(entry))).rejects.toMatchObject({ + reason: "io", + cause: io, + }); + await expect(reader.close()).rejects.toMatchObject({ + cleanup: { reason: "private close EIO" }, + }); + expect(failedOwner?.handle.fd).toBeGreaterThanOrEqual(0); + } finally { + read.mockRestore(); + close.mockRestore(); + await reader.close(); + } + expect(failedOwner?.handle.fd).toBe(-1); + expect(borrowed.fd).toBeGreaterThanOrEqual(0); + }); +}); + +it.each([false, true])( + "shares owned carrier admission across callers and joins late admission (owner closes: %s)", + async (ownerCloses) => { + const bytes = archive(Buffer.from("shared admission")); + await withPyInstallerReader( + bytes, + async (_borrowedReader, borrowed, path) => { + const reader = new PyInstallerArtifactReader(path); + const entered = deferred(), + release = deferred(); + const handles = new Set(); + const stat = borrowed.stat; + const spy = vi + .spyOn(Object.getPrototypeOf(borrowed), "stat") + .mockImplementation(async function ( + this: FileHandle, + ...args: unknown[] + ) { + if (this !== borrowed) { + handles.add(this); + entered.resolve(); + await release.promise; + } + return Reflect.apply(stat, this, args); + }); + const caller = new AbortController(), + reason = new Error("one admission waiter leaves"); + const first = reader + .prepareContainer(hash(bytes), caller.signal) + .catch((cause: unknown) => cause); + const peer = reader + .prepareContainer(hash(bytes)) + .catch((cause: unknown) => cause); + let closing: Promise | undefined; + try { + await entered.promise; + await setImmediate(); + expect(handles.size).toBe(1); + caller.abort(reason); + if (ownerCloses) { + let closed = false; + closing = reader.close().then(() => { + closed = true; + }); + await setImmediate(); + expect(closed).toBe(false); + } + release.resolve(); + expect(await first).toMatchObject({ + reason: "cancelled", + cause: reason, + }); + if (ownerCloses) { + expect(await peer).toMatchObject({ reason: "cancelled" }); + await closing; + } else expect(await peer).toBeUndefined(); + await reader.close(); + for (const handle of handles) expect(handle.fd).toBe(-1); + } finally { + release.resolve(); + await Promise.allSettled([first, peer]); + spy.mockRestore(); + await reader.close(); + await closing; + for (const handle of handles) { + await handle.close(); + } + } + }, + ); + }, +); diff --git a/src/artifacts/pyinstaller/PyInstallerBytes.ts b/src/artifacts/pyinstaller/PyInstallerBytes.ts new file mode 100644 index 0000000000..4dc61eef1b --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerBytes.ts @@ -0,0 +1,60 @@ +import type { FileHandle } from "node:fs/promises"; +import { err, ok, type Result } from "../../domain/result.js"; +import { ArtifactReaderFailure } from "../ArtifactReader.js"; +import { abortIfNeeded } from "../ArtifactHash.js"; +import { + pyInstallerFailure, + type PyInstallerFailure, +} from "./PyInstallerFailure.js"; + +/** Fill one already-budgeted format range; only zero bytes establish EOF. */ +export const readPyInstallerBytes = async ( + handle: Pick, + input: { + readonly position: number; + readonly length: number; + readonly label: string; + readonly signal?: AbortSignal | undefined; + }, +): Promise> => { + abortIfNeeded(input.signal); + if ( + !Number.isSafeInteger(input.position) || + !Number.isSafeInteger(input.length) || + input.position < 0 || + input.length < 0 || + !Number.isSafeInteger(input.position + input.length) + ) + return err( + pyInstallerFailure( + "format", + `${input.label} has an invalid source range`, + ), + ); + const bytes = Buffer.alloc(input.length); + let offset = 0; + while (offset < bytes.length) { + abortIfNeeded(input.signal); + let count: number; + try { + const read = await handle.read( + bytes, + offset, + bytes.length - offset, + input.position + offset, + ); + count = read.bytesRead; + } catch (cause: unknown) { + throw new ArtifactReaderFailure( + "io", + `${input.label} could not be read`, + { cause }, + ); + } + abortIfNeeded(input.signal); + if (count === 0) + return err(pyInstallerFailure("format", `${input.label} is truncated`)); + offset += count; + } + return ok(bytes); +}; diff --git a/src/artifacts/pyinstaller/PyInstallerFailure.ts b/src/artifacts/pyinstaller/PyInstallerFailure.ts new file mode 100644 index 0000000000..8ea582a055 --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerFailure.ts @@ -0,0 +1,10 @@ +/** Typed failure from a pure PyInstaller parser before the reader adapts it. */ +export interface PyInstallerFailure { + readonly reason: "format" | "integrity" | "limit"; + readonly message: string; +} + +export const pyInstallerFailure = ( + reason: PyInstallerFailure["reason"], + message: string, +): PyInstallerFailure => ({ reason, message }); diff --git a/src/artifacts/pyinstaller/PyInstallerFileAdmission.ts b/src/artifacts/pyinstaller/PyInstallerFileAdmission.ts new file mode 100644 index 0000000000..a472459a43 --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerFileAdmission.ts @@ -0,0 +1,36 @@ +import { + NonRegularFileReadError, + RegularFileAdmissionFailure, + RegularFileOperationCancelled, +} from "../../filesystem/RegularFile.js"; +import { ArtifactReaderFailure } from "../ArtifactReader.js"; + +/** Both carrier and private-image admission preserve cancellation and cleanup. */ +export const pyInstallerFileAdmissionFailure = ( + path: string, + cause: unknown, +): ArtifactReaderFailure => { + if (cause instanceof ArtifactReaderFailure) return cause; + if (cause instanceof RegularFileOperationCancelled) + return new ArtifactReaderFailure( + "cancelled", + "PyInstaller file admission cancelled", + { cause: cause.reason }, + ); + if (cause instanceof RegularFileAdmissionFailure) + return ArtifactReaderFailure.withCleanup( + pyInstallerFileAdmissionFailure(path, cause.cause), + ArtifactReaderFailure.cleanupObservation(cause.cleanupCause, path), + ); + if (cause instanceof NonRegularFileReadError) + return new ArtifactReaderFailure( + "format", + `PyInstaller source is not a regular file: ${path}`, + { cause }, + ); + return new ArtifactReaderFailure( + "io", + "PyInstaller archive could not be opened", + { cause }, + ); +}; diff --git a/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts b/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts new file mode 100644 index 0000000000..08cfb4f884 --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts @@ -0,0 +1,1213 @@ +import { buffer } from "node:stream/consumers"; +import { readFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { deflateSync } from "node:zlib"; +import { expect, it, vi } from "vitest"; +import { withPyInstallerReader } from "../../../tests/fixtures/pyinstallerReader.js"; +import { + scanCanonicalArtifactInventory, + inventoryArtifact, + materializeArtifactInventory, +} from "../../../tests/fixtures/artifactInventory.js"; +import { + cArchiveExecutable, + pyzArchive, + PYTHON_39_MAGIC, +} from "./CArchive.fixture.js"; +import { readPyzArchive } from "./PyzArchive.js"; +import { createEvidence } from "../../domain/evidence.js"; +import { jsonValueSchema } from "../../domain/jsonValue.js"; +import { parseArtifactInventoryEvidence } from "../../domain/artifactInventoryEvidence.js"; +import { + ArtifactReaderFailure, + type ArtifactEntry, +} from "../ArtifactReader.js"; + +it("keeps compressed source locations unknown when stored and decoded lengths coincide", async () => { + const payload = Buffer.alloc(11, 0x61); + expect(deflateSync(payload).length).toBe(payload.length); + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [{ name: "data", typeCode: "x", payload, compress: true }], + }), + async (_reader, _carrier, path) => { + const inventory = await scanCanonicalArtifactInventory(path); + expect( + inventory.occurrences.find((row) => row.logical_path === "data"), + ).toMatchObject({ + hash_status: "verified", + source_location: null, + format_metadata: { compression_flag: 1 }, + }); + }, + ); +}); + +it("bounds an admitted drive-prefixed CArchive path before diagnostic serialization", async () => { + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "C:" + "x".repeat(10 * 1024 * 1024), + typeCode: "x", + payload: Buffer.from("x"), + compress: false, + }, + ], + }), + async (reader) => { + const failure = await reader + .entries() + [Symbol.asyncIterator]() + .next() + .then( + () => undefined, + (cause: unknown) => cause, + ); + expect(failure).toMatchObject({ reason: "path" }); + expect((failure as Error).message.length).toBeLessThan(1024); + }, + ); +}); + +it.each(["magic", "marshal"])( + "reuses the private bytes of an opaque outer-compressed PYZ (%s)", + async (kind) => { + const payload = pyzArchive({ + magic: kind === "magic" ? Buffer.alloc(4) : PYTHON_39_MAGIC, + modules: [], + ...(kind === "marshal" ? { toc: Buffer.from([0x4e]) } : {}), + }); + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [{ name: "PYZ.pyz", typeCode: "z", payload, compress: true }], + }), + async (reader, carrier) => { + const entries: ArtifactEntry[] = []; + for await (const entry of reader.entries()) entries.push(entry); + const entry = entries[0]!; + expect(entry.kind).toBe("file"); + const read = vi + .spyOn(carrier, "read") + .mockRejectedValue( + new Error("opaque image must not be reinflated from carrier"), + ); + try { + for (let run = 0; run < 2; run++) + expect(await buffer(await reader.open(entry))).toEqual(payload); + } finally { + read.mockRestore(); + } + }, + ); + }, +); + +it.each([true, false])( + "authenticates mixed-kind duplicate pages in any merge order (option first: %s)", + async (optionFirst) => { + const logical = `__pyinstaller__/runtime-options/${optionFirst ? 0 : 1}`; + const option = { + name: "pyi-option", + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }; + const file = { + name: logical, + typeCode: "x", + payload: Buffer.from("first"), + compress: false, + }; + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + ...(optionFirst ? [option, file] : [file, option]), + { + ...file, + name: logical.replaceAll("/", "\\"), + payload: Buffer.from("second"), + }, + ], + }), + async (_reader, _handle, path) => { + const inventory = await inventoryArtifact(path); + const group = inventory.occurrences.filter( + (row) => row.logical_path === logical, + ); + expect(group).toHaveLength(3); + const pages = group.map((row, index) => + createEvidence( + { + path, + sha256: inventory.manifest.root_sha256, + format: "pyinstaller", + }, + { + id: "rea-artifact-graph", + name: "artifact inventory", + version: "1", + }, + { + operation: "inventory_artifact", + parameters: { page: index }, + confidence: "observed", + authority: "shipped-artifact", + result: jsonValueSchema.parse({ + ...inventory, + nodes: index === 0 ? inventory.nodes : [], + edges: index === 0 ? inventory.edges : [], + occurrences: + index === 0 + ? [ + ...inventory.occurrences.filter( + (item) => item.logical_path !== logical, + ), + row, + ] + : [row], + }), + }, + ), + ); + for (const order of [ + [0, 1, 2], + [0, 2, 1], + [1, 0, 2], + [1, 2, 0], + [2, 0, 1], + [2, 1, 0], + ]) { + const merged = parseArtifactInventoryEvidence( + order.map((index) => pages[index]), + ).inventory; + expect(merged.complete).toBe(true); + expect(merged.manifest).toEqual(inventory.manifest); + expect(merged.occurrences).toEqual(inventory.occurrences); + } + }, + ); + }, +); + +it.each(["duplicate", "size", "offset", "overlap"])( + "bounds %s diagnostics for large nondeclaration names", + async (mode) => { + const name = "member".repeat(200_000); + const first = { + name: mode === "overlap" ? name + "-a" : name, + typeCode: "x", + payload: Buffer.from("body"), + compress: false, + ...(mode === "size" ? { declaredUncompressedSize: 5 } : {}), + }; + const bytes = cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: + mode === "duplicate" || mode === "overlap" + ? [ + first, + { ...first, name: mode === "duplicate" ? name : name + "-b" }, + ] + : [first], + }); + const toc = 64 + bytes.readUInt32BE(bytes.length - 88 + 12); + if (mode === "offset") + bytes.writeUInt32BE( + bytes.readUInt32BE(bytes.length - 88 + 12) + 1, + toc + 4, + ); + if (mode === "overlap") + bytes.writeUInt32BE(0, toc + bytes.readUInt32BE(toc) + 4); + await withPyInstallerReader(bytes, async (_reader, _handle, path) => { + const failure: unknown = await scanCanonicalArtifactInventory(path).then( + () => undefined, + (cause: unknown) => cause, + ); + expect(failure).toBeInstanceOf(ArtifactReaderFailure); + if (!(failure instanceof ArtifactReaderFailure)) + throw new Error("Expected parser failure"); + expect(failure.reason).toBe(mode === "offset" ? "format" : "integrity"); + expect(failure.message.length).toBeLessThan(128); + expect(failure.message).toContain( + `table index ${mode === "duplicate" || mode === "overlap" ? 1 : 0}`, + ); + }); + }, +); + +it.each(["empty", "span"])( + "bounds PYZ %s diagnostics for a maximum-size marshal name", + async (mode) => { + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "n".repeat(65_536), typeCode: 0, body: Buffer.from("body") }, + ], + }); + image.writeInt32LE( + mode === "empty" ? 0 : image.length + 1, + image.length - 4, + ); + await withPyInstallerReader(image, async (_reader, handle) => { + const parsed = await readPyzArchive(handle, 0, image.length); + const message = !parsed.ok + ? parsed.error.message + : parsed.value.kind === "opaque" + ? parsed.value.limitation + : undefined; + expect(message).toBeDefined(); + expect(message?.length).toBeLessThan(256); + expect(message).toContain("table index 0"); + }); + }, +); + +it("keeps malformed-declaration diagnostics bounded while identifying the TOC row", async () => { + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "option".repeat(200_000), + typeCode: "o", + payload: Buffer.from("unexpected"), + compress: false, + }, + ], + }), + async (_reader, _handle, path) => { + const failure: unknown = await scanCanonicalArtifactInventory(path).then( + () => undefined, + (cause: unknown) => cause, + ); + expect(failure).toBeInstanceOf(ArtifactReaderFailure); + if (!(failure instanceof ArtifactReaderFailure)) + throw new Error("Expected a parser failure"); + expect(failure.reason).toBe("integrity"); + expect(failure.message.length).toBeLessThan(128); + expect(failure.message).toContain("table index 0"); + }, + ); +}); + +for (const typeCode of ["o", "d"] as const) { + it.each(["stored", "compressed", "flag-only"])( + `rejects local payload/compression on ${typeCode} declarations (%s)`, + async (mode) => { + const bytes = cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: typeCode === "d" ? "../owner:member" : "pyi-option", + typeCode, + payload: + mode === "flag-only" + ? Buffer.alloc(0) + : Buffer.from("local payload"), + compress: mode === "compressed", + }, + ], + }); + if (mode === "flag-only") { + const archiveStart = 64; + const toc = bytes.readUInt32BE(bytes.length - 88 + 12); + bytes[archiveStart + toc + 16] = 1; + } + await withPyInstallerReader(bytes, async (_reader, _handle, path) => { + await expect( + scanCanonicalArtifactInventory(path), + ).rejects.toMatchObject({ + reason: "integrity", + message: expect.stringContaining("declaration"), + }); + }); + }, + ); + it(`does not claim a destination case collision for a ${typeCode} query path`, async () => { + const logical = `__pyinstaller__/${typeCode === "o" ? "runtime-options" : "dependencies"}/0`; + const realPath = logical.toUpperCase(); + const payload = Buffer.from("real file"); + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: typeCode === "d" ? "../owner:member" : "pyi-option", + typeCode, + payload: Buffer.alloc(0), + compress: false, + }, + { name: realPath, typeCode: "x", payload, compress: false }, + ], + }), + async (_reader, _handle, path) => { + const inventory = await scanCanonicalArtifactInventory(path); + expect( + [ + ...inventory.limitations, + ...inventory.occurrences.flatMap((row) => row.limitations), + ].some((note) => note.includes("case-insensitive destination")), + ).toBe(false); + const output = join(dirname(path), "extracted"); + await materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot: output, + environment: process.env, + integrityPolicy: "fail", + }, + path, + inventory, + ); + expect(await readFile(join(output, realPath))).toEqual(payload); + }, + ); + }); +} + +it("keeps genuine destination collisions on files without annotating same-path metadata", async () => { + const logical = "__pyinstaller__/runtime-options/0"; + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "pyi-option", + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }, + ...[logical, logical.toUpperCase()].map((name) => ({ + name, + typeCode: "x", + payload: Buffer.from(name), + compress: false, + })), + ], + }), + async (_reader, _handle, path) => { + const inventory = await scanCanonicalArtifactInventory(path); + expect( + inventory.limitations.some((note) => + note.includes("case-insensitive destination"), + ), + ).toBe(true); + const metadata = inventory.occurrences.find( + (row) => row.entry_kind === "metadata", + ); + expect( + metadata?.limitations.some((note) => + note.includes("case-insensitive destination"), + ), + ).toBe(false); + expect( + inventory.occurrences + .filter( + (row) => row.entry_kind === "file" && row.logical_path !== ".", + ) + .every((row) => + row.limitations.some((note) => + note.includes("case-insensitive destination"), + ), + ), + ).toBe(true); + }, + ); +}); + +it("anchors declarations to their source container rather than a same-named directory", async () => { + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "../owner:member", + typeCode: "d", + payload: Buffer.alloc(0), + compress: false, + }, + { + name: "__pyinstaller__", + typeCode: "z", + compress: false, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "dependencies", typeCode: 3, body: Buffer.alloc(0) }, + ], + }), + }, + ], + }), + async (_reader, _handle, path) => { + const inventory = await scanCanonicalArtifactInventory(path); + const root = inventory.occurrences.find( + (row) => row.logical_path === ".", + ); + const directory = inventory.occurrences.find( + (row) => row.logical_path === "__pyinstaller__/dependencies", + ); + const metadata = inventory.occurrences.find( + (row) => row.entry_kind === "metadata", + ); + expect(directory).toBeDefined(); + expect(root).toBeDefined(); + expect(metadata).toBeDefined(); + expect(metadata?.parent_occurrence_id).toBe(root?.occurrence_id); + expect(metadata?.parent_occurrence_id).not.toBe(directory?.occurrence_id); + }, + ); +}); + +it("resolves a unique source-map file despite a metadata query-path alias", async () => { + const logical = "__pyinstaller__/runtime-options/0"; + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "pyi-option", + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }, + { + name: logical, + typeCode: "x", + payload: Buffer.from("let value = 1;"), + compress: false, + }, + { + name: logical + ".map", + typeCode: "x", + payload: Buffer.from( + JSON.stringify({ + version: 3, + sources: ["source.js"], + names: [], + mappings: "AAAA", + }), + ), + compress: false, + }, + ], + }), + async (_reader, _handle, path) => { + const inventory = await scanCanonicalArtifactInventory(path); + const source = inventory.occurrences.find( + (row) => row.logical_path === logical && row.entry_kind === "file", + ); + const map = inventory.occurrences.find( + (row) => row.logical_path === logical + ".map", + ); + expect(source?.artifact_id).toBeTruthy(); + expect(map).toBeDefined(); + expect( + inventory.edges.find( + (edge) => edge.occurrence_id === map?.occurrence_id, + ), + ).toMatchObject({ + relation: "maps-source", + parent_artifact_id: source?.artifact_id, + }); + expect( + map?.limitations.some((note) => note.includes("multiple occurrences")), + ).toBe(false); + }, + ); +}); + +it.each(["x", "s", "empty-module"])( + "does not treat a same-named %s as a crypto-key module", + async (kind) => { + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "pyimod00_crypto_key", + typeCode: kind === "empty-module" ? "m" : kind, + payload: + kind === "empty-module" + ? Buffer.alloc(0) + : Buffer.from("ordinary content"), + compress: false, + }, + { + name: "PYZ.pyz", + typeCode: "z", + compress: false, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { + name: "app", + typeCode: 0, + body: Buffer.from("readable module"), + }, + ], + }), + }, + ], + }), + async (_reader, _handle, path) => { + const inventory = await scanCanonicalArtifactInventory(path); + expect( + inventory.occurrences.find((row) => row.logical_path === ".") + ?.format_metadata, + ).toMatchObject({ encryption_marker_present: false }); + expect( + inventory.occurrences.find( + (row) => row.logical_path === "PYZ.pyz/app.pyc", + ), + ).toMatchObject({ + encrypted: false, + hash_status: "verified", + format_metadata: { encryption_source: null }, + }); + const output = join(dirname(path), "extracted"); + await materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot: output, + environment: process.env, + integrityPolicy: "fail", + }, + path, + inventory, + ); + expect( + (await readFile(join(output, "PYZ.pyz/app.pyc"))).subarray(16), + ).toEqual(Buffer.from("readable module")); + }, + ); + }, +); + +it.each([ + "module-in-toc", + "module-crossing-toc", + "namespace-in-toc", + "namespace-past-image", +])("rejects PYZ positions outside the payload region (%s)", async (kind) => { + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { + name: "app", + typeCode: kind.startsWith("namespace") ? 3 : 0, + body: Buffer.from("payload"), + }, + ], + }); + const toc = image.readUInt32BE(8); + if (kind === "module-crossing-toc") + image.writeInt32LE(toc - 12 + 1, image.length - 4); + else + image.writeInt32LE( + kind === "namespace-past-image" ? image.length + 1 : toc + 1, + image.length - 9, + ); + await withPyInstallerReader(image, async (_reader, handle) => { + await expect( + readPyzArchive(handle, 0, image.length), + ).resolves.toMatchObject({ ok: false, error: { reason: "integrity" } }); + }); +}); + +it("retains legacy namespace payload extents without materializing them", async () => { + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { + name: "ns", + typeCode: 3, + legacyNamespacePayload: true, + body: Buffer.from("legacy namespace code object"), + }, + ], + }); + const length = image.readInt32LE(image.length - 4); + expect(length).toBeGreaterThan(0); + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { name: "PYZ.pyz", typeCode: "z", compress: false, payload: image }, + ], + }), + async (_reader, _handle, path) => { + const inventory = await scanCanonicalArtifactInventory(path); + expect( + inventory.occurrences.find((row) => row.logical_path === "PYZ.pyz/ns"), + ).toMatchObject({ + entry_kind: "directory", + declared_size: 0, + compressed_size: length, + format_metadata: { + relative_offset: 12, + type_code: 3, + compression_flag: 1, + }, + limitations: expect.arrayContaining([ + expect.stringContaining("not materialized"), + ]), + }); + const result = await materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot: join(dirname(path), "out"), + environment: process.env, + integrityPolicy: "fail", + }, + path, + inventory, + ); + expect(result.artifacts).toEqual([]); + }, + ); +}); + +it.each(["magic", "marshal"])( + "retains unknown traversal coverage for an opaque PYZ %s", + async (kind) => { + const known = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "known", typeCode: 0, body: Buffer.from("known") }], + }); + const opaque = pyzArchive({ + magic: kind === "magic" ? Buffer.alloc(4) : PYTHON_39_MAGIC, + modules: [], + ...(kind === "marshal" ? { toc: Buffer.from([0x4e]) } : {}), + }); + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { name: "known.pyz", typeCode: "z", payload: known, compress: false }, + { + name: "opaque.pyz", + typeCode: "z", + payload: opaque, + compress: false, + }, + ], + }), + async (reader, _root, path) => { + for (let run = 0; run < 2; run++) { + const observed: unknown[] = []; + const listed: ArtifactEntry[] = []; + for await (const entry of reader.entries(undefined, { + onCoverage: (coverage) => observed.push(coverage), + })) + listed.push(entry); + expect(listed).toHaveLength(3); + expect(observed).toEqual([ + expect.objectContaining({ status: "unknown" }), + ]); + } + const inventory = await scanCanonicalArtifactInventory(path); + expect(inventory.coverage).toMatchObject({ status: "unknown" }); + expect(inventory.limitations.join(" ")).toContain("opaque"); + expect( + inventory.occurrences.find( + (row) => row.logical_path === "known.pyz/known.pyc", + ), + ).toMatchObject({ hash_status: "verified" }); + expect( + inventory.occurrences.find((row) => row.logical_path === "opaque.pyz") + ?.format_metadata, + ).toMatchObject({ pyz_status: "opaque" }); + }, + ); + }, +); + +it.each([false, true])( + "derives namespace encryption from stored payload and the marker (%s)", + async (marker) => { + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { + name: "legacy", + typeCode: 3, + legacyNamespacePayload: true, + body: Buffer.from("declared namespace payload"), + }, + { name: "empty", typeCode: 3, body: Buffer.alloc(0) }, + ], + }); + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + ...(marker + ? [ + { + name: "pyimod00_crypto_key", + typeCode: "m", + payload: Buffer.from("key module"), + compress: false, + }, + ] + : []), + { name: "PYZ.pyz", typeCode: "z", payload: image, compress: false }, + ], + }), + async (_reader, _root, path) => { + const inventory = await scanCanonicalArtifactInventory(path); + expect( + inventory.occurrences.find( + (row) => row.logical_path === "PYZ.pyz/legacy", + ), + ).toMatchObject({ + entry_kind: "directory", + encrypted: marker, + compressed_size: expect.any(Number), + format_metadata: { + encryption_source: marker ? "carchive-marker" : null, + }, + }); + expect( + inventory.occurrences.find( + (row) => row.logical_path === "PYZ.pyz/empty", + ), + ).toMatchObject({ + entry_kind: "directory", + encrypted: false, + compressed_size: 0, + format_metadata: { encryption_source: null }, + }); + }, + ); + }, +); + +it.each([ + ["sequence", Buffer.from("5ba1860100", "hex")], + ["string", Buffer.from("7501000100", "hex")], + [ + "depth", + Buffer.concat([ + ...Array.from({ length: 18 }, () => Buffer.from([0x29, 1])), + Buffer.from("6900000000", "hex"), + ]), + ], + [ + "nodes", + Buffer.concat([ + Buffer.from("5ba0860100", "hex"), + Buffer.alloc(100_000 * 5).fill(Buffer.from("6900000000", "hex")), + ]), + ], +])( + "preserves the marshal %s resource failure through inventory", + async (_guard, toc) => { + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + compress: false, + payload: pyzArchive({ magic: PYTHON_39_MAGIC, modules: [], toc }), + }, + ], + }), + async (_reader, _handle, path) => { + await expect( + scanCanonicalArtifactInventory(path), + ).rejects.toMatchObject({ + reason: "limit", + message: expect.stringContaining("PYZ marshal"), + }); + }, + ); + }, +); + +it.each([1, 2])( + "rejects namespace data extending into the table (length %s)", + async (length) => { + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "ns", typeCode: 3, body: Buffer.alloc(0) }], + }); + // The fixture's last two marshal integers are the namespace offset/length. + image.writeInt32LE(12, image.length - 9); + image.writeInt32LE(length, image.length - 4); + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { name: "PYZ.pyz", typeCode: "z", compress: false, payload: image }, + ], + }), + async (_reader, _handle, path) => { + await expect( + scanCanonicalArtifactInventory(path), + ).rejects.toMatchObject({ + reason: "integrity", + message: expect.stringContaining("payload region"), + }); + }, + ); + }, +); + +it("retains the producer's nonzero namespace offset without inventing data", async () => { + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + compress: false, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "ns", typeCode: 3, body: Buffer.alloc(0) }], + }), + }, + ], + }), + async (_reader, _handle, path) => { + const inventory = await scanCanonicalArtifactInventory(path); + expect( + inventory.occurrences.find((row) => row.logical_path === "PYZ.pyz/ns"), + ).toMatchObject({ + entry_kind: "directory", + declared_size: 0, + compressed_size: 0, + format_metadata: { + relative_offset: 12, + type_code: 3, + compression_flag: 0, + }, + }); + }, + ); +}); + +it("retains multipackage references as distinct unresolved declarations", async () => { + const names = [ + "../owner/app:assets/x:y", + "../owner/app:assets/x:y", + "missing-colon", + ":empty-owner", + ]; + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: names.map((name) => ({ + name, + typeCode: "d", + payload: Buffer.alloc(0), + compress: false, + })), + }), + async (reader) => { + const entries: ArtifactEntry[] = []; + for await (const entry of reader.entries()) entries.push(entry); + expect(reader.formatMetadata()).toMatchObject({ + external_dependency_count: names.length, + runtime_option_count: 0, + }); + expect(new Set(entries.map((entry) => entry.adapterKey)).size).toBe( + names.length, + ); + for (const [index, entry] of entries.entries()) { + expect(entry.kind).toBe("metadata"); + expect(entry.path).toBe( + `__pyinstaller__/dependencies/${String(index)}`, + ); + expect(entry.formatMetadata).toMatchObject({ + source_name: names[index], + type_code: "d", + dependency_status: index < 2 ? "unresolved" : "invalid-reference", + dependency_archive: index < 2 ? "../owner/app" : null, + dependency_member: index < 2 ? "assets/x:y" : null, + }); + await expect(reader.open(entry)).rejects.toMatchObject({ + reason: "format", + }); + } + }, + ); +}); + +for (const container of ["carchive", "outer-pyz", "inner-pyz"] as const) { + it.each(["junk", "second-stream"])( + `rejects unused compressed input in ${container} (%s)`, + async (suffixKind) => { + const suffix = + suffixKind === "junk" + ? Buffer.from("trailing bytes") + : deflateSync(Buffer.from("another stream")); + const body = Buffer.from("first stream"); + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { + name: "app", + typeCode: 0, + body, + ...(container === "inner-pyz" ? { compressedSuffix: suffix } : {}), + }, + ], + }); + const bytes = cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: container === "carchive" ? "data" : "PYZ.pyz", + typeCode: container === "carchive" ? "x" : "z", + payload: container === "carchive" ? body : image, + compress: container !== "inner-pyz", + ...(container === "inner-pyz" ? {} : { compressedSuffix: suffix }), + }, + ], + }); + await withPyInstallerReader(bytes, async (reader) => { + const consume = async () => { + for await (const entry of reader.entries()) + if (entry.kind === "file") await buffer(await reader.open(entry)); + }; + await expect(consume()).rejects.toMatchObject({ + reason: "integrity", + message: expect.stringContaining("complete declared source span"), + }); + }); + }, + ); +} + +it("fills successful short reads through actual classification, TOCs and selected module hashing", async () => { + const bytes = cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "PYZ.pyz", + typeCode: "z", + compress: false, + payload: pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [ + { name: "app", typeCode: 0, body: Buffer.from("module body") }, + ], + }), + }, + ], + }); + await withPyInstallerReader(bytes, async (_reader, handle, path) => { + const read = handle.read; + let shortReads = 0; + const spy = vi + .spyOn(Object.getPrototypeOf(handle), "read") + .mockImplementation(function (this: unknown, ...args: unknown[]) { + if (typeof args[2] === "number" && args[2] > 7) { + args[2] = 7; + shortReads += 1; + } + return Reflect.apply(read, this, args); + }); + try { + const inventory = await scanCanonicalArtifactInventory(path); + expect(inventory.manifest.root_format).toBe("pyinstaller"); + expect( + inventory.occurrences.find( + (row) => row.logical_path === "PYZ.pyz/app.pyc", + ), + ).toMatchObject({ + hash_status: "verified", + format_metadata: { source_name: "app" }, + }); + expect(shortReads).toBeGreaterThan(20); + } finally { + spy.mockRestore(); + } + }); +}); + +it("distinguishes actual EOF after a short PYZ header read", async () => { + const bytes = pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }); + await withPyInstallerReader(bytes, async (_reader, handle) => { + const read = handle.read.bind(handle); + let calls = 0; + const spy = vi + .spyOn(handle, "read") + .mockImplementation((...args: unknown[]) => { + calls += 1; + if (calls > 1) + return Promise.resolve({ bytesRead: 0, buffer: args[0] }); + args[2] = 3; + return Reflect.apply(read, handle, args); + }); + try { + await expect( + readPyzArchive(handle, 0, bytes.length), + ).resolves.toMatchObject({ + ok: false, + error: { reason: "format", message: expect.stringContaining("header") }, + }); + expect(calls).toBe(2); + } finally { + spy.mockRestore(); + } + }); +}); + +it("retains native I/O failure after a short read and simultaneous cancellation", async () => { + const bytes = pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }); + await withPyInstallerReader(bytes, async (_reader, handle) => { + const read = handle.read.bind(handle), + caller = new AbortController(); + const failure = Object.assign(new Error("late native EIO"), { + code: "EIO", + }); + let calls = 0; + const spy = vi + .spyOn(handle, "read") + .mockImplementation((...args: unknown[]) => { + calls += 1; + if (calls > 1) { + caller.abort(new Error("caller leaves")); + return Promise.reject(failure); + } + args[2] = 3; + return Reflect.apply(read, handle, args); + }); + try { + await expect( + readPyzArchive(handle, 0, bytes.length, caller.signal), + ).rejects.toMatchObject({ reason: "io", cause: failure }); + } finally { + spy.mockRestore(); + } + }); +}); + +it("rejects path-component expansion before normalization or consumer trie allocation", async () => { + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: "a/".repeat(100_000) + "x", + typeCode: "x", + payload: Buffer.alloc(0), + compress: false, + }, + ], + }), + async (reader) => { + const list = async () => { + for await (const entry of reader.entries()) void entry; + }; + await expect(list()).rejects.toMatchObject({ + reason: "limit", + boundary: "memory", + message: expect.stringContaining("namespace budget"), + }); + }, + ); +}); + +it.each([true, false])( + "extracts a real file sharing a metadata query path (option first: %s)", + async (optionFirst) => { + const path = `__pyinstaller__/runtime-options/${optionFirst ? "0" : "1"}`; + const option = { + name: "pyi-option value", + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }; + const file = { + name: path, + typeCode: "x", + payload: Buffer.from("real file payload"), + compress: false, + }; + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: optionFirst ? [option, file] : [file, option], + }), + async (_reader, _handle, source) => { + const inventory = await scanCanonicalArtifactInventory(source); + const output = join(dirname(source), "extracted"); + await materializeArtifactInventory( + { + inputPath: source, + inputFormat: "elf", + outputRoot: output, + environment: process.env, + integrityPolicy: "fail", + }, + source, + inventory, + ); + expect(await readFile(join(output, path))).toEqual(file.payload); + }, + ); + }, +); + +it("reserves possible case-annotation expansion along deep sibling paths", async () => { + const names = ["a/".repeat(4000) + "x", "A/".repeat(4000) + "X"]; + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: names.map((name) => ({ + name, + typeCode: "x", + payload: Buffer.alloc(0), + compress: false, + })), + }), + async (reader) => { + const list = async () => { + for await (const entry of reader.entries()) void entry; + }; + await expect(list()).rejects.toMatchObject({ + reason: "limit", + boundary: "memory", + }); + }, + ); +}); diff --git a/src/artifacts/pyinstaller/PyInstallerMemberStream.ts b/src/artifacts/pyinstaller/PyInstallerMemberStream.ts new file mode 100644 index 0000000000..480df2d002 --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerMemberStream.ts @@ -0,0 +1,147 @@ +import { Readable, Transform } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { createInflate } from "node:zlib"; +import { ArtifactReaderFailure } from "../ArtifactReader.js"; +import { abortIfNeeded } from "../ArtifactHash.js"; + +interface MemberStreamOptions { + readonly sourceBytes: number; + readonly compressed: boolean; + readonly header: Buffer | undefined; + readonly expectedBytes: number | undefined; + readonly maxBytes: number; + readonly signal: AbortSignal; + readonly charge?: (bytes: number) => void; + readonly complete?: (bytes: number) => Promise; +} + +/** Backpressure bounds plaintext independently of the untrusted declared size. */ +export const pyInstallerMemberStream = ( + source: Readable, + options: MemberStreamOptions, +): { readonly output: Readable; readonly completion: Promise } => { + let bytes = 0; + let headerSent = false; + let completing = false; + const prependHeader = (): void => { + if (!headerSent && options.header !== undefined) + output.push(options.header); + headerSent = true; + }; + const output = new Transform({ + transform(chunk: Buffer, _encoding, callback) { + try { + abortIfNeeded(options.signal); + bytes += chunk.length; + if (bytes > options.maxBytes) + throw new ArtifactReaderFailure( + "limit", + "PyInstaller member exceeds its plaintext budget", + ); + options.charge?.(chunk.length); + prependHeader(); + callback(null, chunk); + } catch (cause: unknown) { + callback(asError(cause)); + } + }, + flush(callback) { + completing = true; + void (async () => { + abortIfNeeded(options.signal); + if ( + inflate !== undefined && + inflate.bytesWritten !== options.sourceBytes + ) + throw new ArtifactReaderFailure( + "integrity", + "Compressed PyInstaller member did not consume its complete declared source span", + ); + if ( + options.expectedBytes !== undefined && + bytes !== options.expectedBytes + ) + throw new ArtifactReaderFailure( + "integrity", + "PyInstaller member size disagrees with its declaration or previous observation", + ); + await options.complete?.(bytes); + abortIfNeeded(options.signal); + prependHeader(); + })().then( + () => { + completing = false; + try { + abortIfNeeded(options.signal); + } catch (cause: unknown) { + callback(asError(cause)); + return; + } + callback(); + }, + (cause: unknown) => { + completing = false; + callback(asError(cause)); + }, + ); + }, + }); + // A consumer can retire its reader before attaching its stream error handler. + output.on("error", () => undefined); + const onAbort = (): void => { + // Verification and private-file cleanup own their accepted native I/O. + if (completing) return; + // The descriptor source owns cancellation until its native read settles. + // Retiring it downstream earlier would hide a native I/O rejection. + if (!source.readableEnded && !source.destroyed) return; + output.destroy( + new ArtifactReaderFailure( + "cancelled", + "PyInstaller member read cancelled", + { cause: options.signal.reason }, + ), + ); + }; + options.signal.addEventListener("abort", onAbort, { once: true }); + const onSourceFailure = (cause: unknown): void => { + output.destroy( + cause instanceof ArtifactReaderFailure + ? cause + : new ArtifactReaderFailure( + "io", + "PyInstaller member source could not be read", + { cause }, + ), + ); + }; + source.on("error", onSourceFailure); + const inflate = options.compressed ? createInflate() : undefined; + inflate?.on("error", (cause: unknown) => + output.destroy( + new ArtifactReaderFailure( + "format", + "Compressed PyInstaller member is not a complete zlib stream", + { cause }, + ), + ), + ); + const completion = ( + inflate === undefined + ? pipeline(source, output) + : pipeline(source, inflate, output) + ).finally(() => { + options.signal.removeEventListener("abort", onAbort); + source.removeListener("error", onSourceFailure); + }); + // Preserve rejection for the owning scope without an early unhandled promise. + void completion.catch(() => undefined); + if (options.signal.aborted) onAbort(); + return { output, completion }; +}; + +const asError = (cause: unknown): Error => + cause instanceof Error + ? cause + : new ArtifactReaderFailure("format", "PyInstaller member read failed", { + cause, + }); diff --git a/src/artifacts/pyinstaller/PyInstallerMetadata.ts b/src/artifacts/pyinstaller/PyInstallerMetadata.ts new file mode 100644 index 0000000000..8319721064 --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerMetadata.ts @@ -0,0 +1,179 @@ +import type { + PyInstallerMetadata, + PyInstallerEntryMetadata, +} from "../../domain/artifactFormatMetadata.js"; +import type { CArchiveEntry, PyInstallerCookie } from "./CArchiveLayout.js"; +import type { PyzArchive, PyzEntry } from "./PyzArchive.js"; +import { pythonVersionForMagic } from "./PythonBytecode.js"; + +const pythonVersion = (value: number) => + value >= 100 + ? { major: Math.floor(value / 100), minor: value % 100 } + : value >= 20 + ? { major: Math.floor(value / 10), minor: value % 10 } + : undefined; + +export const cookieMetadata = ( + cookie: PyInstallerCookie, + entries: readonly CArchiveEntry[], + encryptionMarker: boolean, +): PyInstallerMetadata => { + const version = pythonVersion(cookie.pythonVersion); + return { + kind: "pyinstaller", + layout_version: cookie.layout, + archive_start: cookie.archiveStart, + archive_size: cookie.packageLength, + cookie_offset: cookie.cookieOffset, + toc_offset: cookie.tocOffset, + toc_size: cookie.tocLength, + entry_count: entries.length, + runtime_option_count: entries.filter((entry) => entry.typeCode === "o") + .length, + external_dependency_count: entries.filter((entry) => entry.typeCode === "d") + .length, + python_version_raw: cookie.pythonVersion, + python_major: version?.major ?? null, + python_minor: version?.minor ?? null, + python_library: cookie.pythonLibrary, + encryption_marker_present: encryptionMarker, + }; +}; + +/** A sibling PYZ header is usable for scripts only when every image and the cookie agree. */ +export const scriptPythonMagic = ( + cookie: PyInstallerCookie, + archives: Iterable, +): Buffer | undefined => { + const version = pythonVersion(cookie.pythonVersion); + if (version === undefined) return undefined; + let selected: Buffer | undefined; + for (const archive of archives) { + const magic = archive.pythonMagic; + if ( + pythonVersionForMagic(magic) !== + `${String(version.major)}.${String(version.minor)}` || + (selected !== undefined && !selected.equals(magic)) + ) + return undefined; + selected = magic; + } + return selected; +}; + +const entryMetadata = ( + source: Pick< + PyInstallerEntryMetadata, + | "container" + | "container_name" + | "index" + | "source_name" + | "type_code" + | "relative_offset" + | "compression_flag" + >, + bytecode: boolean, + header: Buffer | undefined, +): PyInstallerEntryMetadata => ({ + kind: "pyinstaller-entry", + ...source, + bytecode_header: bytecode + ? header === undefined + ? "unknown" + : "derived" + : "not-bytecode", + bytecode_magic: header?.subarray(0, 4).toString("hex") ?? null, + pyz_status: "not-pyz", + directory_record_offset: null, + directory_record_size: null, + pyz_sha256: null, + pyz_python_magic: null, + pyz_python_version: null, + pyz_entry_count: null, + encryption_source: null, +}); + +export const cArchiveEntryMetadata = ( + entry: CArchiveEntry, + header?: Buffer, + archive?: PyzArchive, + encryptionMarker = false, +): PyInstallerEntryMetadata => ({ + ...entryMetadata( + { + container: "carchive", + container_name: null, + index: entry.index, + source_name: entry.name, + type_code: entry.typeCode, + relative_offset: entry.offset, + compression_flag: entry.compressionFlag === 1 ? 1 : 0, + }, + ["s", "m", "M"].includes(entry.typeCode), + header, + ), + ...(entry.typeCode === "d" ? dependencyMetadata(entry.name) : {}), + pyz_status: + archive?.kind === "entries" + ? "decoded" + : archive === undefined + ? "not-pyz" + : "opaque", + directory_record_offset: entry.tocRecordOffset, + directory_record_size: entry.tocRecordLength, + pyz_sha256: archive?.sha256 ?? null, + pyz_python_magic: archive?.pythonMagic.toString("hex") ?? null, + pyz_python_version: + archive === undefined + ? null + : (pythonVersionForMagic(archive.pythonMagic) ?? null), + pyz_entry_count: archive?.kind === "entries" ? archive.entries.length : null, + encryption_source: + archive !== undefined && encryptionMarker ? "carchive-marker" : null, +}); + +/** The bootloader splits at the first colon; neither part is a local read grant. */ +const dependencyMetadata = ( + source: string, +): Pick< + PyInstallerEntryMetadata, + "dependency_status" | "dependency_archive" | "dependency_member" +> => { + const colon = source.indexOf(":"); + const valid = colon > 0 && colon < source.length - 1; + return { + dependency_status: valid ? "unresolved" : "invalid-reference", + dependency_archive: valid ? source.slice(0, colon) : null, + dependency_member: valid ? source.slice(colon + 1) : null, + }; +}; + +/** Legacy writers encrypt nonempty namespace payloads as well as module payloads. */ +export const pyzEntryEncrypted = ( + entry: PyzEntry, + encryptionMarker: boolean, +): boolean => encryptionMarker && entry.length > 0; + +export const pyzEntryMetadata = ( + container: CArchiveEntry, + entry: PyzEntry, + header: Buffer | undefined, + encryptionMarker: boolean, +): PyInstallerEntryMetadata => ({ + ...entryMetadata( + { + container: "pyz", + container_name: container.name, + index: entry.index, + source_name: entry.name, + type_code: entry.typeCode, + relative_offset: entry.offset, + compression_flag: entry.compressionFlag, + }, + entry.typeCode === 0 || entry.typeCode === 1, + header, + ), + encryption_source: pyzEntryEncrypted(entry, encryptionMarker) + ? "carchive-marker" + : null, +}); diff --git a/src/artifacts/pyinstaller/PyInstallerPyzStore.ts b/src/artifacts/pyinstaller/PyInstallerPyzStore.ts new file mode 100644 index 0000000000..04d17b17c1 --- /dev/null +++ b/src/artifacts/pyinstaller/PyInstallerPyzStore.ts @@ -0,0 +1,255 @@ +import type { FileHandle } from "node:fs/promises"; +import { lstat, mkdtemp, open, rm } from "node:fs/promises"; +import type { Stats } from "node:fs"; +import { Readable } from "node:stream"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js"; +import { + openRegularFile, + RegularFileAdmissionFailure, + sameRegularFileState, +} from "../../filesystem/RegularFile.js"; +import { ArtifactReaderFailure } from "../ArtifactReader.js"; +import { + ArtifactResourceScope, + type ArtifactResourceOwner, +} from "../ArtifactResourceScope.js"; +import { abortIfNeeded } from "../ArtifactHash.js"; +import { readArtifactFileChunks } from "../readArtifactFileChunks.js"; +import { pyInstallerMemberStream } from "./PyInstallerMemberStream.js"; +import type { CArchiveEntry } from "./CArchiveLayout.js"; +import type { PyInstallerMember } from "./CArchiveMembers.js"; +import { pyInstallerFileAdmissionFailure } from "./PyInstallerFileAdmission.js"; + +// Each outer-compressed image retains one inode and its identity until close. +// This is a per-reader filesystem capacity budget, independent of byte budgets. +const MAX_PRIVATE_PYZ_IMAGES = 128; + +interface StoredImage { + readonly path: string; + readonly initial: Stats; +} +export interface PyInstallerMemberSource { + readonly source: Readable; + readonly verify?: () => Promise; + readonly close?: (primary?: unknown) => Promise; +} + +/** The reader's operation scope drains work before retiring these private files. */ +export class PyInstallerPyzStore { + #root: string | undefined; + #serial = 0; + readonly #owners = new Map< + FileHandle, + Extract + >(); + readonly #cleanup = new ArtifactResourceScope(); + readonly #images = new Map(); + + reserveImages(count: number): void { + if (count > MAX_PRIVATE_PYZ_IMAGES) + throw new ArtifactReaderFailure( + "limit", + "PyInstaller compressed PYZ images exceed the 128-file private storage budget", + { boundary: "capacity" }, + ); + } + + async spool( + handle: FileHandle, + archiveStart: number, + entry: CArchiveEntry, + signal?: AbortSignal, + ): Promise<{ handle: FileHandle; start: number }> { + abortIfNeeded(signal); + if (entry.compressedSize === 0) + throw new ArtifactReaderFailure( + "format", + "Compressed PYZ image has no zlib bytes", + ); + this.#root ??= await mkdtemp(join(tmpdir(), "rea-pyz-")); + const path = join(this.#root, String(this.#serial++)); + const writer = await open(path, "wx", 0o600); + this.#retain(path, new OwnedFileHandle(writer)); + abortIfNeeded(signal); + const source = readArtifactFileChunks(handle, { + start: archiveStart + entry.offset, + end: archiveStart + entry.offset + entry.compressedSize - 1, + signal, + }); + const stream = pyInstallerMemberStream(source, { + sourceBytes: entry.compressedSize, + compressed: true, + header: undefined, + expectedBytes: entry.uncompressedSize, + maxBytes: entry.uncompressedSize, + signal: signal ?? new AbortController().signal, + }); + const copied = this.#writeImage(writer, stream.output, signal); + const outcomes = await Promise.allSettled([stream.completion, copied]); + // A failed native write can retire the upstream stream. Preserve that I/O + // failure instead of replacing it with a secondary premature-close error. + const copy = outcomes[1]; + if (copy?.status === "rejected") throw copy.reason; + const decoded = outcomes[0]; + if (decoded?.status === "rejected") throw decoded.reason; + await this.release(writer); + abortIfNeeded(signal); + const opened = await this.#admit(path, signal); + this.#images.set(opened, { path, initial: await opened.stat() }); + return { handle: opened, start: 0 }; + } + + async release(handle: FileHandle): Promise { + const owner = this.#owners.get(handle); + if (owner === undefined) return; + const outcome = await this.#cleanup.release(owner); + if (outcome.kind === "failed") throw outcome.cause; + this.#owners.delete(handle); + } + + async memberSource( + read: NonNullable, + root: FileHandle | undefined, + signal: AbortSignal, + ): Promise { + let handle = read.kind === "carchive" ? root : read.handle; + if (handle === undefined) + throw new ArtifactReaderFailure( + "unavailable", + "PyInstaller archive is closed", + ); + const image = read.kind === "pyz" ? this.#images.get(handle) : undefined; + let verify: (() => Promise) | undefined; + let close: ((primary?: unknown) => Promise) | undefined; + if (image !== undefined) { + handle = await this.#admit(image.path, signal); + const opened = handle; + verify = () => this.#verifyImage(opened, image); + close = async (primary) => { + try { + await this.release(opened); + } catch (cause: unknown) { + throw ArtifactReaderFailure.withCleanup( + primary ?? cause, + ArtifactReaderFailure.cleanupObservation(cause, image.path), + ); + } + }; + await verify(); + } + abortIfNeeded(signal); + const start = + read.kind === "carchive" + ? read.archiveStart + read.source.offset + : read.offset; + const length = + read.kind === "carchive" ? read.source.compressedSize : read.length; + const source = + length === 0 + ? Readable.from([]) + : readArtifactFileChunks(handle, { + start, + end: start + length - 1, + signal, + }); + source.on("error", () => undefined); + return { + source, + ...(verify === undefined ? {} : { verify }), + ...(close === undefined ? {} : { close }), + }; + } + + async close(): Promise { + await this.#cleanup.close(); + this.#owners.clear(); + if (this.#root !== undefined) { + try { + await rm(this.#root, { recursive: true, force: true }); + } catch (cause: unknown) { + throw ArtifactReaderFailure.withCleanup( + cause, + ArtifactReaderFailure.cleanupObservation(cause, this.#root), + ); + } + this.#root = undefined; + } + this.#images.clear(); + } + + async #admit(path: string, signal?: AbortSignal): Promise { + try { + const handle = await openRegularFile(path, { + symlinks: "reject", + signal, + }); + this.#retain(path, new OwnedFileHandle(handle)); + return handle; + } catch (cause: unknown) { + if (cause instanceof RegularFileAdmissionFailure) + this.#retain(path, cause.owner); + throw pyInstallerFileAdmissionFailure(path, cause); + } + } + + #retain(path: string, handle: OwnedFileHandle): void { + const owner = { kind: "file-handle" as const, resource: path, handle }; + this.#owners.set(handle.handle, owner); + this.#cleanup.retain(owner); + } + + async #writeImage( + handle: FileHandle, + source: Readable, + signal?: AbortSignal, + ): Promise { + let position = 0; + for await (const chunk of source as AsyncIterable) { + // Stop writing after cancellation; drain only already-produced bytes so + // the descriptor source can report its in-flight I/O outcome truthfully. + if (signal?.aborted) continue; + let offset = 0; + try { + while (offset < chunk.length) { + const { bytesWritten } = await handle.write( + chunk, + offset, + chunk.length - offset, + position, + ); + if (bytesWritten === 0) + throw new ArtifactReaderFailure( + "io", + "Private PYZ write made no progress", + ); + offset += bytesWritten; + position += bytesWritten; + if (signal?.aborted) break; + } + } catch (cause: unknown) { + const failure = + cause instanceof ArtifactReaderFailure + ? cause + : new ArtifactReaderFailure( + "io", + "Private PYZ image could not be written", + { cause }, + ); + source.destroy(failure); + throw failure; + } + } + abortIfNeeded(signal); + } + + async #verifyImage(handle: FileHandle, image: StoredImage): Promise { + const states = await Promise.all([handle.stat(), lstat(image.path)]); + if (states.some((state) => !sameRegularFileState(image.initial, state))) + throw new ArtifactReaderFailure( + "integrity", + "Private PYZ image changed after parsing", + ); + } +} diff --git a/src/artifacts/pyinstaller/PythonBytecode.ts b/src/artifacts/pyinstaller/PythonBytecode.ts new file mode 100644 index 0000000000..cefcc35174 --- /dev/null +++ b/src/artifacts/pyinstaller/PythonBytecode.ts @@ -0,0 +1,33 @@ +/** + * Final CPython bytecode magics for the minor versions whose 16-byte `.pyc` + * header this reader can reconstruct. Other magics leave the PYZ opaque. + * Each value is the four-byte little-endian magic, including the trailing CR LF. + */ +const PYTHON_BYTECODE_MAGICS: Readonly> = { + "420d0d0a": "3.7", + "550d0d0a": "3.8", + "610d0d0a": "3.9", + "6f0d0d0a": "3.10", + a70d0d0a: "3.11", + cb0d0d0a: "3.12", + f30d0d0a: "3.13", + "2b0e0d0a": "3.14", +}; + +/** Limitation recorded when a member's digest covers a derived `.pyc` header. */ +export const RECONSTRUCTED_PYTHON_HEADER_LIMITATION = + "Python bytecode header was reconstructed from the archive Python magic; this digest covers the derived file."; + +/** Dotted Python version for a supported bytecode magic, when one is known. */ +export const pythonVersionForMagic = (magic: Buffer): string | undefined => + PYTHON_BYTECODE_MAGICS[magic.toString("hex")]; + +/** + * 16-byte `.pyc` header used since Python 3.7: magic, zero flags, zero + * timestamp, and zero source size. The flags word selects the timestamp form. + */ +export const reconstructedPycHeader = (magic: Buffer): Buffer => { + const header = Buffer.alloc(16); + magic.copy(header, 0, 0, 4); + return header; +}; diff --git a/src/artifacts/pyinstaller/PyzArchive.test.ts b/src/artifacts/pyinstaller/PyzArchive.test.ts new file mode 100644 index 0000000000..6ea1fedf16 --- /dev/null +++ b/src/artifacts/pyinstaller/PyzArchive.test.ts @@ -0,0 +1,137 @@ +import { open, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; + +import { readPyzArchive, MAX_PYZ_TABLE_BYTES } from "./PyzArchive.js"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import { PYTHON_39_MAGIC, pyzArchive } from "./CArchive.fixture.js"; + +it("lists modules from a supported PYZ and leaves an unknown magic opaque", async () => { + const body = Buffer.from("module"); + const parsed = await readImage( + pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "json.decoder", typeCode: 0, body }], + }), + ); + expect(parsed.ok).toBe(true); + if (!parsed.ok || parsed.value.kind !== "entries") return; + expect(parsed.value.pythonVersion).toBe("3.9"); + expect(parsed.value.entries).toMatchObject([ + { name: "json.decoder", typeCode: 0, package: false }, + ]); + + const opaque = await readImage( + pyzArchive({ + magic: Buffer.alloc(4), + modules: [{ name: "json.decoder", typeCode: 0, body }], + }), + ); + expect(opaque.ok).toBe(true); + if (!opaque.ok || opaque.value.kind !== "opaque") return; + expect(opaque.value.pythonMagic).toEqual(Buffer.alloc(4)); + expect(opaque.value.limitation).toContain("left opaque"); +}); + +it("leaves a PYZ opaque when the table uses an unsupported marshal type", async () => { + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [], + toc: Buffer.from([0x4e]), + }); + const parsed = await readImage(image); + expect(parsed.ok).toBe(true); + if (!parsed.ok || parsed.value.kind !== "opaque") return; + expect(parsed.value.limitation).toContain("left opaque"); + expect(parsed.value.pythonMagic).toEqual(PYTHON_39_MAGIC); +}); + +it("reports an entry that extends outside the PYZ as an integrity error", async () => { + const image = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "missing", typeCode: 0, body: Buffer.from("x") }], + }); + image.writeInt32LE(100_000, image.length - 4); + const parsed = await readImage(image); + expect(parsed.ok).toBe(false); + if (parsed.ok) return; + expect(parsed.error.reason).toBe("integrity"); +}); + +it("rejects an oversized sparse PYZ table before allocating its bytes", async () => { + const root = await createTestTempDirectory("rea-pyz-table-budget-"); + const path = join(root, "image.pyz"); + const header = Buffer.alloc(12); + header.write("PYZ\0"); + PYTHON_39_MAGIC.copy(header, 4); + header.writeUInt32BE(12, 8); + await writeFile(path, header); + const handle = await open(path, "r+"); + try { + const size = 12 + MAX_PYZ_TABLE_BYTES + 1; + await handle.truncate(size); + expect(await readPyzArchive(handle, 0, size)).toMatchObject({ + ok: false, + error: { reason: "limit" }, + }); + } finally { + await handle.close(); + } +}); + +it("rejects a PYZ table whose back-references repeat one long row", async () => { + // One namespace row with a 16 KiB name, flagged for reference, then 999 + // back-references to it. The table is about 21 KiB but would expand to + // 16 MiB of entry names. + const name = Buffer.alloc(16 * 1024, 0x61); + const refs = 1_000; + const row = Buffer.concat([ + Buffer.from([0x28 | 0x80]), + int32(2), + Buffer.from([0x61]), + int32(name.length), + name, + Buffer.from([0x29, 3, 0x69]), + int32(3), + Buffer.from([0x69]), + int32(0), + Buffer.from([0x69]), + int32(0), + ]); + const reference = Buffer.concat([Buffer.from([0x72]), int32(0)]); + const table = Buffer.concat([ + Buffer.from([0x5b]), + int32(refs), + row, + ...Array.from({ length: refs - 1 }, () => reference), + ]); + const header = Buffer.alloc(12); + header.write("PYZ\0"); + PYTHON_39_MAGIC.copy(header, 4); + header.writeUInt32BE(12, 8); + const image = Buffer.concat([header, table]); + expect(image.length).toBeLessThan(32 * 1024); + + const parsed = await readImage(image); + expect(parsed).toMatchObject({ ok: false, error: { reason: "limit" } }); + if (parsed.ok) return; + expect(parsed.error.message).toContain("expand past"); +}); + +const int32 = (value: number): Buffer => { + const bytes = Buffer.alloc(4); + bytes.writeInt32LE(value); + return bytes; +}; + +const readImage = async (bytes: Buffer) => { + const root = await createTestTempDirectory("rea-pyz-file-"); + const path = join(root, "image.pyz"); + await writeFile(path, bytes); + const handle = await open(path, "r"); + try { + return await readPyzArchive(handle, 0, bytes.length); + } finally { + await handle.close(); + } +}; diff --git a/src/artifacts/pyinstaller/PyzArchive.ts b/src/artifacts/pyinstaller/PyzArchive.ts new file mode 100644 index 0000000000..c398cfae0f --- /dev/null +++ b/src/artifacts/pyinstaller/PyzArchive.ts @@ -0,0 +1,266 @@ +import type { FileHandle } from "node:fs/promises"; +import { hashReadable } from "../ArtifactHash.js"; +import { readArtifactFileChunks } from "../readArtifactFileChunks.js"; +import { readPyInstallerBytes } from "./PyInstallerBytes.js"; + +import { err, ok, type Result } from "../../domain/result.js"; +import { readMarshalSubset, type MarshalValue } from "./MarshalSubset.js"; +import { + pyInstallerFailure, + type PyInstallerFailure, +} from "./PyInstallerFailure.js"; +import { pythonVersionForMagic } from "./PythonBytecode.js"; + +const PYZ_MAGIC = Buffer.from("PYZ\0"); +const PYZ_HEADER_BYTES = 12; + +/** One PYZ table entry. Type codes follow the archive's module, package, data, and namespace values. */ +export interface PyzEntry { + readonly index: number; + readonly name: string; + readonly typeCode: number; + readonly offset: number; + readonly length: number; + /** PYZ compresses every nonempty payload, independently of its logical module type. */ + readonly compressionFlag: 0 | 1; + readonly package: boolean; + readonly namespace: boolean; +} + +export type PyzArchive = + | { + readonly kind: "entries"; + readonly pythonMagic: Buffer; + readonly pythonVersion: string; + readonly sha256: string; + readonly entries: readonly PyzEntry[]; + } + | { + readonly kind: "opaque"; + readonly pythonMagic: Buffer; + readonly sha256: string; + readonly limitation: string; + }; + +/** Bound the transient marshal table allocation independently of image expansion. */ +export const MAX_PYZ_TABLE_BYTES = 16 * 1024 * 1024; + +/** Parse only the header and table in memory; hash the complete file range in chunks. */ +export const readPyzArchive = async ( + handle: FileHandle, + start: number, + size: number, + signal?: AbortSignal, +): Promise> => { + if (size < PYZ_HEADER_BYTES) + return err( + pyInstallerFailure("format", "PYZ archive magic does not match"), + ); + const headerRead = await readPyInstallerBytes(handle, { + position: start, + length: PYZ_HEADER_BYTES, + label: "PYZ header", + signal, + }); + if (!headerRead.ok) return headerRead; + const header = headerRead.value; + const tocOffset = + header.length === PYZ_HEADER_BYTES ? header.readUInt32BE(8) : size; + const validOffset = tocOffset >= PYZ_HEADER_BYTES && tocOffset <= size; + const knownMagic = pythonVersionForMagic(header.subarray(4, 8)) !== undefined; + const tableSize = validOffset && knownMagic ? size - tocOffset : 0; + if (tableSize > MAX_PYZ_TABLE_BYTES) + return err( + pyInstallerFailure( + "limit", + "PYZ table exceeds the 16 MiB in-memory table budget", + ), + ); + const read = await readPyInstallerBytes(handle, { + position: start + tocOffset, + length: tableSize, + label: "PYZ table", + signal, + }); + if (!read.ok) return read; + const table = read.value; + const digest = await hashReadable( + readArtifactFileChunks(handle, { + start, + end: start + size - 1, + ...(signal === undefined ? {} : { signal }), + }), + signal, + ); + if (digest.bytes !== size) + return err( + pyInstallerFailure("format", "PYZ image extends outside the file"), + ); + return parsePyzParts(header, table, size, digest.sha256); +}; + +const parsePyzParts = ( + header: Buffer, + table: Buffer, + size: number, + sha256: string, +): Result => { + if ( + header.length < PYZ_HEADER_BYTES || + !header.subarray(0, 4).equals(PYZ_MAGIC) + ) + return err( + pyInstallerFailure("format", "PYZ archive magic does not match"), + ); + const pythonMagic = Buffer.from(header.subarray(4, 8)); + const pythonVersion = pythonVersionForMagic(pythonMagic); + if (pythonVersion === undefined) + return ok({ + kind: "opaque", + pythonMagic, + sha256, + limitation: `PYZ Python magic ${pythonMagic.toString("hex")} is not in the supported set; the member was left opaque.`, + }); + const tocOffset = header.readUInt32BE(8); + if (tocOffset < PYZ_HEADER_BYTES || tocOffset > size) + return err( + pyInstallerFailure( + "format", + "PYZ table of contents is outside the archive", + ), + ); + const marshalled = readMarshalSubset(table); + if (!marshalled.ok) { + if (marshalled.error.reason !== "format") return marshalled; + return ok({ + kind: "opaque", + pythonMagic, + sha256, + limitation: `PYZ table of contents could not be parsed (${marshalled.error.message}); the member was left opaque.`, + }); + } + const entries = pyzEntries(marshalled.value, tocOffset, table.length); + if (!entries.ok) + return entries.error.reason === "format" + ? ok({ + kind: "opaque", + pythonMagic, + sha256, + limitation: `${entries.error.message}; the member was left opaque.`, + }) + : entries; + return ok({ + kind: "entries", + pythonMagic, + pythonVersion, + sha256, + entries: entries.value, + }); +}; + +/** + * Marshal back-references let one stored row appear many times, so a small + * table can name far more bytes than it holds. Each genuine entry stores its + * own name inline, so the expanded names can never exceed the table itself. + */ +const pyzEntries = ( + value: MarshalValue, + payloadEnd: number, + tableBytes: number, +): Result => { + if (!Array.isArray(value)) + return err( + pyInstallerFailure("format", "PYZ table of contents is not a list"), + ); + const entries: PyzEntry[] = []; + let nameBytes = 0; + for (const item of value) { + const entry = onePyzEntry(item, payloadEnd, entries.length); + if (!entry.ok) return entry; + nameBytes += Buffer.byteLength(entry.value.name, "utf8"); + if (nameBytes > tableBytes) + return err( + pyInstallerFailure( + "limit", + `PYZ table entry names expand past the ${String(tableBytes)}-byte table that stores them`, + ), + ); + entries.push(entry.value); + } + return ok(entries); +}; + +const onePyzEntry = ( + item: MarshalValue, + payloadEnd: number, + index: number, +): Result => { + if (!Array.isArray(item) || item.length !== 2) + return err( + pyInstallerFailure( + "format", + "PYZ table entry is not a name and position pair", + ), + ); + const [name, position] = item; + if ( + typeof name !== "string" || + !Array.isArray(position) || + position.length !== 3 + ) + return err( + pyInstallerFailure( + "format", + "PYZ table entry is not a name, type, offset, and length", + ), + ); + const [typeCode, offset, length] = position; + if ( + !Number.isSafeInteger(typeCode) || + !Number.isSafeInteger(offset) || + !Number.isSafeInteger(length) + ) + return err( + pyInstallerFailure( + "format", + "PYZ table entry positions are not integers", + ), + ); + if (typeCode < 0 || typeCode > 3) + return err( + pyInstallerFailure( + "format", + `PYZ table entry type ${String(typeCode)} is not a known module, package, data, or namespace code`, + ), + ); + if (typeCode !== 3 && length === 0) + return err( + pyInstallerFailure( + "format", + `PYZ entry at table index ${String(index)} has no compressed bytes`, + ), + ); + if ( + length < 0 || + offset < 0 || + offset > payloadEnd || + length > payloadEnd - offset || + (length > 0 && offset < PYZ_HEADER_BYTES) + ) + return err( + pyInstallerFailure( + "integrity", + `PYZ entry at table index ${String(index)} extends outside the archive payload region`, + ), + ); + return ok({ + index, + name, + typeCode, + offset, + length, + compressionFlag: length === 0 ? 0 : 1, + package: typeCode === 1 || typeCode === 3, + namespace: typeCode === 3, + }); +}; diff --git a/src/browser/CdpObservationSession.ts b/src/browser/CdpObservationSession.ts index 51adfe5612..ccb2d6b1ad 100644 --- a/src/browser/CdpObservationSession.ts +++ b/src/browser/CdpObservationSession.ts @@ -69,7 +69,7 @@ export const observeCdpSession = async ( const capture = new TimelineCapture(allowedOrigins, mainFrameId, initialUrl); let removeListener = (): void => {}; const removeDisconnectListener = context.connection.onDisconnect(() => - capture.targetTerminated(), + capture.transportDisconnected(), ); try { await context.connection.send( @@ -236,9 +236,11 @@ class TimelineCapture { this.#lifecycle(params); break; case "Inspector.targetCrashed": - case "Target.detachedFromTarget": this.targetTerminated(params); break; + case "Target.detachedFromTarget": + this.sessionDetached(params); + break; } } @@ -306,6 +308,32 @@ class TimelineCapture { this.end("target_terminated"); } + sessionDetached(params: UnknownRecord = {}): void { + if (this.#endReason !== undefined) return; + this.#add({ + type: "session_detached", + params, + timestamp: null, + rawUrl: null, + requestId: null, + detail: "session_detached", + }); + this.end("session_detached"); + } + + transportDisconnected(): void { + if (this.#endReason !== undefined) return; + this.#add({ + type: "transport_disconnected", + params: {}, + timestamp: null, + rawUrl: null, + requestId: null, + detail: "transport_disconnected", + }); + this.end("transport_disconnected"); + } + setFinalUrl(rawUrl: string): void { if (this.#scopeExcluded) return; const destination = scopedUrl(rawUrl, this.allowedOrigins); diff --git a/src/browser/execution/CdpExecutionCapture.ts b/src/browser/execution/CdpExecutionCapture.ts index 14c5bb6414..1448ac2296 100644 --- a/src/browser/execution/CdpExecutionCapture.ts +++ b/src/browser/execution/CdpExecutionCapture.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { BrowserObservationError } from "../../domain/browserObservationError.js"; import { AnalysisCancelledError, @@ -37,9 +38,18 @@ export const captureWebExecution = async ( sources.check(); window.check(); if (window.reason !== undefined) - throw new BrowserObservationError(session.operation, "target_changed", { - detail: `Selected target ${session.target.target_id} ended with ${window.reason} before execution observation could be armed.`, - }); + throw new BrowserObservationError( + session.operation, + window.reason === "session_detached" + ? "target_session_detached" + : "target_changed", + { + detail: executionWindowEndDetail( + session.target.target_id, + window.reason, + ), + }, + ); const waiting = window.start(input.observation_ms); // Cancellation may arrive while asynchronous progress is being delivered. void waiting.catch(() => undefined); @@ -64,7 +74,7 @@ export const captureWebExecution = async ( raw = await session.command("Profiler.takePreciseCoverage"); } collection.freezeMetadata(); - await session.command("Profiler.stopPreciseCoverage"); + await stopPreciseCoverage(session, reason); session.preciseCoverageMayBeActive = false; const requestItems = requests.result(); const parsed = @@ -161,3 +171,36 @@ export const captureWebExecution = async ( collection.close(); } }; + +const stopPreciseCoverage = async ( + session: CdpRuntimeSources["session"], + reason: WebExecution["window"]["end_reason"], +): Promise => { + try { + await session.command("Profiler.stopPreciseCoverage"); + } catch (cause: unknown) { + if ( + (reason !== "target_terminated" && reason !== "session_detached") || + !(cause instanceof AnalysisError) + ) + throw cause; + cause.retainDiagnostics({ + observed_window: { + target_id: session.target.target_id, + end_reason: reason, + }, + }); + throw cause; + } +}; + +const executionWindowEndDetail = ( + targetId: string, + reason: WebExecution["window"]["end_reason"], +): string => { + if (reason === "session_detached") + return `The debugging session for selected target ${targetId} detached before execution observation could be armed; the target may still exist.`; + if (reason === "target_terminated") + return `Selected target ${targetId} crashed before execution observation could be armed.`; + return `Selected target ${targetId} changed documents before execution observation could be armed.`; +}; diff --git a/src/browser/execution/CdpExecutionCollection.ts b/src/browser/execution/CdpExecutionCollection.ts index f96c296047..df37bf48c2 100644 --- a/src/browser/execution/CdpExecutionCollection.ts +++ b/src/browser/execution/CdpExecutionCollection.ts @@ -40,8 +40,8 @@ export class CdpExecutionCollection { } this.window.ingest(event); }), - session.transport.connection.onDisconnect(() => - this.window.end("target_terminated"), + session.transport.connection.onDisconnect((error) => + this.window.fail(error), ), session.transport.connection.onProtocolFailure((error) => this.window.fail(error), diff --git a/src/browser/execution/CdpExecutionWindow.ts b/src/browser/execution/CdpExecutionWindow.ts index fddfe10329..ccbb1f9626 100644 --- a/src/browser/execution/CdpExecutionWindow.ts +++ b/src/browser/execution/CdpExecutionWindow.ts @@ -25,10 +25,11 @@ export class CdpExecutionWindow { /** Collection ends on a main-document replacement, never follows source URLs into another context. */ ingest(event: CdpEvent): void { - if ( - event.method === "Target.detachedFromTarget" || - event.method === "Inspector.targetCrashed" - ) { + if (event.method === "Target.detachedFromTarget") { + this.end("session_detached"); + return; + } + if (event.method === "Inspector.targetCrashed") { this.end("target_terminated"); return; } @@ -101,7 +102,7 @@ export class CdpExecutionWindow { }); } - /** Transport loss ends the window; instrumentation cleanup still requires confirmation. */ + /** A producer lifecycle fact ends the window; instrumentation cleanup still requires confirmation. */ end(reason: EndReason): void { if (this.reason !== undefined || this.#failure !== undefined) return; this.reason = reason; diff --git a/src/cli/rustCommands.ts b/src/cli/rustCommands.ts index 119892afb7..e71546d5cf 100644 --- a/src/cli/rustCommands.ts +++ b/src/cli/rustCommands.ts @@ -2,7 +2,14 @@ import { z } from "incur"; import { createRustBinaryService } from "../composition/rust.js"; import { CLI_COMMANDS } from "../cliCommandNames.js"; import { logCliCommand } from "../cliLogging.js"; -import { finishAnalysisCommand } from "./finishAnalysisCommand.js"; +import { + analysisEvidenceDeliveryOptions, + deliverCliAnalysisEvidence, +} from "./analysisEvidenceDelivery.js"; +import { inspectRustBinaryRequestSchema } from "../domain/rust/rustBinaryQuery.js"; +import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; +import { AnalysisInputError } from "../domain/analysisErrorCore.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import type { Logger } from "pino"; import type { CliInstance } from "./types.js"; import { withCommandCancellation } from "./commandCancellation.js"; @@ -19,22 +26,68 @@ export const registerRustCommands = ( args: z.object({ path: z.string().describe("Local ELF, PE or thin Mach-O image"), }), - run: ({ args }) => - withCommandCancellation((signal) => { - const service = createRustBinaryService(); - return logCliCommand(logger, CLI_COMMANDS.inspectRustBinary, () => - finishAnalysisCommand( - () => - service.inspect( - { path: absoluteCallerPath(args.path) }, - { signal }, + options: z.object({ + ...analysisEvidenceDeliveryOptions, + detail: z + .enum(["complete", "summary"]) + .optional() + .describe( + "Default complete; a view selects compact delivery with --evidence_output", + ), + view: z + .string() + .optional() + .describe( + "JSON package or edge selection; retains complete Evidence with --evidence_output", + ), + }), + run: ({ args, options }) => + withCommandCancellation((signal) => + logCliCommand(logger, CLI_COMMANDS.inspectRustBinary, async () => { + let view: unknown; + try { + view = + options.view === undefined ? undefined : JSON.parse(options.view); + } catch (cause) { + return projectAnalysisError( + new AnalysisInputError("inspect_rust_binary", { cause }, [ + { + path: ["view"], + reason: "invalid_format", + message: "view must be a JSON package or edge selection", + }, + ]), + ); + } + const input = inspectRustBinaryRequestSchema.safeParse({ + path: absoluteCallerPath(args.path), + ...(options.detail === undefined ? {} : { detail: options.detail }), + ...(view === undefined ? {} : { view }), + }); + if (!input.success) + return projectAnalysisError( + analysisInputErrorFromIssues( + "inspect_rust_binary", + input.error.issues, + { view }, ), + ); + const service = createRustBinaryService(); + return deliverCliAnalysisEvidence({ + run: () => service.inspect({ path: input.data.path }, { signal }), service, - "rust-auditable", - "inspect_rust_binary", + providerId: "rust-auditable", + operation: "inspect_rust_binary", signal, - ), - ); - }), + detail: + input.data.detail ?? + (input.data.view === undefined ? "complete" : "summary"), + evidenceOutput: options.evidence_output, + ...(input.data.view === undefined + ? {} + : { view: { kind: "rust" as const, ...input.data.view } }), + }); + }), + ), }); }; diff --git a/src/cli/sqliteCommands.ts b/src/cli/sqliteCommands.ts index 355559e422..f1b8e464f0 100644 --- a/src/cli/sqliteCommands.ts +++ b/src/cli/sqliteCommands.ts @@ -7,6 +7,7 @@ import type { CliInstance } from "./types.js"; import { withCommandCancellation } from "./commandCancellation.js"; import { absoluteCallerPath } from "../filesystem/callerPath.js"; import { finishAnalysisCommand } from "./finishAnalysisCommand.js"; +import { inspectSqliteDatabaseInputSchema } from "../domain/sqlite/sqliteDatabase.js"; /** Resolve an operator-selected SQLite snapshot through the shared inspection workflow. */ export const registerSqliteCommands = ( @@ -22,6 +23,7 @@ export const registerSqliteCommands = ( path: z.string().describe("Local SQLite database snapshot"), }), options: z.object({ + workerTimeoutMs: inspectSqliteDatabaseInputSchema.shape.worker_timeout_ms, table: z .string() .optional() @@ -55,6 +57,9 @@ export const registerSqliteCommands = ( service.inspect( { path: absoluteCallerPath(args.path), + ...(options.workerTimeoutMs === undefined + ? {} + : { worker_timeout_ms: options.workerTimeoutMs }), ...(options.table === undefined ? {} : { table: options.table }), diff --git a/src/cli/wasmCommands.ts b/src/cli/wasmCommands.ts index 1a3217c6ac..cee9d5d571 100644 --- a/src/cli/wasmCommands.ts +++ b/src/cli/wasmCommands.ts @@ -10,6 +10,7 @@ import type { CliInstance } from "./types.js"; import { withCommandCancellation } from "./commandCancellation.js"; import { absoluteCallerPath } from "../filesystem/callerPath.js"; import { finishAnalysisCommand } from "./finishAnalysisCommand.js"; +import { inspectWasmArtifactInputSchema } from "../domain/wasm/wasmArtifact.js"; /** Resolve all caller-selected artifact paths through the shared workflow. */ export const registerWasmCommands = ( @@ -47,6 +48,7 @@ export const registerWasmCommands = ( path: z.string().describe("Local WASM artifact"), }), options: z.object({ + commandTimeoutMs: inspectWasmArtifactInputSchema.shape.command_timeout_ms, glue_paths: z .array(z.string()) .optional() @@ -64,6 +66,9 @@ export const registerWasmCommands = ( service.inspect( { path: absoluteCallerPath(args.path), + ...(options.commandTimeoutMs === undefined + ? {} + : { command_timeout_ms: options.commandTimeoutMs }), glue_paths: options.glue_paths?.map((path) => absoluteCallerPath(path), ), diff --git a/src/contracts/analysisViewToolContracts.ts b/src/contracts/analysisViewToolContracts.ts index 3adc35f12c..9fa79e1250 100644 --- a/src/contracts/analysisViewToolContracts.ts +++ b/src/contracts/analysisViewToolContracts.ts @@ -17,7 +17,7 @@ export const ANALYSIS_VIEW_TOOL_CONTRACTS = [ ...toolContractMetadata("inspect_analysis_view"), kind: "application", description: - 'Inspect selected views of completed Frida session status, GDB/objdump command observations, Go build metadata, binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_macho, decode_interface_builder, inspect_evm_interface, inspect_managed_artifact, inspect_managed_members, inspect_android_package, inspect_dart_aot, inspect_il2cpp_metadata, inspect_model_artifact or inspect_web_network_capture Evidence without repeating analysis. Command summary views preserve ordered GDB records or opaque objdump stdout/stderr once, source parameters, completion facts and parent identity. They do not establish current target state. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. Go metadata uses kind go, collection dependencies/settings, exact paths/keys or indices, fields and pagination. Row index refers to the original collection; empty selections return no rows, and dependency coverage remains unknown. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. The environment-manifest view projects the same HAR requests on the primary origin (most JSON-like records) into a replayable tool surface: view {kind: "environment-manifest", template_rules: ["after-uid-literal", "opaque-token", "sibling-variance"], include_query_parameter_names: true}. Each tool has a deterministic name, a path template and parameters from the four format rules plus the selected opt-in rules, observed query parameter names, a value-free body shape for POST/PUT/PATCH, status and content-type sets and record ordinals; records are pointers into the parent, excluded groups carry a reason and three fixed unknowns are always stated. Templates, parameters and names are inferences; the view copies no header, cookie or body values and refuses parents inspected with sensitive values. A capture_browser_scenario parent that selected response bodies or header values projects the same way, one record per network transaction in request order; it relies on capture-time redaction of declared secrets instead of that refusal, and a transaction whose URL was redacted is left unrepresented. The environment-state view extracts, from the same parents, the initial state the recorded reads observed: view {kind: "environment-state", template_rules: [...]} with the rules of the manifest it accompanies. It reads response bodies recorded before the first related write and reports entity types with an identity field (proved by a path parameter or inferred from a field name), entities in collection order with per-view values where views differ, documents for non-parameterized reads that are not entities and have one initial request line (item-shaped paths stay recorded), and a render plan per read tool: item and list views reproduce their latest recordings from the state, and every other read tool is a recorded view with a reason. It also reports a write plan per state-changing tool (create, update, delete or document, built from its first successful call) and write_clock; a plan is served only when replaying the capture shows a later read confirming its effect and none contradicting it. Types and identity are inferences; values are recorded data, so it refuses parents inspected with sensitive values, and a scenario parent must have selected response bodies; its values keep the capture redaction markers, and a write whose URL it redacted ends the state of every type. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. A completed Evidence JSON file on this host uses {kind: "evidence-file", path: ""}: REA reads and authenticates it, then projects it like retained Evidence, so a large capture is bounded by the file reader rather than by inline-input accounting. For layout, JavaScript, inventory, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Typed module references use view {kind: "javascript", collection: "modules", projection: "references", where: {path: "main.js", kind: "javascript-module"}, offset: 0, limit: 64}; where matches exact retained strings (null selects unavailable paths). Rows include canonical identity, retained source-range count and observation support states. Source coverage and selection remain explicit; exhausting matching rows does not prove runtime loading or complete analysis. For exact file/native facts and processing gaps, use view {kind: "javascript-facts", where: {paths: ["package.json"]}, property_keys: ["name", "operation", "analysis_status"]}. Rows page original observations with canonical identities and Evidence; missing properties remain omitted. Paths match recorded artifact-path/source-range locations; optional kinds filter nodes. To discover field names without values, opt into include_available_property_keys: true with property_keys: []. Canonical identity and Evidence are separate from properties; keep each node_id/observation_id pair together when composing results. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions. Occurrences accept exact logical_paths and occurrence_ids; nodes accept artifact_ids. Batch filters intersect (omitted means all available, [] means none); fields selects columns while retaining identity links. source_complete describes retained graph membership; source_coverage describes producer coverage, including unknown encrypted PCK directories. Page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. Interface Builder views use kind interface-builder and collection documents/nodes/edges/coverage; nodes and edges accept an exact document relative_path. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Managed member summaries keep identity, selection with unmatched names, coverage and per-collection counts; managed-members pages use collection types/exported_types/fields/methods/member_refs/call_edges/field_accesses/user_strings and carry the source selection and coverage. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. Protobuf views use kind protobuf, collection bodies/frames/fields, with record_ordinals, body_pointers and fields; field pages also select frame_indices, field_ids, field_numbers or schema_names. Byte identity, offset basis and body status remain present. Text and schema_value are opt-in page columns. Model views use kind model-artifact and collection metadata (keys), tensors (names/types) or types (derived totals), with indices, fields and paging. Header SHA-256 scopes declarations; full-file identity remains unknown unless full hashing was selected. Arrays retain typed eight-element previews with explicit omissions. IL2CPP metadata views use kind il2cpp-metadata and collection sections/images/types/methods/fields/properties/events/parameters/string_literals/code_gen_modules/method_addresses/return_types/field_types/parameter_types. Select indices, fields and pages; named rows accept names, types also accept image_indices/namespaces/declaring_types, and methods/members accept declaring_types. Parameters and parameter_types accept method_indices; code_gen_modules accept image_indices; method_addresses accept addresses/states, preserving every folded alternative. Row index is scoped to the metadata digest; native pages add binary_source. Uncollected native tables have null counts and remain unavailable. Desktop directory views use kind desktop-application and collection files/omissions/candidates/components/next_probes, with fields and pagination; files and omissions accept exact paths or their stable IDs, and candidates accept families. Source coverage and stable row IDs remain available. Native inventory Evidence from list_names, list/search_procedures or list/search_strings uses view: {kind: "native-inventory", offset: 0, limit: 64, fields: ["value"]}. Rows keep source_index and address even with selected fields. Summary counts describe retained rows; provider discovery coverage remains unknown. Native function Evidence requires kind native: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', + 'Inspect selected views of completed Frida session status, GDB/objdump command observations, Go build metadata, Rust embedded dependencies, binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_macho, decode_interface_builder, inspect_evm_interface, inspect_managed_artifact, inspect_managed_members, inspect_android_package, inspect_dart_aot, inspect_il2cpp_metadata, inspect_model_artifact or inspect_web_network_capture Evidence without repeating analysis. Command summary views preserve ordered GDB records or opaque objdump stdout/stderr once, source parameters, completion facts and parent identity. They do not establish current target state. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. Go metadata uses kind go, collection dependencies/settings, exact paths/keys or indices, fields and pagination. Row index refers to the original collection; empty selections return no rows, and dependency coverage remains unknown. Rust metadata uses kind rust, collection packages/edges, exact names/versions or original indices, selected fields, edge from_indices/to_indices and matching-row pagination. Embedded-record coverage is separate from unknown runtime dependencies. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. The environment-manifest view projects the same HAR requests on the primary origin (most JSON-like records) into a replayable tool surface: view {kind: "environment-manifest", template_rules: ["after-uid-literal", "opaque-token", "sibling-variance"], include_query_parameter_names: true}. Each tool has a deterministic name, a path template and parameters from the four format rules plus the selected opt-in rules, observed query parameter names, a value-free body shape for POST/PUT/PATCH, status and content-type sets and record ordinals; records are pointers into the parent, excluded groups carry a reason and three fixed unknowns are always stated. Templates, parameters and names are inferences; the view copies no header, cookie or body values and refuses parents inspected with sensitive values. A capture_browser_scenario parent that selected response bodies or header values projects the same way, one record per network transaction in request order; it relies on capture-time redaction of declared secrets instead of that refusal, and a transaction whose URL was redacted is left unrepresented. The environment-state view extracts, from the same parents, the initial state the recorded reads observed: view {kind: "environment-state", template_rules: [...]} with the rules of the manifest it accompanies. It reads response bodies recorded before the first related write and reports entity types with an identity field (proved by a path parameter or inferred from a field name), entities in collection order with per-view values where views differ, documents for non-parameterized reads that are not entities and have one initial request line (item-shaped paths stay recorded), and a render plan per read tool: item and list views reproduce their latest recordings from the state, and every other read tool is a recorded view with a reason. It also reports a write plan per state-changing tool (create, update, delete or document, built from its first successful call) and write_clock; a plan is served only when replaying the capture shows a later read confirming its effect and none contradicting it. Types and identity are inferences; values are recorded data, so it refuses parents inspected with sensitive values, and a scenario parent must have selected response bodies; its values keep the capture redaction markers, and a write whose URL it redacted ends the state of every type. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. A completed Evidence JSON file on this host uses {kind: "evidence-file", path: ""}: REA reads and authenticates it, then projects it like retained Evidence, so a large capture is bounded by the file reader rather than by inline-input accounting. For layout, JavaScript, inventory, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Typed module references use view {kind: "javascript", collection: "modules", projection: "references", where: {path: "main.js", kind: "javascript-module"}, offset: 0, limit: 64}; where matches exact retained strings (null selects unavailable paths). Rows include canonical identity, retained source-range count and observation support states. Source coverage and selection remain explicit; exhausting matching rows does not prove runtime loading or complete analysis. For exact file/native facts and processing gaps, use view {kind: "javascript-facts", where: {paths: ["package.json"]}, property_keys: ["name", "operation", "analysis_status"]}. Rows page original observations with canonical identities and Evidence; missing properties remain omitted. Paths match recorded artifact-path/source-range locations; optional kinds filter nodes. To discover field names without values, opt into include_available_property_keys: true with property_keys: []. Canonical identity and Evidence are separate from properties; keep each node_id/observation_id pair together when composing results. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions. Occurrences accept exact logical_paths and occurrence_ids; nodes accept artifact_ids. Batch filters intersect (omitted means all available, [] means none); fields selects columns while retaining identity links. source_complete describes retained graph membership; source_coverage describes producer coverage, including unknown encrypted PCK directories. Page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. Interface Builder views use kind interface-builder and collection documents/nodes/edges/coverage; nodes and edges accept an exact document relative_path. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Managed member summaries keep identity, selection with unmatched names, coverage and per-collection counts; managed-members pages use collection types/exported_types/fields/methods/member_refs/call_edges/field_accesses/user_strings and carry the source selection and coverage. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. Protobuf views use kind protobuf, collection bodies/frames/fields, with record_ordinals, body_pointers and fields; field pages also select frame_indices, field_ids, field_numbers or schema_names. Byte identity, offset basis and body status remain present. Text and schema_value are opt-in page columns. Model views use kind model-artifact and collection metadata (keys), tensors (names/types) or types (derived totals), with indices, fields and paging. Header SHA-256 scopes declarations; full-file identity remains unknown unless full hashing was selected. Arrays retain typed eight-element previews with explicit omissions. IL2CPP metadata views use kind il2cpp-metadata and collection sections/images/types/methods/fields/properties/events/parameters/string_literals/code_gen_modules/method_addresses/return_types/field_types/parameter_types. Select indices, fields and pages; named rows accept names, types also accept image_indices/namespaces/declaring_types, and methods/members accept declaring_types. Parameters and parameter_types accept method_indices; code_gen_modules accept image_indices; method_addresses accept addresses/states, preserving every folded alternative. Row index is scoped to the metadata digest; native pages add binary_source. Uncollected native tables have null counts and remain unavailable. Desktop directory views use kind desktop-application and collection files/omissions/candidates/components/next_probes, with fields and pagination; files and omissions accept exact paths or their stable IDs, and candidates accept families. Source coverage and stable row IDs remain available. Native inventory Evidence from list_names, list/search_procedures or list/search_strings uses view: {kind: "native-inventory", offset: 0, limit: 64, fields: ["value"]}. Rows keep source_index and address even with selected fields. Summary counts describe retained rows; provider discovery coverage remains unknown. Native function Evidence requires kind native: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', inputSchema: inspectAnalysisViewInputSchema, outputSchema: evidenceResultOf(analysisViewResultSchema), examples: [ diff --git a/src/contracts/rust/rustToolContracts.ts b/src/contracts/rust/rustToolContracts.ts index 7f827dfbed..966a057dbe 100644 --- a/src/contracts/rust/rustToolContracts.ts +++ b/src/contracts/rust/rustToolContracts.ts @@ -1,7 +1,10 @@ +import { rustBinarySchema } from "../../domain/rust/rustBinary.js"; +import { inspectRustBinaryRequestSchema } from "../../domain/rust/rustBinaryQuery.js"; +import { z } from "zod"; import { - inspectRustBinaryInputSchema, - rustBinarySchema, -} from "../../domain/rust/rustBinary.js"; + rustBinarySummaryViewResultSchema, + rustBinaryPageViewResultSchema, +} from "../../domain/analysisView/analysisView.js"; import type { ToolContract } from "../toolContractTypes.js"; import { toolContractMetadata } from "../toolEffects.js"; import { evidenceResultOf } from "../toolOutputSchemaPrimitives.js"; @@ -13,9 +16,15 @@ export const RUST_TOOL_CONTRACTS = [ ...toolContractMetadata("inspect_rust_binary"), kind: "artifact-provider", description: - "Inspect the cargo-auditable crate dependency list embedded in an explicit local ELF, PE or thin Mach-O image. Returns the artifact SHA-256, the .dep-v0 section file offset and size, compressed and uncompressed sizes, format version, packages in their original order, the root package index, and dependency edges by index. Reads zlib-compressed JSON without installing Rust, opening a disassembler, executing the target, writing temporary files or accessing the network. A missing section is null metadata with a limitation and is not proof that the binary has no dependencies. A rustc commit hash is reported only as an inference when a /rustc/<40-hex>/ path is present in read-only data. Resource guards bound files to 256 MiB, dependency sections to 1 MiB, inflated JSON to the smaller of 8 MiB and 64 times the compressed size, and the complete report to 8 MiB. Malformed, unsupported and oversized inputs receive typed failures rather than partial success.", - inputSchema: inspectRustBinaryInputSchema, - outputSchema: evidenceResultOf(rustBinarySchema), + "Inspect the cargo-auditable crate dependency list embedded in an explicit local ELF, PE or thin Mach-O image. Returns the artifact SHA-256, the .dep-v0 section file offset and size, compressed and uncompressed sizes, format version, packages in their original order, the root package index, and dependency edges by index. Omitted or explicit complete detail preserves the full report. Optional detail summary retains that complete Evidence and returns source ranges, counts and parent_evidence_id. Optional view directly selects packages or edges in this call; it cannot be combined with explicit complete detail. Query the same retained facts with inspect_analysis_view kind rust: exact names/versions or package indices, edge from_indices/to_indices, fields and matching-row pagination. Row indices are original and scoped to the parent Evidence; empty filters select none. Default package fields omit dependency vectors; explicitly select dependencies for direct index references. Every matching observed version is retained. No current runtime state is inferred from historical reads. Reads zlib-compressed JSON without installing Rust, opening a disassembler, executing the target, writing temporary files or accessing the network. A missing section is null metadata with a limitation and is not proof that the binary has no dependencies. A rustc commit hash is reported only as an inference when a /rustc/<40-hex>/ path is present in read-only data. Resource guards bound files to 256 MiB, dependency sections to 1 MiB, inflated JSON to the smaller of 8 MiB and 64 times the compressed size, and the complete report to 8 MiB. Malformed, unsupported and oversized inputs receive typed failures rather than partial success.", + inputSchema: inspectRustBinaryRequestSchema, + outputSchema: evidenceResultOf( + z.union([ + rustBinarySchema, + rustBinarySummaryViewResultSchema, + rustBinaryPageViewResultSchema, + ]), + ), examples: [ { title: "Inspect an embedded Rust dependency list", diff --git a/src/contracts/sqlite/sqliteToolContracts.ts b/src/contracts/sqlite/sqliteToolContracts.ts index 6ee837c0c5..f0288b86e2 100644 --- a/src/contracts/sqlite/sqliteToolContracts.ts +++ b/src/contracts/sqlite/sqliteToolContracts.ts @@ -13,7 +13,7 @@ export const SQLITE_TOOL_CONTRACTS = [ ...toolContractMetadata("inspect_sqlite_database"), kind: "artifact-provider", description: - "Inspect an explicit local SQLite database snapshot and its existing sibling WAL without modifying source files or opening a native analysis provider. Requires Node.js 24.x >=24.15 or 26+ with native DatabaseSync.limits support, checked by the worker. Returns observed Evidence with database/WAL SHA-256 and a compact schema inventory. Selecting an exact ordinary table expands only its schema and samples rows (default 20, subject to byte and worker resource limits). Select exact columns in the requested order to avoid unrelated or large values; schema_scope=database explicitly expands the whole schema; returned rows retain 64-bit integers, text bytes and blobs with explicit truncation. No arbitrary SQL, view or virtual-table execution, extension loading, database repair, encryption support or live-snapshot atomicity claim. At most two inspections per provider are admitted; occupied capacity is retryable. An isolated temporary copy and owned worker enforce input, value/record, SQL-statement, JSON-reply, JavaScript-heap and deadline limits; overall native heap is not capped. Supply a quiescent snapshot for coherent database/WAL evidence.", + "Inspect an explicit local SQLite database snapshot and its existing sibling WAL without modifying source files or opening a native analysis provider. Requires Node.js 24.x >=24.15 or 26+ with native DatabaseSync.limits support, checked by the worker. Returns observed Evidence with database/WAL SHA-256 and a compact schema inventory. Selecting an exact ordinary table expands only its schema and samples rows (default 20, subject to byte and worker resource limits). Select exact columns in the requested order to avoid unrelated or large values; schema_scope=database explicitly expands the whole schema; returned rows retain 64-bit integers, text bytes and blobs with explicit truncation. Adjust worker_timeout_ms (default 30000 ms) for worker startup, queries and reply writing; allow separate client headroom for snapshot acquisition, reply reads, cleanup and delivery. The result retains the actual budget; byte/heap safety limits stay fixed. No arbitrary SQL, view or virtual-table execution, extension loading, database repair, encryption support or live-snapshot atomicity claim. At most two inspections per provider are admitted; occupied capacity is retryable. An isolated temporary copy and owned worker enforce input, value/record, SQL-statement, JSON-reply, JavaScript-heap and deadline limits; overall native heap is not capped. Supply a quiescent snapshot for coherent database/WAL evidence.", inputSchema: inspectSqliteDatabaseInputSchema, outputSchema: evidenceResultOf(sqliteDatabaseSchema), examples: [ diff --git a/src/contracts/wasm/wasmToolContracts.ts b/src/contracts/wasm/wasmToolContracts.ts index cf05166d29..06abd7d784 100644 --- a/src/contracts/wasm/wasmToolContracts.ts +++ b/src/contracts/wasm/wasmToolContracts.ts @@ -22,7 +22,7 @@ export const INSPECT_WASM_ARTIFACT_CONTRACT = { ...toolContractMetadata("inspect_wasm_artifact"), kind: "artifact-provider", description: - 'Validate an explicit local WASM artifact with caller-supplied WABT 1.0.42. Omitted or complete detail returns exact byte SHA-256, sections, upstream import/export WAT forms and decoded WAT inline as Evidence. For counts, validation and WAT identity without WAT text, pass detail: "summary" on this first call; it retains the complete original Evidence and returns the same typed summary as inspect_analysis_view, avoiding oversized complete delivery. Read exact imports/exports or WAT pages with inspect_analysis_view using normalized_result.parent_evidence_id. Delivery selection does not change analysis, provider inputs or source identity. Optionally associate selected JavaScript glue literals with explicit local candidates while preserving ambiguous URL/basename matches. Configure absolute REA_WABT_BIN_DIRECTORY. No installation, implicit fetch, WASM execution or original-source claim.', + 'Validate an explicit local WASM artifact with caller-supplied WABT 1.0.42. Omitted or complete detail returns exact byte SHA-256, sections, upstream import/export WAT forms and decoded WAT inline as Evidence. For counts, validation and WAT identity without WAT text, pass detail: "summary" on this first call; it retains the complete original Evidence and returns the same typed summary as inspect_analysis_view, avoiding oversized complete delivery. Read exact imports/exports or WAT pages with inspect_analysis_view using normalized_result.parent_evidence_id. Delivery selection does not change analysis, provider inputs or source identity. Adjust command_timeout_ms for each WABT command (default 30000 ms), separately from the client deadline; input/output safety limits stay fixed. Optionally associate selected JavaScript glue literals with explicit local candidates while preserving ambiguous URL/basename matches. Configure absolute REA_WABT_BIN_DIRECTORY. No installation, implicit fetch, WASM execution or original-source claim.', inputSchema: wasmArtifactInspectionInputSchema, outputSchema: analysisDetailOutputSchemaOf( wasmArtifactSchema, diff --git a/src/domain/analysisErrorBase.ts b/src/domain/analysisErrorBase.ts index 1e4ceef10e..f408f471ee 100644 --- a/src/domain/analysisErrorBase.ts +++ b/src/domain/analysisErrorBase.ts @@ -143,17 +143,21 @@ export abstract class AnalysisError extends Error { this.#ownedCleanupResources = [...resources]; } - /** Retain a later lifecycle failure without replacing the original typed error. */ + /** Retain diagnostic facts without replacing the original typed error. */ + retainDiagnostics(diagnostics: Readonly>): this { + Object.assign(this, { + diagnostics: structuredClone({ ...this.diagnostics, ...diagnostics }), + }); + return this; + } + retainSecondaryFailure( failure: AnalysisError, diagnostics?: Readonly>, ): this { // Secondary resources remain owned by their cleanup lifecycle; preserve // the primary error's live cleanup status and original output. - if (diagnostics !== undefined) - Object.assign(this, { - diagnostics: structuredClone({ ...this.diagnostics, ...diagnostics }), - }); + if (diagnostics !== undefined) this.retainDiagnostics(diagnostics); this.#secondaryFailures.push(failure); return this; } diff --git a/src/domain/analysisErrorPresentation.ts b/src/domain/analysisErrorPresentation.ts index 3d3578a887..ac9599e35d 100644 --- a/src/domain/analysisErrorPresentation.ts +++ b/src/domain/analysisErrorPresentation.ts @@ -204,6 +204,7 @@ const browserErrorCategory = ( reason === "target_not_allowed" || reason === "service_worker_blocked" || reason === "target_changed" || + reason === "target_session_detached" || reason === "endpoint_unreachable" || reason === "disconnected" ) diff --git a/src/domain/analysisErrorProjection.ts b/src/domain/analysisErrorProjection.ts index 05bb2aec9a..43cd286d37 100644 --- a/src/domain/analysisErrorProjection.ts +++ b/src/domain/analysisErrorProjection.ts @@ -180,6 +180,7 @@ const BROWSER_ERROR_CODES: Readonly< target_not_allowed: "target_unavailable", service_worker_blocked: "target_unavailable", target_changed: "target_unavailable", + target_session_detached: "target_unavailable", protocol_error: "unreadable_output", disconnected: "provider_unavailable", payload_limit: "truncated", diff --git a/src/domain/analysisSnapshot.ts b/src/domain/analysisSnapshot.ts index 92a49ea60c..eb975a835e 100644 --- a/src/domain/analysisSnapshot.ts +++ b/src/domain/analysisSnapshot.ts @@ -90,6 +90,7 @@ const subjectFormatSchema = z.enum([ "unknown", "javascript-bundle", "entitlements", + "pyinstaller", ]); // Snapshot entries retain the same raw-only architecture boundary as targets. const subjectSchema = z.discriminatedUnion("format", [ diff --git a/src/domain/analysisView/EnvironmentManifestAccumulator.ts b/src/domain/analysisView/EnvironmentManifestAccumulator.ts index d9d8928021..1319f74e20 100644 --- a/src/domain/analysisView/EnvironmentManifestAccumulator.ts +++ b/src/domain/analysisView/EnvironmentManifestAccumulator.ts @@ -14,17 +14,19 @@ import { completeWithinViewCoverage } from "./analysisViewCoverage.js"; import type { UnsignedAnalysisView } from "./analysisView.js"; import { WebNetworkProjectionAccumulator } from "./WebNetworkProjectionAccumulator.js"; import { - applyTemplateRules, compareTools, parameterNames, pathTemplate, - SIBLING_VARIANCE_SEGMENT_LIMIT, toolName, toolNameRegistry, +} from "./environmentManifestNames.js"; +import { + applyTemplateRules, + SEGMENT_LIMITED_RULES, + SIBLING_VARIANCE_SEGMENT_LIMIT, type BodyShape, type JsonTypeName, type LiteralPart, - type ManifestRule, type TemplateRule, type ToolGroup, } from "./environmentManifestRules.js"; @@ -40,6 +42,7 @@ import { scanRouteTemplate, textBytes, type RecordedRequestLine, + type RouteRule, type RouteTemplate, } from "./webNetworkRouteTemplates.js"; @@ -72,7 +75,7 @@ type MutablePart = | LiteralPart | { readonly kind: "param"; - readonly rule: ManifestRule; + readonly rule: RouteRule; readonly values: Set; }; @@ -453,12 +456,10 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula ) ) return []; - return (["sibling-variance", "sibling-pair-variance"] as const) - .filter((rule) => this.#rules.has(rule)) - .map( - (rule) => - `Routes with more than ${String(SIBLING_VARIANCE_SEGMENT_LIMIT)} segments are not compared by ${rule}; they keep their grouped templates.`, - ); + return SEGMENT_LIMITED_RULES.filter((rule) => this.#rules.has(rule)).map( + (rule) => + `Routes with more than ${String(SIBLING_VARIANCE_SEGMENT_LIMIT)} segments are not compared by ${rule}; they keep their grouped templates.`, + ); } /** Project the complete manifest; tools, records and exclusions are sorted deterministically. */ diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index c16b443ad6..0020f4aec4 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -23,6 +23,13 @@ import { goBinaryViewRequestSchema, projectGoBinaryView, } from "./goBinaryView.js"; +import { + rustBinarySummarySchema, + rustBinaryViewContextSchema, + rustBinaryViewPageSchema, + rustBinaryViewRequestSchema, + projectRustBinaryView, +} from "./rustBinaryView.js"; import { projectNativeFunctionMatchingView, nativeFunctionMatchingViewRequestSchema, @@ -277,6 +284,7 @@ export const analysisViewRequestSchema = z.discriminatedUnion("kind", [ nativeFunctionMatchingViewRequestSchema, summaryViewSchema, goBinaryViewRequestSchema, + rustBinaryViewRequestSchema, desktopViewRequestSchema, facetViewSchema, itemViewSchema, @@ -462,6 +470,24 @@ export const goBinarySummaryViewResultSchema = z.strictObject({ summary: goBinarySummarySchema, }); +/** Compact source identity and counts, retaining the complete embedded graph. */ +export const rustBinarySummaryViewResultSchema = z.strictObject({ + ...summaryViewResultBase, + ...rustBinaryViewContextSchema.shape, + parent_operation: z.literal("inspect_rust_binary"), + summary: rustBinarySummarySchema, +}); + +export const rustBinaryPageViewResultSchema = z.strictObject({ + ...viewResultBase, + ...rustBinaryViewContextSchema.shape, + kind: z.literal("rust"), + parent_operation: z.literal("inspect_rust_binary"), + view: rustBinaryViewRequestSchema, + rust: rustBinarySummarySchema, + page: rustBinaryViewPageSchema, +}); + /** Summary `inspect_binary_layout` detail returns; no other view kind. */ export const binaryLayoutSummaryViewResultSchema = z.strictObject({ ...summaryViewResultBase, @@ -665,6 +691,7 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ objdumpSummaryViewResultSchema, fridaSessionStatusSummaryViewResultSchema, goBinarySummaryViewResultSchema, + rustBinarySummaryViewResultSchema, binaryLayoutSummaryViewResultSchema, desktopApplicationSummaryViewResultSchema, javascriptApplicationSummaryViewResultSchema, @@ -738,6 +765,7 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ view: goBinaryViewRequestSchema, page: goBinaryViewPageSchema, }), + rustBinaryPageViewResultSchema, z.strictObject({ kind: z.literal("item"), view: itemViewSchema, @@ -954,6 +982,7 @@ const retainedViewProjectors = new Map([ ["gdb_console", projectCommandObservationView], ["inspect_with_objdump", projectCommandObservationView], ["inspect_go_binary", projectGoBinaryView], + ["inspect_rust_binary", projectRustBinaryView], ...NATIVE_INVENTORY_OPERATIONS.map( (operation) => [operation, projectNativeInventoryView] as const, ), diff --git a/src/domain/analysisView/artifactInventoryView.ts b/src/domain/analysisView/artifactInventoryView.ts index 89e1c8ce95..730d200116 100644 --- a/src/domain/analysisView/artifactInventoryView.ts +++ b/src/domain/analysisView/artifactInventoryView.ts @@ -51,6 +51,9 @@ export const artifactInventoryViewRequestSchema = z.discriminatedUnion( collection: z.literal("occurrences"), occurrence_ids: z.array(prefixedDigestSchema("occ")).optional(), logical_paths: z.array(z.string()).optional(), + entry_kinds: z + .array(artifactOccurrenceSchema.shape.entry_kind) + .optional(), fields: z.array(artifactOccurrenceSchema.keyof()).optional(), }), z.strictObject({ @@ -104,6 +107,7 @@ export const artifactInventorySummarySchema = z.strictObject({ directory: z.number().int().nonnegative(), symlink: z.number().int().nonnegative(), slice: z.number().int().nonnegative(), + metadata: z.number().int().nonnegative().exactOptional(), }), source_complete: z.boolean(), inventory_evidence_ids: z.array(prefixedDigestSchema("ev")).min(1), @@ -140,6 +144,7 @@ interface InventorySource { index?: { byId: ReadonlyMap; byPath: ReadonlyMap; + byKind: ReadonlyMap; }; counts?: InventoryCounts; } @@ -241,7 +246,9 @@ export const projectArtifactInventoryView = ( ?.format_metadata ?? null; const rootMetadata = source.rootMetadata; const includeCoverage = - view.kind === "inventory" || inventory.manifest.root_format === "godot-pck"; + view.kind === "inventory" || + inventory.manifest.root_format === "godot-pck" || + inventory.manifest.root_format === "pyinstaller"; const sourceCoverage = includeCoverage ? { source_coverage: inventory.coverage } : {}; @@ -309,7 +316,15 @@ export const projectArtifactInventoryView = ( edges: inventory.edges.length, integrity_contradictions: inventory.integrityContradictions.length, }, - observed_entry_kind_counts: counts.kinds, + observed_entry_kind_counts: { + file: counts.kinds.file, + directory: counts.kinds.directory, + symlink: counts.kinds.symlink, + slice: counts.kinds.slice, + ...(counts.kinds.metadata === 0 + ? {} + : { metadata: counts.kinds.metadata }), + }, ...(includeCoverage ? { available_hash_status_counts: counts.hashes } : {}), @@ -406,7 +421,7 @@ const nodePage = ( const inventoryCounts = (source: InventorySource): InventoryCounts => { if (source.counts !== undefined) return source.counts; - const kinds = { file: 0, directory: 0, symlink: 0, slice: 0 }; + const kinds = { file: 0, directory: 0, symlink: 0, slice: 0, metadata: 0 }; const hashes = { verified: 0, mismatched: 0, @@ -435,27 +450,43 @@ const occurrenceIndices = ( source: InventorySource, view: OccurrenceView, ): number[] | undefined => { - if (view.occurrence_ids === undefined && view.logical_paths === undefined) + if ( + view.occurrence_ids === undefined && + view.logical_paths === undefined && + view.entry_kinds === undefined + ) return undefined; - if (view.occurrence_ids?.length === 0 || view.logical_paths?.length === 0) + if ( + view.occurrence_ids?.length === 0 || + view.logical_paths?.length === 0 || + view.entry_kinds?.length === 0 + ) return []; if (source.index === undefined) { const byId = new Map(); const byPath = new Map(); + const byKind = new Map(); for (const [index, row] of source.inventory.occurrences.entries()) { byId.set(row.occurrence_id, index); const existing = byPath.get(row.logical_path); if (existing === undefined) byPath.set(row.logical_path, [index]); else existing.push(index); + const kind = byKind.get(row.entry_kind); + if (kind === undefined) byKind.set(row.entry_kind, [index]); + else kind.push(index); } - source.index = { byId, byPath }; + source.index = { byId, byPath, byKind }; } const lookup = source.index; const paths = view.logical_paths === undefined ? undefined : new Set(view.logical_paths); const selected = view.occurrence_ids === undefined - ? [...(paths ?? [])].flatMap((path) => lookup.byPath.get(path) ?? []) + ? paths === undefined + ? [...new Set(view.entry_kinds)].flatMap( + (kind) => lookup.byKind.get(kind) ?? [], + ) + : [...paths].flatMap((path) => lookup.byPath.get(path) ?? []) : [...new Set(view.occurrence_ids)].flatMap((id) => { const index = lookup.byId.get(id); return index === undefined || @@ -464,7 +495,15 @@ const occurrenceIndices = ( ? [] : [index]; }); - return selected.sort((left, right) => left - right); + const kinds = + view.entry_kinds === undefined ? undefined : new Set(view.entry_kinds); + return selected + .filter( + (index) => + kinds === undefined || + kinds.has(source.inventory.occurrences[index]!.entry_kind), + ) + .sort((left, right) => left - right); }; const nodeIndices = ( diff --git a/src/domain/analysisView/environmentManifestNames.ts b/src/domain/analysisView/environmentManifestNames.ts new file mode 100644 index 0000000000..62854e466a --- /dev/null +++ b/src/domain/analysisView/environmentManifestNames.ts @@ -0,0 +1,157 @@ +import { + compareUnicodeCodePoints as compare, + compositeKey, +} from "../unicodeCodePointOrder.js"; +import { VERSION_TOKEN, type RoutePart } from "./environmentManifestRules.js"; +import { ROUTE_RULES } from "./webNetworkRouteTemplates.js"; + +const DROPPED_NAME_LITERALS = new Set(["api", "apis", "namespaces", "default"]); +const TOOL_NAME_LIMIT = 60; +const BASE_RULES: ReadonlySet = new Set(ROUTE_RULES); + +const sanitizeToken = (text: string): string => + text + .toLowerCase() + .replaceAll(/[^a-z0-9_]+/gu, "_") + .replaceAll(/_+/gu, "_") + .replace(/^_/u, "") + .replace(/_$/u, ""); + +const singular = (word: string): string => { + if (word.endsWith("ies") && word.length > 3) return `${word.slice(0, -3)}y`; + if (word.endsWith("s") && !word.endsWith("ss")) return word.slice(0, -1); + return word; +}; + +const paramBaseName = (parts: readonly RoutePart[], index: number): string => { + const part = parts[index]; + if (part?.kind !== "param") return ""; + // A prefix-variance parameter is named as the parameter it shares a position with. + switch (part.rule === "prefix-variance" ? part.sharedRule : part.rule) { + case "numeric": + return "id"; + case "uuid": + return "uuid"; + case "hex": + return "hash"; + case "iso-date": + return "date"; + case "after-uid-literal": + case "opaque-token": + return "uid"; + case "sibling-variance": { + const previous = parts[index - 1]; + const stem = + previous?.kind === "literal" + ? sanitizeToken(singular(previous.value)) + : ""; + return stem.length > 0 ? `${stem}_id` : "id"; + } + case "sibling-pair-variance": { + // The role was fixed when the pair formed; the plural literal precedes the pair's first half. + const role = part.pairRole ?? "namespace"; + const plural = parts[role === "name" ? index - 2 : index - 1]; + const stem = + plural?.kind === "literal" ? sanitizeToken(singular(plural.value)) : ""; + return stem.length > 0 ? `${stem}_${role}` : role; + } + } +}; + +/** Parameter names in segment order; repeated names get numeric suffixes. */ +export const parameterNames = (parts: readonly RoutePart[]): string[] => { + const used = new Map(); + return parts.flatMap((part, index) => { + if (part.kind !== "param") return []; + const base = paramBaseName(parts, index); + const count = used.get(base) ?? 0; + used.set(base, count + 1); + return [count === 0 ? base : `${base}_${String(count + 1)}`]; + }); +}; + +/** Format-rule placeholders keep their api-surface spelling; opt-in rules use the parameter name. */ +export const pathTemplate = ( + parts: readonly RoutePart[], + names: readonly string[], +): string => { + let next = 0; + return `/${parts + .map((part) => { + if (part.kind === "literal") return part.value; + const name = names[next] ?? ""; + next += 1; + return BASE_RULES.has(part.rule) ? `{${part.rule}}` : `{${name}}`; + }) + .join("/")}`; +}; + +/** Names assigned in one manifest and the next suffix to try for each colliding truncated base. */ +export interface ToolNameRegistry { + readonly used: Set; + readonly nextSuffix: Map; +} +export const toolNameRegistry = (): ToolNameRegistry => ({ + used: new Set(), + nextSuffix: new Map(), +}); +/** Deterministic tool name from the method and route literals, unique within a manifest. */ +export const toolName = ( + method: string, + parts: readonly RoutePart[], + names: readonly string[], + registry: ToolNameRegistry, +): string => { + const literals = parts.flatMap((part, index) => { + if (part.kind !== "literal") return []; + const value = part.value; + if ( + value.length === 0 || + DROPPED_NAME_LITERALS.has(value) || + VERSION_TOKEN.test(value) || + (value === "uid" && parts[index + 1]?.kind === "param") + ) + return []; + return [value.split(".")[0] ?? ""]; + }); + const base = + sanitizeToken( + [ + method.toLowerCase(), + ...literals, + ...(names.length > 0 ? ["by", ...names] : []), + ].join("_"), + ) || "tool"; + const truncate = (text: string, limit: number): string => + text.slice(0, limit).replace(/_+$/u, ""); + let candidate = truncate(base, TOOL_NAME_LIMIT); + if (registry.used.has(candidate)) { + let suffix = 2; + for (;;) { + const tail = `_${String(suffix)}`; + const prefix = truncate(base, TOOL_NAME_LIMIT - tail.length); + // Suffixes can truncate distinct full names to the same prefix. Share + // progress for that actual candidate family, separately for each width: + // a longer suffix can change the prefix and must not skip free short names. + const namespace = compositeKey([prefix, tail.length]); + const next = registry.nextSuffix.get(namespace) ?? suffix; + if (next > suffix) { + suffix = next; + continue; + } + candidate = `${prefix}${tail}`; + registry.nextSuffix.set(namespace, suffix + 1); + if (!registry.used.has(candidate)) break; + suffix += 1; + } + } + registry.used.add(candidate); + return candidate; +}; + +/** Sort tools by template then method with locale-independent ordering. */ +export const compareTools = ( + left: { readonly template: string; readonly method: string }, + right: { readonly template: string; readonly method: string }, +): number => + compare(left.template, right.template) || compare(left.method, right.method); diff --git a/src/domain/analysisView/environmentManifestRules.test.ts b/src/domain/analysisView/environmentManifestRules.test.ts index 1584734300..ece5943af8 100644 --- a/src/domain/analysisView/environmentManifestRules.test.ts +++ b/src/domain/analysisView/environmentManifestRules.test.ts @@ -1,5 +1,5 @@ import { expect, it, vi } from "vitest"; -import { toolName, toolNameRegistry } from "./environmentManifestRules.js"; +import { toolName, toolNameRegistry } from "./environmentManifestNames.js"; it("shares suffix work when distinct names truncate to the same suffixed prefix", () => { const registry = toolNameRegistry(); diff --git a/src/domain/analysisView/environmentManifestRules.ts b/src/domain/analysisView/environmentManifestRules.ts index d5bc200ae1..1bdf605df2 100644 --- a/src/domain/analysisView/environmentManifestRules.ts +++ b/src/domain/analysisView/environmentManifestRules.ts @@ -1,8 +1,10 @@ +import { compositeKey } from "../unicodeCodePointOrder.js"; import { - compareUnicodeCodePoints as compare, - compositeKey, -} from "../unicodeCodePointOrder.js"; -import { ROUTE_RULES, type RouteRule } from "./webNetworkRouteTemplates.js"; + spelledLikeObserved, + spellingSummary, + type SpellingSummary, +} from "./environmentManifestSpelling.js"; +import type { RouteRule } from "./webNetworkRouteTemplates.js"; /** Opt-in template rules applied after the four exact format rules. */ export const TEMPLATE_RULES = [ @@ -10,9 +12,11 @@ export const TEMPLATE_RULES = [ "opaque-token", "sibling-variance", "sibling-pair-variance", + "prefix-variance", ] as const; export type TemplateRule = (typeof TEMPLATE_RULES)[number]; -export type ManifestRule = RouteRule | TemplateRule; +/** The opt-in rules whose parameters a prefix-variance parameter can share; never itself, so shared evidence does not chain. */ +type SharedRule = Exclude; export type JsonTypeName = | "string" @@ -32,13 +36,19 @@ export interface LiteralPart { readonly kind: "literal"; readonly value: string; } -export interface ParamPart { +interface ParamFields { readonly kind: "param"; - readonly rule: ManifestRule; readonly values: ReadonlySet; /** Which half of a sibling-pair-variance pair; fixed when the pair forms so later merges cannot change it. */ readonly pairRole?: "namespace" | "name"; } +export type ParamPart = + | (ParamFields & { readonly rule: RouteRule | SharedRule }) + | (ParamFields & { + readonly rule: "prefix-variance"; + /** The rule of the parameter this one shares a position with, which also names it. */ + readonly sharedRule: SharedRule; + }); export type RoutePart = LiteralPart | ParamPart; /** One candidate tool: a merged request-line group on the primary origin. */ @@ -56,7 +66,7 @@ export interface ToolGroup { } const OPAQUE_TOKEN = /^(?=.*\d)(?=.*[a-z])[a-z0-9]{12,}$/u; -const VERSION_TOKEN = /^v\d/u; +export const VERSION_TOKEN = /^v\d/u; const STOP_WORDS = new Set([ "search", "settings", @@ -71,14 +81,17 @@ const STOP_WORDS = new Set([ "dto", "markdown", ]); -const DROPPED_NAME_LITERALS = new Set(["api", "apis", "namespaces", "default"]); -const TOOL_NAME_LIMIT = 60; /** Longest route sibling-variance compares; every position rebuilds a key over the whole route. */ export const SIBLING_VARIANCE_SEGMENT_LIMIT = 32; -const BASE_RULES: ReadonlySet = new Set(ROUTE_RULES); +/** The rules that compare routes and so skip those beyond the segment limit. */ +export const SEGMENT_LIMITED_RULES = [ + "sibling-variance", + "sibling-pair-variance", + "prefix-variance", +] as const; const OPT_IN_RULES: ReadonlySet = new Set(TEMPLATE_RULES); -const param = (rule: ManifestRule, values: Iterable): ParamPart => ({ +const param = (rule: SharedRule, values: Iterable): ParamPart => ({ kind: "param", rule, values: new Set(values), @@ -139,19 +152,25 @@ const mergeGroups = ( }; }; -const rebucket = (groups: readonly ToolGroup[]): ToolGroup[] => { +const rebucket = ( + groups: readonly ToolGroup[], + keyOf: (part: RoutePart) => unknown = partKey, + shapeOf: (bucket: readonly ToolGroup[]) => ToolGroup = earliest, +): ToolGroup[] => { const buckets = new Map(); for (const group of groups) { const key = compositeKey([ group.method, group.origin, - ...group.parts.map(partKey), + ...group.parts.map(keyOf), ]); const bucket = buckets.get(key); if (bucket === undefined) buckets.set(key, [group]); else bucket.push(group); } - return [...buckets.values()].map((bucket) => mergeGroups(bucket)); + return [...buckets.values()].map((bucket) => + mergeGroups(bucket, shapeOf(bucket)), + ); }; const templateAfterUidLiteral = (parts: readonly RoutePart[]): RoutePart[] => @@ -258,12 +277,7 @@ const mergeSiblings = ( /** Merge groups that differ in exactly one identifier-like segment at index two or later. */ const mergeSiblingVariance = (groups: readonly ToolGroup[]): ToolGroup[] => { - const beyondLimit = groups.filter( - (group) => group.parts.length > SIBLING_VARIANCE_SEGMENT_LIMIT, - ); - let current = groups.filter( - (group) => group.parts.length <= SIBLING_VARIANCE_SEGMENT_LIMIT, - ); + let current = [...groups]; const longest = Math.max(0, ...current.map((group) => group.parts.length)); for (let position = 2; position < longest; position += 1) { const buckets = new Map(); @@ -289,7 +303,7 @@ const mergeSiblingVariance = (groups: readonly ToolGroup[]): ToolGroup[] => { output.push(...mergeSiblings(bucket, position)); current = output; } - return [...current, ...beyondLimit]; + return current; }; /** Fewest distinct literal pairs that make two adjacent positions one templated pair. */ @@ -357,12 +371,7 @@ const mergePairBucket = ( /** Merge groups that differ in two adjacent literals after a plural literal, such as `/repos/{owner}/{repo}/…`. */ const mergeSiblingPairs = (groups: readonly ToolGroup[]): ToolGroup[] => { - const beyondLimit = groups.filter( - (group) => group.parts.length > SIBLING_VARIANCE_SEGMENT_LIMIT, - ); - let current = groups.filter( - (group) => group.parts.length <= SIBLING_VARIANCE_SEGMENT_LIMIT, - ); + let current = [...groups]; const longest = Math.max(0, ...current.map((group) => group.parts.length)); for (let position = 2; position + 1 < longest; position += 1) { const buckets = new Map(); @@ -388,9 +397,146 @@ const mergeSiblingPairs = (groups: readonly ToolGroup[]): ToolGroup[] => { output.push(...mergePairBucket(bucket, position)); current = output; } - return [...current, ...beyondLimit]; + return current; }; +/** A part by the rule that names it: a prefix-variance parameter keys as the parameter it shares. */ +const nameKey = (part: RoutePart): unknown => + part.kind === "literal" + ? part.value + : [ + "param", + part.rule === "prefix-variance" ? part.sharedRule : part.rule, + part.pairRole ?? null, + ]; + +const prefixKey = (group: ToolGroup, position: number): string => + compositeKey([group.origin, ...group.parts.slice(0, position).map(nameKey)]); + +/** A parameter a position can share: the earliest tool's parameter of one naming rule, and every value tools named by that rule observed there. */ +interface SharedDonor { + readonly part: ParamPart & { readonly rule: SharedRule }; + readonly values: Set; +} + +const isShared = ( + part: RoutePart | undefined, +): part is ParamPart & { readonly rule: SharedRule } => + part?.kind === "param" && + part.rule !== "prefix-variance" && + OPT_IN_RULES.has(part.rule); + +/** A position's donors, earliest first, with each one's spelling summarised once for every literal it is compared with. */ +type SharedPosition = readonly (SharedDonor & { + readonly spelling: SpellingSummary; +})[]; + +/** Opt-in parameters at one position by their preceding parts, one donor per naming rule, so a value stays with the parameter that observed it. */ +const sharedPositions = ( + groups: readonly ToolGroup[], + position: number, +): Map => { + const donors = new Map>(); + const ordered = [...groups].sort( + (left, right) => (left.positions[0] ?? 0) - (right.positions[0] ?? 0), + ); + for (const group of ordered) { + const part = group.parts[position]; + if (!isShared(part)) continue; + const key = prefixKey(group, position); + const byRule = donors.get(key) ?? new Map(); + donors.set(key, byRule); + const rule = compositeKey([nameKey(part)]); + const known = byRule.get(rule); + if (known === undefined) + byRule.set(rule, { part, values: new Set(part.values) }); + else for (const value of part.values) known.values.add(value); + } + return new Map( + [...donors].map(([key, byRule]) => [ + key, + [...byRule.values()].map((donor) => ({ + ...donor, + spelling: spellingSummary(donor.values), + })), + ]), + ); +}; + +/** The donor a literal shares a position with: one that observed it, else one it is spelled like, earliest first; never for an empty literal, a stop word or a version. */ +const donorFor = ( + value: string, + donors: SharedPosition, +): SharedDonor | undefined => + value === "" || STOP_WORDS.has(value) || VERSION_TOKEN.test(value) + ? undefined + : (donors.find((donor) => donor.values.has(value)) ?? + donors.find((donor) => spelledLikeObserved(value, donor.spelling))); + +/** The bucket's earliest group, with each position's parameter taken from a group that templated it by its own rule when one did. */ +const sharedShape = (bucket: readonly ToolGroup[]): ToolGroup => { + const first = earliest(bucket); + return withParts( + first, + first.parts.map((part, index) => + part.kind === "param" && part.rule === "prefix-variance" + ? (bucket.map((group) => group.parts[index]).find(isShared) ?? part) + : part, + ), + ); +}; + +/** Template literals at positions another tool templates by an opt-in rule after the same preceding parts, at any method or length. */ +const mergePrefixVariance = (groups: readonly ToolGroup[]): ToolGroup[] => { + let current = [...groups]; + const longest = Math.max(0, ...current.map((group) => group.parts.length)); + for (let position = 0; position < longest; position += 1) { + const shared = sharedPositions(current, position); + if (shared.size === 0) continue; + current = current.map((group) => { + const part = group.parts[position]; + const donors = + part?.kind === "literal" + ? shared.get(prefixKey(group, position)) + : undefined; + const known = + part?.kind === "literal" && donors !== undefined + ? donorFor(part.value, donors) + : undefined; + if (part?.kind !== "literal" || known === undefined) return group; + // Copies the shared parameter's pair role; the values are this literal only. + const templated: ParamPart = { + ...known.part, + rule: "prefix-variance", + sharedRule: known.part.rule, + values: new Set([part.value]), + }; + return withParts( + group, + group.parts.map((other, index) => + index === position ? templated : other, + ), + ); + }); + } + return rebucket(current, nameKey, sharedShape); +}; + +/** Run a comparing pass over routes within the segment limit; longer routes keep their groups. */ +const withinSegmentLimit = ( + groups: readonly ToolGroup[], + pass: (groups: readonly ToolGroup[]) => ToolGroup[], +): ToolGroup[] => [ + ...pass( + groups.filter( + (group) => group.parts.length <= SIBLING_VARIANCE_SEGMENT_LIMIT, + ), + ), + ...groups.filter( + (group) => group.parts.length > SIBLING_VARIANCE_SEGMENT_LIMIT, + ), +]; + /** Apply the selected opt-in rules in fixed order; every merge is an inference. */ export const applyTemplateRules = ( groups: readonly ToolGroup[], @@ -408,153 +554,12 @@ export const applyTemplateRules = ( current = rebucket(current); // Pairs merge first: a single-position merge could otherwise template one // half of a pair and leave too few literal pairs to recognize. - if (rules.has("sibling-pair-variance")) current = mergeSiblingPairs(current); - if (rules.has("sibling-variance")) current = mergeSiblingVariance(current); + if (rules.has("sibling-pair-variance")) + current = withinSegmentLimit(current, mergeSiblingPairs); + if (rules.has("sibling-variance")) + current = withinSegmentLimit(current, mergeSiblingVariance); + // Last, so it can share the parameters every other rule produced. + if (rules.has("prefix-variance")) + current = withinSegmentLimit(current, mergePrefixVariance); return current; }; - -const sanitizeToken = (text: string): string => - text - .toLowerCase() - .replaceAll(/[^a-z0-9_]+/gu, "_") - .replaceAll(/_+/gu, "_") - .replace(/^_/u, "") - .replace(/_$/u, ""); - -const singular = (word: string): string => { - if (word.endsWith("ies") && word.length > 3) return `${word.slice(0, -3)}y`; - if (word.endsWith("s") && !word.endsWith("ss")) return word.slice(0, -1); - return word; -}; - -const paramBaseName = (parts: readonly RoutePart[], index: number): string => { - const part = parts[index]; - if (part?.kind !== "param") return ""; - switch (part.rule) { - case "numeric": - return "id"; - case "uuid": - return "uuid"; - case "hex": - return "hash"; - case "iso-date": - return "date"; - case "after-uid-literal": - case "opaque-token": - return "uid"; - case "sibling-variance": { - const previous = parts[index - 1]; - const stem = - previous?.kind === "literal" - ? sanitizeToken(singular(previous.value)) - : ""; - return stem.length > 0 ? `${stem}_id` : "id"; - } - case "sibling-pair-variance": { - // The role was fixed when the pair formed; the plural literal precedes the pair's first half. - const role = part.pairRole ?? "namespace"; - const plural = parts[role === "name" ? index - 2 : index - 1]; - const stem = - plural?.kind === "literal" ? sanitizeToken(singular(plural.value)) : ""; - return stem.length > 0 ? `${stem}_${role}` : role; - } - } -}; - -/** Parameter names in segment order; repeated names get numeric suffixes. */ -export const parameterNames = (parts: readonly RoutePart[]): string[] => { - const used = new Map(); - return parts.flatMap((part, index) => { - if (part.kind !== "param") return []; - const base = paramBaseName(parts, index); - const count = used.get(base) ?? 0; - used.set(base, count + 1); - return [count === 0 ? base : `${base}_${String(count + 1)}`]; - }); -}; - -/** Format-rule placeholders keep their api-surface spelling; opt-in rules use the parameter name. */ -export const pathTemplate = ( - parts: readonly RoutePart[], - names: readonly string[], -): string => { - let next = 0; - return `/${parts - .map((part) => { - if (part.kind === "literal") return part.value; - const name = names[next] ?? ""; - next += 1; - return BASE_RULES.has(part.rule) ? `{${part.rule}}` : `{${name}}`; - }) - .join("/")}`; -}; - -/** Names assigned in one manifest and the next suffix to try for each colliding truncated base. */ -export interface ToolNameRegistry { - readonly used: Set; - readonly nextSuffix: Map; -} -export const toolNameRegistry = (): ToolNameRegistry => ({ - used: new Set(), - nextSuffix: new Map(), -}); -/** Deterministic tool name from the method and route literals, unique within a manifest. */ -export const toolName = ( - method: string, - parts: readonly RoutePart[], - names: readonly string[], - registry: ToolNameRegistry, -): string => { - const literals = parts.flatMap((part, index) => { - if (part.kind !== "literal") return []; - const value = part.value; - if ( - value.length === 0 || - DROPPED_NAME_LITERALS.has(value) || - VERSION_TOKEN.test(value) || - (value === "uid" && parts[index + 1]?.kind === "param") - ) - return []; - return [value.split(".")[0] ?? ""]; - }); - const base = - sanitizeToken( - [ - method.toLowerCase(), - ...literals, - ...(names.length > 0 ? ["by", ...names] : []), - ].join("_"), - ) || "tool"; - const truncate = (text: string, limit: number): string => - text.slice(0, limit).replace(/_+$/u, ""); - let candidate = truncate(base, TOOL_NAME_LIMIT); - if (registry.used.has(candidate)) { - let suffix = 2; - for (;;) { - const tail = `_${String(suffix)}`; - const prefix = truncate(base, TOOL_NAME_LIMIT - tail.length); - // Suffixes can truncate distinct full names to the same prefix. Share - // progress for that actual candidate family, separately for each width: - // a longer suffix can change the prefix and must not skip free short names. - const namespace = compositeKey([prefix, tail.length]); - const next = registry.nextSuffix.get(namespace) ?? suffix; - if (next > suffix) { - suffix = next; - continue; - } - candidate = `${prefix}${tail}`; - registry.nextSuffix.set(namespace, suffix + 1); - if (!registry.used.has(candidate)) break; - suffix += 1; - } - } - registry.used.add(candidate); - return candidate; -}; - -/** Sort tools by template then method with locale-independent ordering. */ -export const compareTools = ( - left: { readonly template: string; readonly method: string }, - right: { readonly template: string; readonly method: string }, -): number => - compare(left.template, right.template) || compare(left.method, right.method); diff --git a/src/domain/analysisView/environmentManifestSpelling.test.ts b/src/domain/analysisView/environmentManifestSpelling.test.ts new file mode 100644 index 0000000000..2277a44e49 --- /dev/null +++ b/src/domain/analysisView/environmentManifestSpelling.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; + +import { + spelledLikeObserved, + spellingSummary, +} from "./environmentManifestSpelling.js"; + +describe("spelledLikeObserved", () => { + it("compares digit-bearing values by length and character classes, and digit-free ones by family", () => { + const observed = spellingSummary(["ydr9n16753tzr7zc"]); + expect(spelledLikeObserved("ab1cd23ef4gh5ij6", observed)).toBe(true); + expect(spelledLikeObserved("fkmwxmxngagvsxfg", observed)).toBe(true); + expect(spelledLikeObserved("FkMwxmXNGaGVsXFG", observed)).toBe(false); + expect(spelledLikeObserved("notifications", observed)).toBe(false); + }); + + it("compares no value longer than the spelling limit, observed or candidate", () => { + const long = (length: number, last: string) => + `${"a".repeat(length - 1)}${last}`; + expect( + spelledLikeObserved(long(128, "2"), spellingSummary([long(128, "1")])), + ).toBe(true); + const beyond = spellingSummary([long(129, "1")]); + expect(beyond).toEqual({ shapes: new Set(), family: undefined }); + expect(spelledLikeObserved(long(129, "2"), beyond)).toBe(false); + // A multi-megabyte segment is summarised without classifying its characters. + expect(spellingSummary([`${"é".repeat(4_000_000)}1`]).shapes.size).toBe(0); + }); +}); diff --git a/src/domain/analysisView/environmentManifestSpelling.ts b/src/domain/analysisView/environmentManifestSpelling.ts new file mode 100644 index 0000000000..92369d688e --- /dev/null +++ b/src/domain/analysisView/environmentManifestSpelling.ts @@ -0,0 +1,84 @@ +import { + compareUnicodeCodePoints as compare, + compositeKey, +} from "../unicodeCodePointOrder.js"; + +/** Fewest characters of a fixed-length identifier family whose members need not carry a digit. */ +const FIXED_LENGTH_MINIMUM = 8; + +/** Longest value, in UTF-16 units, compared by spelling; a longer one shares a position only as an observed value. */ +const SPELLED_LENGTH_LIMIT = 128; + +/** The distinct classes of a text's characters, digits, lower case, upper case and each other character, in one pass; undefined past the spelling limit. */ +const characterClasses = (text: string): string | undefined => { + if (text.length > SPELLED_LENGTH_LIMIT) return undefined; + const classes = new Set(); + for (const character of text) + classes.add( + character >= "0" && character <= "9" + ? "0" + : character >= "a" && character <= "z" + ? "a" + : character >= "A" && character <= "Z" + ? "A" + : character, + ); + return [...classes].sort(compare).join(""); +}; + +const shapeOf = (length: number, classes: string): string => + compositeKey([length, classes]); + +/** How a set of observed identifiers is spelled, summarised once so each candidate is checked without rescanning them. */ +export interface SpellingSummary { + /** Each observed value's length and character classes. */ + readonly shapes: ReadonlySet; + /** The one length of eight or more every observed value has, with the classes they use together. */ + readonly family: + | { readonly length: number; readonly classes: ReadonlySet } + | undefined; +} + +export const spellingSummary = ( + observed: Iterable, +): SpellingSummary => { + const shapes = new Set(); + const lengths = new Set(); + const classes = new Set(); + for (const value of observed) { + lengths.add(value.length); + const kinds = characterClasses(value); + if (kinds === undefined) continue; + shapes.add(shapeOf(value.length, kinds)); + for (const kind of kinds) classes.add(kind); + } + const [length] = lengths; + return { + shapes, + family: + lengths.size === 1 && + length! >= FIXED_LENGTH_MINIMUM && + length! <= SPELLED_LENGTH_LIMIT + ? { length: length!, classes } + : undefined, + }; +}; + +/** Spelled like an observed identifier: a digit-bearing value with the length and character classes of one, or a member of their fixed-length family. */ +export const spelledLikeObserved = ( + value: string, + observed: SpellingSummary, +): boolean => { + const classes = characterClasses(value); + if (classes === undefined) return false; + const { family } = observed; + if ( + family !== undefined && + value.length === family.length && + [...classes].every((kind) => family.classes.has(kind)) + ) + return true; + return ( + /\d/u.test(value) && observed.shapes.has(shapeOf(value.length, classes)) + ); +}; diff --git a/src/domain/analysisView/environmentManifestView.test.ts b/src/domain/analysisView/environmentManifestView.test.ts index 1fdd211ab3..0910f6e3fa 100644 --- a/src/domain/analysisView/environmentManifestView.test.ts +++ b/src/domain/analysisView/environmentManifestView.test.ts @@ -1174,3 +1174,246 @@ it.each([ expect(projected.records).toHaveLength(values.length); }, ); + +it("templates a literal that another method's opt-in parameter observed at the same position, at any depth", () => { + const routes = syntheticCapture([ + ["GET", "https://k.test/api/v1/collections/0RZ51AB"], + ["GET", "https://k.test/api/v1/collections/0RZ52CD"], + ["GET", "https://k.test/api/v1/collections/0RZ53EF"], + ["PATCH", "https://k.test/api/v1/collections/0RZ53EF", null, "{}"], + ["DELETE", "https://k.test/api/v1/collections/0RZ53EF", null], + ["GET", "https://k.test/api/v1/collections/0RZ52CD/books"], + ["GET", "https://k.test/api/v1/collections/layout"], + ["POST", "https://k.test/api/v1/collections/search", null, "{}"], + ]); + const shared = manifest(["sibling-variance", "prefix-variance"], routes); + expect(shared.tools.map(line)).toEqual([ + "GET /api/v1/collections/layout", + "POST /api/v1/collections/search", + "DELETE /api/v1/collections/{collection_id}", + "GET /api/v1/collections/{collection_id}", + "PATCH /api/v1/collections/{collection_id}", + "GET /api/v1/collections/{collection_id}/books", + ]); + expect(shared.tools.find((tool) => tool.method === "PATCH")).toMatchObject({ + name: "patch_collections_by_collection_id", + api_surface_template: "/api/v1/collections/0RZ53EF", + params: [ + { + index: 3, + name: "collection_id", + rule: "prefix-variance", + basis: "inferred", + observed_values: ["0RZ53EF"], + }, + ], + record_ordinals: [3], + }); + expect( + shared.tools.find((tool) => tool.path_template.endsWith("/books"))?.name, + ).toBe("get_collections_books_by_collection_id"); + expect(manifest(["sibling-variance"], routes).tools.map(line)).toEqual([ + "GET /api/v1/collections/0RZ52CD/books", + "DELETE /api/v1/collections/0RZ53EF", + "PATCH /api/v1/collections/0RZ53EF", + "GET /api/v1/collections/layout", + "POST /api/v1/collections/search", + "GET /api/v1/collections/{collection_id}", + ]); +}); + +it("templates an unobserved literal only when it carries a digit and is spelled like an observed value", () => { + const projected = manifest( + ["sibling-variance", "prefix-variance"], + syntheticCapture([ + ["GET", "https://k.test/api/readlists/0RZ51KSZKA53J"], + ["GET", "https://k.test/api/readlists/0RZ56NR37AC1T"], + ["PATCH", "https://k.test/api/readlists/0RZ576R5KA57B", null, "{}"], + ["PUT", "https://k.test/api/readlists/0rz576r5ka57b", null, "{}"], + ["DELETE", "https://k.test/api/readlists/ABCDEFGHIJ", null], + ["POST", "https://k.test/api/readlists/Metrics::12", null, "{}"], + ["GET", "https://k.test/api/items/a1"], + ["GET", "https://k.test/api/items/b2"], + ["PUT", "https://k.test/api/items/v2", null, "{}"], + ]), + ); + expect(projected.tools.map(line)).toEqual([ + "PUT /api/items/v2", + "GET /api/items/{item_id}", + "PUT /api/readlists/0rz576r5ka57b", + "DELETE /api/readlists/ABCDEFGHIJ", + "POST /api/readlists/Metrics::12", + "GET /api/readlists/{readlist_id}", + "PATCH /api/readlists/{readlist_id}", + ]); +}); + +it("merges tools whose routes now match and keeps the parameter a tool templated by its own rule", () => { + const routes = syntheticCapture([ + ["GET", "https://b.test/api/boards/b-2/cards/c-3"], + ["GET", "https://b.test/api/boards/b-1"], + ["GET", "https://b.test/api/boards/b-2"], + ["GET", "https://b.test/api/boards/b-1/cards/c-1"], + ["GET", "https://b.test/api/boards/b-1/cards/c-2"], + ["GET", "https://b.test/api/subjects/red-one"], + ["GET", "https://b.test/api/subjects/blue-two"], + ["GET", "https://b.test/api/subjects/green-three"], + ["GET", "https://b.test/api/subjects/red-one/versions/1"], + ["GET", "https://b.test/api/subjects/blue-two/versions/2"], + ]); + const projected = manifest(["sibling-variance", "prefix-variance"], routes); + expect(projected.tools.map(line)).toEqual([ + "GET /api/boards/{board_id}", + "GET /api/boards/{board_id}/cards/{card_id}", + "GET /api/subjects/{subject_id}", + "GET /api/subjects/{subject_id}/versions/{numeric}", + ]); + const params = (template: string) => + projected.tools + .find((tool) => tool.path_template === template) + ?.params.map((param) => [param.name, param.rule, param.observed_values]); + expect(params("/api/boards/{board_id}/cards/{card_id}")).toEqual([ + ["board_id", "prefix-variance", ["b-1", "b-2"]], + ["card_id", "sibling-variance", ["c-1", "c-2", "c-3"]], + ]); + expect(params("/api/subjects/{subject_id}/versions/{numeric}")).toEqual([ + ["subject_id", "prefix-variance", ["blue-two", "red-one"]], + ["id", "numeric", ["1", "2"]], + ]); + expect(manifest(["sibling-variance"], routes).tools.map(line)).toEqual([ + "GET /api/boards/b-1/cards/{card_id}", + "GET /api/boards/b-2/cards/c-3", + "GET /api/boards/{board_id}", + "GET /api/subjects/blue-two/versions/{numeric}", + "GET /api/subjects/red-one/versions/{numeric}", + "GET /api/subjects/{subject_id}", + ]); +}); + +it("shares no format-rule position and compares no route beyond the segment limit", () => { + const tail = Array.from( + { length: 36 }, + (_, index) => `s${String(index)}`, + ).join("/"); + const projected = manifest( + ["sibling-variance", "prefix-variance"], + syntheticCapture([ + ["GET", "https://t.test/api/teams/1"], + ["GET", "https://t.test/api/teams/2"], + ["PATCH", "https://t.test/api/teams/t3", null, "{}"], + ["GET", "https://t.test/api/items/a1"], + ["GET", "https://t.test/api/items/b2"], + ["GET", `https://t.test/api/items/b2/${tail}`], + ]), + ); + expect(projected.tools.map(line)).toEqual([ + `GET /api/items/b2/${tail}`, + "GET /api/items/{item_id}", + "PATCH /api/teams/t3", + "GET /api/teams/{numeric}", + ]); + expect(projected.limitations).toContain( + "Routes with more than 32 segments are not compared by prefix-variance; they keep their grouped templates.", + ); +}); + +it("shares both halves of a templated pair and keeps each half's name", () => { + const projected = manifest( + ["sibling-pair-variance", "prefix-variance"], + syntheticCapture([ + ["GET", "https://g.test/api/repos/lighthouse-rota/lamp-log/issues"], + ["GET", "https://g.test/api/repos/granite-ledger/ledger-core/issues"], + ["GET", "https://g.test/api/repos/blue-heron/recipe-book/issues"], + [ + "POST", + "https://g.test/api/repos/granite-ledger/ledger-core/issues", + null, + "{}", + ], + ["DELETE", "https://g.test/api/repos/blue-heron/recipe-book", null], + ]), + ); + expect(projected.tools.map(line)).toEqual([ + "DELETE /api/repos/{repo_namespace}/{repo_name}", + "GET /api/repos/{repo_namespace}/{repo_name}/issues", + "POST /api/repos/{repo_namespace}/{repo_name}/issues", + ]); + expect( + projected.tools[0]?.params.map((param) => [param.name, param.rule]), + ).toEqual([ + ["repo_namespace", "prefix-variance"], + ["repo_name", "prefix-variance"], + ]); +}); + +it("templates a digit-free literal only as a member of a fixed-length identifier family", () => { + const projected = manifest( + ["sibling-variance", "prefix-variance"], + syntheticCapture([ + ["GET", "https://n.test/api/workflows/yDr9n16753TZr7Zc"], + ["GET", "https://n.test/api/workflows/5LPA94ML1gYbgAQx"], + ["PATCH", "https://n.test/api/workflows/FkMwxmXNGaGVsXFG", null, "{}"], + ["DELETE", "https://n.test/api/workflows/listForAccounts", null], + ["PUT", "https://n.test/api/workflows/fkmw-xmxn-gagv-x", null, "{}"], + ["GET", "https://n.test/api/tags/a1"], + ["GET", "https://n.test/api/tags/b22"], + ["DELETE", "https://n.test/api/tags/cc", null], + ["GET", "https://n.test/api/labels/ab12cd34"], + ["GET", "https://n.test/api/labels/ef56gh78"], + ["DELETE", "https://n.test/api/labels/abcdefgh", null], + ]), + ); + expect(projected.tools.map(line)).toEqual([ + "DELETE /api/labels/{label_id}", + "GET /api/labels/{label_id}", + "DELETE /api/tags/cc", + "GET /api/tags/{tag_id}", + "PUT /api/workflows/fkmw-xmxn-gagv-x", + "DELETE /api/workflows/listForAccounts", + "GET /api/workflows/{workflow_id}", + "PATCH /api/workflows/{workflow_id}", + ]); +}); + +it("shares a literal with the parameter that observed it when parameters of different rules share a position", () => { + const projected = manifest( + ["opaque-token", "sibling-variance", "prefix-variance"], + syntheticCapture([ + ["GET", "https://i.test/api/items/aaaa12bb34cc/details"], + ["GET", "https://i.test/api/items/a1"], + ["GET", "https://i.test/api/items/b2"], + ["PATCH", "https://i.test/api/items/b2", null, "{}"], + ]), + ); + // The token parameter is earlier, but only the sibling parameter observed b2. + expect(projected.tools.map(line)).toEqual([ + "GET /api/items/{item_id}", + "PATCH /api/items/{item_id}", + "GET /api/items/{uid}/details", + ]); + expect( + projected.tools.map((tool) => + tool.params.map((param) => [param.name, param.rule]), + ), + ).toEqual([ + [["item_id", "sibling-variance"]], + [["item_id", "prefix-variance"]], + [["uid", "opaque-token"]], + ]); +}); + +it("templates a digit-free literal in a fixed-length family that one opt-in parameter observed once", () => { + const projected = manifest( + ["opaque-token", "prefix-variance"], + syntheticCapture([ + ["GET", "https://w.test/api/workflows/ydr9n16753tzr7zc"], + ["PATCH", "https://w.test/api/workflows/fkmwxmxngagvsxfg", null, "{}"], + ["DELETE", "https://w.test/api/workflows/notifications", null], + ]), + ); + expect(projected.tools.map(line)).toEqual([ + "DELETE /api/workflows/notifications", + "GET /api/workflows/{uid}", + "PATCH /api/workflows/{uid}", + ]); +}); diff --git a/src/domain/analysisView/environmentManifestView.ts b/src/domain/analysisView/environmentManifestView.ts index dd68558c4f..931839a95a 100644 --- a/src/domain/analysisView/environmentManifestView.ts +++ b/src/domain/analysisView/environmentManifestView.ts @@ -30,7 +30,7 @@ export const templateRulesSchema = z export const environmentManifestViewRequestSchema = z.strictObject({ kind: z.literal("environment-manifest"), template_rules: templateRulesSchema.describe( - "Opt-in template rules applied after the four format rules, each listed at most once: after-uid-literal templates the segment following a literal uid, opaque-token templates long mixed alphanumeric segments, sibling-variance merges groups that differ in one identifier-like segment, sibling-pair-variance merges groups that differ in two adjacent literals between a plural literal and another literal when at least three distinct pairs vary in both. Literal sibling folding into existing opt-in parameters also requires sibling-variance. Merged parameters are inferences.", + "Opt-in template rules applied after the four format rules, each listed at most once: after-uid-literal templates the segment following a literal uid, opaque-token templates long mixed alphanumeric segments, sibling-variance merges groups that differ in one identifier-like segment, sibling-pair-variance merges groups that differ in two adjacent literals between a plural literal and another literal when at least three distinct pairs vary in both, prefix-variance templates a literal at a position another tool templates by one of the other opt-in rules after the same preceding segments, at any method or length, when the literal is one of that parameter's observed values, carries a digit and has the length and character classes of one, or has the single length (eight or more characters) every observed value has and only their character classes. Literal sibling folding into existing opt-in parameters also requires sibling-variance. Merged parameters are inferences.", ), include_query_parameter_names: z .boolean() diff --git a/src/domain/analysisView/goBinaryView.ts b/src/domain/analysisView/goBinaryView.ts index bbc1c511a5..7e43774d29 100644 --- a/src/domain/analysisView/goBinaryView.ts +++ b/src/domain/analysisView/goBinaryView.ts @@ -22,6 +22,7 @@ import { } from "./analysisViewCoverage.js"; import { analysisViewInputError } from "./analysisViewErrors.js"; import { parseAnalysisViewResult } from "./analysisViewParent.js"; +import { selectViewRows } from "./selectViewRows.js"; const indexSchema = z.number().int().nonnegative(); const count = z.number().int().nonnegative(); @@ -123,23 +124,6 @@ const parseReport = ( return ok(report); }; -const selectRows = ( - rows: readonly Source[], - view: { readonly offset: number; readonly limit: number }, - matches: (row: Source, index: number) => boolean, - project: (row: Source, index: number) => Row, -): { rows: Row[]; total: number } => { - const selected: Row[] = []; - let total = 0; - for (const [index, row] of rows.entries()) { - if (!matches(row, index)) continue; - if (total >= view.offset && selected.length < view.limit) - selected.push(project(row, index)); - total++; - } - return { rows: selected, total }; -}; - /** Query the existing metadata owner's facts; never reopen or reinterpret a file. */ export const projectGoBinaryView = ( parent: AnalysisViewParent, @@ -259,7 +243,7 @@ export const projectGoBinaryView = ( view.fields === undefined ? undefined : new Set(view.fields); if (view.collection === "dependencies") { const paths = view.paths === undefined ? undefined : new Set(view.paths); - const selected = selectRows( + const selected = selectViewRows( info?.module.dependencies ?? [], view, (row, index) => @@ -287,7 +271,7 @@ export const projectGoBinaryView = ( }); } const keys = view.keys === undefined ? undefined : new Set(view.keys); - const selected = selectRows( + const selected = selectViewRows( info?.module.settings ?? [], view, (row, index) => diff --git a/src/domain/analysisView/rustBinaryView.test.ts b/src/domain/analysisView/rustBinaryView.test.ts new file mode 100644 index 0000000000..074ac8dd2f --- /dev/null +++ b/src/domain/analysisView/rustBinaryView.test.ts @@ -0,0 +1,173 @@ +import { expect, it } from "vitest"; +import { createEvidence } from "../evidence.js"; +import type { RustBinary } from "../rust/rustBinary.js"; +import { + projectAnalysisView, + type AnalysisViewParent, +} from "./analysisView.js"; + +const report: RustBinary = { + artifact: { path: "/artifacts/rust", sha256: "a".repeat(64), bytes: 512 }, + format: "elf", + architecture: "amd64", + bits: 64, + byte_order: "little", + auditable_metadata: { + section_name: ".dep-v0", + file_offset: 32, + section_bytes: 64, + compressed_bytes: 64, + uncompressed_bytes: 128, + format_version: null, + packages: [ + { + name: "root", + version: "1", + source: "local", + kind: null, + root: true, + dependencies: [0], + }, + ], + root_package_index: 0, + dependency_edges: [{ from_index: 0, to_index: 0 }], + cyclic_dependency_indices: [0], + }, + rustc_commit: null, + limitations: [], +}; +const parent = (value = structuredClone(report)): AnalysisViewParent => { + const evidence = createEvidence( + { + path: value.artifact.path, + sha256: value.artifact.sha256, + format: "elf", + architecture: "x86_64", + }, + { id: "rust-auditable", name: "Rust reader", version: "1" }, + { + operation: "inspect_rust_binary", + parameters: { path: value.artifact.path }, + result: value, + }, + ); + return { + evidenceId: evidence.evidence_id, + operation: evidence.operation, + subject: evidence.subject, + normalizedResult: value, + limitations: [], + }; +}; + +it("revalidates mutable and merely frozen Rust imports despite a claimed seal", () => { + const mutable = structuredClone(report); + const input = { ...parent(mutable), sealedResult: true }; + expect(projectAnalysisView(input, { kind: "summary" }).ok).toBe(true); + Object.freeze(mutable); + mutable.auditable_metadata!.dependency_edges[0]!.to_index = 99; + expect( + projectAnalysisView(input, { + kind: "rust", + collection: "edges", + offset: 0, + limit: 1, + }).ok, + ).toBe(false); +}); + +it("checks exact artifact identity and subject architecture for every Rust view", () => { + for (const changed of [ + { local_path: "/different" }, + { format: "pe" as const }, + { architecture: "arm64" as const }, + { digest: { sha256: "b".repeat(64) } }, + ]) { + const source = parent(); + if (source.subject == null || source.subject.format !== "elf") + throw new Error("Missing ELF source"); + const input = { ...source, subject: { ...source.subject, ...changed } }; + expect(projectAnalysisView(input, { kind: "summary" }).ok).toBe(false); + expect( + projectAnalysisView(input, { + kind: "rust", + collection: "packages", + offset: 0, + limit: 1, + }).ok, + ).toBe(false); + } +}); + +it("bounds inherited warnings and retains cycle counts without walking a cyclic graph", () => { + const input = { ...parent(), limitations: ["x".repeat(300_000)] }; + const result = projectAnalysisView(input, { kind: "summary" }); + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + summary: { + auditable_metadata: { + format_version: null, + counts: { packages: 1, edges: 1, cyclic_packages: 1 }, + }, + }, + source_coverage: { + embedded_records: "complete", + runtime_dependencies: "unknown", + }, + }); + expect(JSON.stringify(result.value).length).toBeLessThan(5000); + expect(result.value.limitations.join(" ")).toContain(input.evidenceId); +}); + +it("pages matching original indices and omits dependency vectors unless selected", () => { + const value = structuredClone(report); + value.auditable_metadata!.packages.push({ + name: "other", + version: "1", + source: "local", + kind: null, + root: false, + dependencies: [], + }); + const source = parent(value); + const first = projectAnalysisView(source, { + kind: "rust", + collection: "packages", + offset: 0, + limit: 1, + }); + if (!first.ok || first.value.kind !== "rust") + throw new Error("Rust page failed"); + expect(first.value.page.rows).toEqual([ + { + index: 0, + name: "root", + version: "1", + source: "local", + kind: null, + root: true, + }, + ]); + expect(first.value.coverage).toMatchObject({ + total: 2, + examined: 1, + exhausted: false, + next_offset: 1, + }); + const next = projectAnalysisView(source, { + kind: "rust", + collection: "packages", + offset: 1, + limit: 1, + fields: [], + }); + if (!next.ok || next.value.kind !== "rust") + throw new Error("Rust continuation failed"); + expect(next.value.page.rows).toEqual([{ index: 1 }]); + expect(next.value.coverage).toMatchObject({ + total: 2, + examined: 1, + exhausted: true, + next_offset: null, + }); +}); diff --git a/src/domain/analysisView/rustBinaryView.ts b/src/domain/analysisView/rustBinaryView.ts new file mode 100644 index 0000000000..e192c245b6 --- /dev/null +++ b/src/domain/analysisView/rustBinaryView.ts @@ -0,0 +1,277 @@ +import { z } from "zod"; +import type { AnalysisError } from "../analysisErrorBase.js"; +import { + rustArchitectureForSubject, + rustBinarySchema, + rustAuditableMetadataSchema, + rustcCommitSchema, + type RustBinary, +} from "../rust/rustBinary.js"; +export { + rustBinaryViewPageSchema, + rustBinaryViewRequestSchema, +} from "../rust/rustBinaryQuery.js"; +import { isImmutableJsonSnapshot } from "../immutableJson.js"; +import { err, ok, type Result } from "../result.js"; +import type { + AnalysisViewParent, + AnalysisViewRequest, + UnsignedAnalysisView, +} from "./analysisView.js"; +import { + boundedViewMessages, + completeWithinViewCoverage, + pageViewCoverage, +} from "./analysisViewCoverage.js"; +import { analysisViewInputError } from "./analysisViewErrors.js"; +import { parseAnalysisViewResult } from "./analysisViewParent.js"; +import { selectViewRows } from "./selectViewRows.js"; + +const count = z.number().int().nonnegative(); +export const rustBinarySummarySchema = z.strictObject({ + bytes: count, + format: rustBinarySchema.shape.format, + architecture: rustBinarySchema.shape.architecture, + bits: rustBinarySchema.shape.bits, + byte_order: rustBinarySchema.shape.byte_order, + auditable_metadata: rustAuditableMetadataSchema + .omit({ + packages: true, + dependency_edges: true, + cyclic_dependency_indices: true, + }) + .extend({ + counts: z.strictObject({ + packages: count, + edges: count, + cyclic_packages: count, + }), + }) + .nullable(), + rustc_commit: rustcCommitSchema + .omit({ hashes: true }) + .extend({ hashes_count: count }) + .nullable(), +}); +export const rustBinaryViewContextSchema = z.strictObject({ + source_coverage: z.strictObject({ + auditable_metadata_available: z.boolean(), + embedded_records: z.enum(["complete", "unknown"]), + runtime_dependencies: z.literal("unknown"), + }), +}); + +// Exact immutable Evidence identity, not a path or a live-file result cache. +// Mutable/merely frozen imports are validated normally; no failures are cached. +const checkedReports = new WeakMap(); +const parseReport = ( + parent: AnalysisViewParent, +): Result => { + const key = parent.normalizedResult; + const reusable = + parent.sealedResult === true && + typeof key === "object" && + key !== null && + isImmutableJsonSnapshot(key); + const known = reusable ? checkedReports.get(key) : undefined; + if (known !== undefined) return ok(known); + const parsed = parseAnalysisViewResult(rustBinarySchema, parent); + if (!parsed.ok) return parsed; + const report = reusable ? (key as RustBinary) : parsed.value; + if (reusable) checkedReports.set(key, report); + return ok(report); +}; + +const summarize = ( + report: RustBinary, +): z.infer => { + const { + artifact, + auditable_metadata: metadata, + rustc_commit: compiler, + } = report; + const section = + metadata === null + ? null + : { + section_name: metadata.section_name, + file_offset: metadata.file_offset, + section_bytes: metadata.section_bytes, + compressed_bytes: metadata.compressed_bytes, + uncompressed_bytes: metadata.uncompressed_bytes, + format_version: metadata.format_version, + root_package_index: metadata.root_package_index, + counts: { + packages: metadata.packages.length, + edges: metadata.dependency_edges.length, + cyclic_packages: metadata.cyclic_dependency_indices.length, + }, + }; + return { + format: report.format, + architecture: report.architecture, + bits: report.bits, + byte_order: report.byte_order, + bytes: artifact.bytes, + auditable_metadata: section, + rustc_commit: + compiler === null + ? null + : { + evidence_kind: compiler.evidence_kind, + hashes_count: compiler.hashes.length, + }, + }; +}; + +/** Query owner-normalized facts without reopening a source or walking dependency paths. */ +export const projectRustBinaryView = ( + parent: AnalysisViewParent, + view: AnalysisViewRequest, +): Result => { + if (view.kind !== "summary" && view.kind !== "rust") + return err( + analysisViewInputError([ + { + path: ["view"], + reason: "invalid_value", + message: "Rust Evidence supports summary and rust views.", + }, + ]), + ); + const parsed = parseReport(parent); + if (!parsed.ok) return parsed; + const report = parsed.value; + const subject = parent.subject; + if ( + subject == null || + subject.local_path !== report.artifact.path || + subject.digest.sha256 !== report.artifact.sha256 || + subject.format !== (report.format === "macho" ? "mach-o" : report.format) || + subject.architecture !== + (rustArchitectureForSubject(report.architecture) ?? null) + ) + return err( + analysisViewInputError([ + { + path: ["source", "subject"], + reason: "invalid_value", + message: + "Rust parent subject must identify the report's exact artifact.", + }, + ]), + ); + const metadata = report.auditable_metadata; + const unknown = + "Embedded dependency records do not authenticate build provenance or establish complete runtime dependencies or target behavior."; + const shared = { + parent_evidence_id: parent.evidenceId, + parent_operation: "inspect_rust_binary" as const, + parent_digest: parent.evidenceId.slice(3), + artifact: { path: report.artifact.path, sha256: report.artifact.sha256 }, + source_coverage: { + auditable_metadata_available: metadata !== null, + embedded_records: + metadata === null ? ("unknown" as const) : ("complete" as const), + runtime_dependencies: "unknown" as const, + }, + limitations: boundedViewMessages( + [...parent.limitations, ...report.limitations], + parent.evidenceId, + "limitations", + ), + unknowns: boundedViewMessages( + [ + unknown, + ...(metadata === null + ? [ + "No cargo-auditable metadata was observed; missing metadata does not establish an empty dependency set.", + ] + : []), + ], + parent.evidenceId, + "unknowns", + [unknown], + ), + }; + const summary = summarize(report); + if (view.kind === "summary") + return ok({ + kind: "summary", + view, + ...shared, + summary, + coverage: completeWithinViewCoverage(1, 1), + }); + const indices = + view.indices === undefined ? undefined : new Set(view.indices); + const fields = + view.fields === undefined ? undefined : new Set(view.fields); + if (view.collection === "packages") { + const names = view.names === undefined ? undefined : new Set(view.names); + const versions = + view.versions === undefined ? undefined : new Set(view.versions); + const selected = selectViewRows( + metadata?.packages ?? [], + view, + (row, index) => + (indices?.has(index) ?? true) && + (names?.has(row.name) ?? true) && + (versions?.has(row.version) ?? true), + (row, index) => ({ + index, + ...((fields?.has("name") ?? true) && { name: row.name }), + ...((fields?.has("version") ?? true) && { version: row.version }), + ...((fields?.has("source") ?? true) && { source: row.source }), + ...((fields?.has("kind") ?? true) && { kind: row.kind }), + ...((fields?.has("root") ?? true) && { root: row.root }), + ...(fields?.has("dependencies") === true && { + dependencies: row.dependencies, + }), + }), + ); + return ok({ + kind: "rust", + view, + ...shared, + rust: summary, + page: { collection: "packages", rows: selected.rows }, + coverage: pageViewCoverage( + view.offset, + selected.rows.length, + selected.total, + ), + }); + } + const from = + view.from_indices === undefined ? undefined : new Set(view.from_indices); + const to = + view.to_indices === undefined ? undefined : new Set(view.to_indices); + const selected = selectViewRows( + metadata?.dependency_edges ?? [], + view, + (row, index) => + (indices?.has(index) ?? true) && + (from?.has(row.from_index) ?? true) && + (to?.has(row.to_index) ?? true), + (row, index) => ({ + index, + ...((fields?.has("from_index") ?? true) && { + from_index: row.from_index, + }), + ...((fields?.has("to_index") ?? true) && { to_index: row.to_index }), + }), + ); + return ok({ + kind: "rust", + view, + ...shared, + rust: summary, + page: { collection: "edges", rows: selected.rows }, + coverage: pageViewCoverage( + view.offset, + selected.rows.length, + selected.total, + ), + }); +}; diff --git a/src/domain/analysisView/selectViewRows.ts b/src/domain/analysisView/selectViewRows.ts new file mode 100644 index 0000000000..53ec6f34a4 --- /dev/null +++ b/src/domain/analysisView/selectViewRows.ts @@ -0,0 +1,17 @@ +/** Preserve original indices and count every match while projecting only the requested page. */ +export const selectViewRows = ( + rows: readonly Source[], + view: { readonly offset: number; readonly limit: number }, + matches: (row: Source, index: number) => boolean, + project: (row: Source, index: number) => Row, +): { rows: Row[]; total: number } => { + const selected: Row[] = []; + let total = 0; + for (const [index, row] of rows.entries()) { + if (!matches(row, index)) continue; + if (total >= view.offset && selected.length < view.limit) + selected.push(project(row, index)); + total++; + } + return { rows: selected, total }; +}; diff --git a/src/domain/apple/artifactTargetApplicability.ts b/src/domain/apple/artifactTargetApplicability.ts new file mode 100644 index 0000000000..38a9412dfd --- /dev/null +++ b/src/domain/apple/artifactTargetApplicability.ts @@ -0,0 +1,46 @@ +import { isMachOFormat, type BinaryTarget } from "../binaryTargetTypes.js"; + +interface ArtifactTargetRefusal { + readonly code: "target_kind_unsupported" | "target_format_unsupported"; + readonly reason: string; +} + +/** Target prerequisites shared by artifact execution and session discovery. */ +export const artifactTargetRefusal = ( + operation: string, + target: BinaryTarget, +): ArtifactTargetRefusal | undefined => { + const bundle = + target.kind === "executable" && target.bundleInfoPlist !== undefined; + const refuse = (reason: string): ArtifactTargetRefusal => ({ + code: "target_kind_unsupported", + reason, + }); + switch (operation) { + case "decode_interface_builder": + case "inspect_asset_catalog": + return bundle + ? undefined + : refuse(`${operation} requires an active .app bundle target`); + case "inspect_keyed_archive": + return bundle || (target.kind === "artifact" && target.format === "plist") + ? undefined + : refuse( + "inspect_keyed_archive requires an active plist or .app bundle", + ); + case "trace_dylib_resolution": + if (target.kind !== "executable") + return refuse( + "trace_dylib_resolution requires an active Mach-O or .app bundle target", + ); + return isMachOFormat(target.format) + ? undefined + : { + code: "target_format_unsupported", + reason: + "trace_dylib_resolution requires an active Mach-O or .app bundle target", + }; + default: + return undefined; + } +}; diff --git a/src/domain/apple/dylibResolution.test.ts b/src/domain/apple/dylibResolution.test.ts index dd4c18df28..c143738225 100644 --- a/src/domain/apple/dylibResolution.test.ts +++ b/src/domain/apple/dylibResolution.test.ts @@ -398,6 +398,9 @@ describe("dyld resolution outcomes: checking candidate format and compatibility" ({ outcome }) => outcome, ), ).toEqual(["not-mach-o", "malformed", "architecture-missing"]); + expect(trace.findings[0]?.explanation).toContain( + "No usable candidate for @rpath/lib.dylib exists in the analyzed root; found Contents/MacOS/a/lib.dylib (not a Mach-O image), Contents/MacOS/b/lib.dylib (malformed), Contents/MacOS/c/lib.dylib (missing the process architecture)", + ); expect(trace.coverage).toEqual({ status: "partial", unparsed_images: ["Contents/MacOS/b/lib.dylib"], @@ -441,26 +444,6 @@ describe("dyld resolution outcomes: checking candidate format and compatibility" }); }); - it("preserves unknown platform compatibility before a later known candidate", async () => { - const trace = await traceDylibLoading( - memoryView({ - [MAIN]: executable({ - rpaths: ["@executable_path/a", "@executable_path/b"], - dependencies: [dependency("@rpath/lib.dylib")], - }), - "Contents/MacOS/a/lib.dylib": parsed(slice({ platform: null })), - "Contents/MacOS/b/lib.dylib": parsed(slice()), - }), - { roots: [MAIN] }, - ); - const edge = edgeFor(trace, MAIN, "@rpath/lib.dylib"); - expect(edge.candidates.map(({ outcome }) => outcome)).toEqual([ - "undetermined", - "resolved", - ]); - expect(edge.resolution.status).toBe("conditional"); - }); - it("applies dyld Catalyst and zippered platform compatibility", async () => { const catalystProcess = await traceDylibLoading( memoryView({ @@ -506,6 +489,104 @@ describe("dyld resolution outcomes: checking candidate format and compatibility" }); }); +describe("platform-less dylib compatibility", () => { + it("keeps an unknown candidate unresolved and traces the later known fallback", async () => { + const trace = await traceDylibLoading( + memoryView({ + [MAIN]: executable({ + platform: 6, + platforms: [6], + rpaths: ["@executable_path/a", "@executable_path/b"], + dependencies: [dependency("@rpath/lib.dylib")], + }), + "Contents/MacOS/a/lib.dylib": parsed(slice({ platform: null })), + "Contents/MacOS/b/lib.dylib": parsed( + slice({ + platform: 6, + platforms: [6], + dependencies: [dependency("@loader_path/libChild.dylib")], + }), + ), + "Contents/MacOS/b/libChild.dylib": parsed( + slice({ platform: 6, platforms: [6] }), + ), + }), + { roots: [MAIN] }, + ); + const edge = edgeFor(trace, MAIN, "@rpath/lib.dylib"); + expect(edge.candidates.map(({ outcome }) => outcome)).toEqual([ + "undetermined", + "resolved", + ]); + expect(edge.resolution).toEqual({ + status: "conditional", + image: "Contents/MacOS/b/lib.dylib", + }); + expect(edge.candidates[0]).not.toHaveProperty("platform_basis"); + expect(trace.edges.map(({ loader }) => loader)).toContain( + "Contents/MacOS/b/lib.dylib", + ); + expect(trace.images.map(({ path }) => path)).toContain( + "Contents/MacOS/b/libChild.dylib", + ); + expect(trace.coverage.status).toBe("partial"); + expect(trace.limitations).toContainEqual( + expect.stringContaining("Contents/MacOS/a/lib.dylib"), + ); + }); + + it("does not infer a platform-less image's platform when process platform is unknown", async () => { + const trace = await traceDylibLoading( + memoryView({ + [MAIN]: executable({ + platform: null, + platforms: [], + rpaths: ["@executable_path/a"], + dependencies: [dependency("@rpath/lib.dylib")], + }), + "Contents/MacOS/a/lib.dylib": parsed(slice({ platform: null })), + }), + { roots: [MAIN] }, + ); + const edge = edgeFor(trace, MAIN, "@rpath/lib.dylib"); + expect(edge.candidates[0]).toMatchObject({ + outcome: "undetermined", + resolved_path: "Contents/MacOS/a/lib.dylib", + }); + expect(edge.candidates[0]).not.toHaveProperty("platform_basis"); + expect(edge.resolution).toEqual({ status: "undetermined", image: null }); + expect(trace.coverage.status).toBe("partial"); + }); + + it("does not treat a selected dylib root as an executable process context", async () => { + const library = "Contents/Frameworks/libRoot.dylib"; + const child = "Contents/Frameworks/libChild.dylib"; + const trace = await traceDylibLoading( + memoryView({ + [library]: parsed( + slice({ + file_type: "dylib", + file_type_code: 6, + platform: 1, + platforms: [1], + dependencies: [dependency("@loader_path/libChild.dylib")], + }), + ), + [child]: parsed(slice({ platform: null })), + }), + { roots: [library] }, + ); + const edge = edgeFor(trace, library, "@loader_path/libChild.dylib"); + expect(edge.candidates[0]).toMatchObject({ + outcome: "undetermined", + resolved_path: child, + }); + expect(edge.candidates[0]).not.toHaveProperty("platform_basis"); + expect(edge.resolution).toEqual({ status: "undetermined", image: null }); + expect(trace.coverage.status).toBe("partial"); + }); +}); + describe("dyld load order", () => { it("loads each image once per process and reuses matching install names", async () => { const a = "Contents/Frameworks/libA.dylib"; diff --git a/src/domain/apple/dylibResolution.ts b/src/domain/apple/dylibResolution.ts index 1adfdc79fc..3fdf5a518c 100644 --- a/src/domain/apple/dylibResolution.ts +++ b/src/domain/apple/dylibResolution.ts @@ -51,6 +51,13 @@ const dependencyCommandSchema = z.enum([ "LC_LOAD_UPWARD_DYLIB", ]); +const IMPLICIT_MACOS_PLATFORM = 1; +const MACOS_DYLD_IMPLICIT_PLATFORM_CPU_TYPES = new Set([ + 7, // CPU_TYPE_I386 + 0x01000007, // CPU_TYPE_X86_64 + 0x0100000c, // CPU_TYPE_ARM64, including arm64 subtypes +]); + const dependencySchema = z.strictObject({ command: dependencyCommandSchema, encoding: z.enum([ @@ -145,6 +152,7 @@ const candidateSchema = z.strictObject({ "undetermined", ]), resolved_path: z.string().nullable(), + platform_basis: z.enum(["macos-dyld-implicit"]).optional(), }); const edgeSchema = z.strictObject({ @@ -227,7 +235,7 @@ interface LoadedImage { readonly slice: MachoSlice; /** Images from the process root to this image; the rpath stack, outermost first. */ readonly chain: readonly string[]; - /** Some edge on the chain was conditional: an unknown earlier candidate may load instead. */ + /** Some edge on the chain has unresolved candidate or platform facts. */ readonly conditional: boolean; } @@ -240,6 +248,8 @@ interface ProcessContext { >; readonly searchOverrides: readonly DyldSearchOverride[]; readonly environmentRoots: Set; + readonly unknownPlatformCandidates: Set; + readonly implicitMacOSPlatformCandidates: Set; readonly platforms: readonly number[]; readonly executable: string | null; readonly loaded: Map; @@ -346,47 +356,75 @@ const evaluateCandidate = async ( // also admits main executables and bundles for different operations. else if (slice.file_type_code !== 6) outcome = "not-loadable"; else { - const platformResult = platformLoadability( - context.platforms, - slice.platforms, - slice.file_type_code, - context.processIdentity.architecture, - ); + const platformResult = platformLoadability(context, slice); outcome = - platformResult === "compatible" + platformResult.outcome === "compatible" ? "resolved" - : platformResult === "incompatible" + : platformResult.outcome === "incompatible" ? "platform-mismatch" : "undetermined"; + if (outcome === "undetermined") + context.unknownPlatformCandidates.add(lookup.path); + if (platformResult.basis !== undefined) { + context.implicitMacOSPlatformCandidates.add(lookup.path); + return { + ...base, + outcome, + resolved_path: lookup.path, + platform_basis: platformResult.basis, + }; + } } } return { ...base, outcome, resolved_path: lookup.path }; }; type PlatformLoadability = "compatible" | "incompatible" | "unknown"; +type PlatformLoadabilityResult = { + readonly outcome: PlatformLoadability; + readonly basis?: Candidate["platform_basis"]; +}; /** Apply dyld's documented cross-platform load cases and retain uncertain host cases. */ const platformLoadability = ( - processPlatforms: readonly number[], - imagePlatforms: readonly number[], - fileType: number, - architecture: string, -): PlatformLoadability => { - if (processPlatforms.length === 0 || imagePlatforms.length === 0) - return "unknown"; + context: Pick, + image: MachoSlice, +): PlatformLoadabilityResult => { + const { platforms: processPlatforms } = context; + const { file_type_code: fileType, platforms: imagePlatforms } = image; + let effectiveImagePlatforms = imagePlatforms; + let basis: Candidate["platform_basis"]; + if (imagePlatforms.length === 0) { + // macOS dyld supplies an implicit macOS platform to old dylibs on these + // CPU types. Keep the parsed platform facts untouched; this is only the + // compatibility interpretation used for an explicitly macOS process. + if ( + context.executable !== null && + processPlatforms.length === 1 && + processPlatforms[0] === IMPLICIT_MACOS_PLATFORM && + MACOS_DYLD_IMPLICIT_PLATFORM_CPU_TYPES.has(image.cpu_type) + ) { + effectiveImagePlatforms = [IMPLICIT_MACOS_PLATFORM]; + basis = "macos-dyld-implicit"; + } else return { outcome: "unknown" }; + } + if (processPlatforms.length === 0) return { outcome: "unknown" }; const outcomes = processPlatforms.map((processPlatform) => imageLoadabilityForPlatform( processPlatform, - imagePlatforms, + effectiveImagePlatforms, fileType, - architecture, + image.architecture, ), ); if (outcomes.every((outcome) => outcome === "compatible")) - return "compatible"; + return { outcome: "compatible", ...(basis === undefined ? {} : { basis }) }; if (outcomes.every((outcome) => outcome === "incompatible")) - return "incompatible"; - return "unknown"; + return { + outcome: "incompatible", + ...(basis === undefined ? {} : { basis }), + }; + return { outcome: "unknown", ...(basis === undefined ? {} : { basis }) }; }; const imageLoadabilityForPlatform = ( @@ -572,7 +610,7 @@ const traceProcess = async ( context.loaded.set(image, { slice, chain: [...current.chain, image], - // dyld may load an unknown earlier candidate instead of this fallback. + // A modeled earlier search uncertainty can make this image conditional. conditional: current.conditional || edge.resolution.status === "conditional", }); @@ -589,6 +627,69 @@ const traceProcess = async ( return edges; }; +const projectTraceResult = (input: { + readonly images: ReadonlyMap; + readonly roots: DylibTrace["roots"]; + readonly rootSlices: readonly TracedRootSlice[]; + readonly edges: readonly Edge[]; + readonly withoutArchitecture: readonly string[]; + readonly environmentRoots: ReadonlySet; + readonly unknownPlatformCandidates: ReadonlySet; + readonly implicitMacOSPlatformCandidates: ReadonlySet; +}): DylibTrace => { + const reported = [...input.images.entries()] + .filter(([, facts]) => facts.status !== "not-mach-o") + .sort(([left], [right]) => compare(left, right)); + const unparsed = reported + .filter(([, facts]) => facts.status !== "parsed") + .map(([path]) => path); + return { + roots: input.roots, + images: reported.map(([path, facts]) => ({ + path, + parse_status: + facts.status === "not-mach-o" ? "unsupported" : facts.status, + reason: + facts.status === "parsed" || facts.status === "not-mach-o" + ? null + : facts.reason, + slices: + facts.status === "parsed" ? facts.slices.map(withoutDependencies) : [], + })), + edges: [...input.edges], + findings: deriveFindings(input.edges, input.rootSlices), + coverage: { + status: + unparsed.length === 0 && + input.withoutArchitecture.length === 0 && + input.environmentRoots.size === 0 && + input.unknownPlatformCandidates.size === 0 + ? "complete" + : "partial", + unparsed_images: unparsed, + roots_without_architecture: [...input.withoutArchitecture], + }, + limitations: [ + ...DYLIB_RESOLUTION_LIMITATIONS, + ...(input.environmentRoots.size === 0 + ? [] + : [ + `Roots ${[...input.environmentRoots].join(", ")} request applicable image-selection inputs through LC_DYLD_ENVIRONMENT that are not modeled; affected dependencies or inserted images remain uncertain.`, + ]), + ...(input.unknownPlatformCandidates.size === 0 + ? [] + : [ + `In-root dylib candidates ${[...input.unknownPlatformCandidates].sort(compare).join(", ")} have unknown platform compatibility and are not treated as incompatible. Their dependent graphs may be incomplete.`, + ]), + ...(input.implicitMacOSPlatformCandidates.size === 0 + ? [] + : [ + `In-root dylib candidates ${[...input.implicitMacOSPlatformCandidates].sort(compare).join(", ")} have no observed platform load command; macOS compatibility was derived from dyld's implicit-platform rule for an executable root with explicit macOS platform and a supported CPU type. Reported platform facts remain unchanged.`, + ]), + ], + }; +}; + /** Trace dyld load paths from each process root through the analyzed root. */ export const traceDylibLoading = async ( view: DylibTreeView, @@ -608,6 +709,8 @@ export const traceDylibLoading = async ( const edges: Edge[] = []; const withoutArchitecture: string[] = []; const environmentRoots = new Set(); + const unknownPlatformCandidates = new Set(); + const implicitMacOSPlatformCandidates = new Set(); for (const root of request.roots) { const facts = await view.image(root); images.set(root, facts); @@ -647,6 +750,8 @@ export const traceDylibLoading = async ( }, searchOverrides, environmentRoots, + unknownPlatformCandidates, + implicitMacOSPlatformCandidates, platforms: slice.platforms, executable, loaded: new Map(), @@ -659,46 +764,16 @@ export const traceDylibLoading = async ( ); } } - const reported = [...images.entries()] - .filter(([, facts]) => facts.status !== "not-mach-o") - .sort(([left], [right]) => compare(left, right)); - const unparsed = reported - .filter(([, facts]) => facts.status !== "parsed") - .map(([path]) => path); - return { + return projectTraceResult({ + images, roots, - images: reported.map(([path, facts]) => ({ - path, - parse_status: - facts.status === "not-mach-o" ? "unsupported" : facts.status, - reason: - facts.status === "parsed" || facts.status === "not-mach-o" - ? null - : facts.reason, - slices: - facts.status === "parsed" ? facts.slices.map(withoutDependencies) : [], - })), + rootSlices, edges, - findings: deriveFindings(edges, rootSlices), - coverage: { - status: - unparsed.length === 0 && - withoutArchitecture.length === 0 && - environmentRoots.size === 0 - ? "complete" - : "partial", - unparsed_images: unparsed, - roots_without_architecture: withoutArchitecture, - }, - limitations: [ - ...DYLIB_RESOLUTION_LIMITATIONS, - ...(environmentRoots.size === 0 - ? [] - : [ - `Roots ${[...environmentRoots].join(", ")} request applicable image-selection inputs through LC_DYLD_ENVIRONMENT that are not modeled; affected dependencies or inserted images remain uncertain.`, - ]), - ], - }; + withoutArchitecture, + environmentRoots, + unknownPlatformCandidates, + implicitMacOSPlatformCandidates, + }); }; const withoutDependencies = ( diff --git a/src/domain/apple/dylibResolutionFindings.ts b/src/domain/apple/dylibResolutionFindings.ts index 7783ab91db..c7f37bb935 100644 --- a/src/domain/apple/dylibResolutionFindings.ts +++ b/src/domain/apple/dylibResolutionFindings.ts @@ -70,7 +70,10 @@ const unresolvedFinding = (edge: Edge, index: number): Finding => { basis: "derived", } as const; const lazy = isLazyLoadCommand(edge.command); - const missing = `No candidate for ${dependencyPhrase(edge, lazy)} exists in the analyzed root`; + const missing = missingCandidateExplanation( + edge, + dependencyPhrase(edge, lazy), + ); // Below a conditional fallback, every consequence applies only if it loads. const when = edge.loader_conditional ? `. ${edge.loader} loads only conditionally; if it loads, ` @@ -94,6 +97,52 @@ const unresolvedFinding = (edge: Edge, index: number): Finding => { }; }; +/** Use the resolver's physical-candidate facts without rechecking source paths. */ +const missingCandidateExplanation = ( + edge: Edge, + dependency: string, +): string => { + const present = edge.candidates.flatMap(({ outcome, resolved_path }) => + resolved_path === null ? [] : [{ outcome, path: resolved_path }], + ); + if (present.length === 0) + return `No candidate for ${dependency} exists in the analyzed root`; + const descriptions = present.map( + ({ outcome, path }) => `${path} (${candidateRejection(outcome)})`, + ); + return `No usable candidate for ${dependency} exists in the analyzed root; found ${descriptions.join(", ")}`; +}; + +/** Explain a found candidate from the resolver's outcome, not by reopening it. */ +const candidateRejection = ( + outcome: Edge["candidates"][number]["outcome"], +): string => { + switch (outcome) { + case "not-mach-o": + return "not a Mach-O image"; + case "malformed": + return "malformed"; + case "architecture-missing": + return "missing the process architecture"; + case "not-loadable": + return "not a loadable dylib"; + case "platform-mismatch": + return "incompatible with the process platform"; + case "unsupported": + return "unsupported by the parser"; + case "resolved": + return "loadable"; + case "absent": + return "absent"; + case "outside-target": + return "outside the analyzed root"; + case "escapes-target": + return "escapes the analyzed root"; + case "undetermined": + return "of undetermined loadability"; + } +}; + /** Name the dependency the way the finding's consequence treats it. */ const dependencyPhrase = (edge: Edge, lazy: boolean): string => { if (lazy && edge.weak) diff --git a/src/domain/apple/plistValue.ts b/src/domain/apple/plistValue.ts index 6f406cc833..9459f0bbc7 100644 --- a/src/domain/apple/plistValue.ts +++ b/src/domain/apple/plistValue.ts @@ -28,6 +28,10 @@ export interface ProjectedPlistValue { export const unknownDateLimitation = (count: number): string => `${String(count)} date value(s) are reported as { "$plist_type": "date", "iso8601": null } because the decoder could not represent their timestamps.`; +/** Limitation when shared binary bytes prevent safely normalizing date precision. */ +export const binaryDatePrecisionOverlapLimitation = (count: number): string => + `${String(count)} date value(s) are reported with unknown timestamps because their sub-millisecond source records overlap another reachable binary object and the public date projection cannot express their exact instants.`; + /** Limitation for XML scalar literals whose values were not established. */ export const unknownScalarLimitation = (count: number): string => `${String(count)} XML scalar value(s) could not be established; their exact literals and reasons are retained in { "$plist_type": "unknown", "element": ..., "literal": ..., "reason": ... } wrappers.`; diff --git a/src/domain/artifactFormatMetadata.ts b/src/domain/artifactFormatMetadata.ts index d31b936521..4ba7f6475d 100644 --- a/src/domain/artifactFormatMetadata.ts +++ b/src/domain/artifactFormatMetadata.ts @@ -29,9 +29,70 @@ export const godotPckEntryMetadataSchema = z.strictObject({ flags: uint32, md5_status: z.enum(["declared", "all-zero"]), }); + +/** Source cookie fields and derived directory counts without Python execution. */ +export const pyInstallerMetadataSchema = z.strictObject({ + kind: z.literal("pyinstaller"), + layout_version: z.enum(["2.0", "2.1"]), + archive_start: offset, + archive_size: offset, + cookie_offset: offset, + toc_offset: offset, + toc_size: offset, + entry_count: offset, + runtime_option_count: offset, + external_dependency_count: offset, + python_version_raw: uint32, + python_major: uint32.nullable(), + python_minor: uint32.nullable(), + python_library: z.string().nullable(), + encryption_marker_present: z.boolean(), +}); +export type PyInstallerMetadata = z.infer; + +/** Primitive facts preserve source spelling and keep metadata copies independent. */ +export const pyInstallerEntryMetadataSchema = z.strictObject({ + kind: z.literal("pyinstaller-entry"), + container: z.enum(["carchive", "pyz"]), + container_name: z.string().nullable(), + index: offset, + source_name: z.string(), + type_code: z.union([z.string().length(1), uint32]), + relative_offset: offset, + directory_record_offset: offset.nullable(), + directory_record_size: offset.nullable(), + compression_flag: z.union([z.literal(0), z.literal(1)]), + bytecode_header: z.enum(["not-bytecode", "unknown", "derived"]), + bytecode_magic: z + .string() + .regex(/^[0-9a-f]{8}$/u) + .nullable(), + pyz_status: z.enum(["not-pyz", "decoded", "opaque"]), + pyz_sha256: z + .string() + .regex(/^[0-9a-f]{64}$/u) + .nullable(), + pyz_python_magic: z + .string() + .regex(/^[0-9a-f]{8}$/u) + .nullable(), + pyz_python_version: z.string().nullable(), + pyz_entry_count: offset.nullable(), + encryption_source: z.literal("carchive-marker").nullable(), + dependency_status: z + .enum(["unresolved", "invalid-reference"]) + .exactOptional(), + dependency_archive: z.string().nullable().exactOptional(), + dependency_member: z.string().nullable().exactOptional(), +}); +export type PyInstallerEntryMetadata = z.infer< + typeof pyInstallerEntryMetadataSchema +>; export const artifactFormatMetadataSchema = z.discriminatedUnion("kind", [ godotPckMetadataSchema, godotPckEntryMetadataSchema, + pyInstallerMetadataSchema, + pyInstallerEntryMetadataSchema, ]); export type ArtifactFormatMetadata = z.infer< typeof artifactFormatMetadataSchema diff --git a/src/domain/artifactGraph.ts b/src/domain/artifactGraph.ts index 91c97d6930..61d0be7717 100644 --- a/src/domain/artifactGraph.ts +++ b/src/domain/artifactGraph.ts @@ -83,6 +83,7 @@ const artifactFormatSchema = z.enum([ "entitlements", "directory", "file", + "pyinstaller", "unknown", ]); @@ -138,7 +139,7 @@ export const artifactOccurrenceSchema = z.object({ artifact_id: artifactIdSchema.nullable(), parent_occurrence_id: occurrenceIdSchema.nullable(), logical_path: boundedRelativePathSchema, - entry_kind: z.enum(["file", "directory", "symlink", "slice"]), + entry_kind: z.enum(["file", "directory", "symlink", "slice", "metadata"]), artifact_kind: artifactKindSchema, artifact_format: artifactFormatSchema, declared_size: z diff --git a/src/domain/artifactInventoryEvidence.ts b/src/domain/artifactInventoryEvidence.ts index 395915c69a..c7fcc3ec13 100644 --- a/src/domain/artifactInventoryEvidence.ts +++ b/src/domain/artifactInventoryEvidence.ts @@ -348,19 +348,20 @@ const orderOccurrences = ( .flatMap(([path, group]) => { if (group.length < 2) return group; const ordinalById = new Map(); - const first = group[0]; - if (first === undefined) - throw new TypeError("Duplicate occurrence group is empty"); + const kinds = new Set(group.map(({ entry_kind }) => entry_kind)); for (let ordinal = 0; ordinal < group.length; ordinal++) { - ordinalById.set( - occurrenceIdForLocation({ - rootArtifactId, - logicalPath: path, - entryKind: first.entry_kind, - ...(ordinal === 0 ? {} : { duplicateOrdinal: ordinal }), - }), - ordinal, - ); + // Entry kinds come from a fixed schema enum, so work stays linear in + // the group size without changing or guessing producer identities. + for (const entryKind of kinds) + ordinalById.set( + occurrenceIdForLocation({ + rootArtifactId, + logicalPath: path, + entryKind, + ...(ordinal === 0 ? {} : { duplicateOrdinal: ordinal }), + }), + ordinal, + ); } return group.sort( (left, right) => diff --git a/src/domain/browserObservationError.ts b/src/domain/browserObservationError.ts index a519900b79..261ad704b3 100644 --- a/src/domain/browserObservationError.ts +++ b/src/domain/browserObservationError.ts @@ -51,6 +51,8 @@ const observationFailureMessage = ( return `${operation} could not reach the selected loopback debugging endpoint. Verify that the target process is running with debugging enabled and that the endpoint's port matches it, then retry.`; if (reason === "target_not_found") return `${operation} could not find the selected target at the debugging endpoint. Refresh target discovery for that endpoint and retry with a returned target ID.`; + if (reason === "target_session_detached") + return `${operation} lost its debugging session while the target may still exist. Refresh target discovery and attach to the selected target again.`; if (reason === "service_worker_blocked") return `${operation} refused the selected page because it is controlled by or has a registered service worker while the service-worker policy is block. Set environment.service_workers to allow to permit observation, or remove the service worker registration through the browser.`; if (reason === "disconnected") diff --git a/src/domain/browserObservationErrors.ts b/src/domain/browserObservationErrors.ts index 89b2b73e0b..3027553f51 100644 --- a/src/domain/browserObservationErrors.ts +++ b/src/domain/browserObservationErrors.ts @@ -26,6 +26,7 @@ export type BrowserObservationFailureReason = | "target_not_allowed" | "service_worker_blocked" | "target_changed" + | "target_session_detached" | "protocol_error" | "disconnected" | "payload_limit" diff --git a/src/domain/browserSession.ts b/src/domain/browserSession.ts index d36737795c..a8c7b83390 100644 --- a/src/domain/browserSession.ts +++ b/src/domain/browserSession.ts @@ -29,6 +29,8 @@ const timelineEventSchema = z.object({ "load_failed", "lifecycle", "target_terminated", + "session_detached", + "transport_disconnected", ]), timestamp: z .number() @@ -63,6 +65,8 @@ export const webObservationSessionSchema = z.object({ "window_elapsed", "target_left_scope", "target_terminated", + "session_detached", + "transport_disconnected", ]), }), timeline: z.array(timelineEventSchema), diff --git a/src/domain/dosMz.test.ts b/src/domain/dosMz.test.ts index bd98ca7e89..0a591d23af 100644 --- a/src/domain/dosMz.test.ts +++ b/src/domain/dosMz.test.ts @@ -109,15 +109,32 @@ describe("DOS MZ load-module validation", () => { }); it.each([ - ["page count", (bytes: Buffer) => bytes.writeUInt16LE(0, 4)], - ["final-page size", (bytes: Buffer) => bytes.writeUInt16LE(512, 2)], - ["header size", (bytes: Buffer) => bytes.writeUInt16LE(1, 8)], - ["entry point", (bytes: Buffer) => bytes.writeUInt16LE(128, 20)], + [ + "page count", + (bytes: Buffer) => bytes.writeUInt16LE(0, 4), + "invalid DOS MZ page count or final-page size", + ], + [ + "final-page size", + (bytes: Buffer) => bytes.writeUInt16LE(512, 2), + "invalid DOS MZ page count or final-page size", + ], + [ + "header size", + (bytes: Buffer) => bytes.writeUInt16LE(1, 8), + "invalid DOS MZ header size or empty load module", + ], + [ + "entry point", + (bytes: Buffer) => bytes.writeUInt16LE(128, 20), + "DOS MZ entry point lies outside its initialized load module", + ], [ "relocation-table bounds", (bytes: Buffer) => { bytes.writeUInt16LE(2, 6); }, + "DOS MZ relocation table lies outside its header", ], [ "relocation target", @@ -125,11 +142,12 @@ describe("DOS MZ load-module validation", () => { bytes.writeUInt16LE(1, 6); bytes.writeUInt16LE(127, 28); }, + "DOS MZ relocation 0 points outside its load module", ], - ] as const)("rejects invalid %s", (_name, damage) => { + ] as const)("rejects invalid %s", (_name, damage, error) => { const bytes = dosMz(); damage(bytes); - expect(parseDosMzHeader(bytes).ok).toBe(false); + expect(parseDosMzHeader(bytes)).toEqual({ ok: false, error }); }); it("rejects a truncated relocation table even when the file size is known", () => { diff --git a/src/domain/evidence.ts b/src/domain/evidence.ts index 92c9400b90..cf32dee6c1 100644 --- a/src/domain/evidence.ts +++ b/src/domain/evidence.ts @@ -70,6 +70,7 @@ const subjectFormatSchema = z.enum([ "mach-o-universal", "javascript-bundle", "entitlements", + "pyinstaller", ]); const subjectBaseSchema = z.strictObject({ name: z.string().min(1), diff --git a/src/domain/hopperErrors.test.ts b/src/domain/hopperErrors.test.ts new file mode 100644 index 0000000000..a955843027 --- /dev/null +++ b/src/domain/hopperErrors.test.ts @@ -0,0 +1,28 @@ +import { expect, it } from "vitest"; + +import { HopperCancelledError } from "./hopperErrors.js"; + +it("omits a cause when the caller does not supply one", () => { + const error = new HopperCancelledError(); + + expect(error._tag).toBe("HopperCancelledError"); + expect(error.cause).toBeUndefined(); + expect(Object.hasOwn(error, "cause")).toBe(false); +}); + +it("retains an explicitly undefined cause separately from omission", () => { + const error = new HopperCancelledError(undefined, { + diagnostics: { stage: "queue" }, + }); + + expect(error.cause).toBeUndefined(); + expect(Object.hasOwn(error, "cause")).toBe(true); + expect(error.diagnostics).toEqual({ stage: "queue" }); +}); + +it("retains an opaque caller reason by identity", () => { + const reason = { source: "caller", opaque: 7 }; + const error = new HopperCancelledError(reason); + + expect(error.cause).toBe(reason); +}); diff --git a/src/domain/hopperErrors.ts b/src/domain/hopperErrors.ts index 935dd1b48a..ddc67a152b 100644 --- a/src/domain/hopperErrors.ts +++ b/src/domain/hopperErrors.ts @@ -61,16 +61,37 @@ export class HopperCancelledError extends HopperError { readonly _tag = "HopperCancelledError"; constructor( - cause?: unknown, - options: Omit = {}, + ...parameters: + | readonly [] + | readonly [cause: unknown] + | readonly [cause: unknown, options: Omit] ) { - super( - "Hopper operation was cancelled", - cause === undefined ? options : { ...options, cause }, - ); + const errorOptions: AnalysisErrorOptions = { ...parameters[1] }; + // Argument presence matters: an explicit undefined is the caller's reason. + if (parameters.length > 0) + Object.defineProperty(errorOptions, "cause", { + configurable: true, + enumerable: true, + value: parameters[0], + writable: true, + }); + super("Hopper operation was cancelled", errorOptions); } } +/** + * Classify caller cancellation from a signal. + * + * An aborted signal keeps its exact reason, including undefined. A signal that + * has not aborted does not invent a reason. + */ +export const hopperCancelledBySignal = ( + signal: AbortSignal | undefined, +): HopperCancelledError => + signal?.aborted === true + ? new HopperCancelledError(signal.reason) + : new HopperCancelledError(); + /** Hopper returned bytes or JSON that do not satisfy the NDJSON RPC contract. */ export class HopperProtocolError extends HopperError { readonly _tag = "HopperProtocolError"; diff --git a/src/domain/native/objcSwiftMetadata.ts b/src/domain/native/objcSwiftMetadata.ts index fca04b09c7..57612ab16a 100644 --- a/src/domain/native/objcSwiftMetadata.ts +++ b/src/domain/native/objcSwiftMetadata.ts @@ -214,8 +214,14 @@ export type MachoImportBinding = z.infer; /** Swift protocol conformance and its associated witness-table location. */ export const swiftConformanceSchema = z.strictObject({ type_name: z.string().min(1).nullable(), + /** Raw three-bit Swift ABI type-reference kind; absent in older Evidence. */ + type_reference_kind: z.number().int().min(0).max(7).optional(), + /** Address of the descriptor, class-name string, or resolved ObjC class reference. */ + type_reference_target: nativeMetadataLocationSchema.optional(), type_binding: machoImportBindingSchema.nullable().optional(), protocol_name: z.string().min(1).nullable(), + /** Resolved protocol descriptor address, retained when its bytes are unavailable. */ + protocol_target: nativeMetadataLocationSchema.optional(), protocol_binding: machoImportBindingSchema.nullable().optional(), module: z.string().nullable(), witness_table: nativeMetadataLocationSchema, diff --git a/src/domain/propertyListKeys.test.ts b/src/domain/propertyListKeys.test.ts index f29fc1c305..cf771ced15 100644 --- a/src/domain/propertyListKeys.test.ts +++ b/src/domain/propertyListKeys.test.ts @@ -315,7 +315,7 @@ it("bounds nested plist wrappers before recursive decoding", () => { ).toEqual(["x"]); }); -it.each(["\n", "\r\n", "\r", "\u0085", "\u2028", "\u2029"])( +it.each(["\n", "\r\n", "\r"])( "preserves unrelated source values around %j line endings", (newline) => { const text = plist( @@ -390,3 +390,18 @@ describe("OpenStep plist nesting preflight", () => { }); }); }); + +it.each(["\u0085", "\u2028", "\u2029"])( + "preserves key identity and omission around inter-element separator %j", + (separator) => { + const parsed = parseXmlPropertyList( + plist( + `${separator}kept${separator}keya${separator}b${separator}__proto__omitted`, + ), + ); + expect(parsed).toEqual({ + value: { [`kept${separator}key`]: `a${separator}b` }, + omittedPrototypeKeys: 1, + }); + }, +); diff --git a/src/domain/propertyListXml.test.ts b/src/domain/propertyListXml.test.ts index b349cec112..4de709e4be 100644 --- a/src/domain/propertyListXml.test.ts +++ b/src/domain/propertyListXml.test.ts @@ -290,6 +290,13 @@ it("binds raw literals to their nodes across normalized line endings and UTF-16 ]); }); +it("preserves Unicode line separators in plist string values", () => { + const value = "a\u0085b\u2028c\u2029d"; + expect( + parseXmlPropertyList(`${value}`).value, + ).toBe(value); +}); + it.each(["\u0085", "\u2028", "\u2029"])( "does not normalize unsupported leading whitespace %j into an observed scalar", (prefix) => { diff --git a/src/domain/propertyListXmlSource.ts b/src/domain/propertyListXmlSource.ts index 26e29ab92c..cc1f13575d 100644 --- a/src/domain/propertyListXmlSource.ts +++ b/src/domain/propertyListXmlSource.ts @@ -1,15 +1,12 @@ import type { Document, Node } from "@xmldom/xmldom"; import { MAX_PLIST_DEPTH } from "./apple/plistValue.js"; import { err, ok, type Result } from "./result.js"; -import { parseXmlDocument } from "./xmlDocument.js"; +import { parseXmlDocument, XML_LINE_ENDING } from "./xmlDocument.js"; import { decodeXmlPlistData, XmlPlistDataError, } from "./propertyListXmlData.js"; -// Preserve DOM compatibility normalization while retaining raw scalar spans. -const LINE_ENDING = /\r[\n\u0085]|[\r\n\u0085\u2028\u2029]/gu; - interface ScalarSpan { readonly start: number; readonly end: number; @@ -25,14 +22,14 @@ export class XmlPlistSourceError extends TypeError { /** One source owner binds lexical scalar values to their validated DOM nodes. */ export const readXmlPlistSource = (text: string) => { - const normalized = text.replace(LINE_ENDING, "\n"); + const normalized = text.replace(XML_LINE_ENDING, "\n"); const parsed = parseXmlDocument(normalized, { doctype: "allow", lineEndings: "preserve", }); if (!parsed.ok) throw parsed.error; const document = parsed.value; - const lineEndings = new RegExp(LINE_ENDING); + const lineEndings = new RegExp(XML_LINE_ENDING); let line = 1; let lineStart = 0; // The depth walk visits elements in source order, so location lookup is diff --git a/src/domain/rawBinary.test.ts b/src/domain/rawBinary.test.ts index 558e98e009..72da0f6dc0 100644 --- a/src/domain/rawBinary.test.ts +++ b/src/domain/rawBinary.test.ts @@ -33,7 +33,10 @@ describe("raw-binary entry addresses", () => { it.each(["8000000", "0x", "0xgg", "0X8000000", ""])( "rejects malformed entry %s", (value) => { - expect(parseRawBinaryEntryPoint(value).ok).toBe(false); + expect(parseRawBinaryEntryPoint(value)).toEqual({ + ok: false, + error: "entry_point must be a hexadecimal address of the form 0x…", + }); }, ); }); diff --git a/src/domain/rust/rustBinary.ts b/src/domain/rust/rustBinary.ts index 95cc60a807..2047b627aa 100644 --- a/src/domain/rust/rustBinary.ts +++ b/src/domain/rust/rustBinary.ts @@ -1,6 +1,21 @@ import { z } from "zod"; import { digestSchema } from "../digests.js"; import { localPathStringSchema } from "../localPath.js"; +import { BINARY_ARCHITECTURES } from "../binaryTargetTypes.js"; + +/** Preserve the format owner's architecture spelling separately from Evidence vocabulary. */ +export const rustArchitectureForSubject = (architecture: string) => { + switch (architecture) { + case "386": + return "x86"; + case "amd64": + return "x86_64"; + default: + return BINARY_ARCHITECTURES.find( + (supported) => supported === architecture, + ); + } +}; /** Known revisions used to interpret compatible cargo-auditable documents. */ export const RUST_AUDITABLE_FORMAT_VERSIONS = [0, 1, 8] as const; diff --git a/src/domain/rust/rustBinaryQuery.ts b/src/domain/rust/rustBinaryQuery.ts new file mode 100644 index 0000000000..e4a31376ca --- /dev/null +++ b/src/domain/rust/rustBinaryQuery.ts @@ -0,0 +1,69 @@ +import { z } from "zod"; +import { + inspectRustBinaryInputSchema, + rustPackageSchema, + rustAuditableMetadataSchema, +} from "./rustBinary.js"; + +const edgeSchema = rustAuditableMetadataSchema.shape.dependency_edges.element; +const index = z.number().int().nonnegative(); +const page = { + indices: z.array(index).optional(), + offset: index.default(0), + limit: z.number().int().positive().default(64), +}; +const packages = z.strictObject({ + ...page, + collection: z.literal("packages"), + names: z.array(z.string()).optional(), + versions: z.array(z.string()).optional(), + fields: z + .array(rustPackageSchema.keyof()) + .optional() + .describe( + "Original index always retained. Default attributes omit dependency vectors; select dependencies explicitly for direct index references.", + ), +}); +const edges = z.strictObject({ + ...page, + collection: z.literal("edges"), + from_indices: z.array(index).optional(), + to_indices: z.array(index).optional(), + fields: z.array(edgeSchema.keyof()).optional(), +}); + +/** Filters intersect; empty arrays select none and offsets count matching rows. */ +export const rustBinarySelectionSchema = z.discriminatedUnion("collection", [ + packages, + edges, +]); +export const rustBinaryViewRequestSchema = z.discriminatedUnion("collection", [ + packages.extend({ kind: z.literal("rust") }), + edges.extend({ kind: z.literal("rust") }), +]); +export const rustBinaryViewPageSchema = z.discriminatedUnion("collection", [ + z.strictObject({ + collection: z.literal("packages"), + rows: z.array(rustPackageSchema.partial().extend({ index })), + }), + z.strictObject({ + collection: z.literal("edges"), + rows: z.array(edgeSchema.partial().extend({ index })), + }), +]); + +/** Delivery choices never enter the file-reader port or change original Evidence identity. */ +export const inspectRustBinaryRequestSchema = inspectRustBinaryInputSchema + .extend({ + detail: z.enum(["complete", "summary"]).optional(), + view: rustBinarySelectionSchema.optional(), + }) + .superRefine((value, context) => { + if (value.view !== undefined && value.detail === "complete") + context.addIssue({ + code: "custom", + path: ["view"], + message: + "view selects compact delivery; omit view for complete Evidence", + }); + }); diff --git a/src/domain/sqlite/sqliteDatabase.ts b/src/domain/sqlite/sqliteDatabase.ts index cb126098e7..90e4a8e660 100644 --- a/src/domain/sqlite/sqliteDatabase.ts +++ b/src/domain/sqlite/sqliteDatabase.ts @@ -11,6 +11,10 @@ export const SQLITE_PROVIDER_IDENTITY = { /** Default ordinary-table sample size when the caller selects a table. */ export const SQLITE_ROW_LIMIT_DEFAULT = 20; +/** Execution budget, bounded by the runtime's signed 32-bit timer range. */ +export const DEFAULT_SQLITE_WORKER_TIMEOUT_MS = 30_000; +export const MAX_SQLITE_WORKER_TIMEOUT_MS = 2_147_483_647; +const workerTimeout = z.number().int().min(1).max(MAX_SQLITE_WORKER_TIMEOUT_MS); const nonnegative = z.number().int().nonnegative(); const hex = z.string().regex(/^(?:[0-9a-f]{2})*$/); @@ -27,6 +31,11 @@ export const inspectSqliteDatabaseInputSchema = z path: localPathStringSchema.describe( "Absolute path to an offline SQLite database snapshot", ), + worker_timeout_ms: workerTimeout + .optional() + .describe( + `Deadline in milliseconds for worker startup, queries and reply writing (default ${String(DEFAULT_SQLITE_WORKER_TIMEOUT_MS)}), not snapshot acquisition or the whole call. Allow client headroom for acquisition, reply reads, cleanup and delivery. Fixed byte/heap limits do not change.`, + ), table: z .string() .optional() @@ -114,6 +123,11 @@ export const sqliteDatabaseSchema = z name: z.literal("SQLite"), version: z.string().min(1), }), + worker_timeout_ms: workerTimeout + .exactOptional() + .describe( + "Actual parent-supervised worker deadline in milliseconds; absent in historical Evidence means unknown, not the default.", + ), database_encoding: encoding, schema: z.strictObject({ scope: z.enum(["inventory", "selected_table", "database"]), @@ -220,6 +234,9 @@ export const sqliteInspectionParameters = ( input: InspectSqliteDatabaseInput, ): Readonly> => ({ path: input.path, + ...(input.worker_timeout_ms === undefined + ? {} + : { worker_timeout_ms: input.worker_timeout_ms }), ...(input.table === undefined ? {} : { table: input.table }), ...(input.columns === undefined ? {} : { columns: input.columns }), ...(input.schema_scope === undefined diff --git a/src/domain/wasm/wasmArtifact.ts b/src/domain/wasm/wasmArtifact.ts index a15735125d..52abf38db2 100644 --- a/src/domain/wasm/wasmArtifact.ts +++ b/src/domain/wasm/wasmArtifact.ts @@ -11,6 +11,9 @@ const artifact = z.strictObject({ sha256: digest, bytes: z.number().int().nonnegative(), }); +/** Default execution budget; upper bound is the runtime's signed 32-bit timer range. */ +export const DEFAULT_WASM_COMMAND_TIMEOUT_MS = 30_000; +export const MAX_WASM_COMMAND_TIMEOUT_MS = 2_147_483_647; /** All artifacts are explicitly selected local files; references never trigger acquisition. */ export const inspectWasmArtifactInputSchema = z.strictObject({ path: localPathStringSchema.describe( @@ -18,6 +21,15 @@ export const inspectWasmArtifactInputSchema = z.strictObject({ ), glue_paths: z.array(localPathStringSchema).max(16).default([]), candidate_paths: z.array(localPathStringSchema).max(16).default([]), + command_timeout_ms: z + .number() + .int() + .min(1) + .max(MAX_WASM_COMMAND_TIMEOUT_MS) + .exactOptional() + .describe( + `Deadline in milliseconds for each WABT version/analysis command (default ${String(DEFAULT_WASM_COMMAND_TIMEOUT_MS)}), not the whole inspection. Allow additional client time for all commands, file checks and cleanup. Input/output safety limits remain fixed.`, + ), }); /** Upstream output stays intact; import/export names are never split on ambiguous punctuation. */ export const wasmArtifactSchema = z diff --git a/src/domain/webExecution.ts b/src/domain/webExecution.ts index b1344e390f..459757c83d 100644 --- a/src/domain/webExecution.ts +++ b/src/domain/webExecution.ts @@ -40,6 +40,7 @@ export const webExecutionSchema = z.object({ "window_elapsed", "document_changed", "target_terminated", + "session_detached", ]), producer_started_seconds: z.number().min(0), producer_sampled_seconds: z.number().min(0).nullable(), diff --git a/src/domain/xmlDocument.test.ts b/src/domain/xmlDocument.test.ts index c8ad068ab6..469bc31600 100644 --- a/src/domain/xmlDocument.test.ts +++ b/src/domain/xmlDocument.test.ts @@ -73,3 +73,17 @@ it("applies XML line-ending normalization only when the consumer selects it", () expect(parsed.value.documentElement?.getAttribute("attr")).toBe("a b"); expect(parsed.value.documentElement?.textContent).toBe("\nx"); }); + +it("preserves Unicode separators under XML 1.0 normalization", () => { + const parsed = parseXmlDocument( + 'a\r\u0085b\r\u2028c\r\n d\u2029', + { doctype: "allow", lineEndings: "normalize" }, + ); + if (!parsed.ok) throw parsed.error; + expect(parsed.value.documentElement?.getAttribute("attr")).toBe( + "a\u0085b\u2028c\u2029d", + ); + expect(parsed.value.documentElement?.textContent).toBe( + "a\n\u0085b\n\u2028c\n\rd\u2029", + ); +}); diff --git a/src/domain/xmlDocument.ts b/src/domain/xmlDocument.ts index af4f636cc1..3022f4c6cd 100644 --- a/src/domain/xmlDocument.ts +++ b/src/domain/xmlDocument.ts @@ -1,12 +1,10 @@ -import { - DOMParser, - normalizeLineEndings, - onErrorStopParsing, - type Document, -} from "@xmldom/xmldom"; +import { DOMParser, onErrorStopParsing, type Document } from "@xmldom/xmldom"; import { SaxesParser } from "saxes"; import { err, ok, type Result } from "./result.js"; +/** XML 1.0 literal line endings; Unicode separators remain source characters. */ +export const XML_LINE_ENDING = /\r\n?|\n/gu; + export class XmlDocumentParseError extends Error { readonly _tag = "XmlDocumentParseError"; constructor( @@ -48,7 +46,7 @@ export const parseXmlDocument = ( normalizeLineEndings: policy.lineEndings === "preserve" ? (text) => text - : normalizeLineEndings, + : (text) => text.replace(XML_LINE_ENDING, "\n"), onError: onErrorStopParsing, }).parseFromString(source, "application/xml"), ); diff --git a/src/evm/EvmWorkerLimits.test.ts b/src/evm/EvmWorkerLimits.test.ts index fcbf8197a8..a7631689e5 100644 --- a/src/evm/EvmWorkerLimits.test.ts +++ b/src/evm/EvmWorkerLimits.test.ts @@ -43,13 +43,44 @@ it("independently preserves CPU and file output restrictions", () => { ).toMatchObject({ ok: true, value: { cpuSeconds: 5, fileSizeBytes: 4096 } }); }); it.each([ - ["not-text", null], - ["soft-unlimited-hard-finite", limits("unlimited", "2147483648")], - ["soft-exceeds-hard", limits("3", "2")], - ["missing-row", limits().replace("Max file size", "Max other size")], - ["duplicate-row", limits() + "Max cpu time 1 1 seconds\n"], - ["wrong-unit", limits().replace("bytes", "pages")], - ["oversized", "x".repeat(65537)], -])("rejects malformed/ambiguous resource observations: %s", (_label, input) => { - expect(selectEvmWorkerLimits(input).ok).toBe(false); -}); + [ + "not-text", + null, + "Linux resource limits must be text within the 64 KiB parsing budget.", + ], + [ + "soft-unlimited-hard-finite", + limits("unlimited", "2147483648"), + "Inherited Max address space soft limit exceeds its hard limit.", + ], + [ + "soft-exceeds-hard", + limits("3", "2"), + "Inherited Max address space soft limit exceeds its hard limit.", + ], + [ + "missing-row", + limits().replace("Max file size", "Max other size"), + "Linux resource limits must report exactly one Max file size row.", + ], + [ + "duplicate-row", + limits() + "Max cpu time 1 1 seconds\n", + "Linux resource limits must report exactly one Max cpu time row.", + ], + [ + "wrong-unit", + limits().replace("bytes", "pages"), + "Malformed inherited Linux resource limit: Max address space.", + ], + [ + "oversized", + "x".repeat(65537), + "Linux resource limits must be text within the 64 KiB parsing budget.", + ], +])( + "rejects malformed/ambiguous resource observations: %s", + (_label, input, error) => { + expect(selectEvmWorkerLimits(input)).toEqual({ ok: false, error }); + }, +); diff --git a/src/ghidra/GhidraSessionValues.test.ts b/src/ghidra/GhidraSessionValues.test.ts index a68ed16159..84a5c3653d 100644 --- a/src/ghidra/GhidraSessionValues.test.ts +++ b/src/ghidra/GhidraSessionValues.test.ts @@ -36,19 +36,30 @@ const session = ( }, }); +const rejectedHandshake = (value: ReturnType) => { + const parsed = parseGhidraSessionInfo(value, expected); + if (parsed.ok) throw new Error("expected an invalid Ghidra handshake"); + return parsed.error; +}; + describe("Ghidra DOS import commitment", () => { it("accepts the exact admitted real-mode language and compiler", () => { - expect(parseGhidraSessionInfo(session(), expected).ok).toBe(true); + const parsed = parseGhidraSessionInfo(session(), expected); + if (!parsed.ok) throw parsed.error; + expect(parsed.value.target).toMatchObject({ + language_id: expected.expectedLanguageId, + compiler_spec_id: expected.expectedCompilerSpecId, + }); }); it("rejects a 32-bit interpretation even when artifact and profile hashes match", () => { - expect( - parseGhidraSessionInfo(session("x86:LE:32:default"), expected).ok, - ).toBe(false); + expect(rejectedHandshake(session("x86:LE:32:default")).message).toBe( + "Ghidra bridge handshake identity is invalid", + ); }); it("rejects compiler drift independently of the language", () => { - expect( - parseGhidraSessionInfo(session(undefined, "windows"), expected).ok, - ).toBe(false); + expect(rejectedHandshake(session(undefined, "windows")).message).toBe( + "Ghidra bridge handshake identity is invalid", + ); }); }); diff --git a/src/go/GoBinaryImage.ts b/src/go/GoBinaryImage.ts index c9e2366c8c..8519b97451 100644 --- a/src/go/GoBinaryImage.ts +++ b/src/go/GoBinaryImage.ts @@ -1,25 +1,14 @@ import { - GoBinaryFormatFailure, - GoBinaryResourceFailure, - GoBinaryReader, - goUtf8, + BinaryFormatFailure, + BinaryResourceFailure, + BinaryReader, mappedFileOffset, isZeroFilledSpan, - type GoBinaryContainer, - type GoFileMapping, -} from "./GoBinaryContainer.js"; -import { - classifyFatContainerIdentity, - readFatContainerHeader, -} from "../domain/apple/machoContainer.js"; -import { readGoElfImage } from "./GoElfImage.js"; -import { readGoMachoImage } from "./GoMachoImage.js"; -import { readGoPeImage } from "./GoPeImage.js"; - -export { - GoBinaryFormatFailure, - GoBinaryResourceFailure, -} from "./GoBinaryContainer.js"; + type BinaryContainer, + type FileMapping, +} from "../native/metadata/BinaryContainer.js"; +import { readBinaryContainer } from "../native/metadata/readBinaryContainer.js"; +import { goBuildInfoSearch } from "./GoBuildInfoSearch.js"; const MAGIC = Buffer.from("ff20476f206275696c64696e663a", "hex"); const MODULE_START = Buffer.from("3077af0c9274080241e1c107e6d618e6", "hex"); @@ -56,19 +45,19 @@ export interface GoBinaryImage { /** Inspect bounded executable bytes using native mappings, without loading or executing code. */ export const readGoBinaryImage = (bytes: Buffer): GoBinaryImage => { - const container = readGoBinaryContainer(bytes); + const container = readBinaryContainer(bytes); const { format, architecture, bits, byte_order } = container; - let selected: { search: GoFileMapping; header: number } | null = null; + let selected: { search: FileMapping; header: number } | null = null; const searched = new Set(); let remainingSearchBytes = MAX_HEADER_SEARCH_BYTES; - for (const search of container.search) { + for (const search of goBuildInfoSearch(bytes, container)) { // Identical file spans and alignment inspect the same candidate bytes. // Partially overlapping spans still consume a cumulative work budget. const identity = `${search.offset}:${search.size}:${search.address % 16n}`; if (searched.has(identity)) continue; searched.add(identity); if (search.size > remainingSearchBytes) - throw new GoBinaryResourceFailure( + throw new BinaryResourceFailure( "search", MAX_HEADER_SEARCH_BYTES, "Go build-info discovery exceeds the cumulative 256 MiB header-search budget", @@ -93,22 +82,22 @@ export const readGoBinaryImage = (bytes: Buffer): GoBinaryImage => { container.zero_fills, ) !== header ) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Go build-info bytes disagree with their virtual mapping", ); }; verifyMapping(32); - const reader = new GoBinaryReader(bytes, byte_order === "little"); + const reader = new BinaryReader(bytes, byte_order === "little"); const ptrSize = bytes[header + 14]; const flags = bytes[header + 15] ?? 0; if ((flags & ~3) !== 0) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "unsupported", "Go build-info flags contain an unsupported format bit", ); if ((flags & 2) === 0 && (flags & 1) !== (byte_order === "big" ? 1 : 0)) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Go build-info byte order disagrees with the image", ); @@ -125,12 +114,12 @@ export const readGoBinaryImage = (bytes: Buffer): GoBinaryImage => { verifyMapping(module.location.offset + module.location.bytes - header); } else { if (ptrSize !== 4 && ptrSize !== 8) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Go build-info pointer width must be 4 or 8", ); if (ptrSize * 8 !== bits) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Go build-info pointer width disagrees with the image", ); @@ -149,13 +138,13 @@ export const readGoBinaryImage = (bytes: Buffer): GoBinaryImage => { ); } if (version.location.bytes + module.location.bytes > MAX_BUILD_INFO_BYTES) - throw new GoBinaryResourceFailure( + throw new BinaryResourceFailure( "build-info", MAX_BUILD_INFO_BYTES, "Go build-info strings exceed the aggregate 1 MiB limit", ); if (version.bytes.length === 0) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Go build-info compiler version is empty", ); @@ -168,7 +157,7 @@ export const readGoBinaryImage = (bytes: Buffer): GoBinaryImage => { !module.bytes.subarray(-16).equals(MODULE_END) || module.bytes[module.bytes.length - 17] !== 10 ) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Go module string framing is invalid", ); @@ -194,9 +183,9 @@ export const readGoBinaryImage = (bytes: Buffer): GoBinaryImage => { const observedText = (bytes: Buffer): string | null => { try { - return goUtf8(bytes, "Go build string"); + return observedUtf8(bytes, "Go build string"); } catch (cause) { - if (cause instanceof GoBinaryFormatFailure) return null; + if (cause instanceof BinaryFormatFailure) return null; throw cause; } }; @@ -206,45 +195,7 @@ interface LocatedString { readonly location: GoStringLocation; } -/** Identify an ELF, PE, or thin Mach-O image and its named file sections. */ -export const readGoBinaryContainer = (bytes: Buffer): GoBinaryContainer => { - if (bytes.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) - return readGoElfImage(bytes); - if (bytes.subarray(0, 2).equals(Buffer.from("MZ"))) - return readGoPeImage(bytes); - if (bytes.length >= 4) { - const magic = bytes.readUInt32BE(); - if ([0xcafebabe, 0xcafebabf, 0xbebafeca, 0xbfbafeca].includes(magic)) { - const fatHeader = readFatContainerHeader(bytes, bytes.length); - if (fatHeader.state === "malformed") - throw new GoBinaryFormatFailure("malformed", fatHeader.reason); - const identity = - fatHeader.state === "parsed" - ? classifyFatContainerIdentity(fatHeader, (offset) => { - if (offset >= bytes.length) return undefined; - if (offset + 4 > bytes.length) return null; - return bytes.readUInt32BE(offset); - }) - : undefined; - // An invalid FAT-shaped table is not proof of a Mach-O container: JVM - // constant-pool bytes can share both the magic and a known CPU value. - const confirmedFatMachO = identity?.state === "mach-o"; - if (confirmedFatMachO) - throw new GoBinaryFormatFailure( - "unsupported", - "Universal/fat Mach-O containers require selecting a thin slice", - ); - } - if ([0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(magic)) - return readGoMachoImage(bytes); - } - throw new GoBinaryFormatFailure( - "unsupported", - "Only ELF, PE, and thin Mach-O native images are supported", - ); -}; - -const findHeader = (bytes: Buffer, search: GoFileMapping): number | null => { +const findHeader = (bytes: Buffer, search: FileMapping): number | null => { if (search.size === 0) return null; const end = search.offset + search.size; const selected = bytes.subarray(search.offset, end); @@ -256,7 +207,7 @@ const findHeader = (bytes: Buffer, search: GoFileMapping): number | null => { const address = search.address + BigInt(candidate - search.offset); if (address % 16n === 0n) { if (end - candidate < 32) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Go build-info header is truncated", ); @@ -268,34 +219,34 @@ const findHeader = (bytes: Buffer, search: GoFileMapping): number | null => { }; const readInlineString = ( - reader: GoBinaryReader, + reader: BinaryReader, offset: number, remaining: number, ): LocatedString => { let length = 0n; for (let index = 0; index < 10; index++) { if (index >= remaining) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Go build-info varint is truncated", ); const byte = reader.bytes[offset + index] ?? 0; if (index === 9 && byte > 1) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Go build-info varint overflows 64 bits", ); length |= BigInt(byte & 127) << BigInt(index * 7); if ((byte & 128) === 0) { if (length > BigInt(MAX_BUILD_INFO_BYTES)) - throw new GoBinaryResourceFailure( + throw new BinaryResourceFailure( "build-info", MAX_BUILD_INFO_BYTES, "Go build-info string length exceeds the 1 MiB limit", ); const size = Number(length); if (size > remaining - index - 1) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Go build-info string range is truncated", ); @@ -306,15 +257,15 @@ const readInlineString = ( }; } } - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Go build-info varint is unterminated", ); }; const readPointerString = ( - reader: GoBinaryReader, - container: GoBinaryContainer, + reader: BinaryReader, + container: BinaryContainer, address: bigint, ): LocatedString => { const { mappings, zero_fills: zeroFills, bits } = container; @@ -323,7 +274,7 @@ const readPointerString = ( const valueAddress = reader.word(header, bits); const length = reader.word(header + width, bits); if (length > BigInt(MAX_BUILD_INFO_BYTES)) - throw new GoBinaryResourceFailure( + throw new BinaryResourceFailure( "build-info", MAX_BUILD_INFO_BYTES, "Go build-info pointer string length exceeds the 1 MiB limit", @@ -341,8 +292,8 @@ const readPointerString = ( }; const readOptionalModuleString = ( - reader: GoBinaryReader, - container: GoBinaryContainer, + reader: BinaryReader, + container: BinaryContainer, address: bigint, pointerField: number, ): LocatedString => @@ -354,3 +305,17 @@ const readOptionalModuleString = ( ) ? { bytes: Buffer.alloc(0), location: { offset: pointerField, bytes: 0 } } : readPointerString(reader, container, address); + +/** Decode metadata text without replacing invalid bytes or stripping an initial BOM. */ +export const observedUtf8 = (bytes: Buffer, label: string): string => { + try { + return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( + bytes, + ); + } catch (cause) { + throw new BinaryFormatFailure( + "malformed", + `${label} contains malformed UTF-8: ${String(cause)}`, + ); + } +}; diff --git a/src/go/GoBinaryProvider.ts b/src/go/GoBinaryProvider.ts index 7bbeb5c1d5..9488f07d14 100644 --- a/src/go/GoBinaryProvider.ts +++ b/src/go/GoBinaryProvider.ts @@ -30,11 +30,11 @@ import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { err, ok, type Result } from "../domain/result.js"; +import { readGoBinaryImage } from "./GoBinaryImage.js"; import { - readGoBinaryImage, - GoBinaryFormatFailure, - GoBinaryResourceFailure, -} from "./GoBinaryImage.js"; + BinaryFormatFailure, + BinaryResourceFailure, +} from "../native/metadata/BinaryContainer.js"; import { parseGoModuleBytes, parseGoModuleText } from "./GoModuleText.js"; /** Bundled format reader identity; embedded compiler versions are separate observed facts. */ @@ -57,7 +57,7 @@ const failureForPrimary = (cause: unknown, path: string): AnalysisError => { return new AnalysisCancelledError(OPERATION, { cause: artifactFileCancellationCause(cause), }); - if (cause instanceof GoBinaryResourceFailure) + if (cause instanceof BinaryResourceFailure) return new AnalysisResourceConstraintError( OPERATION, cause.boundary === "search" ? "cpu" : "memory", @@ -75,7 +75,7 @@ const failureForPrimary = (cause: unknown, path: string): AnalysisError => { ), }, ); - if (cause instanceof GoBinaryFormatFailure) + if (cause instanceof BinaryFormatFailure) return cause.kind === "unsupported" ? new AnalysisUnsupportedTargetError(OPERATION, path, cause.message, { cause, diff --git a/src/go/GoBuildInfoSearch.ts b/src/go/GoBuildInfoSearch.ts new file mode 100644 index 0000000000..7227acb6f8 --- /dev/null +++ b/src/go/GoBuildInfoSearch.ts @@ -0,0 +1,137 @@ +import { + BinaryFormatFailure, + BinaryReader, + mappedFileOffset, + type BinaryContainer, + type FileMapping, +} from "../native/metadata/BinaryContainer.js"; + +const fallbackSearchRanges = ( + container: BinaryContainer, +): readonly FileMapping[] => { + if (container.format === "elf") + return (container.load_regions ?? []) + .filter((region) => region.write && !region.execute) + .map((region) => region.mapping); + if (container.format === "macho") + return (container.load_regions ?? []) + .filter( + (region) => + region.mapping.address !== 0n && + region.protection_masks?.current === 3 && + region.protection_masks.maximum === 3, + ) + .slice(0, 1) + .map((region) => region.mapping); + if (container.image_base === undefined) return []; + const section = container.sections.find( + (candidate) => + candidate.format === "pe" && + candidate.virtual_address !== 0 && + candidate.raw_size !== 0 && + (candidate.characteristics & ~0x00600000) >>> 0 === 0xc0000040, + ); + return section?.format === "pe" + ? [ + { + address: container.image_base + BigInt(section.virtual_address), + offset: section.offset, + size: section.size, + }, + ] + : []; +}; + +/** Go owns build-info section selection and validity; native readers only own container facts. */ +export const goBuildInfoSearch = ( + bytes: Buffer, + container: BinaryContainer, +): readonly FileMapping[] => { + const reader = new BinaryReader(bytes, container.byte_order === "little"); + const named: FileMapping[] = []; + for (const section of container.sections) { + if (section.format === "elf" && section.name === ".go.buildinfo") { + if (named.length !== 0) + throw new BinaryFormatFailure( + "malformed", + "ELF contains ambiguous build-info sections", + ); + if (section.type !== 1) + throw new BinaryFormatFailure( + "malformed", + "ELF build-info section is not file-backed data", + ); + if (section.compressed) + throw new BinaryFormatFailure( + "unsupported", + "Compressed ELF build-info sections are unsupported", + ); + const search = { + ...reader.fileRange( + section.declared_range.offset, + section.declared_range.size, + "ELF build-info section", + ), + address: section.address, + }; + if ( + mappedFileOffset( + container.mappings, + search.address, + search.size, + container.zero_fills, + ) !== search.offset + ) + throw new BinaryFormatFailure( + "malformed", + "ELF build-info section disagrees with its load mapping", + ); + named.push(search); + } else if ( + section.format === "macho" && + section.section_name === "__go_buildinfo" + ) { + if (section.segment_unmapped) + throw new BinaryFormatFailure( + "malformed", + "Mach-O build-info section is in an unmapped segment", + ); + if (section.segment_mapping === null) + throw new BinaryFormatFailure( + "malformed", + "Mach-O build-info section is not file mapped", + ); + if ([1, 0xc, 0x12].includes(section.type)) + throw new BinaryFormatFailure( + "malformed", + "Mach-O build-info section is zero-fill data", + ); + const search = { + ...reader.fileRange( + section.declared_range.offset, + section.declared_range.size, + "Mach-O build-info section", + ), + address: section.address, + }; + if ( + mappedFileOffset( + [section.segment_mapping], + search.address, + search.size, + ) !== search.offset + ) + throw new BinaryFormatFailure( + "malformed", + "Mach-O build-info section disagrees with its segment", + ); + named.push(search); + } + } + if (named.length > 1) + throw new BinaryFormatFailure( + "malformed", + "Mach-O has ambiguous build-info sections", + ); + return named.length > 0 ? named : fallbackSearchRanges(container); +}; diff --git a/src/go/GoMachoImage.alignment.test.ts b/src/go/GoMachoImage.alignment.test.ts index 07a378a86b..a2ad09b488 100644 --- a/src/go/GoMachoImage.alignment.test.ts +++ b/src/go/GoMachoImage.alignment.test.ts @@ -7,7 +7,7 @@ it("rejects a 12-byte load command in a 64-bit Go Mach-O image", () => { readGoBinaryImage(Buffer.from(machoImageWithUnknownCommand(true, 12))), ).toThrow( expect.objectContaining({ - name: "GoBinaryFormatFailure", + name: "BinaryFormatFailure", kind: "malformed", }), ); diff --git a/src/hopper/BridgeLauncher.ts b/src/hopper/BridgeLauncher.ts index 99431606f8..76c4776770 100644 --- a/src/hopper/BridgeLauncher.ts +++ b/src/hopper/BridgeLauncher.ts @@ -4,6 +4,7 @@ import { basename, dirname, extname, join } from "node:path"; import { promisify } from "node:util"; import { + hopperCancelledBySignal, HopperCancelledError, HopperProcessError, HopperStartError, @@ -331,7 +332,7 @@ export class HopperApplicationLauncher implements BridgeLauncher { ]; let launched = false; try { - if (signal?.aborted === true) return err(new HopperCancelledError()); + if (signal?.aborted === true) return err(hopperCancelledBySignal(signal)); let expectedLinuxSha256: string | undefined; if (usesLinuxDemo(this.options)) { const inspected = await ( @@ -339,7 +340,7 @@ export class HopperApplicationLauncher implements BridgeLauncher { )(this.options.launcherPath, this.options.environment, signal); if (!inspected.ok) return inspected.error.reason === "cancelled" - ? err(new HopperCancelledError()) + ? err(cancelledLauncherVerification(signal, inspected.error)) : err( new HopperStartError({ userMessage: inspected.error.message, @@ -366,7 +367,7 @@ export class HopperApplicationLauncher implements BridgeLauncher { this.options.environment, signal, ); - if (!prepared) return err(new HopperCancelledError()); + if (!prepared) return err(hopperCancelledBySignal(signal)); const linuxDemo = linuxDemoLaunch( this.options, session, @@ -383,7 +384,7 @@ export class HopperApplicationLauncher implements BridgeLauncher { linuxDemo, signal, }); - if (started === null) return err(new HopperCancelledError()); + if (started === null) return err(hopperCancelledBySignal(signal)); launched = true; const cleanup = () => cleanupOwnedProcessGroup(started.ownership); const ownedLauncher = { @@ -518,7 +519,7 @@ export class HopperApplicationLauncher implements BridgeLauncher { ...(signal === undefined ? {} : { signal }), }); if (signal?.aborted === true) - return { ok: false, error: new HopperCancelledError() }; + return { ok: false, error: hopperCancelledBySignal(signal) }; return selection.ok ? { ok: true, strategy: selection.strategy } : { @@ -637,12 +638,22 @@ export const linuxDemoLaunch = ( }; }; +const cancelledLauncherVerification = ( + signal: AbortSignal | undefined, + error: Error, +): HopperCancelledError => { + if (signal?.aborted === true) return new HopperCancelledError(signal.reason); + return Object.hasOwn(error, "cause") + ? new HopperCancelledError(error.cause) + : new HopperCancelledError(); +}; + const hopperLaunchFailure = ( cause: unknown, signal: AbortSignal | undefined, ): HopperCancelledError | HopperStartError => signal?.aborted === true - ? new HopperCancelledError() + ? new HopperCancelledError(signal.reason) : new HopperStartError({ cause }); /** Select the publisher-verified demo adapter from the caller's explicit launch mode. */ diff --git a/src/hopper/HopperClient.ts b/src/hopper/HopperClient.ts index 6a0dd077d3..cbd99cf9b4 100644 --- a/src/hopper/HopperClient.ts +++ b/src/hopper/HopperClient.ts @@ -9,7 +9,8 @@ import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup. import { analysisClientFailureContext } from "../providers/AnalysisClientFailure.js"; import { AnalysisCapabilityUnavailableError } from "../domain/analysisErrorCore.js"; import { - HopperCancelledError, + hopperCancelledBySignal, + type HopperCancelledError, type HopperError, type HopperLauncherOutcome, HopperProcessError, @@ -313,11 +314,11 @@ export class HopperClient { /** Launch the bridge once while each caller retains its own cancellation. */ start(signal?: AbortSignal): Promise { if (isAborted(signal)) - return Promise.resolve(err(new HopperCancelledError())); + return Promise.resolve(err(hopperCancelledBySignal(signal))); if (this.#closePromise !== undefined) return waitForAbortable(this.#closePromise, signal).then((closed) => closed === ABORTED - ? err(new HopperCancelledError()) + ? err(hopperCancelledBySignal(signal)) : closed.ok ? this.start(signal) : closed, @@ -355,7 +356,7 @@ export class HopperClient { const waited = await attempt.wait(signal); if (waited.status === "completed") return waited.value; if (waited.status === "stopped" && !waited.value.ok) return waited.value; - return err(new HopperCancelledError()); + return err(hopperCancelledBySignal(signal)); } /** Cleanup belongs to the abandoned attempt, never to a prospective retry. */ @@ -365,7 +366,7 @@ export class HopperClient { ): Promise { const waited = await waitForAbortable(attempt.promise, signal); if (waited === ABORTED || (waited.ok && isAborted(signal))) - return err(new HopperCancelledError()); + return err(hopperCancelledBySignal(signal)); if (waited.ok || waited.error.cleanupIncomplete) return waited; return this.start(signal); } @@ -373,7 +374,7 @@ export class HopperClient { async #start( signal?: AbortSignal, ): Promise> { - if (isAborted(signal)) return err(new HopperCancelledError()); + if (isAborted(signal)) return err(hopperCancelledBySignal(signal)); if ( this.#socket !== undefined || this.#resources.launch !== undefined || @@ -835,7 +836,7 @@ export class HopperClient { if (deadline.signal.reason === STARTUP_OWNER_CLOSED) return new HopperProtocolError("Hopper session closed during startup"); if (deadline.interruption === "cancelled") - return new HopperCancelledError(); + return hopperCancelledBySignal(deadline.signal); return new HopperTimeoutError( deadline.timeoutMs, undefined, @@ -969,7 +970,7 @@ export class HopperClient { ); } if (options.signal?.aborted === true) - return err(new HopperCancelledError()); + return err(hopperCancelledBySignal(options.signal)); const startedAt = performance.now(); const result = await this.#requests.run(id, method, params, { ...(options.signal === undefined ? {} : { signal: options.signal }), diff --git a/src/hopper/HopperRequestQueue.test.ts b/src/hopper/HopperRequestQueue.test.ts index 183d67bae4..2cc32e9785 100644 --- a/src/hopper/HopperRequestQueue.test.ts +++ b/src/hopper/HopperRequestQueue.test.ts @@ -103,9 +103,9 @@ describe("HopperRequestQueue deadlines", () => { { timeoutMs: 25, signal: controller.signal }, ); controller.abort(); - await expect(pending).resolves.toEqual({ + await expect(pending).resolves.toMatchObject({ ok: false, - error: new HopperCancelledError(), + error: { cause: controller.signal.reason }, }); expect(vi.getTimerCount()).toBe(0); queue.accept(1, ok(null)); @@ -118,6 +118,7 @@ describe("HopperRequestQueue deadlines", () => { const send = vi.fn(); const queue = new HopperRequestQueue(send); const controller = new AbortController(); + const reason = { source: "progress" }; const pending = queue.run( 1, "echo", @@ -127,7 +128,7 @@ describe("HopperRequestQueue deadlines", () => { signal: controller.signal, progress: { report: () => { - controller.abort(); + controller.abort(reason); return Promise.resolve(); }, }, @@ -135,7 +136,7 @@ describe("HopperRequestQueue deadlines", () => { ); await expect(pending).resolves.toMatchObject({ ok: false, - error: { _tag: "HopperCancelledError" }, + error: { cause: reason }, }); expect(vi.getTimerCount()).toBe(0); expect(send).not.toHaveBeenCalled(); @@ -298,3 +299,56 @@ describe("Hopper deadlines before timer dispatch", () => { expect(vi.getTimerCount()).toBe(0); }); }); + +describe("HopperRequestQueue caller abort reasons", () => { + const reason = { source: "caller", opaque: 7 }; + + it("retains an opaque reason for a signal aborted before admission", async () => { + const queue = new HopperRequestQueue(() => {}); + const controller = new AbortController(); + controller.abort(reason); + const result = await queue.run( + 1, + "echo", + {}, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected cancellation"); + expect(result.error).toBeInstanceOf(HopperCancelledError); + expect(result.error.cause).toBe(reason); + }); + + it("retains an opaque reason when the active caller aborts", async () => { + const sent: number[] = []; + const queue = new HopperRequestQueue(({ id }) => sent.push(id)); + const controller = new AbortController(); + const pending = queue.run(1, "echo", {}, { signal: controller.signal }); + controller.abort(reason); + const result = await pending; + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected cancellation"); + expect(result.error.cause).toBe(reason); + expect(sent).toEqual([1]); + expect(queue.accept(1, ok(null))).toBe(true); + expect(result.error.cause).toBe(reason); + expect(queue.activity()).toBeNull(); + }); + + it("retains an opaque reason for a queued caller without transmitting it", async () => { + const sent: number[] = []; + const queue = new HopperRequestQueue(({ id }) => sent.push(id)); + const active = queue.run(1, "active", {}, {}); + const controller = new AbortController(); + const queued = queue.run(2, "queued", {}, { signal: controller.signal }); + controller.abort(reason); + const result = await queued; + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected cancellation"); + expect(result.error.cause).toBe(reason); + expect(queue.hasQueued(2)).toBe(false); + queue.accept(1, ok(null)); + await active; + expect(sent).toEqual([1]); + }); +}); diff --git a/src/hopper/HopperRequestQueue.ts b/src/hopper/HopperRequestQueue.ts index 43f4f2821d..9dc9fb500f 100644 --- a/src/hopper/HopperRequestQueue.ts +++ b/src/hopper/HopperRequestQueue.ts @@ -5,7 +5,7 @@ import type { import { hopperOperationStage } from "./HopperOperationStage.js"; import type { HopperBridgeEvent } from "./protocol.js"; import { - HopperCancelledError, + hopperCancelledBySignal, type HopperError, HopperProcessError, HopperProtocolError, @@ -77,7 +77,7 @@ export class HopperRequestQueue { options: HopperRequestQueueOptions, ): Promise { if (options.signal?.aborted === true) - return Promise.resolve(err(new HopperCancelledError())); + return Promise.resolve(err(hopperCancelledBySignal(options.signal))); const timeoutMs = options.timeoutMs; if ( timeoutMs !== undefined && @@ -239,7 +239,11 @@ export class HopperRequestQueue { return; } if (entry.signal?.aborted === true) { - this.#settleCaller(entry, err(new HopperCancelledError()), "cancelled"); + this.#settleCaller( + entry, + err(hopperCancelledBySignal(entry.signal)), + "cancelled", + ); this.#drain(); return; } @@ -316,7 +320,7 @@ export class HopperRequestQueue { #cancel( entry: QueuedRequest, - error: HopperError = new HopperCancelledError(), + error: HopperError = hopperCancelledBySignal(entry.signal), ): void { if (entry.callerSettled) return; if (this.#active === entry) { diff --git a/src/hopper/HopperSocketConnection.test.ts b/src/hopper/HopperSocketConnection.test.ts new file mode 100644 index 0000000000..0d67984da5 --- /dev/null +++ b/src/hopper/HopperSocketConnection.test.ts @@ -0,0 +1,22 @@ +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; + +import { HopperCancelledError } from "../domain/hopperErrors.js"; +import { connectHopperSocketOnce } from "./HopperSocketConnection.js"; + +it("retains the caller abort reason when the socket signal is already aborted", async () => { + const reason = { source: "caller", opaque: 7 }; + const controller = new AbortController(); + controller.abort(reason); + const result = await connectHopperSocketOnce( + join(tmpdir(), "rea-hopper-cancelled.sock"), + controller.signal, + ); + + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected cancellation"); + expect(result.error).toBeInstanceOf(HopperCancelledError); + expect(result.error.cause).toBe(reason); +}); diff --git a/src/hopper/HopperSocketConnection.ts b/src/hopper/HopperSocketConnection.ts index 8333bb164a..77fe8ef300 100644 --- a/src/hopper/HopperSocketConnection.ts +++ b/src/hopper/HopperSocketConnection.ts @@ -1,7 +1,8 @@ import { createConnection, type Socket } from "node:net"; import { - HopperCancelledError, + hopperCancelledBySignal, + type HopperCancelledError, HopperStartError, } from "../domain/hopperErrors.js"; import { err, ok, type Result } from "../domain/result.js"; @@ -30,7 +31,7 @@ export const connectHopperSocketOnce = ( const onAbort = (): void => { detach(); socket.destroy(); - resolve(err(new HopperCancelledError())); + resolve(err(hopperCancelledBySignal(signal))); }; socket.once("connect", onConnect); socket.once("error", onError); diff --git a/src/ida/IdaProvider.test.ts b/src/ida/IdaProvider.test.ts index e0d5a2180e..9ce6d84045 100644 --- a/src/ida/IdaProvider.test.ts +++ b/src/ida/IdaProvider.test.ts @@ -1,8 +1,84 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; import { parseConfig } from "../config/parseConfig.js"; import { IdaProvider } from "./IdaProvider.js"; +it.each([ + { state: "unconfigured", selected: false, content: undefined }, + { state: "missing", selected: true, content: undefined }, + { state: "malformed", selected: true, content: "{" }, + { state: "invalid", selected: true, content: "{}" }, +])( + "preserves the owning registration setting and selection for $state IDA readiness", + async ({ selected, content }) => { + const root = await createTestTempDirectory("rea-ida-recovery-"); + const path = join(root, "registration.json"); + if (content !== undefined) await writeFile(path, content); + const config = parseConfig(selected ? { REA_IDA_MCP_CONFIG: path } : {}); + if (!config.ok) throw config.error; + const connect = vi.fn(() => { + throw new Error("Readiness must not connect to IDA"); + }); + const provider = new IdaProvider(config.value, {}, connect); + try { + const snapshot = await provider.inspectReadinessSnapshot(); + expect(snapshot.availability).toMatchObject({ + status: "unavailable", + code: selected ? "open_options_invalid" : "not_configured", + reason: expect.stringContaining("REA_IDA_MCP_CONFIG"), + diagnostics: { + configured: selected, + registration_path: selected ? path : null, + live_connection_probed: false, + configuration_details: { + settings: [ + { setting: "REA_IDA_MCP_CONFIG", constraint: expect.any(String) }, + ], + }, + configuration_remediation: + expect.stringContaining("REA_IDA_MCP_CONFIG"), + }, + }); + expect(connect).not.toHaveBeenCalled(); + } finally { + const closed = await provider.closeOwnedResources(); + if (!closed.ok) throw closed.error; + } + }, +); + +it("keeps a selected valid registration distinct from a live IDA connection", async () => { + const root = await createTestTempDirectory("rea-ida-configured-"); + const path = join(root, "registration.json"); + await writeFile( + path, + JSON.stringify({ mode: "attached", url: "http://127.0.0.1:13337/mcp" }), + ); + const config = parseConfig({ REA_IDA_MCP_CONFIG: path }); + if (!config.ok) throw config.error; + const connect = vi.fn(() => { + throw new Error("Readiness must not connect to IDA"); + }); + const provider = new IdaProvider(config.value, {}, connect); + try { + expect(await provider.inspectAvailability()).toMatchObject({ + status: "available", + diagnostics: { + configured: true, + registration_path: path, + live_connection_probed: false, + }, + }); + expect(connect).not.toHaveBeenCalled(); + } finally { + const closed = await provider.closeOwnedResources(); + if (!closed.ok) throw closed.error; + } +}); + describe("IDA provider target support", () => { it("declines headerless raw-binary targets", () => { const config = parseConfig({}); diff --git a/src/ida/IdaProvider.ts b/src/ida/IdaProvider.ts index f48dbccc58..88bc8a3b30 100644 --- a/src/ida/IdaProvider.ts +++ b/src/ida/IdaProvider.ts @@ -7,6 +7,7 @@ import { } from "../domain/binaryTargetTypes.js"; import { err, ok } from "../domain/result.js"; import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import type { AnalysisClient, AnalysisProviderBoundClientFactory, @@ -171,7 +172,31 @@ export class IdaProvider implements AnalysisProviderCandidate { config.value.protocol === "native", ) : this.capabilities(); - if (config.ok && config.value.mode === "headless") { + const registrationDiagnostics = { + configured: this.config.idaMcpConfigPath !== undefined, + registration_path: this.config.idaMcpConfigPath ?? null, + live_connection_probed: false, + }; + if (!config.ok) { + const failure = projectAnalysisError(config.error); + return { + availability: { + status: "unavailable", + code: + this.config.idaMcpConfigPath === undefined + ? "not_configured" + : "open_options_invalid", + reason: failure.message, + diagnostics: { + ...registrationDiagnostics, + configuration_details: failure.details ?? null, + configuration_remediation: failure.remediation.action, + }, + }, + capabilities, + }; + } + if (config.value.mode === "headless") { const privacy = privateRuntimeRootCapability(); if (!privacy.available) return { @@ -180,9 +205,8 @@ export class IdaProvider implements AnalysisProviderCandidate { code: privacy.code, reason: privacy.reason, diagnostics: { - configured: true, + ...registrationDiagnostics, mode: "headless", - live_connection_probed: false, private_workspace_available: false, windows_native_failure: { stage: privacy.failureStage, @@ -193,28 +217,19 @@ export class IdaProvider implements AnalysisProviderCandidate { capabilities, }; } - const availability: ProviderAvailability = config.ok - ? { - status: "available" as const, - code: null, - reason: null, - diagnostics: { - configured: true, - mode: config.value.mode, - transport: "command" in config.value ? "stdio" : "http", - live_connection_probed: false, - }, - } - : { - status: "unavailable" as const, - code: - this.config.idaMcpConfigPath === undefined - ? "not_configured" - : "open_options_invalid", - reason: config.error.message, - diagnostics: { configured: false, live_connection_probed: false }, - }; - return { availability, capabilities }; + return { + availability: { + status: "available" as const, + code: null, + reason: null, + diagnostics: { + ...registrationDiagnostics, + mode: config.value.mode, + transport: "command" in config.value ? "stdio" : "http", + }, + }, + capabilities, + }; } inspectTargetSupport(target: BinaryTarget): ProviderTargetSupport { const diagnostics = { diff --git a/src/il2cpp/ElfRelocations.ts b/src/il2cpp/ElfRelocations.ts index f06a846d47..ed8b540093 100644 --- a/src/il2cpp/ElfRelocations.ts +++ b/src/il2cpp/ElfRelocations.ts @@ -1,10 +1,10 @@ import { - GoBinaryFormatFailure, + BinaryFormatFailure, mappedFileOffset, isZeroFilledSpan, - type GoFileMapping, - type GoZeroFillMapping, -} from "../go/GoBinaryContainer.js"; + type FileMapping, + type ZeroFillMapping, +} from "../native/metadata/BinaryContainer.js"; import { Il2cppBinaryFormatFailure } from "./Il2cppBinaryFormat.js"; const DT_RELA = 7n, @@ -26,8 +26,8 @@ interface RelocatedImage { } interface RelocationSource { readonly image: Buffer; - readonly mappings: readonly GoFileMapping[]; - readonly zeroFills: readonly GoZeroFillMapping[]; + readonly mappings: readonly FileMapping[]; + readonly zeroFills: readonly ZeroFillMapping[]; readonly relative: number; readonly unknownPointerKinds: ReadonlySet; } @@ -101,14 +101,14 @@ function* dynamicTags(image: Buffer): Generator> { } const fileOffset = ( - mappings: readonly GoFileMapping[], + mappings: readonly FileMapping[], address: bigint, size: number, ): number | null => { try { return mappedFileOffset(mappings, address, size); } catch (cause) { - if (cause instanceof GoBinaryFormatFailure) return null; + if (cause instanceof BinaryFormatFailure) return null; throw cause; } }; @@ -178,8 +178,8 @@ function* applyRelaTable( /** Materialize relative pointers in a private copy, preserving unknown and .bss facts. */ export function* applyElfRelativeRelocationsSteps( image: Buffer, - mappings: readonly GoFileMapping[], - zeroFills: readonly GoZeroFillMapping[], + mappings: readonly FileMapping[], + zeroFills: readonly ZeroFillMapping[], ): Generator { if (image.length < 64) throw malformed("ELF header is truncated"); const kinds = RELOCATION_KINDS.get(image.readUInt16LE(18)); diff --git a/src/il2cpp/Il2cppBinaryImage.ts b/src/il2cpp/Il2cppBinaryImage.ts index 9c14aa08b2..9540da86b1 100644 --- a/src/il2cpp/Il2cppBinaryImage.ts +++ b/src/il2cpp/Il2cppBinaryImage.ts @@ -3,13 +3,13 @@ import { validateFatSlice, } from "../domain/apple/machoContainer.js"; import { - GoBinaryFormatFailure, - GoBinaryResourceFailure, + BinaryFormatFailure, + BinaryResourceFailure, mappedFileOffset, isZeroFilledSpan, - type GoBinaryContainer, -} from "../go/GoBinaryContainer.js"; -import { readGoBinaryContainer } from "../go/GoBinaryImage.js"; + type BinaryContainer, +} from "../native/metadata/BinaryContainer.js"; +import { readBinaryContainer } from "../native/metadata/readBinaryContainer.js"; import { applyElfRelativeRelocationsSteps } from "./ElfRelocations.js"; import { Il2cppBinaryFormatFailure } from "./Il2cppBinaryFormat.js"; const MACHO_CPU = new Map([ @@ -64,7 +64,7 @@ const selectImage = ( export interface NativeImage { readonly bytes: Buffer; - readonly container: GoBinaryContainer; + readonly container: BinaryContainer; readonly imageBase: bigint; readonly unresolvedPointerSlots: ReadonlySet; readonly zeroFillPointers: ReadonlyMap; @@ -75,16 +75,16 @@ export function* openImageSteps( architecture: string | undefined, ): Generator { let image = selectImage(bytes, architecture); - let container: GoBinaryContainer; + let container: BinaryContainer; try { - container = readGoBinaryContainer(image); + container = readBinaryContainer(image); } catch (cause) { - if (cause instanceof GoBinaryResourceFailure) + if (cause instanceof BinaryResourceFailure) throw new Il2cppBinaryFormatFailure( "native-index-capacity", cause.message, ); - if (cause instanceof GoBinaryFormatFailure) + if (cause instanceof BinaryFormatFailure) throw new Il2cppBinaryFormatFailure( cause.kind === "unsupported" ? "unsupported-container" @@ -121,7 +121,7 @@ export function* openImageSteps( container.zero_fills, ); } catch (cause) { - if (cause instanceof GoBinaryFormatFailure) + if (cause instanceof BinaryFormatFailure) throw new Il2cppBinaryFormatFailure("malformed-container", cause.message); throw cause; } @@ -172,7 +172,7 @@ export const readPointer = ( ); return image.bytes.readBigUInt64LE(offset); } catch (cause) { - if (cause instanceof GoBinaryFormatFailure) return null; + if (cause instanceof BinaryFormatFailure) return null; throw cause; } }; @@ -195,7 +195,7 @@ export const readUInt32 = ( ), ); } catch (cause) { - if (cause instanceof GoBinaryFormatFailure) return null; + if (cause instanceof BinaryFormatFailure) return null; throw cause; } }; @@ -205,7 +205,7 @@ export const isMapped = (image: NativeImage, address: bigint, size: number) => { mappedFileOffset(image.container.mappings, address, size); return true; } catch (cause) { - if (cause instanceof GoBinaryFormatFailure) return false; + if (cause instanceof BinaryFormatFailure) return false; throw cause; } }; @@ -262,7 +262,7 @@ export const hasPointerArrayCoverage = ( ); } } catch (cause) { - if (cause instanceof GoBinaryFormatFailure) return false; + if (cause instanceof BinaryFormatFailure) return false; throw cause; } cursor += BigInt(size); @@ -284,7 +284,7 @@ export const isExecutable = (image: NativeImage, address: bigint): boolean => { mappedFileOffset(image.container.executable_mappings ?? [], address, 1) ); } catch (cause) { - if (cause instanceof GoBinaryFormatFailure) return false; + if (cause instanceof BinaryFormatFailure) return false; throw cause; } }; @@ -312,7 +312,7 @@ export const readSlot = ( try { return isZeroFilled(image, address, 8) ? 0n : readPointer(image, address); } catch (cause) { - if (cause instanceof GoBinaryFormatFailure) + if (cause instanceof BinaryFormatFailure) throw new Il2cppBinaryFormatFailure("malformed-container", cause.message); throw cause; } diff --git a/src/il2cpp/Il2cppMetadataFile.ts b/src/il2cpp/Il2cppMetadataFile.ts index b5a2c0c6d9..730b4adf00 100644 --- a/src/il2cpp/Il2cppMetadataFile.ts +++ b/src/il2cpp/Il2cppMetadataFile.ts @@ -253,15 +253,25 @@ const FIXED_WIDTHS: Il2cppMetadataDecoded["index_widths"] = { type_width_source: "fixed-width-version", }; +const GENERIC_CONTAINER_RECORD_BYTES = 16; + const resolveWidths = ( section: (name: SectionName) => Il2cppMetadataSection, ): Il2cppMetadataDecoded["index_widths"] => { + const genericContainers = section("genericContainers"); + const genericContainerBytes = recordBytes(genericContainers); + if ( + genericContainerBytes !== null && + genericContainerBytes !== GENERIC_CONTAINER_RECORD_BYTES + ) + throw new Il2cppMetadataFormatFailure( + "record-size-mismatch", + `section genericContainers records are ${genericContainerBytes} bytes, but each record is ${GENERIC_CONTAINER_RECORD_BYTES}`, + ); const type_definition = il2cppIndexWidth( headerCount(section("typeDefinitions")), ); - const generic_container = il2cppIndexWidth( - headerCount(section("genericContainers")), - ); + const generic_container = il2cppIndexWidth(headerCount(genericContainers)); const parameter = il2cppIndexWidth(headerCount(section("parameters"))); // TypeIndex counts native types, so its width is recovered from records. const interfaceOffset = recordBytes(section("interfaceOffsets")); diff --git a/src/model/GgufHeader.ts b/src/model/GgufHeader.ts index 7f598502de..1f2899f82c 100644 --- a/src/model/GgufHeader.ts +++ b/src/model/GgufHeader.ts @@ -1,3 +1,4 @@ +import { isUtf8 } from "node:buffer"; import { TextDecoder } from "node:util"; import { setImmediate } from "node:timers/promises"; import type { JsonValue } from "../domain/jsonValue.js"; @@ -73,6 +74,13 @@ export class GgufHeader { rejectModel("GGUF name exceeds its format byte limit"); if (retain) this.#budget.charge(6 * length + 2); this.reader.guarantee(length); + // Omitted preview entries still need validation, but not decoded strings. + // Keep the streaming decoder for values spanning multiple read chunks. + if (!retain && length <= 65536) { + if (!isUtf8(await this.reader.take(length))) + rejectModel("GGUF string is not valid UTF-8"); + return ""; + } const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); let result = ""; try { @@ -239,16 +247,21 @@ export class GgufHeader { tensors: ModelTensor[]; }> { const versionBytes = await this.reader.take(4); - this.#little = - versionBytes.readUInt32LE() === 2 || versionBytes.readUInt32LE() === 3; - const version = this.#little - ? versionBytes.readUInt32LE() - : versionBytes.readUInt32BE(); - if (version !== 2 && version !== 3) + const littleVersion = versionBytes.readUInt32LE(); + const bigVersion = versionBytes.readUInt32BE(); + let version: number; + if (littleVersion === 2 || littleVersion === 3) { + this.#little = true; + version = littleVersion; + } else if (bigVersion === 2 || bigVersion === 3) { + this.#little = false; + version = bigVersion; + } else { throw new ModelFormatFailure( "unsupported", - `GGUF version ${String(version)} is unsupported; supported versions are 2 and 3`, + `GGUF version bytes ${versionBytes.toString("hex")} are unsupported (little-endian ${String(littleVersion)}, big-endian ${String(bigVersion)}; byte order unknown); supported versions are 2 and 3`, ); + } if (!this.#little && version !== 3) rejectModel("Big-endian GGUF requires version 3"); const tensorCount = await this.#unsigned(8); diff --git a/src/native/AppleDispatchMetadata.test.ts b/src/native/AppleDispatchMetadata.test.ts index 92250d71bc..9793811c12 100644 --- a/src/native/AppleDispatchMetadata.test.ts +++ b/src/native/AppleDispatchMetadata.test.ts @@ -121,7 +121,7 @@ const fixture = ( const provenance = { path: "/fixture/app", sha256: "a".repeat(64) }; const swiftExternalSwiftBindingImage = ( - boundField: "protocol" | "type" = "protocol", + boundField: "protocol" | "type" | "objc-class" = "protocol", ) => { const bytes = fixture(); const sectionCommand = 32; @@ -141,7 +141,9 @@ const swiftExternalSwiftBindingImage = ( const bind = Buffer.from([ 0x11, // SET_DYLIB_ORDINAL_IMM 1 0x40, - ...Buffer.from(`${boundField === "protocol" ? "_$sSQMp" : "_$sSiMn"}\0`), + ...Buffer.from( + `${boundField === "protocol" ? "_$sSQMp" : boundField === "objc-class" ? "_OBJC_CLASS_$_NSObject" : "_$sSiMn"}\0`, + ), 0x70, 0x80, 0x16, // SET_SEGMENT_AND_OFFSET_ULEB segment 0 + offset 0xb00 @@ -163,12 +165,15 @@ const swiftExternalSwiftBindingImage = ( bytes.writeInt32LE(0x10, 0xa00); bytes.writeInt32LE(boundField === "protocol" ? 0x1ec : 0xec, 0xa14); bytes.writeInt32LE(0, 0xa18); // Nullable witness-table pattern. - bytes.writeUInt32LE(boundField === "protocol" ? 0 : 8, 0xa1c); + bytes.writeUInt32LE( + boundField === "protocol" ? 0 : boundField === "objc-class" ? 24 : 8, + 0xa1c, + ); bytes.writeInt32LE(boundField === "protocol" ? 0xf1 : 0x2f0, 0xa10); // Protocol reference: imported indirect slot or local descriptor. bytes.writeBigUInt64LE(0n, 0xb00); // Bound slot has no in-image pointer. bytes.writeUInt32LE(16, 0xc00); bytes.writeInt32LE(0x1f8, 0xc08); - if (boundField === "type") { + if (boundField !== "protocol") { bytes.writeUInt32LE(3, 0xd00); bytes.writeInt32LE(0xf8, 0xd08); bytes.write("Equatable\0", 0xe00); @@ -237,6 +242,28 @@ const swiftBoundWitnessSlotImage = () => { return bytes; }; +const swiftConformanceTypeReferenceImage = ( + typeKind: number, + input: { + readonly typeTargetOffset?: number; + readonly indirectClassTargetOffset?: number; + } = {}, +) => { + const bytes = swiftBoundWitnessSlotImage(); + const typeTargetOffset = input.typeTargetOffset ?? 0xe60; + bytes.writeUInt32LE(typeKind << 3, 0xa1c); + bytes.writeInt32LE(typeTargetOffset - 0xa14, 0xa14); + if (typeKind === 2 && typeTargetOffset === 0xe60) { + bytes.write("NSObject\0", typeTargetOffset); + } else if (typeKind === 3 && typeTargetOffset === 0xe60) { + bytes.writeBigUInt64LE( + base + BigInt(input.indirectClassTargetOffset ?? 0xe80), + typeTargetOffset, + ); + } + return bytes; +}; + describe("Swift imported protocol references", () => { it("preserves bind identity without reading VM address zero and marks coverage partial", () => { const result = decodeAppleDispatchMetadata( @@ -282,6 +309,7 @@ describe("Swift imported protocol references", () => { expect(result.swift_conformances).toMatchObject([ { type_name: null, + type_reference_kind: 1, type_binding: { symbol: "_$sSiMn", requested_address: "0x100000b00", @@ -298,6 +326,27 @@ describe("Swift imported protocol references", () => { ).toMatchObject({ status: "partial" }); }); + it("preserves a kind-3 Objective-C class import without inventing a class name", () => { + const result = decodeAppleDispatchMetadata( + swiftExternalSwiftBindingImage("objc-class"), + 100, + provenance, + ); + expect(result.swift_conformances).toMatchObject([ + { + type_name: null, + type_reference_kind: 3, + type_reference_target: { address: null, file_offset: null }, + type_binding: { + symbol: "_OBJC_CLASS_$_NSObject", + requested_address: "0x100000b00", + }, + protocol_name: "Equatable", + decode: { status: "partial" }, + }, + ]); + }); + it("reports only the legacy stream mechanisms actually examined", () => { const bytes = swiftBoundWitnessSlotImage(); const dyldInfo = 32 + 232; @@ -325,6 +374,8 @@ describe("Swift imported protocol references", () => { expect(result.swift_conformances).toMatchObject([ { type_name: "Widget", + type_reference_kind: 0, + type_reference_target: { address: "0x100000c00", file_offset: 3072 }, protocol_name: "Equatable", witness_table: { address: "0x100000e00" }, decode: { status: "decoded" }, @@ -350,6 +401,256 @@ describe("Swift imported protocol references", () => { }); }); +describe("Swift conformance type-reference kinds", () => { + it("keeps a direct Objective-C class name and its local witness evidence", () => { + const result = decodeAppleDispatchMetadata( + swiftConformanceTypeReferenceImage(2), + 100, + provenance, + ); + + expect(result.swift_conformances).toMatchObject([ + { + type_name: "NSObject", + type_reference_kind: 2, + type_reference_target: { address: "0x100000e60", file_offset: 3680 }, + protocol_name: "Equatable", + witness_table: { address: "0x100000e00" }, + decode: { status: "decoded" }, + }, + ]); + expect(result.swift_dispatch_slots[0]).toMatchObject({ + owner: "NSObject: Equatable", + table_kind: "witness_table", + implementation_address: null, + implementation_binding: { + symbol: "_externalWitnessImpl", + requested_address: "0x100000e08", + }, + }); + }); +}); + +describe("Swift partial conformance reference retention", () => { + it("retains type and witness facts when the protocol target is unmapped", () => { + const bytes = swiftConformanceTypeReferenceImage(2); + bytes.writeInt32LE(0x5000 - 0xa10, 0xa10); + const result = decodeAppleDispatchMetadata(bytes, 100, provenance); + + expect(result.swift_conformances).toMatchObject([ + { + type_name: "NSObject", + type_reference_target: { address: "0x100000e60", file_offset: 3680 }, + protocol_name: null, + protocol_target: { address: "0x100005000", file_offset: null }, + witness_table: { address: "0x100000e00", file_offset: 3584 }, + decode: { status: "partial" }, + }, + ]); + expect(result.swift_dispatch_slots).toEqual([]); + }); + + it("preserves a typed type binding beside an unmapped protocol without a guessed name", () => { + const bytes = swiftExternalSwiftBindingImage("objc-class"); + bytes.writeInt32LE(0x5000 - 0xa10, 0xa10); + const result = decodeAppleDispatchMetadata(bytes, 100, provenance); + + expect(result.swift_conformances).toMatchObject([ + { + type_name: null, + type_reference_kind: 3, + type_reference_target: { address: null, file_offset: null }, + type_binding: { + symbol: "_OBJC_CLASS_$_NSObject", + requested_address: "0x100000b00", + }, + protocol_name: null, + protocol_target: { address: "0x100005000", file_offset: null }, + decode: { status: "partial" }, + }, + ]); + }); + + it("retains protocol and type targets when the protocol name is unmapped", () => { + const bytes = swiftConformanceTypeReferenceImage(2); + bytes.writeInt32LE(0x5000 - 0xd08, 0xd08); + const result = decodeAppleDispatchMetadata(bytes, 100, provenance); + expect(result.swift_conformances).toMatchObject([ + { + type_name: "NSObject", + protocol_name: null, + protocol_target: { address: "0x100000d00", file_offset: 3328 }, + witness_table: { address: "0x100000e00", file_offset: 3584 }, + decode: { status: "partial" }, + }, + ]); + expect(result.swift_dispatch_slots).toEqual([]); + }); + + it("retains other facts when the protocol descriptor kind is unsupported", () => { + const bytes = swiftConformanceTypeReferenceImage(2); + bytes.writeUInt32LE(4, 0xd00); + const result = decodeAppleDispatchMetadata(bytes, 100, provenance); + + expect(result.swift_conformances).toMatchObject([ + { + type_name: "NSObject", + protocol_name: null, + protocol_target: { address: "0x100000d00", file_offset: 3328 }, + witness_table: { address: "0x100000e00", file_offset: 3584 }, + decode: { status: "partial" }, + }, + ]); + }); + + it("retains other facts when a protocol name is invalid UTF-8", () => { + const bytes = swiftConformanceTypeReferenceImage(2); + bytes.writeUInt8(0xff, 0xe50); + const result = decodeAppleDispatchMetadata(bytes, 100, provenance); + + expect(result.swift_conformances).toMatchObject([ + { + type_name: "NSObject", + protocol_name: null, + protocol_target: { address: "0x100000d00", file_offset: 3328 }, + witness_table: { address: "0x100000e00", file_offset: 3584 }, + decode: { status: "partial" }, + }, + ]); + }); + + it("retains mixed readable witness slots when a later slot is unmapped", () => { + const bytes = swiftConformanceTypeReferenceImage(2); + bytes.writeUInt32LE(2, 0xd10); + bytes.writeInt32LE(0xff0 - 0xa18, 0xa18); + bytes.writeBigUInt64LE(base + 0xa10n, 0xff0); + const result = decodeAppleDispatchMetadata(bytes, 100, provenance); + + expect(result.swift_conformances).toMatchObject([ + { + type_name: "NSObject", + protocol_name: "Equatable", + witness_table: { address: "0x100000ff0", file_offset: 4080 }, + }, + ]); + expect(result.swift_dispatch_slots).toMatchObject([ + { + slot_index: 1, + location: { address: "0x100000ff8", file_offset: 4088 }, + }, + ]); + expect(result.swift_dispatch_slots).toHaveLength(1); + expect( + result.coverage.find( + ({ facet }) => facet === "swift_conformances_static_witness_slots", + ), + ).toMatchObject({ status: "partial" }); + }); +}); + +describe("Swift unresolved conformance type references", () => { + it.each([3, 4, 7] as const)( + "retains an unresolved type-reference kind %i with conformance and witness locations", + (typeKind) => { + const result = decodeAppleDispatchMetadata( + swiftConformanceTypeReferenceImage(typeKind), + 100, + provenance, + ); + + expect(result.swift_conformances).toMatchObject([ + { + type_name: null, + type_reference_kind: typeKind, + ...(typeKind === 3 + ? { + type_reference_target: { + address: "0x100000e80", + file_offset: 3712, + }, + } + : {}), + protocol_name: "Equatable", + witness_table: { address: "0x100000e00" }, + decode: { status: "partial" }, + }, + ]); + expect(result.swift_dispatch_slots).toEqual([]); + expect( + result.coverage.find( + ({ facet }) => facet === "swift_conformances_static_witness_slots", + ), + ).toMatchObject({ status: "partial" }); + }, + ); + + it("retains a kind-2 source location when its inline class name is empty", () => { + const bytes = swiftConformanceTypeReferenceImage(2); + bytes.writeUInt8(0, 0xe60); + const result = decodeAppleDispatchMetadata(bytes, 100, provenance); + + expect(result.swift_conformances).toMatchObject([ + { + type_name: null, + type_reference_kind: 2, + type_reference_target: { address: "0x100000e60", file_offset: 3680 }, + protocol_name: "Equatable", + witness_table: { address: "0x100000e00" }, + decode: { status: "partial" }, + }, + ]); + expect(result.swift_dispatch_slots).toEqual([]); + }); + + it.each([ + { kind: 0, target: 0x5000, actual: 0x5000 }, + { kind: 1, target: 0x5000, actual: 0x5000 }, + { kind: 2, target: 0x5000, actual: 0x5000 }, + { kind: 3, target: 0x5000, actual: 0x5000 }, + { + kind: 3, + target: 0xe60, + indirectClassTarget: 0x5000, + actual: 0x5000, + }, + { kind: 4, target: 0x5000, actual: 0x5000 }, + ])( + "retains partial conformance evidence when kind $kind targets non-file-backed address $target", + ({ kind, target, indirectClassTarget, actual }) => { + const result = decodeAppleDispatchMetadata( + swiftConformanceTypeReferenceImage(kind, { + typeTargetOffset: target, + ...(indirectClassTarget === undefined + ? {} + : { indirectClassTargetOffset: indirectClassTarget }), + }), + 100, + provenance, + ); + + expect(result.swift_conformances).toMatchObject([ + { + type_name: null, + type_reference_kind: kind, + type_reference_target: { + address: `0x${(base + BigInt(actual)).toString(16)}`, + file_offset: null, + }, + protocol_name: "Equatable", + witness_table: { address: "0x100000e00" }, + decode: { status: "partial" }, + }, + ]); + expect(result.swift_dispatch_slots).toEqual([]); + expect( + result.coverage.find( + ({ facet }) => facet === "swift_conformances_static_witness_slots", + ), + ).toMatchObject({ status: "partial" }); + }, + ); +}); + const withProtocolListSection = (bytes: Buffer): Buffer => { const pointer = (at: number, offset: number) => bytes.writeBigUInt64LE(base + BigInt(offset), at); diff --git a/src/native/AppleDispatchMetadata.ts b/src/native/AppleDispatchMetadata.ts index 007e44f8b5..ced870abda 100644 --- a/src/native/AppleDispatchMetadata.ts +++ b/src/native/AppleDispatchMetadata.ts @@ -44,6 +44,7 @@ import { createSwiftRelativeReader, decodeSwiftDispatchFacets, type DispatchRecordBudget, + type SwiftRelativeTargetLocation, } from "./AppleSwiftDispatchFacets.js"; import { decodeSwiftClassVtables } from "./AppleSwiftVtables.js"; import { inspectObjcClassImportBindings } from "./AppleObjcClassImportBindings.js"; @@ -97,6 +98,16 @@ export const decodeAppleDispatchMetadata = ( address: `0x${address.toString(16)}`, file_offset: offset(address), }); + const targetLocation = (address: bigint): SwiftRelativeTargetLocation => { + let fileOffset: number | null; + try { + fileOffset = offset(address); + } catch (cause) { + if (!(cause instanceof RangeError)) throw cause; + fileOffset = null; + } + return { address: `0x${address.toString(16)}`, file_offset: fileOffset }; + }; const evidence = (address: bigint, description: string) => [ { kind: "binary_metadata" as const, @@ -164,6 +175,7 @@ export const decodeAppleDispatchMetadata = ( bound, string, location, + targetLocation, evidence, offset, }; diff --git a/src/native/AppleSwiftConformanceTypeReference.test.ts b/src/native/AppleSwiftConformanceTypeReference.test.ts new file mode 100644 index 0000000000..9f8ab40df3 --- /dev/null +++ b/src/native/AppleSwiftConformanceTypeReference.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; + +import { decodeSwiftConformanceTypeReference } from "./AppleSwiftConformanceTypeReference.js"; +import type { SwiftDispatchReaders } from "./AppleSwiftDispatchFacets.js"; + +describe("Swift conformance type-reference read failures", () => { + it("propagates unexpected reader errors instead of reporting malformed evidence", () => { + const failure = new Error("reader invariant failed"); + const read = { + u32: () => { + throw failure; + }, + } as unknown as SwiftDispatchReaders; + + expect(() => + decodeSwiftConformanceTypeReference({ + descriptor: 0x100n, + flags: 0, + read, + relative: () => ({ kind: "target", target: 0x200n }), + unresolvedReason: () => "unresolved", + }), + ).toThrow(failure); + }); +}); diff --git a/src/native/AppleSwiftConformanceTypeReference.ts b/src/native/AppleSwiftConformanceTypeReference.ts new file mode 100644 index 0000000000..a076574374 --- /dev/null +++ b/src/native/AppleSwiftConformanceTypeReference.ts @@ -0,0 +1,265 @@ +import type { PointerResolution } from "./AppleMachoFixups.js"; +import type { + SwiftDispatchReaders, + SwiftRelativeField, + SwiftRelativeValue, +} from "./AppleSwiftDispatchFacets.js"; +import { expectedSwiftMetadataReadFailure } from "./AppleSwiftMetadataReadFailure.js"; + +const hex = (value: bigint) => `0x${value.toString(16)}`; + +export const swiftImportBinding = ( + requestedAddress: bigint, + binding: Extract, +) => ({ + requested_address: hex(requestedAddress), + symbol: binding.symbol, + binding_source: binding.binding_source, + library_ordinal: binding.library_ordinal, + library: binding.library, + weak: binding.weak, + addend: binding.addend.toString(), +}); + +export interface SwiftConformanceTypeReference { + readonly kind: number; + /** Descriptor, inline class-name, local ObjC class pointer, or raw target. */ + readonly target: bigint | null; + readonly name: string | null; + readonly binding: ReturnType | null; + readonly reason: string | null; + readonly staticWitnessIdentity: boolean; + readonly generic: boolean | null; +} + +interface SwiftTypeReferenceInput { + readonly descriptor: bigint; + readonly flags: number; + readonly read: SwiftDispatchReaders; + readonly relative: ( + field: bigint, + kind: SwiftRelativeField, + ) => SwiftRelativeValue; + readonly unresolvedReason: ( + value: Exclude, + field: SwiftRelativeField, + ) => string; +} + +const typeReferenceResult = (input: { + readonly kind: number; + readonly target: bigint | null; + readonly name?: string | null; + readonly binding?: ReturnType | null; + readonly reason?: string | null; + readonly generic?: boolean | null; +}): SwiftConformanceTypeReference => ({ + kind: input.kind, + target: input.target, + name: input.name ?? null, + binding: input.binding ?? null, + reason: input.reason ?? null, + staticWitnessIdentity: + input.name !== null && input.name !== undefined && input.kind <= 2, + generic: input.generic ?? null, +}); + +export const requiredSwiftRelativeTarget = ( + relative: (field: bigint, kind: SwiftRelativeField) => SwiftRelativeValue, + field: bigint, + kind: SwiftRelativeField, + unresolvedReason: SwiftTypeReferenceInput["unresolvedReason"], +): bigint => { + const value = relative(field, kind); + if (value.kind !== "target") + throw new RangeError(unresolvedReason(value, kind)); + return value.target; +}; + +const decodeNominalTypeReference = ( + kind: 0 | 1, + input: SwiftTypeReferenceInput, + value: SwiftRelativeValue, +): SwiftConformanceTypeReference => { + const { read, relative } = input; + let target = value.kind === "target" ? value.target : null; + let binding = + value.kind === "binding" + ? swiftImportBinding(value.requestedAddress, value.binding) + : null; + let reason = + value.kind === "unknown" + ? value.reason + : value.kind === "null" + ? "Null Swift conformance type reference" + : null; + try { + if (kind === 1 && target !== null) { + const pointer = read.resolvePointer(target); + if (pointer.kind === "bind") { + binding = swiftImportBinding(target, pointer); + target = null; + } else if (pointer.kind === "raw" && pointer.target === 0n) { + target = null; + reason = "Null indirect Swift conformance type reference"; + } else if (pointer.kind === "raw" || pointer.kind === "rebase") { + target = pointer.target; + } else { + target = null; + reason = pointer.reason; + } + } + if (target === null) + return typeReferenceResult({ kind, target, binding, reason }); + + const descriptorFlags = read.u32(target); + if (![16, 17, 18].includes(descriptorFlags & 31)) + return typeReferenceResult({ + kind, + target, + binding, + reason: "Unsupported Swift type descriptor kind", + }); + + const name = read.string( + requiredSwiftRelativeTarget( + relative, + target + 8n, + "typeName", + input.unresolvedReason, + ), + ); + return typeReferenceResult({ + kind, + target, + name: name.length === 0 ? null : name, + binding, + reason: name.length === 0 ? "Empty Swift conformance type name" : null, + generic: (descriptorFlags & 0x80) !== 0, + }); + } catch (cause) { + const reason = expectedSwiftMetadataReadFailure(cause); + if (reason === null) throw cause; + return typeReferenceResult({ + kind, + target, + binding, + reason, + }); + } +}; + +const decodeDirectObjcClassName = ( + kind: 2, + read: SwiftDispatchReaders, + value: SwiftRelativeValue, +): SwiftConformanceTypeReference => { + if (value.kind !== "target") + return typeReferenceResult({ + kind, + target: null, + reason: + value.kind === "unknown" + ? value.reason + : "Null Swift conformance type reference", + }); + try { + const name = read.string(value.target); + return typeReferenceResult({ + kind, + target: value.target, + name: name.length === 0 ? null : name, + reason: name.length === 0 ? "Empty inline Objective-C class name" : null, + }); + } catch (cause) { + const reason = expectedSwiftMetadataReadFailure(cause); + if (reason === null) throw cause; + return typeReferenceResult({ + kind, + target: value.target, + reason, + }); + } +}; + +const decodeIndirectObjcClass = ( + kind: 3, + read: SwiftDispatchReaders, + value: SwiftRelativeValue, +): SwiftConformanceTypeReference => { + if (value.kind !== "target") + return typeReferenceResult({ + kind, + target: null, + reason: + value.kind === "unknown" + ? value.reason + : "Null Swift conformance type reference", + }); + let pointer: PointerResolution; + try { + pointer = read.resolvePointer(value.target); + } catch (cause) { + const reason = expectedSwiftMetadataReadFailure(cause); + if (reason === null) throw cause; + return typeReferenceResult({ + kind, + target: value.target, + reason, + }); + } + if (pointer.kind === "bind") + return typeReferenceResult({ + kind, + target: null, + binding: swiftImportBinding(value.target, pointer), + reason: "Indirect Objective-C class metadata requires runtime lookup", + }); + if (pointer.kind === "raw" || pointer.kind === "rebase") + return typeReferenceResult({ + kind, + target: pointer.target === 0n ? null : pointer.target, + reason: "Indirect Objective-C class metadata requires runtime lookup", + }); + return typeReferenceResult({ + kind, + target: null, + reason: pointer.reason, + }); +}; + +/** Decode source-encoded class identity; ObjC runtime lookup remains unavailable. */ +export const decodeSwiftConformanceTypeReference = ( + input: SwiftTypeReferenceInput, +): SwiftConformanceTypeReference => { + const kind = (input.flags >>> 3) & 7; + try { + const value = input.relative(input.descriptor + 4n, "conformanceType"); + switch (kind) { + case 0: + case 1: + return decodeNominalTypeReference(kind, input, value); + case 2: + return decodeDirectObjcClassName(kind, input.read, value); + case 3: + return decodeIndirectObjcClass(kind, input.read, value); + default: + return typeReferenceResult({ + kind, + target: value.kind === "target" ? value.target : null, + reason: + value.kind === "unknown" + ? value.reason + : `Unsupported Swift conformance type-reference kind ${kind}`, + }); + } + } catch (cause) { + const reason = expectedSwiftMetadataReadFailure(cause); + if (reason === null) throw cause; + return typeReferenceResult({ + kind, + target: null, + reason, + }); + } +}; diff --git a/src/native/AppleSwiftDispatchFacets.ts b/src/native/AppleSwiftDispatchFacets.ts index 3c41cf7007..433d748ec5 100644 --- a/src/native/AppleSwiftDispatchFacets.ts +++ b/src/native/AppleSwiftDispatchFacets.ts @@ -2,6 +2,12 @@ import type { ObjcSwiftMetadata } from "../domain/native/objcSwiftMetadata.js"; import type { PointerResolution } from "./AppleMachoFixups.js"; import type { Segment, Section } from "./AppleMachoSelection.js"; import { issue, type FacetDecodeFacts } from "./AppleDispatchDecodeFacts.js"; +import { + decodeSwiftConformanceTypeReference, + requiredSwiftRelativeTarget, + swiftImportBinding as importBinding, +} from "./AppleSwiftConformanceTypeReference.js"; +import { expectedSwiftMetadataReadFailure } from "./AppleSwiftMetadataReadFailure.js"; /** Resolved file-backed metadata location from the Mach-O layout. */ export interface ResolvedMetadataLocation { @@ -9,6 +15,12 @@ export interface ResolvedMetadataLocation { file_offset: number; } +/** Address fact for a relative target that may not map to file-backed bytes. */ +export interface SwiftRelativeTargetLocation { + address: string; + file_offset: number | null; +} + /** Binary-metadata evidence produced by the Apple dispatch readers. */ export type BinaryMetadataEvidence = { kind: "binary_metadata"; @@ -84,6 +96,84 @@ export const unresolvedSwiftRelativeReason = ( return `Swift relative pointer for ${field} has an unresolved external binding at ${hex(value.requestedAddress)}`; }; +interface SwiftProtocolFacts { + readonly name: string | null; + readonly readable: boolean; + readonly reason: string | null; +} + +const readSwiftProtocolFacts = (input: { + readonly target: bigint | null; + readonly relative: ( + field: bigint, + kind: SwiftRelativeField, + ) => SwiftRelativeValue; + readonly read: SwiftDispatchReaders; +}): SwiftProtocolFacts => { + const { target, relative, read } = input; + if (target === null) + return { + name: null, + readable: false, + reason: "Protocol descriptor target is unresolved", + }; + let descriptorKind: number; + try { + descriptorKind = read.u32(target) & 31; + } catch (cause) { + const reason = expectedSwiftMetadataReadFailure(cause); + if (reason === null) throw cause; + return { name: null, readable: false, reason }; + } + if (descriptorKind !== 3) + return { + name: null, + readable: false, + reason: "Unsupported Swift protocol descriptor kind", + }; + let nameAddress: bigint; + try { + nameAddress = requiredSwiftRelativeTarget( + relative, + target + 8n, + "protocolName", + unresolvedSwiftRelativeReason, + ); + } catch (cause) { + const reason = expectedSwiftMetadataReadFailure(cause); + if (reason === null) throw cause; + return { name: null, readable: false, reason }; + } + try { + return { name: read.string(nameAddress), readable: true, reason: null }; + } catch (cause) { + const reason = expectedSwiftMetadataReadFailure(cause); + if (reason === null) throw cause; + return { name: null, readable: false, reason }; + } +}; + +const readWitnessConformanceHeader = ( + read: SwiftDispatchReaders, + witness: bigint, + descriptor: bigint, +): string | null => { + try { + const header = read.resolvePointer(witness); + if (header.kind !== "raw" && header.kind !== "rebase") + return header.kind === "unknown" || header.kind === "unsupported" + ? header.reason + : "Witness-table conformance header is an external binding"; + return header.target === descriptor + ? null + : "Witness-table conformance header is unresolved or encoded"; + } catch (cause) { + const reason = expectedSwiftMetadataReadFailure(cause); + if (reason === null) throw cause; + return reason; + } +}; + /** Byte readers for Swift relative-pointer and conformance decode facets. */ export interface SwiftDispatchReaders { u32(address: bigint): number; @@ -91,25 +181,13 @@ export interface SwiftDispatchReaders { resolvePointer(address: bigint): PointerResolution; string(address: bigint): string; location(address: bigint): ResolvedMetadataLocation; + targetLocation(address: bigint): SwiftRelativeTargetLocation; evidence(address: bigint, description: string): BinaryMetadataEvidence; offset(address: bigint, size?: number): number; } const hex = (value: bigint) => `0x${value.toString(16)}`; const decoded = { status: "decoded" as const, reason: null }; -const importBinding = ( - requestedAddress: bigint, - binding: Extract, -) => ({ - requested_address: hex(requestedAddress), - symbol: binding.symbol, - binding_source: binding.binding_source, - library_ordinal: binding.library_ordinal, - library: binding.library, - weak: binding.weak, - addend: binding.addend.toString(), -}); - /** Create a Swift ABI signed-relative pointer decoder that records relative_pointers. */ export const createSwiftRelativeReader = (input: { bytes: Buffer; @@ -239,17 +317,7 @@ export const decodeSwiftDispatchFacets = (input: { result: ObjcSwiftMetadata; }): FacetDecodeFacts => { const { sections, segments, readers: read, relative, budget, result } = input; - const swiftFailures: string[] = []; const issues = [] as ReturnType[]; - const requiredRelative = ( - field: bigint, - kind: SwiftRelativeField, - ): bigint => { - const value = relative(field, kind); - if (value.kind !== "target") - throw new RangeError(unresolvedSwiftRelativeReason(value, kind)); - return value.target; - }; let swiftExamined = 0; let decodedRecords = 0; for (const section of sections.filter( @@ -262,7 +330,12 @@ export const decodeSwiftDispatchFacets = (input: { swiftExamined++; const field = section.address + BigInt(index * 4); try { - const descriptor = requiredRelative(field, "conformanceRecord"); + const descriptor = requiredSwiftRelativeTarget( + relative, + field, + "conformanceRecord", + unresolvedSwiftRelativeReason, + ); const flags = read.u32(descriptor + 12n); const protocolValue = relative(descriptor, "conformanceProtocol"); const protocol = @@ -274,12 +347,12 @@ export const decodeSwiftDispatchFacets = (input: { "conformanceProtocol", ) : null; - if (protocol !== null && (read.u32(protocol) & 31) !== 3) - throw new TypeError("Unsupported Swift protocol descriptor kind"); - const protocolName = - protocol === null - ? null - : read.string(requiredRelative(protocol + 8n, "protocolName")); + const protocolFacts = readSwiftProtocolFacts({ + target: protocol, + relative, + read, + }); + const protocolName = protocolFacts.name; const protocolBinding = protocolValue.kind === "binding" ? importBinding( @@ -287,107 +360,85 @@ export const decodeSwiftDispatchFacets = (input: { protocolValue.binding, ) : null; - const typeKind = (flags >>> 3) & 7; - if (typeKind !== 0 && typeKind !== 1) - throw new TypeError( - "Unsupported Swift conformance type-reference kind", - ); - const typeValue = relative(descriptor + 4n, "conformanceType"); - let type = typeValue.kind === "target" ? typeValue.target : null; - let typeBinding = - typeValue.kind === "binding" - ? importBinding(typeValue.requestedAddress, typeValue.binding) - : null; - let unresolvedTypeReason = - typeValue.kind === "unknown" ? typeValue.reason : null; - if (typeKind === 1 && type !== null) { - const pointer = read.resolvePointer(type); - if (pointer.kind === "bind") { - typeBinding = importBinding(type, pointer); - type = null; - } else if (pointer.kind === "raw" && pointer.target === 0n) { - type = null; - unresolvedTypeReason = - "Null indirect Swift conformance type reference"; - } else if (pointer.kind === "rebase" || pointer.kind === "raw") { - type = pointer.target; - } else { - type = null; - unresolvedTypeReason = pointer.reason; - } - } - if (type !== null && ![16, 17, 18].includes(read.u32(type) & 31)) - throw new TypeError("Unsupported Swift type descriptor kind"); - const typeName = - type === null - ? null - : read.string(requiredRelative(type + 8n, "typeName")); - const witnessValue = - protocol !== null && type !== null - ? relative(descriptor + 8n, "witnessTablePattern") - : { kind: "null" as const }; + const typeReference = decodeSwiftConformanceTypeReference({ + descriptor, + flags, + read, + relative, + unresolvedReason: unresolvedSwiftRelativeReason, + }); + const witnessValue = relative(descriptor + 8n, "witnessTablePattern"); const witness = witnessValue.kind === "target" ? witnessValue.target : null; const unresolvedWitnessReason = witnessValue.kind === "unknown" || witnessValue.kind === "binding" ? unresolvedSwiftRelativeReason(witnessValue, "witnessTablePattern") : null; - const staticWitness = + let staticWitness = witness !== null && protocol !== null && - type !== null && - (flags & ~0x38) === 0 && - (read.u32(type) & 0x80) === 0; + protocolFacts.readable && + typeReference.staticWitnessIdentity && + (flags & ~0x38) === 0; + staticWitness &&= + typeReference.kind === 2 || typeReference.generic === false; let witnessReason: string | null = staticWitness ? null : (unresolvedProtocolReason ?? - unresolvedTypeReason ?? + protocolFacts.reason ?? + typeReference.reason ?? unresolvedWitnessReason ?? - (protocolBinding !== null || typeBinding !== null + (protocolBinding !== null || typeReference.binding !== null ? "Conformance references an external Mach-O binding" : "Conditional, generic, resilient or missing static witness table is not decoded")); - if (staticWitness) { - const header = read.resolvePointer(witness); - if (header.kind !== "raw" && header.kind !== "rebase") - witnessReason = - header.kind === "unknown" || header.kind === "unsupported" - ? header.reason - : "Witness-table conformance header is an external binding"; - else if (header.target !== descriptor) - witnessReason = - "Witness-table conformance header is unresolved or encoded"; + if (staticWitness && witness !== null) { + witnessReason = readWitnessConformanceHeader( + read, + witness, + descriptor, + ); } const conformanceDecode = witnessReason === null ? { status: "decoded" as const, reason: null } : { status: "partial" as const, reason: witnessReason }; result.swift_conformances.push({ - type_name: typeName, - type_binding: typeBinding, + type_name: typeReference.name, + type_reference_kind: typeReference.kind, + type_reference_target: + typeReference.target === null + ? { address: null, file_offset: null } + : read.targetLocation(typeReference.target), + type_binding: typeReference.binding, protocol_name: protocolName, + ...(protocol === null + ? {} + : { protocol_target: read.targetLocation(protocol) }), protocol_binding: protocolBinding, module: null, witness_table: witness === null ? { address: null, file_offset: null } - : read.location(witness), + : read.targetLocation(witness), location: read.location(descriptor), decode: conformanceDecode, evidence: read.evidence( descriptor, - "Swift protocol conformance descriptor and directly encoded type/protocol names", + "Swift protocol conformance descriptor, type reference and protocol name", ), }); if (conformanceDecode.status === "decoded") decodedRecords++; if ( witnessReason !== null || witness === null || - typeName === null || - protocolName === null + typeReference.name === null || + protocolName === null || + !protocolFacts.readable ) { const reason = - witnessReason ?? "Missing static witness table is not decoded"; - swiftFailures.push(`${typeName ?? ""}: ${reason}`); + witnessReason ?? + typeReference.reason ?? + "Missing static witness table is not decoded"; issues.push( issue( "swift_conformances_static_witness_slots", @@ -405,75 +456,87 @@ export const decodeSwiftDispatchFacets = (input: { for (let slot = 0; slot < count; slot++) { if (!budget.admit()) break; const entry = witness + BigInt((slot + 1) * 8); - const implementationResolution = read.resolvePointer(entry); - const implementation = - implementationResolution.kind === "raw" || - implementationResolution.kind === "rebase" - ? implementationResolution.target - : null; - const requirementFlags = read.u32( - protocol! + 24n + BigInt(signatureCount * 12 + slot * 8), - ); - const kind = requirementFlags & 15; - const synchronousFunction = - kind >= 1 && kind <= 4 && (requirementFlags & 0x20) === 0; - const executable = - implementation !== null && - synchronousFunction && - segments.some( - (segment) => - segment.executable && - implementation >= segment.address && - implementation < segment.address + segment.size, + try { + const implementationResolution = read.resolvePointer(entry); + const implementation = + implementationResolution.kind === "raw" || + implementationResolution.kind === "rebase" + ? implementationResolution.target + : null; + const requirementFlags = read.u32( + protocol! + 24n + BigInt(signatureCount * 12 + slot * 8), ); - const slotDecode = executable - ? { status: "decoded" as const, reason: null } - : { - status: "partial" as const, - reason: - implementationResolution.kind === "bind" - ? "Witness implementation is an external Mach-O binding" - : implementationResolution.kind === "unknown" || - implementationResolution.kind === "unsupported" - ? implementationResolution.reason - : "Witness slot is non-function, async, nil, external, authenticated or encoded; implementation target is unresolved", - }; - result.swift_dispatch_slots.push({ - owner: `${typeName}: ${protocolName}`, - table_kind: "witness_table", - slot_index: slot + 1, - requirement: null, - implementation: null, - implementation_address: executable ? hex(implementation) : null, - ...(implementationResolution.kind === "bind" - ? { - implementation_binding: importBinding( - entry, - implementationResolution, - ), - } - : {}), - thunk_address: null, - location: read.location(entry), - decode: slotDecode, - evidence: read.evidence( - entry, - "Static Swift witness slot, indexed after the conformance-descriptor header", - ), - }); - if (slotDecode.status !== "decoded") + const kind = requirementFlags & 15; + const synchronousFunction = + kind >= 1 && kind <= 4 && (requirementFlags & 0x20) === 0; + const executable = + implementation !== null && + synchronousFunction && + segments.some( + (segment) => + segment.executable && + implementation >= segment.address && + implementation < segment.address + segment.size, + ); + const slotDecode = executable + ? { status: "decoded" as const, reason: null } + : { + status: "partial" as const, + reason: + implementationResolution.kind === "bind" + ? "Witness implementation is an external Mach-O binding" + : implementationResolution.kind === "unknown" || + implementationResolution.kind === "unsupported" + ? implementationResolution.reason + : "Witness slot is non-function, async, nil, external, authenticated or encoded; implementation target is unresolved", + }; + result.swift_dispatch_slots.push({ + owner: `${typeReference.name}: ${protocolName}`, + table_kind: "witness_table", + slot_index: slot + 1, + requirement: null, + implementation: null, + implementation_address: executable ? hex(implementation) : null, + ...(implementationResolution.kind === "bind" + ? { + implementation_binding: importBinding( + entry, + implementationResolution, + ), + } + : {}), + thunk_address: null, + location: read.location(entry), + decode: slotDecode, + evidence: read.evidence( + entry, + "Static Swift witness slot, indexed after the conformance-descriptor header", + ), + }); + if (slotDecode.status !== "decoded") + issues.push( + issue( + "swift_conformances_static_witness_slots", + "witness_slot_unresolved", + slotDecode.reason, + hex(entry), + ), + ); + } catch (cause) { + if (!(cause instanceof RangeError)) throw cause; issues.push( issue( "swift_conformances_static_witness_slots", "witness_slot_unresolved", - slotDecode.reason, + cause.message, hex(entry), ), ); + break; + } } } catch (cause) { const message = cause instanceof Error ? cause.message : String(cause); - swiftFailures.push(`${hex(field)}: ${message}`); issues.push( issue( "swift_conformances_static_witness_slots", diff --git a/src/native/AppleSwiftMetadataReadFailure.ts b/src/native/AppleSwiftMetadataReadFailure.ts new file mode 100644 index 0000000000..8eceddf87f --- /dev/null +++ b/src/native/AppleSwiftMetadataReadFailure.ts @@ -0,0 +1,13 @@ +/** Expected format-boundary failures from bounded Swift metadata reads. */ +export const expectedSwiftMetadataReadFailure = ( + cause: unknown, +): string | null => { + if (cause instanceof RangeError) return cause.message; + if ( + cause instanceof TypeError && + "code" in cause && + cause.code === "ERR_ENCODING_INVALID_ENCODED_DATA" + ) + return cause.message; + return null; +}; diff --git a/src/native/AppleSwiftRelativePointer.test.ts b/src/native/AppleSwiftRelativePointer.test.ts index e394a8ddd2..2d7d60e8d0 100644 --- a/src/native/AppleSwiftRelativePointer.test.ts +++ b/src/native/AppleSwiftRelativePointer.test.ts @@ -18,6 +18,10 @@ const readers = (bytes: Buffer) => ({ address: `0x${address.toString(16)}`, file_offset: Number(address), }), + targetLocation: (address: bigint) => ({ + address: `0x${address.toString(16)}`, + file_offset: Number(address), + }), evidence: (address: bigint, description: string) => [ { kind: "binary_metadata" as const, diff --git a/src/native/AppleSwiftVtables.test.ts b/src/native/AppleSwiftVtables.test.ts index c1e5fb4b8f..ddb47fc98a 100644 --- a/src/native/AppleSwiftVtables.test.ts +++ b/src/native/AppleSwiftVtables.test.ts @@ -52,6 +52,10 @@ it("preserves a required conformance pointer's unknown reason", () => { address: `0x${address.toString(16)}`, file_offset: Number(address), }), + targetLocation: (address) => ({ + address: `0x${address.toString(16)}`, + file_offset: Number(address), + }), evidence: () => [], offset: (address) => Number(address), }, @@ -203,6 +207,10 @@ it("returns a located issue for an emitted unresolved witness slot", () => { address: `0x${address.toString(16)}`, file_offset: Number(address), }), + targetLocation: (address) => ({ + address: `0x${address.toString(16)}`, + file_offset: Number(address), + }), evidence: (address, description) => [ { kind: "binary_metadata", diff --git a/src/native/NativeMacOSProvider.ts b/src/native/NativeMacOSProvider.ts index e48422cc7e..8c8f9ebc57 100644 --- a/src/native/NativeMacOSProvider.ts +++ b/src/native/NativeMacOSProvider.ts @@ -1,9 +1,12 @@ import { plistSourceFormat, + isBinaryPlistDescription, + readNativePlistBytes, requireRegularPlist, nativePlistFailure, resolveNativePlistSelection, unreadablePlist, + type NativePlistSelection, } from "./NativePlistSource.js"; import { bindArtifactFileCleanup } from "../artifacts/ArtifactFileOperation.js"; import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; @@ -50,7 +53,6 @@ import type { EvidenceLocation } from "../domain/evidence.js"; import { AnalysisCancelledError, AnalysisCapabilityUnavailableError, - AnalysisInputError, AnalysisOutputError, AnalysisResourceConstraintError, AnalysisTimeoutError, @@ -105,8 +107,10 @@ import { parseDemangledSymbols } from "./parsers/demangle.js"; import { parseLipoOutput } from "./parsers/lipo.js"; import { parsePlistJson, + parsePlistBinary, parsePlistXml, plistJsonNeedsNumberTypes, + type ParsedPlist, } from "./parsers/plist.js"; import { architectureLocations, @@ -128,6 +132,12 @@ type SignatureRuntimeRootFactory = () => Promise< Pick >; +interface NativePlistReadContext { + readonly selection: NativePlistSelection; + readonly resources: ArtifactResourceScope; + readonly signal: AbortSignal | undefined; +} + /** Read-only semantic provider composed from Xcode command-line utilities. */ export class NativeMacOSProvider implements AnalysisProvider { readonly #capabilities: readonly CapabilityDescriptor[]; @@ -667,6 +677,10 @@ class NativeMacOSClient implements AnalysisClient { binding: SignatureTargetBinding, signal?: AbortSignal, ): Promise> { + const executablePaths = [ + binding.snapshotPath, + ...(await canonicalPath(binding.snapshotPath)), + ]; const display = await this.#captureSignature( binding, ["-d", "--verbose=4"], @@ -676,10 +690,12 @@ class NativeMacOSClient implements AnalysisClient { const unsigned = isNonzeroUnsignedObservation( display.value, binding.snapshotPath, + executablePaths, ); const displayFailure = codeSignCaptureFailure( display.value, binding.snapshotPath, + executablePaths, ); if (displayFailure !== null) return err(displayFailure); const requirements = await this.#captureSignature( @@ -691,6 +707,7 @@ class NativeMacOSClient implements AnalysisClient { const requirementsFailure = codeSignCaptureFailure( requirements.value, binding.snapshotPath, + executablePaths, ); if (requirementsFailure !== null) return err(requirementsFailure); const entitlements = await this.#captureSignature( @@ -702,13 +719,15 @@ class NativeMacOSClient implements AnalysisClient { const entitlementsFailure = codeSignCaptureFailure( entitlements.value, binding.snapshotPath, + executablePaths, ); if (entitlementsFailure !== null) return err(entitlementsFailure); // codesign echoes the resolved executable path in its diagnostics. - const parsed = parseCodeSignature(display.value.stderr, unsigned, [ - binding.snapshotPath, - ...(await canonicalPath(binding.snapshotPath)), - ]); + const parsed = parseCodeSignature( + display.value.stderr, + unsigned, + executablePaths, + ); // The requirement is printed to stdout; stderr echoes the path. const requirementText = /designated\s*=>\s*(.+)$/mu.exec(requirements.value.stdout)?.[1] ?? null; @@ -727,49 +746,79 @@ class NativeMacOSClient implements AnalysisClient { ]; const mixedSigning = !unsigned && - (isNonzeroUnsignedObservation(requirements.value, binding.snapshotPath) || - isNonzeroUnsignedObservation(entitlements.value, binding.snapshotPath)); + (isNonzeroUnsignedObservation( + requirements.value, + binding.snapshotPath, + executablePaths, + ) || + isNonzeroUnsignedObservation( + entitlements.value, + binding.snapshotPath, + executablePaths, + )); + const observation = () => + inspectSignatureSchema.parse({ + ...parsed, + designated_requirement: requirementText, + entitlements: entitlementValue.value, + provenance: captures.map((capture) => + invocation(capture, binding.snapshotPath), + ), + limitations, + }); if (mixedSigning) { + limitations.push( + "Partial signature coverage: aggregate signing metadata describes codesign’s default slice; Mach-O slices have mixed signing states.", + ); const slices = await this.#inspectMixedSignatureSlices( binding, parsed.format, + executablePaths, + { captures, limitations }, { requirements: requirements.value.exitCode, entitlements: entitlements.value.exitCode, }, signal, ); - if (!slices.ok) return slices; - captures.push(...slices.value.captures); - limitations.push(...slices.value.limitations); + if (!slices.ok) { + limitations.push( + "Architecture-specific signature inspection did not complete; remaining slice signing states are unknown.", + ); + return err(slices.error.retainPartialObservation(observation())); + } } - const provenance = captures.map((capture) => - invocation(capture, binding.snapshotPath), - ); const version = await verifySignatureTarget(this.target, binding, signal); - if (!version.ok) return version; - const result = inspectSignatureSchema.parse({ - ...parsed, - designated_requirement: requirementText, - entitlements: entitlementValue.value, - provenance, - limitations, - }); - return ok(result); + if (!version.ok) + return err(version.error.retainPartialObservation(observation())); + return ok(observation()); } async #inspectMixedSignatureSlices( binding: SignatureTargetBinding, format: string | null, + executablePaths: readonly string[], + observation: { captures: NativeCommandCapture[]; limitations: string[] }, aggregateExits: Readonly<{ requirements: number | null; entitlements: number | null; }>, signal?: AbortSignal, - ): Promise> { - const captures: NativeCommandCapture[] = []; + ): Promise> { + const { captures, limitations } = observation; const unsigned: string[] = []; - const unclassified: string[] = []; + const unsignedNote = limitations.length; + limitations.push( + "One or more Mach-O slices are unsigned, but their architectures could not be classified.", + ); + if (aggregateExits.requirements !== 0) + limitations.push( + "The aggregate designated requirement is unavailable because Mach-O slices have mixed signing states.", + ); + if (aggregateExits.entitlements !== 0) + limitations.push( + "The aggregate entitlements are unavailable because Mach-O slices have mixed signing states.", + ); for (const architecture of codeSignArchitectures(format, this.target)) { const slice = await this.#captureSignature( binding, @@ -779,28 +828,17 @@ class NativeMacOSClient implements AnalysisClient { if (!slice.ok) return slice; captures.push(slice.value); if (slice.value.exitCode === 0) continue; - if (isNonzeroUnsignedObservation(slice.value, binding.snapshotPath)) - unsigned.push(architecture); - else unclassified.push(architecture); + const failure = codeSignCaptureFailure( + slice.value, + binding.snapshotPath, + executablePaths, + ); + if (failure !== null) return err(failure); + unsigned.push(architecture); + limitations[unsignedNote] = + `Unsigned Mach-O slices: ${unsigned.join(", ")}.`; } - const limitations = [ - unsigned.length > 0 - ? `Unsigned Mach-O slices: ${unsigned.join(", ")}.` - : "One or more Mach-O slices are unsigned, but their architectures could not be classified.", - ]; - if (unclassified.length > 0) - limitations.push( - `Signature state could not be classified for Mach-O slices: ${unclassified.join(", ")}.`, - ); - if (aggregateExits.requirements !== 0) - limitations.push( - "The aggregate designated requirement is unavailable because Mach-O slices have mixed signing states.", - ); - if (aggregateExits.entitlements !== 0) - limitations.push( - "The aggregate entitlements are unavailable because Mach-O slices have mixed signing states.", - ); - return ok({ captures, limitations }); + return ok(undefined); } async #captureSignature( @@ -859,7 +897,7 @@ class NativeMacOSClient implements AnalysisClient { requested, ); if (!plist.ok) return plist; - const fileInput = { + const fileInput: NativePlistReadContext = { selection: plist.value, resources: this.#plistResources, signal, @@ -873,98 +911,12 @@ class NativeMacOSClient implements AnalysisClient { { signal }, ); if (!classified.ok) return classified; - const json = await this.#run( - "inspect_plist", - "plutil", - ["-convert", "json", "-o", "-", "--", plist.value.path], - { signal, acceptNonZero: true }, - ); - if (!json.ok) return json; - // JSON cannot express data, dates, or non-finite reals; plutil rejects - // such plists, so decode its lossless XML conversion instead. JSON also - // prints an integral like an , so a number beyond the - // exact JSON range takes its element type from the XML conversion. - const jsonDecoded = json.value.exitCode === 0; - const xml = - jsonDecoded && !plistJsonNeedsNumberTypes(json.value.stdout) - ? undefined - : await this.#run( - "inspect_plist", - "plutil", - ["-convert", "xml1", "-o", "-", "--", plist.value.path], - { signal, acceptNonZero: !jsonDecoded }, - ); - if (xml !== undefined && !xml.ok) return xml; - if (xml !== undefined && xml.value.exitCode !== 0) { - const capture = failureCapture(xml.value, false); - return err( - unreadablePlist( - plist.value, - `plutil could not decode it as a property list (${plutilDiagnostic(capture, plist.value.path)})`, - { - capturedOutput: { - stdout: capture.stdout, - stderr: capture.stderr, - truncated: capture.truncated, - stdout_bytes: capture.stdoutBytes, - stderr_bytes: capture.stderrBytes, - exit_code: capture.exitCode, - signal: capture.signal, - }, - }, - ), - ); - } - const parsed = - xml === undefined - ? parsePlistJson(json.value.stdout) - : jsonDecoded - ? parsePlistJson(json.value.stdout, xml.value.stdout) - : parsePlistXml(xml.value.stdout); - if (!parsed.ok) return parsed; - const provenance = [ + const binary = await this.#inspectBinaryPlist( + fileInput, classified.value, - json.value, - ...(xml === undefined ? [] : [xml.value]), - ].map((item) => invocation(item, plist.value.path, "$PLIST")); - const converted = xml?.value ?? json.value; - const format = await plistSourceFormat(classified.value.stdout, { - ...fileInput, - capturedOutput: { - stdout: converted.stdout, - stderr: converted.stderr, - truncated: false, - stdout_bytes: converted.stdoutBytes, - stderr_bytes: converted.stderrBytes, - exit_code: converted.exitCode, - signal: converted.signal, - }, - }); - if (!format.ok) return format; - const result = inspectPlistSchema.parse({ - format: format.value, - value: parsed.value.value, - bundle: parsed.value.bundle, - source_path: plist.value.path, - ...(this.bundleResourceRoot === undefined - ? {} - : { - resource_root: { - caller_path: this.bundleResourceRoot.callerPath, - canonical_path: this.bundleResourceRoot.canonicalPath, - info_plist_path: this.bundleResourceRoot.infoPlistPath, - layout: this.bundleResourceRoot.layout, - executable_sha256: this.bundleResourceRoot.executableSha256, - }, - }), - provenance, - limitations: parsed.value.limitations, - }); - return ok({ - result: jsonValueSchema.parse(result), - limitations: parsed.value.limitations, - locations: [{ kind: "artifact-path", path: plist.value.path }], - }); + ); + if (binary !== undefined) return binary; + return await this.#inspectConvertedPlist(fileInput, classified.value); } catch (cause: unknown) { // Producer failures stay opaque; only scope admission failures reach the file mapper. return err(nativeOutputFailure("inspect_plist", cause)); @@ -975,6 +927,122 @@ class NativeMacOSClient implements AnalysisClient { } } + async #inspectBinaryPlist( + fileInput: NativePlistReadContext, + classified: NativeCommandCapture, + ): Promise | undefined> { + if (!isBinaryPlistDescription(classified.stdout)) return undefined; + const source = await readNativePlistBytes(fileInput); + if (!source.ok) return source; + const parsed = parsePlistBinary(source.value); + if (!parsed.ok) { + return parsed.error instanceof AnalysisOutputError + ? err( + unreadablePlist( + fileInput.selection, + "binary property list could not be decoded", + { cause: parsed.error }, + ), + ) + : parsed; + } + return ok( + nativePlistObservation({ + parsed: parsed.value, + format: "binary", + path: fileInput.selection.path, + resourceRoot: this.bundleResourceRoot, + provenance: [ + invocation(classified, fileInput.selection.path, "$PLIST"), + ], + }), + ); + } + + async #inspectConvertedPlist( + fileInput: NativePlistReadContext, + classified: NativeCommandCapture, + ): Promise> { + const { path } = fileInput.selection; + const { signal } = fileInput; + const json = await this.#run( + "inspect_plist", + "plutil", + ["-convert", "json", "-o", "-", "--", path], + { signal, acceptNonZero: true }, + ); + if (!json.ok) return json; + // JSON cannot express data, dates, or non-finite reals; plutil rejects + // such plists, so decode its XML conversion instead. JSON also prints an + // integral like an , so XML identifies unsafe number types. + const jsonDecoded = json.value.exitCode === 0; + const xml = + jsonDecoded && !plistJsonNeedsNumberTypes(json.value.stdout) + ? undefined + : await this.#run( + "inspect_plist", + "plutil", + ["-convert", "xml1", "-o", "-", "--", path], + { signal, acceptNonZero: !jsonDecoded }, + ); + if (xml !== undefined && !xml.ok) return xml; + if (xml !== undefined && xml.value.exitCode !== 0) { + const capture = failureCapture(xml.value, false); + return err( + unreadablePlist( + fileInput.selection, + `plutil could not decode it as a property list (${plutilDiagnostic(capture, path)})`, + { + capturedOutput: { + stdout: capture.stdout, + stderr: capture.stderr, + truncated: capture.truncated, + stdout_bytes: capture.stdoutBytes, + stderr_bytes: capture.stderrBytes, + exit_code: capture.exitCode, + signal: capture.signal, + }, + }, + ), + ); + } + const parsed = + xml === undefined + ? parsePlistJson(json.value.stdout) + : jsonDecoded + ? parsePlistJson(json.value.stdout, xml.value.stdout) + : parsePlistXml(xml.value.stdout); + if (!parsed.ok) return parsed; + const provenance = [ + classified, + json.value, + ...(xml === undefined ? [] : [xml.value]), + ].map((item) => invocation(item, path, "$PLIST")); + const converted = xml?.value ?? json.value; + const format = await plistSourceFormat(classified.stdout, { + ...fileInput, + capturedOutput: { + stdout: converted.stdout, + stderr: converted.stderr, + truncated: false, + stdout_bytes: converted.stdoutBytes, + stderr_bytes: converted.stderrBytes, + exit_code: converted.exitCode, + signal: converted.signal, + }, + }); + if (!format.ok) return format; + return ok( + nativePlistObservation({ + parsed: parsed.value, + format: format.value, + path, + resourceRoot: this.bundleResourceRoot, + provenance, + }), + ); + } + async #inspectMacho( signal?: AbortSignal, ): Promise> { @@ -1131,11 +1199,6 @@ interface NativeObservation { readonly locations: readonly EvidenceLocation[]; } -interface MixedSignatureSlices { - readonly captures: readonly NativeCommandCapture[]; - readonly limitations: readonly string[]; -} - const isNativeOperation = ( operation: AnalysisOperation, ): operation is NativeToolName => @@ -1318,13 +1381,15 @@ const translateCodeSignExitFailure = ( ); /** - * Accept only codesign's exact unsigned diagnostic for the bound snapshot. + * Accept only codesign's exact unsigned diagnostic for the bound snapshot, + * optionally preceded by its verified Executable path echo. * The operand is framed byte-for-byte so caller-controlled path text cannot * supply the phrase that determines signature state. */ const isNonzeroUnsignedObservation = ( capture: NativeCommandCapture, snapshotPath: string, + executablePaths: readonly string[], ): boolean => { if ( capture.exitCode === null || @@ -1335,8 +1400,16 @@ const isNonzeroUnsignedObservation = ( ) return false; const diagnostic = `${snapshotPath}: code object is not signed at all`; - return [diagnostic, `${diagnostic}\n`, `${diagnostic}\r\n`].includes( - capture.stderr, + return [ + "", + ...executablePaths.flatMap((path) => [ + `Executable=${path}\n`, + `Executable=${path}\r\n`, + ]), + ].some((prefix) => + [diagnostic, `${diagnostic}\n`, `${diagnostic}\r\n`].some( + (body) => capture.stderr === `${prefix}${body}`, + ), ); }; @@ -1351,8 +1424,10 @@ const canonicalPath = async (path: string): Promise => { const codeSignCaptureFailure = ( capture: NativeCommandCapture, snapshotPath: string, + executablePaths: readonly string[], ): AnalysisError | null => - capture.exitCode === 0 || isNonzeroUnsignedObservation(capture, snapshotPath) + capture.exitCode === 0 || + isNonzeroUnsignedObservation(capture, snapshotPath, executablePaths) ? null : translateCodeSignExitFailure(capture); @@ -1397,6 +1472,40 @@ const invocation = ( stderr_bytes: capture.stderrBytes, }); +const nativePlistObservation = (input: { + readonly parsed: ParsedPlist; + readonly format: z.infer["format"]; + readonly path: string; + readonly resourceRoot: ResolvedAppBundleResourceRoot | undefined; + readonly provenance: readonly NativeCommandInvocation[]; +}): NativeObservation => { + const { parsed, format, path, resourceRoot, provenance } = input; + const result = inspectPlistSchema.parse({ + format, + value: parsed.value, + bundle: parsed.bundle, + source_path: path, + ...(resourceRoot === undefined + ? {} + : { + resource_root: { + caller_path: resourceRoot.callerPath, + canonical_path: resourceRoot.canonicalPath, + info_plist_path: resourceRoot.infoPlistPath, + layout: resourceRoot.layout, + executable_sha256: resourceRoot.executableSha256, + }, + }), + provenance, + limitations: parsed.limitations, + }); + return { + result: jsonValueSchema.parse(result), + limitations: parsed.limitations, + locations: [{ kind: "artifact-path", path }], + }; +}; + const parseEntitlements = ( output: string, ): { diff --git a/src/native/NativePlistSource.ts b/src/native/NativePlistSource.ts index 8c6f2e5828..6b66a9f345 100644 --- a/src/native/NativePlistSource.ts +++ b/src/native/NativePlistSource.ts @@ -11,9 +11,11 @@ import type { AnalysisCapturedOutput, } from "../domain/analysisErrorBase.js"; import { + AnalysisArtifactChangedError, AnalysisCancelledError, AnalysisCapabilityUnavailableError, AnalysisInputError, + AnalysisResourceConstraintError, } from "../domain/analysisErrorCore.js"; import { BinaryTargetError } from "../domain/configurationErrors.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; @@ -26,10 +28,12 @@ import { artifactFilePrimaryFailure, artifactFileCancellationCause, bindArtifactFileCleanup, + assertArtifactReadActive, withArtifactRegularFile, } from "../artifacts/ArtifactFileOperation.js"; import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { NATIVE_MACOS_PROVIDER_IDENTITY as IDENTITY } from "./NativeMacOSProviderMetadata.js"; +import { MAX_NATIVE_PLIST_BYTES } from "./parsers/plist.js"; /** Resolved lookup identity and selection intent, independent of file readiness. */ export interface NativePlistSelection { @@ -197,6 +201,29 @@ export const nativePlistFailure = ( cause.message, { cause, capturedOutput }, ); + if ( + selection !== undefined && + cause instanceof ArtifactReaderFailure && + cause.reason === "integrity" + ) + return new AnalysisArtifactChangedError( + "inspect_plist", + selection.path, + cause.message, + { cause, capturedOutput }, + ); + if (cause instanceof ArtifactReaderFailure && cause.reason === "limit") { + const resource = cause.boundary ?? "memory"; + return new AnalysisResourceConstraintError( + "inspect_plist", + resource, + cause.message, + resource === "file-size" + ? { maximum_input_bytes: MAX_NATIVE_PLIST_BYTES } + : null, + { cause, capturedOutput, retryable: false }, + ); + } if (selection === undefined) return new ProviderAdapterError(IDENTITY.id, "inspect_plist", { cause, @@ -263,6 +290,51 @@ export const requireRegularPlist = ( ): Promise> => readPlistFile(input, async () => null); +/** Read bounded source bytes through the same retained descriptor owner. */ +export const readNativePlistBytes = ( + input: PlistFileInput, +): Promise> => + readPlistFile(input, async (file) => { + const before = await file.stat({ bigint: true }); + if (before.size > BigInt(MAX_NATIVE_PLIST_BYTES)) + throw new ArtifactReaderFailure( + "limit", + `Selected plist exceeds ${String(MAX_NATIVE_PLIST_BYTES)} bytes: ${input.selection.path}`, + { boundary: "file-size" }, + ); + const bytes = Buffer.allocUnsafe(Number(before.size)); + let position = 0; + while (position < bytes.length) { + assertArtifactReadActive(input.signal); + const length = Math.min(64 * 1024, bytes.length - position); + const { bytesRead } = await file.read(bytes, position, length, position); + if (bytesRead === 0) break; + position += bytesRead; + } + const after = await file.stat({ bigint: true }); + if (after.size > BigInt(MAX_NATIVE_PLIST_BYTES)) + throw new ArtifactReaderFailure( + "limit", + `Selected plist grew beyond ${String(MAX_NATIVE_PLIST_BYTES)} bytes: ${input.selection.path}`, + { boundary: "file-size" }, + ); + if ( + !after.isFile() || + before.dev !== after.dev || + before.ino !== after.ino || + before.size !== after.size || + before.mtimeNs !== after.mtimeNs || + before.ctimeNs !== after.ctimeNs || + BigInt(position) !== before.size + ) + throw new ArtifactReaderFailure( + "integrity", + `Selected plist changed during read: ${input.selection.path}`, + ); + assertArtifactReadActive(input.signal); + return bytes; + }); + /** Bytes examined for leading markup when `file` reports only text. */ const PLIST_MARKUP_PROBE_BYTES = 4096; @@ -273,7 +345,7 @@ export const plistSourceFormat = async ( ): Promise< Result["format"], AnalysisError> > => { - if (/binary property list/iu.test(description)) return ok("binary"); + if (isBinaryPlistDescription(description)) return ok("binary"); if (/\bXML\b/iu.test(description)) return ok("xml"); if (/\bJSON\b/iu.test(description)) return ok("json"); if (!/\btext\b/iu.test(description)) return ok("unknown"); @@ -296,3 +368,6 @@ export const plistSourceFormat = async ( : "unknown"; }); }; + +export const isBinaryPlistDescription = (description: string): boolean => + /binary property list/iu.test(description); diff --git a/src/go/GoBinaryContainer.ts b/src/native/metadata/BinaryContainer.ts similarity index 69% rename from src/go/GoBinaryContainer.ts rename to src/native/metadata/BinaryContainer.ts index 354a04c472..6d0766be20 100644 --- a/src/go/GoBinaryContainer.ts +++ b/src/native/metadata/BinaryContainer.ts @@ -1,69 +1,112 @@ /** Distinguish corrupt recognized binaries from containers outside parser coverage. */ -export class GoBinaryFormatFailure extends Error { +export class BinaryFormatFailure extends Error { constructor( readonly kind: "malformed" | "unsupported", message: string, ) { super(message); - this.name = "GoBinaryFormatFailure"; + this.name = "BinaryFormatFailure"; } } /** Resource boundary exceeded while decoding metadata or native container structure. */ -export class GoBinaryResourceFailure extends Error { +export class BinaryResourceFailure extends Error { constructor( readonly boundary: "build-info" | "structure" | "search", readonly maximum_bytes: number, message: string, ) { super(message); - this.name = "GoBinaryResourceFailure"; + this.name = "BinaryResourceFailure"; } } /** One file-backed virtual range; addresses stay exact until mapped to file offsets. */ -export interface GoFileMapping { +export interface FileMapping { readonly address: bigint; readonly offset: number; readonly size: number; } /** Declared zero-initialized virtual memory with no corresponding file bytes. */ -export interface GoZeroFillMapping { +export interface ZeroFillMapping { readonly address: bigint; readonly size: bigint; } -/** One named file section. Invalid declared ranges stay visible with `file_backed: false`. */ -export interface GoImageSection { +/** Native loader region facts; language-specific fallback selection belongs to consumers. */ +export interface LoadRegion { + readonly mapping: FileMapping; + readonly memory_size: bigint; + readonly read: boolean; + readonly write: boolean; + readonly execute: boolean; + /** Raw Mach-O VM_PROT masks preserve bits not represented by booleans. */ + readonly protection_masks?: { + readonly current: number; + readonly maximum: number; + }; +} + +/** Parsed section facts; consumer metadata validity is separate from native structure. */ +export type ImageSection = { readonly name: string; readonly offset: number; readonly size: number; readonly file_backed: boolean; readonly read_only: boolean; readonly compressed: boolean; -} +} & ( + | { + readonly format: "elf"; + readonly type: number; + readonly address: bigint; + readonly declared_range: { + readonly offset: bigint; + readonly size: bigint; + }; + } + | { + readonly format: "pe"; + readonly virtual_address: number; + readonly virtual_size: number; + readonly raw_size: number; + readonly characteristics: number; + } + | { + readonly format: "macho"; + readonly section_name: string; + readonly type: number; + readonly address: bigint; + readonly declared_range: { + readonly offset: bigint; + readonly size: bigint; + }; + readonly segment_mapping: FileMapping | null; + readonly segment_unmapped: boolean; + } +); -/** Native container facts and its ordered producer-defined build-info search ranges. */ -export interface GoBinaryContainer { +/** Native container facts, without language metadata search policy. */ +export interface BinaryContainer { readonly format: "elf" | "pe" | "macho"; readonly architecture: string; readonly bits: 32 | 64; readonly byte_order: "little" | "big"; - /** Declared preferred base when the container supplies one (PE). */ + /** Declared PE preferred base, when available. */ readonly image_base?: bigint; - /** Mach-O load-command observation; omitted for other formats. */ + /** Mach-O load-command observation, when available. */ readonly macho_chained_fixups?: boolean; - readonly mappings: readonly GoFileMapping[]; - /** File-backed ranges the native loader marks executable; omitted if unobserved. */ - readonly executable_mappings?: readonly GoFileMapping[]; - readonly zero_fills: readonly GoZeroFillMapping[]; - readonly sections: readonly GoImageSection[]; - readonly search: readonly GoFileMapping[]; + readonly mappings: readonly FileMapping[]; + readonly load_regions?: readonly LoadRegion[]; + /** File-backed ranges the native loader marks executable. */ + readonly executable_mappings?: readonly FileMapping[]; + readonly zero_fills: readonly ZeroFillMapping[]; + readonly sections: readonly ImageSection[]; } /** Reader that checks every structural range before decoding numeric fields. */ -export class GoBinaryReader { +export class BinaryReader { constructor( readonly bytes: Buffer, readonly little: boolean, @@ -78,7 +121,7 @@ export class GoBinaryReader { offset > this.bytes.length || size > this.bytes.length - offset ) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", `${label} range is truncated or outside the file`, ); @@ -101,12 +144,12 @@ export class GoBinaryReader { return this.little ? value.readBigUInt64LE() : value.readBigUInt64BE(); } - fileRange(offset: bigint, size: bigint, label: string): GoFileMapping { + fileRange(offset: bigint, size: bigint, label: string): FileMapping { if ( offset > BigInt(this.bytes.length) || size > BigInt(this.bytes.length) - offset ) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", `${label} range is outside the file`, ); @@ -118,7 +161,7 @@ export class GoBinaryReader { table(offset: bigint, count: number, width: number, label: string): number { const size = BigInt(count) * BigInt(width); if (size > 16n * 1024n * 1024n) - throw new GoBinaryResourceFailure( + throw new BinaryResourceFailure( "structure", 16 * 1024 * 1024, `${label} exceeds the 16 MiB structural decoding budget`, @@ -155,10 +198,10 @@ export const declaredFileRange = ( /** Map a virtual span without losing precision or selecting an ambiguous alias. */ export const mappedFileOffset = ( - mappings: readonly GoFileMapping[], + mappings: readonly FileMapping[], address: bigint, size: number, - zeroFills: readonly GoZeroFillMapping[] = [], + zeroFills: readonly ZeroFillMapping[] = [], ): number => { let found: number | undefined; for (const mapping of mappings) { @@ -171,20 +214,20 @@ export const mappedFileOffset = ( continue; const offset = mapping.offset + Number(delta); if (found !== undefined && found !== offset) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Virtual address has ambiguous file mappings", ); found = offset; } if (found === undefined) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Virtual address or string range is not file mapped", ); for (const zeroFill of zeroFills) { if (spansOverlap(address, BigInt(size), zeroFill.address, zeroFill.size)) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Virtual address has ambiguous file and zero-fill mappings", ); @@ -199,7 +242,7 @@ export const mappedFileOffset = ( BigInt(found) + start - address !== BigInt(mapping.offset) + start - mapping.address ) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Virtual address has ambiguous file mappings", ); @@ -209,8 +252,8 @@ export const mappedFileOffset = ( /** Recognize a wholly zero-filled span while refusing even partial file aliases. */ export const isZeroFilledSpan = ( - mappings: readonly GoFileMapping[], - zeroFills: readonly GoZeroFillMapping[], + mappings: readonly FileMapping[], + zeroFills: readonly ZeroFillMapping[], address: bigint, size: number, ): boolean => { @@ -225,7 +268,7 @@ export const isZeroFilledSpan = ( return false; for (const mapping of mappings) { if (spansOverlap(address, bytes, mapping.address, BigInt(mapping.size))) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Virtual address has ambiguous file and zero-fill mappings", ); @@ -243,17 +286,3 @@ const spansOverlap = ( otherSize > 0n && address < otherAddress + otherSize && otherAddress < address + size; - -/** Decode metadata text without replacing invalid bytes or stripping an initial BOM. */ -export const goUtf8 = (bytes: Buffer, label: string): string => { - try { - return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - bytes, - ); - } catch (cause) { - throw new GoBinaryFormatFailure( - "malformed", - `${label} contains malformed UTF-8: ${String(cause)}`, - ); - } -}; diff --git a/src/go/GoElfImage.ts b/src/native/metadata/ElfImage.ts similarity index 67% rename from src/go/GoElfImage.ts rename to src/native/metadata/ElfImage.ts index a53fbae62a..71e9f99f3c 100644 --- a/src/go/GoElfImage.ts +++ b/src/native/metadata/ElfImage.ts @@ -1,22 +1,22 @@ import { declaredFileRange, - GoBinaryFormatFailure, - GoBinaryReader, - mappedFileOffset, - type GoBinaryContainer, - type GoFileMapping, - type GoImageSection, - type GoZeroFillMapping, -} from "./GoBinaryContainer.js"; -import { readElfFixedHeader } from "../domain/elfFixedHeader.js"; + BinaryFormatFailure, + BinaryReader, + type BinaryContainer, + type FileMapping, + type ImageSection, + type LoadRegion, + type ZeroFillMapping, +} from "./BinaryContainer.js"; +import { readElfFixedHeader } from "../../domain/elfFixedHeader.js"; -/** Read ELF build-info sections and file-backed program mappings. */ -export const readGoElfImage = (bytes: Buffer): GoBinaryContainer => { +/** Read ELF sections and file-backed program mappings without interpreting language metadata. */ +export const readElfImage = (bytes: Buffer): BinaryContainer => { const fixedHeader = readElfFixedHeader(bytes); if (!fixedHeader.ok) - throw new GoBinaryFormatFailure("malformed", fixedHeader.error); + throw new BinaryFormatFailure("malformed", fixedHeader.error); const { elfClass: bits, headerSize } = fixedHeader.value; - const reader = new GoBinaryReader( + const reader = new BinaryReader( bytes, fixedHeader.value.byteOrder === "little", ); @@ -32,7 +32,7 @@ export const readGoElfImage = (bytes: Buffer): GoBinaryContainer => { (phCount !== 0 && phOffset < BigInt(headerSize)) || (shOffset !== 0n && shOffset < BigInt(headerSize)) ) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "ELF header table overlaps the native header", ); @@ -41,7 +41,7 @@ export const readGoElfImage = (bytes: Buffer): GoBinaryContainer => { (shCount === 0 || phCount === 0xffff || namesIndex === 0xffff) ) { if (shWidth < (wide ? 64 : 40)) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "ELF extended section header width is invalid", ); @@ -51,14 +51,14 @@ export const readGoElfImage = (bytes: Buffer): GoBinaryContainer => { "ELF section zero", ).offset; if (reader.u32(first + 4) !== 0) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "ELF initial section for extended numbering must have SHT_NULL type", ); if (shCount === 0) { const count = reader.word(first + (wide ? 32 : 20), bits); if (count < 0xff00n || count > BigInt(Number.MAX_SAFE_INTEGER)) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "ELF extended section count is invalid", ); @@ -67,7 +67,7 @@ export const readGoElfImage = (bytes: Buffer): GoBinaryContainer => { if (phCount === 0xffff) { phCount = reader.u32(first + (wide ? 44 : 28)); if (phCount < 0xffff) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "ELF extended program count is below its reserved range", ); @@ -75,7 +75,7 @@ export const readGoElfImage = (bytes: Buffer): GoBinaryContainer => { if (namesIndex === 0xffff) { namesIndex = reader.u32(first + (wide ? 40 : 24)); if (namesIndex < 0xff00) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "ELF extended section name index is below its reserved range", ); @@ -84,13 +84,10 @@ export const readGoElfImage = (bytes: Buffer): GoBinaryContainer => { shOffset === 0n && (shCount !== 0 || namesIndex !== 0 || phCount === 0xffff) ) { - throw new GoBinaryFormatFailure( - "malformed", - "ELF section table is missing", - ); + throw new BinaryFormatFailure("malformed", "ELF section table is missing"); } if (phCount !== 0 && phWidth < (wide ? 56 : 32)) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "ELF program header width is invalid", ); @@ -100,11 +97,11 @@ export const readGoElfImage = (bytes: Buffer): GoBinaryContainer => { phWidth, "ELF program headers", ); - const mappings: GoFileMapping[] = []; - const executableMappings: GoFileMapping[] = []; - const zeroFills: GoZeroFillMapping[] = []; - const sections: GoImageSection[] = []; - const fallback: GoFileMapping[] = []; + const mappings: FileMapping[] = []; + const loadRegions: LoadRegion[] = []; + const executableMappings: FileMapping[] = []; + const zeroFills: ZeroFillMapping[] = []; + const sections: ImageSection[] = []; for (let index = 0; index < phCount; index++) { const offset = phStart + index * phWidth; if (reader.u32(offset) !== 1) continue; @@ -114,7 +111,7 @@ export const readGoElfImage = (bytes: Buffer): GoBinaryContainer => { const size = reader.word(offset + (wide ? 32 : 16), bits); const memorySize = reader.word(offset + (wide ? 40 : 20), bits); if (size > memorySize || address + memorySize > 1n << BigInt(bits)) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "ELF load segment sizes or addresses are invalid", ); @@ -126,16 +123,21 @@ export const readGoElfImage = (bytes: Buffer): GoBinaryContainer => { }; if (mapping.size !== 0) { mappings.push(mapping); + loadRegions.push({ + mapping, + memory_size: memorySize, + read: (flags & 4) !== 0, + write: (flags & 2) !== 0, + execute: (flags & 1) !== 0, + }); if ((flags & 1) !== 0) executableMappings.push(mapping); } if (memorySize > size) zeroFills.push({ address: address + size, size: memorySize - size }); - if (mapping.size !== 0 && (flags & 3) === 2) fallback.push(mapping); } - let search: GoFileMapping | null = null; if (shCount !== 0) { if (shWidth < (wide ? 64 : 40) || namesIndex >= shCount) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "ELF section width or name-table index is invalid", ); @@ -148,7 +150,7 @@ export const readGoElfImage = (bytes: Buffer): GoBinaryContainer => { if (namesIndex !== 0) { const namesHeader = shStart + namesIndex * shWidth; if (reader.u32(namesHeader + 4) !== 3) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "ELF section name table is not a string table", ); @@ -161,7 +163,7 @@ export const readGoElfImage = (bytes: Buffer): GoBinaryContainer => { const offset = shStart + index * shWidth; const nameOffset = reader.u32(offset); if (nameOffset >= names.size) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "ELF section name offset is outside its string table", ); @@ -174,73 +176,29 @@ export const readGoElfImage = (bytes: Buffer): GoBinaryContainer => { wide, header: offset, }); - // Comparing the complete fixed name avoids quadratic scans through malicious string tables. - const expectedName = Buffer.from(".go.buildinfo\0"); - if ( - expectedName.length > names.size - nameOffset || - !bytes - .subarray( - names.offset + nameOffset, - names.offset + nameOffset + expectedName.length, - ) - .equals(expectedName) - ) - continue; - if (search !== null) - throw new GoBinaryFormatFailure( - "malformed", - "ELF contains ambiguous build-info sections", - ); - if (reader.u32(offset + 4) !== 1) - throw new GoBinaryFormatFailure( - "malformed", - "ELF build-info section is not file-backed data", - ); - if ((reader.word(offset + 8, bits) & 0x800n) !== 0n) - throw new GoBinaryFormatFailure( - "unsupported", - "Compressed ELF build-info sections are unsupported", - ); - const address = reader.word(offset + (wide ? 16 : 12), bits); - search = { - ...reader.fileRange( - reader.word(offset + (wide ? 24 : 16), bits), - reader.word(offset + (wide ? 32 : 20), bits), - "ELF build-info section", - ), - address, - }; - if ( - mappedFileOffset(mappings, address, search.size, zeroFills) !== - search.offset - ) - throw new GoBinaryFormatFailure( - "malformed", - "ELF build-info section disagrees with its load mapping", - ); } } } return { format: "elf", - architecture: goArchitectureForElfMachine(fixedHeader.value.machine), + architecture: architectureForElfMachine(fixedHeader.value.machine), bits, byte_order: reader.little ? "little" : "big", mappings, + load_regions: loadRegions, executable_mappings: executableMappings, zero_fills: zeroFills, sections, - search: search === null ? fallback : [search], }; }; const recordElfSection = ( - sections: GoImageSection[], + sections: ImageSection[], input: { readonly bytes: Buffer; readonly names: { readonly offset: number; readonly size: number }; readonly nameOffset: number; - readonly reader: GoBinaryReader; + readonly reader: BinaryReader; readonly bits: 32 | 64; readonly wide: boolean; readonly header: number; @@ -299,18 +257,22 @@ const elfImageSection = ({ header, name, }: { - readonly reader: GoBinaryReader; + readonly reader: BinaryReader; readonly bits: 32 | 64; readonly wide: boolean; readonly header: number; readonly name: string; -}): GoImageSection => { +}): ImageSection => { const type = reader.u32(header + 4); const flags = reader.word(header + 8, bits); + const declared_range = { + offset: reader.word(header + (wide ? 24 : 16), bits), + size: reader.word(header + (wide ? 32 : 20), bits), + }; const declared = declaredFileRange( reader.bytes.length, - reader.word(header + (wide ? 24 : 16), bits), - reader.word(header + (wide ? 32 : 20), bits), + declared_range.offset, + declared_range.size, ); // SHT_NOBITS offsets and sizes do not describe stored bytes. const range = @@ -323,13 +285,22 @@ const elfImageSection = ({ (flags & 1n) === 0n && (flags & 4n) === 0n && (flags & 2n) !== 0n; - return { name, ...range, read_only: readOnly, compressed }; + return { + format: "elf", + name, + type, + address: reader.word(header + (wide ? 16 : 12), bits), + declared_range, + ...range, + read_only: readOnly, + compressed, + }; }; const SHT_NOBITS = 8; -/** Go build-info vocabulary, including families outside REA target admission. */ -const goArchitectureForElfMachine = (machine: number): string => { +/** Map ELF machine identifiers to stable architecture names. */ +const architectureForElfMachine = (machine: number): string => { const names = new Map([ [3, "386"], [8, "mips"], diff --git a/src/native/metadata/Il2cppContainer.test.ts b/src/native/metadata/Il2cppContainer.test.ts new file mode 100644 index 0000000000..a9bea21e3c --- /dev/null +++ b/src/native/metadata/Il2cppContainer.test.ts @@ -0,0 +1,50 @@ +import { expect, it } from "vitest"; +import { readGoBinaryImage } from "../../go/GoBinaryImage.js"; +import { createRustImageWithGoSection } from "../../../tests/fixtures/rust/image.js"; +import { openImageSteps } from "../../il2cpp/Il2cppBinaryImage.js"; +import { readBinaryContainer } from "./readBinaryContainer.js"; + +it.each(["unmapped", "zero-fill", "duplicate", "out-of-file"] as const)( + "keeps IL2CPP admission independent of a Mach-O Go build-info %s failure", + (fault) => { + const goNamed = createRustImageWithGoSection("macho", fault); + const control = Buffer.from(goNamed); + const sectionName = Buffer.from("__other\0".padEnd(16, "\0"), "ascii"); + for (let index = 0; index < (fault === "duplicate" ? 2 : 1); index++) + sectionName.copy(control, 408 + index * 152); + + expect(() => readGoBinaryImage(goNamed)).toThrow(/build-info|mapped/i); + expect(readGoBinaryImage(control).build_info).toBeNull(); + expect(readBinaryContainer(goNamed).load_regions).toEqual( + readBinaryContainer(control).load_regions, + ); + + const image = openImageSteps(goNamed, undefined).next(); + expect(image.done).toBe(true); + if (!image.done) + throw new Error("container admission unexpectedly yielded"); + expect(image.value.bytes).toBe(goNamed); + expect(image.value.container).toMatchObject({ + format: "macho", + macho_chained_fixups: false, + }); + expect(image.value.container.executable_mappings?.length).toBeGreaterThan( + 0, + ); + expect( + image.value.container.sections.filter( + (section) => + section.format === "macho" && + section.section_name === "__go_buildinfo", + ), + ).toHaveLength(fault === "duplicate" ? 2 : 1); + + const controlImage = openImageSteps(control, undefined).next(); + expect(controlImage.done).toBe(true); + if (!controlImage.done) + throw new Error("control container admission unexpectedly yielded"); + expect(controlImage.value.container.mappings).toEqual( + image.value.container.mappings, + ); + }, +); diff --git a/src/go/GoMachoImage.ts b/src/native/metadata/MachoImage.ts similarity index 68% rename from src/go/GoMachoImage.ts rename to src/native/metadata/MachoImage.ts index ee5e000848..3e3c41d071 100644 --- a/src/go/GoMachoImage.ts +++ b/src/native/metadata/MachoImage.ts @@ -1,21 +1,21 @@ import { declaredFileRange, - GoBinaryFormatFailure, - GoBinaryReader, - mappedFileOffset, - type GoBinaryContainer, - type GoFileMapping, - type GoImageSection, - type GoZeroFillMapping, -} from "./GoBinaryContainer.js"; -import { machoLoadCommandAlignment } from "../domain/apple/machoContainer.js"; + BinaryFormatFailure, + BinaryReader, + type BinaryContainer, + type FileMapping, + type ImageSection, + type LoadRegion, + type ZeroFillMapping, +} from "./BinaryContainer.js"; +import { machoLoadCommandAlignment } from "../../domain/apple/machoContainer.js"; /** Read thin Mach-O images, retaining the slice's exact file-backed mappings. */ -export const readGoMachoImage = (bytes: Buffer): GoBinaryContainer => { +export const readMachoImage = (bytes: Buffer): BinaryContainer => { const magic = bytes.readUInt32BE(0); const little = magic === 0xcefaedfe || magic === 0xcffaedfe; const bits = magic === 0xfeedfacf || magic === 0xcffaedfe ? 64 : 32; - const reader = new GoBinaryReader(bytes, little); + const reader = new BinaryReader(bytes, little); const headerSize = bits === 64 ? 32 : 28; reader.range(0, headerSize, "Mach-O header"); const count = reader.u32(16); @@ -27,18 +27,17 @@ export const readGoMachoImage = (bytes: Buffer): GoBinaryContainer => { "Mach-O load commands", ); if (count > Math.floor(commandBytes / 8)) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Mach-O command count exceeds its load-command range", ); - const mappings: GoFileMapping[] = []; - const executableMappings: GoFileMapping[] = []; - const zeroFills: GoZeroFillMapping[] = []; - const sections: GoImageSection[] = []; - const named: GoFileMapping[] = []; - let fallback: GoFileMapping | null = null; - let offset = commandStart; + const mappings: FileMapping[] = []; + const loadRegions: LoadRegion[] = []; + const executableMappings: FileMapping[] = []; + const zeroFills: ZeroFillMapping[] = []; + const sections: ImageSection[] = []; let chainedFixups = false; + let offset = commandStart; const end = commandStart + commandBytes; for (let index = 0; index < count; index++) { const command = reader.u32(offset); @@ -48,7 +47,7 @@ export const readGoMachoImage = (bytes: Buffer): GoBinaryContainer => { size % machoLoadCommandAlignment(bits === 64) !== 0 || size > end - offset ) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Mach-O load command size is invalid or truncated", ); @@ -56,14 +55,14 @@ export const readGoMachoImage = (bytes: Buffer): GoBinaryContainer => { if (command === 1 || command === 0x19) { const wide = command === 0x19; if (wide !== (bits === 64)) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Mach-O segment command does not match the header width", ); const segmentHeader = wide ? 72 : 56; const sectionWidth = wide ? 80 : 68; if (size < segmentHeader) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Mach-O segment header is truncated", ); @@ -79,12 +78,12 @@ export const readGoMachoImage = (bytes: Buffer): GoBinaryContainer => { (!unmapped && fileSize > memorySize) || address + memorySize > 1n << BigInt(bits) ) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Mach-O segment addresses or sizes are invalid", ); if (sectionCount > Math.floor((size - segmentHeader) / sectionWidth)) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Mach-O section table exceeds its segment command", ); @@ -101,6 +100,17 @@ export const readGoMachoImage = (bytes: Buffer): GoBinaryContainer => { ); if (mapping !== undefined && name !== "__PAGEZERO") { mappings.push(mapping); + loadRegions.push({ + mapping, + memory_size: memorySize, + read: (protection & 1) !== 0, + write: (protection & 2) !== 0, + execute: (protection & 4) !== 0, + protection_masks: { + current: protection, + maximum: maxProtection, + }, + }); if ((protection & 4) !== 0) executableMappings.push(mapping); } if (memorySize > fileSize && name !== "__PAGEZERO") @@ -108,14 +118,6 @@ export const readGoMachoImage = (bytes: Buffer): GoBinaryContainer => { address: address + fileSize, size: memorySize - fileSize, }); - if ( - fallback === null && - mapping !== undefined && - address !== 0n && - protection === 3 && - maxProtection === 3 - ) - fallback = mapping; for (let sectionIndex = 0; sectionIndex < sectionCount; sectionIndex++) { const section = offset + segmentHeader + sectionIndex * sectionWidth; const sectionName = fixedName( @@ -129,6 +131,9 @@ export const readGoMachoImage = (bytes: Buffer): GoBinaryContainer => { header: section, name: `${fixedName(reader.range(section + 16, 16, "Mach-O section segment"))},${sectionName}`, protection, + sectionName, + segmentMapping: mapping ?? null, + segmentUnmapped: unmapped, unmapped: unmapped || mapping === undefined, segmentAddress: address, segmentMemorySize: memorySize, @@ -136,56 +141,15 @@ export const readGoMachoImage = (bytes: Buffer): GoBinaryContainer => { segmentFileSize: fileSize, }), ); - if (sectionName !== "__go_buildinfo") continue; - if (unmapped) - throw new GoBinaryFormatFailure( - "malformed", - "Mach-O build-info section is in an unmapped segment", - ); - if (mapping === undefined) - throw new GoBinaryFormatFailure( - "malformed", - "Mach-O build-info section is not file mapped", - ); - const sectionFlags = reader.u32(section + (wide ? 64 : 56)); - if ([1, 0xc, 0x12].includes(sectionFlags & 0xff)) - throw new GoBinaryFormatFailure( - "malformed", - "Mach-O build-info section is zero-fill data", - ); - const sectionAddress = reader.word(section + 32, bits); - const sectionSize = reader.word(section + (wide ? 40 : 36), bits); - const sectionOffset = BigInt(reader.u32(section + (wide ? 48 : 40))); - const info = { - ...reader.fileRange( - sectionOffset, - sectionSize, - "Mach-O build-info section", - ), - address: sectionAddress, - }; - if ( - mappedFileOffset([mapping], sectionAddress, info.size) !== info.offset - ) - throw new GoBinaryFormatFailure( - "malformed", - "Mach-O build-info section disagrees with its segment", - ); - named.push(info); } } offset += size; } if (offset !== end) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "Mach-O load-command byte count does not match its commands", ); - if (named.length > 1) - throw new GoBinaryFormatFailure( - "malformed", - "Mach-O has ambiguous build-info sections", - ); const cpu = reader.u32(4); const names = new Map([ [7, "386"], @@ -202,10 +166,10 @@ export const readGoMachoImage = (bytes: Buffer): GoBinaryContainer => { bits, byte_order: little ? "little" : "big", mappings, + load_regions: loadRegions, executable_mappings: executableMappings, zero_fills: zeroFills, sections, - search: named.length > 0 ? named : fallback === null ? [] : [fallback], }; }; @@ -216,24 +180,30 @@ const machoImageSection = ({ header, name, protection, + sectionName, + segmentMapping, + segmentUnmapped, unmapped, segmentAddress, segmentMemorySize, segmentFileOffset, segmentFileSize, }: { - readonly reader: GoBinaryReader; + readonly reader: BinaryReader; readonly bits: 32 | 64; readonly wide: boolean; readonly header: number; readonly name: string; readonly protection: number; + readonly sectionName: string; + readonly segmentMapping: FileMapping | null; + readonly segmentUnmapped: boolean; readonly unmapped: boolean; readonly segmentAddress: bigint; readonly segmentMemorySize: bigint; readonly segmentFileOffset: bigint; readonly segmentFileSize: bigint; -}): GoImageSection => { +}): ImageSection => { const flags = reader.u32(header + (wide ? 64 : 56)); const zeroFill = [1, 0xc, 0x12].includes(flags & 0xff); const sectionAddress = reader.word(header + 32, bits); @@ -251,11 +221,7 @@ const machoImageSection = ({ sectionSize <= segmentFileSize - fileDelta; const range = unmapped ? { offset: 0, size: 0, file_backed: false } - : declaredFileRange( - reader.bytes.length, - BigInt(reader.u32(header + (wide ? 48 : 40))), - reader.word(header + (wide ? 40 : 36), bits), - ); + : declaredFileRange(reader.bytes.length, sectionOffset, sectionSize); const fileBacked = range.file_backed && !zeroFill && inSegmentMemory && inSegmentFile; const readOnly = @@ -264,7 +230,14 @@ const machoImageSection = ({ (protection & 2) === 0 && (flags & 0x80000400) === 0; return { + format: "macho", name, + section_name: sectionName, + type: flags & 0xff, + address: sectionAddress, + declared_range: { offset: sectionOffset, size: sectionSize }, + segment_mapping: segmentMapping, + segment_unmapped: segmentUnmapped, ...range, file_backed: fileBacked, read_only: readOnly, diff --git a/src/go/GoPeImage.test.ts b/src/native/metadata/PeImage.test.ts similarity index 55% rename from src/go/GoPeImage.test.ts rename to src/native/metadata/PeImage.test.ts index 6119176c7c..70f33dfb34 100644 --- a/src/go/GoPeImage.test.ts +++ b/src/native/metadata/PeImage.test.ts @@ -1,10 +1,10 @@ import { expect, it } from "vitest"; -import { readGoBinaryImage } from "./GoBinaryImage.js"; -import { readGoPeImage } from "./GoPeImage.js"; +import { readGoBinaryImage } from "../../go/GoBinaryImage.js"; +import { readPeImage } from "./PeImage.js"; import { createGoBinaryFixture, setGoPeVirtualSize, -} from "../../tests/fixtures/go/image.js"; +} from "../../../tests/fixtures/go/image.js"; it.each([32, 64] as const)( "uses raw-size fallback for PE%d Go build-info and keeps RVA mappings intact", @@ -15,7 +15,7 @@ it.each([32, 64] as const)( expect(expected).not.toBeNull(); setGoPeVirtualSize(fixture.bytes, 0); - const container = readGoPeImage(fixture.bytes); + const container = readPeImage(fixture.bytes); expect(container.sections[0]).toMatchObject({ name: ".data", offset: 1024, @@ -27,16 +27,35 @@ it.each([32, 64] as const)( offset: 1024, size: 2048, }); - expect(container.search[0]).toMatchObject({ - address: 0x10400n, - offset: 1024, - size: 2048, + expect(container.sections[0]).toMatchObject({ + virtual_address: 0x400, + virtual_size: 0, + raw_size: 2048, + characteristics: 0xc0000040, }); + expect(container).not.toHaveProperty("data_ranges"); + expect(container.image_base).toBe(0x10000n); + expect(container.executable_mappings).toEqual([]); expect(readGoBinaryImage(fixture.bytes).build_info).toEqual(expected); } }, ); +it("keeps Go fallback candidates when a PE section has no printable name", () => { + const fixture = createGoBinaryFixture({ format: "pe", bits: 64 }); + const expected = readGoBinaryImage(fixture.bytes).build_info; + const pe = fixture.bytes.readUInt32LE(0x3c); + const section = pe + 24 + fixture.bytes.readUInt16LE(pe + 20); + fixture.bytes.fill(0, section, section + 8); + + const container = readPeImage(fixture.bytes); + expect(container.sections[0]).toMatchObject({ + name: "", + characteristics: 0xc0000040, + }); + expect(readGoBinaryImage(fixture.bytes).build_info).toEqual(expected); +}); + it.each(["inline", "pointer"] as const)( "does not search PE raw alignment padding beyond a nonzero VirtualSize (%s)", (encoding) => { @@ -51,13 +70,19 @@ it.each(["inline", "pointer"] as const)( fixture.bytes.fill(0, fixture.headerOffset, fixture.headerOffset + 512); setGoPeVirtualSize(fixture.bytes, 512); - const container = readGoPeImage(fixture.bytes); + const container = readPeImage(fixture.bytes); expect(container.sections[0]).toMatchObject({ size: 512, file_backed: true, }); expect(container.mappings[0]).toMatchObject({ size: 2048, offset: 1024 }); - expect(container.search[0]).toMatchObject({ size: 512, offset: 1024 }); + expect(container.sections[0]).toMatchObject({ + size: 512, + offset: 1024, + virtual_size: 512, + raw_size: 2048, + }); + expect(container).not.toHaveProperty("data_ranges"); expect(readGoBinaryImage(fixture.bytes).build_info).toBeNull(); }, ); diff --git a/src/go/GoPeImage.ts b/src/native/metadata/PeImage.ts similarity index 70% rename from src/go/GoPeImage.ts rename to src/native/metadata/PeImage.ts index dfb9f9c579..f694550c71 100644 --- a/src/go/GoPeImage.ts +++ b/src/native/metadata/PeImage.ts @@ -1,38 +1,41 @@ -import { observeMzWindowsHeader, parseDosMzHeader } from "../domain/dosMz.js"; +import { + observeMzWindowsHeader, + parseDosMzHeader, +} from "../../domain/dosMz.js"; import { declaredFileRange, - GoBinaryFormatFailure, - GoBinaryReader, - type GoBinaryContainer, - type GoFileMapping, - type GoImageSection, - type GoZeroFillMapping, -} from "./GoBinaryContainer.js"; + BinaryFormatFailure, + BinaryReader, + type BinaryContainer, + type FileMapping, + type ImageSection, + type ZeroFillMapping, +} from "./BinaryContainer.js"; -const peHeaderOffset = (reader: GoBinaryReader): number => { +const peHeaderOffset = (reader: BinaryReader): number => { const windowsHeader = observeMzWindowsHeader(reader.bytes); if (windowsHeader.state !== "declared") { const dos = parseDosMzHeader(reader.bytes); - if (!dos.ok) throw new GoBinaryFormatFailure("malformed", dos.error); - throw new GoBinaryFormatFailure( + if (!dos.ok) throw new BinaryFormatFailure("malformed", dos.error); + throw new BinaryFormatFailure( "unsupported", "DOS MZ images are outside the supported ELF, PE and thin Mach-O formats", ); } const { offset: pe } = windowsHeader; if (pe < 64) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "PE signature overlaps the DOS header", ); reader.range(pe, 24, "PE signature and COFF header"); if (["NE", "LE", "LX"].includes(windowsHeader.signature)) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "unsupported", `${windowsHeader.signature} images are outside the supported ELF, PE and thin Mach-O formats`, ); if (windowsHeader.signature !== "PE") - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "PE signature is invalid or overlaps the DOS header", ); @@ -40,41 +43,41 @@ const peHeaderOffset = (reader: GoBinaryReader): number => { }; const optionalHeaderMagic = ( - reader: GoBinaryReader, + reader: BinaryReader, offset: number, size: number, ): number => { reader.range(offset, size, "PE optional header"); if (size < 2) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "PE optional header is missing its complete magic field", ); return reader.u16(offset); }; -/** Read PE32/PE32+ native image mappings and the linker data section. */ -export const readGoPeImage = (bytes: Buffer): GoBinaryContainer => { - const reader = new GoBinaryReader(bytes, true); +/** Read PE32/PE32+ native image mappings and section facts. */ +export const readPeImage = (bytes: Buffer): BinaryContainer => { + const reader = new BinaryReader(bytes, true); const pe = peHeaderOffset(reader); const optional = pe + 24; const optionalSize = reader.u16(pe + 20); const magic = optionalHeaderMagic(reader, optional, optionalSize); const bits = magic === 0x10b ? 32 : magic === 0x20b ? 64 : null; if (bits === null) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "unsupported", "Only PE32 and PE32+ image optional headers are supported", ); if (optionalSize < (bits === 64 ? 112 : 96)) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "PE optional header is truncated", ); const directoriesOffset = bits === 64 ? 112 : 96; const directories = reader.u32(optional + directoriesOffset - 4); if (directories * 8 > optionalSize - directoriesOffset) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "PE data directories extend beyond the optional header", ); @@ -86,11 +89,10 @@ export const readGoPeImage = (bytes: Buffer): GoBinaryContainer => { 40, "PE sections", ); - const mappings: GoFileMapping[] = []; - const executableMappings: GoFileMapping[] = []; - const zeroFills: GoZeroFillMapping[] = []; - const sections: GoImageSection[] = []; - let search: GoFileMapping | null = null; + const mappings: FileMapping[] = []; + const executableMappings: FileMapping[] = []; + const zeroFills: ZeroFillMapping[] = []; + const sections: ImageSection[] = []; for (let index = 0; index < count; index++) { const entry = start + index * 40; const sectionName = peSectionName( @@ -103,19 +105,21 @@ export const readGoPeImage = (bytes: Buffer): GoBinaryContainer => { const characteristics = reader.u32(entry + 36); // SizeOfRawData includes file-alignment padding. VirtualSize is the content. const contentSize = virtualSize === 0 ? size : Math.min(virtualSize, size); - if (sectionName.length > 0) - sections.push( - peImageSection({ - reader, - name: sectionName, - size: contentSize, - offset, - characteristics, - }), - ); + sections.push( + peImageSection({ + reader, + name: sectionName, + size: contentSize, + offset, + characteristics, + virtualAddress, + virtualSize, + rawSize: size, + }), + ); const address = imageBase + BigInt(virtualAddress); if (address + BigInt(Math.max(size, virtualSize)) > 1n << BigInt(bits)) - throw new GoBinaryFormatFailure( + throw new BinaryFormatFailure( "malformed", "PE section virtual address overflows its address width", ); @@ -135,15 +139,6 @@ export const readGoPeImage = (bytes: Buffer): GoBinaryContainer => { address: address + BigInt(size), size: BigInt(virtualSize - size), }); - // Match debug/buildinfo's first initialized, readable, writable data section. - if ( - search === null && - virtualAddress !== 0 && - size !== 0 && - (characteristics & ~0x00600000) >>> 0 === 0xc0000040 - ) { - search = { ...mapping, size: contentSize }; - } } const machine = reader.u16(pe + 4); const names = new Map([ @@ -164,7 +159,6 @@ export const readGoPeImage = (bytes: Buffer): GoBinaryContainer => { executable_mappings: executableMappings, zero_fills: zeroFills, sections, - search: search === null ? [] : [search], }; }; @@ -179,13 +173,19 @@ const peImageSection = ({ size, offset, characteristics, + virtualAddress, + virtualSize, + rawSize, }: { - readonly reader: GoBinaryReader; + readonly reader: BinaryReader; readonly name: string; readonly size: number; readonly offset: number; readonly characteristics: number; -}): GoImageSection => { + readonly virtualAddress: number; + readonly virtualSize: number; + readonly rawSize: number; +}): ImageSection => { const range = declaredFileRange( reader.bytes.length, BigInt(offset), @@ -197,5 +197,15 @@ const peImageSection = ({ (characteristics & 0x80000000) === 0 && (characteristics & 0x20000000) === 0 && (characteristics & 0x40) !== 0; - return { name, ...range, read_only: readOnly, compressed: false }; + return { + format: "pe", + name, + virtual_address: virtualAddress, + virtual_size: virtualSize, + raw_size: rawSize, + characteristics, + ...range, + read_only: readOnly, + compressed: false, + }; }; diff --git a/src/native/metadata/readBinaryContainer.ts b/src/native/metadata/readBinaryContainer.ts new file mode 100644 index 0000000000..d8fa65f8a0 --- /dev/null +++ b/src/native/metadata/readBinaryContainer.ts @@ -0,0 +1,48 @@ +import { + BinaryFormatFailure, + type BinaryContainer, +} from "./BinaryContainer.js"; +import { + classifyFatContainerIdentity, + readFatContainerHeader, +} from "../../domain/apple/machoContainer.js"; +import { readElfImage } from "./ElfImage.js"; +import { readPeImage } from "./PeImage.js"; +import { readMachoImage } from "./MachoImage.js"; + +/** Identify an ELF, PE, or thin Mach-O image and its named file sections. */ +export const readBinaryContainer = (bytes: Buffer): BinaryContainer => { + if (bytes.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) + return readElfImage(bytes); + if (bytes.subarray(0, 2).equals(Buffer.from("MZ"))) return readPeImage(bytes); + if (bytes.length >= 4) { + const magic = bytes.readUInt32BE(); + if ([0xcafebabe, 0xcafebabf, 0xbebafeca, 0xbfbafeca].includes(magic)) { + const fatHeader = readFatContainerHeader(bytes, bytes.length); + if (fatHeader.state === "malformed") + throw new BinaryFormatFailure("malformed", fatHeader.reason); + const identity = + fatHeader.state === "parsed" + ? classifyFatContainerIdentity(fatHeader, (offset) => { + if (offset >= bytes.length) return undefined; + if (offset + 4 > bytes.length) return null; + return bytes.readUInt32BE(offset); + }) + : undefined; + // An invalid FAT-shaped table is not proof of a Mach-O container: JVM + // constant-pool bytes can share both the magic and a known CPU value. + const confirmedFatMachO = identity?.state === "mach-o"; + if (confirmedFatMachO) + throw new BinaryFormatFailure( + "unsupported", + "Universal/fat Mach-O containers require selecting a thin slice", + ); + } + if ([0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(magic)) + return readMachoImage(bytes); + } + throw new BinaryFormatFailure( + "unsupported", + "Only ELF, PE, and thin Mach-O native images are supported", + ); +}; diff --git a/src/native/parsers/plist.test.ts b/src/native/parsers/plist.test.ts index 6cfde85b7e..468131ef0c 100644 --- a/src/native/parsers/plist.test.ts +++ b/src/native/parsers/plist.test.ts @@ -1,10 +1,438 @@ import { expect, it } from "vitest"; +import { buildBinary } from "plist"; +import { + decodeBinaryPlistScalars, + scanBinaryPlistScalars, +} from "../../artifacts/apple/PlistScalars.js"; import { + parsePlistBinary, parsePlistJson, parsePlistXml, plistJsonNeedsNumberTypes, } from "./plist.js"; +import { estimateBinaryPlistExpansion } from "../../artifacts/apple/PropertyListDecodeBudget.js"; +import { + binaryPlistObjectTable, + binaryPlistWithOverlappingDateAndData, +} from "../../../tests/fixtures/binaryPlistObjectTable.js"; + +const parseWithoutMutatingSource = (bytes: Buffer) => { + const original = Buffer.from(bytes); + const parsed = parsePlistBinary(bytes); + expect(parsed.ok).toBe(true); + if (!parsed.ok) throw parsed.error; + expect(bytes).toEqual(original); + return parsed.value; +}; + +const binaryPlistWithNegativeZero = (): Buffer => { + const bytes = Buffer.from( + buildBinary({ + negative: -1.25, + positive: 0, + note: "a\u0001b", + payload: Buffer.from([0]), + fraction: new Date("2020-05-05T01:02:03.100Z"), + whole: new Date("2020-05-05T01:02:03.000Z"), + nel: "a\u0085b", + ls: "a\u2028b", + ps: "a\u2029b", + largeInteger: 9007199254740992, + }), + ); + const negativeOneAndQuarter = Buffer.from([ + 0x23, 0xbf, 0xf4, 0, 0, 0, 0, 0, 0, + ]); + const offset = bytes.indexOf(negativeOneAndQuarter); + if (offset < 0) throw new Error("binary plist fixture lost its real"); + bytes.fill(0, offset + 1, offset + 9); + bytes[offset + 1] = 0x80; + const largeReal = Buffer.alloc(9); + largeReal[0] = 0x23; + largeReal.writeDoubleBE(9007199254740992, 1); + const largeOffset = bytes.indexOf(largeReal); + if (largeOffset < 0) throw new Error("binary plist fixture lost its integer"); + bytes[largeOffset] = 0x13; + bytes.writeBigInt64BE(9007199254740992n, largeOffset + 1); + return bytes; +}; + +const binaryDate = (secondsSince2001: number): Buffer => { + const bytes = Buffer.from( + buildBinary({ date: new Date("2001-01-01T00:00:00.000Z") }), + ); + const metadata = scanBinaryPlistScalars(bytes); + const date = metadata.dates[0]; + if (date === undefined) throw new Error("binary plist fixture lost its date"); + bytes.writeDoubleBE(secondsSince2001, date.payloadOffset); + return bytes; +}; + +const binarySharedSubmillisecondDate = (): Buffer => { + // Array object 0 references the same date object 1 twice. + const bytes = Buffer.alloc(66); + bytes[8] = 0xa2; + bytes[9] = 1; + bytes[10] = 1; + bytes[11] = 0x33; + bytes.writeDoubleBE(0.1005, 12); + return binaryPlistObjectTable(bytes, [8, 11], 20); +}; + +const binaryWithUnreachableMalformedDate = (): Buffer => { + const bytes = Buffer.alloc(54); + bytes.set([0x52, 0x6f, 0x6b, 0x30], 8); // Root "ok"; unused malformed date. + return binaryPlistObjectTable(bytes, [8, 11], 20); +}; + +const binaryWithReachableAndUnreachableDate = (): Buffer => { + const bytes = Buffer.alloc(67); + bytes.set([0xa1, 1, 0x33], 8); + bytes.writeDoubleBE(0.1005, 11); + bytes[19] = 0x30; // Unreachable malformed date marker. + return binaryPlistObjectTable(bytes, [8, 10, 19], 20); +}; + +const binaryWithExactOverlappingDateAndNull = (): Buffer => { + const bytes = Buffer.alloc(55); + bytes.set([0xa2, 1, 2, 0x33], 8); + bytes.writeDoubleBE(0, 12); + return binaryPlistObjectTable(bytes, [8, 11, 18], 20); +}; + +const binaryWithDateInsideData = (): Buffer => + Buffer.from( + "62706c6973743030a201024a00333fb9ba5e353f7cee080b0d0000000000000101000000000000000300000000000000000000000000000016", + "hex", + ); + +const binaryWithOverlappingAndIndependentDates = (): Buffer => { + const bytes = Buffer.alloc(68); + bytes.set([0xa3, 1, 2, 3, 0x4a, 0, 0x33], 8); + bytes.writeDoubleBE(0.1005, 15); + bytes.set([0x33], 23); + bytes.writeDoubleBE(0.1, 24); + return binaryPlistObjectTable(bytes, [8, 12, 14, 23], 32); +}; + +const binaryWithOverlappingIntegerDate = (): Buffer => + Buffer.from( + "62706c6973743030a20102333fb999999999102a080b120000000000000101000000000000000300000000000000000000000000000014", + "hex", + ); + +const binaryWithSharedOverlappingDate = (): Buffer => { + const bytes = Buffer.alloc(57); + bytes.set([0xa3, 1, 2, 3, 0x33], 8); + bytes.writeDoubleBE(0.1005, 13); + bytes[20] = 0x40; + return binaryPlistObjectTable(bytes, [8, 12, 12, 20], 21); +}; + +const binaryInt128 = (marker = 0x14): Buffer => { + const bytes = Buffer.alloc(58); + bytes[8] = marker; + bytes.writeBigUInt64BE(0n, 9); + bytes.writeBigUInt64BE(9007199254740993n, 17); + bytes[25] = 8; + return binaryPlistObjectTable(bytes, [8], 25); +}; + +const dateProjection = (secondsSince2001: number) => { + const source = binaryDate(secondsSince2001); + const metadata = scanBinaryPlistScalars(source); + const datePlan = estimateBinaryPlistExpansion(source, 1024 * 1024, { + numberProjection: true, + }); + const decodedInput = decodeBinaryPlistScalars(source, metadata, datePlan); + const parsed = parsePlistBinary(source); + if (!parsed.ok) throw parsed.error; + return { source, decodedInput, metadata, parsed: parsed.value }; +}; + +it.each([ + [0, "2001-01-01T00:00:00.000Z"], + [0.1, "2001-01-01T00:00:00.100Z"], + [-0.1, "2000-12-31T23:59:59.900Z"], + [ + (Date.parse("1900-01-01T00:00:00.900Z") - + Date.parse("2001-01-01T00:00:00.000Z")) / + 1000, + "1900-01-01T00:00:00.900Z", + ], +])( + "preserves binary dates exactly representable in milliseconds: %s", + (seconds, iso) => { + const result = dateProjection(seconds); + expect(result.parsed.value).toEqual({ + date: { $plist_type: "date", iso8601: iso }, + }); + expect(result.decodedInput).toBe(result.source); + }, +); + +it.each([0.1005, -0.1005, 0.0005])( + "marks binary sub-millisecond source seconds unknown rather than rounding: %s", + (seconds) => { + const result = dateProjection(seconds); + expect(result.metadata.dates[0]?.secondsSince2001).toBe(seconds); + expect(result.decodedInput).not.toBe(result.source); + expect( + result.source.readDoubleBE(result.metadata.dates[0]!.payloadOffset), + ).toBe(seconds); + expect(result.parsed.value).toEqual({ + date: { $plist_type: "date", iso8601: null }, + }); + expect(result.parsed.limitations).toContainEqual( + expect.stringContaining("date value(s)"), + ); + }, +); + +it("keeps distinct source dates separate when they round to the same millisecond", () => { + const bytes = Buffer.from( + buildBinary([ + new Date("2001-01-01T00:00:00.100Z"), + new Date("2001-01-01T00:00:00.100Z"), + ]), + ); + const metadata = scanBinaryPlistScalars(bytes); + expect(metadata.dates).toHaveLength(2); + bytes.writeDoubleBE(0.1005, metadata.dates[1]!.payloadOffset); + const parsed = parsePlistBinary(bytes); + if (!parsed.ok) throw parsed.error; + expect(parsed.value.value).toEqual([ + { $plist_type: "date", iso8601: "2001-01-01T00:00:00.100Z" }, + { $plist_type: "date", iso8601: null }, + ]); +}); + +it("projects a shared sub-millisecond date reference as unknown at each use", () => { + const source = binarySharedSubmillisecondDate(); + const metadata = scanBinaryPlistScalars(source); + expect(metadata.dates).toHaveLength(1); + expect(metadata.dates[0]?.secondsSince2001).toBe(0.1005); + const parsed = parsePlistBinary(source); + if (!parsed.ok) throw parsed.error; + expect(parsed.value.value).toEqual([ + { $plist_type: "date", iso8601: null }, + { $plist_type: "date", iso8601: null }, + ]); +}); + +it("retains out-of-range date source seconds without rewriting them", () => { + const result = dateProjection(1e20); + expect(result.metadata.dates[0]?.secondsSince2001).toBe(1e20); + expect(result.decodedInput).toBe(result.source); + expect(result.parsed.value).toEqual({ + date: { $plist_type: "date", iso8601: null }, + }); +}); + +it("does not treat a malformed date marker as a valid date scalar", () => { + const bytes = binaryDate(0.1005); + const valid = scanBinaryPlistScalars(bytes); + bytes[valid.dates[0]!.offset] = 0x30; + const metadata = scanBinaryPlistScalars(bytes); + expect(metadata.literals.incomplete).toBe(true); + expect(metadata.dates).toEqual([]); + expect( + decodeBinaryPlistScalars(bytes, metadata, { + reachableDateOffsets: [], + overlappingDateOffsets: [], + reachableWideIntegerOffsets: [], + overlappingWideIntegerOffsets: [], + reachableNumericOffsets: [], + }), + ).toBe(bytes); +}); + +it("rejects a reachable date with a malformed marker", () => { + const bytes = binaryDate(0.1); + const date = scanBinaryPlistScalars(bytes).dates[0]; + if (date === undefined) throw new Error("binary plist fixture lost its date"); + bytes[date.offset] = 0x30; + + const parsed = parsePlistBinary(bytes); + + expect(parsed.ok).toBe(false); + if (parsed.ok) throw new Error("Expected malformed reachable date rejection"); + expect(parsed.error.message).toContain("date marker 0x30 is invalid"); +}); + +it("does not reject an unreachable malformed date object as a decoded value", () => { + const parsed = parsePlistBinary(binaryWithUnreachableMalformedDate()); + + expect(parsed.ok).toBe(true); + if (!parsed.ok) throw parsed.error; + expect(parsed.value.value).toBe("ok"); + expect(parsed.value.limitations).toContainEqual( + expect.stringContaining("number metadata could not be completely observed"), + ); +}); + +it("keeps reachable date uncertainty when another object is malformed", () => { + const parsed = parsePlistBinary(binaryWithReachableAndUnreachableDate()); + + expect(parsed.ok).toBe(true); + if (!parsed.ok) throw parsed.error; + expect(parsed.value.value).toEqual([{ $plist_type: "date", iso8601: null }]); + expect(parsed.value.limitations).toContainEqual( + expect.stringContaining("number metadata could not be completely observed"), + ); +}); + +it("retains reachable date source facts when an unreachable offset is invalid", () => { + const bytes = binaryWithReachableAndUnreachableDate(); + bytes[22] = 0xff; + const metadata = scanBinaryPlistScalars(bytes); + const parsed = parsePlistBinary(bytes); + + expect(metadata.dates).toHaveLength(1); + expect(metadata.dates[0]?.secondsSince2001).toBe(0.1005); + expect(parsed.ok).toBe(true); + if (!parsed.ok) throw parsed.error; + expect(parsed.value.value).toEqual([{ $plist_type: "date", iso8601: null }]); + expect(parsed.value.limitations).toContainEqual( + expect.stringContaining("number metadata could not be completely observed"), + ); +}); + +it("does not rewrite shared bytes when a reachable date overlaps another object", () => { + const bytes = binaryPlistWithOverlappingDateAndData(); + const parsed = parseWithoutMutatingSource(bytes); + + expect(parsed.value).toEqual([ + { $plist_type: "date", iso8601: null }, + { $plist_type: "data", base64: "" }, + ]); + expect(parsed.limitations).toContainEqual( + expect.stringContaining("sub-millisecond source records overlap"), + ); +}); + +it("keeps an exactly millisecond-representable overlapping date known", () => { + const parsed = parsePlistBinary(binaryWithExactOverlappingDateAndNull()); + + expect(parsed.ok).toBe(true); + if (!parsed.ok) throw parsed.error; + expect(parsed.value.value).toEqual([ + { $plist_type: "date", iso8601: "2001-01-01T00:00:00.000Z" }, + null, + ]); + expect(parsed.value.limitations).not.toContainEqual( + expect.stringContaining("sub-millisecond source records overlap"), + ); +}); + +it("keeps a date unknown when its start is inside another reachable payload", () => { + const bytes = binaryWithDateInsideData(); + const parsed = parseWithoutMutatingSource(bytes); + + expect(parsed.value).toEqual([ + { $plist_type: "data", base64: "ADM/ubpeNT987g==" }, + { $plist_type: "date", iso8601: null }, + ]); + expect(parsed.limitations).toContainEqual( + expect.stringContaining("sub-millisecond source records overlap"), + ); +}); + +it("isolates an overlapping date without losing an independent date", () => { + const parsed = parsePlistBinary(binaryWithOverlappingAndIndependentDates()); + + expect(parsed.ok).toBe(true); + if (!parsed.ok) throw parsed.error; + expect(parsed.value.value).toEqual([ + { $plist_type: "data", base64: "ADM/ubpeNT987g==" }, + { $plist_type: "date", iso8601: null }, + { $plist_type: "date", iso8601: "2001-01-01T00:00:00.100Z" }, + ]); + expect(parsed.value.limitations).toContainEqual( + expect.stringContaining("sub-millisecond source records overlap"), + ); +}); + +it("retains the integer sibling of a sub-millisecond overlapping date", () => { + const bytes = binaryWithOverlappingIntegerDate(); + const parsed = parseWithoutMutatingSource(bytes); + + expect(parsed.value).toEqual([{ $plist_type: "date", iso8601: null }, 42]); + expect(parsed.limitations).toContainEqual( + expect.stringContaining("sub-millisecond source records overlap"), + ); +}); + +it("isolates a shared overlapping date once while preserving both references", () => { + const parsed = parsePlistBinary(binaryWithSharedOverlappingDate()); + + if (!parsed.ok) throw parsed.error; + expect(parsed.value.value).toEqual([ + { $plist_type: "date", iso8601: null }, + { $plist_type: "date", iso8601: null }, + { $plist_type: "data", base64: "" }, + ]); + expect(parsed.value.limitations).toContainEqual( + expect.stringContaining( + "2 date value(s) are reported with unknown timestamps", + ), + ); +}); + +it("preserves valid 16-byte integers while rejecting unsupported numeric markers", () => { + const parsed = parsePlistBinary(binaryInt128()); + if (!parsed.ok) throw parsed.error; + expect(parsed.value.value).toEqual({ + $plist_type: "integer", + decimal: "9007199254740993", + }); + + for (const [marker, message] of [ + [0x15, "integer width is invalid"], + [0x20, "real width is invalid"], + [0x24, "real width is invalid"], + ] as const) { + const rejected = parsePlistBinary(binaryInt128(marker)); + expect(rejected.ok).toBe(false); + if (rejected.ok) + throw new Error("Expected unsupported numeric marker rejection"); + expect(rejected.error.message).toContain(message); + } +}); + +it("keeps binary plist source values lost by XML conversion", () => { + const parsed = parsePlistBinary(binaryPlistWithNegativeZero()); + if (!parsed.ok) throw parsed.error; + expect(parsed.value.value).toEqual({ + negative: { $plist_type: "real", value: "-0" }, + positive: 0, + note: "a\u0001b", + payload: { $plist_type: "data", base64: "AA==" }, + fraction: { $plist_type: "date", iso8601: "2020-05-05T01:02:03.100Z" }, + whole: { $plist_type: "date", iso8601: "2020-05-05T01:02:03.000Z" }, + nel: "a\u0085b", + ls: "a\u2028b", + ps: "a\u2029b", + largeInteger: { $plist_type: "integer", decimal: "9007199254740992" }, + }); + expect(parsed.value.limitations).toContain( + '1 negative-zero real value(s) are reported as { "$plist_type": "real", "value": "-0" } because a JSON number cannot carry the sign of zero.', + ); + expect(parsed.value.limitations).toContainEqual( + expect.stringContaining("1 observed integer literal(s)"), + ); + expect(parsed.value.limitations).toContainEqual( + expect.stringContaining("1 unambiguous integer value(s)"), + ); +}); + +it("rejects a truncated binary plist source", () => { + const parsed = parsePlistBinary(Buffer.from("bplist00bad")); + expect(parsed.ok).toBe(false); + if (parsed.ok) throw new Error("Expected malformed binary plist"); + expect(parsed.error.message).toContain("trailer is truncated"); +}); it("retains complete segmented XML keys in native plist fallback evidence", () => { const parsed = parsePlistXml( diff --git a/src/native/parsers/plist.ts b/src/native/parsers/plist.ts index 0ee5fa3d30..b6863b2e48 100644 --- a/src/native/parsers/plist.ts +++ b/src/native/parsers/plist.ts @@ -1,8 +1,13 @@ import { z } from "zod"; +import { parseBinary } from "plist"; -import { AnalysisOutputError } from "../../domain/analysisErrorCore.js"; +import { + AnalysisOutputError, + AnalysisResourceConstraintError, +} from "../../domain/analysisErrorCore.js"; import type { JsonValue } from "../../domain/jsonValue.js"; import { + binaryDatePrecisionOverlapLimitation, negativeZeroReal, negativeZeroRealLimitation, nonFiniteRealLimitation, @@ -20,8 +25,22 @@ import { err, ok, type Result } from "../../domain/result.js"; import { incompletePlistNumberLimitation, + createPlistNumberProjection, type PlistNumberLiterals, } from "../../domain/apple/plistNumbers.js"; +import { + decodeBinaryPlistScalars, + scanBinaryPlistScalars, +} from "../../artifacts/apple/PlistScalars.js"; +import { + estimateBinaryPlistExpansion, + PropertyListDecodeBudgetFailure, +} from "../../artifacts/apple/PropertyListDecodeBudget.js"; +import { hasBinaryPlistSignature } from "../../domain/apple/appleArchiveFormat.js"; + +/** Input ceiling matches native command output; projection is bounded separately. */ +export const MAX_NATIVE_PLIST_BYTES = 64 * 1024 * 1024; +const MAX_NATIVE_PLIST_DECODE_BYTES = 64 * 1024 * 1024; const plistObject = z.record(z.string(), z.unknown()); @@ -204,16 +223,7 @@ export const parsePlistXml = ( bundle: projectPlistBundle(value), limitations: [ 'plutil cannot express this plist as JSON, so it was decoded from plutil\'s XML conversion; data and date values are typed objects such as { "$plist_type": "data", "base64": ... }.', - ...(projected.nonFiniteRealCount === 0 - ? [] - : [nonFiniteRealLimitation(projected.nonFiniteRealCount)]), - ...negativeZeroLimitations(projected.negativeZeroRealCount), - ...(projected.unknownDateCount === 0 - ? [] - : [unknownDateLimitation(projected.unknownDateCount)]), - ...(projected.unknownScalarCount === 0 - ? [] - : [unknownScalarLimitation(projected.unknownScalarCount)]), + ...projectedScalarLimitations(projected), ...(literals.incomplete ? [incompletePlistNumberLimitation] : []), ...numberLimitations(exactIntegerCount, ambiguousNumberCount), ...prototypeKeyLimitations(omittedPrototypeKeys), @@ -221,9 +231,94 @@ export const parsePlistXml = ( }); }; +/** Decode binary plist source bytes before conversion can erase scalar facts. */ +export const parsePlistBinary = ( + bytes: Buffer, +): Result< + ParsedPlist, + AnalysisOutputError | AnalysisResourceConstraintError +> => { + try { + if (!hasBinaryPlistSignature(bytes)) + throw new TypeError("selected file does not have a binary plist header"); + const expansion = estimateBinaryPlistExpansion( + bytes, + MAX_NATIVE_PLIST_DECODE_BYTES, + { + budget: "the native plist decode budget", + numberProjection: true, + fail: (kind, message) => + kind === "cycle" + ? new TypeError( + "binary plist object references form a cycle, which a property list cannot represent", + ) + : new PropertyListDecodeBudgetFailure(kind, message), + }, + ); + const scalarMetadata = scanBinaryPlistScalars(bytes, expansion); + const decoded = parseBinary( + decodeBinaryPlistScalars(bytes, scalarMetadata, expansion), + ); + const projected = projectPlistValue(decoded); + const numbers = createPlistNumberProjection(scalarMetadata.literals); + const value = numbers.project(projected.value); + return ok({ + value, + bundle: projectPlistBundle(value), + limitations: [ + ...projectedScalarLimitations(projected), + ...(expansion.datePrecisionOverlapCount === 0 + ? [] + : [ + binaryDatePrecisionOverlapLimitation( + expansion.datePrecisionOverlapCount, + ), + ]), + ...numbers.limitations(), + ...(expansion.binaryPrototypeKeyObserved + ? [omittedPrototypeKeysLimitation(null)] + : []), + ], + }); + } catch (cause: unknown) { + if (cause instanceof PropertyListDecodeBudgetFailure) + return err( + new AnalysisResourceConstraintError( + "inspect_plist", + "memory", + cause.message, + { maximum_decoded_bytes: MAX_NATIVE_PLIST_DECODE_BYTES }, + { cause, retryable: false }, + ), + ); + return err( + new AnalysisOutputError( + "inspect_plist", + cause instanceof Error ? cause.message : String(cause), + { cause }, + ), + ); + } +}; + const prototypeKeyLimitations = (count: number): string[] => count === 0 ? [] : [omittedPrototypeKeysLimitation(count)]; +const projectedScalarLimitations = ( + projected: ProjectedPlistValue, +): string[] => [ + ...(projected.nonFiniteRealCount === 0 + ? [] + : [nonFiniteRealLimitation(projected.nonFiniteRealCount)]), + ...negativeZeroLimitations(projected.negativeZeroRealCount), + ...(projected.unknownDateCount === 0 + ? [] + : [unknownDateLimitation(projected.unknownDateCount)]), + ...(projected.unknownScalarCount === 0 + ? [] + : [unknownScalarLimitation(projected.unknownScalarCount)]), +]; + const negativeZeroLimitations = (count: number): string[] => count === 0 ? [] : [negativeZeroRealLimitation(count)]; diff --git a/src/native/pe/PeResourceParser.test.ts b/src/native/pe/PeResourceParser.test.ts index 300c054c13..ecf3ec4195 100644 --- a/src/native/pe/PeResourceParser.test.ts +++ b/src/native/pe/PeResourceParser.test.ts @@ -22,6 +22,59 @@ const parse = (bytes: Buffer, maxEntries = 4096, signal?: AbortSignal) => signal, ); +describe("word-aligned PE resource records", () => { + it.each([false, true])( + "preserves word-aligned resource records after UTF-16 names (PE32+ = %s)", + async (plus) => { + for (const identities of [ + { type: 10, name: 1, language: "AB" }, + { type: "AB", name: 1, language: 1033 }, + ]) { + const payload = Buffer.from("opaque resource"); + const fixture = peResourceFixture( + plus, + [{ ...identities, payload }], + 65536, + 2, + ); + const dataOffset = fixture.dataOffsets[0]; + if (dataOffset === undefined) + throw new Error("Missing fixture data entry"); + expect(fixture.dataOffsets.some((at) => at % 4 === 2)).toBe(true); + if (typeof identities.type === "string") + expect(fixture.directoryOffsets.some((at) => at % 4 === 2)).toBe( + true, + ); + const result = peResourcesSchema.parse(await parse(fixture.bytes)); + expect( + result.directories.map(({ location }) => location.offset), + ).toEqual(fixture.directoryOffsets); + expect(result.coverage).toMatchObject({ + status: "complete", + resources: 1, + }); + expect(result.resources[0]?.data_entry_location).toEqual({ + offset: dataOffset, + bytes: 16, + }); + expect(result.resources[0]?.payload.sha256).toBe( + createHash("sha256").update(payload).digest("hex"), + ); + expect(result.resources[0]?.payload.location.offset).toBe( + fixture.bytes.readUInt32LE(dataOffset) - 0x1c00, + ); + fixture.bytes.writeUInt32LE( + dataOffset - 0x400 + 15, + fixture.directoryAt + 4, + ); + await expect(parse(fixture.bytes)).rejects.toMatchObject({ + reason: "format", + }); + } + }, + ); +}); + describe("portable PE resource parsing", () => { it.each([false, true])( "preserves original identities, ranges, hashes and icon candidates (PE32+ = %s)", diff --git a/src/native/pe/PeResourceParser.ts b/src/native/pe/PeResourceParser.ts index 1f266defb3..6a56ac15c2 100644 --- a/src/native/pe/PeResourceParser.ts +++ b/src/native/pe/PeResourceParser.ts @@ -120,7 +120,6 @@ export const parsePeResources = async ( ancestors: ReadonlySet, ): Promise => { if (ancestors.has(at)) peFailure("Cyclic PE resource directory."); - if (at % 4 !== 0) peFailure("Unaligned PE resource directory."); const header = relative(at, 16); const named = bytes.readUInt16LE(header + 12); const ids = bytes.readUInt16LE(header + 14); @@ -177,7 +176,6 @@ export const parsePeResources = async ( "Unsupported resource leaf outside the type/name/language profile.", ); const data = relative(offset, 16); - if (offset % 4 !== 0) peFailure("Unaligned resource data entry."); const rva = bytes.readUInt32LE(data); const size = bytes.readUInt32LE(data + 4); const payloadOffset = layout.map(rva, size); diff --git a/src/native/pe/PeResources.fixture.ts b/src/native/pe/PeResources.fixture.ts index d50136e291..45c2e51f57 100644 --- a/src/native/pe/PeResources.fixture.ts +++ b/src/native/pe/PeResources.fixture.ts @@ -38,6 +38,7 @@ export const peResourceFixture = ( { type: 14, name: 42, language: 1033, payload: iconGroup() }, ], sectionBytes = 65536, + recordAlignment: 2 | 4 = 4, ) => { const tree: Tree = { children: new Map() }; for (const resource of resources) { @@ -61,7 +62,7 @@ export const peResourceFixture = ( } const section = Buffer.alloc(sectionBytes); let cursor = 0; - const allocate = (size: number, alignment = 4): number => { + const allocate = (size: number, alignment = recordAlignment): number => { cursor = Math.ceil(cursor / alignment) * alignment; const at = cursor; cursor += size; @@ -110,7 +111,7 @@ export const peResourceFixture = ( }; emit(tree); for (const { dataAt, resource } of leaves) { - const payloadAt = allocate(resource.payload.length); + const payloadAt = allocate(resource.payload.length, 4); resource.payload.copy(section, payloadAt); section.writeUInt32LE(0x2000 + payloadAt, dataAt); section.writeUInt32LE(resource.payload.length, dataAt + 4); diff --git a/src/pcap/PcapOutput.test.ts b/src/pcap/PcapOutput.test.ts index 12569151f0..251abb39e2 100644 --- a/src/pcap/PcapOutput.test.ts +++ b/src/pcap/PcapOutput.test.ts @@ -16,6 +16,93 @@ const capture = (layers: ReadonlyArray>) => })), ); +it.each([ + [[], []], + [["", ""], []], + [["user_dlt:data", ""], [{ protocol: "user_dlt:data", packet_count: 1 }]], + [["", "user_dlt:data"], [{ protocol: "user_dlt:data", packet_count: 1 }]], + [ + ["user_dlt:data", "", "user_dlt:data"], + [{ protocol: "user_dlt:data", packet_count: 2 }], + ], +] as const)( + "counts every frame while aggregating only reported protocol stacks: %j", + async (stacks, expected) => { + const result = await parseTsharkJsonOutput( + capture(stacks.map((stack) => ({ "frame.protocols": [stack] }))), + ); + expect(result.packetCount).toBe(stacks.length); + expect(result.protocolDistribution).toEqual(expected); + expect(result.protocolCount).toBe(expected.length); + expect(result.partial).toBe(false); + }, +); + +it.each([ + { "frame.number": ["1"] }, + { "frame.number": ["1"], "frame.protocols": [] }, + { "frame.number": ["1"], "frame.protocols": ["", "ip"] }, + { "frame.number": ["1"], "frame.protocols": ["ip", "ip"] }, + { "frame.number": ["1"], "frame.protocols": [null] }, + { "frame.number": ["1"], "frame.protocols": "" }, + { "frame.protocols": [""] }, + { "frame.number": ["2"], "frame.protocols": [""] }, +])("rejects malformed or ambiguous frame inventories: %j", async (layers) => { + await expect( + parseTsharkJsonOutput(JSON.stringify([{ _source: { layers } }])), + ).rejects.toMatchObject({ _tag: "AnalysisProtocolError" }); +}); + +it.each(["before", "after"] as const)( + "retains endpoint and DNS facts with an empty stack %s the response", + async (position) => { + const empty = { "frame.protocols": [""] }; + const response = { + "frame.protocols": ["eth:ethertype:ip:udp:dns"], + "ip.src": ["192.0.2.53"], + "ip.dst": ["192.0.2.1"], + "dns.qry.name": ["api.example"], + "dns.flags.response": ["True"], + "dns.a": ["203.0.113.5"], + }; + const result = await parseTsharkJsonOutput( + capture(position === "before" ? [empty, response] : [response, empty]), + ); + expect(result).toMatchObject({ + packetCount: 2, + protocolCount: 1, + protocolDistribution: [ + { protocol: "eth:ethertype:ip:udp:dns", packet_count: 1 }, + ], + endpointCount: 2, + endpoints: [ + { address: "192.0.2.1", port: null, packet_count: 1 }, + { address: "192.0.2.53", port: null, packet_count: 1 }, + ], + dnsQueryCount: 1, + dnsQueries: [ + { + query: "api.example", + response_addresses: ["203.0.113.5"], + response_address_count: 1, + }, + ], + partial: false, + }); + }, +); + +it("rejects duplicate frame identities even when protocol stacks are empty", async () => { + await expect( + parseTsharkJsonOutput( + capture([ + { "frame.protocols": [""] }, + { "frame.number": ["1"], "frame.protocols": [""] }, + ]), + ), + ).rejects.toMatchObject({ _tag: "AnalysisProtocolError" }); +}); + it("binds interleaved DNS answers to the question in their response packet", async () => { const result = await parseTsharkJsonOutput( capture([ diff --git a/src/pcap/PcapOutput.ts b/src/pcap/PcapOutput.ts index 7226e46370..1f6da7ee6e 100644 --- a/src/pcap/PcapOutput.ts +++ b/src/pcap/PcapOutput.ts @@ -269,13 +269,14 @@ export const parseTsharkJsonOutput = async ( return protocolFailure( "frame inventory is missing, duplicated or out of order", ); - if ( - field("frame.protocols").length !== 1 || - field("frame.protocols")[0] === "" - ) + const protocolStack = field("frame.protocols"); + if (protocolStack.length !== 1) return protocolFailure( "packet protocol inventory is missing or ambiguous", ); + // TShark reports [""] for a frame with no dissected protocol, including + // zero-length packets. Keep the frame without inventing a protocol row. + const protocol = protocolStack[0]!; // Count one occurrence per packet even for loopback or nested IP layers. for (const address of new Set([ ...field("ip.src"), @@ -314,10 +315,9 @@ export const parseTsharkJsonOutput = async ( } for (const value of field("tls.handshake.extensions_server_name")) if (value !== "") add(sni, value); - for (const value of new Set(field("frame.protocols"))) { - if (value === "") continue; - if (!protocols.has(value)) charge(value); - protocols.set(value, (protocols.get(value) ?? 0) + 1); + if (protocol !== "") { + if (!protocols.has(protocol)) charge(protocol); + protocols.set(protocol, (protocols.get(protocol) ?? 0) + 1); } for (const value of field("http.request.full_uri")) { if (value === "" || observedUris.has(value)) continue; diff --git a/src/process/DarwinProcessRunTokenReader.ts b/src/process/DarwinProcessRunTokenReader.ts index dfdfd2811e..feb04bac97 100644 --- a/src/process/DarwinProcessRunTokenReader.ts +++ b/src/process/DarwinProcessRunTokenReader.ts @@ -518,6 +518,9 @@ export const createDarwinProcessRunTokenReader = ( } catch (cause: unknown) { operation.primaryFailure = { cause }; if (operationRoot !== undefined) { + // execFileOutput has settled, so inherited compiler pipes are closed. + // That is not proof for a child that did not inherit them; this owner + // does not signal processes it did not launch. try { await cleanupRoot(operationRoot); } catch (cleanup: unknown) { diff --git a/src/process/ExecFileOutput.test.ts b/src/process/ExecFileOutput.test.ts index a385d80719..21f2f826ba 100644 --- a/src/process/ExecFileOutput.test.ts +++ b/src/process/ExecFileOutput.test.ts @@ -1,4 +1,5 @@ -import { access, readFile, rm } from "node:fs/promises"; +import { execFile } from "node:child_process"; +import { access, mkdir, readFile, rm } from "node:fs/promises"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -292,6 +293,114 @@ describe.skipIf(process.platform === "win32")( }, ); +const descendantCommand = async (pid: number): Promise => { + if (process.platform === "linux") + return readFile(`/proc/${pid}/cmdline`, "utf8").catch(() => ""); + return new Promise((resolve) => { + execFile( + "ps", + ["-p", String(pid), "-o", "command="], + { encoding: "utf8" }, + (error, stdout) => { + resolve(error ? "" : stdout); + }, + ); + }); +}; + +/** Signal only a still-running descendant whose command line contains `marker`. */ +const stopOwnedDescendant = async ( + pid: number, + marker: string, +): Promise => { + if (!Number.isInteger(pid) || pid <= 0 || pid === process.pid) + return undefined; + if (!(await descendantCommand(pid)).includes(marker)) return undefined; + try { + process.kill(pid, "SIGKILL"); + } catch (cause: unknown) { + if (cause instanceof Error && "code" in cause && cause.code === "ESRCH") + return undefined; + return cause; + } + return undefined; +}; + +describe.skipIf(process.platform === "win32")( + "execFileOutput inherited pipes", + () => { + it("keeps scratch files until an inherited-pipe descendant closes them", async () => { + const directory = await createTestTempDirectory("rea-exec-inherited-"); + const scratch = join(directory, "scratch"); + const finished = join(directory, "finished.json"); + const pidPath = join(directory, "child.pid"); + await mkdir(scratch); + const childSource = [ + 'const fs = require("node:fs");', + "setTimeout(() => {", + ` fs.writeFileSync(${JSON.stringify(finished)}, JSON.stringify({ scratchExists: fs.existsSync(${JSON.stringify(scratch)}) }));`, + " process.exit(0);", + "}, 7000);", + ].join("\n"); + const script = [ + 'const { spawn } = require("node:child_process");', + 'const fs = require("node:fs");', + `const child = spawn(process.execPath, ["-e", ${JSON.stringify(childSource)}], { stdio: "inherit" });`, + `fs.writeFileSync(${JSON.stringify(pidPath)}, String(child.pid));`, + "process.exit(0);", + ].join("\n"); + const settled = execFileOutput(process.execPath, ["-e", script], { + timeout: 200, + stopSignal: "SIGTERM", + }) + .finally(async () => { + await rm(scratch, { recursive: true, force: true }); + }) + .then( + () => ({ ok: true as const }), + (cause: unknown) => ({ ok: false as const, cause }), + ); + let killFailure: unknown; + let descendantSettled = false; + try { + const deadline = Date.now() + 15_000; + let text: string | undefined; + while (Date.now() < deadline) { + text = await readFile(finished, "utf8").then( + (value) => value, + (cause: unknown) => { + if ( + cause instanceof Error && + "code" in cause && + cause.code === "ENOENT" + ) + return undefined; + throw cause; + }, + ); + if (text !== undefined) break; + await new Promise((resolve) => setTimeout(resolve, 20)); + } + expect(text).toBeDefined(); + expect(JSON.parse(text ?? "")).toEqual({ scratchExists: true }); + const outcome = await settled; + descendantSettled = true; + expect(outcome).toMatchObject({ + ok: false, + cause: { code: "ETIMEDOUT", killed: true }, + }); + } finally { + if (!descendantSettled) { + const pid = Number(await readFile(pidPath, "utf8").catch(() => "")); + killFailure = await stopOwnedDescendant(pid, finished); + } + await settled; + } + if (killFailure !== undefined) throw killFailure; + }); + }, +); + describe("execFileOutput failure metadata", () => { it("preserves raw byte counts on a failed command with invalid UTF-8", async () => { const failure: unknown = await execFileOutput(process.execPath, [ diff --git a/src/process/ExecFileOutput.ts b/src/process/ExecFileOutput.ts index 2e80e337a0..022254b337 100644 --- a/src/process/ExecFileOutput.ts +++ b/src/process/ExecFileOutput.ts @@ -10,10 +10,12 @@ type ExecFileOutputOptions = Omit< readonly maxBuffer?: number; /** * Signal that stops the process on abort or timeout. The promise then - * settles only after the process exits, so the caller can remove files it - * writes; a process still running after a grace period is killed. Without - * it, Node's execFile sends SIGTERM on abort and rejects before the - * process exits. + * settles only after that process's stdio pipes close, so a caller can + * remove files the command wrote. A descendant that inherited those pipes + * keeps them open until it exits; destroying the readers is not + * process-family completion and must not unblock this settlement. A process + * still running after a grace period is killed. Without this signal, Node's + * execFile sends SIGTERM on abort and rejects before the process exits. */ readonly stopSignal?: NodeJS.Signals; }; diff --git a/src/process/ProcessOwnership.fixture.ts b/src/process/ProcessOwnership.fixture.ts index fead795dad..af86243a71 100644 --- a/src/process/ProcessOwnership.fixture.ts +++ b/src/process/ProcessOwnership.fixture.ts @@ -1,7 +1,25 @@ -import { vi } from "vitest"; +import { expect, vi } from "vitest"; import type { ProcessOwnershipHost } from "./ProcessOwnership.js"; import { ProcessOwnershipEvidenceOwner } from "./ProcessOwnershipEvidence.js"; +export const verifiedLineage = ( + descendants: readonly { + readonly pid: number; + readonly parentPid: number; + readonly processGroupId: number; + }[], +) => ({ + status: "verified", + observedAt: expect.any(String), + lineage: { + runId: "run-token", + launcherPid: 100, + launcherParentPid: 1, + processGroupId: 100, + descendants, + }, +}); + export const createOwnership = () => ({ runId: "run-token", leaderPid: 100, diff --git a/src/process/ProcessOwnership.part2.test.ts b/src/process/ProcessOwnership.part2.test.ts index 9984ccfbe8..e88fb64799 100644 --- a/src/process/ProcessOwnership.part2.test.ts +++ b/src/process/ProcessOwnership.part2.test.ts @@ -5,7 +5,11 @@ import { type ProcessOwnershipHost, } from "./ProcessOwnership.js"; import { observeOwnedProcessLineage } from "./ProcessOwnershipObservation.js"; -import { createOwnership, host } from "./ProcessOwnership.fixture.js"; +import { + createOwnership, + host, + verifiedLineage, +} from "./ProcessOwnership.fixture.js"; let ownership = createOwnership(); beforeEach(() => { @@ -173,20 +177,12 @@ describe("owned process-group cleanup discovery", () => { }); await expect( observeOwnedProcessLineage(ownership, adapter), - ).resolves.toEqual({ - status: "verified", - observedAt: expect.any(String), - lineage: { - runId: "run-token", - launcherPid: 100, - launcherParentPid: 1, - processGroupId: 100, - descendants: [ - { pid: 101, parentPid: 100, processGroupId: 100 }, - { pid: 102, parentPid: 100, processGroupId: 100 }, - ], - }, - }); + ).resolves.toEqual( + verifiedLineage([ + { pid: 101, parentPid: 100, processGroupId: 100 }, + { pid: 102, parentPid: 100, processGroupId: 100 }, + ]), + ); }); }); diff --git a/src/process/ProcessOwnership.part3.test.ts b/src/process/ProcessOwnership.part3.test.ts index bab43d7a97..6707deb12c 100644 --- a/src/process/ProcessOwnership.part3.test.ts +++ b/src/process/ProcessOwnership.part3.test.ts @@ -1,11 +1,60 @@ import { describe, expect, it, vi } from "vitest"; -import type { ProcessOwnershipHost } from "./ProcessOwnership.js"; +import type { + ProcessOwnershipHost, + ProcessTableEntry, +} from "./ProcessOwnership.js"; import { observeOwnedProcessLineage } from "./ProcessOwnershipObservation.js"; +import { verifiedLineage } from "./ProcessOwnership.fixture.js"; const ownership = { runId: "run-token", leaderPid: 100, processGroupId: 100, }; +const row = ( + pid: number, + parentPid: number, + processGroupId = pid, + command = `pid-${String(pid)}`, +): ProcessTableEntry => ({ + pid, + parentPid, + processGroupId, + state: "S", + command, +}); +const environmentForExited = (...pids: number[]) => + vi.fn((pid: number) => + pids.includes(pid) + ? Promise.reject(new Error("process exited during environment read")) + : Promise.resolve({ REA_PROCESS_RUN_ID: "run-token" }), + ); +const environmentForReplacementReads = ( + runIdAtRead: (read: number) => string | undefined, +) => { + let reads = 0; + const environment = vi.fn((pid: number) => { + if (pid === 101 || pid === 103) + return Promise.reject( + new Error("process exited during environment read"), + ); + if (pid !== 102) + return Promise.resolve({ REA_PROCESS_RUN_ID: "run-token" }); + reads += 1; + const runId = runIdAtRead(reads); + return Promise.resolve(runId ? { REA_PROCESS_RUN_ID: runId } : {}); + }); + return { environment, reads: () => reads }; +}; +const observeLineage = ( + listProcesses: ProcessOwnershipHost["listProcesses"], + environment: ProcessOwnershipHost["environment"], + processOwnership = ownership, +) => + observeOwnedProcessLineage(processOwnership, { + listProcesses, + environment, + signalGroup: vi.fn(), + }); describe("owned process-group cleanup discovery", () => { it("walks the PPID tree across process-group boundaries", async () => { const environment = vi.fn((pid: number) => @@ -50,20 +99,12 @@ describe("owned process-group cleanup discovery", () => { }; await expect( observeOwnedProcessLineage(ownership, adapter), - ).resolves.toEqual({ - status: "verified", - observedAt: expect.any(String), - lineage: { - runId: "run-token", - launcherPid: 100, - launcherParentPid: 1, - processGroupId: 100, - descendants: [ - { pid: 101, parentPid: 100, processGroupId: 101 }, - { pid: 102, parentPid: 101, processGroupId: 102 }, - ], - }, - }); + ).resolves.toEqual( + verifiedLineage([ + { pid: 101, parentPid: 100, processGroupId: 101 }, + { pid: 102, parentPid: 101, processGroupId: 102 }, + ]), + ); }); it("omits a descendant that exits while lineage is validated", async () => { const launcher = { @@ -87,10 +128,7 @@ describe("owned process-group cleanup discovery", () => { .mockResolvedValueOnce([launcher]); const host: ProcessOwnershipHost = { listProcesses, - environment: (pid) => - pid === 101 - ? Promise.reject(new Error("process exited during environment read")) - : Promise.resolve({ REA_PROCESS_RUN_ID: "run-token" }), + environment: environmentForExited(101), signalGroup: vi.fn(), }; @@ -106,7 +144,362 @@ describe("owned process-group cleanup discovery", () => { }, }); }); +}); +describe("identityless lineage refresh keeps replacement rows", () => { + it("does not publish an empty lineage when exit refresh reveals a token-owned replacement", async () => { + const launcher = row(100, 1, 100, "root"); + const child = row(101, 100, 100, "child"); + const replacement = row(102, 1, 102, "reparented-grandchild"); + const environment = environmentForExited(101); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([launcher, child]) + .mockResolvedValue([launcher, replacement]); + await expect( + observeLineage(listProcesses, environment), + ).resolves.toMatchObject({ + status: "unavailable", + reason: + "process lineage refresh contained owned PID(s) outside the observed descendants: 102", + }); + expect(environment).toHaveBeenCalledWith(102); + }); + + it("publishes a descendant that appears in the exit refresh and still belongs to the launcher", async () => { + const launcher = row(100, 1, 100, "root"); + const child = row(101, 100, 100, "child"); + const grandchild = row(102, 100, 102, "grandchild"); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([launcher, child]) + .mockResolvedValue([launcher, grandchild]); + await expect( + observeLineage(listProcesses, environmentForExited(101)), + ).resolves.toMatchObject({ + status: "verified", + lineage: { + descendants: [{ pid: 102, parentPid: 100, processGroupId: 102 }], + }, + }); + }); +}); + +describe("identityless lineage refresh ignores the prior process table", () => { + it("does not token-read a pre-existing row when a descendant exits", async () => { + const environment = vi.fn((pid: number) => + pid === 101 + ? Promise.reject(new Error("process exited during environment read")) + : pid === 50 + ? Promise.reject(new Error("unreadable pre-existing process")) + : Promise.resolve({ REA_PROCESS_RUN_ID: "run-token" }), + ); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([row(100, 1, 100), row(101, 100, 100), row(50, 1)]) + .mockResolvedValue([row(100, 1, 100), row(50, 1)]); + await expect( + observeLineage(listProcesses, environment), + ).resolves.toMatchObject({ + status: "verified", + lineage: { descendants: [] }, + }); + expect(environment.mock.calls.map(([pid]) => pid)).not.toContain(50); + }); + + it("re-reads a refresh row after it moves under the launcher", async () => { + const { environment, reads } = environmentForReplacementReads((read) => + read === 1 ? undefined : "run-token", + ); + const outside = row(102, 1, 102); + const joined = row(102, 100, 102); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([row(100, 1, 100), row(101, 100), row(103, 100)]) + .mockResolvedValueOnce([row(100, 1, 100), row(103, 100), outside]) + .mockResolvedValue([row(100, 1, 100), joined]); + await expect( + observeLineage(listProcesses, environment), + ).resolves.toMatchObject({ + status: "verified", + lineage: { + descendants: [{ pid: 102, parentPid: 100, processGroupId: 102 }], + }, + }); + expect(reads()).toBe(2); + }); +}); + +describe("identityless lineage refresh follows tree membership", () => { + it("token-checks an initial outsider after it joins the launcher", async () => { + const environment = environmentForExited(101); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([row(100, 1, 100), row(101, 100), row(102, 1)]) + .mockResolvedValue([row(100, 1, 100), row(102, 100)]); + await expect( + observeLineage(listProcesses, environment), + ).resolves.toMatchObject({ + status: "verified", + lineage: { + descendants: [{ pid: 102, parentPid: 100, processGroupId: 102 }], + }, + }); + expect(environment).toHaveBeenCalledWith(102); + }); + + it("rejects a cached foreign row after an ancestor joins the launcher", async () => { + let outsiderReads = 0; + const environment = vi.fn((pid: number) => { + if (pid === 101 || pid === 103) + return Promise.reject( + new Error("process exited during environment read"), + ); + if (pid !== 102) + return Promise.resolve({ REA_PROCESS_RUN_ID: "run-token" }); + outsiderReads += 1; + return Promise.resolve({}); + }); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([ + row(100, 1, 100), + row(101, 100), + row(103, 100), + row(50, 1), + ]) + .mockResolvedValueOnce([ + row(100, 1, 100), + row(103, 100), + row(50, 1), + row(102, 50), + ]) + .mockResolvedValue([row(100, 1, 100), row(50, 100), row(102, 50)]); + await expect( + observeOwnedProcessLineage(ownership, { + listProcesses, + environment, + signalGroup: vi.fn(), + }), + ).resolves.toMatchObject({ + status: "unavailable", + reason: "process lineage contains an unowned or PID-reused process", + }); + expect(outsiderReads).toBe(2); + expect(environment.mock.calls.filter(([pid]) => pid === 50)).toHaveLength( + 1, + ); + }); + + it("re-reads a cached descendant when its relation is unchanged", async () => { + const { environment, reads } = environmentForReplacementReads((read) => + read === 1 ? "run-token" : undefined, + ); + const descendant = row(102, 100, 102); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([row(100, 1, 100), row(101, 100), row(103, 100)]) + .mockResolvedValueOnce([row(100, 1, 100), row(103, 100), descendant]) + .mockResolvedValue([row(100, 1, 100), descendant]); + await expect( + observeOwnedProcessLineage(ownership, { + listProcesses, + environment, + signalGroup: vi.fn(), + }), + ).resolves.toMatchObject({ + status: "unavailable", + reason: "process lineage contains an unowned or PID-reused process", + }); + expect(reads()).toBe(2); + }); +}); + +describe("identityless lineage refresh drops exited descendants", () => { + it("omits a validated descendant that is gone from the latest refresh", async () => { + const environment = vi.fn((pid: number) => + pid === 101 || pid === 103 + ? Promise.reject(new Error("process exited during environment read")) + : Promise.resolve({ REA_PROCESS_RUN_ID: "run-token" }), + ); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([ + row(100, 1, 100), + row(102, 100), + row(101, 100), + row(103, 100), + ]) + .mockResolvedValueOnce([row(100, 1, 100), row(102, 100), row(103, 100)]) + .mockResolvedValue([row(100, 1, 100)]); + const result = await observeLineage(listProcesses, environment); + expect(result.status).toBe("verified"); + if (result.status !== "verified") return; + expect(result.lineage.descendants).toEqual([]); + }); +}); + +describe("identityless lineage refresh rechecks the launcher relation", () => { + it.each([ + [ + "process group", + row(100, 1, 999, "root"), + { ...ownership, expectedCommand: "root" }, + "owned launcher process-group identity did not match", + ], + [ + "command", + row(100, 1, 100, "replacement"), + { ...ownership, expectedCommand: "root" }, + "owned launcher command identity did not match (observed=replacement; expected=root)", + ], + [ + "parent", + row(100, 7, 100, "root"), + { ...ownership, expectedParentPid: 1, expectedCommand: "root" }, + "owned launcher parent identity did not match", + ], + ] as const)( + "rejects a same-token launcher whose %s changed", + async (_identity, currentLauncher, processOwnership, reason) => { + const listProcesses = vi + .fn() + .mockResolvedValueOnce([ + row(100, 1, 100, "root"), + row(102, 100), + row(101, 100), + ]) + .mockResolvedValue([currentLauncher, row(102, 100)]); + await expect( + observeLineage( + listProcesses, + environmentForExited(101), + processOwnership, + ), + ).resolves.toMatchObject({ status: "unavailable", reason }); + }, + ); +}); + +describe("identityless lineage refresh rechecks the launcher", () => { + it("does not publish lineage when the launcher PID is gone on a later refresh", async () => { + const listProcesses = vi + .fn() + .mockResolvedValueOnce([row(100, 1, 100), row(102, 100), row(101, 100)]) + .mockResolvedValue([row(102, 100)]); + await expect( + observeOwnedProcessLineage(ownership, { + listProcesses, + environment: environmentForExited(101), + signalGroup: vi.fn(), + }), + ).resolves.toMatchObject({ + status: "unavailable", + reason: "owned launcher exited during lineage validation", + }); + }); + + it("rejects a refresh whose launcher PID is no longer owned", async () => { + let launcherReads = 0; + const environment = vi.fn((pid: number) => { + if (pid === 101) + return Promise.reject( + new Error("process exited during environment read"), + ); + if (pid !== 100) + return Promise.resolve({ REA_PROCESS_RUN_ID: "run-token" }); + launcherReads += 1; + return Promise.resolve( + launcherReads === 1 ? { REA_PROCESS_RUN_ID: "run-token" } : {}, + ); + }); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([row(100, 1, 100), row(102, 100), row(101, 100)]) + .mockResolvedValue([row(100, 1, 100), row(102, 100)]); + await expect( + observeOwnedProcessLineage(ownership, { + listProcesses, + environment, + signalGroup: vi.fn(), + }), + ).resolves.toMatchObject({ + status: "unavailable", + reason: "process lineage contains an unowned or PID-reused process", + }); + expect(launcherReads).toBe(2); + }); +}); + +describe("identityless lineage refresh rechecks an initial descendant", () => { + it("rejects a reused PID after an earlier descendant token matched", async () => { + let reads = 0; + const environment = vi.fn((pid: number) => { + if (pid === 101) + return Promise.reject( + new Error("process exited during environment read"), + ); + if (pid !== 102) + return Promise.resolve({ REA_PROCESS_RUN_ID: "run-token" }); + reads += 1; + return Promise.resolve( + reads === 1 ? { REA_PROCESS_RUN_ID: "run-token" } : {}, + ); + }); + const descendant = row(102, 100, 102); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([row(100, 1, 100), descendant, row(101, 100)]) + .mockResolvedValue([row(100, 1, 100), descendant]); + await expect( + observeOwnedProcessLineage(ownership, { + listProcesses, + environment, + signalGroup: vi.fn(), + }), + ).resolves.toMatchObject({ + status: "unavailable", + reason: "process lineage contains an unowned or PID-reused process", + }); + expect(reads).toBe(2); + }); +}); + +describe("identityless lineage refresh publishes one row per PID", () => { + it("keeps a single row when a descendant returns to its original group", async () => { + const environment = vi.fn((pid: number) => + pid === 101 || pid === 104 + ? Promise.reject(new Error("process exited during environment read")) + : Promise.resolve({ REA_PROCESS_RUN_ID: "run-token" }), + ); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([ + row(100, 1, 100), + row(101, 100), + row(102, 100, 100), + row(104, 100), + ]) + .mockResolvedValueOnce([ + row(100, 1, 100), + row(102, 100, 102), + row(104, 100), + ]) + .mockResolvedValue([row(100, 1, 100), row(102, 100, 100)]); + const result = await observeOwnedProcessLineage(ownership, { + listProcesses, + environment, + signalGroup: vi.fn(), + }); + expect(result.status).toBe("verified"); + if (result.status !== "verified") return; + expect(result.lineage.descendants).toEqual([ + { pid: 102, parentPid: 100, processGroupId: 100 }, + ]); + }); +}); + +describe("owned process-group cleanup discovery", () => { it("does not publish lineage when the launcher exits during validation", async () => { const launcher = { pid: 100, diff --git a/src/process/ProcessOwnership.ts b/src/process/ProcessOwnership.ts index c8a74a83af..5ca6d021aa 100644 --- a/src/process/ProcessOwnership.ts +++ b/src/process/ProcessOwnership.ts @@ -348,7 +348,23 @@ export const cleanupOwnedProcessGroup = async ( ); if ("cleaned" in plan) return plan; let signaled = false; - for (const processGroupId of plan.signalOrder) { + const pendingGroups = [...plan.signalOrder]; + const pending = new Set(pendingGroups); + const passes = new Map(); + while (pendingGroups.length > 0) { + const processGroupId = pendingGroups.shift(); + if (processGroupId === undefined) break; + pending.delete(processGroupId); + const seen = passes.get(processGroupId) ?? 0; + if (seen >= OWNED_GROUP_REVALIDATION_LIMIT) + return mergeUnverifiedObservations( + { + cleaned: false, + reason: `owned process group ${String(processGroupId)} reappeared after revalidation`, + }, + plan.unverified, + ); + passes.set(processGroupId, seen + 1); const revalidation = await revalidateOwnedProcessGroup( processGroupId, ownership, @@ -358,6 +374,19 @@ export const cleanupOwnedProcessGroup = async ( ); if ("cleaned" in revalidation) return mergeUnverifiedObservations(revalidation, plan.unverified); + for (const groupId of revalidation.escapedProcessGroupIds) { + if (groupId === processGroupId || pending.has(groupId)) continue; + if ((passes.get(groupId) ?? 0) >= OWNED_GROUP_REVALIDATION_LIMIT) + return mergeUnverifiedObservations( + { + cleaned: false, + reason: `owned process group ${String(groupId)} reappeared after revalidation`, + }, + plan.unverified, + ); + pending.add(groupId); + pendingGroups.push(groupId); + } if (revalidation.empty) continue; try { host.signalGroup(processGroupId, "SIGKILL"); @@ -497,6 +526,12 @@ interface OwnedCleanupPlan { readonly unverified: readonly OwnershipSweepUnverifiedProcess[]; } +/** Process table retained after launcher-exit settlement, plus groups proven by token. */ +interface SettledProcessTable { + readonly processes: readonly ProcessTableEntry[]; + readonly escapedProcessGroupIds: readonly number[]; +} + interface ProcessRelationSnapshot { readonly currentRelations: readonly ProcessTableEntry[]; readonly identities: ReadonlyMap; @@ -561,6 +596,169 @@ const readProcessRelationSnapshot = async ( return { currentRelations, identities, failures }; }; +const settledProcessTable = ( + processes: readonly ProcessTableEntry[], +): SettledProcessTable => ({ + processes, + escapedProcessGroupIds: [], +}); + +/** Confirmation tables taken after an unreadable exit-refresh row disappears. */ +const EXIT_REFRESH_CONFIRMATION_LIMIT = 8; + +/** Parent and group recorded for a PID in the pre-exit process table. */ +interface InitialProcessRelation { + readonly parentPid: number; + readonly processGroupId: number; +} + +/** + * Newcomers are token-checked. A PID from the pre-exit table is checked only + * when its parent or group changed, which is the identityless sign that the + * PID may have been reused. An unchanged pre-existing row stays unread. + */ +const isExitRefreshCandidate = ( + process: ProcessTableEntry, + ownership: OwnedProcessGroup, + initialRelations: ReadonlyMap, +): boolean => { + if (process.processGroupId === ownership.processGroupId) return false; + const initial = initialRelations.get(process.pid); + return ( + initial === undefined || + initial.parentPid !== process.parentPid || + initial.processGroupId !== process.processGroupId + ); +}; + +/** + * Times one group may be revalidated in a single cleanup. The second pass + * covers a group that reappears after it was already processed. Another + * appearance stays unresolved instead of reporting success. + */ +const OWNED_GROUP_REVALIDATION_LIMIT = 2; + +/** + * Token-check rows that exist only in the post-exit table. A prior PID, group, + * or ancestry link is handled by the caller; these rows have none of those. + * Foreign tokens are not owned. A matching token authorizes only a group whose + * leader also carries that token. An unreadable live row stays unresolved. + */ +const classifyUnlinkedExitRows = async ( + ownership: OwnedProcessGroup, + refreshed: readonly ProcessTableEntry[], + unlinked: readonly ProcessTableEntry[], + host: ProcessOwnershipHost, + initialRelations: ReadonlyMap, +): Promise< + | { readonly kind: "clear"; readonly processes: readonly ProcessTableEntry[] } + | { readonly kind: "uncertain"; readonly reason: string } + | { + readonly kind: "escaped"; + readonly processes: readonly ProcessTableEntry[]; + readonly groupIds: readonly number[]; + } +> => { + let current = refreshed; + let pending = unlinked; + let confirmations = 0; + const matched: ProcessTableEntry[] = []; + const inspected = new Set(); + // A readable token does not expire when the PID's parent and group return + // to the pre-exit values. Those PIDs are read again on the next table. + const tokenRead = new Set(); + while (pending.length > 0) { + const due = pending.filter((process) => !inspected.has(process.pid)); + if (due.length === 0) break; + for (const process of due) inspected.add(process.pid); + const unreadable: { readonly pid: number; readonly reason: string }[] = []; + for await (const { process, observation } of readProcessRunTokens( + host, + due, + )) { + if (observation.state === "readable") { + tokenRead.add(process.pid); + if (observation.runId === ownership.runId) matched.push(process); + continue; + } + unreadable.push({ pid: process.pid, reason: observation.reason }); + } + if (unreadable.length === 0) break; + let liveRows: readonly ProcessTableEntry[] | undefined; + try { + liveRows = liveProcesses(await host.listProcesses()); + } catch (cause: unknown) { + void cause; + liveRows = undefined; + } + const live = + liveRows === undefined + ? undefined + : new Set(liveRows.map(({ pid }) => pid)); + const stillLive = unreadable.filter( + (candidate) => live === undefined || live.has(candidate.pid), + ); + if (stillLive.length > 0) { + const only = stillLive.length === 1 ? stillLive[0] : undefined; + const pids = stillLive + .map(({ pid }) => pid) + .sort((left, right) => left - right); + return { + kind: "uncertain", + reason: + only === undefined + ? `process ownership could not be revalidated for PID(s): ${pids.join(", ")}` + : `process ownership could not be revalidated for PID ${String(only.pid)}: ${only.reason}`, + }; + } + if (liveRows === undefined) break; + confirmations += 1; + if (confirmations > EXIT_REFRESH_CONFIRMATION_LIMIT) + return { + kind: "uncertain", + reason: + "process ownership could not be revalidated: exit refresh confirmation limit exceeded", + }; + // The confirmation read is its own table. A PID classified on the previous + // table, including a readable foreign token, does not identify the process + // now using that PID. Read newcomers again, read a pre-exit PID again + // when its parent or group changed, and read a PID again when its token + // was already read even if that parent and group have returned. + current = liveRows; + inspected.clear(); + matched.length = 0; + pending = current.filter( + (process) => + tokenRead.has(process.pid) || + isExitRefreshCandidate(process, ownership, initialRelations), + ); + } + const livePids = new Set(current.map(({ pid }) => pid)); + const stillMatched = matched.filter((process) => livePids.has(process.pid)); + if (stillMatched.length === 0) return { kind: "clear", processes: current }; + const matchedPids = new Set(stillMatched.map(({ pid }) => pid)); + const groupIds = new Set(); + for (const process of stillMatched) { + const leader = current.find( + (candidate) => + candidate.pid === process.processGroupId && + candidate.processGroupId === process.processGroupId, + ); + if (leader === undefined || !matchedPids.has(leader.pid)) + return { + kind: "uncertain", + reason: + "descendant process-group leader identity could not be verified", + }; + groupIds.add(process.processGroupId); + } + return { + kind: "escaped", + processes: current, + groupIds: [...groupIds].sort((left, right) => left - right), + }; +}; + // A process-table read can race exit and retain the old state with an altered // command. Native terminal facts can settle that mismatch; exit-in-progress // must be confirmed first. Surviving members still require normal token checks. @@ -570,11 +768,11 @@ const settleLauncherExit = async ( host: ProcessOwnershipHost, exitSettlement: ProcessExitSettlementBudget, ): Promise< - | readonly ProcessTableEntry[] + | SettledProcessTable | Extract > => { const launcher = processes.find(({ pid }) => pid === ownership.leaderPid); - if (launcher === undefined) return processes; + if (launcher === undefined) return settledProcessTable(processes); let identityCheckedLauncher = launcher; if (host.processIdentities !== undefined) { const initialIdentities = await readNativeProcessIdentityMap( @@ -605,7 +803,9 @@ const settleLauncherExit = async ( reason: "owned launcher exit could not be confirmed", }; if (processIdentityHasExited(identity)) - return processes.filter(({ pid }) => pid !== launcher.pid); + return settledProcessTable( + processes.filter(({ pid }) => pid !== launcher.pid), + ); if ( identity?.state !== "readable" || identity.relation?.state !== "observed" @@ -621,7 +821,7 @@ const settleLauncherExit = async ( }; } const reason = launcherIdentityFailure(identityCheckedLauncher, ownership); - if (reason === null) return processes; + if (reason === null) return settledProcessTable(processes); // A native identity read already says that the launcher still exists and // supplies its current parent/group relation. Refreshing the process table // cannot repair a command or launch-relation mismatch for that generation. @@ -654,7 +854,36 @@ const settleLauncherExit = async ( ) ) return { cleaned: false, reason }; - return refreshed.processes; + const initialRelations = new Map( + processes.map((process) => [ + process.pid, + { + parentPid: process.parentPid, + processGroupId: process.processGroupId, + }, + ]), + ); + const unlinked = refreshed.processes.filter( + (process) => + isExitRefreshCandidate(process, ownership, initialRelations) && + !descendantPids.has(process.pid) && + !descendantGroupIds.has(process.processGroupId) && + !refreshedDescendantPids.has(process.pid), + ); + const classified = await classifyUnlinkedExitRows( + ownership, + refreshed.processes, + unlinked, + host, + initialRelations, + ); + if (classified.kind === "uncertain") + return { cleaned: false, reason: classified.reason }; + return { + processes: classified.processes, + escapedProcessGroupIds: + classified.kind === "escaped" ? classified.groupIds : [], + }; } return { cleaned: false, reason }; }; @@ -665,13 +894,15 @@ const createOwnedCleanupPlan = async ( host: ProcessOwnershipHost, exitSettlement: ProcessExitSettlementBudget, ): Promise => { - const processes = await settleLauncherExit( + const settled = await settleLauncherExit( ownership, initialProcesses, host, exitSettlement, ); - if ("cleaned" in processes) return processes; + if ("cleaned" in settled) return settled; + const processes = settled.processes; + const escapedProcessGroupIds = new Set(settled.escapedProcessGroupIds); const relationSnapshot = await readProcessRelationSnapshot( ownership, processes, @@ -714,6 +945,7 @@ const createOwnedCleanupPlan = async ( launcher === undefined && rootMembers.length === 0 && tokenOwned.owned.length === 0 && + escapedProcessGroupIds.size === 0 && !relationSnapshot.currentRelations.some(({ processGroupId }) => sampledProcessGroupIds.has(processGroupId), ) @@ -748,6 +980,8 @@ const createOwnedCleanupPlan = async ( processGroupIds.add(process.processGroupId); for (const processGroupId of sampledProcessGroupIds) processGroupIds.add(processGroupId); + for (const processGroupId of escapedProcessGroupIds) + processGroupIds.add(processGroupId); const signalableGroups = new Set(); const unresolved = [...tokenOwned.failures, ...relationSnapshot.failures]; const unverifiableSampledGroups = new Set(); @@ -757,9 +991,12 @@ const createOwnedCleanupPlan = async ( ({ pid, processGroupId: observedGroupId }) => pid === processGroupId && observedGroupId === processGroupId, ); + const provenGroup = + tokenOwnedGroupIds.has(processGroupId) || + escapedProcessGroupIds.has(processGroupId); if ( (groupLeader === undefined || !descendantPids.has(groupLeader.pid)) && - !tokenOwnedGroupIds.has(processGroupId) && + !provenGroup && !sampledProcessGroupIds.has(processGroupId) ) return { @@ -769,7 +1006,7 @@ const createOwnedCleanupPlan = async ( }; if ( (groupLeader === undefined || !descendantPids.has(groupLeader.pid)) && - !tokenOwnedGroupIds.has(processGroupId) + !provenGroup ) unverifiableSampledGroups.add(processGroupId); } @@ -864,16 +1101,25 @@ const revalidateOwnedProcessGroup = async ( host: ProcessOwnershipHost, expectedIdentities: ReadonlyMap, exitSettlement: ProcessExitSettlementBudget, -): Promise<{ readonly empty: boolean } | ProcessCleanupResult> => { +): Promise< + | { + readonly empty: boolean; + readonly escapedProcessGroupIds: readonly number[]; + } + | ProcessCleanupResult +> => { let members: readonly ProcessTableEntry[]; + let escapedProcessGroupIds: readonly number[] = []; try { - const processes = await settleLauncherExit( + const settled = await settleLauncherExit( ownership, liveProcesses(await host.listProcesses()), host, exitSettlement, ); - if ("cleaned" in processes) return processes; + if ("cleaned" in settled) return settled; + escapedProcessGroupIds = settled.escapedProcessGroupIds; + const processes = settled.processes; const relationSnapshot = await readProcessRelationSnapshot( ownership, processes, @@ -910,7 +1156,7 @@ const revalidateOwnedProcessGroup = async ( reason: "descendant process-group leader identity could not be revalidated", }; - return { empty: members.length === 0 }; + return { empty: members.length === 0, escapedProcessGroupIds }; }; const cleanupValidationFailure = ( diff --git a/src/process/ProcessOwnership.validation.test.ts b/src/process/ProcessOwnership.validation.test.ts index a2ee26b029..e7fa1ce7e9 100644 --- a/src/process/ProcessOwnership.validation.test.ts +++ b/src/process/ProcessOwnership.validation.test.ts @@ -24,6 +24,24 @@ const launcher: ProcessTableEntry = { state: "S", command: "fixture", }; +const nonSignalingObserver = ( + listProcesses: ProcessOwnershipHost["listProcesses"], +): ProcessOwnershipHost => ({ + listProcesses, + environment: () => Promise.resolve({}), + signalGroup: () => { + throw new Error("Observation must never signal processes"); + }, +}); +const expectUnownedLineage = (adapter: ProcessOwnershipHost) => + expect(observeOwnedProcessLineage(ownership, adapter)).resolves.toEqual({ + status: "unavailable", + observedAt: expect.any(String), + runId: "run-token", + launcherPid: 100, + processGroupId: 100, + reason: "process lineage contains an unowned or PID-reused process", + }); describe("owned process-group cleanup validation: ownership and lineage", () => { it("settles unreadable observations for Darwin processes exiting in the kernel", async () => { @@ -79,32 +97,14 @@ describe("owned process-group cleanup validation: ownership and lineage", () => }), signalGroup: vi.fn(), }; - await expect( - observeOwnedProcessLineage(ownership, adapter), - ).resolves.toEqual({ - status: "unavailable", - observedAt: expect.any(String), - runId: "run-token", - launcherPid: 100, - processGroupId: 100, - reason: "process lineage contains an unowned or PID-reused process", - }); + await expectUnownedLineage(adapter); }); it("does not publish lineage when any member fails ownership checks", async () => { const { adapter } = host({ 100: { REA_PROCESS_RUN_ID: "run-token" }, 101: { REA_PROCESS_RUN_ID: "other-run" }, }); - await expect( - observeOwnedProcessLineage(ownership, adapter), - ).resolves.toEqual({ - status: "unavailable", - observedAt: expect.any(String), - runId: "run-token", - launcherPid: 100, - processGroupId: 100, - reason: "process lineage contains an unowned or PID-reused process", - }); + await expectUnownedLineage(adapter); }); it("fails closed for stale metadata or an unrelated concurrent process", async () => { const { adapter, signalGroup } = host({ @@ -310,6 +310,577 @@ describe("owned process-group cleanup validation: refreshed descendant lineage", ); }); +describe("identityless exit refresh signals a token-owned replacement", () => { + it("token-checks a reparented grandchild that appears only after the launcher exits", async () => { + const child = { ...launcher, pid: 101, parentPid: 100 }; + const escaped = { + ...launcher, + pid: 102, + parentPid: 1, + processGroupId: 102, + command: "reparented-grandchild", + }; + const signalGroup = vi.fn(); + const environment = vi.fn((pid: number) => + Promise.resolve({ + REA_PROCESS_RUN_ID: pid === 102 ? ownership.runId : "run-token", + }), + ); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([{ ...launcher, command: "[MainThread]" }, child]) + .mockResolvedValue([escaped]); + await expect( + cleanupOwnedProcessGroup( + { ...ownership, expectedCommand: "fixture" }, + { listProcesses, environment, signalGroup }, + ), + ).resolves.toEqual({ cleaned: true, signaled: true }); + expect(environment).toHaveBeenCalledWith(102); + expect(signalGroup.mock.calls).toEqual([[102, "SIGKILL"]]); + }); + + it("does not token-read rows that were already outside the exited snapshot", async () => { + const unrelated = { + ...launcher, + pid: 50, + parentPid: 1, + processGroupId: 50, + command: "unrelated", + }; + const escaped = { + ...launcher, + pid: 102, + parentPid: 1, + processGroupId: 102, + command: "reparented-grandchild", + }; + const signalGroup = vi.fn(); + const environment = vi.fn((pid: number) => + Promise.resolve({ + REA_PROCESS_RUN_ID: pid === 102 ? ownership.runId : "other-run", + }), + ); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([ + { ...launcher, command: "[MainThread]" }, + { ...launcher, pid: 101, parentPid: 100 }, + unrelated, + ]) + .mockResolvedValue([unrelated, escaped]); + await expect( + cleanupOwnedProcessGroup( + { ...ownership, expectedCommand: "fixture" }, + { listProcesses, environment, signalGroup }, + ), + ).resolves.toEqual({ cleaned: true, signaled: true }); + expect(environment.mock.calls.map(([pid]) => pid)).not.toContain(50); + expect(signalGroup.mock.calls).toEqual([[102, "SIGKILL"]]); + }); +}); + +describe("identityless exit refresh leaves unrelated replacements unresolved", () => { + it.each([ + ["foreign", "other-run"], + ["absent", undefined], + ] as const)( + "does not signal a %s replacement row after the launcher exits", + async (_label, runId) => { + const signalGroup = vi.fn(); + const environment = vi.fn(() => + Promise.resolve( + runId === undefined ? {} : { REA_PROCESS_RUN_ID: runId }, + ), + ); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([ + { ...launcher, command: "[MainThread]" }, + { ...launcher, pid: 101, parentPid: 100 }, + ]) + .mockResolvedValue([ + { + ...launcher, + pid: 102, + parentPid: 1, + processGroupId: 102, + command: "foreign-replacement", + }, + ]); + await expect( + cleanupOwnedProcessGroup( + { ...ownership, expectedCommand: "fixture" }, + { listProcesses, environment, signalGroup }, + ), + ).resolves.toEqual({ cleaned: true, signaled: false }); + expect(environment).toHaveBeenCalledWith(102); + expect(signalGroup).not.toHaveBeenCalled(); + }, + ); + + it("keeps cleanup unresolved when a replacement row's token cannot be read", async () => { + const signalGroup = vi.fn(); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([ + { ...launcher, command: "[MainThread]" }, + { ...launcher, pid: 101, parentPid: 100 }, + ]) + .mockResolvedValue([ + { + ...launcher, + pid: 102, + parentPid: 1, + processGroupId: 102, + command: "unreadable-replacement", + }, + ]); + await expect( + cleanupOwnedProcessGroup( + { ...ownership, expectedCommand: "fixture" }, + { + listProcesses, + environment: () => + Promise.reject(new Error("environment unavailable")), + signalGroup, + }, + ), + ).resolves.toMatchObject({ + cleaned: false, + reason: + "process ownership could not be revalidated for PID 102: environment unavailable", + }); + expect(signalGroup).not.toHaveBeenCalled(); + }); + + it("does not signal a token-owned replacement whose group leader is absent", async () => { + const signalGroup = vi.fn(); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([ + { ...launcher, command: "[MainThread]" }, + { ...launcher, pid: 101, parentPid: 100 }, + ]) + .mockResolvedValue([ + { + ...launcher, + pid: 102, + parentPid: 1, + processGroupId: 999, + command: "member-without-leader", + }, + ]); + await expect( + cleanupOwnedProcessGroup( + { ...ownership, expectedCommand: "fixture" }, + { + listProcesses, + environment: () => + Promise.resolve({ REA_PROCESS_RUN_ID: ownership.runId }), + signalGroup, + }, + ), + ).resolves.toMatchObject({ + cleaned: false, + reason: "descendant process-group leader identity could not be verified", + }); + expect(signalGroup).not.toHaveBeenCalled(); + }); +}); + +describe("identityless exit refresh keeps later confirmation rows", () => { + it("token-checks a replacement born in the liveness table after an unreadable row exits", async () => { + const unreadable = { + ...launcher, + pid: 102, + parentPid: 1, + processGroupId: 102, + command: "unreadable-replacement", + }; + const escaped = { + ...launcher, + pid: 103, + parentPid: 1, + processGroupId: 103, + command: "owned-replacement", + }; + const signalGroup = vi.fn(); + const environment = vi.fn((pid: number) => + pid === 102 + ? Promise.reject(new Error("environment unavailable")) + : Promise.resolve({ REA_PROCESS_RUN_ID: ownership.runId }), + ); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([ + { ...launcher, command: "[MainThread]" }, + { ...launcher, pid: 101, parentPid: 100 }, + ]) + .mockResolvedValueOnce([unreadable]) + .mockResolvedValue([escaped]); + await expect( + cleanupOwnedProcessGroup( + { ...ownership, expectedCommand: "fixture" }, + { listProcesses, environment, signalGroup }, + ), + ).resolves.toEqual({ cleaned: true, signaled: true }); + expect(environment).toHaveBeenCalledWith(103); + expect(signalGroup.mock.calls).toEqual([[103, "SIGKILL"]]); + }); + + it("signals an escaped group discovered while revalidating the planned group", async () => { + const escaped = { + ...launcher, + pid: 102, + parentPid: 1, + processGroupId: 102, + command: "reparented-grandchild", + }; + const signalGroup = vi.fn(); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([launcher]) + .mockResolvedValueOnce([{ ...launcher, command: "[MainThread]" }]) + .mockResolvedValue([escaped]); + await expect( + cleanupOwnedProcessGroup( + { ...ownership, expectedCommand: "fixture" }, + { + listProcesses, + environment: () => + Promise.resolve({ REA_PROCESS_RUN_ID: ownership.runId }), + signalGroup, + }, + ), + ).resolves.toEqual({ cleaned: true, signaled: true }); + expect(signalGroup.mock.calls).toEqual([[102, "SIGKILL"]]); + }); + + it("stays unresolved when exit-refresh confirmations do not settle", async () => { + const signalGroup = vi.fn(); + let nextPid = 200; + const listProcesses = vi + .fn() + .mockResolvedValueOnce([ + { ...launcher, command: "[MainThread]" }, + { ...launcher, pid: 101, parentPid: 100 }, + ]) + .mockImplementation(() => { + const pid = nextPid; + nextPid += 1; + return Promise.resolve([ + { + ...launcher, + pid, + parentPid: 1, + processGroupId: pid, + command: "churn", + }, + ]); + }); + await expect( + cleanupOwnedProcessGroup( + { ...ownership, expectedCommand: "fixture" }, + { + listProcesses, + environment: () => + Promise.reject(new Error("environment unavailable")), + signalGroup, + }, + ), + ).resolves.toMatchObject({ + cleaned: false, + reason: + "process ownership could not be revalidated: exit refresh confirmation limit exceeded", + }); + expect(signalGroup).not.toHaveBeenCalled(); + }); +}); + +describe("identityless exit refresh follows a helper that changes groups", () => { + it("signals the group a token-owned member joins during confirmation", async () => { + const leader = { + ...launcher, + pid: 102, + parentPid: 1, + processGroupId: 102, + command: "owned-leader", + }; + const member = { + ...launcher, + pid: 103, + parentPid: 102, + processGroupId: 102, + command: "owned-member", + }; + const moved = { ...member, parentPid: 1, processGroupId: 103 }; + const unreadable = { + ...launcher, + pid: 104, + parentPid: 1, + processGroupId: 104, + command: "unreadable", + }; + const signalGroup = vi.fn(); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([ + { ...launcher, command: "[MainThread]" }, + { ...launcher, pid: 101, parentPid: 100 }, + ]) + .mockResolvedValueOnce([leader, member, unreadable]) + .mockResolvedValue([leader, moved]); + await expect( + cleanupOwnedProcessGroup( + { ...ownership, expectedCommand: "fixture" }, + { + listProcesses, + environment: (pid) => + pid === 104 + ? Promise.reject(new Error("environment unavailable")) + : Promise.resolve({ REA_PROCESS_RUN_ID: ownership.runId }), + signalGroup, + }, + ), + ).resolves.toEqual({ cleaned: true, signaled: true }); + expect(signalGroup.mock.calls).toEqual([ + [102, "SIGKILL"], + [103, "SIGKILL"], + ]); + }); +}); + +describe("identityless exit refresh rechecks a prior PID whose relation changes", () => { + it("token-checks a pre-existing PID after its parent changes", async () => { + const unrelated = { + ...launcher, + pid: 50, + parentPid: 999, + processGroupId: 50, + command: "unrelated", + }; + const stable = { + ...launcher, + pid: 60, + parentPid: 1, + processGroupId: 60, + command: "stable-outsider", + }; + const signalGroup = vi.fn(); + const environment = vi.fn((pid: number) => + Promise.resolve({ + REA_PROCESS_RUN_ID: pid === 50 ? ownership.runId : "other-run", + }), + ); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([ + { ...launcher, command: "[MainThread]" }, + { ...launcher, pid: 101, parentPid: 100 }, + unrelated, + stable, + ]) + .mockResolvedValue([{ ...unrelated, parentPid: 1 }, stable]); + await expect( + cleanupOwnedProcessGroup( + { ...ownership, expectedCommand: "fixture" }, + { listProcesses, environment, signalGroup }, + ), + ).resolves.toEqual({ cleaned: true, signaled: true }); + expect(environment).toHaveBeenCalledWith(50); + expect(environment.mock.calls.map(([pid]) => pid)).not.toContain(60); + expect(signalGroup.mock.calls).toEqual([[50, "SIGKILL"]]); + }); +}); + +describe("identityless exit refresh keeps a token-read helper", () => { + it("rechecks a matched helper after it returns to its original group", async () => { + const helper = { + ...launcher, + pid: 50, + parentPid: 1, + processGroupId: 50, + command: "helper", + }; + const leader = { + ...launcher, + pid: 60, + parentPid: 1, + processGroupId: 60, + command: "owned-leader", + }; + const stable = { + ...launcher, + pid: 70, + parentPid: 1, + processGroupId: 70, + command: "stable-outsider", + }; + const unreadable = { + ...launcher, + pid: 104, + parentPid: 1, + processGroupId: 104, + command: "unreadable", + }; + const signalGroup = vi.fn(); + const environment = vi.fn((pid: number) => + pid === 104 + ? Promise.reject(new Error("environment unavailable")) + : Promise.resolve({ REA_PROCESS_RUN_ID: ownership.runId }), + ); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([ + { ...launcher, command: "[MainThread]" }, + { ...launcher, pid: 101, parentPid: 100 }, + helper, + stable, + ]) + .mockResolvedValueOnce([ + leader, + { ...helper, parentPid: 60, processGroupId: 60 }, + unreadable, + stable, + ]) + .mockResolvedValue([leader, helper, stable]); + await expect( + cleanupOwnedProcessGroup( + { ...ownership, expectedCommand: "fixture" }, + { listProcesses, environment, signalGroup }, + ), + ).resolves.toEqual({ cleaned: true, signaled: true }); + expect(environment.mock.calls.map(([pid]) => pid)).not.toContain(70); + expect(signalGroup.mock.calls).toEqual([ + [50, "SIGKILL"], + [60, "SIGKILL"], + ]); + }); +}); + +describe("identityless exit refresh re-reads a reused foreign PID", () => { + it("token-checks a PID again after a foreign row exits into an owned replacement", async () => { + const reused = { + ...launcher, + pid: 102, + parentPid: 1, + processGroupId: 102, + command: "reused-pid", + }; + const unreadable = { + ...launcher, + pid: 104, + parentPid: 1, + processGroupId: 104, + command: "unreadable", + }; + let reads = 0; + const signalGroup = vi.fn(); + const environment = vi.fn((pid: number) => { + if (pid === 104) + return Promise.reject(new Error("environment unavailable")); + if (pid !== 102) + return Promise.resolve({ REA_PROCESS_RUN_ID: ownership.runId }); + reads += 1; + return Promise.resolve({ + REA_PROCESS_RUN_ID: reads === 1 ? "other-run" : ownership.runId, + }); + }); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([ + { ...launcher, command: "[MainThread]" }, + { ...launcher, pid: 101, parentPid: 100 }, + ]) + .mockResolvedValueOnce([reused, unreadable]) + .mockResolvedValue([reused]); + await expect( + cleanupOwnedProcessGroup( + { ...ownership, expectedCommand: "fixture" }, + { listProcesses, environment, signalGroup }, + ), + ).resolves.toEqual({ cleaned: true, signaled: true }); + expect(reads).toBeGreaterThan(1); + expect(signalGroup.mock.calls).toEqual([[102, "SIGKILL"]]); + }); +}); + +describe("identityless cleanup revalidates a group that reappears", () => { + const leader = (pid: number, command = "fixture"): ProcessTableEntry => ({ + ...launcher, + pid, + parentPid: pid === 100 ? 1 : 100, + processGroupId: pid, + command, + }); + + it("signals a token-owned group that reappears after it was already processed", async () => { + const signalGroup = vi.fn(); + const listProcesses = vi + .fn() + .mockResolvedValueOnce([leader(100), leader(101)]) + .mockResolvedValueOnce([leader(100)]) + .mockResolvedValueOnce([leader(100, "[MainThread]")]) + .mockResolvedValueOnce([leader(102)]) + .mockResolvedValueOnce([leader(100, "[MainThread]")]) + .mockResolvedValueOnce([leader(101)]) + .mockResolvedValue([leader(101)]); + await expect( + cleanupOwnedProcessGroup( + { ...ownership, expectedCommand: "fixture" }, + { + listProcesses, + environment: () => + Promise.resolve({ REA_PROCESS_RUN_ID: ownership.runId }), + signalGroup, + }, + ), + ).resolves.toEqual({ cleaned: true, signaled: true }); + expect(signalGroup.mock.calls).toEqual([ + [100, "SIGKILL"], + [101, "SIGKILL"], + ]); + }); + + it("stays unresolved when a processed group keeps reappearing", async () => { + const signalGroup = vi.fn(); + const tables = [ + [leader(100), leader(101)], + [leader(100)], + [leader(100, "[MainThread]")], + [leader(102)], + [leader(100, "[MainThread]")], + [leader(101)], + [leader(100, "[MainThread]")], + [leader(102)], + [leader(100, "[MainThread]")], + [leader(101)], + ]; + let reads = 0; + const listProcesses = vi.fn(() => { + const table = tables[reads] ?? [leader(101)]; + reads += 1; + return Promise.resolve(table); + }); + await expect( + cleanupOwnedProcessGroup( + { ...ownership, expectedCommand: "fixture" }, + { + listProcesses, + environment: () => + Promise.resolve({ REA_PROCESS_RUN_ID: ownership.runId }), + signalGroup, + }, + ), + ).resolves.toMatchObject({ + cleaned: false, + reason: "owned process group 101 reappeared after revalidation", + }); + expect(signalGroup.mock.calls).toEqual([[100, "SIGKILL"]]); + }); +}); + describe("owned process-group cleanup validation: launcher exit races", () => { it.each([101, 999])( "retains cleanup uncertainty for a detached descendant in group %s after launcher exit", @@ -453,17 +1024,11 @@ describe("owned process-group cleanup validation: exited members", () => { it("preserves cancellation during process-table observation instead of reporting unverifiable ownership", async () => { const controller = new AbortController(); const reason = new Error("cancelled process observation"); - const adapter: ProcessOwnershipHost = { - listProcesses: async (signal) => { - controller.abort(reason); - signal?.throwIfAborted(); - return []; - }, - environment: () => Promise.resolve({}), - signalGroup: () => { - throw new Error("Observation must never signal processes"); - }, - }; + const adapter = nonSignalingObserver(async (signal) => { + controller.abort(reason); + signal?.throwIfAborted(); + return []; + }); await expect( observeOwnedProcessGroups( ownership.runId, @@ -477,15 +1042,9 @@ describe("owned process-group cleanup validation: exited members", () => { const controller = new AbortController(); const reason = new Error("already cancelled observation"); controller.abort(reason); - const adapter: ProcessOwnershipHost = { - listProcesses: () => { - throw new Error("Cancelled observation must not inspect processes"); - }, - environment: () => Promise.resolve({}), - signalGroup: () => { - throw new Error("Observation must never signal processes"); - }, - }; + const adapter = nonSignalingObserver(() => { + throw new Error("Cancelled observation must not inspect processes"); + }); await expect( observeOwnedProcessGroups( ownership.runId, diff --git a/src/process/ProcessOwnershipObservation.ts b/src/process/ProcessOwnershipObservation.ts index fd287a031f..24608c2e3f 100644 --- a/src/process/ProcessOwnershipObservation.ts +++ b/src/process/ProcessOwnershipObservation.ts @@ -239,6 +239,27 @@ export const observeOwnedProcessLineage = async ( return unavailableLineage(ownership, identityFailure); const descendants = descendantsOf([launcher.pid], processes); const verifiedDescendants: ProcessTableEntry[] = []; + // Seed every initial row. A later refresh reads a row again when its parent, + // group, or launcher-tree membership changes. A row whose token was read is + // read again on every later refresh: parent and group are not a generation. + const initialTree = new Set(descendants.map(({ pid }) => pid)); + const initialRelations = new Map(); + const examinedRefresh = new Map(); + for (const process of processes) { + if (process.pid === launcher.pid) continue; + const relation: ExaminedLineageRefresh = { + parentPid: process.parentPid, + processGroupId: process.processGroupId, + inTree: initialTree.has(process.pid), + tokenChecked: false, + descendant: undefined, + }; + initialRelations.set(process.pid, relation); + examinedRefresh.set(process.pid, relation); + } + let refreshDescendants: readonly ProcessTableEntry[] = []; + let suppressedRefreshPids = new Set(); + let latestRefreshPids: ReadonlySet | undefined; for await (const { process: member, observation } of readProcessRunTokens( host, [launcher, ...descendants], @@ -250,19 +271,43 @@ export const observeOwnedProcessLineage = async ( "process lineage contains an unowned or PID-reused process", ); if (member.pid !== launcher.pid) verifiedDescendants.push(member); + const seeded = examinedRefresh.get(member.pid); + if (seeded !== undefined) + examinedRefresh.set(member.pid, { ...seeded, tokenChecked: true }); } else { - if (await processIsGone(host, member.pid)) { - if (member.pid === launcher.pid) - return unavailableLineage( - ownership, - "owned launcher exited during lineage validation", - ); - continue; + let refreshed: readonly ProcessTableEntry[]; + try { + refreshed = liveProcesses(await host.listProcesses()); + } catch (cause: unknown) { + void cause; + return unavailableLineage( + ownership, + `process ownership could not be revalidated for PID ${member.pid}: ${observation.reason}`, + ); } - return unavailableLineage( + if (refreshed.some((process) => process.pid === member.pid)) + return unavailableLineage( + ownership, + `process ownership could not be revalidated for PID ${member.pid}: ${observation.reason}`, + ); + if (member.pid === launcher.pid) + return unavailableLineage( + ownership, + "owned launcher exited during lineage validation", + ); + const classified = await classifyIdentitylessLineageRefresh( ownership, - `process ownership could not be revalidated for PID ${member.pid}: ${observation.reason}`, + launcher.pid, + refreshed, + host, + examinedRefresh, + initialRelations, ); + if (classified.status === "unavailable") + return unavailableLineage(ownership, classified.reason); + refreshDescendants = classified.descendants; + suppressedRefreshPids = new Set(classified.suppressedPids); + latestRefreshPids = new Set(refreshed.map(({ pid }) => pid)); } } return { @@ -273,7 +318,15 @@ export const observeOwnedProcessLineage = async ( launcherPid: launcher.pid, launcherParentPid: launcher.parentPid, processGroupId: launcher.processGroupId, - descendants: verifiedDescendants + descendants: [ + ...verifiedDescendants.filter( + ({ pid }) => + (latestRefreshPids === undefined || latestRefreshPids.has(pid)) && + !suppressedRefreshPids.has(pid) && + !refreshDescendants.some((descendant) => descendant.pid === pid), + ), + ...refreshDescendants, + ] .sort((left, right) => left.pid - right.pid) .map(({ pid, parentPid, processGroupId }) => ({ pid, @@ -431,6 +484,145 @@ const observeNativeOwnedProcessLineage = async ( }; }; +/** Relation already classified for one PID seen by an identityless refresh. */ +interface ExaminedLineageRefresh { + readonly parentPid: number; + readonly processGroupId: number; + /** True when this row was a launcher descendant at this classification. */ + readonly inTree: boolean; + /** True after this refresh has read the row's run token. */ + readonly tokenChecked: boolean; + readonly descendant: ProcessTableEntry | undefined; +} + +/** + * A liveness refresh is a process table, not a boolean. New rows still under + * the launcher join the published lineage after a matching token. A matching + * token outside that tree, or an unreadable live row, blocks a verified result. + * A readable foreign token outside the tree is not part of this run. A row is + * read again when its parent, process group, or launcher-tree membership + * changes. A row whose token was already read is read again on every later + * refresh, because those fields do not identify a process generation. + * Unchanged rows from the initial snapshot are not read again. The launcher + * is read on every refresh; a missing, unowned, or relation-mismatched + * launcher PID is unavailable. + */ +const classifyIdentitylessLineageRefresh = async ( + ownership: OwnedProcessGroup, + launcherPid: number, + refreshed: readonly ProcessTableEntry[], + host: ProcessOwnershipHost, + examined: Map, + initialRelations: ReadonlyMap, +): Promise< + | { + readonly status: "classified"; + readonly descendants: readonly ProcessTableEntry[]; + readonly suppressedPids: readonly number[]; + } + | { readonly status: "unavailable"; readonly reason: string } +> => { + const refreshedLauncher = refreshed.find( + (process) => process.pid === launcherPid, + ); + if (refreshedLauncher === undefined) + return { + status: "unavailable", + reason: "owned launcher exited during lineage validation", + }; + for await (const { observation } of readProcessRunTokens(host, [ + refreshedLauncher, + ])) { + if (observation.state !== "readable") + return { + status: "unavailable", + reason: `process ownership could not be revalidated for PID ${String(launcherPid)}: ${observation.reason}`, + }; + if (observation.runId !== ownership.runId) + return { + status: "unavailable", + reason: "process lineage contains an unowned or PID-reused process", + }; + } + const identityFailure = launcherIdentityFailure(refreshedLauncher, ownership); + if (identityFailure !== null) + return { status: "unavailable", reason: identityFailure }; + const inTree = new Set( + descendantsOf([launcherPid], refreshed).map(({ pid }) => pid), + ); + const due = refreshed.filter((process) => { + if (process.pid === launcherPid) return false; + const previous = examined.get(process.pid); + return ( + previous === undefined || + previous.tokenChecked || + previous.parentPid !== process.parentPid || + previous.processGroupId !== process.processGroupId || + previous.inTree !== inTree.has(process.pid) + ); + }); + if (due.length > 0) { + const included = new Map(); + const ownedOutside: number[] = []; + for await (const { process, observation } of readProcessRunTokens( + host, + due, + )) { + if (observation.state !== "readable") + return { + status: "unavailable", + reason: `process ownership could not be revalidated for PID ${String(process.pid)}: ${observation.reason}`, + }; + if (observation.runId !== ownership.runId) { + if (inTree.has(process.pid)) + return { + status: "unavailable", + reason: "process lineage contains an unowned or PID-reused process", + }; + continue; + } + if (inTree.has(process.pid)) included.set(process.pid, process); + else ownedOutside.push(process.pid); + } + if (ownedOutside.length > 0) + return { + status: "unavailable", + reason: `process lineage refresh contained owned PID(s) outside the observed descendants: ${ownedOutside + .sort((left, right) => left - right) + .join(", ")}`, + }; + for (const process of due) + examined.set(process.pid, { + parentPid: process.parentPid, + processGroupId: process.processGroupId, + inTree: inTree.has(process.pid), + tokenChecked: true, + descendant: included.get(process.pid), + }); + } + const livePids = new Set(refreshed.map(({ pid }) => pid)); + for (const pid of examined.keys()) + if (!livePids.has(pid)) examined.delete(pid); + const suppressedPids = [...examined.keys()].filter((pid) => { + const seed = initialRelations.get(pid); + const entry = examined.get(pid); + return ( + seed !== undefined && + entry !== undefined && + (seed.parentPid !== entry.parentPid || + seed.processGroupId !== entry.processGroupId || + seed.inTree !== entry.inTree) + ); + }); + return { + status: "classified", + descendants: [...examined.values()].flatMap((entry) => + entry.descendant === undefined ? [] : [entry.descendant], + ), + suppressedPids, + }; +}; + const processWithNativeRelation = ( process: ProcessTableEntry, identity: ProcessIdentityObservation, diff --git a/src/rust/RustBinaryImage.ts b/src/rust/RustBinaryImage.ts index ae077d21ca..01d0d9a66d 100644 --- a/src/rust/RustBinaryImage.ts +++ b/src/rust/RustBinaryImage.ts @@ -1,9 +1,9 @@ import { - GoBinaryFormatFailure, - GoBinaryResourceFailure, - type GoImageSection, -} from "../go/GoBinaryContainer.js"; -import { readGoBinaryContainer } from "../go/GoBinaryImage.js"; + BinaryFormatFailure, + BinaryResourceFailure, + type ImageSection, +} from "../native/metadata/BinaryContainer.js"; +import { readBinaryContainer } from "../native/metadata/readBinaryContainer.js"; import type { RustAuditableMetadata, RustBinary, @@ -48,13 +48,13 @@ export const readRustBinaryImage = (bytes: Buffer): RustBinaryImage => { const readContainer = (bytes: Buffer) => { try { - return readGoBinaryContainer(bytes); + return readBinaryContainer(bytes); } catch (cause) { - if (cause instanceof GoBinaryFormatFailure) + if (cause instanceof BinaryFormatFailure) throw new RustAuditableFormatFailure(cause.kind, cause.message, { cause, }); - if (cause instanceof GoBinaryResourceFailure) + if (cause instanceof BinaryResourceFailure) throw new RustAuditableResourceFailure( { resource: "memory", @@ -71,8 +71,8 @@ const readContainer = (bytes: Buffer) => { const dependencySection = ( format: RustBinaryImage["format"], - sections: readonly GoImageSection[], -): GoImageSection | null => { + sections: readonly ImageSection[], +): ImageSection | null => { const expected = format === "macho" ? "__DATA,.dep-v0" : ".dep-v0"; const matches = sections.filter((section) => section.name === expected); if (matches.length > 1) @@ -98,7 +98,7 @@ const dependencySection = ( const readDependency = ( bytes: Buffer, format: RustBinaryImage["format"], - section: GoImageSection, + section: ImageSection, ): RustAuditableMetadata => readAuditableSection( format === "macho" ? "__DATA,.dep-v0" : ".dep-v0", @@ -108,7 +108,7 @@ const readDependency = ( const rustcCommit = ( bytes: Buffer, - sections: readonly GoImageSection[], + sections: readonly ImageSection[], ): RustBinaryImage["rustc_commit"] => { const hashes: { hash: string; file_offset: number }[] = []; const seen = new Set(); @@ -120,7 +120,7 @@ const rustcCommit = ( }; const mergedReadOnlyRanges = ( - sections: readonly GoImageSection[], + sections: readonly ImageSection[], ): { readonly offset: number; readonly end: number }[] => { const ranges = sections .filter((section) => section.read_only && section.size > 0) diff --git a/src/server/createServer.ts b/src/server/createServer.ts index 03fef3ef53..4d01bb3b7a 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -777,6 +777,7 @@ const registerConfiguredAnalysisTools = ( toolLogger, recordEvidence, withAdmittedAnalysis, + evidenceById, ); registerWasmTools( server, diff --git a/src/server/registerRustTools.ts b/src/server/registerRustTools.ts index 4f674cd82f..fbf9099caf 100644 --- a/src/server/registerRustTools.ts +++ b/src/server/registerRustTools.ts @@ -1,4 +1,5 @@ import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; +import type { EvidenceLookup } from "../application/EvidenceInputResolver.js"; import type { RustBinaryService } from "../application/rust/RustBinaryService.js"; import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; import { toolContract } from "../contracts/toolContracts.js"; @@ -14,11 +15,13 @@ export const registerRustTools = ( logger: Logger, recordEvidence?: EvidenceWriter["recordEvidence"], withAdmittedAnalysis?: WithAdmittedAnalysis, + evidenceById?: EvidenceLookup, ): void => { const contract = toolContract("inspect_rust_binary"); const handler = createEvidenceToolHandler( server, - (input, options) => service.inspect(input, options), + (input, options) => + service.inspect(input, options, { recordEvidence, evidenceById }), logger, recordEvidence, withAdmittedAnalysis, diff --git a/src/signals/SigrokRecordingProvider.ts b/src/signals/SigrokRecordingProvider.ts index 6b8918d576..05417d1af9 100644 --- a/src/signals/SigrokRecordingProvider.ts +++ b/src/signals/SigrokRecordingProvider.ts @@ -7,6 +7,10 @@ import { DecoderDiscoveryBudgetError, parseDecoderDiscovery, } from "./parseDecoderDiscovery.js"; +import { + parseSigrokSelectorRejection, + type SigrokSelectorRejection, +} from "./sigrokSelectorRejection.js"; import { BOUNDED_PRODUCER_LIMITS as LIMITS, prlimitArgumentVector, @@ -166,7 +170,12 @@ export class SigrokRecordingProvider implements SignalRecordingPort { root = await this.createRuntime(); this.#resources.retain(root, undefined); const snapshotPath = join(root.path, "recording"); - const execute = async (args: readonly string[]) => { + const execute = async ( + args: readonly string[], + rejectedSelector?: ( + stderr: string, + ) => SigrokSelectorRejection | undefined, + ) => { capturedOutput = undefined; const output = await runOwnedCommand( { @@ -222,6 +231,16 @@ export class SigrokRecordingProvider implements SignalRecordingPort { ); phase = previousPhase; resourcePath = "path" in input ? input.path : command; + const rejection = + output.exitCode === 0 + ? undefined + : rejectedSelector?.(output.stderr.text); + if (rejection !== undefined) + throw new AnalysisInputError( + operation, + { capturedOutput, userMessage: rejection.message }, + [rejection], + ); if (output.exitCode !== 0) throw new ProviderAdapterError( SIGROK_PROVIDER_IDENTITY.id, @@ -311,7 +330,9 @@ export class SigrokRecordingProvider implements SignalRecordingPort { ? [] : [`--protocol-decoder-annotations=${input.annotations}`]), ]; - const output = await execute(args); + const output = await execute(args, (stderr) => + parseSigrokSelectorRejection(stderr, input), + ); phase = "integrity"; resourcePath = snapshotPath; if ( diff --git a/src/signals/sigrokSelectorRejection.ts b/src/signals/sigrokSelectorRejection.ts new file mode 100644 index 0000000000..d37bfa10f4 --- /dev/null +++ b/src/signals/sigrokSelectorRejection.ts @@ -0,0 +1,52 @@ +import type { AnalysisInputIssue } from "../domain/analysisErrorCore.js"; +import type { DecodeSignalRecordingInput } from "../domain/signals/signalRecording.js"; + +export type SigrokSelectorRejection = AnalysisInputIssue & { + readonly message: string; +}; + +const rejectedSelector = ( + field: "decoders" | "annotations", + producerMessage: string, +): SigrokSelectorRejection => ({ + path: [field], + reason: "invalid_value", + message: `sigrok-cli rejected the ${field} selector: ${producerMessage} Use inspect_signal_decoders to list installed decoder IDs, or pass a decoder ID to show its options and annotation classes.`, +}); + +/** + * Recognize only observed sigrok-cli 0.7.2 selector refusals whose rejected + * name equals a caller-supplied selector; other failures stay unclassified. + */ +export const parseSigrokSelectorRejection = ( + stderr: string, + input: Pick, +): SigrokSelectorRejection | undefined => { + const lines = new Set(stderr.split("\n")); + // sigrok-cli stacks decoders with "," and appends options with ":". + for (const entry of input.decoders.split(",")) { + const id = entry.split(":")[0] ?? ""; + const producerMessage = `Failed to load protocol decoder ${id}.`; + // A missing module named exactly after the decoder separates an unknown ID + // from an installed decoder whose own imports or code fail to load. + if ( + id !== "" && + lines.has(producerMessage) && + lines.has( + `srd: ModuleNotFoundError: Failed to load decoder ${id}: import by name failed: No module named '${id}'`, + ) + ) + return rejectedSelector("decoders", producerMessage); + } + // Annotation selections are "decoder=class:class,decoder". + for (const entry of input.annotations?.split(",") ?? []) { + const [decoder = "", classes, ...rest] = entry.split("="); + if (decoder === "" || classes === undefined || rest.length > 0) continue; + for (const annotation of classes.split(":")) { + const producerMessage = `Annotation '${annotation}' not found for protocol decoder '${decoder}'.`; + if (annotation !== "" && lines.has(producerMessage)) + return rejectedSelector("annotations", producerMessage); + } + } + return undefined; +}; diff --git a/src/sqlite/SqliteDatabaseFailures.ts b/src/sqlite/SqliteDatabaseFailures.ts index eb9a3863ed..9b2fc071d4 100644 --- a/src/sqlite/SqliteDatabaseFailures.ts +++ b/src/sqlite/SqliteDatabaseFailures.ts @@ -8,7 +8,9 @@ import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { AnalysisError, type AnalysisCapturedOutput, + type AnalysisErrorOptions, } from "../domain/analysisErrorBase.js"; +import type { JsonValue } from "../domain/jsonValue.js"; import { AnalysisAccessDeniedError, AnalysisArtifactChangedError, @@ -23,8 +25,11 @@ import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { OwnedCommandFailure } from "../process/OwnedCommand.js"; import type { ProviderProcessSnapshot } from "../process/ProviderProcess.js"; -import { SQLITE_PROVIDER_IDENTITY } from "../domain/sqlite/sqliteDatabase.js"; -import { SQLITE_DATABASE_LIMITS } from "./SqliteDatabaseLimits.js"; +import { + SQLITE_PROVIDER_IDENTITY, + DEFAULT_SQLITE_WORKER_TIMEOUT_MS, + MAX_SQLITE_WORKER_TIMEOUT_MS, +} from "../domain/sqlite/sqliteDatabase.js"; /** Preserve diagnostics independently of worker exit and private-root cleanup. */ export const capturedSqliteOutput = ( @@ -35,6 +40,38 @@ export const capturedSqliteOutput = ( truncated: snapshot.diagnosticTruncated === true, }); +/** Keep typed cleanup history and one canonical output owner, including after resolution. */ +export const createSqliteCleanupError = ( + resources: readonly string[], + diagnostics: Readonly>, + options: Omit, +): ProviderCleanupError => { + const capturedOutput = + options.capturedOutput ?? options.cleanupPrimary?.capturedOutput; + let previous = + options.cleanupPrimary === undefined + ? null + : projectAnalysisError(options.cleanupPrimary); + // History carries facts/recovery; the wrapper delivers output only once. + if (capturedOutput !== undefined && previous?.details !== undefined) { + const { captured_output: _capturedOutput, ...metadata } = previous.details; + previous = { ...previous, details: metadata }; + } + return new ProviderCleanupError( + SQLITE_PROVIDER_IDENTITY.id, + resources, + { + ...diagnostics, + previous_error: previous ?? diagnostics.previous_error ?? null, + }, + { + ...options, + operation: "inspect_sqlite_database", + ...(capturedOutput === undefined ? {} : { capturedOutput }), + }, + ); +}; + /** The file-set member a filesystem error names, if it is the database, its WAL or its journal. */ const busyMember = (cause: Error, path: string): string => { const failed = "path" in cause ? cause.path : undefined; @@ -50,6 +87,7 @@ export const sqliteDatabaseFailure = ( path: string, phase: string, signal?: AbortSignal, + workerTimeoutMs = DEFAULT_SQLITE_WORKER_TIMEOUT_MS, ): AnalysisError => { const operation = "inspect_sqlite_database"; if (cause instanceof AnalysisError) return cause; @@ -61,28 +99,36 @@ export const sqliteDatabaseFailure = ( : undefined; let previous: AnalysisError | undefined; if (physicalRead?.kind === "failed") - previous = sqliteDatabaseFailure(physicalRead.cause, path, phase); + previous = sqliteDatabaseFailure( + physicalRead.cause, + path, + phase, + undefined, + workerTimeoutMs, + ); else if (cause.reason === "cancelled") previous = new AnalysisCancelledError(operation, { cause: artifactFileCancellationCause(cause), }); else if (physicalRead?.kind === "completed") previous = undefined; else if (cause.cause !== undefined) - previous = sqliteDatabaseFailure(cause.cause, path, phase); + previous = sqliteDatabaseFailure( + cause.cause, + path, + phase, + undefined, + workerTimeoutMs, + ); else if (signal?.aborted) previous = new AnalysisCancelledError(operation, { cause: signal.reason, }); - const error = new ProviderCleanupError( - SQLITE_PROVIDER_IDENTITY.id, + const error = createSqliteCleanupError( cause.cleanup.resources, { reason: cause.cleanup.reason, - previous_error: - previous === undefined ? null : projectAnalysisError(previous), }, { - operation, cause, ...(previous === undefined ? {} : { cleanupPrimary: previous }), }, @@ -91,7 +137,13 @@ export const sqliteDatabaseFailure = ( return error; } if (cause.reason === "io" && cause.cause !== undefined) - return sqliteDatabaseFailure(cause.cause, path, phase, signal); + return sqliteDatabaseFailure( + cause.cause, + path, + phase, + signal, + workerTimeoutMs, + ); if (cause.reason === "integrity") return new AnalysisArtifactChangedError(operation, path, cause.message, { cause, @@ -116,7 +168,7 @@ export const sqliteDatabaseFailure = ( : cause, }); if (cause instanceof OwnedCommandFailure) { - const projected = ownedCommandFailure(cause); + const projected = ownedCommandFailure(cause, workerTimeoutMs); if (projected !== undefined) return projected; } if (phase === "artifact-read" && cause instanceof Error && "code" in cause) { @@ -173,6 +225,7 @@ export const sqliteDatabaseFailure = ( const ownedCommandFailure = ( cause: OwnedCommandFailure, + workerTimeoutMs: number, ): AnalysisError | undefined => { const operation = "inspect_sqlite_database"; const capturedOutput = @@ -181,21 +234,37 @@ const ownedCommandFailure = ( cause, ...(capturedOutput === undefined ? {} : { capturedOutput }), }; + // Preserve the same typed primary whether or not owned cleanup also fails. + const timeout = + cause.reason === "timeout" + ? new AnalysisTimeoutError(operation, workerTimeoutMs, { + ...options, + userMessage: `The SQLite worker reached its ${String(workerTimeoutMs)} ms server/provider deadline (worker_timeout_ms). This is not a whole-inspection or MCP client deadline.`, + remediationAction: `${ + workerTimeoutMs < MAX_SQLITE_WORKER_TIMEOUT_MS + ? `Increase worker_timeout_ms above ${String(workerTimeoutMs)} (integer milliseconds, maximum ${String(MAX_SQLITE_WORKER_TIMEOUT_MS)}), or use rea inspect-sqlite-database --worker-timeout-ms on the CLI.` + : `worker_timeout_ms is already at the maximum supported ${String(MAX_SQLITE_WORKER_TIMEOUT_MS)} ms; no higher worker deadline is supported. Report the blocker or narrow the question with caller approval.` + } This per-call setting needs no server restart. It covers worker startup, queries and reply writing, not snapshot acquisition or final reply reads/cleanup. Allow client headroom for the whole call; increasing only the client deadline does not change the worker budget. Keep the selected database/WAL and requested facts; narrow table/columns/schema_scope/row_limit only if the question permits it. Byte and heap safety limits remain fixed. Resolve incomplete cleanup before retrying; do not repeat the unchanged failed call.`, + }) + : undefined; if (cause.cleanupFailure !== null) - return new ProviderCleanupError( - SQLITE_PROVIDER_IDENTITY.id, + return createSqliteCleanupError( cause.resources, { reason: cause.cleanupFailure, - previous_error: { - failure_kind: cause.reason, - message: cause.message, - }, - ...(capturedOutput === undefined - ? {} - : { captured_output: { ...capturedOutput } }), + ...(timeout === undefined + ? { + previous_error: { + failure_kind: cause.reason, + message: cause.message, + }, + } + : {}), + }, + { + ...options, + ...(timeout === undefined ? {} : { cleanupPrimary: timeout }), }, - { operation, cause }, ); if (cause.reason === "cancelled") return new AnalysisCancelledError(operation, { @@ -203,12 +272,7 @@ const ownedCommandFailure = ( cause: cause.cancellation === undefined ? cause : cause.cancellation.reason, }); - if (cause.reason === "timeout") - return new AnalysisTimeoutError( - operation, - SQLITE_DATABASE_LIMITS.timeoutMs, - options, - ); + if (timeout !== undefined) return timeout; if (cause.reason === "output-limit") return new AnalysisOutputError(operation, cause.message, options); if ( diff --git a/src/sqlite/SqliteDatabaseInspection.ts b/src/sqlite/SqliteDatabaseInspection.ts index 9153991504..a41ce65592 100644 --- a/src/sqlite/SqliteDatabaseInspection.ts +++ b/src/sqlite/SqliteDatabaseInspection.ts @@ -549,7 +549,7 @@ export const inspectSqliteDatabaseSnapshot = ( "SQLite reports current schema and selected committed rows from its valid database/WAL state. Deleted records, uncommitted or invalid WAL tails, encrypted databases, forensic recovery, and full integrity verification are outside this profile.", "Stored CREATE statements preserve index/view/trigger definitions. Views, triggers, and virtual/shadow table rows are not executed; virtual-table column expansion is unsupported.", "Selected rows have unspecified order and are a bounded sample. Text display may replace invalid sequences; bytes_hex and encoding preserve SQLite's text byte representation, not original on-disk page offsets.", - "Resource boundaries: at most two concurrent inspections per provider; combined input 256 MiB per inspection, complete worker JSON 16 MiB, native SQLite value/record and SQL statement lengths 16 MiB, owned-worker JavaScript heap 256 MiB and supervised deadline 30 seconds. These are distinct from total resident memory; an overall native SQLite heap cap is not claimed. Failures return no partial success.", + "Resource boundaries: at most two concurrent inspections per provider; combined input 256 MiB per inspection, complete worker JSON 16 MiB, native SQLite value/record and SQL statement lengths 16 MiB, owned-worker JavaScript heap 256 MiB. The parent applies a separate supervised worker deadline, reported in worker_timeout_ms. These are distinct from total resident memory; an overall native SQLite heap cap is not claimed. Failures return no partial success.", ]; const budget = new OutputBudget(); const { schema, selectedTable } = readSchema(db, input, { diff --git a/src/sqlite/SqliteDatabaseLimits.ts b/src/sqlite/SqliteDatabaseLimits.ts index 408fc4bc30..50ab3978f2 100644 --- a/src/sqlite/SqliteDatabaseLimits.ts +++ b/src/sqlite/SqliteDatabaseLimits.ts @@ -3,7 +3,6 @@ export const SQLITE_DATABASE_LIMITS = { concurrentInspections: 2, inputBytes: 256 * 1024 * 1024, outputBytes: 16 * 1024 * 1024, - timeoutMs: 30_000, diagnosticBytes: 1024 * 1024, } as const; /** Internal rejection retains the failed boundary without returning partial success. */ diff --git a/src/sqlite/SqliteDatabaseProvider.ts b/src/sqlite/SqliteDatabaseProvider.ts index 8b0a1166da..50a8a4f68b 100644 --- a/src/sqlite/SqliteDatabaseProvider.ts +++ b/src/sqlite/SqliteDatabaseProvider.ts @@ -27,6 +27,7 @@ import { sqliteDatabaseSchema, type InspectSqliteDatabaseInput, SQLITE_PROVIDER_IDENTITY, + DEFAULT_SQLITE_WORKER_TIMEOUT_MS, sqliteInspectionParameters, } from "../domain/sqlite/sqliteDatabase.js"; import { isAbsolute } from "node:path"; @@ -39,7 +40,10 @@ import type { ProviderProcessStopResult, ProviderProcessSupervisor, } from "../process/ProviderProcess.js"; -import { sqliteDatabaseFailure } from "./SqliteDatabaseFailures.js"; +import { + createSqliteCleanupError, + sqliteDatabaseFailure, +} from "./SqliteDatabaseFailures.js"; import { captureSqliteDatabaseSnapshot } from "./SqliteDatabaseSnapshot.js"; import { SQLITE_DATABASE_LIMITS } from "./SqliteDatabaseLimits.js"; @@ -257,6 +261,8 @@ export class SqliteDatabaseProvider implements SqliteDatabasePort { let retainedOutput: AnalysisCapturedOutput | undefined; let phase = "configuration"; let result: Result; + const workerTimeoutMs = + input.worker_timeout_ms ?? DEFAULT_SQLITE_WORKER_TIMEOUT_MS; try { if (options?.signal?.aborted) throw new AnalysisCancelledError(operation, { @@ -283,6 +289,7 @@ export class SqliteDatabaseProvider implements SqliteDatabasePort { input, root: root.path, snapshotPath: snapshot.snapshotPath, + timeoutMs: workerTimeoutMs, environment: this.environment, resourceScope: this.#artifactResources, ...(this.launcher === undefined ? {} : { launcher: this.launcher }), @@ -299,6 +306,7 @@ export class SqliteDatabaseProvider implements SqliteDatabasePort { artifact: snapshot.artifact, wal: snapshot.wal, ...inspection, + worker_timeout_ms: workerTimeoutMs, }); if (!validated.success) throw new AnalysisOutputError( @@ -322,6 +330,7 @@ export class SqliteDatabaseProvider implements SqliteDatabasePort { input.path, phase, options?.signal, + workerTimeoutMs, ); retainedOutput ??= failure.capturedOutput; result = err(failure); @@ -495,17 +504,14 @@ const sqliteCleanupFailure = ( const partialObservation = result.ok ? completedSqliteEvidence(input, result.value) : result.error.partialObservation; - return new ProviderCleanupError( - SQLITE_PROVIDER_IDENTITY.id, + return createSqliteCleanupError( resources, { ...details, reason, - previous_error: result.ok ? null : projectAnalysisError(result.error), - ...(output === undefined ? {} : { captured_output: { ...output } }), }, { - operation, + ...(output === undefined ? {} : { capturedOutput: output }), ...(result.ok ? {} : { cleanupPrimary: result.error }), ...(cause === undefined ? {} : { cause }), ...(partialObservation === undefined ? {} : { partialObservation }), diff --git a/src/sqlite/SqliteDatabaseWorkerExecution.ts b/src/sqlite/SqliteDatabaseWorkerExecution.ts index d3fea33924..bdd82ad1ee 100644 --- a/src/sqlite/SqliteDatabaseWorkerExecution.ts +++ b/src/sqlite/SqliteDatabaseWorkerExecution.ts @@ -8,7 +8,6 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { z } from "zod"; -import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import type { AnalysisCapturedOutput } from "../domain/analysisErrorBase.js"; import { AnalysisCancelledError, @@ -36,6 +35,7 @@ const operation = "inspect_sqlite_database"; const { artifact: _artifactSchema, wal: _walSchema, + worker_timeout_ms: _workerTimeoutSchema, ...inspectionShape } = sqliteDatabaseSchema.shape; const inspectionSchema = z.strictObject(inspectionShape); @@ -70,6 +70,7 @@ const rejectedArgument = { const rejectedReply = ( reply: z.output & { ok: false }, + selectedPath: string, capturedOutput: AnalysisCapturedOutput, ) => { if (reply.reason === "unavailable") @@ -100,14 +101,21 @@ const rejectedReply = ( }, { capturedOutput, - ...fixedAllowanceOptions("The SQLite worker reply allowance", "report"), + retryable: false, + remediationAction: + "The SQLite value/record and worker JSON reply allowances are fixed at 16 MiB; worker_timeout_ms and client deadlines do not raise them. If it preserves the requested facts, select exact columns, reduce row_limit, or use schema_scope: inventory instead of database. This worker refusal leaves no completed Evidence to retrieve. Do not repeat the unchanged request or truncate required database bytes.", }, ); + const argument = rejectedArgument[reply.reason]; return new AnalysisInputError(operation, { capturedOutput }, [ { - path: [rejectedArgument[reply.reason]], + path: [argument], reason: "invalid_format", - message: reply.message, + // The worker sees only the private snapshot, so name the caller's selection here. + message: + argument === "path" + ? `Selected SQLite database is invalid (${reply.message}): ${selectedPath}` + : reply.message, }, ]); }; @@ -117,6 +125,7 @@ export const executeSqliteDatabaseWorker = async (request: { readonly input: InspectSqliteDatabaseInput; readonly root: string; readonly snapshotPath: string; + readonly timeoutMs: number; readonly environment: Readonly; readonly resourceScope: ArtifactResourceScope; readonly launcher?: SqliteWorkerLauncher; @@ -154,7 +163,7 @@ export const executeSqliteDatabaseWorker = async (request: { hostEnvironment: environment, }, { - timeoutMs: SQLITE_DATABASE_LIMITS.timeoutMs, + timeoutMs: request.timeoutMs, diagnosticBytes: SQLITE_DATABASE_LIMITS.diagnosticBytes, }, { @@ -210,6 +219,6 @@ export const executeSqliteDatabaseWorker = async (request: { bindArtifactFileCleanup(cause, error); throw error; } - if (!reply.ok) throw rejectedReply(reply, capturedOutput); + if (!reply.ok) throw rejectedReply(reply, request.input.path, capturedOutput); return reply.inspection; }; diff --git a/src/wasm/WabtArtifactProvider.test.ts b/src/wasm/WabtArtifactProvider.test.ts index b09d6ea2be..546ce0844e 100644 --- a/src/wasm/WabtArtifactProvider.test.ts +++ b/src/wasm/WabtArtifactProvider.test.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; -import { access, chmod, readFile, writeFile } from "node:fs/promises"; +import { access, chmod, readFile, realpath, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { expect, it, vi } from "vitest"; +import { expect, it, onTestFinished, vi } from "vitest"; import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; import { WabtArtifactProvider } from "./WabtArtifactProvider.js"; import { wasmArtifactSchema } from "../domain/wasm/wasmArtifact.js"; @@ -194,6 +194,20 @@ const assertModeOutcome = ( result: Awaited>, path: string, ): void => { + if (mode.startsWith("invalid")) + expect(result).toMatchObject({ + error: { + partialObservation: { + parameters: { command_timeout_ms: 42_000 }, + normalized_result: { + tool_profile: { + timeout_ms: 42_000, + output_budget_bytes: WABT_LIMITS.outputBytes, + }, + }, + }, + }, + }); if (mode === "success") expect(result).toMatchObject({ ok: true, @@ -228,6 +242,25 @@ const assertModeOutcome = ( }); }; +const assertValidationCleanupRecovery = async ( + provider: WabtArtifactProvider, + result: Awaited>, + cleanupAttempts: () => number, +): Promise => { + expect(result).toMatchObject({ error: { cleanupIncomplete: true } }); + await provider.close(); + expect(cleanupAttempts()).toBe(2); + if (result.ok) throw new Error("Expected validation failure"); + expect(result.error.cleanupIncomplete).toBe(false); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "invalid_request", + details: { + captured_output: { stderr: "module.wasm: invalid bytes\n" }, + cleanup_history: expect.any(Object), + }, + }); +}; + it("returns actionable unavailable without creating a workspace when tools are absent", async () => { const provider = new WabtArtifactProvider({}, undefined, () => { throw new Error("Must not allocate"); @@ -247,6 +280,75 @@ it("returns actionable unavailable without creating a workspace when tools are a }); }); +it.each([ + { tool: "wasm-validate", stdout: "1.0.41\n", stderr: "" }, + { tool: "wasm-objdump", stdout: "", stderr: "" }, + { tool: "wasm2wat", stdout: "1.0.42\n", stderr: "launcher warning\n" }, + { + tool: "wasm-objdump", + stdout: "", + stderr: "version launcher failed\n", + exitCode: 2, + }, +])( + "retains the exact rejected $tool version probe and configuration recovery", + async ({ tool, stdout, stderr, exitCode = 0 }) => { + const root = await realpath( + await createTestTempDirectory("rea-wabt-version-recovery-"), + ); + const suffix = process.platform === "win32" ? ".exe" : ""; + for (const name of ["wasm-validate", "wasm-objdump", "wasm2wat"]) + await writeFile(join(root, name + suffix), "owned executable seam", { + mode: 0o700, + }); + const commands: string[] = []; + const provider = new WabtArtifactProvider( + { REA_WABT_BIN_DIRECTORY: root }, + (input) => { + expect(input.arguments).toEqual(["--version"]); + commands.push(input.command); + const selected = input.command === join(root, tool + suffix); + return spawnOwnedProviderProcess({ + ...input, + command: process.execPath, + arguments: [ + "-e", + `process.stdout.write(${JSON.stringify(selected ? stdout : "1.0.42\n")}); process.stderr.write(${JSON.stringify(selected ? stderr : "")}); process.exit(${String(selected ? exitCode : 0)});`, + ], + expectedCommand: null, + }); + }, + ); + try { + const result = await provider.inspect({ + path: "/must-not-read-before-configuration.wasm", + glue_paths: [], + candidate_paths: [], + }); + if (result.ok) throw new Error("Expected WABT configuration rejection"); + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + code: "capability_unavailable", + retryable: false, + details: { + diagnostics: { + configuration_key: "REA_WABT_BIN_DIRECTORY", + executable_path: join(root, tool + suffix), + tool, + expected_version: "1.0.42", + }, + captured_output: { stdout, stderr, truncated: false }, + }, + }); + expect(projected.remediation.action).toContain("--version"); + expect(projected.remediation.action).toContain("restart"); + expect(commands.at(-1)).toBe(join(root, tool + suffix)); + } finally { + await provider.close(); + } + }, +); + it("retries a retained input descriptor before admitting the next WABT operation", async () => { const root = await createTestTempDirectory("rea-wabt-read-cleanup-"); const path = join(root, "selected.wasm"); @@ -468,6 +570,60 @@ const launchWabtOutcome = async ( return launched; }; +it("enforces a selected command deadline and completes a corrected retry after cleanup", async () => { + const fixture = await createWabtArtifactFixture(); + let first = true; + let runtimePath = ""; + const provider = new WabtArtifactProvider( + { REA_WABT_BIN_DIRECTORY: fixture.selectedRoot }, + async (input) => { + if (first) { + first = false; + return spawnOwnedProviderProcess({ + ...input, + command: process.execPath, + arguments: [ + "-e", + 'setTimeout(() => process.stdout.write("1.0.42\\n"), 250)', + ], + expectedCommand: null, + }); + } + return launchWabtOutcome(input, "success", { + ...fixture, + controller: new AbortController(), + }); + }, + async () => { + const root = await PrivateRuntimeRoot.create({ + prefix: "rea-wabt-budget-", + }); + runtimePath = root.path; + return root; + }, + ); + onTestFinished(() => provider.close()); + const input = { path: fixture.path, glue_paths: [], candidate_paths: [] }; + const timed = await provider.inspect({ ...input, command_timeout_ms: 20 }); + if (timed.ok) throw new Error("Selected 20 ms deadline was not enforced"); + expect(projectAnalysisError(timed.error)).toMatchObject({ + code: "provider_timeout", + details: { timeout_ms: 20 }, + remediation: { action: expect.stringContaining("command_timeout_ms") }, + }); + expect(timed.error.cleanupIncomplete).toBe(false); + await expect(access(runtimePath)).rejects.toMatchObject({ code: "ENOENT" }); + const corrected = await provider.inspect({ + ...input, + command_timeout_ms: 2_000, + }); + if (!corrected.ok) throw corrected.error; + const report = wasmArtifactSchema.parse(corrected.value.result); + expect(report.tool_profile.timeout_ms).toBe(2_000); + expect(report.limitations.join(" ")).not.toContain("30-second"); + await expect(readFile(fixture.path)).resolves.toEqual(fixture.bytes); +}); + const glueSourcesForOutcome = (mode: OutcomeMode) => { const literal = 'fetch("./selected.wasm");'; return mode === "success" @@ -567,6 +723,7 @@ const runOwnedArtifactOutcome = async ( path, glue_paths: glueInputs.map(({ path }) => path), candidate_paths: [], + ...(mode.startsWith("invalid") ? { command_timeout_ms: 42_000 } : {}), }, { signal: controller.signal }, ); @@ -592,21 +749,12 @@ const runOwnedArtifactOutcome = async ( cleanupAttempts: () => cleanupAttempts, cancellationReason: controller.signal.reason, }); - if (mode === "invalid-close-failure") { - expect(result).toMatchObject({ error: { cleanupIncomplete: true } }); - await provider.close(); - expect(cleanupAttempts).toBe(2); - if (!result.ok) { - expect(result.error.cleanupIncomplete).toBe(false); - expect(projectAnalysisError(result.error)).toMatchObject({ - code: "invalid_request", - details: { - captured_output: { stderr: "module.wasm: invalid bytes\n" }, - cleanup_history: expect.any(Object), - }, - }); - } - } + if (mode === "invalid-close-failure") + await assertValidationCleanupRecovery( + provider, + result, + () => cleanupAttempts, + ); if ( mode === "snapshot-close-failure" || mode === "snapshot-read-failure-late-abort" diff --git a/src/wasm/WabtArtifactProvider.ts b/src/wasm/WabtArtifactProvider.ts index 2436a387f0..95b30b77dc 100644 --- a/src/wasm/WabtArtifactProvider.ts +++ b/src/wasm/WabtArtifactProvider.ts @@ -33,14 +33,23 @@ import { createEvidence } from "../domain/evidence.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { wasmArtifactSchema, + DEFAULT_WASM_COMMAND_TIMEOUT_MS, type InspectWasmArtifactInput, } from "../domain/wasm/wasmArtifact.js"; import { err, ok, type Result } from "../domain/result.js"; import { runOwnedCommand } from "../process/OwnedCommand.js"; import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js"; import { WabtRuntime } from "./WabtRuntime.js"; -import { WABT_PROVIDER_IDENTITY, WABT_LIMITS } from "./WabtRelease.js"; -import { wabtFailure } from "./WabtFailures.js"; +import { + WABT_PROVIDER_IDENTITY, + WABT_LIMITS, + WABT_VERSION, +} from "./WabtRelease.js"; +import { + wabtFailure, + wabtConfigurationRecoveryAction, + type WabtVersionProbe, +} from "./WabtFailures.js"; import { parseWabtOutput } from "./WabtOutput.js"; import { parseWabtWat } from "./WabtWat.js"; import { associateWasmGlue } from "./WasmGlueAdapter.js"; @@ -102,6 +111,9 @@ export class WabtArtifactProvider implements WasmArtifactPort { let phase: "configuration" | "artifact" | "producer" = "configuration"; let result: Result; let capturedOutput: AnalysisCapturedOutput | undefined; + let versionProbe: WabtVersionProbe | undefined; + const commandTimeoutMs = + input.command_timeout_ms ?? DEFAULT_WASM_COMMAND_TIMEOUT_MS; try { if (options?.signal?.aborted) throw new AnalysisCancelledError(OPERATION); const directory = this.environment.REA_WABT_BIN_DIRECTORY ?? ""; @@ -179,7 +191,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { hostEnvironment: this.environment, }, { - timeoutMs: WABT_LIMITS.timeoutMs, + timeoutMs: commandTimeoutMs, diagnosticBytes: WABT_LIMITS.outputBytes, acceptNonZeroExit, }, @@ -206,16 +218,46 @@ export class WabtArtifactProvider implements WasmArtifactPort { return snapshot; }; for (const command of commands) { + versionProbe = { + configuration_key: "REA_WABT_BIN_DIRECTORY", + executable_path: command.path, + tool: command.tool, + expected_version: WABT_VERSION, + }; const version = await execute(command, ["--version"]); if ( - version.stdout.text.trim() !== "1.0.42" || + version.stdout.text.trim() !== WABT_VERSION || version.stderr.text !== "" - ) - throw new Error( - `Unsupported ${command.tool} version banner; expected 1.0.42.`, + ) { + const reason = `Unsupported ${command.tool} version banner; expected ${WABT_VERSION}.`; + throw new AnalysisCapabilityUnavailableError( + WABT_PROVIDER_IDENTITY.id, + OPERATION, + reason, + { + userMessage: reason, + diagnostics: versionProbe, + capturedOutput: { + stdout: version.stdout.text, + stderr: version.stderr.text, + truncated: version.diagnosticTruncated === true, + }, + remediationAction: wabtConfigurationRecoveryAction(), + }, ); + } command.version_banner = version.stdout.text; + versionProbe = undefined; } + // Success and validation failure report the same admitted tool/execution facts. + const toolProfile = { + source: "https://github.com/WebAssembly/wabt", + release: WABT_VERSION, + source_commit: "ff0ef7e0009402740c805a9744c09b05be063e48", + commands, + timeout_ms: commandTimeoutMs, + output_budget_bytes: WABT_LIMITS.outputBytes, + }; phase = "artifact"; const selected = await readStableArtifact( input.path, @@ -298,12 +340,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { result: { artifact, validation: "rejected-by-selected-profile", - tool_profile: { - source: "https://github.com/WebAssembly/wabt", - release: "1.0.42", - source_commit: "ff0ef7e0009402740c805a9744c09b05be063e48", - commands, - }, + tool_profile: toolProfile, exit_code: validation.exitCode ?? null, stdout: validation.stdout.text, stderr: validation.stderr.text, @@ -375,7 +412,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { "Imports and exports retain exact top-level WABT WAT forms. Names remain escaped producer text; unescaped objdump display names are not parsed as unambiguous identifiers.", "JavaScript references are AST-observed static literals only. Candidate matches are associations, not proven data flow, URL-to-file identity or runtime loading. Computed references and aliases are not resolved.", "No target execution or implicit fetch. WABT tools are trusted caller-supplied executables; SHA-256 identifies their bytes, not publisher authenticity. No OS memory sandbox is claimed.", - "Each subprocess has a 30-second deadline and 32 MiB combined output budget. Oversized output fails instead of producing truncated success.", + `Each WABT command has a ${String(commandTimeoutMs)} ms deadline and a fixed 32 MiB combined output budget. The deadline is not a whole-inspection budget. Oversized output fails instead of producing truncated success.`, ]; const report = wasmArtifactSchema.parse({ artifact, @@ -391,14 +428,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { sha256: digest(wat.stdout.text), representation: "decoded-wat", }, - tool_profile: { - source: "https://github.com/WebAssembly/wabt", - release: "1.0.42", - source_commit: "ff0ef7e0009402740c805a9744c09b05be063e48", - commands, - timeout_ms: WABT_LIMITS.timeoutMs, - output_budget_bytes: WABT_LIMITS.outputBytes, - }, + tool_profile: toolProfile, candidates, glue, runtime_execution: "not-performed", @@ -417,7 +447,10 @@ export class WabtArtifactProvider implements WasmArtifactPort { }), ); } catch (cause: unknown) { - const failure = wabtFailure(cause, phase, input.path); + const failure = wabtFailure(cause, phase, input.path, { + commandTimeoutMs, + ...(versionProbe === undefined ? {} : { versionProbe }), + }); runtime?.retainCommandFailure(cause, failure, this.#resources); result = err( failure instanceof AnalysisOutputError && diff --git a/src/wasm/WabtFailures.test.ts b/src/wasm/WabtFailures.test.ts index e7c8b09cd2..648d8fc999 100644 --- a/src/wasm/WabtFailures.test.ts +++ b/src/wasm/WabtFailures.test.ts @@ -3,6 +3,164 @@ import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { wabtFailure } from "./WabtFailures.js"; import { WABT_LIMITS } from "./WabtRelease.js"; +import { OwnedCommandFailure } from "../process/OwnedCommand.js"; + +it.each(["configuration", "producer"] as const)( + "retains timeout recovery under %s cleanup failure without making cleanup retryable", + (phase) => { + const cause = new OwnedCommandFailure( + "timeout", + "Command deadline elapsed.", + null, + "owned stop unconfirmed", + undefined, + ["pid:owned-fixture"], + ); + const failure = wabtFailure(cause, phase, "/module.wasm", { + commandTimeoutMs: 1234, + }); + expect(failure.cause).toBe(cause); + const projected = projectAnalysisError(failure); + expect(projected).toMatchObject({ + code: "cleanup_incomplete", + retryable: false, + details: { + resources: ["pid:owned-fixture"], + diagnostics: { + previous_error: { + code: "provider_timeout", + details: { timeout_ms: 1234 }, + recovery: { kind: "inspect_timeout_state", timeout_ms: 1234 }, + remediation: { + action: expect.stringContaining("command_timeout_ms above 1234"), + }, + }, + }, + }, + }); + expect(projected.remediation.action).not.toContain("Increase"); + }, +); + +it("retains timeout command output once, including when cleanup wraps its recovery", () => { + for (const cleanup of [null, "stop unconfirmed"]) { + const cause = new OwnedCommandFailure( + "timeout", + "deadline", + { + stdout: { text: "partial-owned-output", bytes: 20, observedBytes: 20 }, + stderr: { text: "", bytes: 0, observedBytes: 0 }, + exitCode: null, + signal: null, + }, + cleanup, + ); + const projected = projectAnalysisError( + wabtFailure(cause, "producer", "/module.wasm", { + commandTimeoutMs: 1234, + }), + ); + expect( + JSON.stringify(projected).split("partial-owned-output"), + ).toHaveLength(2); + expect(projected.code).toBe( + cleanup === null ? "provider_timeout" : "cleanup_incomplete", + ); + } +}); + +it.each([1234, 2_147_483_647])( + "reports the actual %i ms command budget without an unsupported higher timer", + (commandTimeoutMs) => { + const projected = projectAnalysisError( + wabtFailure( + new OwnedCommandFailure("timeout", "deadline"), + "producer", + "/module.wasm", + { commandTimeoutMs }, + ), + ); + expect(projected).toMatchObject({ + code: "provider_timeout", + details: { timeout_ms: commandTimeoutMs }, + recovery: { kind: "inspect_timeout_state", timeout_ms: commandTimeoutMs }, + }); + expect(projected.remediation.action).toContain("no server restart"); + expect(projected.remediation.action).toContain("cleanup"); + if (commandTimeoutMs === 2_147_483_647) { + expect(projected.remediation.action).toContain( + "no higher command deadline", + ); + expect(projected.remediation.action).not.toContain("Increase"); + } else + expect(projected.remediation.action).toContain("--command-timeout-ms"); + }, +); + +it.each([ + { reason: "timeout" as const, cleanup: null, code: "provider_timeout" }, + { reason: "cancelled" as const, cleanup: null, code: "cancelled" }, + { + reason: "process" as const, + cleanup: "owned stop failed", + code: "cleanup_incomplete", + }, +])( + "retains version-probe facts without changing $reason cleanup=$cleanup", + ({ reason, cleanup, code }) => { + const cause = new OwnedCommandFailure( + reason, + "version command failed", + null, + cleanup, + ); + const probe = { + configuration_key: "REA_WABT_BIN_DIRECTORY" as const, + executable_path: "/tools/wasm2wat", + tool: "wasm2wat", + expected_version: "1.0.42" as const, + }; + const failure = wabtFailure(cause, "configuration", "/selected.wasm", { + commandTimeoutMs: 1234, + versionProbe: probe, + }); + const original = wabtFailure(cause, "configuration", "/selected.wasm"); + expect(failure._tag).toBe(original._tag); + expect(failure.cleanupIncomplete).toBe(original.cleanupIncomplete); + expect(failure.cause).toBe(original.cause); + expect(projectAnalysisError(failure)).toMatchObject({ + code, + details: { diagnostics: probe }, + }); + }, +); + +it.each([ + { code: "EACCES", action: "read and execute access" }, + { + code: "ENOENT", + action: "contains wasm-validate, wasm-objdump and wasm2wat", + }, + { code: "ENOTDIR", action: "directory, not an individual executable" }, +])( + "explains $code configuration recovery without blaming the WASM input", + ({ code, action }) => { + const failure = wabtFailure( + Object.assign(new Error(`tool admission failed: ${code}`), { code }), + "configuration", + "/selected.wasm", + ); + const projected = projectAnalysisError(failure); + expect(projected).toMatchObject({ + code: "capability_unavailable", + retryable: false, + remediation: { action: expect.stringContaining(action) }, + }); + expect(projected.remediation.action).toContain("REA_WABT_BIN_DIRECTORY"); + expect(projected.remediation.action).toContain("restart"); + expect(projected.remediation.action).toContain("Do not repeat"); + }, +); it.each([false, true])( "preserves the file-size boundary with cleanup=%s when an input exceeds its fixed allowance", diff --git a/src/wasm/WabtFailures.ts b/src/wasm/WabtFailures.ts index 8dfb9b774e..9c9979f2f5 100644 --- a/src/wasm/WabtFailures.ts +++ b/src/wasm/WabtFailures.ts @@ -6,6 +6,7 @@ import { import { boundedUtf8Text } from "../process/boundedUtf8Text.js"; import type { ProviderProcessSnapshot } from "../process/ProviderProcess.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { AnalysisCancelledError, AnalysisArtifactChangedError, @@ -19,9 +20,45 @@ import { import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { OwnedCommandFailure } from "../process/OwnedCommand.js"; import { PrivateRuntimeRootUnavailableError } from "../process/PrivateRuntimeRootUnavailableError.js"; -import { WABT_LIMITS, WABT_PROVIDER_IDENTITY } from "./WabtRelease.js"; +import { + DEFAULT_WASM_COMMAND_TIMEOUT_MS, + MAX_WASM_COMMAND_TIMEOUT_MS, +} from "../domain/wasm/wasmArtifact.js"; +import { + WABT_LIMITS, + WABT_PROVIDER_IDENTITY, + WABT_VERSION, +} from "./WabtRelease.js"; const OUTPUT_LIMIT_EXCERPT_BYTES = 4096; const OUTPUT_LIMIT_REASON = `WABT output exceeded the ${String(WABT_LIMITS.outputBytes)}-byte output budget`; + +/** Facts owned before dispatch, even when a version command does not return successfully. */ +export type WabtVersionProbe = { + readonly configuration_key: "REA_WABT_BIN_DIRECTORY"; + readonly executable_path: string; + readonly tool: string; + readonly expected_version: typeof WABT_VERSION; +}; + +type WabtFailureContext = { + readonly commandTimeoutMs: number; + readonly versionProbe?: WabtVersionProbe; +}; + +/** Configuration is captured by the server; fixing files and changing its environment are distinct repairs. */ +export const wabtConfigurationRecoveryAction = (cause?: unknown): string => { + const code = + cause instanceof Error && "code" in cause ? cause.code : undefined; + const action = + code === "EACCES" || code === "EPERM" + ? "Check this REA process has read and execute access to the reported caller-supplied tools, or select an accessible trusted copy. " + : code === "ENOTDIR" + ? "Select the tools' containing directory, not an individual executable. " + : code === "ENOENT" + ? "The reported configured tool is missing. " + : ""; + return `${action}Set absolute REA_WABT_BIN_DIRECTORY to a directory that contains wasm-validate, wasm-objdump and wasm2wat from WABT ${WABT_VERSION}; each --version must report ${WABT_VERSION} on stdout with empty stderr. Correct the reported tool path, access or release before retrying, not the WASM input. Set this in the shell for the next CLI call, or in the MCP server launch environment and restart that server; a shell export does not update a running MCP server. Do not repeat the unchanged call or install tools implicitly.`; +}; /** Past the output budget, retained text is a prefix of an oversized stream; keep a bounded excerpt of each. */ const outputLimitExcerpt = (snapshot: ProviderProcessSnapshot) => ({ stdout: boundedUtf8Text(snapshot.stdout.text, OUTPUT_LIMIT_EXCERPT_BYTES) @@ -40,8 +77,9 @@ export const wabtFailure = ( cause: unknown, phase: "configuration" | "artifact" | "producer", path: string, + execution?: WabtFailureContext, ): AnalysisError => { - const failure = classifyWabtFailure(cause, phase, path); + const failure = classifyWabtFailure(cause, phase, path, execution); bindArtifactFileCleanup(cause, failure); return failure; }; @@ -50,7 +88,9 @@ const classifyWabtFailure = ( cause: unknown, phase: "configuration" | "artifact" | "producer", path: string, + execution: WabtFailureContext | undefined, ): AnalysisError => { + const versionProbe = execution?.versionProbe; if (cause instanceof AnalysisError) return cause; const artifactCleanup = cause instanceof ArtifactReaderFailure ? cause.cleanup : undefined; @@ -79,22 +119,46 @@ const classifyWabtFailure = ( stderr: cause.snapshot.stderr.text, truncated: cause.snapshot.diagnosticTruncated ?? false, }; - const options = - capturedOutput === undefined ? { cause } : { cause, capturedOutput }; + const options = { + cause, + ...(capturedOutput === undefined ? {} : { capturedOutput }), + ...(versionProbe === undefined ? {} : { diagnostics: versionProbe }), + }; + // Classify once before cleanup takes precedence, so the primary remains actionable. + const timeoutMs = + execution?.commandTimeoutMs ?? DEFAULT_WASM_COMMAND_TIMEOUT_MS; + const timeout = + cause.reason === "timeout" + ? new AnalysisTimeoutError("inspect_wasm_artifact", timeoutMs, { + ...options, + // The cleanup wrapper retains output once; its nested primary needs only recovery facts. + capturedOutput: + cause.cleanupFailure === null ? capturedOutput : undefined, + userMessage: `A WABT command reached its ${String(timeoutMs)} ms server/provider deadline (command_timeout_ms), not the whole-inspection or MCP client deadline.`, + remediationAction: `${ + timeoutMs < MAX_WASM_COMMAND_TIMEOUT_MS + ? `Increase command_timeout_ms above ${String(timeoutMs)} (integer milliseconds, maximum ${String(MAX_WASM_COMMAND_TIMEOUT_MS)}), or use rea inspect-wasm-artifact --command-timeout-ms on the CLI.` + : `command_timeout_ms is already at the maximum supported ${String(MAX_WASM_COMMAND_TIMEOUT_MS)} ms; no higher command deadline is supported. Report this blocker or select another suitable complete artifact with caller approval.` + } This is a per-call setting; no server restart is needed. Allow the MCP client deadline to cover all version/analysis commands, file checks, cleanup and delivery; increasing only that client deadline does not change this budget. Input/output safety limits stay fixed, and detail: summary reduces delivery size, not analysis work. Resolve any incomplete cleanup before retrying; do not repeat the unchanged failed call.`, + }) + : undefined; if (cause.cleanupFailure !== null) return new ProviderCleanupError( WABT_PROVIDER_IDENTITY.id, cause.resources, { + ...versionProbe, reason: cause.cleanupFailure, previous_error: - cause.reason === "output-limit" - ? { - failure_kind: "output-limit", - message: OUTPUT_LIMIT_REASON, - max_output_bytes: WABT_LIMITS.outputBytes, - } - : reason, + timeout !== undefined + ? projectAnalysisError(timeout) + : cause.reason === "output-limit" + ? { + failure_kind: "output-limit", + message: OUTPUT_LIMIT_REASON, + max_output_bytes: WABT_LIMITS.outputBytes, + } + : reason, ...(capturedOutput === undefined ? {} : { captured_output: capturedOutput }), @@ -103,12 +167,7 @@ const classifyWabtFailure = ( ); if (cause.reason === "cancelled") return new AnalysisCancelledError("inspect_wasm_artifact", options); - if (cause.reason === "timeout") - return new AnalysisTimeoutError( - "inspect_wasm_artifact", - WABT_LIMITS.timeoutMs, - options, - ); + if (timeout !== undefined) return timeout; if (phase !== "configuration" && cause.reason === "output-limit") return new AnalysisResourceConstraintError( "inspect_wasm_artifact", @@ -128,8 +187,8 @@ const classifyWabtFailure = ( reason, { ...options, - userMessage: - "Configured WABT could not start. Check executable/interpreter access and WABT 1.0.42 profile; inspect retained diagnostics.", + userMessage: `Configured WABT could not start. Check executable/interpreter access and WABT ${WABT_VERSION} profile; inspect retained diagnostics.`, + remediationAction: wabtConfigurationRecoveryAction(cause), }, ); return new AnalysisOutputError("inspect_wasm_artifact", reason, options); @@ -149,7 +208,8 @@ const classifyWabtFailure = ( { cause, ...cleanupOptions, - userMessage: `${reason} Supply WABT 1.0.42 via absolute REA_WABT_BIN_DIRECTORY; REA does not install tools.`, + userMessage: reason, + remediationAction: wabtConfigurationRecoveryAction(cause), }, ); if ( diff --git a/src/wasm/WabtInterface.ts b/src/wasm/WabtInterface.ts index 65e6fc0fa4..df47208995 100644 --- a/src/wasm/WabtInterface.ts +++ b/src/wasm/WabtInterface.ts @@ -7,7 +7,7 @@ import type { WasmInterfaceSignature, WasmInterfaceValueType, } from "../domain/wasm/wasmInterface.js"; -import { watString } from "./WabtString.js"; +import { watString, watStringEnd } from "./WabtString.js"; /** Split direct WAT children only; function instruction bodies are never materialized. */ const parts = (form: string): string[] => { @@ -40,12 +40,12 @@ const parts = (form: string): string[] => { } const start = i; if (form[i] === '"') { - i = watString(form, i + 1).end; + i = watStringEnd(form, i + 1); } else if (form[i] === "(") { let depth = 1; i++; while (i < form.length && depth > 0) { - if (form[i] === '"') i = watString(form, i + 1).end; + if (form[i] === '"') i = watStringEnd(form, i + 1); else { if (form[i] === "(") depth++; else if (form[i] === ")") depth--; diff --git a/src/wasm/WabtRelease.ts b/src/wasm/WabtRelease.ts index 5920d95235..2280106fa9 100644 --- a/src/wasm/WabtRelease.ts +++ b/src/wasm/WabtRelease.ts @@ -1,12 +1,12 @@ /** Exact upstream release whose producer output is admitted by this adapter. */ +export const WABT_VERSION = "1.0.42"; export const WABT_PROVIDER_IDENTITY = { id: "wabt-artifact", name: "REA WABT artifact adapter", - version: "wabt@1.0.42", + version: `wabt@${WABT_VERSION}`, } as const; export const WABT_LIMITS = { inputBytes: 8 * 1024 * 1024, toolBytes: 64 * 1024 * 1024, outputBytes: 32 * 1024 * 1024, - timeoutMs: 30_000, } as const; diff --git a/src/wasm/WabtString.test.ts b/src/wasm/WabtString.test.ts new file mode 100644 index 0000000000..bdbaf40afe --- /dev/null +++ b/src/wasm/WabtString.test.ts @@ -0,0 +1,56 @@ +import { expect, it } from "vitest"; +import { parseWabtWat } from "./WabtWat.js"; +import { watString, watStringEnd } from "./WabtString.js"; + +it("frames escaped quotes and trailing backslashes without consuming the next form", () => { + const source = String.raw`prefix "a\"b\\" (export "next" (func 0))`; + const end = source.indexOf(" (export"); + expect(watStringEnd(source, 8)).toBe(end); + expect(watString(source, 8)).toEqual({ + bytes: Buffer.from('a"b\\'), + end, + }); + expect(watStringEnd('""', 1)).toBe(2); + for (const count of [1, 2, 3, 4, 5]) { + for (const tail of ['"tail', ""]) { + const value = "\\".repeat(count) + tail; + const encoded = JSON.stringify(value); + const source = encoded + ' (export "next" (func 0))'; + expect(watStringEnd(source, 1)).toBe(encoded.length); + expect(watString(source, 1).bytes).toEqual(Buffer.from(value)); + } + } +}); + +it("rejects missing terminators, including a dangling escape or escaped last quote", () => { + for (const source of ['"', '"payload', '"payload\\', '"payload\\"']) { + expect(() => watStringEnd(source, 1)).toThrow("Incomplete WABT string"); + expect(() => watString(source, 1)).toThrow("Incomplete WABT string"); + expect(() => parseWabtWat(`(module (import ${source}`)).toThrow(); + } +}); + +it("preserves quoted WAT punctuation, nested comments, escaped names and unknown signatures", () => { + const result = parseWabtWat(String.raw`(module + (; outer " (; inner ;) ;) + (import "env" "a\"b\\(;);;\00" (func (param i32))) + (memory 1 (future-proposal "ignored\ff")) + (export "memory" (memory 0)) + (data (i32.const 0) "(; );) ;; \" \\"))`); + expect(result.interface.entries).toMatchObject([ + { + direction: "import", + name: 'a"b\\(;);;\0', + signature: { kind: "func", params: ["i32"] }, + }, + { + direction: "export", + name: "memory", + signature: null, + issue: expect.any(String), + }, + ]); + expect(() => parseWabtWat('(module (import "env" "\\q" (func)))')).toThrow( + "Unsupported WABT name escape", + ); +}); diff --git a/src/wasm/WabtString.ts b/src/wasm/WabtString.ts index 79615b193d..715e4cb565 100644 --- a/src/wasm/WabtString.ts +++ b/src/wasm/WabtString.ts @@ -1,13 +1,21 @@ -/** Decode a producer WAT string to frame objdump's raw display text. */ -export const watString = (form: string, start: number) => { - let end = start; - while (end < form.length && form[end] !== '"') { - if (form[end] === "\\") end++; - end++; +/** Locate a WAT string's exclusive end without decoding or copying its payload. */ +export const watStringEnd = (form: string, start: number): number => { + for ( + let end = form.indexOf('"', start); + end !== -1; + end = form.indexOf('"', end + 1) + ) { + let escapes = end; + while (escapes > start && form[escapes - 1] === "\\") escapes--; + if ((end - escapes) % 2 === 0) return end + 1; } - if (end === form.length) - throw new Error("Incomplete WABT custom-section name."); - const name = form.slice(start, end); + throw new Error("Incomplete WABT string."); +}; + +/** Decode a producer WAT string only where its bytes are interpreted. */ +export const watString = (form: string, start: number) => { + const end = watStringEnd(form, start); + const name = form.slice(start, end - 1); // Escapes shrink; literal UTF-8 has the same byte extent as its source. const bytes = Buffer.alloc(Buffer.byteLength(name, "utf8")); let length = 0; @@ -42,5 +50,5 @@ export const watString = (form: string, start: number) => { index += 2; } } - return { bytes: bytes.subarray(0, length), end: end + 1 }; + return { bytes: bytes.subarray(0, length), end }; }; diff --git a/src/wasm/WabtWat.ts b/src/wasm/WabtWat.ts index 01b9f16bc5..669c04c8e5 100644 --- a/src/wasm/WabtWat.ts +++ b/src/wasm/WabtWat.ts @@ -1,5 +1,5 @@ import { parseWabtInterface } from "./WabtInterface.js"; -import { watString } from "./WabtString.js"; +import { watString, watStringEnd } from "./WabtString.js"; /** Read only module-name metadata from WABT's preserved name-section payload. */ const moduleNames = (bytes: Uint8Array): string[] => { @@ -48,22 +48,24 @@ export const parseWabtWat = (text: string) => { const debugModuleNames: string[] = []; let depth = 0; let start = 0; - let quoted = false; let commentDepth = 0; let lineComment = false; let closed = false; + const boundaries = /[()";]/gu; for (let index = 0; index < text.length; index++) { + // Instructions and whitespace inside a form cannot change its framing. + if (depth > 0 && !lineComment && commentDepth === 0) { + boundaries.lastIndex = index; + const boundary = boundaries.exec(text); + if (boundary === null) break; + index = boundary.index; + } const char = text[index]; const next = text[index + 1]; if (lineComment) { if (char === "\n") lineComment = false; continue; } - if (quoted) { - if (char === "\\") index++; - else if (char === '"') quoted = false; - continue; - } if (commentDepth > 0) { if (char === "(" && next === ";") { commentDepth++; @@ -87,7 +89,7 @@ export const parseWabtWat = (text: string) => { if (char === '"') { if (depth === 0) throw new Error("Unexpected string outside WABT WAT module."); - quoted = true; + index = watStringEnd(text, index + 1) - 1; continue; } if (char === "(") { @@ -133,7 +135,7 @@ export const parseWabtWat = (text: string) => { } else if (depth === 0 && char !== undefined && !/\s/u.test(char)) throw new Error("Unexpected text outside WABT WAT module."); } - if (depth !== 0 || quoted || commentDepth !== 0) + if (depth !== 0 || commentDepth !== 0) throw new Error("Incomplete WABT WAT output."); return { imports, diff --git a/src/windows/WindowsHandleClose.test.ts b/src/windows/WindowsHandleClose.test.ts new file mode 100644 index 0000000000..b51c7ab1cf --- /dev/null +++ b/src/windows/WindowsHandleClose.test.ts @@ -0,0 +1,53 @@ +import { execFile, spawnSync } from "node:child_process"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +const run = promisify(execFile); +const compilerProbe = spawnSync("g++", ["--version"], { stdio: "ignore" }); +const compilerMissing = + compilerProbe.error instanceof Error && + "code" in compilerProbe.error && + compilerProbe.error.code === "ENOENT"; + +describe.skipIf(process.platform === "win32" && compilerMissing)( + "Windows native handle close gate", + () => { + it("retains a value unless close succeeds or the table no longer contains it", async () => { + // Injected close and liveness results. This does not call CloseHandle. + const include = fileURLToPath( + new URL("../../native/windows/src/", import.meta.url), + ); + const source = join(include, "handle_close_test.cc"); + const directory = await mkdtemp(join(tmpdir(), "rea-handle-close-")); + const binary = join(directory, "handle-close-test"); + try { + await run( + "g++", + [ + "-std=c++17", + "-Wall", + "-Wextra", + "-Werror", + `-I${include}`, + source, + "-o", + binary, + ], + { encoding: "utf8", timeout: 60_000 }, + ); + const result = await run(binary, [], { + encoding: "utf8", + timeout: 30_000, + }); + expect(result.stdout).toBe(""); + expect(result.stderr).toBe(""); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + }, +); diff --git a/src/windows/WindowsOwnedProcess.test.ts b/src/windows/WindowsOwnedProcess.test.ts index 0950b93939..865c740295 100644 --- a/src/windows/WindowsOwnedProcess.test.ts +++ b/src/windows/WindowsOwnedProcess.test.ts @@ -135,10 +135,13 @@ describe("Windows native release retry", () => { ); shouldFailClose = false; + const callsBeforeRetry = calls.length; const cleanup = child.cleanup(); - await vi.advanceTimersByTimeAsync(25); - expect(await cleanup).toMatchObject({ cleaned: true }); + expect(await cleanup).toMatchObject({ cleaned: true, signaled: false }); await closed; + expect( + calls.slice(callsBeforeRetry).map(({ operation }) => operation), + ).toEqual(["process_close"]); expect(child.stdout.writableEnded).toBe(true); expect(child.stderr.writableEnded).toBe(true); } finally { diff --git a/src/windows/WindowsOwnedProcess.ts b/src/windows/WindowsOwnedProcess.ts index 73ac7aefdd..b0193d0c7c 100644 --- a/src/windows/WindowsOwnedProcess.ts +++ b/src/windows/WindowsOwnedProcess.ts @@ -65,6 +65,10 @@ export class WindowsOwnedProcess extends EventEmitter { readonly #settled: Promise; #settle: (() => void) | undefined; #closed = false; + // Settlement was observed and stream teardown finished, but process_close + // has not released every native handle. Later cleanup retries that close + // alone; kill and poll still need the job or process handle. + #releasePending = false; #failure: string | undefined; #timer: NodeJS.Timeout | undefined; #blockedWake: NodeJS.Timeout | undefined; @@ -247,6 +251,17 @@ export class WindowsOwnedProcess extends EventEmitter { async #stop(): Promise { if (this.#closed) return { cleaned: true, signaled: false }; + if (this.#releasePending) { + try { + this.#release(); + return { cleaned: true, signaled: false }; + } catch (cause: unknown) { + return { + cleaned: false, + reason: cause instanceof Error ? cause.message : String(cause), + }; + } + } let timer: NodeJS.Timeout | undefined; try { const signaled = this.kill(); @@ -323,6 +338,17 @@ export class WindowsOwnedProcess extends EventEmitter { #poll(): void { this.#timer = undefined; if (this.#closed) return; + if (this.#releasePending) { + try { + this.#release(); + } catch (cause: unknown) { + const failure = + cause instanceof Error ? cause : new Error(String(cause)); + this.#failure = failure.message; + if (this.listenerCount("error") > 0) this.emit("error", failure); + } + return; + } try { const observed: unknown = this.#authority.call("process_poll", [ this.#handle, @@ -361,7 +387,8 @@ export class WindowsOwnedProcess extends EventEmitter { // Native lifecycle failures belong to the error/cleanup channels, never // the captured stderr observation produced by the child itself. if (this.listenerCount("error") > 0) this.emit("error", failure); - // Retain the original job so explicit cleanup can retry its observation. + // Poll failures keep the job for another observation. A process_close + // failure leaves #releasePending so cleanup retries that close alone. } } @@ -385,14 +412,18 @@ export class WindowsOwnedProcess extends EventEmitter { #release(): void { if (this.#closed) return; - if (this.#timer !== undefined) clearTimeout(this.#timer); - this.#clearBlockedWake(); - this.stdin?.destroy(); - if (!this.stdout.destroyed) this.stdout.end(); - if (!this.stderr.destroyed) this.stderr.end(); - // End local streams before releasing the native handle. If release fails, - // the retained job can be retried without leaving output streams open. + if (!this.#releasePending) { + if (this.#timer !== undefined) clearTimeout(this.#timer); + this.#timer = undefined; + this.#clearBlockedWake(); + this.stdin?.destroy(); + if (!this.stdout.destroyed) this.stdout.end(); + if (!this.stderr.destroyed) this.stderr.end(); + this.#releasePending = true; + } + // Streams are already ended. A failed close retries this call alone. this.#authority.call("process_close", [this.#handle]); + this.#releasePending = false; this.#closed = true; this.#settle?.(); this.emit("close", this.exitCode, null); diff --git a/src/windows/WindowsPrivateRuntime.test.ts b/src/windows/WindowsPrivateRuntime.test.ts index e32cf2c068..947d71d21e 100644 --- a/src/windows/WindowsPrivateRuntime.test.ts +++ b/src/windows/WindowsPrivateRuntime.test.ts @@ -151,6 +151,100 @@ describe("Windows private runtime cleanup retry", () => { }); }); +const admittedFile = { + requestedPath: "C:\\rea-private\\endpoint", + finalPath: "C:\\rea-private\\endpoint", + filesystem: "NTFS" as const, + volumeSerial: "0123456789abcdef", + fileId: "fedcba9876543210fedcba9876543210", + size: 2, + directory: false, + handle: { kind: "admitted-file" }, +}; + +describe("Windows private runtime admitted file close", () => { + it("retries the same file wrapper before the runtime tree walk", async () => { + const failure = nativeFailure(5); + const operations: string[] = []; + let fileCloses = 0; + nativeCall.mockImplementation((operation: string) => { + operations.push(operation); + if (operation === "runtime_create") return observation; + if (operation === "runtime_open") return admittedFile; + if (operation === "read") return Buffer.from("ok"); + if (operation === "close" && fileCloses++ === 0) throw failure; + return null; + }); + const runtime = WindowsPrivateRuntime.create("C:\\", "rea-"); + const readFailure: unknown = (() => { + try { + runtime.readFile("endpoint"); + return undefined; + } catch (cause: unknown) { + return cause; + } + })(); + expect(readFailure).toBe(failure); + await runtime.close(); + + expect(fileCloses).toBe(2); + expect(operations).toEqual([ + "runtime_create", + "runtime_open", + "read", + "close", + "close", + "runtime_close", + ]); + }); + + it("does not walk the runtime while the admitted file close is still failing", async () => { + const denied = nativeFailure(5); + const operations: string[] = []; + nativeCall.mockImplementation((operation: string) => { + operations.push(operation); + if (operation === "runtime_create") return observation; + if (operation === "runtime_open") return admittedFile; + if (operation === "read") return Buffer.from("ok"); + if (operation === "close") throw denied; + return null; + }); + const runtime = WindowsPrivateRuntime.create("C:\\", "rea-"); + const readFailure: unknown = (() => { + try { + runtime.readFile("endpoint"); + return undefined; + } catch (cause: unknown) { + return cause; + } + })(); + expect(readFailure).toBe(denied); + + await expect(runtime.close()).rejects.toBe(denied); + + expect(operations).toEqual([ + "runtime_create", + "runtime_open", + "read", + "close", + "close", + ]); + expect(windowsPrivateRuntime(observation.path)).toBe(runtime); + nativeCall.mockImplementation((operation: string) => { + operations.push(operation); + if (operation === "runtime_create") return observation; + if (operation === "runtime_open") return admittedFile; + if (operation === "read") return Buffer.from("ok"); + return null; + }); + await runtime.close(); + expect(operations.slice(-2)).toEqual(["close", "runtime_close"]); + expect(() => windowsPrivateRuntime(observation.path)).toThrow( + /No native private runtime/u, + ); + }); +}); + it.each([995, 5])( "classifies native snapshot code %i independently of a late abort", async (code) => { diff --git a/src/windows/WindowsPrivateRuntime.ts b/src/windows/WindowsPrivateRuntime.ts index ea28a5398f..a32488aaa8 100644 --- a/src/windows/WindowsPrivateRuntime.ts +++ b/src/windows/WindowsPrivateRuntime.ts @@ -63,6 +63,10 @@ export class WindowsPrivateRuntime { #pending: Promise | undefined; #closed = false; #closePromise: Promise | undefined; + // Admitted files whose close failed. Removal retries these wrappers before + // the native tree walk. Dropping them leaves ancestor handles until GC, and + // those handles can make runtime_close fail with a sharing violation. + #unreleasedFiles: object[] = []; private constructor( readonly observation: z.infer, private readonly authority: WindowsNativeAuthority, @@ -151,6 +155,8 @@ export class WindowsPrivateRuntime { relativePath, ]), ); + let text: string; + let closeFailure: unknown; try { const chunks: Buffer[] = []; for (let offset = 0; offset < file.size;) { @@ -170,10 +176,17 @@ export class WindowsPrivateRuntime { offset += value.length; chunks.push(value); } - return Buffer.concat(chunks).toString("utf8"); + text = Buffer.concat(chunks).toString("utf8"); } finally { - this.authority.call("close", [file.handle]); + try { + this.authority.call("close", [file.handle]); + } catch (cause: unknown) { + this.#unreleasedFiles.push(file.handle); + closeFailure = cause; + } } + if (closeFailure !== undefined) throw closeFailure; + return text; } /** Remove owned objects by handle, unlinking reparse entries without traversal. */ @@ -195,9 +208,25 @@ export class WindowsPrivateRuntime { runtimes.delete(windowsRuntimeIdentityKey(this.observation)); } + /** Close admitted files that survived readFile before walking the tree. */ + #releaseAdmittedFiles(): void { + const pending = this.#unreleasedFiles.splice(0); + let failure: unknown; + for (const handle of pending) { + try { + this.authority.call("close", [handle]); + } catch (cause: unknown) { + this.#unreleasedFiles.push(handle); + failure ??= cause; + } + } + if (failure !== undefined) throw failure; + } + async #closeRuntime(): Promise { for (let attempt = 1; ; attempt++) { try { + this.#releaseAdmittedFiles(); this.authority.call("runtime_close", [this.observation.handle]); return; } catch (cause: unknown) { diff --git a/tests/acceptance/applications/applicationWorkflowCli.test.ts b/tests/acceptance/applications/applicationWorkflowCli.test.ts index 7a96b08fca..3ecb567882 100644 --- a/tests/acceptance/applications/applicationWorkflowCli.test.ts +++ b/tests/acceptance/applications/applicationWorkflowCli.test.ts @@ -12,6 +12,11 @@ import { afterAll, describe, expect, it } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { writeElectronApiAliasApplication } from "../../fixtures/electronApiAliasApplication.js"; +import { + createCliCommandSession, + type CliCommandSession, +} from "../../support/cli/cliCommandSession.js"; +import { definedCliEnvironment } from "../../support/cli/cliFixture.js"; import { JAVASCRIPT_FEATURE_TRACE_EXAMPLE, @@ -34,6 +39,7 @@ import { javaScriptSemanticTraceResultSchema } from "../../../src/domain/javascr import type { JavaScriptSemanticQueryInput } from "../../../src/domain/javascript/javascriptSemanticQuerySchemas.js"; const execute = promisify(execFile); +let cliSession: CliCommandSession | undefined; const requireFixture = createRequire(import.meta.url); // Analysis requests are stateless; retain one compiled server and its SDK // validators while every scenario still owns a separate application directory. @@ -57,9 +63,13 @@ const connectedMcpClient = async (): Promise => { afterAll(async () => { try { - await mcpClient.close(); + try { + await mcpClient.close(); + } finally { + await mcpTransport.close(); + } } finally { - await mcpTransport.close(); + await cliSession?.close(); } }); const WEBPACK_FACTORY_BODY = ` @@ -1809,27 +1819,19 @@ const runCli = async ( arguments_: readonly string[], environment: Readonly> = {}, ): Promise => { - try { - const { stdout } = await execute( - process.execPath, - ["scripts/rea.mjs", ...arguments_], - { - cwd: process.cwd(), - env: { ...process.env, ...environment }, - maxBuffer: 16 * 1_024 * 1_024, - }, - ); - return JSON.parse(stdout); - } catch (cause: unknown) { - if ( - typeof cause === "object" && - cause !== null && - "stdout" in cause && - typeof cause.stdout === "string" - ) - return JSON.parse(cause.stdout); - throw cause; - } + cliSession ??= createCliCommandSession(); + const result = await cliSession.run({ + arguments: arguments_, + cwd: process.cwd(), + environment: definedCliEnvironment({ ...process.env, ...environment }), + }); + const stdout = result.stdout.trim(); + if (stdout.length > 0) return JSON.parse(stdout) as unknown; + throw new Error( + result.stderr.trim().length > 0 + ? result.stderr + : `Compiled CLI command failed with status ${String(result.exitCode)}`, + ); }; /** diff --git a/tests/acceptance/applications/binaryPlistDates.test.ts b/tests/acceptance/applications/binaryPlistDates.test.ts new file mode 100644 index 0000000000..bdc78b8f4d --- /dev/null +++ b/tests/acceptance/applications/binaryPlistDates.test.ts @@ -0,0 +1,271 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { build, buildBinary, parseBinary } from "plist"; +import { expect } from "vitest"; + +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { interfaceBuilderAnalysisSchema } from "../../../src/domain/apple/interfaceBuilderGraph.js"; +import { keyedArchiveResultSchema } from "../../../src/domain/apple/keyedArchive.js"; +import { thinMach } from "../../../src/domain/binaryTarget.fixture.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { embedSubmillisecondDateInWideIntegerPayload } from "../../fixtures/binaryPlistScalarAlias.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const MUTATION_SOURCE_SECONDS = 0.2; +const SUBMILLISECOND_SECONDS = 0.1005; + +const makeArchiveBytes = (): Buffer => + Buffer.from( + buildBinary({ + $archiver: "NSKeyedArchiver", + $version: 100000, + $top: { root: { UID: 1 } }, + $objects: [ + "$null", + { + $class: { UID: 2 }, + wide: Buffer.alloc(16), + canonicalDate: new Date("2001-01-01T00:00:00.100Z"), + preciseDate: new Date("2001-01-01T00:00:00.200Z"), + }, + { $classname: "UIView", $classes: ["UIView", "NSObject"] }, + ], + }), + ); + +const mutateDate = ( + source: Buffer, + sourceSeconds: number, + mutation: "fraction" | "marker", +): Buffer => { + const bytes = Buffer.from(source); + const date = Buffer.alloc(9); + date[0] = 0x33; + date.writeDoubleBE(sourceSeconds, 1); + const offset = bytes.indexOf(date); + expect(offset).toBeGreaterThanOrEqual(0); + expect(bytes.indexOf(date, offset + date.length)).toBe(-1); + if (mutation === "fraction") + bytes.writeDoubleBE(SUBMILLISECOND_SECONDS, offset + 1); + else bytes[offset] = 0x30; + return bytes; +}; + +const dateValues = (value: unknown): unknown[] => { + if (Array.isArray(value)) return value.flatMap(dateValues); + if (value === null || typeof value !== "object") return []; + if ("$plist_type" in value && value.$plist_type === "date") return [value]; + return Object.values(value).flatMap(dateValues); +}; + +const makeApp = async (root: string, bytes: Buffer): Promise => { + const app = join(root, "DateFixture.app"); + const contents = join(app, "Contents"); + const resources = join(contents, "Resources"); + await mkdir(resources, { recursive: true }); + await mkdir(join(contents, "MacOS")); + await writeFile( + join(contents, "MacOS", "Fixture"), + thinMach(0xfeedfacf, 0x0100000c), + ); + await writeFile( + join(contents, "Info.plist"), + build({ CFBundleExecutable: "Fixture" }), + ); + await writeFile(join(resources, "Main.nib"), bytes); + return app; +}; + +cliTest( + "preserves binary plist date precision through keyed archive and Interface Builder CLI and MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-binary-plist-dates-"); + const sourceBytes = makeArchiveBytes(); + const parsedFixture = parseBinary(sourceBytes) as { + $top: { root: { UID: number } }; + }; + expect(parsedFixture.$top.root).toEqual({ UID: 1 }); + const bytes = mutateDate(sourceBytes, MUTATION_SOURCE_SECONDS, "fraction"); + const malformedBytes = mutateDate( + sourceBytes, + MUTATION_SOURCE_SECONDS, + "marker", + ); + const archivePath = join(root, "archive.plist"); + const malformedArchivePath = join(root, "malformed.plist"); + await Promise.all([ + writeFile(archivePath, bytes), + writeFile(malformedArchivePath, malformedBytes), + ]); + + const archiveOutput = await cli.run({ + arguments: ["inspect-keyed-archive", archivePath, "--json"], + environment: { HOME: root, USERPROFILE: root, REA_LOG_LEVEL: "silent" }, + }); + expect(archiveOutput.exitCode, archiveOutput.stderr).toBe(0); + const archiveResult = keyedArchiveResultSchema.parse( + parseEvidence(archiveOutput.json).normalized_result, + ); + expect(dateValues(archiveResult.objects)).toEqual( + expect.arrayContaining([ + { $plist_type: "date", iso8601: "2001-01-01T00:00:00.100Z" }, + { $plist_type: "date", iso8601: null }, + ]), + ); + expect(archiveResult.limitations).toContain( + '1 date value(s) are reported as { "$plist_type": "date", "iso8601": null } because the decoder could not represent their timestamps.', + ); + + const malformedOutput = await cli.run({ + arguments: ["inspect-keyed-archive", malformedArchivePath, "--json"], + environment: { HOME: root, USERPROFILE: root, REA_LOG_LEVEL: "silent" }, + }); + expect(malformedOutput.exitCode).not.toBe(0); + + const malformedApp = await makeApp(root, bytes); + const malformedNib = join(malformedApp, "Contents/Resources/Broken.nib"); + await writeFile(malformedNib, malformedBytes); + const ibOutput = await cli.run({ + arguments: ["decode-interface-builder", malformedApp, "--json"], + environment: { REA_LOG_LEVEL: "silent", REA_ANALYSIS_PROVIDER: "auto" }, + }); + expect(ibOutput.exitCode, ibOutput.stderr).toBe(0); + const ibResult = interfaceBuilderAnalysisSchema.parse( + parseEvidence(ibOutput.json).normalized_result, + ); + const view = ibResult.graph.nodes.find( + ({ attributes }) => attributes.interface_builder_object_id === "1", + ); + expect(view).toBeDefined(); + expect(dateValues(view?.attributes)).toEqual( + expect.arrayContaining([ + { $plist_type: "date", iso8601: "2001-01-01T00:00:00.100Z" }, + { $plist_type: "date", iso8601: null }, + ]), + ); + expect( + ibResult.documents.map(({ relative_path }) => relative_path), + ).toEqual(["Contents/Resources/Main.nib"]); + expect(ibResult.graph.coverage).toContainEqual( + expect.objectContaining({ + facet: "archive_decode", + status: "partial", + reason: "one_or_more_archives_invalid", + }), + ); + expect(ibResult.limitations).toContain( + 'Contents/Resources/Main.nib: 1 date value(s) are reported as { "$plist_type": "date", "iso8601": null } because the decoder could not represent their timestamps.', + ); + expect(ibResult.limitations).toContainEqual( + expect.stringContaining( + "Contents/Resources/Broken.nib: binary plist date marker 0x30 is invalid", + ), + ); + + const { call } = await connectLocalToolsMcp({ + HOME: root, + USERPROFILE: root, + REA_LOG_LEVEL: "silent", + REA_ANALYSIS_PROVIDER: "auto", + }); + expect((await call("open_binary", { path: archivePath })).isError).not.toBe( + true, + ); + const archiveMcp = await call("inspect_keyed_archive", {}); + expect(archiveMcp.isError, JSON.stringify(archiveMcp)).not.toBe(true); + const validatedArchive = toolContract( + "inspect_keyed_archive", + ).outputSchema.parse(archiveMcp.structuredContent); + expect( + keyedArchiveResultSchema.parse( + parseEvidence(validatedArchive).normalized_result, + ), + ).toEqual(archiveResult); + + const openedBundle = await call("open_bundle_resources", { + path: malformedApp, + }); + expect(openedBundle.isError, JSON.stringify(openedBundle)).not.toBe(true); + toolContract("open_bundle_resources").outputSchema.parse( + openedBundle.structuredContent, + ); + const ibMcp = await call("decode_interface_builder", {}); + expect(ibMcp.isError, JSON.stringify(ibMcp)).not.toBe(true); + const validatedIb = toolContract( + "decode_interface_builder", + ).outputSchema.parse(ibMcp.structuredContent); + expect( + interfaceBuilderAnalysisSchema.parse( + parseEvidence(validatedIb).normalized_result, + ), + ).toEqual(ibResult); + + expect(await readFile(archivePath)).toEqual(bytes); + expect(await readFile(malformedArchivePath)).toEqual(malformedBytes); + expect( + await readFile(join(malformedApp, "Contents/Resources/Main.nib")), + ).toEqual(bytes); + expect(await readFile(malformedNib)).toEqual(malformedBytes); + expect(createHash("sha256").update(bytes).digest("hex")).toBe( + ibResult.documents[0]?.archive_sha256, + ); + }, + 60_000, +); + +cliTest( + "preserves aliased wide-integer and unknown-date values through keyed archive CLI and MCP", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-binary-plist-scalar-alias-", + ); + const { bytes, integerDecimal } = + embedSubmillisecondDateInWideIntegerPayload(makeArchiveBytes(), 0.2); + const archivePath = join(root, "aliased-archive.plist"); + await writeFile(archivePath, bytes); + const environment = { + HOME: root, + USERPROFILE: root, + REA_LOG_LEVEL: "silent", + REA_ANALYSIS_PROVIDER: "auto", + }; + + const cliOutput = await cli.run({ + arguments: ["inspect-keyed-archive", archivePath, "--json"], + environment, + }); + expect(cliOutput.exitCode, cliOutput.stderr).toBe(0); + const cliResult = keyedArchiveResultSchema.parse( + parseEvidence(cliOutput.json).normalized_result, + ); + const archivedValues = cliResult.objects.find(({ id }) => id === 1)?.value; + expect(archivedValues).toMatchObject({ + wide: { $plist_type: "integer", decimal: integerDecimal }, + canonicalDate: { + $plist_type: "date", + iso8601: "2001-01-01T00:00:00.100Z", + }, + preciseDate: { $plist_type: "date", iso8601: null }, + }); + + const { call } = await connectLocalToolsMcp(environment); + expect((await call("open_binary", { path: archivePath })).isError).not.toBe( + true, + ); + const response = await call("inspect_keyed_archive", {}); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const validated = toolContract("inspect_keyed_archive").outputSchema.parse( + response.structuredContent, + ); + const mcpResult = keyedArchiveResultSchema.parse( + parseEvidence(validated).normalized_result, + ); + expect(mcpResult).toEqual(cliResult); + expect(await readFile(archivePath)).toEqual(bytes); + }, + 60_000, +); diff --git a/tests/acceptance/applications/exportedFunctionBindingsCli.test.ts b/tests/acceptance/applications/exportedFunctionBindingsCli.test.ts index 97c470a4f1..9a6d0cd66c 100644 --- a/tests/acceptance/applications/exportedFunctionBindingsCli.test.ts +++ b/tests/acceptance/applications/exportedFunctionBindingsCli.test.ts @@ -7,19 +7,17 @@ import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, describe, expect, it } from "vitest"; import { z } from "zod"; import { closeVerifierResources } from "../../../scripts/lib/verifier-cleanup.mjs"; import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; import { javaScriptExportShapeComparisonResultSchema } from "../../../src/domain/javascript/javascriptExportShapeComparisonSchemas.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliCompileCacheEnvironment } from "../../support/cli/cliFixture.js"; import { - createTestWorkspace, - removeTestWorkspace, - type TestWorkspace, -} from "../../support/workspace/workspaceFixture.js"; + startCompiledCliSession, + type CompiledCliSession, +} from "../../support/cli/compiledCliSession.js"; const execute = promisify(execFile); const requireFixture = createRequire(import.meta.url); @@ -29,14 +27,11 @@ const comparisonEnvelope = z.object({ const applicationEnvelope = z.object({ normalized_result: javascriptApplicationAnalysisResultSchema, }); -let cliCacheWorkspace: TestWorkspace | undefined; +let cliSession: CompiledCliSession | undefined; -beforeAll(async () => { - cliCacheWorkspace = await createTestWorkspace("rea-export-bindings-cache-"); -}); - -// Source semantics use the real stdio workflow throughout the matrix. The -// missing-export and count-change cases also check compiled CLI adapter parity. +// Source semantics use one real stdio MCP session. Cases that also check the +// compiled CLI adapter share one warm process: it loads the production command +// runtime once and serves the same JSON commands as scripts/rea.mjs. let mcpConnection: | { readonly client: Client; @@ -61,8 +56,7 @@ afterAll(async () => { await mcpConnection?.transport.close(); }, async () => { - if (cliCacheWorkspace !== undefined) - await removeTestWorkspace(cliCacheWorkspace.root); + await cliSession?.close(); }, ]); }); @@ -1213,20 +1207,8 @@ const expectCountChange = ( }; const runCli = async (args: readonly string[]): Promise => { - if (cliCacheWorkspace === undefined) - throw new Error("CLI compile-cache workspace was not initialized"); - const { stdout } = await execute( - process.execPath, - [resolve("scripts/rea.mjs"), ...args], - { - maxBuffer: 16 * 1024 * 1024, - env: cliCompileCacheEnvironment( - { ...process.env, REA_LOG_LEVEL: "silent" }, - cliCacheWorkspace.path("node-compile"), - ), - }, - ); - return JSON.parse(stdout); + cliSession ??= startCompiledCliSession(); + return cliSession.run(args); }; const analyzeVersions = async ( diff --git a/tests/acceptance/applications/nativePlistBinaryScalars.test.ts b/tests/acceptance/applications/nativePlistBinaryScalars.test.ts new file mode 100644 index 0000000000..4a2231d082 --- /dev/null +++ b/tests/acceptance/applications/nativePlistBinaryScalars.test.ts @@ -0,0 +1,187 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { buildBinary } from "plist"; +import { describe, expect } from "vitest"; + +import { inspectPlistSchema } from "../../../src/domain/native/nativeInspection.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { thinMach } from "../../../src/domain/binaryTarget.fixture.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const patchReal = (bytes: Buffer, original: number, replacement: number) => { + const marker = Buffer.alloc(9); + marker[0] = 0x23; + marker.writeDoubleBE(original, 1); + const offset = bytes.indexOf(marker); + expect(offset).toBeGreaterThanOrEqual(0); + expect(bytes.indexOf(marker, offset + marker.length)).toBe(-1); + bytes.writeDoubleBE(replacement, offset + 1); +}; + +const makeBinaryPlist = () => { + const bytes = Buffer.from( + buildBinary({ + note: "a\u0001b", + payload: Buffer.from([0, 1, 2]), + fractional: new Date("2020-05-05T01:02:03.100Z"), + whole: new Date("2020-05-05T01:02:03.000Z"), + negative: 1.5, + positive: 0.25, + wide: 9007199254740992, + wide128: Buffer.alloc(16), + plusInfinity: Number.POSITIVE_INFINITY, + minusInfinity: Number.NEGATIVE_INFINITY, + nan: Number.NaN, + }), + ); + patchReal(bytes, 1.5, -0); + patchReal(bytes, 0.25, 0); + + const wideReal = Buffer.alloc(9); + wideReal[0] = 0x23; + wideReal.writeDoubleBE(9007199254740992, 1); + const wideOffset = bytes.indexOf(wideReal); + expect(wideOffset).toBeGreaterThanOrEqual(0); + expect(bytes.indexOf(wideReal, wideOffset + wideReal.length)).toBe(-1); + bytes[wideOffset] = 0x13; + bytes.writeBigInt64BE(9007199254740993n, wideOffset + 1); + + const trailer = bytes.length - 32; + const offsetWidth = bytes[trailer + 6] ?? 0; + const objectCount = Number(bytes.readBigUInt64BE(trailer + 8)); + const offsetTable = Number(bytes.readBigUInt64BE(trailer + 24)); + let wide128Offset: number | undefined; + for (let index = 0; index < objectCount; index += 1) { + let offset = 0; + const entry = offsetTable + index * offsetWidth; + for (let byte = 0; byte < offsetWidth; byte += 1) + offset = offset * 256 + (bytes[entry + byte] ?? 0); + if (bytes[offset] === 0x4f) { + expect(wide128Offset).toBeUndefined(); + wide128Offset = offset; + } + } + expect(wide128Offset).toBeDefined(); + if (wide128Offset === undefined) + throw new Error("plist fixture lost wide128"); + bytes[wide128Offset] = 0x14; + bytes.writeBigUInt64BE(1n, wide128Offset + 1); + bytes.writeBigUInt64BE(42n, wide128Offset + 9); + return bytes; +}; + +describe.skipIf(process.platform !== "darwin")( + "native binary plist scalar fidelity", + () => { + cliTest( + "preserves binary plist scalars through compiled CLI and stdio MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-plist-binary-scalars-"); + const target = join(root, "target"); + const binaryPath = join(root, "selected-binary.plist"); + const xmlPath = join(root, "selected-xml.plist"); + const binaryBytes = makeBinaryPlist(); + const xmlBytes = Buffer.from( + "separatorsa\u0085b\u2028c\u2029dpayloadAAEC", + ); + await Promise.all([ + writeFile(target, thinMach(0xfeedfacf, 0x0100000c)), + writeFile(binaryPath, binaryBytes), + writeFile(xmlPath, xmlBytes), + ]); + const environment = { + HOME: root, + USERPROFILE: root, + REA_LOG_LEVEL: "silent", + REA_ANALYSIS_PROVIDER: "auto", + }; + + const inspectCli = async (path: string) => { + const output = await cli.run({ + arguments: ["inspect-plist", path, "--json"], + environment, + }); + expect(output.exitCode, output.stderr).toBe(0); + return inspectPlistSchema.parse( + parseEvidence(output.json).normalized_result, + ); + }; + const binaryCliResult = await inspectCli(binaryPath); + const xmlCliResult = await inspectCli(xmlPath); + + const { call } = await connectLocalToolsMcp(environment); + const opened = await call("open_binary", { path: target }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const inspectMcp = async (path: string) => { + const response = await call("inspect_plist", { path }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + return inspectPlistSchema.parse( + parseEvidence(response.structuredContent).normalized_result, + ); + }; + const binaryMcpResult = await inspectMcp(binaryPath); + const xmlMcpResult = await inspectMcp(xmlPath); + + expect(binaryMcpResult).toEqual(binaryCliResult); + expect(binaryCliResult).toMatchObject({ + format: "binary", + source_path: binaryPath, + value: { + note: "a\u0001b", + payload: { $plist_type: "data", base64: "AAEC" }, + fractional: { + $plist_type: "date", + iso8601: "2020-05-05T01:02:03.100Z", + }, + whole: { + $plist_type: "date", + iso8601: "2020-05-05T01:02:03.000Z", + }, + negative: { $plist_type: "real", value: "-0" }, + positive: 0, + wide: { $plist_type: "integer", decimal: "9007199254740993" }, + wide128: { + $plist_type: "integer", + decimal: "18446744073709551658", + }, + plusInfinity: { $plist_type: "real", value: "+infinity" }, + minusInfinity: { $plist_type: "real", value: "-infinity" }, + nan: { $plist_type: "real", value: "nan" }, + }, + provenance: [{ tool: "file" }], + }); + expect(binaryCliResult.limitations).toEqual( + expect.arrayContaining([ + expect.stringContaining("1 negative-zero real value(s)"), + expect.stringContaining("3 non-finite real value(s)"), + expect.stringContaining("2 unambiguous integer value(s)"), + ]), + ); + expect(binaryCliResult.limitations).not.toContainEqual( + expect.stringContaining("XML conversion"), + ); + expect(xmlMcpResult).toEqual(xmlCliResult); + expect(xmlCliResult).toMatchObject({ + format: "xml", + source_path: xmlPath, + value: { + separators: "a\u0085b\u2028c\u2029d", + payload: { $plist_type: "data", base64: "AAEC" }, + }, + provenance: [ + { tool: "file" }, + { command: expect.arrayContaining(["json"]), exit: { code: 1 } }, + { command: expect.arrayContaining(["xml1"]), exit: { code: 0 } }, + ], + limitations: [expect.stringContaining("XML conversion")], + }); + expect(await readFile(binaryPath)).toEqual(binaryBytes); + expect(await readFile(xmlPath)).toEqual(xmlBytes); + }, + 60_000, + ); + }, +); diff --git a/tests/acceptance/applications/peResources.test.ts b/tests/acceptance/applications/peResources.test.ts index 5e2d7cc88d..73a49a144c 100644 --- a/tests/acceptance/applications/peResources.test.ts +++ b/tests/acceptance/applications/peResources.test.ts @@ -92,6 +92,59 @@ const expectPeInputFailuresAcrossAdapters = async (input: { const contract = toolContract("inspect_pe_resources"); +const expectPeWordAlignedRecordsAcrossAdapters = async (input: { + readonly cli: TestCli; + readonly client: Client; + readonly root: string; + readonly definition: Awaited< + ReturnType + >["tools"][number]; +}): Promise => { + const { cli, client, root, definition } = input; + for (const plus of [false, true]) { + for (const [index, identities] of [ + { type: 10, name: 1, language: "AB" }, + { type: "AB", name: 1, language: 1033 }, + ].entries()) { + const fixture = peResourceFixture( + plus, + [{ ...identities, payload: Buffer.from("opaque resource") }], + 65536, + 2, + ); + const path = join(root, `word-aligned-${plus}-${index}.exe`); + await writeFile(path, fixture.bytes); + const result = await cli.run({ + arguments: ["inspect-pe-resources", path, "--json"], + }); + expect(result.exitCode, result.stderr).toBe(0); + const evidence = contract.outputSchema.parse(result.json); + const report = peResourcesSchema.parse(evidence.normalized_result); + expect(report.coverage).toMatchObject({ + status: "complete", + resources: 1, + }); + expect(report.resources[0]?.data_entry_location).toEqual({ + offset: fixture.dataOffsets[0], + bytes: 16, + }); + expect(report.resources[0]?.payload.sha256).toBe( + createHash("sha256").update("opaque resource").digest("hex"), + ); + const response = await client.callTool( + { name: definition.name, arguments: { path } }, + { toolDefinition: definition }, + ); + expect(response.isError).not.toBe(true); + expect( + contract.outputSchema.parse(response.structuredContent) + .normalized_result, + ).toEqual(evidence.normalized_result); + expect(await readFile(path)).toEqual(fixture.bytes); + } + } +}; + const expectPeMappingOwnershipAcrossAdapters = async ( cli: TestCli, client: Client, @@ -284,6 +337,11 @@ cliTest( try { await client.connect(transport); const inventory = await client.listTools(); + const definition = inventory.tools.find( + ({ name }) => name === contract.name, + ); + if (definition === undefined) + throw new Error("Missing PE resource tool contract"); expect(inventory.tools).toContainEqual( expect.objectContaining({ name: contract.name, @@ -301,6 +359,12 @@ cliTest( "inspect_pe_resources", ); await expectPeInventoriesAcrossAdapters(cli, client, root); + await expectPeWordAlignedRecordsAcrossAdapters({ + cli, + client, + root, + definition, + }); await expectPeMappingOwnershipAcrossAdapters(cli, client, root); await expectPePaddingFailureAcrossAdapters({ cli, diff --git a/tests/acceptance/artifactSymlinkLoopCli.test.ts b/tests/acceptance/artifactSymlinkLoopCli.test.ts new file mode 100644 index 0000000000..d44e122f19 --- /dev/null +++ b/tests/acceptance/artifactSymlinkLoopCli.test.ts @@ -0,0 +1,35 @@ +import { readlink, symlink } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +cliTest.skipIf(process.platform === "win32").for([ + { command: "inspect-artifact", link: "loop.db", selected: "loop.db" }, + { command: "extract-artifact", link: "tree", selected: join("tree", "file") }, +])( + "$command reports a stable symbolic link loop as a path resolution failure", + async ({ command, link, selected }, { cli }) => { + const root = await createTestTempDirectory("rea-artifact-symlink-loop-"); + await symlink(link, join(root, link)); + const path = join(root, selected); + const result = await cli.run({ + arguments: [command, path, "--json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(result.exitCode).toBe(1); + const reason = expect.stringMatching( + /^target path could not be resolved: too many symbolic links or a symbolic link loop: ELOOP/u, + ); + expect(result.json).toMatchObject({ + code: "target_unavailable", + category: "unavailable", + retryable: false, + message: expect.stringContaining("symbolic link loop"), + details: { path, reason }, + }); + expect(result.stdout).not.toContain("changed while being opened"); + expect(await readlink(join(root, link))).toBe(link); + }, +); diff --git a/tests/acceptance/goBinaryCli.test.ts b/tests/acceptance/goBinaryCli.test.ts index a17c09bab7..a5dfdf0bb8 100644 --- a/tests/acceptance/goBinaryCli.test.ts +++ b/tests/acceptance/goBinaryCli.test.ts @@ -16,7 +16,7 @@ import { GO_MODULE_TEXT, } from "../fixtures/go/image.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; -import { cliTest } from "../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../support/cli/cliFixture.js"; import { parseEvidence } from "../../src/domain/evidence.js"; import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; import { parseMcpToolError } from "../fixtures/mcpToolError.js"; @@ -28,12 +28,12 @@ cliTest( const path = join(root, "application.elf"); const output = join(root, "source.json"); await writeFile(path, createGoBinaryFixture().bytes); - const complete = await cli.run({ + const complete = await runFileCli(cli, { arguments: ["inspect-go-binary", path, "--json"], }); expect(complete.exitCode, complete.stderr).toBe(0); const full = evidenceResultOf(goBinarySchema).parse(complete.json); - const summary = await cli.run({ + const summary = await runFileCli(cli, { arguments: [ "inspect-go-binary", path, @@ -62,7 +62,7 @@ cliTest( .map((record) => parseEvidence(record)) .find((record) => record.evidence_id === full.evidence_id), ).toEqual(full); - const refused = await cli.run({ + const refused = await runFileCli(cli, { arguments: [ "inspect-go-binary", join(root, "missing"), @@ -86,7 +86,7 @@ cliTest( const root = await createTestTempDirectory("rea-go-overlap-"); const path = join(root, "overlapping.elf"); await writeFile(path, createGoOverlappingElfFixture()); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-go-binary", path, "--json"], }); expect(result.exitCode).toBe(1); @@ -120,7 +120,7 @@ cliTest( const fixture = createGoSplitElfFixture({ sectionless }); const path = join(root, sectionless ? "sectionless.elf" : "original.elf"); await writeFile(path, fixture.bytes); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-go-binary", path, "--json"], }); expect(result.exitCode, result.stderr).toBe(0); @@ -156,7 +156,7 @@ for (const bits of [32, 64] as const) { ] as const) { const path = join(root, `${label}.exe`); await writeFile(path, bytes); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-go-binary", path, "--json"], }); expect(result.exitCode, result.stderr).toBe(0); @@ -208,7 +208,7 @@ cliTest( path, machoImageWithUnknownCommand(fixture.wide, fixture.size), ); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-go-binary", path, "--json"], }); const response = await call("inspect_go_binary", { path }); @@ -248,7 +248,7 @@ for (const expectedErrno of ["ELOOP", "ENAMETOOLONG"] as const) { root, expectedErrno, ); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-go-binary", path, "--json"], }); expect(result.exitCode).toBe(1); @@ -279,7 +279,7 @@ cliTest( const path = join(root, "byte-strings.elf"); const fixture = createGoBinaryByteStringFixture(); await writeFile(path, fixture.bytes); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-go-binary", path, "--json"], }); expect(result.exitCode, result.stderr).toBe(0); @@ -297,7 +297,7 @@ for (const fixture of createGoBinaryDiagnosticFixtures()) { const root = await createTestTempDirectory("rea-go-cli-mz-"); const path = join(root, `${fixture.name}.exe`); await writeFile(path, fixture.bytes); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-go-binary", path, "--json"], }); expect(result.exitCode).toBe(1); @@ -318,7 +318,7 @@ cliTest( const root = await createTestTempDirectory("rea-go-cli-"); const path = join(root, "text.bin"); await writeFile(path, "This is not an executable container."); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-go-binary", path, "--json"], }); expect(result.exitCode).toBe(1); @@ -338,7 +338,7 @@ for (const format of ["elf", "pe", "macho"] as const) { const fixture = createGoBinaryFixture({ format, encoding }); const path = join(root, "application.bin"); await writeFile(path, fixture.bytes); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-go-binary", "application.bin", "--json"], cwd: root, }); @@ -409,7 +409,7 @@ cliTest("preserves MIPS identity in Go CLI Evidence", async ({ cli }) => { fixture.bytes.writeUInt16LE(8, 18); // ELF EM_MIPS. const path = join(root, "application.elf"); await writeFile(path, fixture.bytes); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-go-binary", path, "--json"], }); expect(result.exitCode, result.stderr).toBe(0); @@ -428,7 +428,7 @@ cliTest( const root = await createTestTempDirectory("rea-go-cli-"); const path = join(root, "metadata-free.elf"); await writeFile(path, createGoBinaryFixture({ omitBuildInfo: true }).bytes); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-go-binary", path, "--json"], }); expect(result.exitCode).toBe(0); @@ -452,7 +452,7 @@ cliTest( const root = await createTestTempDirectory("rea-go-gopath-cli-"); const path = join(root, "application.elf"); await writeFile(path, createGoBinaryFixture({ moduleText }).bytes); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-go-binary", path, "--json"], }); expect(result.exitCode, result.stderr).toBe(0); diff --git a/tests/acceptance/il2cppMetadataCli.test.ts b/tests/acceptance/il2cppMetadataCli.test.ts index ab69cca9dd..95ff5bf731 100644 --- a/tests/acceptance/il2cppMetadataCli.test.ts +++ b/tests/acceptance/il2cppMetadataCli.test.ts @@ -40,7 +40,7 @@ import { import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; import { parseMcpToolError } from "../fixtures/mcpToolError.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; -import { cliTest } from "../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../support/cli/cliFixture.js"; const parseCliMetadata = (value: unknown) => il2cppMetadataSchema.parse( @@ -52,6 +52,80 @@ const parseMetadataSummary = (value: unknown) => parseEvidence(value).normalized_result, ); +cliTest( + "validates generic-container extents before returning CLI or MCP metadata", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-il2cpp-container-extents-"); + const { call } = await connectLocalToolsMcp(); + for (const count of [0, 1, 300]) { + const { bytes } = createIl2cppMetadataFixture({ + wide: { fillerTypes: 0, genericContainers: count, parameters: 0 }, + }); + const path = join(root, `valid-${count}.dat`); + await writeFile(path, bytes); + const output = await runFileCli(cli, { + arguments: ["inspect-il2cpp-metadata", path, "--json"], + }); + expect(output.exitCode, output.stderr).toBe(0); + const report = parseCliMetadata(output.json); + expect(report.index_widths.generic_container).toBe(count > 255 ? 2 : 1); + const response = await call("inspect_il2cpp_metadata", { + path, + detail: "complete", + }); + expect(response.isError).not.toBe(true); + expect(parseCliMetadata(response.structuredContent)).toEqual(report); + expect(await readFile(path)).toEqual(bytes); + } + for (const [records, declaredCount, declaredBytes] of [ + [0, 1, 0], + [0, 300, 0], + [1, 1, 1], + [1, 0, 16], + ] as const) { + const fixture = createIl2cppMetadataFixture({ + wide: { fillerTypes: 0, genericContainers: records, parameters: 0 }, + }); + fixture.bytes.writeInt32LE( + declaredCount, + fixture.header("genericContainers", "count"), + ); + fixture.bytes.writeInt32LE( + declaredBytes, + fixture.header("genericContainers", "size"), + ); + const path = join(root, `invalid-${declaredCount}-${declaredBytes}.dat`); + await writeFile(path, fixture.bytes); + const output = await runFileCli(cli, { + arguments: ["inspect-il2cpp-metadata", path, "--json"], + }); + expect(output.exitCode).toBe(1); + const expected = { + code: "invalid_request", + details: { + operation: "inspect_il2cpp_metadata", + issues: [ + { + path: ["path"], + reason: "invalid_format", + message: expect.stringContaining("record-size-mismatch"), + }, + ], + }, + }; + expect(output.json).toMatchObject(expected); + const response = await call("inspect_il2cpp_metadata", { + path, + detail: "complete", + }); + expect(response.isError).toBe(true); + expect(parseMcpToolError(response).error).toMatchObject(expected); + expect(await readFile(path)).toEqual(fixture.bytes); + } + }, + 30000, +); + cliTest( "returns identical complete inventories through CLI and MCP and pages a retained summary after removal", async ({ cli }) => { @@ -62,7 +136,7 @@ cliTest( propertiesAndEvents: true, }); await writeFile(path, bytes); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-il2cpp-metadata", path, "--json"], }); expect(result.exitCode, result.stderr).toBe(0); @@ -211,7 +285,7 @@ cliTest( ] as const) { const path = join(root, name); await writeFile(path, bytes); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-il2cpp-metadata", path, "--json"], }); expect(result.exitCode).toBe(1); @@ -261,7 +335,7 @@ cliTest( }, }, }; - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-il2cpp-metadata", path, "--json"], }); expect(result.exitCode).toBe(1); @@ -569,7 +643,7 @@ cliTest( const root = await createTestTempDirectory("rea-il2cpp-cli-export-"); const path = join(root, "global-metadata.dat"); const bundle = join(root, "evidence.json"); - const refused = await cli.run({ + const refused = await runFileCli(cli, { arguments: [ "inspect-il2cpp-metadata", join(root, "must-not-be-read"), @@ -593,14 +667,14 @@ cliTest( bundle, "--json", ]; - const result = await cli.run({ arguments: args }); + const result = await runFileCli(cli, { arguments: args }); expect(result.exitCode, result.stderr).toBe(0); expect(result.json).toMatchObject({ evidence_bundle: { path: bundle, bytes: expect.any(Number) }, summary: { normalized_result: { kind: "summary" } }, }); const before = await readFile(bundle); - const overwrite = await cli.run({ arguments: args }); + const overwrite = await runFileCli(cli, { arguments: args }); expect(overwrite.exitCode).toBe(1); expect(await readFile(bundle)).toEqual(before); const saved = await readEvidenceBundle(bundle); @@ -626,7 +700,7 @@ cliTest( }, }), ); - const selected = await cli.run({ + const selected = await runFileCli(cli, { arguments: ["inspect-analysis-view", query, "--json"], }); expect(selected.exitCode, selected.stderr).toBe(0); @@ -653,7 +727,7 @@ cliTest.skipIf(process.platform === "win32")( const decoy = join(root, "evidence.json"); await writeFile(decoy, "unrelated caller data"); const selected = `${root}/link/../evidence.json`; - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: [ "inspect-il2cpp-metadata", path, @@ -672,7 +746,7 @@ cliTest.skipIf(process.platform === "win32")( expect( JSON.parse(await readFile(join(actual, "evidence.json"), "utf8")), ).toEqual(saved.value); - const read = await cli.run({ + const read = await runFileCli(cli, { arguments: [ "inspect-evidence-bundle", JSON.stringify({ path: selected, detail: "summary" }), @@ -707,7 +781,7 @@ cliTest( rva: `0x${(address - FIXTURE_BINARY_BASE).toString(16)}`, }, ); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: [ "inspect-il2cpp-metadata", path, @@ -794,7 +868,7 @@ cliTest( { machine: 62 }, ), ); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: [ "inspect-il2cpp-metadata", path, @@ -826,7 +900,7 @@ cliTest( await writeFile(path, createIl2cppMetadataFixture().bytes); await writeFile(binaryPath, createElfFixture(createIl2cppBinaryFixture())); const run = (output: string, extra: readonly string[] = []) => - cli.run({ + runFileCli(cli, { arguments: [ "inspect-il2cpp-metadata", path, @@ -881,7 +955,7 @@ cliTest( }); expect(await readFile(seeds, "utf8")).toBe("kept\n"); - const unlinked = await cli.run({ + const unlinked = await runFileCli(cli, { arguments: [ "inspect-il2cpp-metadata", path, diff --git a/tests/acceptance/pyinstallerArchive.test.ts b/tests/acceptance/pyinstallerArchive.test.ts new file mode 100644 index 0000000000..1039e14c1b --- /dev/null +++ b/tests/acceptance/pyinstallerArchive.test.ts @@ -0,0 +1,203 @@ +import { execFile } from "node:child_process"; +import { readFile, readdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import { expect, it } from "vitest"; + +import { + PYTHON_39_MAGIC, + cArchiveExecutable, + pyzArchive, +} from "../../src/artifacts/pyinstaller/CArchive.fixture.js"; +import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js"; +import { artifactExtractionResultSchema } from "../../src/domain/artifactGraph.js"; +import { parseEvidence } from "../../src/domain/evidence.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); +const cli = fileURLToPath(new URL("../../scripts/rea.mjs", import.meta.url)); + +const onefile = async ( + magic = PYTHON_39_MAGIC, + marker = false, + compressPyz = false, +): Promise => { + const directory = await createTestTempDirectory("rea-pyinstaller-cli-"); + const path = join(directory, "hello"); + await writeFile( + path, + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 314, + pythonLibrary: "libpython3.14.dylib", + trailer: Buffer.from("signature"), + members: [ + ...(marker + ? [ + { + name: "pyimod00_crypto_key", + typeCode: "m", + payload: Buffer.from("key"), + compress: false, + }, + ] + : []), + { + name: "pyi-contents-directory _internal", + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }, + { + name: "../shared/app:assets/data.bin", + typeCode: "d", + payload: Buffer.alloc(0), + compress: false, + }, + { + name: "hello", + typeCode: "s", + payload: Buffer.from("print('hi')"), + compress: true, + }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic, + modules: [ + { name: "app", typeCode: 0, body: Buffer.from("app-body") }, + ], + }), + compress: compressPyz, + }, + ], + }), + ); + return path; +}; + +it("lists CArchive members and the Python version through inspect-artifact", async () => { + const path = await onefile(); + const result = await execute(process.execPath, [ + cli, + "inspect-artifact", + path, + "--json", + ]); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(JSON.parse(result.stdout)).normalized_result, + ); + expect(inspection.subject.root_format).toBe("pyinstaller"); + expect(inspection.next_probes).toEqual([]); + const inventory = inspection.substeps[0]?.evidence.normalized_result; + expect(inventory?.manifest.root_format).toBe("pyinstaller"); + const root = inventory?.occurrences.find( + (occurrence) => occurrence.logical_path === ".", + ); + expect(root?.format_metadata).toMatchObject({ + kind: "pyinstaller", + python_version_raw: 314, + python_major: 3, + python_minor: 14, + runtime_option_count: 1, + external_dependency_count: 1, + }); + expect( + inventory?.occurrences.find( + (item) => + item.entry_kind === "metadata" && + item.format_metadata?.kind === "pyinstaller-entry" && + item.format_metadata.type_code === "o", + ), + ).toMatchObject({ + artifact_id: null, + format_metadata: { + source_name: "pyi-contents-directory _internal", + type_code: "o", + }, + }); + expect( + inventory?.occurrences.map((occurrence) => occurrence.logical_path), + ).toEqual(expect.arrayContaining(["hello", "PYZ.pyz", "PYZ.pyz/app.pyc"])); + expect( + inventory?.occurrences.find( + (row) => + row.format_metadata?.kind === "pyinstaller-entry" && + row.format_metadata.type_code === "d", + ), + ).toMatchObject({ + entry_kind: "metadata", + artifact_id: null, + format_metadata: { + dependency_archive: "../shared/app", + dependency_member: "assets/data.bin", + dependency_status: "unresolved", + }, + }); +}); + +it("extracts CArchive members through extract-artifact", async () => { + const path = await onefile(); + const result = await execute(process.execPath, [ + cli, + "extract-artifact", + path, + "--json", + ]); + const extracted = artifactExtractionResultSchema.parse( + parseEvidence(JSON.parse(result.stdout)).normalized_result, + ); + const hello = extracted.artifacts.find( + (artifact) => artifact.relative_path === "hello", + ); + expect(hello).toBeDefined(); + expect(await readFile(join(extracted.output_root, "hello"))).toHaveLength( + hello?.bytes_written ?? 0, + ); + expect( + await readFile(join(extracted.output_root, "PYZ.pyz", "app.pyc")), + ).toHaveLength(16 + Buffer.from("app-body").length); + expect(await readFile(join(extracted.output_root, "hello"))).toEqual( + Buffer.from("print('hi')"), + ); + expect(await readdir(extracted.output_root)).not.toContain("__pyinstaller__"); + expect( + extracted.artifacts.some((item) => + item.relative_path.includes("pyi-contents-directory"), + ), + ).toBe(false); +}); + +it("reports unknown traversal through the CLI for an opaque PYZ", async () => { + const path = await onefile(Buffer.alloc(4), true, true); + const result = await execute(process.execPath, [ + cli, + "inspect-artifact", + path, + "--json", + ]); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(JSON.parse(result.stdout)).normalized_result, + ); + const inventory = inspection.substeps[0]?.evidence.normalized_result; + expect(inventory?.coverage.status).toBe("unknown"); + expect(inventory?.limitations.join(" ")).toContain("opaque"); + const opaque = inventory?.occurrences.find( + (row) => row.logical_path === "PYZ.pyz", + ); + expect(opaque).toMatchObject({ + encrypted: false, + hash_status: "verified", + artifact_id: expect.any(String), + source_location: null, + }); + expect(opaque?.format_metadata).toMatchObject({ + pyz_python_magic: "00000000", + pyz_status: "opaque", + encryption_source: "carchive-marker", + compression_flag: 1, + }); +}); diff --git a/tests/acceptance/rustBinaryCli.test.ts b/tests/acceptance/rustBinaryCli.test.ts index fbd8f69f56..b568065ac5 100644 --- a/tests/acceptance/rustBinaryCli.test.ts +++ b/tests/acceptance/rustBinaryCli.test.ts @@ -1,3 +1,5 @@ +import { rustBinarySchema } from "../../src/domain/rust/rustBinary.js"; +import { createHash } from "node:crypto"; import { deflateSync } from "node:zlib"; import { readFile, writeFile } from "node:fs/promises"; import { pathToFileURL } from "node:url"; @@ -7,6 +9,8 @@ import { toolContract } from "../../src/contracts/toolContracts.js"; import { machoImageWithUnknownCommand } from "../../src/artifacts/apple/MachoImage.fixture.js"; import { createRustImage, + createRustImageWithGoSection, + RUSTC_HASH, rustDependencyPayload, } from "../fixtures/rust/image.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; @@ -83,7 +87,9 @@ cliTest( const evidence = toolContract("inspect_rust_binary").outputSchema.parse( result.json, ); - expect(evidence.normalized_result.auditable_metadata).toMatchObject({ + expect( + rustBinarySchema.parse(evidence.normalized_result).auditable_metadata, + ).toMatchObject({ section_name: "__DATA,.dep-v0", format_version: 1, root_package_index: 1, @@ -106,8 +112,9 @@ cliTest("returns null metadata when .dep-v0 is absent", async ({ cli }) => { const evidence = toolContract("inspect_rust_binary").outputSchema.parse( result.json, ); - expect(evidence.normalized_result.auditable_metadata).toBeNull(); - expect(evidence.normalized_result.limitations.join("\n")).toContain( + const report = rustBinarySchema.parse(evidence.normalized_result); + expect(report.auditable_metadata).toBeNull(); + expect(report.limitations.join("\n")).toContain( "does not mean the binary has no dependencies", ); }); @@ -131,13 +138,12 @@ cliTest( const evidence = toolContract("inspect_rust_binary").outputSchema.parse( result.json, ); - expect(evidence.normalized_result.auditable_metadata?.format_version).toBe( - 4, - ); - expect(evidence.normalized_result.limitations.join("\n")).toContain( + const report = rustBinarySchema.parse(evidence.normalized_result); + expect(report.auditable_metadata?.format_version).toBe(4); + expect(report.limitations.join("\n")).toContain( "highest known preceding revision (1)", ); - expect(evidence.normalized_result.limitations.join("\n")).toContain( + expect(report.limitations.join("\n")).toContain( "this specific revision has not been independently verified", ); }, @@ -229,3 +235,94 @@ cliTest( expect(await readFile(path)).toEqual(bytes); }, ); + +cliTest( + "keeps Rust evidence independent of Go section errors through public CLI and stdio MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-rust-cli-go-section-"); + const { client, call } = await connectLocalToolsMcp(); + for (const [format, fault, code] of [ + ["elf", "compressed", "unsupported_target"], + ["elf", "zero-address", "invalid_request"], + ["macho", "zero-fill", "invalid_request"], + ] as const) { + const bytes = createRustImageWithGoSection(format, fault); + if (fault === "zero-address") { + const sectionHeader = 512; + expect(bytes.readUInt32LE(sectionHeader + 4)).toBe(1); + expect(bytes.readBigUInt64LE(sectionHeader + 8)).toBe(0n); + expect(bytes.readBigUInt64LE(sectionHeader + 16)).toBe(0n); + expect(bytes.readBigUInt64LE(sectionHeader + 24)).toBe(3072n); + expect(bytes.readBigUInt64LE(sectionHeader + 32)).toBe(16n); + const nameOffset = bytes.readUInt32LE(sectionHeader); + expect( + bytes.toString( + "ascii", + 768 + nameOffset, + 768 + nameOffset + ".go.buildinfo".length, + ), + ).toBe(".go.buildinfo"); + } + const path = join(root, `mixed.${format}`); + await writeFile(path, bytes); + const result = await cli.run({ + arguments: ["inspect-rust-binary", path, "--json"], + }); + expect(result.exitCode, result.stderr).toBe(0); + const evidence = toolContract("inspect_rust_binary").outputSchema.parse( + result.json, + ); + expect(evidence.normalized_result).toMatchObject({ + artifact: { + path, + sha256: createHash("sha256").update(bytes).digest("hex"), + bytes: bytes.length, + }, + format, + auditable_metadata: { + file_offset: 1024, + root_package_index: 1, + packages: [ + { name: "itoa", version: "1.0.18" }, + { name: "sample", version: "0.1.0", root: true }, + ], + dependency_edges: [{ from_index: 1, to_index: 0 }], + }, + rustc_commit: { + evidence_kind: "inferred", + hashes: [{ hash: RUSTC_HASH, file_offset: 2048 }], + }, + }); + const response = await call("inspect_rust_binary", { path }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(response.structuredContent).toEqual(evidence); + const go = await cli.run({ + arguments: ["inspect-go-binary", path, "--json"], + }); + expect(go.exitCode).toBe(1); + expect(go.json).toMatchObject({ + code, + details: { operation: "inspect_go_binary" }, + }); + if (fault === "zero-address") + expect(go.json).toMatchObject({ + details: { + issues: [{ path: ["path"], reason: "invalid_format" }], + }, + }); + const goResponse = await call("inspect_go_binary", { path }); + expect(parseMcpToolError(goResponse).error).toMatchObject({ + code, + details: { operation: "inspect_go_binary" }, + }); + if (fault === "zero-address") + expect(parseMcpToolError(goResponse).error).toMatchObject({ + details: { + issues: [{ path: ["path"], reason: "invalid_format" }], + }, + }); + } + await client.ping(); + }, + 30000, +); diff --git a/tests/acceptance/rustDependencyViews.test.ts b/tests/acceptance/rustDependencyViews.test.ts new file mode 100644 index 0000000000..6f97f45a24 --- /dev/null +++ b/tests/acceptance/rustDependencyViews.test.ts @@ -0,0 +1,298 @@ +import { readFile, unlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; +import { parseEvidence, createEvidence } from "../../src/domain/evidence.js"; +import { rustBinarySchema } from "../../src/domain/rust/rustBinary.js"; +import { analysisViewResultSchema } from "../../src/domain/analysisView/analysisView.js"; +import { readEvidenceBundle } from "../../src/application/EvidenceBundleFiles.js"; +import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; +import { parseMcpToolError } from "../fixtures/mcpToolError.js"; +import { + createRustImage, + rustDependencyPayload, +} from "../fixtures/rust/image.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +/** Both authenticated retention and portable imports query the same captured facts. */ +const assertRetainedPages = async ( + call: Awaited>["call"], + evidenceId: string, + exported: string, +) => { + for (const [filter, expected] of [ + [ + { names: ["same"], versions: ["2"], fields: ["name", "version"] }, + [{ index: 1, name: "same", version: "2" }], + ], + [{ names: ["same"], indices: [0], fields: [] }, [{ index: 0 }]], + [{ names: ["same"], versions: [], fields: [] }, []], + ] as const) { + const reply = await call("inspect_analysis_view", { + source: { kind: "retained-evidence", evidence_id: evidenceId }, + view: { kind: "rust", collection: "packages", ...filter }, + }); + expect(reply.isError).not.toBe(true); + expect( + parseEvidence(reply.structuredContent).normalized_result, + ).toMatchObject({ page: { rows: expected } }); + } + + for (const source of [ + { kind: "retained-evidence", evidence_id: evidenceId }, + { kind: "evidence-file", path: exported }, + ]) { + const reply = await call("inspect_analysis_view", { + source, + view: { + kind: "rust", + collection: "packages", + names: ["same"], + indices: [0, 1, 99], + offset: 1, + limit: 1, + fields: [], + }, + }); + expect(reply.isError, JSON.stringify(reply)).not.toBe(true); + expect( + parseEvidence(reply.structuredContent).normalized_result, + ).toMatchObject({ + parent_evidence_id: evidenceId, + page: { rows: [{ index: 1 }] }, + coverage: { + total: 2, + examined: 1, + exhausted: true, + next_offset: null, + }, + }); + const edgeReply = await call("inspect_analysis_view", { + source, + view: { + kind: "rust", + collection: "edges", + from_indices: [1, 3], + to_indices: [0], + fields: ["from_index", "to_index"], + }, + }); + expect(edgeReply.isError).not.toBe(true); + expect( + parseEvidence(edgeReply.structuredContent).normalized_result, + ).toMatchObject({ + page: { + collection: "edges", + rows: [ + { index: 1, from_index: 1, to_index: 0 }, + { index: 3, from_index: 3, to_index: 0 }, + ], + }, + }); + const empty = await call("inspect_analysis_view", { + source, + view: { + kind: "rust", + collection: "packages", + names: [], + versions: ["1"], + fields: ["name"], + }, + }); + expect( + parseEvidence(empty.structuredContent).normalized_result, + ).toMatchObject({ + page: { rows: [] }, + coverage: { total: 0, exhausted: true }, + }); + } +}; + +cliTest( + "queries stable Rust packages and edges through real CLI/MCP and portable Evidence", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-rust-views-"); + const { call } = await connectLocalToolsMcp(); + for (const format of ["elf", "pe", "macho"] as const) { + const path = join(root, `source.${format}`); + const output = join(root, `${format}-source.json`); + const payload = rustDependencyPayload({ + packages: [ + { + name: "same", + version: "1", + source: "crates.io", + dependencies: [2], + }, + { + name: "same", + version: "2", + source: "git+https://upstream.example/repo", + dependencies: [0, 2], + }, + { name: "leaf", version: "1", source: "local" }, + { + name: "app", + version: "1", + source: "local", + dependencies: [0, 1], + root: true, + }, + ], + }); + await writeFile(path, createRustImage({ format, payload })); + const fullReply = await call("inspect_rust_binary", { path }); + expect(fullReply.isError).not.toBe(true); + const full = parseEvidence(fullReply.structuredContent); + const report = rustBinarySchema.parse(full.normalized_result); + const explicit = await call("inspect_rust_binary", { + path, + detail: "complete", + }); + expect(parseEvidence(explicit.structuredContent)).toEqual(full); + const selection = { + collection: "packages", + names: ["same"], + fields: ["name", "version", "dependencies"], + }; + const direct = await call("inspect_rust_binary", { + path, + view: selection, + }); + expect(direct.isError, JSON.stringify(direct)).not.toBe(true); + const selected = parseEvidence( + direct.structuredContent, + ).normalized_result; + expect(selected).toMatchObject({ + kind: "rust", + parent_evidence_id: full.evidence_id, + page: { + collection: "packages", + rows: [ + { index: 0, name: "same", version: "1", dependencies: [2] }, + { index: 1, name: "same", version: "2", dependencies: [0, 2] }, + ], + }, + }); + const summary = await call("inspect_rust_binary", { + path, + detail: "summary", + }); + expect(summary.isError).not.toBe(true); + expect( + parseEvidence(summary.structuredContent).normalized_result, + ).toMatchObject({ + kind: "summary", + summary: { + auditable_metadata: { + counts: { packages: 4, edges: 5 }, + format_version: null, + }, + }, + source_coverage: { + embedded_records: "complete", + runtime_dependencies: "unknown", + }, + }); + const delivered = await cli.run({ + arguments: [ + "inspect-rust-binary", + path, + "--view", + JSON.stringify(selection), + "--evidence_output", + output, + "--json", + ], + }); + expect(delivered.exitCode, delivered.stderr).toBe(0); + expect(delivered.json).toMatchObject({ + summary: { normalized_result: selected }, + }); + const bundle = await readEvidenceBundle(output); + if (!bundle.ok) throw bundle.error; + expect( + bundle.value.records.find( + (row) => row.evidence_id === full.evidence_id, + ), + ).toEqual(full); + const exported = join(root, `${format}-record.json`); + await writeFile(exported, JSON.stringify(full)); + await unlink(path); + + await assertRetainedPages(call, full.evidence_id, exported); + const malformed = structuredClone(report); + malformed.auditable_metadata!.dependency_edges[0]!.to_index = 999; + if (full.subject == null) + throw new Error("Missing complete Rust subject"); + const invalid = createEvidence( + { + path, + sha256: report.artifact.sha256, + format: format === "macho" ? "mach-o" : format, + ...(full.subject.architecture === null + ? {} + : { architecture: full.subject.architecture }), + }, + full.provider, + { + operation: full.operation, + parameters: full.parameters, + result: malformed, + }, + ); + expect( + parseMcpToolError( + await call("inspect_analysis_view", { + source: { kind: "inline", evidence: invalid }, + view: { kind: "rust", collection: "edges" }, + }), + ).error, + ).toMatchObject({ code: "invalid_request" }); + } + }, + 60000, +); + +cliTest( + "preserves unknown Rust metadata and refuses invalid compact CLI choices before reading", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-rust-view-missing-"); + const path = join(root, "plain.elf"); + await writeFile( + path, + createRustImage({ format: "elf", omitSection: true }), + ); + const { call } = await connectLocalToolsMcp(); + const summary = await call("inspect_rust_binary", { + path, + detail: "summary", + }); + expect(summary.isError).not.toBe(true); + const view = analysisViewResultSchema.parse( + parseEvidence(summary.structuredContent).normalized_result, + ); + expect(view).toMatchObject({ + summary: { auditable_metadata: null }, + source_coverage: { + embedded_records: "unknown", + runtime_dependencies: "unknown", + }, + }); + const missing = join(root, "never-created"); + for (const args of [ + ["--detail", "summary"], + ["--view", "{"], + ["--detail", "complete", "--view", '{"collection":"packages"}'], + ]) { + const refused = await cli.run({ + arguments: ["inspect-rust-binary", missing, ...args, "--json"], + }); + expect(refused.exitCode).toBe(1); + expect(refused.json).toMatchObject({ code: "invalid_request" }); + expect(JSON.stringify(refused.json)).not.toContain("ENOENT"); + } + expect((await readFile(path)).length).toBeGreaterThan(0); + }, + 30000, +); diff --git a/tests/acceptance/sqliteDatabaseCli.test.ts b/tests/acceptance/sqliteDatabaseCli.test.ts index d0ffb361c0..d871fe31ab 100644 --- a/tests/acceptance/sqliteDatabaseCli.test.ts +++ b/tests/acceptance/sqliteDatabaseCli.test.ts @@ -86,11 +86,20 @@ cliTest( })), ); const cliResult = await cli.run({ - arguments: ["inspect-sqlite-database", path, "--json"], + arguments: [ + "inspect-sqlite-database", + path, + "--worker-timeout-ms", + "60000", + "--json", + ], environment: { REA_LOG_LEVEL: "silent" }, }); const { call } = await connectLocalToolsMcp(); - const mcpResponse = await call("inspect_sqlite_database", { path }); + const mcpResponse = await call("inspect_sqlite_database", { + path, + worker_timeout_ms: 60000, + }); const mcpError = parseMcpToolError(mcpResponse).error; const expectedError = { code: "resource_constraint", @@ -240,10 +249,13 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( path, table: "selected", columns: ["", "id", "id"], + worker_timeout_ms: 42000, }); expect(selected.isError).not.toBe(true); expect(selected.structuredContent).toMatchObject({ + parameters: { worker_timeout_ms: 42000 }, normalized_result: { + worker_timeout_ms: 42000, schema: { scope: "selected_table", tables: [{ name: "selected" }], @@ -283,13 +295,19 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( "", "--row-limit", "1", + "--worker-timeout-ms", + "42000", "--json", ], environment: { REA_LOG_LEVEL: "silent" }, }); expect(cliResult.exitCode, cliResult.stdout + cliResult.stderr).toBe(0); expect(cliResult.json).toMatchObject({ - normalized_result: { rows: { columns: ["id", ""], returned_rows: 1 } }, + parameters: { worker_timeout_ms: 42000 }, + normalized_result: { + worker_timeout_ms: 42000, + rows: { columns: ["id", ""], returned_rows: 1 }, + }, }); for (const input of [ { path, columns: ["id"] }, @@ -678,17 +696,33 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( async ({ cli }) => { const root = await createTestTempDirectory("rea-sqlite-errors-public-"); const path = join(root, "invalid.db"); - await writeFile(path, "not a SQLite database"); const { call } = await connectLocalToolsMcp(); - const corrupt = await cli.run({ - arguments: ["inspect-sqlite-database", path, "--json"], - environment: { REA_LOG_LEVEL: "silent" }, - }); - expect(corrupt.exitCode).toBe(1); - expect(corrupt.json).toMatchObject({ code: "invalid_request" }); - expect((await call("inspect_sqlite_database", { path })).isError).toBe( - true, - ); + // Distinct malformed selections each name their own selected path. + for (const selected of [path, join(root, "other-invalid.db")]) { + await writeFile(selected, "not a SQLite database"); + const expectedError = { + code: "invalid_request", + details: { + issues: [ + { + path: ["path"], + reason: "invalid_format", + message: `Selected SQLite database is invalid (file is not a database): ${selected}`, + }, + ], + }, + }; + const corrupt = await cli.run({ + arguments: ["inspect-sqlite-database", selected, "--json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(corrupt.exitCode).toBe(1); + expect(corrupt.json).toMatchObject(expectedError); + const response = await call("inspect_sqlite_database", { + path: selected, + }); + expect(parseMcpToolError(response).error).toMatchObject(expectedError); + } const valid = join(root, "valid.db"); const database = new DatabaseSync(valid); try { diff --git a/tests/boundary/browser/cdpBrowserProvider.lifecycle_failures.test.ts b/tests/boundary/browser/cdpBrowserProvider.lifecycle_failures.test.ts index 2dba76b7e5..9b9e12c5d0 100644 --- a/tests/boundary/browser/cdpBrowserProvider.lifecycle_failures.test.ts +++ b/tests/boundary/browser/cdpBrowserProvider.lifecycle_failures.test.ts @@ -366,10 +366,10 @@ describe("CdpBrowserProvider: session termination and scope changes", () => { ); if (!result.ok) throw result.error; - expect(result.value.window.end_reason).toBe("target_terminated"); + expect(result.value.window.end_reason).toBe("session_detached"); expect(result.value.timeline.at(-1)).toMatchObject({ - type: "target_terminated", - detail: "target_terminated", + type: "session_detached", + detail: "session_detached", timestamp: null, }); expect(result.value.limitations).toContain( @@ -377,6 +377,28 @@ describe("CdpBrowserProvider: session termination and scope changes", () => { ); }); + it("reports target termination only for an observed target crash", async () => { + const browser = await startFakeCdpBrowser({ + sessionTimeline: "target_crashed", + }); + trackBrowser(browser); + const result = await new CdpBrowserProvider().observeSession( + observeWebSessionInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + observation_ms: 1_000, + }), + ); + + if (!result.ok) throw result.error; + expect(result.value.window.end_reason).toBe("target_terminated"); + expect(result.value.timeline.at(-1)).toMatchObject({ + type: "target_terminated", + detail: "target_terminated", + }); + }); + it("ends a direct page session when its transport disconnects", async () => { const browser = await startFakeCdpBrowser({ pageScopedVersionWebSocket: true, @@ -394,10 +416,10 @@ describe("CdpBrowserProvider: session termination and scope changes", () => { ); if (!result.ok) throw result.error; - expect(result.value.window.end_reason).toBe("target_terminated"); + expect(result.value.window.end_reason).toBe("transport_disconnected"); expect(result.value.timeline.at(-1)).toMatchObject({ - type: "target_terminated", - detail: "target_terminated", + type: "transport_disconnected", + detail: "transport_disconnected", }); const methods = browser.commands.map(({ method }) => method); expect(methods).not.toContain("Target.attachToTarget"); diff --git a/tests/boundary/browser/cdpObservationWindowScope.test.ts b/tests/boundary/browser/cdpObservationWindowScope.test.ts index 7e76b967c8..8331adc17b 100644 --- a/tests/boundary/browser/cdpObservationWindowScope.test.ts +++ b/tests/boundary/browser/cdpObservationWindowScope.test.ts @@ -268,6 +268,8 @@ it("fails closed on an untimestamped out-of-scope redirect during clock sampling it.each([ [ "Inspector.targetCrashed", + "target_terminated", + "target_terminated", { method: "Inspector.targetCrashed", sessionId: "session-1", @@ -276,6 +278,8 @@ it.each([ ], [ "Target.detachedFromTarget", + "session_detached", + "session_detached", { method: "Target.detachedFromTarget", params: { sessionId: "session-1", targetId: "allowed-page" }, @@ -283,7 +287,7 @@ it.each([ ], ] as const)( "ends on buffered untimestamped %s before collecting later events", - async (_method, terminalEvent) => { + async (_method, endReason, eventType, terminalEvent) => { const { observation } = await observeWithFake({ eventsBeforeReply: ["Performance.getMetrics"], commandResult: (command) => @@ -310,10 +314,10 @@ it.each([ }, }); - expect(observation.window.end_reason).toBe("target_terminated"); + expect(observation.window.end_reason).toBe(endReason); expect(observation.timeline).toContainEqual( expect.objectContaining({ - type: "target_terminated", + type: eventType, timestamp: null, }), ); diff --git a/tests/boundary/browser/cdpWebRuntimeTermination.test.ts b/tests/boundary/browser/cdpWebRuntimeTermination.test.ts index ec2e633a40..588088225b 100644 --- a/tests/boundary/browser/cdpWebRuntimeTermination.test.ts +++ b/tests/boundary/browser/cdpWebRuntimeTermination.test.ts @@ -1,11 +1,12 @@ -import { describe, expect, it, onTestFinished } from "vitest"; +import { describe, expect, it, onTestFinished, vi } from "vitest"; import { CdpWebRuntimeProvider } from "../../../src/browser/execution/CdpWebRuntimeProvider.js"; import type { CdpEvent } from "../../../src/browser/CdpConnection.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { observeWebExecutionInputSchema } from "../../../src/domain/webExecution.js"; import { startRuntimeBrowser } from "../../fixtures/webRuntime.js"; describe("runtime flat-session lifecycle", () => { - it("rejects a producer detachment between counter startup and local arming", async () => { + it("rejects session detachment between counter startup and local arming", async () => { let emitEvent: ((event: CdpEvent) => void) | undefined; const browser = await startRuntimeBrowser({ commandResult: (command) => { @@ -37,14 +38,223 @@ describe("runtime flat-session lifecycle", () => { }, ); if (result.ok) - throw new Error("A terminated target cannot arm an execution window"); - expect(result.error).toMatchObject({ reason: "target_changed" }); + throw new Error("A detached session cannot arm an execution window"); + expect(result.error).toMatchObject({ reason: "target_session_detached" }); expect(reportedArmed).toBe(false); }); - it("ends immediately on its root detachment without confusing another session", async () => { + it.each([ + { + case: "matching detach after an unrelated session detaches", + events: [ + { + method: "Target.detachedFromTarget", + params: { sessionId: "another-session" }, + }, + { + method: "Target.detachedFromTarget", + params: { sessionId: "session-1" }, + }, + ] as CdpEvent[], + observationMs: 2_147_483_647, + endReason: "session_detached", + }, + { + case: "detach for another flat session", + events: [ + { + method: "Target.detachedFromTarget", + params: { sessionId: "another-session" }, + }, + ] as CdpEvent[], + observationMs: 30, + endReason: "window_elapsed", + }, + { + case: "the observed crash event", + events: [ + { + method: "Inspector.targetCrashed", + sessionId: "session-1", + params: {}, + }, + ] as CdpEvent[], + observationMs: 2_147_483_647, + endReason: "target_terminated", + }, + ])("preserves lifecycle identity for $case", async (scenario) => { const browser = await startRuntimeBrowser(); onTestFinished(() => browser.close()); + const result = await new CdpWebRuntimeProvider().observeExecution( + observeWebExecutionInputSchema.parse({ + cdp_endpoint: browser.endpoint, + target_id: "allowed-page", + observation_ms: scenario.observationMs, + }), + { + signal: AbortSignal.timeout(2_000), + progress: { + report: async (update) => { + if (update.completed === 1) + for (const event of scenario.events) browser.emitEvent(event); + }, + }, + }, + ); + if (!result.ok) throw result.error; + expect(result.value.window.end_reason).toBe(scenario.endReason); + if (scenario.endReason !== "window_elapsed") { + expect(result.value.coverage.state).toBe("unavailable"); + expect(result.value.instrumentation.takes_one_resetting_sample).toBe( + false, + ); + } + }); +}); + +describe("runtime stop lifecycle", () => { + it.each(["cleanup failure", "caller cancellation"] as const)( + "retains explicit session detachment during precise-coverage stop with %s", + async (outcome) => { + const controller = new AbortController(); + const reason = { caller: "stop precise coverage" }; + const browser = await startRuntimeBrowser( + outcome === "cleanup failure" + ? { + commandError: (command) => + command.method === "Profiler.stopPreciseCoverage" + ? { code: -32_000, message: "Target session is detached" } + : undefined, + } + : { + commandResult: (command) => { + if (command.method === "Profiler.stopPreciseCoverage") + controller.abort(reason); + return undefined; + }, + }, + ); + onTestFinished(() => browser.close()); + const result = await new CdpWebRuntimeProvider().observeExecution( + observeWebExecutionInputSchema.parse({ + cdp_endpoint: browser.endpoint, + target_id: "allowed-page", + observation_ms: 2_147_483_647, + }), + { + signal: + outcome === "cleanup failure" + ? AbortSignal.timeout(2_000) + : controller.signal, + progress: { + report: async (update) => { + if (update.completed === 1) + browser.emitEvent({ + method: "Target.detachedFromTarget", + params: { sessionId: "session-1" }, + }); + }, + }, + }, + ); + if (outcome === "caller cancellation") { + if (result.ok) throw new Error("The aborted stop command must fail"); + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + cause: reason, + }); + return; + } + if (result.ok) + throw new Error("Unconfirmed cleanup must remain a failure"); + + const projection = projectAnalysisError(result.error); + expect(projection.code).toBe("cleanup_incomplete"); + expect(projection.details?.diagnostics).toMatchObject({ + primary_error: { + message: expect.stringContaining("Target session is detached"), + details: { + diagnostics: { + observed_window: { + target_id: "allowed-page", + end_reason: "session_detached", + }, + }, + }, + }, + cleanup_error: { + details: { + diagnostics: { + cleanup_failures: expect.arrayContaining([ + expect.stringContaining("Profiler.stopPreciseCoverage"), + ]), + }, + }, + }, + }); + expect( + browser.commands.filter( + ({ method }) => method === "Profiler.stopPreciseCoverage", + ), + ).toHaveLength(2); + expect(result.error.cleanupPrimary).toMatchObject({ + _tag: "BrowserObservationError", + reason: "protocol_error", + diagnostics: { + observed_window: { + target_id: "allowed-page", + end_reason: "session_detached", + }, + }, + }); + }, + ); +}); + +describe("runtime disconnect ownership", () => { + it("preserves a pending stop-command transport failure after session detachment", async () => { + const browser = await startRuntimeBrowser({ + hangOnMethod: "Profiler.stopPreciseCoverage", + }); + let closed = false; + onTestFinished(async () => { + if (!closed) await browser.close(); + }); + const observed = new CdpWebRuntimeProvider().observeExecution( + observeWebExecutionInputSchema.parse({ + cdp_endpoint: browser.endpoint, + target_id: "allowed-page", + observation_ms: 1, + }), + { signal: AbortSignal.timeout(2_000) }, + ); + await vi.waitFor(() => { + expect( + browser.commands.some( + ({ method }) => method === "Profiler.stopPreciseCoverage", + ), + ).toBe(true); + }); + browser.emitEvent({ + method: "Target.detachedFromTarget", + params: { sessionId: "session-1" }, + }); + await browser.close(); + closed = true; + const result = await observed; + if (result.ok) throw new Error("The pending stop request must fail"); + expect(result.error.cleanupPrimary).toMatchObject({ + _tag: "BrowserObservationError", + reason: "disconnected", + }); + }); + + it("does not infer target termination from transport disconnect", async () => { + const browser = await startRuntimeBrowser(); + let closed = false; + onTestFinished(async () => { + if (!closed) await browser.close(); + }); const result = await new CdpWebRuntimeProvider().observeExecution( observeWebExecutionInputSchema.parse({ cdp_endpoint: browser.endpoint, @@ -52,52 +262,55 @@ describe("runtime flat-session lifecycle", () => { observation_ms: 2_147_483_647, }), { - signal: AbortSignal.timeout(2_000), progress: { report: async (update) => { - if (update.completed !== 1) return; - browser.emitEvent({ - method: "Target.detachedFromTarget", - params: { sessionId: "another-session" }, - }); - browser.emitEvent({ - method: "Target.detachedFromTarget", - params: { sessionId: "session-1" }, - }); + if (update.completed === 1) { + await browser.close(); + closed = true; + } }, }, }, ); - if (!result.ok) throw result.error; - expect(result.value.window.end_reason).toBe("target_terminated"); - expect(result.value.coverage.state).toBe("unavailable"); - expect(result.value.instrumentation.takes_one_resetting_sample).toBe(false); + if (result.ok) throw new Error("A disconnected transport is not a result"); + expect(result.error.cleanupPrimary).toMatchObject({ + _tag: "BrowserObservationError", + reason: "disconnected", + }); }); +}); - it("ignores root detachment for a different flat session", async () => { - const browser = await startRuntimeBrowser(); - onTestFinished(() => browser.close()); - const result = await new CdpWebRuntimeProvider().observeExecution( - observeWebExecutionInputSchema.parse({ - cdp_endpoint: browser.endpoint, - target_id: "allowed-page", - observation_ms: 30, - }), - { - progress: { - report: async (update) => { - if (update.completed === 1) - browser.emitEvent({ - method: "Target.detachedFromTarget", - params: { sessionId: "another-session" }, - }); - }, - }, - }, - ); - if (!result.ok) throw result.error; - expect(result.value.window.end_reason).toBe("window_elapsed"); - }); +describe("runtime cleanup failure ordering", () => { + it.each([ + ["Profiler.takePreciseCoverage", "Coverage sample failed"], + ["Profiler.stopPreciseCoverage", "Coverage stop failed"], + ] as const)( + "keeps the %s protocol failure primary when transport later closes", + async (method, message) => { + const browser = await startRuntimeBrowser({ + commandError: (command) => + command.method === method ? { code: -32_000, message } : undefined, + closeAfterMethod: method, + }); + onTestFinished(() => browser.close()); + const result = await new CdpWebRuntimeProvider().observeExecution( + observeWebExecutionInputSchema.parse({ + cdp_endpoint: browser.endpoint, + target_id: "allowed-page", + observation_ms: 1, + }), + { signal: AbortSignal.timeout(2_000) }, + ); + if (result.ok) throw new Error("The failed CDP command must fail"); + const primary = result.error.cleanupPrimary; + expect(primary).toMatchObject({ + _tag: "BrowserObservationError", + reason: "protocol_error", + }); + if (method === "Profiler.takePreciseCoverage") + expect(primary?.secondaryFailures).toHaveLength(0); + }, + ); }); describe("coverage URL representation", () => { diff --git a/tests/boundary/cli/cliInputSchemaRequired.test.ts b/tests/boundary/cli/cliInputSchemaRequired.test.ts index 1bf778b7a7..3fae2290dd 100644 --- a/tests/boundary/cli/cliInputSchemaRequired.test.ts +++ b/tests/boundary/cli/cliInputSchemaRequired.test.ts @@ -61,12 +61,19 @@ const parserSchema = ( return schema instanceof z.ZodObject ? schema : undefined; }; +/** + * Reject a key only when an object parse that omits it fails. + * `exactOptional()` rejects explicit `undefined` and still accepts omission. + */ const omissionRejected = (schema: z.ZodObject | undefined): string[] => Object.entries(schema?.shape ?? {}) - .filter(([, field]) => !field.safeParse(undefined).success) + .filter(([key, field]) => !acceptsOmittedKey(key, field)) .map(([key]) => key) .sort(); +const acceptsOmittedKey = (key: string, field: z.ZodType): boolean => + z.object({ [key]: field }).safeParse({}).success; + describe("CLI advertised input schemas", () => { it("require exactly the inputs whose omission the parser rejects", async () => { const cli = createCli({}); @@ -104,6 +111,30 @@ describe("CLI advertised input schemas", () => { expect(mismatches).toEqual([]); }); + it("allows an omitted WASM timeout while validating explicitly supplied values", async () => { + const cli = createCli({}); + const advertised = await advertisedSchema(cli, "inspect-wasm-artifact"); + const options = parserSchema(cli, "inspect-wasm-artifact", "options"); + expect(options).toBeDefined(); + const validate = new Ajv2020({ strict: false }).compile( + advertised.options ?? {}, + ); + + expect(options?.safeParse({}).success).toBe(true); + expect(validate({})).toBe(true); + expect(options?.safeParse({ commandTimeoutMs: undefined }).success).toBe( + false, + ); + for (const commandTimeoutMs of [1, 30_000, 2_147_483_647]) { + expect(options?.safeParse({ commandTimeoutMs }).success).toBe(true); + expect(validate({ commandTimeoutMs })).toBe(true); + } + for (const commandTimeoutMs of [0, 1.5, 2_147_483_648]) { + expect(options?.safeParse({ commandTimeoutMs }).success).toBe(false); + expect(validate({ commandTimeoutMs })).toBe(false); + } + }); + it("accepts the one-argument keyed-archive call and keeps its constraints", async () => { const cli = createCli({}); const advertised = await advertisedSchema(cli, "inspect-keyed-archive"); diff --git a/tests/boundary/cli/compiledCliSession.test.ts b/tests/boundary/cli/compiledCliSession.test.ts new file mode 100644 index 0000000000..0499a5c44c --- /dev/null +++ b/tests/boundary/cli/compiledCliSession.test.ts @@ -0,0 +1,266 @@ +import { EventEmitter, PassThrough } from "node:stream"; +import type { ChildProcessWithoutNullStreams } from "node:child_process"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, onTestFinished, test } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { startCompiledCliSession } from "../../support/cli/compiledCliSession.js"; + +interface FakeCliProcess extends EventEmitter { + stdin: PassThrough; + stdout: PassThrough; + stderr: PassThrough; + pid: number; + exitCode: number | null; + signalCode: NodeJS.Signals | null; + kill(signal?: NodeJS.Signals): boolean; +} + +const fakeProcess = (pid: number): FakeCliProcess => { + const child = new EventEmitter() as FakeCliProcess; + child.stdin = new PassThrough(); + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + child.pid = pid; + child.exitCode = null; + child.signalCode = null; + child.kill = (signal: NodeJS.Signals = "SIGTERM") => { + if (child.exitCode !== null || child.signalCode !== null) return true; + child.signalCode = signal; + child.emit("exit", null, signal); + child.emit("close", null, signal); + return true; + }; + return child; +}; + +const asCliProcess = (child: FakeCliProcess): ChildProcessWithoutNullStreams => + child as unknown as ChildProcessWithoutNullStreams; + +const ready = (child: FakeCliProcess): void => { + child.stdout.write(`${JSON.stringify({ ready: true })}\n`); +}; + +const exitWhenStdinCloses = (child: FakeCliProcess): void => { + child.stdin.on("finish", () => { + if (child.exitCode !== null || child.signalCode !== null) return; + child.exitCode = 0; + child.emit("exit", 0, null); + child.emit("close", 0, null); + }); +}; + +const replyWhenAsked = ( + child: FakeCliProcess, + stdout: string, + exitCode = 0, +): void => { + let buffer = ""; + child.stdin.setEncoding("utf8"); + child.stdin.on("data", (chunk: string) => { + buffer += chunk; + let newline = buffer.indexOf("\n"); + while (newline !== -1) { + const line = buffer.slice(0, newline); + buffer = buffer.slice(newline + 1); + if (line.length > 0) { + const request = JSON.parse(line) as { id: number }; + child.stdout.write( + `${JSON.stringify({ id: request.id, exitCode, stdout })}\n`, + ); + } + newline = buffer.indexOf("\n"); + } + }); +}; + +test("rejects a compiled CLI command when close wins the ready handshake", async () => { + let child: FakeCliProcess | undefined; + let releaseReady: (() => void) | undefined; + const session = startCompiledCliSession({ + stopTimeoutMs: 50, + spawnWorker: () => { + const started = fakeProcess(60); + child = started; + exitWhenStdinCloses(started); + releaseReady = () => { + ready(started); + }; + return asCliProcess(started); + }, + }); + + const pending = session.run(["early"]); + for (let attempt = 0; child === undefined && attempt < 30; attempt += 1) + await new Promise((resolve) => { + setImmediate(resolve); + }); + if (child === undefined) throw new Error("compiled CLI worker did not start"); + const closing = session.close(); + releaseReady?.(); + await expect(pending).rejects.toThrow("Compiled CLI session is closed"); + await closing; +}); + +test("rejects a compiled CLI command that never replies and serves the next one", async () => { + const children: FakeCliProcess[] = []; + const session = startCompiledCliSession({ + requestTimeoutMs: 30, + spawnWorker: () => { + const child = fakeProcess(70 + children.length); + children.push(child); + exitWhenStdinCloses(child); + ready(child); + if (children.length > 1) + replyWhenAsked(child, JSON.stringify({ ok: true })); + return asCliProcess(child); + }, + }); + + await expect(session.run(["stuck"])).rejects.toThrow(/did not respond/); + expect(children[0]?.signalCode).toBe("SIGKILL"); + await expect(session.run(["next"])).resolves.toEqual({ ok: true }); + expect(children).toHaveLength(2); + await session.close(); +}); + +test("keeps a timed-out compiled CLI worker until it exits", async () => { + const children: FakeCliProcess[] = []; + let releaseExit: (() => void) | undefined; + const session = startCompiledCliSession({ + requestTimeoutMs: 20, + stopTimeoutMs: 40, + spawnWorker: () => { + const child = fakeProcess(90 + children.length); + children.push(child); + if (children.length === 1) { + child.kill = () => { + releaseExit = () => { + if (child.exitCode !== null || child.signalCode !== null) return; + child.signalCode = "SIGKILL"; + child.emit("exit", null, "SIGKILL"); + child.emit("close", null, "SIGKILL"); + }; + return true; + }; + } else { + exitWhenStdinCloses(child); + replyWhenAsked(child, JSON.stringify({ ok: true })); + } + ready(child); + return asCliProcess(child); + }, + }); + + const stalled = session.run(["stuck"]); + const next = session.run(["next"]); + await expect(stalled).rejects.toThrow(/did not respond/); + expect(children).toHaveLength(1); + expect(children[0]?.signalCode).toBeNull(); + releaseExit?.(); + await expect(next).resolves.toEqual({ ok: true }); + expect(children).toHaveLength(2); + await session.close(); +}); + +test("retries a timed-out compiled CLI worker that survives SIGKILL", async () => { + let child: FakeCliProcess | undefined; + const signals: NodeJS.Signals[] = []; + const session = startCompiledCliSession({ + requestTimeoutMs: 20, + stopTimeoutMs: 20, + spawnWorker: () => { + child = fakeProcess(91); + child.kill = (signal: NodeJS.Signals = "SIGTERM") => { + signals.push(signal); + return true; + }; + ready(child); + return asCliProcess(child); + }, + }); + + await expect(session.run(["stuck"])).rejects.toThrow(/did not respond/); + const replacement = session.run(["next"]); + await expect(replacement).rejects.toThrow(/stayed running after SIGKILL/); + await expect(session.close()).rejects.toThrow(/stayed running after SIGKILL/); + expect(signals).toEqual(["SIGKILL", "SIGKILL"]); + const stalledChild = child; + if (stalledChild === undefined) throw new Error("missing stalled worker"); + stalledChild.kill = (signal: NodeJS.Signals = "SIGKILL") => { + signals.push(signal); + stalledChild.signalCode = signal; + stalledChild.emit("exit", null, signal); + stalledChild.emit("close", null, signal); + return true; + }; + await session.close(); + expect(stalledChild.signalCode).toBe("SIGKILL"); +}); + +test("reports a compiled CLI session that needed a signal after stdin closed", async () => { + let child: FakeCliProcess | undefined; + const signals: NodeJS.Signals[] = []; + const session = startCompiledCliSession({ + stopTimeoutMs: 20, + spawnWorker: () => { + child = fakeProcess(81); + child.kill = (signal: NodeJS.Signals = "SIGTERM") => { + signals.push(signal); + if (signal !== "SIGKILL") return true; + if (child?.exitCode !== null || child?.signalCode !== null) return true; + child.signalCode = signal; + child.emit("exit", null, signal); + child.emit("close", null, signal); + return true; + }; + ready(child); + replyWhenAsked(child, JSON.stringify({ ok: true })); + return asCliProcess(child); + }, + }); + + await expect(session.run(["once"])).resolves.toEqual({ ok: true }); + await expect(session.close()).rejects.toThrow( + "Compiled CLI session did not exit after stdin closed", + ); + expect(signals).toEqual(["SIGTERM", "SIGKILL"]); + expect(child?.signalCode).toBe("SIGKILL"); +}); + +test("closes a compiled CLI session that exits after stdin closes", async () => { + const session = startCompiledCliSession({ + spawnWorker: () => { + const child = fakeProcess(82); + exitWhenStdinCloses(child); + ready(child); + replyWhenAsked(child, JSON.stringify({ ok: true })); + return asCliProcess(child); + }, + }); + + await expect(session.run(["once"])).resolves.toEqual({ ok: true }); + await session.close(); +}); + +test("keeps a compiled CLI failure status and serves the next command", async () => { + const session = startCompiledCliSession(); + onTestFinished(() => session.close()); + const root = await createTestTempDirectory("rea-warm-cli-status-"); + await writeFile( + join(root, "app.mjs"), + "export function current() { return 1; }\n", + ); + + await expect( + session.run(["compare-javascript-export-shapes", "{", "--json"]), + ).rejects.toThrow(/status 1:.*invalid_request/); + await expect( + session.run(["analyze-javascript-application", root, "--json"]), + ).resolves.toMatchObject({ + normalized_result: expect.any(Object), + }); + await session.close(); +}); diff --git a/tests/boundary/cli/desktopApplicationCli.test.ts b/tests/boundary/cli/desktopApplicationCli.test.ts index c0f4a6e3e8..52026748e6 100644 --- a/tests/boundary/cli/desktopApplicationCli.test.ts +++ b/tests/boundary/cli/desktopApplicationCli.test.ts @@ -1,5 +1,5 @@ import { readEvidenceBundle } from "../../../src/application/EvidenceBundleFiles.js"; -import { mkdir, writeFile } from "node:fs/promises"; +import { chmod, mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect } from "vitest"; import { cliTest } from "../../support/cli/cliFixture.js"; @@ -109,3 +109,60 @@ cliTest( expect(overwrite.exitCode).not.toBe(0); }, ); + +cliTest( + "refuses a non-directory or missing root as a target selection", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-desktop-cli-root-"); + const file = join(root, "regular-file"); + await writeFile(file, "fixture\n"); + const notDirectory = await cli.run({ + arguments: ["inspect-desktop-application", file, "--json"], + }); + expect(notDirectory.exitCode).toBe(1); + expect(notDirectory.json).toMatchObject({ + code: "unsupported_target", + category: "unsupported_target", + details: { operation: "inspect_desktop_application", path: file }, + }); + const missingPath = join(root, "missing-directory"); + const missing = await cli.run({ + arguments: ["inspect-desktop-application", missingPath, "--json"], + }); + expect(missing.exitCode).toBe(1); + expect(missing.json).toMatchObject({ + code: "target_unavailable", + category: "unavailable", + details: { path: missingPath, reason: expect.stringContaining("ENOENT") }, + }); + }, +); + +cliTest.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "reports a read-denied root as host access denial with read-access recovery", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-desktop-cli-denied-"); + const locked = join(root, "locked-directory"); + await mkdir(locked); + await chmod(locked, 0o000); + try { + const denied = await cli.run({ + arguments: ["inspect-desktop-application", locked, "--json"], + }); + expect(denied.exitCode).toBe(1); + expect(denied.json).toMatchObject({ + code: "access_denied", + category: "unavailable", + remediation: { action: expect.stringContaining("read access") }, + details: { + path: locked, + system_code: "EACCES", + boundary: "filesystem-read", + partial_observation: expect.any(Object), + }, + }); + } finally { + await chmod(locked, 0o700); + } + }, +); diff --git a/tests/boundary/cli/doctorCancellation.test.ts b/tests/boundary/cli/doctorCancellation.test.ts index 530427dcc6..7ebcfeae86 100644 --- a/tests/boundary/cli/doctorCancellation.test.ts +++ b/tests/boundary/cli/doctorCancellation.test.ts @@ -40,13 +40,11 @@ cliTest.skipIf(!["darwin", "linux"].includes(process.platform)).for( const executable = provider === "kuna" ? join(root, "kuna") : join(jdk, "bin", "java"); const probeArgument = provider === "kuna" ? "--version" : "--list-modules"; - const runIdField = - provider === "jadx" ? ",runId:process.env.REA_PROCESS_RUN_ID" : ""; await writeFile( executable, `#!${process.execPath} if (!process.argv.slice(2).includes(${JSON.stringify(probeArgument)})) process.exit(7); -require("node:fs").writeFileSync(${JSON.stringify(ready)}, JSON.stringify({pid: process.pid${runIdField}})); +require("node:fs").writeFileSync(${JSON.stringify(ready)}, JSON.stringify({pid: process.pid, runId: process.env.REA_PROCESS_RUN_ID})); process.on("SIGTERM", () => {}); setInterval(() => { if (require("node:fs").existsSync(${JSON.stringify(release)})) process.exit(0); }, 25).unref(); setTimeout(() => process.exit(99), 20000); @@ -97,7 +95,10 @@ setTimeout(() => process.exit(99), 20000); ) .toBe(true); const readyRecord: unknown = JSON.parse(await readFile(ready, "utf8")); - const observed: { readonly pid: number; readonly runId?: string } = + // JADX publishes a UUID. Kuna publishes an opaque + // `rea-kuna-version-` token. Both are required: JSON.stringify + // drops an undefined REA_PROCESS_RUN_ID, and that omission must fail. + const observed = provider === "jadx" ? z .strictObject({ @@ -106,11 +107,12 @@ setTimeout(() => process.exit(99), 20000); }) .parse(readyRecord) : z - .strictObject({ pid: z.number().int().positive() }) + .strictObject({ + pid: z.number().int().positive(), + runId: z.string().min(1), + }) .parse(readyRecord); if (provider === "jadx") { - if (observed.runId === undefined) - throw new Error("Missing JADX run token"); ownership = { leaderPid: observed.pid, processGroupId: observed.pid, @@ -131,8 +133,7 @@ setTimeout(() => process.exit(99), 20000); state: "empty", }); } else { - // Kuna's version probe uses a joined direct execFile child, without a - // detached run-token group. Do not invent a group owner from its PID. + // Do not invent a group owner from the probe PID. expect(() => process.kill(observed.pid, 0)).toThrowError( expect.objectContaining({ code: "ESRCH" }), ); diff --git a/tests/boundary/cli/dylibResolutionCli.test.ts b/tests/boundary/cli/dylibResolutionCli.test.ts index 128c51d161..80ca7260e3 100644 --- a/tests/boundary/cli/dylibResolutionCli.test.ts +++ b/tests/boundary/cli/dylibResolutionCli.test.ts @@ -17,7 +17,11 @@ import { import { dylibResolutionResultSchema } from "../../../src/domain/apple/dylibResolution.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliTest, type TestCli } from "../../support/cli/cliFixture.js"; +import { + cliTest, + runFileCli, + type TestCli, +} from "../../support/cli/cliFixture.js"; const ENVIRONMENT = { REA_LOG_LEVEL: "silent", REA_ANALYSIS_PROVIDER: "auto" }; @@ -46,7 +50,7 @@ const expectRootInputError = async ( readonly message: string; }, ): Promise => { - const rejected = await cli.run({ + const rejected = await runFileCli(cli, { arguments: ["trace-dylib-resolution", app, "--root", root, "--json"], environment: ENVIRONMENT, }); @@ -127,7 +131,7 @@ cliTest( ], }), }); - const output = await cli.run({ + const output = await runFileCli(cli, { arguments: ["trace-dylib-resolution", app, "--json"], environment: ENVIRONMENT, }); @@ -160,7 +164,7 @@ cliTest( describe("trace-dylib-resolution CLI", () => { cliTest("traces every executable root of an app bundle", async ({ cli }) => { const app = await fixtureApp(); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["trace-dylib-resolution", app, "--json"], environment: ENVIRONMENT, }); @@ -211,7 +215,7 @@ describe("trace-dylib-resolution CLI", () => { "narrows roots and architecture, and rejects escaping roots", async ({ cli }) => { const app = await fixtureApp(); - const narrowed = await cli.run({ + const narrowed = await runFileCli(cli, { arguments: [ "trace-dylib-resolution", app, @@ -267,7 +271,7 @@ describe("trace-dylib-resolution CLI", () => { }), "unrelated.txt": "not traversed", }); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: [ "trace-dylib-resolution", join(directory, "tool"), @@ -292,7 +296,7 @@ describe("trace-dylib-resolution CLI", () => { async ({ cli }) => { const app = await fixtureApp(); await withBrokenHelper(app); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["trace-dylib-resolution", app, "--json"], environment: ENVIRONMENT, }); @@ -343,7 +347,7 @@ cliTest( }); } - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["trace-dylib-resolution", app, "--json"], environment: ENVIRONMENT, }); @@ -400,7 +404,7 @@ cliTest( commands: [buildVersionCommand(1)], }), }); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["trace-dylib-resolution", join(directory, "tool"), "--json"], environment: ENVIRONMENT, }); @@ -450,7 +454,7 @@ describe.skipIf(process.platform === "win32" || process.getuid?.() === 0)( }); await chmod(join(directory, "libsecret.dylib"), 0o000); try { - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: [ "trace-dylib-resolution", join(directory, "tool"), @@ -498,7 +502,7 @@ cliTest( ], }), }); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: [ "trace-dylib-resolution", app, @@ -559,7 +563,7 @@ cliTest( { cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) }, ]), }); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: [ "trace-dylib-resolution", app, @@ -609,7 +613,7 @@ cliTest( commands: [buildVersionCommand(1)], }), }); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: [ "trace-dylib-resolution", app, @@ -663,7 +667,7 @@ cliTest( ], }), ); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["trace-dylib-resolution", path, "--json"], environment: ENVIRONMENT, }); @@ -711,7 +715,7 @@ describe("trace-dylib-resolution root enumeration", () => { "classifies every Mach-O among more files than one batch of magic reads", async ({ cli }) => { const app = await crowdedApp(); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["trace-dylib-resolution", app, "--json"], environment: ENVIRONMENT, }); @@ -742,7 +746,7 @@ describe("trace-dylib-resolution root enumeration", () => { await writeFile(secret, "x"); await chmod(secret, 0o000); try { - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["trace-dylib-resolution", app, "--json"], environment: ENVIRONMENT, }); diff --git a/tests/boundary/cli/environmentExportCli.test.ts b/tests/boundary/cli/environmentExportCli.test.ts index 0cb1d7ffe8..434100782c 100644 --- a/tests/boundary/cli/environmentExportCli.test.ts +++ b/tests/boundary/cli/environmentExportCli.test.ts @@ -2,6 +2,7 @@ import { mkdir, readdir, readFile, + realpath, stat, symlink, writeFile, @@ -299,3 +300,57 @@ cliTest.skipIf(process.platform === "win32")( ).rejects.toMatchObject({ code: "ENOENT" }); }, ); + +const HOST_PATH_MAX: Partial> = { + darwin: 1024, + linux: 4096, +}; + +cliTest.skipIf(HOST_PATH_MAX[process.platform] === undefined)( + "reports a later nested-file pathname-limit failure as a host write failure and keeps the partial folder", + async ({ cli }) => { + const limit = HOST_PATH_MAX[process.platform] ?? 0; + const root = await realpath( + await createTestTempDirectory("rea-export-long-path-"), + ); + const files = await writeEnvironmentServeFiles(root); + // Shorter bundle paths fit; the 41-byte nested verifier path reaches the host limit. + const length = limit - 40; + let parent = join(root, "long"); + while (Buffer.byteLength(parent) + 1 + 200 < length - 1) + parent = join(parent, "d".repeat(200)); + await mkdir(parent, { recursive: true }); + const directory = join( + parent, + "l".repeat(length - Buffer.byteLength(parent) - 1), + ); + expect(Buffer.byteLength(directory)).toBe(length); + const failed = join(directory, "artifacts/check_grafana-example/verify.py"); + const result = await cli.run({ + arguments: [ + "environment-export", + files.manifestPath, + "--evidence", + files.capturePath, + directory, + "--json", + ], + timeoutMs: 30_000, + }); + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + code: "unreadable_output", + remediation: { action: expect.stringContaining(directory) }, + details: { + operation: "environment-export", + diagnostics: { output_directory: directory }, + }, + }); + expect(result.json).not.toHaveProperty("details.issues"); + const reason = (result.json as { details: { reason: string } }).details + .reason; + expect(reason).toContain("ENAMETOOLONG"); + expect(reason).toContain(failed); + expect(await readFile(join(directory, "README.md"))).not.toHaveLength(0); + }, +); diff --git a/tests/boundary/cli/environmentExportInputsCli.test.ts b/tests/boundary/cli/environmentExportInputsCli.test.ts new file mode 100644 index 0000000000..17ebf3cc53 --- /dev/null +++ b/tests/boundary/cli/environmentExportInputsCli.test.ts @@ -0,0 +1,193 @@ +import { + appendFile, + readFile, + stat, + truncate, + writeFile, +} from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { expect } from "vitest"; +import { z } from "zod"; + +import { nativeJsonParseRequiredBytes } from "../../../src/application/JsonInputFile.js"; +import { writeEnvironmentServeFiles } from "../../fixtures/environmentServe.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const probe = pathToFileURL( + resolve("tests/fixtures/environmentExportInputProbe.mjs"), +).href; + +for (const role of ["manifest", "evidence"] as const) { + cliTest( + `refuses a real ${role} file changed during its read`, + async ({ cli }) => { + const root = await createTestTempDirectory("rea-export-input-race-cli-"); + const files = await writeEnvironmentServeFiles(root); + const input = + role === "manifest" ? files.manifestPath : files.capturePath; + const before = await readFile(input); + const marker = join(root, "read-marker"); + const directory = join(root, "bundle"); + const result = await cli.run({ + arguments: [ + "environment-export", + files.manifestPath, + "--evidence", + files.capturePath, + directory, + "--json", + ], + environment: { + NODE_OPTIONS: `--import=${probe} --max-old-space-size=256`, + REA_TEST_EXPORT_INPUT: input, + REA_TEST_EXPORT_READ_MARKER: marker, + REA_TEST_EXPORT_READ_ACTION: "append", + }, + timeoutMs: 30_000, + }); + expect(await readFile(marker, "utf8")).toBe("read"); + expect(await readFile(input)).toEqual( + Buffer.concat([before, Buffer.from(" ")]), + ); + expect(result.exitCode, result.stderr).toBe(1); + expect(result.json).toMatchObject({ + code: "invalid_request", + details: { + operation: "environment-export", + issues: [{ path: [role], reason: "invalid_value" }], + }, + }); + await expect(stat(directory)).rejects.toMatchObject({ code: "ENOENT" }); + }, + ); +} + +const heapRefusalSchema = z.object({ + code: z.literal("resource_constraint"), + details: z.object({ + operation: z.literal("environment-export"), + resource: z.literal("memory"), + reported_limits: z.object({ + input_path: z.string(), + input_file_bytes: z.number(), + required_heap_bytes: z.number(), + available_heap_bytes: z.number(), + }), + }), +}); + +cliTest( + "admits neither the whole read nor an unbudgeted snapshot under a real heap limit", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-export-input-heap-cli-"); + const files = await writeEnvironmentServeFiles(root); + const originalBytes = (await stat(files.manifestPath)).size; + const marker = join(root, "read-marker"); + const directory = join(root, "bundle"); + // Whitespace preserves a valid manifest while making the actual file large. + await appendFile(files.manifestPath, Buffer.alloc(8 * 1024 * 1024, " ")); + let available: number | undefined; + for (const phase of ["whole-read", "snapshot"] as const) { + if (available !== undefined) { + // Fit the native parse in two thirds of the previous child's real + // headroom, while the additional snapshot reservation cannot fit. + const overhead = nativeJsonParseRequiredBytes(0); + const perByte = nativeJsonParseRequiredBytes(1) - overhead; + await truncate( + files.manifestPath, + Math.max( + originalBytes, + Math.floor(((available * 2) / 3 - overhead) / perByte), + ), + ); + } + const result = await cli.run({ + arguments: [ + "environment-export", + files.manifestPath, + "--evidence", + files.capturePath, + directory, + "--json", + ], + environment: { + NODE_OPTIONS: `--import=${probe} --max-old-space-size=256`, + REA_TEST_EXPORT_INPUT: files.manifestPath, + REA_TEST_EXPORT_READ_MARKER: marker, + REA_TEST_EXPORT_READ_ACTION: "record", + }, + timeoutMs: 30_000, + }); + expect(result.exitCode, result.stderr).toBe(1); + const limits = heapRefusalSchema.parse(result.json).details + .reported_limits; + expect(limits.input_path).toBe(files.manifestPath); + expect(limits.input_file_bytes).toBe( + (await stat(files.manifestPath)).size, + ); + expect(limits.required_heap_bytes).toBe( + nativeJsonParseRequiredBytes(limits.input_file_bytes, true), + ); + expect(limits.required_heap_bytes).toBeGreaterThan( + limits.available_heap_bytes, + ); + if (phase === "snapshot") + expect( + nativeJsonParseRequiredBytes(limits.input_file_bytes), + ).toBeLessThan(limits.available_heap_bytes); + else + expect( + nativeJsonParseRequiredBytes(limits.input_file_bytes), + ).toBeGreaterThan(limits.available_heap_bytes); + available = limits.available_heap_bytes; + await expect(stat(marker)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(stat(directory)).rejects.toMatchObject({ code: "ENOENT" }); + } + }, +); + +cliTest( + "attributes a pathless descriptor failure during the real tarball copy", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-export-copy-read-cli-"); + const files = await writeEnvironmentServeFiles(root); + const tarball = join(root, "package.tgz"); + await writeFile(tarball, "admitted package"); + const marker = join(root, "read-marker"); + const directory = join(root, "bundle"); + const result = await cli.run({ + arguments: [ + "environment-export", + files.manifestPath, + "--evidence", + files.capturePath, + directory, + "--package-tarball", + tarball, + "--json", + ], + environment: { + NODE_OPTIONS: `--import=${probe} --max-old-space-size=256`, + REA_TEST_EXPORT_INPUT: tarball, + REA_TEST_EXPORT_READ_MARKER: marker, + REA_TEST_EXPORT_READ_ACTION: "fail-copy", + }, + timeoutMs: 30_000, + }); + expect(await readFile(marker, "utf8")).toBe("copy"); + expect(await readFile(tarball, "utf8")).toBe("admitted package"); + expect(result.exitCode, result.stderr).toBe(1); + expect(result.json).toMatchObject({ + code: "unreadable_output", + details: { + reason: expect.stringContaining(`reading ${tarball}`), + diagnostics: { output_directory: directory }, + }, + remediation: { action: expect.stringContaining(tarball) }, + }); + expect(JSON.stringify(result.json)).not.toContain("disk space"); + }, +); diff --git a/tests/boundary/cli/nativePlistCli.test.ts b/tests/boundary/cli/nativePlistCli.test.ts index 3c6f69dff6..9f47e202e1 100644 --- a/tests/boundary/cli/nativePlistCli.test.ts +++ b/tests/boundary/cli/nativePlistCli.test.ts @@ -228,6 +228,7 @@ describe.skipIf(process.platform !== "darwin")( "CFBundleExecutableApp", "CFBundleIdentifiercom.example.types", "BlobAAEC", + "Separatorsa\u0085b\u2028c\u2029d", "Built2020-01-01T00:00:00Z", // Parse the unsigned extended year as a UTC field, without JS string-date inference. "Far11507-08-17T05:20:00Z", @@ -261,6 +262,7 @@ describe.skipIf(process.platform !== "darwin")( value: { CFBundleIdentifier: "com.example.types", Blob: { $plist_type: "data", base64: "AAEC" }, + Separators: "a\u0085b\u2028c\u2029d", Built: { $plist_type: "date", iso8601: "2020-01-01T00:00:00.000Z", @@ -279,15 +281,27 @@ describe.skipIf(process.platform !== "darwin")( ], }, bundle: { identifier: "com.example.types", executable: "App" }, - provenance: [ - { tool: "file" }, - { command: expect.arrayContaining(["json"]), exit: { code: 1 } }, - { command: expect.arrayContaining(["xml1"]), exit: { code: 0 } }, - ], - limitations: [ - expect.stringContaining("XML conversion"), - expect.stringContaining("2 non-finite real"), - ], + provenance: + format === "xml" + ? [ + { tool: "file" }, + { + command: expect.arrayContaining(["json"]), + exit: { code: 1 }, + }, + { + command: expect.arrayContaining(["xml1"]), + exit: { code: 0 }, + }, + ] + : [{ tool: "file" }], + limitations: + format === "xml" + ? [ + expect.stringContaining("XML conversion"), + expect.stringContaining("2 non-finite real"), + ] + : [expect.stringContaining("2 non-finite real")], }, }); }, diff --git a/tests/boundary/cli/sharedJavaScriptCliSession.test.ts b/tests/boundary/cli/sharedJavaScriptCliSession.test.ts new file mode 100644 index 0000000000..3794f70486 --- /dev/null +++ b/tests/boundary/cli/sharedJavaScriptCliSession.test.ts @@ -0,0 +1,577 @@ +import { EventEmitter, PassThrough } from "node:stream"; +import type { ChildProcessWithoutNullStreams } from "node:child_process"; + +import { expect, test } from "vitest"; +import { z } from "zod"; + +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { + cliTest, + type CliInvocation, + type CliResult, +} from "../../support/cli/cliFixture.js"; +import { createCliCommandSession } from "../../support/cli/cliCommandSession.js"; +import { + createCliSessionFrameReader, + encodeCliSessionFrame, + MAX_CLI_SESSION_FRAME_BYTES, +} from "../../support/cli/cliCommandSessionFrames.mjs"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/workspace/writeFixtureFiles.js"; + +const requestSchema = z.object({ + id: z.number(), + arguments: z.array(z.string()), +}); + +const command = ( + arguments_: readonly string[], + environment: Readonly>, +): CliInvocation & { + cwd: string; + environment: Readonly>; +} => ({ + arguments: arguments_, + cwd: process.cwd(), + environment, +}); + +interface FakeCliProcess extends EventEmitter { + stdin: PassThrough; + stdout: PassThrough; + stderr: PassThrough; + pid: number; + exitCode: number | null; + signalCode: NodeJS.Signals | null; + kill(signal?: NodeJS.Signals): boolean; +} + +const fakeProcess = (pid: number): FakeCliProcess => { + const child = new EventEmitter() as FakeCliProcess; + child.stdin = new PassThrough(); + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + child.pid = pid; + child.exitCode = null; + child.signalCode = null; + child.kill = (signal: NodeJS.Signals = "SIGTERM") => { + if (child.exitCode !== null || child.signalCode !== null) return true; + child.signalCode = signal; + child.emit("exit", null, signal); + child.emit("close", null, signal); + return true; + }; + return child; +}; + +const asCliProcess = (child: FakeCliProcess): ChildProcessWithoutNullStreams => + child as unknown as ChildProcessWithoutNullStreams; + +const waitFor = async (ready: () => boolean): Promise => { + for (let attempt = 0; attempt < 30; attempt += 1) { + if (ready()) return; + await new Promise((resolve) => { + setImmediate(resolve); + }); + } + throw new Error("shared CLI process did not reach the expected state"); +}; + +test("reads a CLI session frame split across chunks", () => { + const frames: unknown[] = []; + const reader = createCliSessionFrameReader((frame) => { + frames.push(frame); + }); + const encoded = encodeCliSessionFrame({ ready: true, seq: 1 }); + const second = encodeCliSessionFrame({ id: 2 }); + const combined = Buffer.concat([encoded, second]); + reader(combined.subarray(0, 1)); + expect(frames).toEqual([]); + reader(combined.subarray(1, 3)); + expect(frames).toEqual([]); + reader(combined.subarray(3)); + expect(frames).toEqual([{ ready: true, seq: 1 }, { id: 2 }]); +}); + +test("rejects an oversized CLI session frame", () => { + const reader = createCliSessionFrameReader(() => undefined); + const header = Buffer.alloc(4); + header.writeUInt32BE(MAX_CLI_SESSION_FRAME_BYTES + 1); + expect(() => reader(header)).toThrow(/limit/); +}); + +test("reads a CLI session frame delivered one byte at a time", () => { + const frames: unknown[] = []; + const reader = createCliSessionFrameReader((frame) => { + frames.push(frame); + }); + const encoded = encodeCliSessionFrame({ + id: 7, + note: "x".repeat(64), + }); + for (const byte of encoded) reader(Buffer.from([byte])); + expect(frames).toEqual([{ id: 7, note: "x".repeat(64) }]); +}); + +test("stops an unready shared CLI process when the session closes", async () => { + let spawned: FakeCliProcess | undefined; + const session = createCliCommandSession({ + spawnWorker: () => { + spawned = fakeProcess(41); + return asCliProcess(spawned); + }, + }); + const pending = session.run( + command(["--version"], { PATH: "/usr/bin", REA_LOG_LEVEL: "silent" }), + ); + await waitFor(() => spawned !== undefined); + await session.close(); + await expect(pending).rejects.toThrow("Shared CLI session is closed"); + expect(spawned?.signalCode).toBe("SIGTERM"); + await expect( + session.run( + command(["--version"], { PATH: "/usr/bin", REA_LOG_LEVEL: "silent" }), + ), + ).rejects.toThrow("Shared CLI session is closed"); + await session.close(); +}); + +test("stops an in-flight shared CLI command when the session closes", async () => { + let received = false; + let spawned: FakeCliProcess | undefined; + const session = createCliCommandSession({ + spawnWorker: () => { + const child = fakeProcess(42); + spawned = child; + child.stdout.write(encodeCliSessionFrame({ ready: true })); + child.stdin.on("data", () => { + received = true; + }); + return asCliProcess(child); + }, + }); + const pending = session.run( + command(["--version"], { PATH: "/usr/bin", REA_LOG_LEVEL: "silent" }), + ); + await waitFor(() => received); + await session.close(); + await expect(pending).rejects.toThrow("Shared CLI session is closed"); + expect(spawned?.signalCode).toBe("SIGTERM"); +}); + +test("reuses one process and restarts it when the environment changes", async () => { + let spawns = 0; + const session = createCliCommandSession({ + spawnWorker: (environment) => { + spawns += 1; + const child = fakeProcess(50 + spawns); + const reader = createCliSessionFrameReader((value) => { + const request = requestSchema.parse(value); + child.stdout.write( + encodeCliSessionFrame({ + id: request.id, + exitCode: 0, + stdout: request.arguments.join(" "), + stderr: environment.MARKER ?? "", + }), + ); + }); + child.stdout.write(encodeCliSessionFrame({ ready: true })); + child.stdin.on("data", (chunk) => { + reader(chunk); + }); + return asCliProcess(child); + }, + }); + const first = await session.run( + command(["one"], { PATH: "a", MARKER: "first" }), + ); + const second = await session.run( + command(["two"], { PATH: "a", MARKER: "first" }), + ); + expect(spawns).toBe(1); + expect(session.processId()).toBe(51); + expect(first.stdout).toBe("one"); + expect(second.stdout).toBe("two"); + expect(first.stderr).toBe("first"); + expect(second.stderr).toBe("first"); + + const changed = await session.run( + command(["three"], { PATH: "b", MARKER: "second" }), + ); + expect(spawns).toBe(2); + expect(session.processId()).toBe(52); + expect(changed.stdout).toBe("three"); + expect(changed.stderr).toBe("second"); + await session.close(); + expect(session.processId()).toBeUndefined(); +}); + +const reply = ( + child: FakeCliProcess, + id: number, + fields: { + exitCode?: number; + stdout?: string; + stderr?: string; + rss?: number; + fatal?: boolean; + }, +): void => { + child.stdout.write( + encodeCliSessionFrame({ + id, + exitCode: fields.exitCode ?? 0, + stdout: fields.stdout ?? "", + stderr: fields.stderr ?? "", + ...(fields.rss === undefined ? {} : { rss: fields.rss }), + ...(fields.fatal === undefined ? {} : { fatal: fields.fatal }), + }), + ); +}; + +const respondingProcess = ( + pid: number, + onRequest: ( + request: { id: number; arguments: string[] }, + child: FakeCliProcess, + ) => void, +): FakeCliProcess => { + const child = fakeProcess(pid); + child.stdout.write(encodeCliSessionFrame({ ready: true })); + const reader = createCliSessionFrameReader((value) => { + onRequest(requestSchema.parse(value), child); + }); + child.stdin.on("data", (chunk: Buffer) => { + reader(chunk); + }); + return child; +}; + +test("starts another process after the shared CLI process exits", async () => { + const children: FakeCliProcess[] = []; + const session = createCliCommandSession({ + spawnWorker: () => { + const child = fakeProcess(70 + children.length); + children.push(child); + child.stdout.write(encodeCliSessionFrame({ ready: true })); + if (children.length === 1) { + child.stdin.on("data", () => { + child.kill("SIGTERM"); + }); + } else { + const reader = createCliSessionFrameReader((value) => { + const request = requestSchema.parse(value); + child.stdout.write( + encodeCliSessionFrame({ + id: request.id, + exitCode: 0, + stdout: "recovered", + stderr: "", + }), + ); + }); + child.stdin.on("data", (chunk) => { + reader(chunk); + }); + } + return asCliProcess(child); + }, + }); + await expect( + session.run(command(["fail"], { PATH: "/usr/bin" })), + ).rejects.toThrow(/Shared CLI process exited/); + await expect( + session.run(command(["again"], { PATH: "/usr/bin" })), + ).resolves.toMatchObject({ + exitCode: 0, + stdout: "recovered", + }); + expect(children).toHaveLength(2); + await session.close(); +}); + +test("retires a fatal worker before starting the next command", async () => { + const events: string[] = []; + const session = createCliCommandSession({ + spawnWorker: () => { + const generation = events.filter((event) => + event.startsWith("spawn"), + ).length; + events.push(`spawn:${generation}`); + const child = respondingProcess(80 + generation, (request, current) => { + events.push(`command:${generation}`); + if (generation === 0) { + reply(current, request.id, { + exitCode: 1, + stderr: "boom", + fatal: true, + }); + return; + } + reply(current, request.id, { stdout: "next" }); + }); + const stop = child.kill.bind(child); + child.kill = (signal?: NodeJS.Signals) => { + events.push(`kill:${generation}`); + return stop(signal); + }; + return asCliProcess(child); + }, + }); + const fatal = session.run(command(["fail"], { PATH: "/usr/bin" })); + const next = session.run(command(["next"], { PATH: "/usr/bin" })); + await expect(fatal).resolves.toMatchObject({ exitCode: 1, stderr: "boom" }); + await expect(next).resolves.toMatchObject({ exitCode: 0, stdout: "next" }); + expect(events).toEqual([ + "spawn:0", + "command:0", + "kill:0", + "spawn:1", + "command:1", + ]); + await session.close(); +}); + +test("retires a transport failure before the next command reuses the worker", async () => { + const events: string[] = []; + const session = createCliCommandSession({ + spawnWorker: () => { + const generation = events.filter((event) => + event.startsWith("spawn"), + ).length; + events.push(`spawn:${generation}`); + const child = respondingProcess(120 + generation, (request, current) => { + events.push(`command:${generation}`); + if (generation === 0) { + current.stdout.write(encodeCliSessionFrame({ id: request.id })); + return; + } + reply(current, request.id, { stdout: "recovered" }); + }); + const stop = child.kill.bind(child); + child.kill = (signal: NodeJS.Signals = "SIGTERM") => { + events.push(`kill:${generation}`); + // Leave signalCode unset until a later turn. A synchronous SIGTERM + // does not update it, so reuse cannot depend on that field. + queueMicrotask(() => { + if (child.exitCode !== null || child.signalCode !== null) return; + stop(signal); + }); + return true; + }; + return asCliProcess(child); + }, + }); + const failed = session.run(command(["bad"], { PATH: "/usr/bin" })); + const next = session.run(command(["next"], { PATH: "/usr/bin" })); + await expect(failed).rejects.toThrow(/exit code/); + await expect(next).resolves.toMatchObject({ stdout: "recovered" }); + expect(events).toEqual([ + "spawn:0", + "command:0", + "kill:0", + "spawn:1", + "command:1", + ]); + await session.close(); +}); + +test("restarts the shared CLI process after the command limit", async () => { + const children: FakeCliProcess[] = []; + const session = createCliCommandSession({ + commandLimit: 2, + spawnWorker: () => { + const child = respondingProcess( + 90 + children.length, + (request, current) => { + reply(current, request.id, { + stdout: String(children.indexOf(current)), + }); + }, + ); + children.push(child); + return asCliProcess(child); + }, + }); + const first = await session.run(command(["one"], { PATH: "/usr/bin" })); + const second = await session.run(command(["two"], { PATH: "/usr/bin" })); + expect(children).toHaveLength(1); + expect(first.stdout).toBe("0"); + expect(second.stdout).toBe("0"); + const third = await session.run(command(["three"], { PATH: "/usr/bin" })); + expect(children).toHaveLength(2); + expect(children[0]?.signalCode).toBe("SIGTERM"); + expect(third.stdout).toBe("1"); + expect(session.processId()).toBe(91); + await session.close(); +}); + +test("restarts the shared CLI process after the resident-set limit", async () => { + const children: FakeCliProcess[] = []; + const session = createCliCommandSession({ + commandLimit: 100, + residentLimitBytes: 100, + spawnWorker: () => { + const generation = children.length; + const child = respondingProcess(100 + generation, (request, current) => { + reply(current, request.id, { + stdout: String(generation), + rss: generation === 0 ? 150 : 10, + }); + }); + children.push(child); + return asCliProcess(child); + }, + }); + await expect( + session.run(command(["one"], { PATH: "/usr/bin" })), + ).resolves.toMatchObject({ stdout: "0" }); + expect(session.processId()).toBe(100); + await expect( + session.run(command(["two"], { PATH: "/usr/bin" })), + ).resolves.toMatchObject({ stdout: "1" }); + expect(children[0]?.signalCode).toBe("SIGTERM"); + expect(session.processId()).toBe(101); + await session.close(); +}); + +test("starts a new process after the real worker reports a fatal failure", async () => { + const session = createCliCommandSession(); + const failed = await session.run({ + arguments: ["--version"], + cwd: "/workspace/this-directory-does-not-exist", + environment: { PATH: process.env.PATH ?? "", REA_LOG_LEVEL: "silent" }, + }); + expect(failed.exitCode).toBe(1); + expect(failed.stderr).toMatch(/ENOENT/); + const recovered = await session.run({ + arguments: ["--version"], + cwd: process.cwd(), + environment: { PATH: process.env.PATH ?? "", REA_LOG_LEVEL: "silent" }, + }); + expect(recovered.exitCode).toBe(0); + expect(recovered.stdout.length).toBeGreaterThan(0); + expect(session.processId()).toEqual(expect.any(Number)); + await session.close(); +}); + +const requireSharedCli = (cli: { + sharedCliRun?: (invocation: CliInvocation) => Promise; +}): ((invocation: CliInvocation) => Promise) => { + const run = cli.sharedCliRun; + if (run === undefined) + throw new Error("cliTest did not provide a shared CLI session"); + return run; +}; + +const normalized = (result: CliResult) => + z + .object({ normalized_result: javascriptApplicationAnalysisResultSchema }) + .parse(result.json).normalized_result; + +const analyzeArguments = (root: string): readonly string[] => [ + "analyze-javascript-application", + root, + "--format", + "json", +]; + +const writeApplication = async (prefix: string): Promise => { + const root = await createTestTempDirectory(prefix); + await writeFixtureFiles(root, { + "package.json": JSON.stringify({ main: "main.cjs" }), + "main.cjs": + 'const { BrowserWindow } = require("electron"); new BrowserWindow().loadFile("index.html");', + "index.html": "page", + }); + return root; +}; + +cliTest( + "matches a fresh CLI process and reuses that process for the same environment", + async ({ cli, cliCommandSession }) => { + const sharedCliRun = requireSharedCli(cli); + const fresh = await cli.run({ + arguments: ["--version"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + const shared = await sharedCliRun({ + arguments: ["--version"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(shared.exitCode).toBe(fresh.exitCode); + expect(shared.stdout).toBe(fresh.stdout); + expect(shared.stderr).toBe(fresh.stderr); + const pid = cliCommandSession.processId(); + expect(pid).toEqual(expect.any(Number)); + const again = await sharedCliRun({ + arguments: ["--version"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(again.stdout).toBe(fresh.stdout); + expect(cliCommandSession.processId()).toBe(pid); + + const freshInvalid = await cli.run({ + arguments: ["--version"], + environment: { REA_LOG_LEVEL: "not-a-level" }, + }); + const sharedInvalid = await sharedCliRun({ + arguments: ["--version"], + environment: { REA_LOG_LEVEL: "not-a-level" }, + }); + expect(sharedInvalid.exitCode).toBe(freshInvalid.exitCode); + expect(sharedInvalid.exitCode).not.toBe(0); + expect(sharedInvalid.stdout).toBe(freshInvalid.stdout); + expect(sharedInvalid.stderr).toBe(freshInvalid.stderr); + expect(cliCommandSession.processId()).not.toBe(pid); + + const restored = await sharedCliRun({ + arguments: ["--version"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(restored.exitCode).toBe(0); + expect(restored.stdout).toBe(fresh.stdout); + expect(cliCommandSession.processId()).not.toBe(pid); + }, +); + +cliTest( + "reuses one CLI process across JavaScript application directories", + async ({ cli, cliCommandSession }) => { + const sharedCliRun = requireSharedCli(cli); + const firstRoot = await writeApplication("rea-shared-cli-"); + const fresh = await cli.run({ + arguments: analyzeArguments(firstRoot), + environment: { REA_LOG_LEVEL: "silent" }, + }); + const shared = await sharedCliRun({ + arguments: analyzeArguments(firstRoot), + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(shared.exitCode).toBe(0); + expect(normalized(shared)).toEqual(normalized(fresh)); + const pid = cliCommandSession.processId(); + expect(pid).toEqual(expect.any(Number)); + + const secondRoot = await writeApplication("rea-shared-cli-next-"); + const freshNext = await cli.run({ + arguments: analyzeArguments(secondRoot), + environment: { REA_LOG_LEVEL: "silent" }, + }); + const sharedNext = await sharedCliRun({ + arguments: analyzeArguments(secondRoot), + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(normalized(sharedNext)).toEqual(normalized(freshNext)); + expect(normalized(sharedNext).input_path).not.toBe( + normalized(shared).input_path, + ); + expect(cliCommandSession.processId()).toBe(pid); + + const analyzed = await analyzeJavaScriptDirectory("CLI", cli, secondRoot); + expect(analyzed.input_path).toBe(normalized(freshNext).input_path); + expect(cliCommandSession.processId()).toBe(pid); + }, +); diff --git a/tests/boundary/filesystem/appleXmlDeclarations.test.ts b/tests/boundary/filesystem/appleXmlDeclarations.test.ts index 56763d8b00..df8a1676a5 100644 --- a/tests/boundary/filesystem/appleXmlDeclarations.test.ts +++ b/tests/boundary/filesystem/appleXmlDeclarations.test.ts @@ -18,7 +18,7 @@ const archive = build({ it("preflights repeated unterminated quoted declarations without suffix rescans", async () => { const preflightUrl = new URL( - "../../../dist/artifacts/apple/InterfaceBuilderDecodeBudget.js", + "../../../dist/artifacts/apple/PropertyListDecodeBudget.js", import.meta.url, ).href; const { stdout } = await promisify(execFile)( diff --git a/tests/boundary/filesystem/ciTestHistory.test.ts b/tests/boundary/filesystem/ciTestHistory.test.ts index 84950d49c5..0d2093f7cb 100644 --- a/tests/boundary/filesystem/ciTestHistory.test.ts +++ b/tests/boundary/filesystem/ciTestHistory.test.ts @@ -17,7 +17,7 @@ const now = Date.now(); function artifact(id: number, overrides: Record = {}) { return { id: id + 100, - name: "test-timings", + name: "test-timings-full", expired: false, created_at: new Date(now - 60_000).toISOString(), workflow_run: { @@ -51,6 +51,7 @@ async function invoke( apiError?: boolean; repository?: string; repositoryId?: string; + tier?: string; } = {}, ) { const directory = await createTestTempDirectory("rea-ci-test-history-"); @@ -92,6 +93,7 @@ async function invoke( CI_REPOSITORY: options.repository ?? repository, CI_REPOSITORY_ID: options.repositoryId ?? String(repositoryId), CI_DEFAULT_BRANCH: "main", + CI_TIMING_TIER: options.tier ?? "", GITHUB_OUTPUT: output, }, }); @@ -120,17 +122,21 @@ it("selects the latest trusted full run that actually published timing history", 14: run(14, { path: ".github/workflows/release.yml" }), 11: run(11, { event: "workflow_dispatch" }), }, + { tier: "full" }, ); - expect(JSON.parse(result.stdout)).toEqual({ run_id: "11" }); - expect(output).toBe("run-id=11\n"); - expect(calls).toContain("name=test-timings"); + expect(JSON.parse(result.stdout)).toEqual({ + run_id: "11", + artifact_name: "test-timings-full", + }); + expect(output).toBe("run-id=11\nartifact-name=test-timings-full\n"); + expect(calls).toContain("name=test-timings-full"); expect(calls).toContain("per_page=100"); expect(calls.match(/actions\/runs\//gu)).toHaveLength(3); expect(calls).not.toContain("actions/workflows"); }); -it("rejects expired, foreign, old, and non-full run candidates", async () => { +it("rejects expired, foreign, old, and pull-request candidates", async () => { const { result, output } = await invoke( [ artifact(15, { expired: true }), @@ -145,16 +151,30 @@ it("rejects expired, foreign, old, and non-full run candidates", async () => { { 12: run(12, { event: "pull_request" }), }, + { tier: "full" }, ); - expect(JSON.parse(result.stdout)).toEqual({ run_id: "" }); - expect(output).toBe("run-id=\n"); + expect(JSON.parse(result.stdout)).toEqual({ + run_id: "", + artifact_name: "", + }); + expect(output).toBe("run-id=\nartifact-name=\n"); }); it("writes an empty run id when GitHub history is unavailable", async () => { - const { result, output } = await invoke([], {}, { apiError: true }); - expect(JSON.parse(result.stdout)).toEqual({ run_id: "" }); - expect(output).toBe("run-id=\n"); + const { result, output } = await invoke( + [], + {}, + { + apiError: true, + tier: "full", + }, + ); + expect(JSON.parse(result.stdout)).toEqual({ + run_id: "", + artifact_name: "", + }); + expect(output).toBe("run-id=\nartifact-name=\n"); }); it("bounds run lookups to five artifacts even when inventory has more", async () => { @@ -167,9 +187,13 @@ it("bounds run lookups to five artifacts even when inventory has more", async () run(id, index < 5 ? { conclusion: "failure" } : {}), ]), ), + { tier: "full" }, ); - expect(JSON.parse(result.stdout)).toEqual({ run_id: "" }); + expect(JSON.parse(result.stdout)).toEqual({ + run_id: "", + artifact_name: "", + }); expect(calls.match(/actions\/runs\//gu)).toHaveLength(5); }); @@ -182,7 +206,119 @@ it("returns empty history without querying for an invalid repository identity", repositoryId: "not-an-id", }, ); - expect(JSON.parse(result.stdout)).toEqual({ run_id: "" }); - expect(output).toBe("run-id=\n"); + expect(JSON.parse(result.stdout)).toEqual({ + run_id: "", + artifact_name: "", + }); + expect(output).toBe("run-id=\nartifact-name=\n"); expect(calls).toBe(""); }); + +it("selects fast history without reading the full-tier artifact", async () => { + const { result, calls } = await invoke( + [ + artifact(32, { + created_at: new Date(now - 30_000).toISOString(), + }), + artifact(31, { + name: "test-timings-fast", + created_at: new Date(now - 90_000).toISOString(), + }), + artifact(30, { + name: "test-timings-fast", + created_at: new Date(now - 20_000).toISOString(), + workflow_run: { + ...artifact(30).workflow_run, + head_branch: "feature", + }, + }), + ], + { + 32: run(32, { event: "schedule" }), + 31: run(31, { event: "push" }), + }, + { tier: "fast" }, + ); + + expect(JSON.parse(result.stdout)).toEqual({ + run_id: "31", + artifact_name: "test-timings-fast", + }); + expect(calls).toContain("name=test-timings-fast"); + expect(calls).not.toContain("name=test-timings-full"); + expect(calls.match(/actions\/runs\//gu)).toHaveLength(1); +}); + +it("ignores pull-request fast history instead of borrowing another artifact", async () => { + const { result, calls } = await invoke( + [artifact(41, { name: "test-timings-fast" }), artifact(40)], + { + 41: run(41, { event: "pull_request" }), + 40: run(40, { event: "workflow_dispatch" }), + }, + { tier: "fast" }, + ); + + expect(JSON.parse(result.stdout)).toEqual({ + run_id: "", + artifact_name: "", + }); + expect(calls).toContain("name=test-timings-fast"); + expect(calls).not.toContain("name=test-timings-full"); + expect(calls.match(/actions\/runs\//gu)).toHaveLength(1); +}); + +it("leaves fast history empty when only full-tier history was published", async () => { + const { result, output, calls } = await invoke( + [artifact(70)], + { 70: run(70, { event: "schedule" }) }, + { tier: "fast" }, + ); + + expect(JSON.parse(result.stdout)).toEqual({ + run_id: "", + artifact_name: "", + }); + expect(output).toBe("run-id=\nartifact-name=\n"); + expect(calls).toContain("name=test-timings-fast"); + expect(calls).not.toContain("name=test-timings-full"); + expect(calls).not.toMatch(/actions\/runs\//u); +}); + +it("does not query when the tier has no timing artifact", async () => { + const { result, output, calls } = await invoke([artifact(71)], { + 71: run(71, { event: "schedule" }), + }); + + expect(JSON.parse(result.stdout)).toEqual({ + run_id: "", + artifact_name: "", + }); + expect(output).toBe("run-id=\nartifact-name=\n"); + expect(calls).toBe(""); +}); + +it("selects full-tier history without using a newer fast artifact", async () => { + const { result } = await invoke( + [ + artifact(61, { + name: "test-timings-fast", + created_at: new Date(now - 10_000).toISOString(), + }), + artifact(60, { + name: "test-timings-full", + created_at: new Date(now - 80_000).toISOString(), + }), + ], + { + 61: run(61, { event: "push" }), + 60: run(60, { event: "schedule" }), + }, + { tier: "full" }, + ); + + expect(JSON.parse(result.stdout)).toEqual({ + run_id: "60", + artifact_name: "test-timings-full", + }); +}); diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts index fe65133320..582b8b77fb 100644 --- a/tests/boundary/filesystem/ciTestReports.test.ts +++ b/tests/boundary/filesystem/ciTestReports.test.ts @@ -243,10 +243,22 @@ it("keeps diagnosis available after merge failure and publishes only complete su const diagnosis = steps.find((step) => step.with?.name === "test-results"); expect(diagnosis?.if).toContain("!cancelled()"); expect(diagnosis?.if).not.toContain("success()"); - const history = steps.find((step) => step.with?.name === "test-timings"); + const history = steps.find( + (step) => step.name === "Publish successful timing history", + ); expect(history?.if).toContain("success()"); expect(history?.if).toContain("needs.test-shard.result == 'success'"); expect(history?.if).toContain("needs.changes.outputs.tier == 'full'"); + expect(history?.if).toContain( + "needs.changes.outputs.tier == 'fast' && needs.test-source.result == 'success'", + ); + expect(history?.if).toContain("github.event_name == 'push'"); + expect(history?.if).toContain( + "github.ref == format('refs/heads/{0}', github.event.repository.default_branch)", + ); + expect(history?.with?.name).toBe( + "test-timings-${{ needs.changes.outputs.tier }}", + ); expect(history?.with?.path).toBe(".cache/ci-test-history/test-results.json"); const buildSteps = (await workflow()).build!.steps; const lookup = buildSteps.find((step) => @@ -256,11 +268,14 @@ it("keeps diagnosis available after merge failure and publishes only complete su CI_REPOSITORY: "${{ github.repository }}", CI_REPOSITORY_ID: "${{ github.repository_id }}", CI_DEFAULT_BRANCH: "${{ github.event.repository.default_branch }}", + CI_TIMING_TIER: "${{ needs.changes.outputs.tier }}", }); const download = buildSteps.find( (step) => step.name === "Download previous timing report", ); - expect(download?.with?.name).toBe(history?.with?.name); + expect(download?.with?.name).toBe( + "${{ steps.history.outputs.artifact-name }}", + ); expect(download?.with?.["run-id"]).toBe( "${{ steps.history.outputs.run-id }}", ); diff --git a/tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts b/tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts index 54f54a4cf6..9531fb743f 100644 --- a/tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts +++ b/tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts @@ -1,8 +1,13 @@ +import { readFile } from "node:fs/promises"; + import { expect, it } from "vitest"; import { inspectBundleKeyedArchive } from "../../../src/artifacts/apple/KeyedArchiveReader.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; -import { keyedArchiveWideIntegerFixture } from "../../fixtures/keyedArchiveWideIntegers.js"; +import { + keyedArchiveSharedEntryFixture, + keyedArchiveWideIntegerFixture, +} from "../../fixtures/keyedArchiveWideIntegers.js"; import { archiveNumberMetadataCases, expectKeyedArchiveNumberMetadata, @@ -75,6 +80,55 @@ cliTest( }, ); +const sharedEntryObjects = [ + expect.objectContaining({ + id: 1, + status: "observed", + value: { $plist_type: "integer", decimal: "9007199254740993" }, + }), + expect.objectContaining({ + id: 2, + status: "observed", + value: { $plist_type: "integer", decimal: "9007199254740993" }, + }), + expect.objectContaining({ + id: 3, + status: "observed", + value: { $plist_type: "integer", decimal: "1152921504606846979" }, + }), +]; + +it("keeps exact integers when two object-table entries share one record", async () => { + const fixture = await keyedArchiveSharedEntryFixture(); + const result = await inspectBundleKeyedArchive({ + bundlePath: fixture.root, + targetSha256: fixture.digest, + parameters: { path: "archive.plist" }, + }); + expect(result.archive_sha256).toBe(fixture.digest); + expect(result.objects.slice(1)).toEqual(sharedEntryObjects); + expect(result.limitations).not.toContainEqual( + expect.stringMatching(/metadata could not be completely observed/u), + ); + expect(await readFile(fixture.path)).toEqual(fixture.bytes); +}); + +cliTest( + "compiled CLI keeps exact integers when object-table entries share one record", + async ({ cli }) => { + const fixture = await keyedArchiveSharedEntryFixture(); + const output = await cli.run({ + arguments: ["inspect-keyed-archive", fixture.path, "--json"], + environment: { REA_LOG_LEVEL: "silent", REA_ANALYSIS_PROVIDER: "auto" }, + }); + expect(output.exitCode, output.stderr).toBe(0); + const result = parseEvidence(output.json).normalized_result as { + objects: unknown[]; + }; + expect(result.objects.slice(1)).toEqual(sharedEntryObjects); + }, +); + it.each(archiveNumberMetadataCases)( "filesystem preserves numeric meaning for $name", async (item) => { diff --git a/tests/boundary/go/goBinaryImage.test.ts b/tests/boundary/go/goBinaryImage.test.ts index 9bc91c1853..c45d26f361 100644 --- a/tests/boundary/go/goBinaryImage.test.ts +++ b/tests/boundary/go/goBinaryImage.test.ts @@ -5,11 +5,11 @@ import { fatImage, machoImage, } from "../../../src/artifacts/apple/MachoImage.fixture.js"; -import { - readGoBinaryImage, - GoBinaryResourceFailure, -} from "../../../src/go/GoBinaryImage.js"; -import { readGoMachoImage } from "../../../src/go/GoMachoImage.js"; +import { readGoBinaryImage } from "../../../src/go/GoBinaryImage.js"; +import { goBuildInfoSearch } from "../../../src/go/GoBuildInfoSearch.js"; +import { BinaryResourceFailure } from "../../../src/native/metadata/BinaryContainer.js"; +import { readMachoImage } from "../../../src/native/metadata/MachoImage.js"; +import { readBinaryContainer } from "../../../src/native/metadata/readBinaryContainer.js"; import { createGoBinaryFixture, createGoSplitElfFixture, @@ -17,9 +17,43 @@ import { GO_MODULE_TEXT, } from "../../fixtures/go/image.js"; +it("preserves exact Mach-O protection masks for Go fallback selection", () => { + const { bytes } = createGoBinaryFixture({ + format: "macho", + sectionless: true, + }); + const firstSegment = 32; + const secondSegment = 104; + bytes.writeUInt32LE(2, 16); + bytes.writeUInt32LE(144, 20); + bytes.writeUInt32LE(11, firstSegment + 56); + bytes.writeUInt32LE(11, firstSegment + 60); + bytes.writeUInt32LE(0x19, secondSegment); + bytes.writeUInt32LE(72, secondSegment + 4); + bytes.write("__DATA2", secondSegment + 8, "ascii"); + bytes.writeBigUInt64LE(0x20000n, secondSegment + 24); + bytes.writeBigUInt64LE(BigInt(bytes.length), secondSegment + 32); + bytes.writeBigUInt64LE(0n, secondSegment + 40); + bytes.writeBigUInt64LE(BigInt(bytes.length), secondSegment + 48); + bytes.writeUInt32LE(3, secondSegment + 56); + bytes.writeUInt32LE(3, secondSegment + 60); + + const container = readBinaryContainer(bytes); + expect( + container.load_regions?.map((region) => region.protection_masks), + ).toEqual([ + { current: 11, maximum: 11 }, + { current: 3, maximum: 3 }, + ]); + expect(goBuildInfoSearch(bytes, container)).toMatchObject([ + { address: 0x20000n, offset: 0, size: bytes.length }, + ]); + expect(readGoBinaryImage(bytes).build_info?.go_version).toBe("go1.26.0"); +}); + it("bounds cumulative header searches across overlapping ELF file spans", () => { const bytes = createGoOverlappingElfFixture(); - expect(() => readGoBinaryImage(bytes)).toThrow(GoBinaryResourceFailure); + expect(() => readGoBinaryImage(bytes)).toThrow(BinaryResourceFailure); expect(() => readGoBinaryImage(bytes)).toThrow(/256 MiB.*search/); }); @@ -633,7 +667,7 @@ it("does not mark Mach-O section bytes outside their owning segment as read-only fixture.bytes.writeBigUInt64LE(0x10c00n, section + 32); fixture.bytes.writeBigUInt64LE(128n, section + 40); fixture.bytes.writeUInt32LE(3072, section + 48); - const image = readGoMachoImage(fixture.bytes); + const image = readMachoImage(fixture.bytes); expect(image.sections[0]).toMatchObject({ file_backed: false, read_only: false, diff --git a/tests/boundary/go/goBinaryMappings.test.ts b/tests/boundary/go/goBinaryMappings.test.ts index 96a850e48b..c0b4223225 100644 --- a/tests/boundary/go/goBinaryMappings.test.ts +++ b/tests/boundary/go/goBinaryMappings.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "vitest"; -import { mappedFileOffset } from "../../../src/go/GoBinaryContainer.js"; +import { mappedFileOffset } from "../../../src/native/metadata/BinaryContainer.js"; const selected = { address: 100n, offset: 16, size: 64 }; diff --git a/tests/boundary/il2cpp/il2cppMetadataFile.test.ts b/tests/boundary/il2cpp/il2cppMetadataFile.test.ts index 1402a86d3b..8418e3313a 100644 --- a/tests/boundary/il2cpp/il2cppMetadataFile.test.ts +++ b/tests/boundary/il2cpp/il2cppMetadataFile.test.ts @@ -261,13 +261,6 @@ describe("rejected IL2CPP global-metadata", () => { ), "record-size-mismatch", ); - // 300 generic containers widen GenericContainerIndex to two bytes. - rejects( - mutated((fixture, bytes) => - bytes.writeInt32LE(300, fixture.header("genericContainers", "count")), - ), - "record-size-mismatch", - ); rejects( mutated((fixture, bytes) => bytes.writeInt32LE(2, fixture.header("nestedTypes", "count")), @@ -290,6 +283,62 @@ describe("rejected IL2CPP global-metadata", () => { }); }); +describe("IL2CPP generic-container extents", () => { + it("checks generic-container count against its fixed 16-byte records before resolving widths", () => { + // A positive count with no record bytes. + for (const count of [1, 300]) + rejects( + mutated((fixture, bytes) => + bytes.writeInt32LE( + count, + fixture.header("genericContainers", "count"), + ), + ), + "record-size-mismatch", + ); + const oneRecord = createIl2cppMetadataFixture({ + wide: { fillerTypes: 0, genericContainers: 1, parameters: 0 }, + }); + // A positive count with a partial record. + rejects( + mutated( + (fixture, bytes) => + bytes.writeInt32LE(1, fixture.header("genericContainers", "size")), + oneRecord, + ), + "record-size-mismatch", + ); + // Bytes without a corresponding record. + rejects( + mutated( + (fixture, bytes) => + bytes.writeInt32LE(0, fixture.header("genericContainers", "count")), + oneRecord, + ), + "record-size-mismatch", + ); + }); + + it("retains width selection for empty and complete generic-container records", () => { + const empty = readIl2cppMetadata(createIl2cppMetadataFixture().bytes); + expect(empty.index_widths.generic_container).toBe(1); + + const one = readIl2cppMetadata( + createIl2cppMetadataFixture({ + wide: { fillerTypes: 0, genericContainers: 1, parameters: 0 }, + }).bytes, + ); + expect(one.index_widths.generic_container).toBe(1); + + const wide = readIl2cppMetadata( + createIl2cppMetadataFixture({ + wide: { fillerTypes: 0, genericContainers: 300, parameters: 0 }, + }).bytes, + ); + expect(wide.index_widths.generic_container).toBe(2); + }); +}); + describe("rejected IL2CPP metadata references", () => { it("rejects image, type, method and nested-type indices outside their tables", () => { rejects( diff --git a/tests/boundary/mcp/artifactMcp.test.ts b/tests/boundary/mcp/artifactMcp.test.ts index d84c139100..94f7e5150c 100644 --- a/tests/boundary/mcp/artifactMcp.test.ts +++ b/tests/boundary/mcp/artifactMcp.test.ts @@ -1,3 +1,4 @@ +import { toolContract } from "../../../src/contracts/toolContracts.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; @@ -658,3 +659,151 @@ it("compares summary Evidence through retained complete inspection parents", asy }); const toolEvidence = (value: unknown) => parseEvidence(value); + +it("discovers artifact target prerequisites and preserves typed MCP refusals", async () => { + const root = await createTestTempDirectory("rea-artifact-availability-mcp-"); + const zip = new ZipWriter(new Uint8ArrayWriter()); + await zip.add("a.txt", new TextReader("hello")); + await writeFile(join(root, "sample.zip"), await zip.close()); + await writeFile( + join(root, "plain.plist"), + "helloworld", + ); + await writeFile( + join(root, "keyed.plist"), + "$archiverNSKeyedArchiver$objects$nullhello$toprootCF$UID1", + ); + const header = Buffer.alloc(32); + header.writeUInt32LE(0xfeedfacf, 0); + header.writeUInt32LE(0x01000007, 4); + header.writeUInt32LE(3, 8); + header.writeUInt32LE(2, 12); + await writeFile(join(root, "standalone"), header); + await writeFile(join(root, "Tool.app"), header); + const contents = join(root, "Fixture.app", "Contents"); + await mkdir(join(contents, "MacOS"), { recursive: true }); + await writeFile(join(contents, "MacOS", "Fixture"), header); + await writeFile( + join(contents, "Info.plist"), + "CFBundleExecutableFixture", + ); + const session = createTestBinarySession( + new ArtifactProvider(process.env, "darwin"), + ); + const server = createServer({ kind: "session", session }); + const client = new Client({ + name: "artifact-availability-test", + version: "1", + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + const operations = [ + "decode_interface_builder", + "inspect_asset_catalog", + "inspect_keyed_archive", + "trace_dylib_resolution", + ]; + const availability = async () => { + const response = await client.callTool({ + name: "binary_session", + arguments: {}, + }); + const result = toolContract("binary_session").outputSchema.parse( + response.structuredContent, + ); + return result.result.tool_availability.groups; + }; + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + const before = await availability(); + for (const name of operations) + expect(before.find((group) => group.names.includes(name))).toMatchObject({ + available: false, + reason: "target_required", + }); + for (const filename of [ + "sample.zip", + "plain.plist", + "keyed.plist", + "standalone", + "Tool.app", + "Fixture.app", + ]) { + expect( + ( + await client.callTool({ + name: "open_binary", + arguments: { path: join(root, filename) }, + }) + ).isError, + ).not.toBe(true); + const groups = await availability(); + for (const name of operations) { + const supported = + filename === "Fixture.app" || + (filename.endsWith(".plist") && name === "inspect_keyed_archive") || + (["standalone", "Tool.app"].includes(filename) && + name === "trace_dylib_resolution"); + const group = groups.find((entry) => entry.names.includes(name)); + expect(group, `${filename} ${name}`).toMatchObject({ + available: supported, + reason: supported ? "available" : "target_unsupported", + }); + if (!supported) { + expect(group?.remediation).toMatch(/requires an active/); + const refused = await client.callTool({ name, arguments: {} }); + expect(parseMcpToolError(refused)).toMatchObject({ + error: { code: "unsupported_target" }, + }); + } + } + if (filename.endsWith(".plist")) { + const inspected = await client.callTool({ + name: "inspect_keyed_archive", + arguments: {}, + }); + if (filename === "keyed.plist") + expect(inspected.isError).not.toBe(true); + else + expect(parseMcpToolError(inspected)).toMatchObject({ + error: { code: "unsupported_target" }, + }); + } + await client.callTool({ name: "close_binary", arguments: {} }); + } + await writeFile( + join(contents, "state.plist"), + await readFile(join(root, "keyed.plist")), + ); + for (const format of ["raw-binary", "dos-com"] as const) { + const opened = await client.callTool({ + name: "open_binary", + arguments: { + path: join(root, "Fixture.app"), + format, + ...(format === "raw-binary" + ? { architecture: "x86_64", entry_point: "0x0" } + : {}), + }, + }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const groups = await availability(); + for (const name of operations.filter( + (name) => name !== "trace_dylib_resolution", + )) + expect( + groups.find((group) => group.names.includes(name)), + `${format} ${name}`, + ).toMatchObject({ available: true }); + const inspected = await client.callTool({ + name: "inspect_keyed_archive", + arguments: { path: "Contents/state.plist" }, + }); + expect(inspected.isError, JSON.stringify(inspected)).not.toBe(true); + await client.callTool({ name: "close_binary", arguments: {} }); + } + } finally { + await Promise.allSettled([client.close(), server.close(), session.close()]); + } +}); diff --git a/tests/boundary/mcp/nativeUiAvailability.test.ts b/tests/boundary/mcp/nativeUiAvailability.test.ts index 8dcc1f6895..6e208f4434 100644 --- a/tests/boundary/mcp/nativeUiAvailability.test.ts +++ b/tests/boundary/mcp/nativeUiAvailability.test.ts @@ -1,4 +1,5 @@ -import { rm } from "node:fs/promises"; +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; import type { Client } from "@modelcontextprotocol/client"; import type { CallToolResult } from "@modelcontextprotocol/server"; @@ -23,6 +24,7 @@ import { } from "../../../src/domain/apple/interfaceBuilderGraph.js"; import { inspectNativeDispatchMetadata } from "../../../src/domain/native/objcSwiftMetadata.js"; import { nativeInvestigationTraceSchema } from "../../../src/domain/native/nativeInvestigationGraph.js"; +import { thinMach } from "../../../src/domain/binaryTarget.fixture.js"; import type { JsonValue } from "../../../src/domain/jsonValue.js"; import { err, ok, type Result } from "../../../src/domain/result.js"; import type { AnalysisOperation } from "../../../src/application/AnalysisProvider.js"; @@ -33,13 +35,24 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { createSessionMcpHarness } from "./sessionMcpHarness.js"; const resources: Array<{ close(): Promise }> = []; -let directory: string | undefined; + +const writeEligibleAppBundle = async (root: string): Promise => { + const app = join(root, "Eligible.app"); + const contents = join(app, "Contents"); + await mkdir(join(contents, "MacOS"), { recursive: true }); + await writeFile( + join(contents, "MacOS", "Fixture"), + thinMach(0xfeedfacf, 0x0100000c), + ); + await writeFile( + join(contents, "Info.plist"), + "CFBundleExecutableFixture", + ); + return app; +}; afterEach(async () => { await Promise.all(resources.splice(0).map((resource) => resource.close())); - if (directory !== undefined) - await rm(directory, { recursive: true, force: true }); - directory = undefined; }); const identity = { @@ -205,16 +218,19 @@ describe("native UI action availability through available routes", () => { it.each(routeScenarios)( "reports the $name route available when its observed path succeeds", async (scenario) => { - directory = await createTestTempDirectory("rea-native-ui-availability-"); + const directory = await createTestTempDirectory( + "rea-native-ui-availability-", + ); const calls: ProviderOperation[] = []; - const { mcp, first } = await createSessionMcpHarness( + const { mcp } = await createSessionMcpHarness( directory, () => provider(scenario, calls), resources, ); + const app = await writeEligibleAppBundle(directory); const opened = await mcp.callTool({ name: "open_binary", - arguments: { path: first }, + arguments: { path: app }, }); expect(opened.isError).not.toBe(true); @@ -264,16 +280,19 @@ describe("native UI action availability for unmet routes", () => { ])( "reports unmet route sources for $name without hiding partial results", async (scenario) => { - directory = await createTestTempDirectory("rea-native-ui-no-metadata-"); + const directory = await createTestTempDirectory( + "rea-native-ui-no-metadata-", + ); const calls: ProviderOperation[] = []; - const { mcp, first } = await createSessionMcpHarness( + const { mcp } = await createSessionMcpHarness( directory, () => provider(scenario, calls), resources, ); + const app = await writeEligibleAppBundle(directory); const opened = await mcp.callTool({ name: "open_binary", - arguments: { path: first }, + arguments: { path: app }, }); expect(opened.isError).not.toBe(true); @@ -378,20 +397,21 @@ describe("native UI action availability for failures and target gates", () => { it.each([metadataRuntimeFailureScenario, decodeCancelledScenario])( "preserves runtime failures for $name without trying another declared source", async (scenario) => { - directory = await createTestTempDirectory( + const directory = await createTestTempDirectory( "rea-native-ui-source-failure-", ); const calls: ProviderOperation[] = []; - const { mcp, first } = await createSessionMcpHarness( + const { mcp } = await createSessionMcpHarness( directory, () => provider(scenario, calls), resources, ); + const app = await writeEligibleAppBundle(directory); expect( ( await mcp.callTool({ name: "open_binary", - arguments: { path: first }, + arguments: { path: app }, }) ).isError, ).not.toBe(true); @@ -426,7 +446,7 @@ describe("native UI action availability for failures and target gates", () => { ); it("reports target_required before querying provider operations", async () => { - directory = await createTestTempDirectory("rea-native-ui-no-target-"); + const directory = await createTestTempDirectory("rea-native-ui-no-target-"); const calls: ProviderOperation[] = []; const { mcp } = await createSessionMcpHarness( directory, @@ -441,6 +461,44 @@ describe("native UI action availability for failures and target gates", () => { }); expect(calls).toEqual([]); }); + + it("keeps authored UI dispatch unavailable for a non-bundle target", async () => { + const directory = await createTestTempDirectory( + "rea-native-ui-non-bundle-", + ); + const calls: ProviderOperation[] = []; + const { mcp } = await createSessionMcpHarness( + directory, + () => provider(authoredUiDirectScenario, calls), + resources, + ); + const executable = join(directory, "Standalone"); + await writeFile(executable, thinMach(0xfeedfacf, 0x0100000c)); + expect( + ( + await mcp.callTool({ + name: "open_binary", + arguments: { path: executable }, + }) + ).isError, + ).not.toBe(true); + + try { + const entry = dispatchAvailability(await readInventory(mcp)); + const authored = entry?.modes?.find( + (mode) => mode.name === "authored_ui_dispatch", + ); + expect(authored).toMatchObject({ + available: false, + missing_operations: ["decode_interface_builder"], + }); + expect(calls).toEqual([]); + } finally { + expect( + (await mcp.callTool({ name: "close_binary", arguments: {} })).isError, + ).not.toBe(true); + } + }); }); const readInventory = async (mcp: Client) => { diff --git a/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts b/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts new file mode 100644 index 0000000000..8dfd22fcf8 --- /dev/null +++ b/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts @@ -0,0 +1,277 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import { expect, it } from "vitest"; + +import { createArtifactBinarySession } from "../../../src/composition/binary.js"; +import { + PYTHON_39_MAGIC, + cArchiveExecutable, + pyzArchive, +} from "../../../src/artifacts/pyinstaller/CArchive.fixture.js"; +import { artifactInspectionResultSchema } from "../../../src/domain/artifactInspection.js"; +import { parseEvidence, type Evidence } from "../../../src/domain/evidence.js"; +import { analysisViewResultSchema } from "../../../src/domain/analysisView/analysisView.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const assertMetadataQueries = async ( + client: Client, + parent: Evidence, + expectedCoverage: "complete" | "unknown", +): Promise => { + const definition = (await client.listTools()).tools.find( + (tool) => tool.name === "inspect_analysis_view", + ); + expect(definition?.outputSchema).toBeDefined(); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const inputSchema: Record = definition!.inputSchema; + const outputSchema: Record = definition!.outputSchema!; + const input = ajv.compile(inputSchema); + const output = ajv.compile(outputSchema); + const ids: string[] = []; + const selected: unknown[] = []; + for (const offset of [0, 1, 2, 3]) { + const args = { + source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, + view: { + kind: "inventory", + collection: "occurrences", + entry_kinds: ["metadata"], + fields: ["format_metadata"], + offset, + limit: 1, + }, + }; + expect(input(args), JSON.stringify(input.errors)).toBe(true); + const reply = await client.callTool({ + name: "inspect_analysis_view", + arguments: args, + }); + expect(reply.isError).not.toBe(true); + expect(output(reply.structuredContent), JSON.stringify(output.errors)).toBe( + true, + ); + const evidence = parseEvidence(reply.structuredContent); + const view = analysisViewResultSchema.parse(evidence.normalized_result); + if (view.kind !== "inventory") throw new Error("Expected inventory page"); + expect(view.parent_evidence_id).toBe(parent.evidence_id); + expect(evidence.evidence_links).toContain(parent.evidence_id); + expect(view.coverage.total).toBe(3); + expect(view.source_coverage?.status).toBe(expectedCoverage); + if (offset < 3) { + expect(view.items).toHaveLength(1); + expect(view.items[0]).toMatchObject({ + artifact_id: null, + format_metadata: { kind: "pyinstaller-entry" }, + }); + expect(view.items[0]).not.toHaveProperty("logical_path"); + const row = view.items[0]; + if (row === undefined || !("occurrence_id" in row)) + throw new Error("Expected occurrence identity"); + ids.push(row.occurrence_id); + selected.push(row); + } else expect(view.items).toEqual([]); + } + expect(new Set(ids).size).toBe(3); + expect(selected).toEqual( + expect.arrayContaining([ + ...[0, 1].map((index) => + expect.objectContaining({ + format_metadata: expect.objectContaining({ + index, + type_code: "o", + source_name: "pyi-contents-directory _internal", + }), + }), + ), + expect.objectContaining({ + format_metadata: expect.objectContaining({ + type_code: "d", + source_name: "../shared/app:assets/data.bin", + dependency_status: "unresolved", + dependency_archive: "../shared/app", + dependency_member: "assets/data.bin", + }), + }), + ]), + ); +}; + +const writeExecutable = async (opaque: boolean): Promise => { + const directory = await createTestTempDirectory("rea-pyinstaller-mcp-"); + const path = join(directory, "hello"); + await writeFile( + path, + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 312, + pythonLibrary: "libpython3.12.so", + members: [ + { + name: "pyi-contents-directory _internal", + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }, + { + name: "pyi-contents-directory _internal", + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }, + { + name: "hello", + typeCode: "s", + payload: Buffer.from("print('hi')"), + compress: false, + }, + { + name: "../shared/app:assets/data.bin", + typeCode: "d", + payload: Buffer.alloc(0), + compress: false, + }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: opaque ? Buffer.alloc(4) : PYTHON_39_MAGIC, + modules: [{ name: "app", typeCode: 0, body: Buffer.from("app") }], + }), + compress: opaque, + }, + ...(opaque + ? [ + { + name: "pyimod00_crypto_key", + typeCode: "m", + payload: Buffer.from("key"), + compress: false, + }, + ] + : []), + ], + }), + ); + return path; +}; + +it.each([false, true])( + "inventories a PyInstaller executable through inspect_artifact (opaque: %s)", + async (opaque) => { + const path = await writeExecutable(opaque); + const session = await createArtifactBinarySession(process.env); + const server = createServer({ kind: "session", session }); + const client = new Client({ name: "pyinstaller-mcp-test", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path }, + }); + expect(opened.isError).not.toBe(true); + const inspected = await client.callTool({ + name: "inspect_artifact", + arguments: {}, + }); + expect(inspected.isError).not.toBe(true); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(inspected.structuredContent).normalized_result, + ); + expect(inspection.subject.root_format).toBe("pyinstaller"); + if (!opaque) expect(inspection.next_probes).toEqual([]); + expect( + inspection.substeps[0]?.evidence.normalized_result.coverage.status, + ).toBe(opaque ? "unknown" : "complete"); + if (opaque) + expect( + inspection.substeps[0]?.evidence.normalized_result.occurrences.find( + (row) => row.logical_path === "PYZ.pyz", + ), + ).toMatchObject({ + encrypted: false, + hash_status: "verified", + artifact_id: expect.any(String), + format_metadata: { encryption_source: "carchive-marker" }, + }); + expect( + inspection.substeps[0]?.evidence.normalized_result.occurrences.map( + (occurrence) => occurrence.logical_path, + ), + ).toEqual( + expect.arrayContaining([ + "hello", + ...(opaque ? [] : ["PYZ.pyz/app.pyc"]), + ]), + ); + const parent = parseEvidence(inspected.structuredContent); + const compact = await client.callTool({ + name: "inspect_artifact", + arguments: { detail: "summary" }, + }); + expect(compact.isError).not.toBe(true); + const summary = parseEvidence( + compact.structuredContent, + ).normalized_result; + expect(summary).toMatchObject({ + kind: "summary", + summary: { + root_format_metadata: { + kind: "pyinstaller", + python_version_raw: 312, + external_dependency_count: 1, + }, + observed_entry_kind_counts: { metadata: 3 }, + source_coverage: { status: opaque ? "unknown" : "complete" }, + }, + }); + await assertMetadataQueries( + client, + parent, + opaque ? "unknown" : "complete", + ); + if (opaque) { + const missing = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { + kind: "retained-evidence", + evidence_id: parent.evidence_id, + }, + view: { + kind: "inventory", + collection: "occurrences", + logical_paths: ["PYZ.pyz/app.pyc"], + entry_kinds: ["file"], + fields: [], + offset: 0, + limit: 1, + }, + }, + }); + expect(missing.isError).not.toBe(true); + const page = analysisViewResultSchema.parse( + parseEvidence(missing.structuredContent).normalized_result, + ); + expect(page).toMatchObject({ + kind: "inventory", + items: [], + source_coverage: { status: "unknown" }, + }); + expect(page.unknowns).toContain( + "Producer traversal coverage is unknown; missing matches do not prove source absence.", + ); + } + } finally { + await client.close(); + await server.close(); + await session.close(); + } + }, +); diff --git a/tests/boundary/mcp/rustBinaryMcp.test.ts b/tests/boundary/mcp/rustBinaryMcp.test.ts index d903affd4d..4f7b3452c0 100644 --- a/tests/boundary/mcp/rustBinaryMcp.test.ts +++ b/tests/boundary/mcp/rustBinaryMcp.test.ts @@ -1,3 +1,4 @@ +import { rustBinarySchema } from "../../../src/domain/rust/rustBinary.js"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect } from "vitest"; @@ -43,7 +44,10 @@ mcpTest( const evidence = toolContract("inspect_rust_binary").outputSchema.parse( response.structuredContent, ); - expect(evidence.normalized_result.auditable_metadata?.packages).toEqual([ + expect( + rustBinarySchema.parse(evidence.normalized_result).auditable_metadata + ?.packages, + ).toEqual([ expect.objectContaining({ name: "itoa", version: "1.0.18" }), expect.objectContaining({ name: "sample", @@ -64,7 +68,10 @@ mcpTest( const absentEvidence = toolContract( "inspect_rust_binary", ).outputSchema.parse(absent.structuredContent); - expect(absentEvidence.normalized_result.auditable_metadata).toBeNull(); + expect( + rustBinarySchema.parse(absentEvidence.normalized_result) + .auditable_metadata, + ).toBeNull(); } finally { await session.close(); } diff --git a/tests/boundary/mcp/wasmArtifactMcp.test.ts b/tests/boundary/mcp/wasmArtifactMcp.test.ts index 1be804f61c..a2d394a2d7 100644 --- a/tests/boundary/mcp/wasmArtifactMcp.test.ts +++ b/tests/boundary/mcp/wasmArtifactMcp.test.ts @@ -164,10 +164,14 @@ it.each(["bin", "hop"])( it("advertises exact valid schemas and records inline WASM interface evidence without a binary target", async () => { const value = wasmArtifactFixture(); + value.tool_profile.timeout_ms = 42_000; const close = vi.fn(() => Promise.resolve()); const service = new WasmArtifactService({ close, - inspect: () => Promise.resolve(ok(wasmArtifactExecution(value))), + inspect: (input) => { + expect(input.command_timeout_ms).toBe(42_000); + return Promise.resolve(ok(wasmArtifactExecution(value))); + }, }); const session = createTestBinarySession(() => { throw new Error("deep provider must not start"); @@ -184,6 +188,20 @@ it("advertises exact valid schemas and records inline WASM interface evidence wi ); const { ajv, inputSchema } = schemas; expect(ajv.validate(inputSchema, { path: value.artifact.path })).toBe(true); + for (const command_timeout_ms of [1, 42_000, 2_147_483_647]) + expect( + ajv.validate(inputSchema, { + path: value.artifact.path, + command_timeout_ms, + }), + ).toBe(true); + for (const command_timeout_ms of [0, -1, 1.5, 2_147_483_648, "1000", null]) + expect( + ajv.validate(inputSchema, { + path: value.artifact.path, + command_timeout_ms, + }), + ).toBe(false); expect( ajv.validate(inputSchema, { path: value.artifact.path, @@ -193,7 +211,7 @@ it("advertises exact valid schemas and records inline WASM interface evidence wi ).toBe(false); const response = await client.callTool({ name: "inspect_wasm_artifact", - arguments: { path: value.artifact.path }, + arguments: { path: value.artifact.path, command_timeout_ms: 42_000 }, }); assertRecordedMcpEvidence( response, diff --git a/tests/boundary/model/modelArtifact.test.ts b/tests/boundary/model/modelArtifact.test.ts index 22c2a83737..dd7d9d8683 100644 --- a/tests/boundary/model/modelArtifact.test.ts +++ b/tests/boundary/model/modelArtifact.test.ts @@ -22,6 +22,57 @@ const providerAt = async (bytes: Buffer) => { return { provider, path, root }; }; +it.each([ + ["01000000", 1, 16_777_216], + ["00000001", 16_777_216, 1], + ["04000000", 4, 67_108_864], + ["00000004", 67_108_864, 4], + ["e7030000", 999, 3_875_733_504], + ["000003e7", 3_875_733_504, 999], + ["00000000", 0, 0], + ["ffffffff", 4_294_967_295, 4_294_967_295], +] as const)( + "preserves unsupported GGUF version bytes %s without assuming byte order", + async (versionBytes, little, big) => { + // No counts are needed to explain a version refusal. Do not reinterpret + // an unknown version's subsequent layout or publish a header observation. + const { provider, path } = await providerAt( + Buffer.from(`47475546${versionBytes}`, "hex"), + ); + const result = await provider.inspect({ path, hash: "header" }); + if (result.ok) throw new Error("Unknown version admitted"); + expect(result.error).toMatchObject({ + _tag: "AnalysisUnsupportedTargetError", + reason: `GGUF version bytes ${versionBytes} are unsupported (little-endian ${String(little)}, big-endian ${String(big)}; byte order unknown); supported versions are 2 and 3`, + }); + }, +); + +it("continues reading little-endian GGUF v2 declarations", async () => { + const { provider, path } = await providerAt( + ggufFixture({ version: 2 }).bytes, + ); + const result = await provider.inspect({ path, hash: "header" }); + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ version: 2, byte_order: "little" }); + expect(result.value.tensors.map(({ name }) => name)).toEqual([ + "weight", + "quant", + ]); +}); + +it("continues refusing the unsupported big-endian GGUF v2 layout", async () => { + const { provider, path } = await providerAt( + ggufFixture({ version: 2, bigEndian: true }).bytes, + ); + const result = await provider.inspect({ path, hash: "header" }); + if (result.ok) throw new Error("Big-endian GGUF v2 admitted"); + expect(result.error).toMatchObject({ + _tag: "BinaryTargetError", + constraint: "file_format_rejected", + }); +}); + it.each([false, true])( "reads exact GGUF declarations, endian=%s, without hashing tensor payload", async (bigEndian) => { @@ -368,6 +419,91 @@ it("validates booleans beyond an array preview without allocating the full value }); }); +it.each([false, true])( + "validates omitted UTF-8 strings across chunk boundaries, big endian=%s", + async (bigEndian) => { + const base = ggufFixture({ bigEndian }); + const preview = Array.from({ length: 8 }, (_, n) => `\uFEFFtoken-${n} 🦙`); + const strings = [ + ...preview, + "", + "a".repeat(65533) + "€", + "a".repeat(65535) + "🦙", + "\0tail", + ]; + const fixture = ggufFixture({ + bigEndian, + metadata: [ + { + key: "custom.strings", + type: 9, + value: Buffer.concat([ + base.u32(8), + base.u64(BigInt(strings.length)), + ...strings.map(base.text), + ]), + }, + ], + }); + const { path, provider } = await providerAt(fixture.bytes); + const result = await provider.inspect({ path, hash: "header" }); + if (!result.ok) throw result.error; + expect(result.value.metadata[0]).toMatchObject({ + value: preview, + element_count: strings.length, + preview_complete: false, + }); + expect(result.value.artifact).toMatchObject({ + bytes_read: fixture.header.length, + header_sha256: createHash("sha256").update(fixture.header).digest("hex"), + }); + expect(result.value.tensors).toHaveLength(2); + }, +); + +it.each([ + ["short", Buffer.from([0xc0, 0x80])], + [ + "chunk-sized", + Buffer.concat([Buffer.alloc(65535, 0x61), Buffer.from([0xe2])]), + ], + [ + "cross-chunk", + Buffer.concat([ + Buffer.alloc(65535, 0x61), + Buffer.from([0xf0, 0x9f, 0x61, 0x80]), + ]), + ], +] as const)( + "rejects malformed omitted %s UTF-8 strings", + async (_, invalid) => { + const base = ggufFixture(); + const fixture = ggufFixture({ + metadata: [ + { + key: "custom.strings", + type: 9, + value: Buffer.concat([ + base.u32(8), + base.u64(9n), + ...Array.from({ length: 8 }, () => base.text("valid")), + base.u64(BigInt(invalid.length)), + invalid, + ]), + }, + ], + }); + const { path, provider } = await providerAt(fixture.bytes); + const result = await provider.inspect({ path, hash: "header" }); + if (result.ok) throw new Error("Invalid omitted UTF-8 string was accepted"); + expect(result.error).toMatchObject({ + _tag: "BinaryTargetError", + constraint: "file_format_rejected", + message: expect.stringContaining("GGUF string is not valid UTF-8"), + }); + }, +); + it("refuses an oversized declared header before reading or expanding it", async () => { const bytes = Buffer.alloc(9); bytes.writeBigUInt64LE(100_000_001n); diff --git a/tests/boundary/process/packageNpmInvocation.test.ts b/tests/boundary/process/packageNpmInvocation.test.ts index ccc5c461a1..a3afae696f 100644 --- a/tests/boundary/process/packageNpmInvocation.test.ts +++ b/tests/boundary/process/packageNpmInvocation.test.ts @@ -1,25 +1,25 @@ import { execFile } from "node:child_process"; -import { copyFile, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; +import { copyFile, mkdir, writeFile } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; import { pathToFileURL } from "node:url"; import { promisify } from "node:util"; import { expect, it } from "vitest"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + const execute = promisify(execFile); it("runs the caller-selected npm entry through Node without rewriting its inputs", async () => { - const root = await mkdtemp(join(tmpdir(), "rea-npm-invocation-")); - try { - const npmEntry = join(root, "selected npm entry.mjs"); - await writeFile( - npmEntry, - `process.stdout.write(JSON.stringify({ arguments: process.argv.slice(2), cwd: process.cwd(), marker: process.env.REA_NPM_TEST_MARKER }));\n`, - ); - const helper = pathToFileURL( - resolve("scripts/lib/verify-package-core.mjs"), - ).href; - const program = ` + const root = await createTestTempDirectory("rea-npm-invocation-"); + const npmEntry = join(root, "selected npm entry.mjs"); + await writeFile( + npmEntry, + `process.stdout.write(JSON.stringify({ arguments: process.argv.slice(2), cwd: process.cwd(), marker: process.env.REA_NPM_TEST_MARKER }));\n`, + ); + const helper = pathToFileURL( + resolve("scripts/lib/verify-package-core.mjs"), + ).href; + const program = ` const { execNpm } = await import(${JSON.stringify(helper)}); const result = await execNpm(["pack", "value with spaces"], { cwd: ${JSON.stringify(root)}, @@ -27,19 +27,16 @@ it("runs the caller-selected npm entry through Node without rewriting its inputs }); process.stdout.write(result.stdout); `; - const result = await execute( - process.execPath, - ["--input-type=module", "--eval", program], - { windowsHide: true }, - ); - expect(JSON.parse(result.stdout)).toEqual({ - arguments: ["pack", "value with spaces"], - cwd: root, - marker: "caller-selected", - }); - } finally { - await rm(root, { recursive: true, force: true }); - } + const result = await execute( + process.execPath, + ["--input-type=module", "--eval", program], + { windowsHide: true }, + ); + expect(JSON.parse(result.stdout)).toEqual({ + arguments: ["pack", "value with spaces"], + cwd: root, + marker: "caller-selected", + }); }); const windowsIt = process.platform === "win32" ? it : it.skip; @@ -47,22 +44,21 @@ const windowsIt = process.platform === "win32" ? it : it.skip; windowsIt( "uses the npm entry adjacent to Node when none is selected", async () => { - const root = await mkdtemp(join(tmpdir(), "rea-adjacent-npm-")); - try { - const node = join(root, "node.exe"); - const npmEntry = join(root, "node_modules/npm/bin/npm-cli.js"); - await mkdir(dirname(npmEntry), { recursive: true }); - await Promise.all([ - copyFile(process.execPath, node), - writeFile( - npmEntry, - `process.stdout.write(JSON.stringify({ arguments: process.argv.slice(2), cwd: process.cwd() }));\n`, - ), - ]); - const helper = pathToFileURL( - resolve("scripts/lib/verify-package-core.mjs"), - ).href; - const program = ` + const root = await createTestTempDirectory("rea-adjacent-npm-"); + const node = join(root, "node.exe"); + const npmEntry = join(root, "node_modules/npm/bin/npm-cli.js"); + await mkdir(dirname(npmEntry), { recursive: true }); + await Promise.all([ + copyFile(process.execPath, node), + writeFile( + npmEntry, + `process.stdout.write(JSON.stringify({ arguments: process.argv.slice(2), cwd: process.cwd() }));\n`, + ), + ]); + const helper = pathToFileURL( + resolve("scripts/lib/verify-package-core.mjs"), + ).href; + const program = ` const { execNpm } = await import(${JSON.stringify(helper)}); const environment = { ...process.env }; delete environment.npm_execpath; @@ -72,19 +68,16 @@ windowsIt( }); process.stdout.write(result.stdout); `; - const environment = { ...process.env }; - delete environment.npm_execpath; - const result = await execute( - node, - ["--input-type=module", "--eval", program], - { env: environment, windowsHide: true }, - ); - expect(JSON.parse(result.stdout)).toEqual({ - arguments: ["pack", "value with spaces"], - cwd: root, - }); - } finally { - await rm(root, { recursive: true, force: true }); - } + const environment = { ...process.env }; + delete environment.npm_execpath; + const result = await execute( + node, + ["--input-type=module", "--eval", program], + { env: environment, windowsHide: true }, + ); + expect(JSON.parse(result.stdout)).toEqual({ + arguments: ["pack", "value with spaces"], + cwd: root, + }); }, ); diff --git a/tests/boundary/providers/hopper/bridgeBootstrapCleanup.test.ts b/tests/boundary/providers/hopper/bridgeBootstrapCleanup.test.ts index 81a8ffc5e9..b45e68233f 100644 --- a/tests/boundary/providers/hopper/bridgeBootstrapCleanup.test.ts +++ b/tests/boundary/providers/hopper/bridgeBootstrapCleanup.test.ts @@ -3,6 +3,7 @@ import { join } from "node:path"; import { expect, it } from "vitest"; +import { HopperCancelledError } from "../../../../src/domain/hopperErrors.js"; import { HopperApplicationLauncher } from "../../../../src/hopper/BridgeLauncher.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; @@ -11,7 +12,8 @@ it.each(["failed", "cancelled"] as const)( async (outcome) => { const directory = await createTestTempDirectory("rea-bootstrap-cleanup-"); const controller = new AbortController(); - if (outcome === "cancelled") controller.abort(); + const reason = { source: "caller", opaque: 7 }; + if (outcome === "cancelled") controller.abort(reason); let releases = 0; const launcher = new HopperApplicationLauncher( { @@ -45,11 +47,12 @@ it.each(["failed", "cancelled"] as const)( { signal: controller.signal }, ); expect(result.ok).toBe(false); - if (outcome === "cancelled") - expect(result).toMatchObject({ - ok: false, - error: { _tag: "HopperCancelledError" }, - }); + if (outcome === "cancelled") { + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected cancellation"); + expect(result.error).toBeInstanceOf(HopperCancelledError); + expect(result.error.cause).toBe(reason); + } await expect(lstat(join(directory, "bootstrap.py"))).rejects.toMatchObject({ code: "ENOENT", }); diff --git a/tests/boundary/providers/hopper/hopperClient.startupWaiters.test.ts b/tests/boundary/providers/hopper/hopperClient.startupWaiters.test.ts index 6c2fdb7c23..fd25fc14d8 100644 --- a/tests/boundary/providers/hopper/hopperClient.startupWaiters.test.ts +++ b/tests/boundary/providers/hopper/hopperClient.startupWaiters.test.ts @@ -2,6 +2,7 @@ import { access } from "node:fs/promises"; import { expect, it, vi } from "vitest"; +import { HopperCancelledError } from "../../../../src/domain/hopperErrors.js"; import { HopperClient } from "../../../../src/hopper/HopperClient.js"; import { ok } from "../../../../src/domain/result.js"; import { ProviderRunLineage } from "../../../../src/process/ProviderRunLineage.js"; @@ -16,11 +17,13 @@ it("preserves a warm provider when an already cancelled caller joins", async () const client = trackHopperClient(new HopperClient({ launcher })); await expect(client.start()).resolves.toMatchObject({ ok: true }); const controller = new AbortController(); - controller.abort(); - await expect(client.start(controller.signal)).resolves.toMatchObject({ - ok: false, - error: { _tag: "HopperCancelledError" }, - }); + const reason = { source: "caller", opaque: 7 }; + controller.abort(reason); + const joined = await client.start(controller.signal); + expect(joined.ok).toBe(false); + if (joined.ok) throw new Error("Expected cancellation"); + expect(joined.error).toBeInstanceOf(HopperCancelledError); + expect(joined.error.cause).toBe(reason); expect(client.isReady()).toBe(true); expect(client.operationHealth()).toMatchObject({ state: "idle" }); await expect( @@ -137,13 +140,18 @@ it.each([0, 1])( void waits[cancelledIndex]?.then((result) => { cancelled = result; }); - controllers[cancelledIndex]?.abort(); + const reason = { source: "caller", opaque: cancelledIndex }; + controllers[cancelledIndex]?.abort(reason); await vi.waitFor(() => expect(cancelled).toMatchObject({ ok: false, error: { _tag: "HopperCancelledError" }, }), ); + if (cancelled === undefined || cancelled.ok) + throw new Error("Expected cancellation"); + expect(cancelled.error).toBeInstanceOf(HopperCancelledError); + expect(cancelled.error.cause).toBe(reason); expect(launcher.processes).toHaveLength(0); release?.(); await expect(waits[1 - cancelledIndex]).resolves.toMatchObject({ diff --git a/tests/boundary/providers/native/nativeInventory.golden.test.ts b/tests/boundary/providers/native/nativeInventory.golden.test.ts index 5e1824e802..cee8136c3c 100644 --- a/tests/boundary/providers/native/nativeInventory.golden.test.ts +++ b/tests/boundary/providers/native/nativeInventory.golden.test.ts @@ -1,4 +1,5 @@ -import { writeFile } from "node:fs/promises"; +import { inspectSignatureSchema } from "../../../../src/domain/native/nativeInspection.js"; +import { realpath, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, it } from "vitest"; import { @@ -96,7 +97,7 @@ it("reports unsigned slices when a universal Mach-O has mixed signatures", async ); const client = new NativeMacOSProvider( {}, - signatureRunner({ signed, unsigned, entitlements }), + signatureRunner({ signed, unsigned, entitlements, unsignedHeader: true }), "darwin", ).createClient(await signatureTarget(signedArchitecture)); @@ -110,6 +111,7 @@ it("reports unsigned slices when a universal Mach-O has mixed signatures", async designated_requirement: null, entitlements: { "com.apple.security.app-sandbox": true }, limitations: expect.arrayContaining([ + "Partial signature coverage: aggregate signing metadata describes codesign’s default slice; Mach-O slices have mixed signing states.", "Unsigned Mach-O slices: x86_64.", "The aggregate designated requirement is unavailable because Mach-O slices have mixed signing states.", ]), @@ -155,6 +157,99 @@ it("rejects nonzero supplemental codesign output that is not an unsigned observa }); }); +it.each([ + "extra diagnostic", + "foreign executable", + "slice diagnostic", + "later slice diagnostic", +])("rejects an unsigned response with %s", async (kind) => { + const signed = (await fixture("codesign.txt")).replace( + /^Format=.*$/mu, + "Format=Mach-O universal (x86_64 arm64)", + ); + const replay = signatureRunner({ + signed, + unsigned: "code object is not signed at all", + entitlements: await fixture("entitlements.xml"), + unsignedHeader: true, + }); + const runner: NativeCommandRunner = { + async run(tool, args, options) { + const result = await replay.run(tool, args, options); + if (!result.ok) return result; + if (kind === "later slice diagnostic") { + if (!args.includes("arm64")) return result; + const stderr = "later architecture probe failed\n"; + return ok({ + ...result.value, + exitCode: 1, + stderr, + stderrBytes: Buffer.byteLength(stderr), + }); + } + if (result.value.exitCode === 0) return result; + const isSlice = args.includes("-a"); + if ((kind === "slice diagnostic") !== isSlice) return result; + const stderr = + kind === "foreign executable" + ? result.value.stderr.replace( + /^Executable=.*\n/u, + "Executable=/foreign/program\n", + ) + : `${result.value.stderr}additional codesign failure\n`; + return ok({ + ...result.value, + stderr, + stderrBytes: Buffer.byteLength(stderr), + }); + }, + }; + const result = await new NativeMacOSProvider({}, runner, "darwin") + .createClient(await signatureTarget()) + .execute("inspect_signature", {}); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "ProviderAdapterError" }, + }); + if (kind.endsWith("slice diagnostic") && !result.ok) { + const partial = inspectSignatureSchema.parse( + result.error.partialObservation, + ); + expect(partial).toMatchObject({ + signed: true, + identifier: "com.example.fixture", + entitlements: { "com.apple.security.app-sandbox": true }, + designated_requirement: null, + limitations: expect.arrayContaining([ + expect.stringContaining("Partial signature coverage"), + expect.stringContaining("did not complete"), + ]), + }); + expect(partial.provenance).toHaveLength( + kind === "later slice diagnostic" ? 5 : 4, + ); + expect(partial.provenance[0]?.command).toContain("--verbose=4"); + expect(partial.provenance[1]?.command).toContain("-r-"); + expect(partial.provenance[2]?.command).toContain("--entitlements"); + if (kind === "later slice diagnostic") { + expect(partial.limitations).toEqual( + expect.arrayContaining([ + "Unsigned Mach-O slices: x86_64.", + "The aggregate designated requirement is unavailable because Mach-O slices have mixed signing states.", + ]), + ); + expect(partial.provenance[4]).toMatchObject({ + command: expect.arrayContaining(["-a", "arm64"]), + exit: { code: 1, signal: null }, + }); + } + expect(partial.provenance[3]).toMatchObject({ + command: expect.arrayContaining(["-a", "x86_64"]), + exit: { code: 1, signal: null }, + }); + } +}); + it("retains native inventory facts from captured Apple tool output", async () => { const outputs: Record = {}; for (const tool of ["lipo", "otool", "nm", "dwarfdump", "vtool"]) @@ -403,8 +498,9 @@ const signatureRunner = (outputs: { readonly signed: string; readonly unsigned: string; readonly entitlements: string; + readonly unsignedHeader?: boolean; }): NativeCommandRunner => ({ - run( + async run( tool: string, arguments_: readonly string[], options: NativeCommandOptions, @@ -424,7 +520,7 @@ const signatureRunner = (outputs: { selectedResponse.output.includes("code object is not signed at all") ? { ...selectedResponse, - output: `${arguments_.at(-1)}: code object is not signed at all\n`, + output: `${outputs.unsignedHeader ? `Executable=${await realpath(arguments_.at(-1)!)}\n` : ""}${arguments_.at(-1)}: code object is not signed at all\n`, } : selectedResponse; const capture: NativeCommandCapture = { diff --git a/tests/boundary/providers/native/nativePlistBinarySource.test.ts b/tests/boundary/providers/native/nativePlistBinarySource.test.ts new file mode 100644 index 0000000000..2e40cba12f --- /dev/null +++ b/tests/boundary/providers/native/nativePlistBinarySource.test.ts @@ -0,0 +1,232 @@ +import { open, truncate, writeFile, type FileHandle } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it, onTestFinished, vi } from "vitest"; + +import { + AnalysisCancelledError, + AnalysisInputError, + AnalysisResourceConstraintError, +} from "../../../../src/domain/analysisErrorCore.js"; +import { ArtifactFileCleanupFailure } from "../../../../src/artifacts/ArtifactFileOperation.js"; +import { scanBinaryPlistScalars } from "../../../../src/artifacts/apple/PlistScalars.js"; +import { ProviderAdapterError } from "../../../../src/domain/providerAdapterError.js"; +import { buildBinary } from "plist"; +import { NativeMacOSProvider } from "../../../../src/native/NativeMacOSProvider.js"; +import { MAX_NATIVE_PLIST_BYTES } from "../../../../src/native/parsers/plist.js"; +import { + NativeFixtureRunner, + nativeMachoTarget, +} from "../../../fixtures/nativeCommands.js"; +import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; + +class BinaryNativeRunner extends NativeFixtureRunner { + readonly tools: string[] = []; + + override async run(tool: string, arguments_: readonly string[]) { + this.tools.push(tool); + return super.run(tool, arguments_); + } +} + +const binaryPath = async () => { + const root = await createTestTempDirectory("rea-native-binary-plist-"); + const path = join(root, "selected.plist"); + await writeFile(path, Buffer.from("bplist00")); + return { root, path }; +}; + +const binaryClient = (root: string, runner: BinaryNativeRunner) => { + const client = new NativeMacOSProvider({}, runner, "darwin").createClient( + nativeMachoTarget(join(root, "target")), + ); + onTestFinished(async () => { + await client.close(); + }); + return client; +}; + +it("reports a binary source-size refusal as a typed resource limit", async () => { + const { root, path } = await binaryPath(); + await truncate(path, MAX_NATIVE_PLIST_BYTES + 1); + const runner = new BinaryNativeRunner({ + file: "Apple binary property list\n", + }); + const client = binaryClient(root, runner); + + const result = await client.execute("inspect_plist", { path }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected file-size refusal"); + expect(result.error).toBeInstanceOf(AnalysisResourceConstraintError); + expect(result.error).toMatchObject({ + resource: "file-size", + reportedLimits: { maximum_input_bytes: MAX_NATIVE_PLIST_BYTES }, + retryable: false, + }); + expect(runner.tools).toEqual(["file"]); +}); + +it("keeps a malformed binary plist as a selected-input failure", async () => { + const { root, path } = await binaryPath(); + await writeFile(path, Buffer.from("bplist00bad")); + const runner = new BinaryNativeRunner({ + file: "Apple binary property list\n", + }); + const client = binaryClient(root, runner); + + const result = await client.execute("inspect_plist", { path }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected malformed input failure"); + expect(result.error).toBeInstanceOf(AnalysisInputError); + expect(runner.tools).toEqual(["file"]); +}); + +it("rejects a reachable binary date with an invalid marker before projection", async () => { + const { root, path } = await binaryPath(); + const bytes = Buffer.from( + buildBinary({ date: new Date("2001-01-01T00:00:00.100Z") }), + ); + const date = scanBinaryPlistScalars(bytes).dates[0]; + if (date === undefined) throw new Error("binary plist fixture lost its date"); + bytes[date.offset] = 0x30; + await writeFile(path, bytes); + const runner = new BinaryNativeRunner({ + file: "Apple binary property list\n", + }); + const client = binaryClient(root, runner); + + const result = await client.execute("inspect_plist", { path }); + + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected malformed date input failure"); + expect(result.error).toBeInstanceOf(AnalysisInputError); + expect(result.error).toMatchObject({ + issues: [ + { + path: ["path"], + reason: "invalid_value", + }, + ], + }); + expect(runner.tools).toEqual(["file"]); +}); + +it("preserves the exact caller cancellation reason during a binary read", async () => { + const { root, path } = await binaryPath(); + const probe = await open(path, "r"); + const prototype = Object.getPrototypeOf(probe) as FileHandle; + await probe.close(); + const originalRead = prototype.read; + const reason = { caller: "binary-plist-read" }; + const controller = new AbortController(); + const read = vi + .spyOn(prototype, "read") + .mockImplementationOnce(async function ( + this: FileHandle, + ...args: Parameters + ) { + const result = await Reflect.apply(originalRead, this, args); + controller.abort(reason); + return result; + }); + onTestFinished(() => read.mockRestore()); + const client = binaryClient( + root, + new BinaryNativeRunner({ file: "Apple binary property list\n" }), + ); + + const result = await client.execute( + "inspect_plist", + { path }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected cancellation"); + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cause).toBe(reason); +}); + +it("preserves a physical EIO when a pending binary read is then aborted", async () => { + const { root, path } = await binaryPath(); + const probe = await open(path, "r"); + const prototype = Object.getPrototypeOf(probe) as FileHandle; + await probe.close(); + const reason = { caller: "abort-then-eio" }; + const primary = Object.assign(new Error("controlled binary read EIO"), { + code: "EIO", + }); + const controller = new AbortController(); + const read = vi + .spyOn(prototype, "read") + .mockImplementationOnce(async function (this: FileHandle) { + controller.abort(reason); + throw primary; + }); + onTestFinished(() => read.mockRestore()); + const client = binaryClient( + root, + new BinaryNativeRunner({ file: "Apple binary property list\n" }), + ); + + const result = await client.execute( + "inspect_plist", + { path }, + { signal: controller.signal }, + ); + + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected physical read failure"); + expect(result.error).toBeInstanceOf(ProviderAdapterError); + expect(result.error.cause).toBe(primary); +}); + +it("retains the exact binary descriptor owner when reading and closing fail", async () => { + const { root, path } = await binaryPath(); + const probe = await open(path, "r"); + const prototype = Object.getPrototypeOf(probe) as FileHandle; + await probe.close(); + const primary = Object.assign(new Error("controlled binary read failure"), { + code: "EIO", + }); + const cleanup = new Error("controlled binary descriptor close failure"); + let attempts = 0; + let release: (() => Promise) | undefined; + const read = vi + .spyOn(prototype, "read") + .mockImplementationOnce(async function (this: FileHandle) { + release = this.close.bind(this); + this.close = async () => { + attempts += 1; + if (attempts < 3) throw cleanup; + await release!(); + }; + throw primary; + }); + onTestFinished(async () => { + read.mockRestore(); + await release?.(); + }); + const client = binaryClient( + root, + new BinaryNativeRunner({ file: "Apple binary property list\n" }), + ); + + const result = await client.execute("inspect_plist", { path }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected failed read and close"); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.cleanupResources).toContain(path); + expect(result.error.cause).toBeInstanceOf(ArtifactFileCleanupFailure); + const failure = result.error.cause as ArtifactFileCleanupFailure; + expect(failure.fileCleanupFailure.outcome).toEqual({ + kind: "failed", + cause: primary, + }); + expect(failure.fileCleanupFailure.cleanupCause).toBe(cleanup); + + expect((await client.close()).ok).toBe(false); + expect(attempts).toBe(2); + expect((await client.close()).ok).toBe(true); + expect(attempts).toBe(3); + expect(result.error.cleanupIncomplete).toBe(false); +}); diff --git a/tests/boundary/providers/native/plistSourceFormat.test.ts b/tests/boundary/providers/native/plistSourceFormat.test.ts index ab267b5768..2af7d83871 100644 --- a/tests/boundary/providers/native/plistSourceFormat.test.ts +++ b/tests/boundary/providers/native/plistSourceFormat.test.ts @@ -2,6 +2,7 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, it } from "vitest"; +import { buildBinary } from "plist"; import { inspectPlistSchema } from "../../../../src/domain/native/nativeInspection.js"; import { ok } from "../../../../src/domain/result.js"; @@ -49,6 +50,8 @@ interface PlistSourceCase { readonly xmlFallback: boolean; readonly convertedXml?: string; readonly expectedValue?: unknown; + readonly expectedTools?: readonly string[]; + readonly expectedLimitations?: readonly unknown[]; } it.for([ @@ -110,8 +113,42 @@ it.for([ }, { description: "Apple binary property list\n", + contents: Buffer.from( + buildBinary({ + nel: "a\u0085b", + ls: "a\u2028b", + ps: "a\u2029b", + lf: "a\nb", + payload: Buffer.from([0]), + }), + ), expected: "binary", + xmlFallback: false, + expectedTools: ["file"], + expectedValue: { + nel: "a\u0085b", + ls: "a\u2028b", + ps: "a\u2029b", + lf: "a\nb", + payload: { $plist_type: "data", base64: "AA==" }, + }, + expectedLimitations: [], + }, + { + description: "XML 1.0 document text, ASCII text\n", + contents: + "nela\u0085blsa\u2028bpsa\u2029blfa\nbpayloadAA==", + expected: "xml", xmlFallback: true, + convertedXml: + "nela\u0085blsa\u2028bpsa\u2029blfa\nbpayloadAA==", + expectedValue: { + nel: "a\u0085b", + ls: "a\u2028b", + ps: "a\u2029b", + lf: "a\nb", + payload: { $plist_type: "data", base64: "AA==" }, + }, }, ])( "keeps source format $expected for $description (XML fallback: $xmlFallback)", @@ -123,6 +160,8 @@ it.for([ xmlFallback, convertedXml = XML, expectedValue = { ...VALUE, ...(xmlFallback ? { Blob: DATA } : {}) }, + expectedTools, + expectedLimitations, }, { onTestFinished }, ) => { @@ -146,10 +185,12 @@ it.for([ expect(result.source_path).toBe(path); expect(result.value).toEqual(expectedValue); expect(result.provenance.map(({ tool }) => tool)).toEqual( - xmlFallback ? ["file", "plutil", "plutil"] : ["file", "plutil"], + expectedTools ?? + (xmlFallback ? ["file", "plutil", "plutil"] : ["file", "plutil"]), ); expect(result.limitations).toEqual( - xmlFallback ? [expect.stringContaining("XML conversion")] : [], + expectedLimitations ?? + (xmlFallback ? [expect.stringContaining("XML conversion")] : []), ); }, ); diff --git a/tests/boundary/rust/rustBinaryImage.test.ts b/tests/boundary/rust/rustBinaryImage.test.ts index d4aa12283f..4f94a4cbdc 100644 --- a/tests/boundary/rust/rustBinaryImage.test.ts +++ b/tests/boundary/rust/rustBinaryImage.test.ts @@ -2,11 +2,14 @@ import { expect, it } from "vitest"; import { fat } from "../../../src/domain/binaryTarget.fixture.js"; import { createRustImage, + createRustImageWithGoSection, createRustHashOverflowImage, RUSTC_HASH, rustDependencyPayload, } from "../../fixtures/rust/image.js"; import { readRustBinaryImage } from "../../../src/rust/RustBinaryImage.js"; +import { readBinaryContainer } from "../../../src/native/metadata/readBinaryContainer.js"; +import { readGoBinaryImage } from "../../../src/go/GoBinaryImage.js"; import { RustAuditableFormatFailure } from "../../../src/rust/RustAuditableReader.js"; const packages = [ @@ -139,3 +142,48 @@ it("refuses read-only data with more distinct rustc paths than it retains", () = }), ); }); + +it.each([ + ["elf", "compressed"], + ["elf", "unmapped"], + ["elf", "zero-fill"], + ["elf", "duplicate"], + ["elf", "out-of-file"], + ["elf", "zero-address"], + ["macho", "unmapped"], + ["macho", "zero-fill"], + ["macho", "duplicate"], + ["macho", "out-of-file"], +] as const)( + "retains Rust evidence beside an unrelated %s %s Go section", + (format, fault) => { + const bytes = createRustImageWithGoSection(format, fault); + const control = Buffer.from(bytes); + const sectionOffset = format === "elf" ? 768 + 11 : 408; + const sectionName = Buffer.from( + (format === "elf" ? ".other\0" : "__other\0").padEnd( + format === "elf" ? 13 : 16, + "\0", + ), + "ascii", + ); + if (format === "elf") { + const nameOffset = Number(bytes.readBigUInt64LE(320 + 32)); + sectionName.copy(control, 768 + nameOffset); + } else { + sectionName.copy(control, sectionOffset); + } + expect(readBinaryContainer(bytes).load_regions).toEqual( + readBinaryContainer(control).load_regions, + ); + expect(() => readGoBinaryImage(bytes)).toThrow(/build-info|mapped/i); + expect(readRustBinaryImage(bytes)).toMatchObject({ + format, + auditable_metadata: { file_offset: 1024, packages }, + rustc_commit: { + evidence_kind: "inferred", + hashes: [{ hash: RUSTC_HASH, file_offset: 2048 }], + }, + }); + }, +); diff --git a/tests/boundary/signals/recordingProvider.test.ts b/tests/boundary/signals/recordingProvider.test.ts index 0a75aa5671..d044497f39 100644 --- a/tests/boundary/signals/recordingProvider.test.ts +++ b/tests/boundary/signals/recordingProvider.test.ts @@ -1,5 +1,6 @@ import { AnalysisResourceConstraintError } from "../../../src/domain/analysisErrorCore.js"; import { signalDecodersSchema } from "../../../src/domain/signals/signalDecoders.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { OwnedCommandFailure } from "../../../src/process/OwnedCommand.js"; import { it, expect, onTestFinished } from "vitest"; import { writeFile, readdir, chmod, symlink } from "node:fs/promises"; @@ -28,6 +29,9 @@ const fixture = async (mode = "ok", failCleanup = false) => { if (process.argv.includes('--version')) { console.log(${JSON.stringify(mode)} === 'bad-version' ? 'sigrok-cli 0.7.1' : 'sigrok-cli 0.7.2\\nlibsigrokdecode 0.5.3'); console.error('version warning'); } else if (${JSON.stringify(mode)} === 'bad-version') console.log('unexpected version fallback'); else if (${JSON.stringify(mode)} === 'nonzero') { console.error('invalid decoder'); process.exitCode = 1; } +else if (${JSON.stringify(mode)} === 'unknown-decoder') { console.error("srd: ModuleNotFoundError: Failed to load decoder nosuchdecoder: import by name failed: No module named 'nosuchdecoder'\\nFailed to load protocol decoder nosuchdecoder."); process.exitCode = 1; } +else if (${JSON.stringify(mode)} === 'broken-decoder') { console.error("srd: ModuleNotFoundError: Failed to load decoder uart: import by name failed: No module named 'serial'\\nFailed to load protocol decoder uart."); process.exitCode = 1; } +else if (${JSON.stringify(mode)} === 'unknown-annotation') { console.error("Annotation 'nosuchclass' not found for protocol decoder 'uart'."); process.exitCode = 1; } else if (${JSON.stringify(mode)} === 'malformed-copy') { chmodSync('recording', 0o600); writeFileSync('recording', 'changed'); } else if (${JSON.stringify(mode)} === 'oversized-copy') { chmodSync('recording', 0o600); truncateSync('recording', 32 * 1024 * 1024 + 1); } else if (${JSON.stringify(mode)} === 'line-budget') process.stdout.write('ID: uart' + '\\n'.repeat(65536)); @@ -250,6 +254,68 @@ it.each([ }, ); +it.each([ + [ + "unknown-decoder", + { decoders: "uart:rx=RX,nosuchdecoder" }, + "decoders", + "Failed to load protocol decoder nosuchdecoder.", + ], + [ + "unknown-annotation", + { decoders: "uart", annotations: "uart=rx-data:nosuchclass" }, + "annotations", + "Annotation 'nosuchclass' not found for protocol decoder 'uart'.", + ], +] as const)( + "reports sigrok's %s refusal as invalid caller input", + async (mode, selection, field, refusal) => { + const f = await fixture(mode); + const result = await f.service.decode({ path: f.path, ...selection }); + if (result.ok) throw new Error("Expected selector refusal"); + expect(result.error._tag).toBe("AnalysisInputError"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "invalid_request", + message: expect.stringContaining("inspect_signal_decoders"), + details: { + issues: [ + { + path: [field], + reason: "invalid_value", + message: expect.stringContaining(refusal), + }, + ], + captured_output: { + exit_code: 1, + stderr: expect.stringContaining(refusal), + }, + }, + }); + expect(result.error.cleanupIncomplete).toBe(false); + expect( + (await readdir(f.directory)).filter((name) => + name.startsWith("decoder-"), + ), + ).toEqual([]); + }, +); + +it.each([ + // An installed decoder whose own dependency is missing is an installation fault. + ["broken-decoder", { decoders: "uart" }], + // Refusals naming selectors the caller did not supply stay unclassified. + ["unknown-decoder", { decoders: "uart" }], + ["unknown-annotation", { decoders: "uart", annotations: "uart=rx-data" }], +] as const)( + "keeps %s outside caller selectors as a provider failure", + async (mode, selection) => { + const f = await fixture(mode); + const result = await f.service.decode({ path: f.path, ...selection }); + if (result.ok) throw new Error("Expected decoder failure"); + expect(result.error._tag).toBe("ProviderAdapterError"); + }, +); + it.each(["decode", "discovery"] as const)( "does not attribute the version probe's output to a %s command that fails to launch", async (operation) => { diff --git a/tests/boundary/sqlite/sqliteDatabaseDescriptorCleanup.test.ts b/tests/boundary/sqlite/sqliteDatabaseDescriptorCleanup.test.ts index 1bbba73c7e..97fd11aabd 100644 --- a/tests/boundary/sqlite/sqliteDatabaseDescriptorCleanup.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseDescriptorCleanup.test.ts @@ -296,7 +296,16 @@ it("retains reply-read cancellation and retries the failed reply descriptor clos code: "cleanup_incomplete", details: { resources: [dirname(descriptorCleanup.path)], - diagnostics: { previous_error: cancelled }, + captured_output: cancelled.details.captured_output, + diagnostics: { + previous_error: { + code: cancelled.code, + details: { + execution_failure: "cancelled", + resources: cancelled.details.resources, + }, + }, + }, }, } : cancelled, diff --git a/tests/boundary/sqlite/sqliteDatabaseFailures.test.ts b/tests/boundary/sqlite/sqliteDatabaseFailures.test.ts new file mode 100644 index 0000000000..927151d0bf --- /dev/null +++ b/tests/boundary/sqlite/sqliteDatabaseFailures.test.ts @@ -0,0 +1,88 @@ +import { expect, it } from "vitest"; +import { OwnedCommandFailure } from "../../../src/process/OwnedCommand.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { sqliteDatabaseFailure } from "../../../src/sqlite/SqliteDatabaseFailures.js"; + +it.each([null, "owned stop unconfirmed"])( + "preserves a selected worker deadline with cleanup=%s", + (cleanup) => { + const cause = new OwnedCommandFailure( + "timeout", + "Command deadline elapsed.", + { + stdout: { text: "partial-owned-output", bytes: 20, observedBytes: 20 }, + stderr: { text: "", bytes: 0, observedBytes: 0 }, + exitCode: null, + signal: null, + }, + cleanup, + undefined, + ["pid:owned-fixture"], + ); + const failure = sqliteDatabaseFailure( + cause, + "/selected.db", + "worker", + undefined, + 1234, + ); + expect(failure.cause).toBe(cause); + const projected = projectAnalysisError(failure); + if (cleanup === null) + expect(projected).toMatchObject({ + code: "provider_timeout", + details: { timeout_ms: 1234 }, + remediation: { action: expect.stringContaining("--worker-timeout-ms") }, + }); + else + expect(projected).toMatchObject({ + code: "cleanup_incomplete", + retryable: false, + details: { + resources: ["pid:owned-fixture"], + diagnostics: { + previous_error: { + code: "provider_timeout", + details: { timeout_ms: 1234 }, + remediation: { + action: expect.stringContaining("worker_timeout_ms"), + }, + }, + }, + }, + }); + expect( + JSON.stringify(projected).split("partial-owned-output"), + ).toHaveLength(2); + if (cleanup !== null) { + expect(failure.cleanupPrimary).toMatchObject({ + _tag: "AnalysisTimeoutError", + timeoutMs: 1234, + }); + failure.resolveCleanup(); + const resolved = projectAnalysisError(failure); + expect(resolved).toMatchObject({ + code: "provider_timeout", + details: { timeout_ms: 1234 }, + }); + expect( + JSON.stringify(resolved).split("partial-owned-output"), + ).toHaveLength(2); + } + }, +); + +it("does not recommend a timer above the supported maximum", () => { + const failure = projectAnalysisError( + sqliteDatabaseFailure( + new OwnedCommandFailure("timeout", "deadline"), + "/selected.db", + "worker", + undefined, + 2_147_483_647, + ), + ); + expect(failure.details).toMatchObject({ timeout_ms: 2_147_483_647 }); + expect(failure.remediation.action).toContain("no higher worker deadline"); + expect(failure.remediation.action).not.toContain("Increase"); +}); diff --git a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts index 5075a1944f..b56ba9ba7d 100644 --- a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts @@ -51,6 +51,135 @@ const createFixture = async (wal = false) => { const expectPrivateRootRemoved = (path: string) => expect(access(path)).rejects.toMatchObject({ code: "ENOENT" }); +it("enforces a selected worker budget and completes the corrected same-database retry", async () => { + const { database } = await createFixture(); + const original = await readFile(database.path); + let first = true; + let root = ""; + const provider = new SqliteDatabaseProvider(process.env, (spawn) => { + root = spawn.cwd ?? ""; + if (first) { + first = false; + return spawnOwnedProviderProcess({ + ...spawn, + arguments: [ + spawn.arguments[0] ?? "--max-old-space-size=256", + "-e", + `process.argv[2]=process.argv[1];setTimeout(()=>import(${JSON.stringify(workerUrl.href)}),250)`, + spawn.arguments[2] ?? "", + ], + }); + } + return spawnOwnedProviderProcess({ + ...spawn, + arguments: spawn.arguments.map((arg, index) => + index === 1 ? fileURLToPath(workerUrl) : arg, + ), + }); + }); + onTestFinished(() => provider.close()); + const failed = await provider.inspect({ + path: database.path, + worker_timeout_ms: 50, + }); + if (failed.ok) throw new Error("Worker deadline was ignored"); + expect(projectAnalysisError(failed.error)).toMatchObject({ + code: "provider_timeout", + details: { timeout_ms: 50 }, + }); + expect(failed.error.cleanupIncomplete).toBe(false); + await expectPrivateRootRemoved(root); + const corrected = await provider.inspect({ + path: database.path, + worker_timeout_ms: 2000, + }); + if (!corrected.ok) throw corrected.error; + expect(corrected.value.result).toMatchObject({ + worker_timeout_ms: 2000, + schema: { completeness: "complete" }, + }); + await expectPrivateRootRemoved(root); + expect(await readFile(database.path)).toEqual(original); +}); + +nativeTest( + "retains a timed-out worker's budget and one output copy through owned cleanup retry", + async () => { + const { database } = await createFixture(); + const original = await readFile(database.path); + let cleanupAllowed = false; + let launches = 0; + let root = ""; + const provider = new SqliteDatabaseProvider(process.env, async (spawn) => { + launches += 1; + root = spawn.cwd ?? ""; + const launched = await spawnOwnedProviderProcess({ + ...spawn, + arguments: [ + "-e", + "process.stdout.write('ready\\n');setTimeout(()=>process.stdout.write('sqlite-timeout-marker'),100);setInterval(()=>{},1000)", + ], + }); + return { + ...launched, + cleanup: async () => + cleanupAllowed + ? await (launched.cleanup?.() ?? + cleanupOwnedProcessGroup(launched.ownership)) + : { cleaned: false, reason: "owned timeout cleanup denied" }, + }; + }); + onTestFinished(async () => { + cleanupAllowed = true; + await provider.close(); + }); + const result = await provider.inspect({ + path: database.path, + worker_timeout_ms: 3000, + }); + if (result.ok) throw new Error("Expected timeout with unresolved cleanup"); + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + code: "cleanup_incomplete", + retryable: false, + details: { + resources: expect.arrayContaining([root]), + diagnostics: { + previous_error: { + code: "provider_timeout", + details: { timeout_ms: 3000 }, + remediation: { + action: expect.stringContaining("worker_timeout_ms"), + }, + }, + }, + }, + }); + expect( + JSON.stringify(projected).split("sqlite-timeout-marker"), + ).toHaveLength(2); + await expect(access(root)).resolves.toBeUndefined(); + expect(await provider.inspect({ path: database.path })).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + expect(launches).toBe(1); + cleanupAllowed = true; + await provider.close(); + const resolved = projectAnalysisError(result.error); + expect(resolved).toMatchObject({ + code: "provider_timeout", + details: { timeout_ms: 3000 }, + }); + expect( + JSON.stringify(resolved).split("sqlite-timeout-marker"), + ).toHaveLength(2); + expect(result.error.cleanupResources).toEqual([]); + await expectPrivateRootRemoved(root); + expect(await readFile(database.path)).toEqual(original); + }, +); + it.each(["completed reply", "malformed reply"] as const)( "preserves the primary result for a %s while retaining unconfirmed cleanup ownership", async (replyKind) => { @@ -291,7 +420,13 @@ it.each(["late-cancellation", "cleanup-failure"])( await provider.close(); }); const result = await provider.inspect( - { path: database.path, table: "records", columns: ["id"], row_limit: 1 }, + { + path: database.path, + table: "records", + columns: ["id"], + row_limit: 1, + worker_timeout_ms: 42000, + }, { signal: controller.signal }, ); if (scenario === "late-cancellation") { @@ -313,8 +448,12 @@ it.each(["late-cancellation", "cleanup-failure"])( table: "records", columns: ["id"], row_limit: 1, + worker_timeout_ms: 42000, + }, + normalized_result: { + worker_timeout_ms: 42000, + rows: { columns: ["id"], returned_rows: 1 }, }, - normalized_result: { rows: { columns: ["id"], returned_rows: 1 } }, }, }, }); @@ -338,8 +477,12 @@ it.each(["late-cancellation", "cleanup-failure"])( table: "records", columns: ["id"], row_limit: 1, + worker_timeout_ms: 42000, + }, + normalized_result: { + worker_timeout_ms: 42000, + rows: { columns: ["id"], returned_rows: 1 }, }, - normalized_result: { rows: { columns: ["id"], returned_rows: 1 } }, }, }, }); diff --git a/tests/fixtures/binaryPlistObjectTable.ts b/tests/fixtures/binaryPlistObjectTable.ts new file mode 100644 index 0000000000..a258f529b3 --- /dev/null +++ b/tests/fixtures/binaryPlistObjectTable.ts @@ -0,0 +1,72 @@ +export interface BinaryPlistObjectTableLayout { + readonly offsetWidth: number; + readonly objectCount: number; + readonly tableOffset: number; +} + +/** Read the trailer fields that locate a binary plist's object offset table. */ +export const readBinaryPlistObjectTableLayout = ( + bytes: Buffer, +): BinaryPlistObjectTableLayout => { + const trailer = bytes.length - 32; + return { + offsetWidth: bytes[trailer + 6] ?? 0, + objectCount: Number(bytes.readBigUInt64BE(trailer + 8)), + tableOffset: Number(bytes.readBigUInt64BE(trailer + 24)), + }; +}; + +/** Decode one object's offset-table entry using the table's declared width. */ +export const readBinaryPlistObjectOffset = ( + bytes: Buffer, + table: BinaryPlistObjectTableLayout, + objectId: number, +): number => { + const entry = table.tableOffset + objectId * table.offsetWidth; + let offset = 0; + for (let byte = 0; byte < table.offsetWidth; byte += 1) + offset = offset * 256 + (bytes[entry + byte] ?? 0); + return offset; +}; + +/** Repoint one object while preserving the table's declared offset width. */ +export const writeBinaryPlistObjectOffset = ( + bytes: Buffer, + table: BinaryPlistObjectTableLayout, + objectId: number, + offset: number, +): void => { + const entry = table.tableOffset + objectId * table.offsetWidth; + let remaining = offset; + for (let byte = table.offsetWidth - 1; byte >= 0; byte -= 1) { + bytes[entry + byte] = remaining & 0xff; + remaining = Math.floor(remaining / 256); + } + if (remaining !== 0) throw new Error("fixture offset width is insufficient"); +}; + +/** Finish a small hand-authored binary plist with one-byte object offsets. */ +export const binaryPlistObjectTable = ( + bytes: Buffer, + objectOffsets: readonly number[], + tableOffset: number, +): Buffer => { + Buffer.from("bplist00", "ascii").copy(bytes); + bytes.set(objectOffsets, tableOffset); + const trailer = bytes.length - 32; + bytes[trailer + 6] = 1; + bytes[trailer + 7] = 1; + bytes.writeBigUInt64BE(BigInt(objectOffsets.length), trailer + 8); + bytes.writeBigUInt64BE(0n, trailer + 16); + bytes.writeBigUInt64BE(BigInt(tableOffset), trailer + 24); + return bytes; +}; + +/** Date and empty-data objects deliberately share bytes in a valid offset table. */ +export const binaryPlistWithOverlappingDateAndData = (): Buffer => { + const bytes = Buffer.alloc(55); + bytes.set([0xa2, 1, 2, 0x33], 8); + bytes.writeDoubleBE(0.0005, 12); + bytes[13] = 0x40; + return binaryPlistObjectTable(bytes, [8, 11, 13], 20); +}; diff --git a/tests/fixtures/binaryPlistScalarAlias.ts b/tests/fixtures/binaryPlistScalarAlias.ts new file mode 100644 index 0000000000..9610cb592f --- /dev/null +++ b/tests/fixtures/binaryPlistScalarAlias.ts @@ -0,0 +1,47 @@ +import { scanBinaryPlistScalars } from "../../src/artifacts/apple/PlistScalars.js"; +import { + readBinaryPlistObjectOffset, + readBinaryPlistObjectTableLayout, + writeBinaryPlistObjectOffset, +} from "./binaryPlistObjectTable.js"; + +/** Place a reachable date object inside a 16-byte data object's payload. */ +export const embedSubmillisecondDateInWideIntegerPayload = ( + source: Buffer, + targetDateSeconds?: number, +): { readonly bytes: Buffer; readonly integerDecimal: string } => { + const bytes = Buffer.from(source); + const dates = scanBinaryPlistScalars(bytes).dates; + const date = + targetDateSeconds === undefined + ? dates[0] + : dates.find( + ({ secondsSince2001 }) => secondsSince2001 === targetDateSeconds, + ); + if (date === undefined) throw new Error("binary plist fixture lost its date"); + const objectTable = readBinaryPlistObjectTableLayout(bytes); + let dataOffset: number | undefined; + let dateObjectId: number | undefined; + for (let objectId = 0; objectId < objectTable.objectCount; objectId += 1) { + const offset = readBinaryPlistObjectOffset(bytes, objectTable, objectId); + if (offset === date.offset) dateObjectId = objectId; + if (bytes[offset] === 0x4f) dataOffset = offset; + } + if (dataOffset === undefined || dateObjectId === undefined) + throw new Error( + "binary plist fixture lost its wide integer or date object", + ); + + const dateOffset = dataOffset + 4; + bytes[dataOffset] = 0x14; + bytes.fill(0, dataOffset + 1, dataOffset + 17); + bytes[dateOffset] = 0x33; + bytes.writeDoubleBE(0.1005, dateOffset + 1); + writeBinaryPlistObjectOffset(bytes, objectTable, dateObjectId, dateOffset); + const integer = BigInt.asIntN( + 128, + (bytes.readBigUInt64BE(dataOffset + 1) << 64n) | + bytes.readBigUInt64BE(dataOffset + 9), + ); + return { bytes, integerDecimal: integer.toString() }; +}; diff --git a/tests/fixtures/environmentExportInputProbe.mjs b/tests/fixtures/environmentExportInputProbe.mjs new file mode 100644 index 0000000000..09d19cd302 --- /dev/null +++ b/tests/fixtures/environmentExportInputProbe.mjs @@ -0,0 +1,41 @@ +import { promises as fs } from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; + +const path = process.env.REA_TEST_EXPORT_INPUT; +const marker = process.env.REA_TEST_EXPORT_READ_MARKER; +const action = process.env.REA_TEST_EXPORT_READ_ACTION; +if (!path || !marker || !["append", "record", "fail-copy"].includes(action)) + throw new Error("Expected an explicit export input, read marker and action"); + +// Instrument only the selected real descriptor in this disposable CLI process. +// Reads and metadata checks remain the production filesystem operations. +const open = fs.open; +fs.open = async (...args) => { + const handle = await open(...args); + if (args[0] !== path) return handle; + const read = handle.read.bind(handle); + let recorded = false; + let hashed = false; + handle.read = async (...readArgs) => { + if (action === "fail-copy" && hashed) { + await fs.writeFile(marker, "copy"); + throw Object.assign(new Error("controlled pathless descriptor failure"), { + code: "EIO", + syscall: "read", + }); + } + const result = await read(...readArgs); + if (action === "fail-copy") { + hashed = result.bytesRead === 0; + return result; + } + if (!recorded) { + recorded = true; + await fs.writeFile(marker, "read"); + if (action === "append") await fs.appendFile(path, " "); + } + return result; + }; + return handle; +}; +syncBuiltinESMExports(); diff --git a/tests/fixtures/fakeCdpBrowser.ts b/tests/fixtures/fakeCdpBrowser.ts index f06300f065..646b6c0360 100644 --- a/tests/fixtures/fakeCdpBrowser.ts +++ b/tests/fixtures/fakeCdpBrowser.ts @@ -60,6 +60,8 @@ export const startFakeCdpBrowser = async ( const commandError = options.commandError?.(command); if (commandError !== undefined) { sendCommandReply(socket, command, { error: commandError }); + if (options.closeAfterMethod === command.method) + setImmediate(() => socket.close()); return; } if (options.closeOnMethod === command.method) { diff --git a/tests/fixtures/fakeCdpBrowserEvents.ts b/tests/fixtures/fakeCdpBrowserEvents.ts index 294c2ad567..20d4ef0733 100644 --- a/tests/fixtures/fakeCdpBrowserEvents.ts +++ b/tests/fixtures/fakeCdpBrowserEvents.ts @@ -305,6 +305,8 @@ const emitSessionTimeline = ( sessionId: command.sessionId, targetId: "allowed-page", }); + if (options.sessionTimeline === "target_crashed") + event(socket, "Inspector.targetCrashed", command.sessionId, {}); } }; diff --git a/tests/fixtures/fakeCdpBrowserTypes.ts b/tests/fixtures/fakeCdpBrowserTypes.ts index 56b03659b6..08472f451c 100644 --- a/tests/fixtures/fakeCdpBrowserTypes.ts +++ b/tests/fixtures/fakeCdpBrowserTypes.ts @@ -92,7 +92,8 @@ export interface FakeOptions { readonly sessionTimeline?: | "same_origin" | "outside_policy" - | "target_detached"; + | "target_detached" + | "target_crashed"; readonly sessionTimelineEventCount?: number; readonly sessionRedirectResourceType?: "Document" | "Fetch" | "Script"; readonly closeAfterMethod?: string; diff --git a/tests/fixtures/keyedArchiveWideIntegers.ts b/tests/fixtures/keyedArchiveWideIntegers.ts index 4bf4362488..91e708f475 100644 --- a/tests/fixtures/keyedArchiveWideIntegers.ts +++ b/tests/fixtures/keyedArchiveWideIntegers.ts @@ -27,6 +27,58 @@ const integer64 = (value: bigint): Buffer => { return bytes; }; +/** + * Minimal valid NSKeyedArchiver whose object-table entries 10 and 11 both + * point at one Int64 record, beside an unrelated Int64 sibling (entry 12). + * The shared record is encoded once; only its offset is listed twice. + */ +const sharedEntryArchiveBytes = (): Buffer => { + const objects = [ + Buffer.from([0xd3, 1, 3, 4, 2, 8, 5]), // root dict: three key/value pairs + ascii("$archiver"), + ascii("NSKeyedArchiver"), + ascii("$objects"), + ascii("$top"), + Buffer.from([0xd1, 6, 7]), // $top.root -> UID 1 + ascii("root"), + Buffer.from([0x80, 1]), // UID 1 + Buffer.from([0xa4, 9, 10, 11, 12]), // $objects refs + ascii("$null"), + integer64(9_007_199_254_740_993n), + integer64(1_152_921_504_606_846_979n), + ]; + const header = Buffer.from("bplist00", "ascii"); + const offsets: number[] = []; + let position = header.length; + for (const object of objects) { + offsets.push(position); + position += object.length; + } + // Entries 10 and 11 share the first Int64 record; entry 12 is the sibling. + const [shared, sibling] = [offsets[10] ?? 0, offsets[11] ?? 0]; + offsets.splice(11, 1, shared, sibling); + const trailer = Buffer.alloc(32); + trailer[6] = 1; + trailer[7] = 1; + trailer.writeBigUInt64BE(BigInt(offsets.length), 8); + trailer.writeBigUInt64BE(0n, 16); + trailer.writeBigUInt64BE(BigInt(position), 24); + return Buffer.concat([header, ...objects, Buffer.from(offsets), trailer]); +}; + +export const keyedArchiveSharedEntryFixture = async () => { + const root = await createTestTempDirectory("rea-keyed-shared-entry-"); + const path = join(root, "archive.plist"); + const bytes = sharedEntryArchiveBytes(); + await writeFile(path, bytes); + return { + root, + path, + bytes, + digest: createHash("sha256").update(bytes).digest("hex"), + }; +}; + /** Minimal valid NSKeyedArchiver with a 16-byte integer and an Int64 control. */ const archiveBytes = (): Buffer => { const objects = [ diff --git a/tests/fixtures/pyinstallerReader.ts b/tests/fixtures/pyinstallerReader.ts new file mode 100644 index 0000000000..495ad9942c --- /dev/null +++ b/tests/fixtures/pyinstallerReader.ts @@ -0,0 +1,31 @@ +import { open, writeFile, type FileHandle } from "node:fs/promises"; +import { join } from "node:path"; +import { PyInstallerArtifactReader } from "../../src/artifacts/PyInstallerArtifactReader.js"; +import { createTestTempDirectory } from "./temporaryDirectory.js"; + +export const withPyInstallerReader = async ( + bytes: Buffer, + run: ( + reader: PyInstallerArtifactReader, + handle: FileHandle, + path: string, + ) => Promise, +) => { + const root = await createTestTempDirectory("rea-pyi-boundary-"); + const path = join(root, "app"); + await writeFile(path, bytes); + const handle = await open(path, "r"); + const reader = new PyInstallerArtifactReader(path, { + handle, + initial: await handle.stat(), + }); + try { + await run(reader, handle, path); + } finally { + try { + await reader.close(); + } finally { + await handle.close(); + } + } +}; diff --git a/tests/fixtures/rust/image.ts b/tests/fixtures/rust/image.ts index 6bbac25abe..c80a40ad24 100644 --- a/tests/fixtures/rust/image.ts +++ b/tests/fixtures/rust/image.ts @@ -261,3 +261,75 @@ export const createRustHashOverflowImage = (): Buffer => { } return bytes; }; + +/** Valid Rust evidence beside an independently invalid Go metadata section. */ +export const createRustImageWithGoSection = ( + format: "elf" | "macho", + fault: + | "compressed" + | "unmapped" + | "zero-fill" + | "duplicate" + | "out-of-file" + | "zero-address", +): Buffer => { + const bytes = createRustImage({ format }); + const count = fault === "duplicate" ? 2 : 1; + if (format === "elf") { + const nameOffset = Number(bytes.readBigUInt64LE(320 + 32)); + const name = Buffer.from(".go.buildinfo\0"); + name.copy(bytes, 768 + nameOffset); + bytes.writeBigUInt64LE(BigInt(nameOffset + name.length), 320 + 32); + bytes.writeUInt16LE(4 + count, 60); + for (let index = 0; index < count; index++) { + const header = 512 + index * 64; + bytes.writeUInt32LE(nameOffset, header); + bytes.writeUInt32LE(fault === "zero-fill" ? 8 : 1, header + 4); + bytes.writeBigUInt64LE(fault === "compressed" ? 0x800n : 0n, header + 8); + bytes.writeBigUInt64LE( + fault === "zero-address" + ? 0n + : fault === "unmapped" + ? 0x600000n + : 0x400c00n, + header + 16, + ); + bytes.writeBigUInt64LE( + fault === "out-of-file" ? 4088n : 3072n, + header + 24, + ); + bytes.writeBigUInt64LE(fault === "zero-address" ? 16n : 32n, header + 32); + if (fault === "zero-address") + Buffer.from("additional data\0", "ascii").copy(bytes, 3072); + } + } else { + if (fault === "compressed") throw new Error("Not a Mach-O section flag"); + if (fault === "zero-address") throw new Error("Not an ELF section address"); + bytes.writeUInt32LE(2 + count, 16); + bytes.writeUInt32LE((2 + count) * 152, 20); + for (let index = 0; index < count; index++) { + const command = 336 + index * 152; + const little: RustImageWrite["little"] = (offset, value, width) => { + if (width === 16) bytes.writeUInt16LE(value, offset); + else if (width === 32) bytes.writeUInt32LE(value, offset); + else bytes.writeBigUInt64LE(BigInt(value), offset); + }; + writeMachoSegment({ + bytes, + little, + command, + segment: "__EXTRA", + section: "__go_buildinfo", + body: Buffer.alloc(32), + fileOffset: 3072, + protection: fault === "unmapped" ? 0 : 3, + flags: fault === "zero-fill" ? 1 : 0, + }); + if (fault === "out-of-file") bytes.writeUInt32LE(4088, command + 72 + 48); + bytes.writeBigUInt64LE(0x200000000n, command + 24); + bytes.writeBigUInt64LE(0x200000000n, command + 72 + 32); + if (fault === "unmapped") bytes.writeBigUInt64LE(0n, command + 32); + } + } + return bytes; +}; diff --git a/tests/support/cli/cliCommandSession.ts b/tests/support/cli/cliCommandSession.ts new file mode 100644 index 0000000000..1805836054 --- /dev/null +++ b/tests/support/cli/cliCommandSession.ts @@ -0,0 +1,448 @@ +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +import { z } from "zod"; + +import { waitForExit } from "../process/processFixture.js"; +import { + createCliSessionFrameReader, + encodeCliSessionFrame, +} from "./cliCommandSessionFrames.mjs"; + +const workerPath = fileURLToPath( + new URL("./cliCommandSessionWorker.mjs", import.meta.url), +); +const SHUTDOWN_GRACE_MS = 1_000; +/** Restart after this many commands so one file cannot retain every heap. */ +const COMMAND_REUSE_LIMIT = 32; +/** Restart when the worker reports a resident set at least this large. */ +const RESIDENT_REUSE_LIMIT_BYTES = 1024 * 1024 * 1024; + +const responseSchema = z.object({ + ready: z.boolean().optional(), + id: z.number().optional(), + exitCode: z.number().nullable().optional(), + stdout: z.string().optional(), + stderr: z.string().optional(), + pid: z.number().optional(), + rss: z.number().finite().nonnegative().optional(), + fatal: z.boolean().optional(), +}); + +/** One compiled CLI command sent to the shared process. */ +export interface CliCommandRequest { + readonly arguments: readonly string[]; + readonly cwd: string; + readonly environment: Readonly>; +} + +/** Captured result from one shared CLI command. */ +export interface CliCommandResult { + readonly exitCode: number | null; + readonly signal: NodeJS.Signals | null; + readonly stdout: string; + readonly stderr: string; + readonly json: unknown | undefined; +} + +/** File-scoped compiled CLI process. */ +export interface CliCommandSession { + run(request: CliCommandRequest): Promise; + /** PID of the live process, when one is running. */ + processId(): number | undefined; + close(): Promise; +} + +interface CliCommandSessionDependencies { + readonly spawnWorker?: ( + environment: Readonly>, + ) => ChildProcessWithoutNullStreams; + /** Commands served by one process before the next command starts a new one. */ + readonly commandLimit?: number; + /** Resident-set bytes, reported by the worker, that start a new process. */ + readonly residentLimitBytes?: number; +} + +interface PendingCommand { + readonly id: number; + readonly resolve: (result: CliCommandResult) => void; + readonly reject: (error: unknown) => void; +} + +type SessionResponse = z.infer; + +const closedError = (): Error => new Error("Shared CLI session is closed"); + +const positiveLimit = (value: number | undefined, fallback: number): number => + value !== undefined && Number.isSafeInteger(value) && value > 0 + ? value + : fallback; + +const envKey = (environment: Readonly>): string => + JSON.stringify( + Object.keys(environment) + .sort() + .map((key) => [key, environment[key]]), + ); + +const parseCliStdout = (stdout: string): unknown | undefined => { + const document = stdout.trim(); + if (document.length === 0) return undefined; + try { + return JSON.parse(document) as unknown; + } catch { + return undefined; + } +}; + +const signalChild = ( + child: ChildProcessWithoutNullStreams, + signal: NodeJS.Signals, +): void => { + try { + child.kill(signal); + } catch (error) { + const code = + error instanceof Error && "code" in error ? error.code : undefined; + if (code !== "ESRCH") throw error; + } +}; + +const stopChild = async ( + child: ChildProcessWithoutNullStreams, +): Promise => { + if (child.exitCode !== null || child.signalCode !== null) return; + signalChild(child, "SIGTERM"); + if (await waitForExit(child, SHUTDOWN_GRACE_MS)) return; + if (child.exitCode !== null || child.signalCode !== null) return; + signalChild(child, "SIGKILL"); + if (await waitForExit(child, SHUTDOWN_GRACE_MS)) return; + throw new Error( + `shared CLI process ${String(child.pid)} did not exit after SIGKILL`, + ); +}; + +const commandResult = (message: SessionResponse): CliCommandResult => ({ + exitCode: message.exitCode ?? null, + signal: null, + stdout: message.stdout ?? "", + stderr: message.stderr ?? "", + json: parseCliStdout(message.stdout ?? ""), +}); + +const writeChild = ( + child: ChildProcessWithoutNullStreams, + frame: Buffer, +): Promise => + new Promise((resolve, reject) => { + child.stdin.write(frame, (error) => { + if (error) reject(error); + else resolve(); + }); + }); + +const defaultSpawnWorker = ( + environment: Readonly>, +): ChildProcessWithoutNullStreams => + spawn(process.execPath, [workerPath], { + cwd: process.cwd(), + env: environment, + stdio: ["pipe", "pipe", "pipe"], + }); + +class SharedCliProcess implements CliCommandSession { + readonly #spawnWorker: ( + environment: Readonly>, + ) => ChildProcessWithoutNullStreams; + + readonly #commandLimit: number; + readonly #residentLimitBytes: number; + #child: ChildProcessWithoutNullStreams | undefined; + #childEnvKey: string | undefined; + #commands = 0; + #residentBytes: number | undefined; + #retiring: Promise | undefined; + #closed = false; + #closing: Promise | undefined; + #starting: Promise | undefined; + #nextId = 1; + #pending: PendingCommand | undefined; + #resolveReady: (() => void) | undefined; + #rejectReady: ((error: unknown) => void) | undefined; + #queue: Promise = Promise.resolve(); + + constructor(dependencies: CliCommandSessionDependencies) { + this.#spawnWorker = dependencies.spawnWorker ?? defaultSpawnWorker; + this.#commandLimit = positiveLimit( + dependencies.commandLimit, + COMMAND_REUSE_LIMIT, + ); + this.#residentLimitBytes = positiveLimit( + dependencies.residentLimitBytes, + RESIDENT_REUSE_LIMIT_BYTES, + ); + } + + run(request: CliCommandRequest): Promise { + const result = this.#queue.then(() => this.#invoke(request)); + this.#queue = result.then( + () => undefined, + () => undefined, + ); + return result; + } + + processId(): number | undefined { + const child = this.#child; + if ( + child === undefined || + child.exitCode !== null || + child.signalCode !== null + ) { + return undefined; + } + return child.pid; + } + + close(): Promise { + this.#closed = true; + if (this.#closing !== undefined) return this.#closing; + const attempt = this.#shutdown(); + this.#closing = attempt; + return attempt; + } + + async #shutdown(): Promise { + try { + const owner = this.#child; + if (owner !== undefined) await stopChild(owner); + const retiring = this.#retiring; + if (retiring !== undefined) await retiring; + } catch (error) { + this.#closing = undefined; + throw error; + } + const pendingStart = this.#starting; + if (pendingStart !== undefined) await pendingStart.catch(() => undefined); + } + + async #invoke(request: CliCommandRequest): Promise { + if (this.#closed) throw closedError(); + try { + return await this.#run(request); + } finally { + await this.#finishRetirement(); + } + } + + async #run(request: CliCommandRequest): Promise { + await this.#ensure(request.environment); + const owner = this.#child; + if (this.#closed || owner === undefined) throw closedError(); + const id = this.#nextId; + this.#nextId += 1; + const result = new Promise((resolve, reject) => { + this.#pending = { id, resolve, reject }; + }); + try { + await writeChild( + owner, + encodeCliSessionFrame({ + id, + arguments: request.arguments, + cwd: request.cwd, + env: request.environment, + }), + ); + } catch (error) { + this.#failTransport(owner, error); + } + return await result; + } + + async #finishRetirement(): Promise { + const retiring = this.#retiring; + if (retiring === undefined) return; + try { + await retiring; + } finally { + if (this.#retiring === retiring) this.#retiring = undefined; + } + } + + #reusable( + current: ChildProcessWithoutNullStreams | undefined, + key: string, + ): current is ChildProcessWithoutNullStreams { + if ( + current === undefined || + current.exitCode !== null || + current.signalCode !== null || + this.#childEnvKey !== key + ) { + return false; + } + if (this.#commands >= this.#commandLimit) return false; + return !( + this.#residentBytes !== undefined && + this.#residentBytes >= this.#residentLimitBytes + ); + } + + async #ensure(environment: Readonly>): Promise { + if (this.#closed) throw closedError(); + const key = envKey(environment); + const current = this.#child; + if (this.#reusable(current, key)) return; + if (current !== undefined) await stopChild(current); + if (this.#closed) throw closedError(); + await this.#start(environment); + } + + #start(environment: Readonly>): Promise { + if (this.#closed) return Promise.reject(closedError()); + const started = this.#spawnWorker(environment); + this.#child = started; + this.#childEnvKey = envKey(environment); + this.#commands = 0; + this.#residentBytes = undefined; + const ready = new Promise((resolve, reject) => { + this.#resolveReady = resolve; + this.#rejectReady = reject; + }); + this.#starting = ready; + this.#watch(started); + if (this.#closed) { + signalChild(started, "SIGTERM"); + this.#failReady(closedError()); + } + return ready.finally(() => { + if (this.#starting === ready) this.#starting = undefined; + }); + } + + #watch(started: ChildProcessWithoutNullStreams): void { + const reader = createCliSessionFrameReader((frame) => { + this.#onFrame(started, frame); + }); + started.stdout.on("data", (chunk: Buffer) => { + try { + reader(chunk); + } catch (error) { + this.#failTransport(started, error); + } + }); + started.stderr.resume(); + started.stdin.on("error", (error) => { + this.#failTransport(started, error); + }); + started.once("error", (error) => { + this.#failTransport(started, error); + }); + started.once("close", (code, signal) => { + this.#onClose(started, code, signal); + }); + } + + #onFrame(owner: ChildProcessWithoutNullStreams, value: unknown): void { + if (this.#child !== owner || this.#closed) return; + const parsed = responseSchema.safeParse(value); + if (!parsed.success) { + this.#failTransport( + owner, + new Error("CLI session response was not a command result"), + ); + return; + } + const message = parsed.data; + if (message.ready === true && message.id === undefined) { + const resolve = this.#resolveReady; + this.#resolveReady = undefined; + this.#rejectReady = undefined; + resolve?.(); + return; + } + const current = this.#pending; + if (current === undefined || message.id !== current.id) { + this.#failTransport( + owner, + new Error("CLI session response did not match the command"), + ); + return; + } + if (message.exitCode === undefined) { + this.#failTransport( + owner, + new Error("CLI session response omitted its exit code"), + ); + return; + } + this.#pending = undefined; + if (message.fatal === true) { + this.#retire(owner); + } else { + this.#commands += 1; + if (message.rss !== undefined) this.#residentBytes = message.rss; + } + current.resolve(commandResult(message)); + } + + #retire(owner: ChildProcessWithoutNullStreams): void { + if (this.#child === owner) { + this.#child = undefined; + this.#childEnvKey = undefined; + } + this.#commands = 0; + this.#residentBytes = undefined; + this.#retiring = stopChild(owner); + } + + #onClose( + owner: ChildProcessWithoutNullStreams, + code: number | null, + signal: NodeJS.Signals | null, + ): void { + if (this.#child !== owner) return; + this.#child = undefined; + this.#childEnvKey = undefined; + const error = this.#closed + ? closedError() + : new Error( + `Shared CLI process exited (${String(code ?? signal ?? "unknown")})`, + ); + this.#failReady(error); + this.#failPending(error); + } + + #failTransport(owner: ChildProcessWithoutNullStreams, error: unknown): void { + if (this.#child !== owner) return; + this.#failReady(error); + this.#failPending(error); + // Clear the worker before the queued command runs. SIGTERM does not set + // signalCode until the process actually exits, so #reusable would still + // accept this process and write into it. + this.#retire(owner); + } + + #failPending(error: unknown): void { + const current = this.#pending; + this.#pending = undefined; + current?.reject(error); + } + + #failReady(error: unknown): void { + const reject = this.#rejectReady; + this.#resolveReady = undefined; + this.#rejectReady = undefined; + reject?.(error); + } +} + +/** + * Keep one compiled CLI process for commands that share an environment. + * The process restarts after its command limit, after a reported resident-set + * limit, when the environment changes, or when a command fails fatally. + */ +export const createCliCommandSession = ( + dependencies: CliCommandSessionDependencies = {}, +): CliCommandSession => new SharedCliProcess(dependencies); diff --git a/tests/support/cli/cliCommandSessionFrames.d.mts b/tests/support/cli/cliCommandSessionFrames.d.mts new file mode 100644 index 0000000000..85e6ae65f7 --- /dev/null +++ b/tests/support/cli/cliCommandSessionFrames.d.mts @@ -0,0 +1,13 @@ +/** Largest accepted session message, including the encoded command result. */ +export const MAX_CLI_SESSION_FRAME_BYTES: number; + +/** Length-prefixed JSON frame shared by the session parent and worker. */ +export function encodeCliSessionFrame(value: unknown): Buffer; + +/** + * Parse complete frames from a byte stream. An oversized or malformed frame + * fails the reader instead of resynchronizing on a later length. + */ +export function createCliSessionFrameReader( + onFrame: (value: unknown) => void, +): (chunk: Buffer | string) => void; diff --git a/tests/support/cli/cliCommandSessionFrames.mjs b/tests/support/cli/cliCommandSessionFrames.mjs new file mode 100644 index 0000000000..4f88c70cdd --- /dev/null +++ b/tests/support/cli/cliCommandSessionFrames.mjs @@ -0,0 +1,77 @@ +/** Largest accepted session message, including the encoded command result. */ +export const MAX_CLI_SESSION_FRAME_BYTES = 64 * 1024 * 1024; + +/** Length-prefixed JSON frame shared by the session parent and worker. */ +export const encodeCliSessionFrame = (value) => { + const body = Buffer.from(JSON.stringify(value)); + if (body.length > MAX_CLI_SESSION_FRAME_BYTES) { + throw new Error( + `CLI session frame is ${body.length} bytes; the limit is ${MAX_CLI_SESSION_FRAME_BYTES}`, + ); + } + const frame = Buffer.allocUnsafe(4 + body.length); + frame.writeUInt32BE(body.length, 0); + body.copy(frame, 4); + return frame; +}; + +const frameLimitError = (length) => + new Error( + `CLI session frame is ${length} bytes; the limit is ${MAX_CLI_SESSION_FRAME_BYTES}`, + ); + +/** Read a big-endian length from the start of a chunk list without joining it. */ +const readFrameLength = (chunks) => { + const first = chunks[0]; + if (first.length >= 4) return first.readUInt32BE(0); + const header = Buffer.allocUnsafe(4); + let offset = 0; + for (const chunk of chunks) { + const take = Math.min(chunk.length, 4 - offset); + chunk.copy(header, offset, 0, take); + offset += take; + if (offset === 4) break; + } + return header.readUInt32BE(0); +}; + +/** + * Parse complete frames from a byte stream. An oversized or malformed frame + * fails the reader instead of resynchronizing on a later length. Incomplete + * chunks stay separate until one whole frame is buffered, then that frame is + * joined once. + */ +export const createCliSessionFrameReader = (onFrame) => { + /** @type {Buffer[]} */ + const chunks = []; + let buffered = 0; + return (chunk) => { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + chunks.push(bytes); + buffered += bytes.length; + if (buffered < 4) return; + const length = readFrameLength(chunks); + if (length > MAX_CLI_SESSION_FRAME_BYTES) throw frameLimitError(length); + if (buffered < 4 + length) return; + const pending = Buffer.concat(chunks); + let offset = 0; + try { + while (offset + 4 <= pending.length) { + const frameLength = pending.readUInt32BE(offset); + if (frameLength > MAX_CLI_SESSION_FRAME_BYTES) { + throw frameLimitError(frameLength); + } + const frameEnd = offset + 4 + frameLength; + if (frameEnd > pending.length) break; + const body = pending.toString("utf8", offset + 4, frameEnd); + offset = frameEnd; + onFrame(JSON.parse(body)); + } + } finally { + const rest = pending.subarray(offset); + chunks.length = 0; + buffered = rest.length; + if (rest.length > 0) chunks.push(Buffer.from(rest)); + } + }; +}; diff --git a/tests/support/cli/cliCommandSessionWorker.mjs b/tests/support/cli/cliCommandSessionWorker.mjs new file mode 100644 index 0000000000..63ce69542c --- /dev/null +++ b/tests/support/cli/cliCommandSessionWorker.mjs @@ -0,0 +1,263 @@ +import { Writable } from "node:stream"; +import { access } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + createCliSessionFrameReader, + encodeCliSessionFrame, +} from "./cliCommandSessionFrames.mjs"; + +// One-shot command sequence matches scripts/rea.mjs. Output is captured per +// command so this process can serve the next invocation without restarting. +const packageRoot = resolve( + dirname(fileURLToPath(import.meta.url)), + "../../..", +); +const { default: packageJson } = await import( + new URL("../../../package.json", import.meta.url), + { with: { type: "json" } } +); + +const prepareCliProcess = () => { + process.env.REA_PACKAGE_VERSION = packageJson.version; + // Windows home lookup throws for an empty USERPROFILE. Dependencies resolve + // home during import, before REA's own resolver can apply its account fallback. + if (process.platform === "win32" && process.env.USERPROFILE === "") { + delete process.env.USERPROFILE; + } + if ( + process.platform === "android" && + process.env.PLAYWRIGHT_BROWSERS_PATH === undefined + ) { + const cacheRoot = + process.env.XDG_CACHE_HOME ?? join(process.env.HOME ?? ".", ".cache"); + process.env.PLAYWRIGHT_BROWSERS_PATH = join(cacheRoot, "ms-playwright"); + } +}; + +const runtimeFiles = [ + "dist/cli-runtime/cli.js", + "dist/cli-runtime/cliOutput.js", + "dist/cli-runtime/cliResultOutput.js", +]; + +prepareCliProcess(); +if (!(await compiledRuntimeExists(runtimeFiles))) { + process.stderr.write( + `REA's compiled runtime is missing. Run \`npm ci && npm run build:cached\` in ${packageRoot} to install dependencies and build REA, then restart it. If this is an installed package, reinstall rea-agents.\n`, + ); + process.exit(1); +} + +const { cliConfigurationError, createCli } = + await import("../../../dist/cli-runtime/cli.js"); +const { createCliResultOutput } = + await import("../../../dist/cli-runtime/cliResultOutput.js"); +const { + normalizeCliArguments, + renderCliOutputArgumentError, + renderEmptyFilteredCliOutput, + validateCliOutputArguments, +} = await import("../../../dist/cli-runtime/cliOutput.js"); + +const writeFrame = (value) => + new Promise((resolveWrite, rejectWrite) => { + process.stdout.write(encodeCliSessionFrame(value), (error) => { + if (error) rejectWrite(error); + else resolveWrite(); + }); + }); + +await writeFrame({ ready: true }); + +let chain = Promise.resolve(); +const reader = createCliSessionFrameReader((value) => { + chain = chain.then( + () => handle(value), + () => handle(value), + ); +}); +process.stdin.on("data", (chunk) => { + try { + reader(chunk); + } catch (error) { + process.stderr.write( + `${error instanceof Error ? error.message : String(error)}\n`, + ); + process.exit(1); + } +}); +process.stdin.on("end", () => { + chain.finally(() => process.exit(0)); +}); + +async function handle(value) { + const request = value; + const savedEnv = { ...process.env }; + const savedCwd = process.cwd(); + try { + if ( + request === null || + typeof request !== "object" || + !Array.isArray(request.arguments) || + typeof request.cwd !== "string" || + request.env === null || + typeof request.env !== "object" + ) { + throw new Error( + "CLI session command is missing its arguments, cwd, or environment", + ); + } + for (const key of Object.keys(process.env)) delete process.env[key]; + Object.assign(process.env, request.env); + prepareCliProcess(); + process.chdir(request.cwd); + const result = await runOneShot(request.arguments); + await writeFrame({ + id: request.id, + pid: process.pid, + rss: process.memoryUsage().rss, + ...result, + }); + } catch (error) { + const id = + request !== null && typeof request === "object" ? request.id : undefined; + await writeFrame({ + id, + pid: process.pid, + exitCode: 1, + stdout: "", + stderr: + error instanceof Error ? (error.stack ?? error.message) : String(error), + fatal: true, + }); + process.exit(1); + } finally { + for (const key of Object.keys(process.env)) delete process.env[key]; + Object.assign(process.env, savedEnv); + process.chdir(savedCwd); + } +} + +async function runOneShot(args) { + const stdoutChunks = []; + const stderrChunks = []; + const originalStdoutWrite = process.stdout.write.bind(process.stdout); + const originalStderrWrite = process.stderr.write.bind(process.stderr); + process.stdout.write = captureWrite(stdoutChunks); + process.stderr.write = captureWrite(stderrChunks); + const destination = new Writable({ + write(chunk, encoding, callback) { + process.stdout.write(chunk, encoding, callback); + }, + }); + destination.isTTY = process.stdout.isTTY === true; + const previousExitCode = process.exitCode; + process.exitCode = undefined; + let explicitExit; + try { + const cliArgs = normalizeCliArguments(args); + const outputArguments = validateCliOutputArguments(cliArgs); + const configurationError = cliConfigurationError(process.env); + if (configurationError !== undefined && !isDoctorCommand(cliArgs)) { + process.stderr.write(`${configurationError}\n`); + process.exitCode = 1; + } else if (!outputArguments.ok) { + process.stdout.write(renderCliOutputArgumentError(outputArguments)); + process.exitCode = 1; + } else { + let wroteOutput = false; + const resultOutput = createCliResultOutput(cliArgs, destination); + try { + await createCli(process.env, resultOutput).serve(cliArgs, { + stdout: (output) => { + if (resultOutput?.handled) { + if (resultOutput.failed) process.stderr.write(output); + return; + } + if (output.length > 0) wroteOutput = true; + process.stdout.write(output); + }, + exit: (code) => { + explicitExit = code; + }, + }); + } catch (cause) { + if (wroteOutput) throw cause; + const recovered = await resultOutput?.recoverFormattingFailure(cause); + if (!recovered) throw cause; + process.exitCode = 1; + } + if (!wroteOutput && !resultOutput?.handled) { + const empty = renderEmptyFilteredCliOutput(cliArgs); + if (empty) process.stdout.write(empty); + } + } + } finally { + process.stdout.write = originalStdoutWrite; + process.stderr.write = originalStderrWrite; + } + const exitCode = + explicitExit ?? + (typeof process.exitCode === "number" ? process.exitCode : 0); + process.exitCode = previousExitCode; + return { + exitCode, + stdout: Buffer.concat(stdoutChunks).toString("utf8"), + stderr: Buffer.concat(stderrChunks).toString("utf8"), + }; +} + +function captureWrite(chunks) { + return function patched(chunk, encoding, callback) { + const encodingName = typeof encoding === "string" ? encoding : "utf8"; + chunks.push( + Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk), encodingName), + ); + const done = typeof encoding === "function" ? encoding : callback; + if (typeof done === "function") done(); + return true; + }; +} + +function isDoctorCommand(arguments_) { + const valuedFlags = new Set([ + "--format", + "--filter-output", + "--token-limit", + "--token-offset", + ]); + for (let index = 0; index < arguments_.length; index += 1) { + const argument = arguments_[index]; + if (argument === "--") return arguments_[index + 1] === "doctor"; + if (valuedFlags.has(argument)) { + index += 1; + continue; + } + if (argument.startsWith("-")) continue; + return argument === "doctor"; + } + return false; +} + +async function compiledRuntimeExists(paths) { + for (const path of paths) { + try { + await access(resolve(packageRoot, path)); + } catch (cause) { + if (isMissing(cause)) return false; + throw cause; + } + } + return true; +} + +function isMissing(cause) { + return ( + typeof cause === "object" && + cause !== null && + "code" in cause && + cause.code === "ENOENT" + ); +} diff --git a/tests/support/cli/cliFixture.ts b/tests/support/cli/cliFixture.ts index 9d093c7da4..545ac465a0 100644 --- a/tests/support/cli/cliFixture.ts +++ b/tests/support/cli/cliFixture.ts @@ -10,6 +10,10 @@ import { type ProcessResult, type TestProcesses, } from "../process/processFixture.js"; +import { + createCliCommandSession, + type CliCommandSession, +} from "./cliCommandSession.js"; /** Caller-controlled values for one compiled REA CLI invocation. */ export interface CliInvocation { @@ -40,6 +44,12 @@ export interface TestCli { * drivers created with `createTestCli` alone. */ sharedMcpCall?(): Promise; + /** + * File-scoped compiled CLI process for commands with no stdin. Present on + * `cliTest` drivers. A different environment restarts the process. Stdin and + * explicit deadlines stay on `run`, which starts a fresh process. + */ + sharedCliRun?(invocation: CliInvocation): Promise; } /** One file-scoped stdio MCP session. */ @@ -202,6 +212,34 @@ const parseJson = (stdout: string): unknown | undefined => { } }; +/** Child environment for one compiled CLI process, without the parent environment. */ +export const cliProcessEnvironment = ( + environment: NodeJS.ProcessEnv | undefined, + compileCache?: string, +): NodeJS.ProcessEnv => ({ + PATH: process.env.PATH ?? "", + // Native Windows APIs need the system root and a writable temporary + // directory even when the command's application environment is empty. + ...(process.platform === "win32" + ? { + SystemRoot: process.env.SystemRoot, + TEMP: tmpdir(), + TMP: tmpdir(), + } + : {}), + ...cliCompileCacheEnvironment(environment, compileCache), +}); + +/** Keep string entries. A shared process cannot round-trip an absent variable as JSON. */ +export const definedCliEnvironment = ( + environment: NodeJS.ProcessEnv, +): Record => + Object.fromEntries( + Object.entries(environment).flatMap(([key, value]) => + typeof value === "string" ? [[key, value] as const] : [], + ), + ); + /** Reuse CLI compilation without overriding caller-selected cache or coverage. */ export const cliCompileCacheEnvironment = ( environment: NodeJS.ProcessEnv | undefined, @@ -213,6 +251,20 @@ export const cliCompileCacheEnvironment = ( ...environment, }); +/** + * Run one command on the file-scoped compiled CLI. Stdin and explicit + * deadlines stay on `TestCli.run`, which starts a fresh process. + */ +export const runFileCli = ( + cli: TestCli, + invocation: CliInvocation, +): Promise => { + const run = cli.sharedCliRun; + if (run === undefined) + throw new Error("cliTest did not provide a shared CLI session"); + return run(invocation); +}; + /** Create a compiled CLI driver backed by the supplied process registry. */ export const createTestCli = ( processes: TestProcesses, @@ -230,19 +282,7 @@ export const createTestCli = ( [resolve("scripts/rea.mjs"), ...arguments_], { cwd: cwd ?? process.cwd(), - env: { - PATH: process.env.PATH ?? "", - // Native Windows APIs need the system root and a writable temporary - // directory even when the command's application environment is empty. - ...(process.platform === "win32" - ? { - SystemRoot: process.env.SystemRoot, - TEMP: tmpdir(), - TMP: tmpdir(), - } - : {}), - ...cliCompileCacheEnvironment(environment, compileCache), - }, + env: cliProcessEnvironment(environment, compileCache), ...(input === undefined ? {} : { input }), ...(timeoutMs === undefined ? {} : { timeoutMs }), }, @@ -254,7 +294,7 @@ export const createTestCli = ( /** Vitest API extended with an owned process registry and compiled CLI driver. */ export const cliTest = processTest.extend<{ $test: { cli: TestCli }; - $file: { mcpSession: SharedMcpSession }; + $file: { mcpSession: SharedMcpSession; cliCommandSession: CliCommandSession }; $worker: { cliCompileCache: string }; }>({ cliCompileCache: [ @@ -280,11 +320,40 @@ export const cliTest = processTest.extend<{ }, { scope: "file" }, ], - cli: async ({ processes, cliCompileCache, mcpSession }, use) => { + cliCommandSession: [ + // oxlint-disable-next-line no-empty-pattern -- Vitest parses fixture dependencies from object destructuring. + async ({}, use) => { + const session = createCliCommandSession(); + try { + await use(session); + } finally { + await session.close(); + } + }, + { scope: "file" }, + ], + cli: async ( + { processes, cliCompileCache, mcpSession, cliCommandSession }, + use, + ) => { const cli = createTestCli(processes, cliCompileCache); await use({ ...cli, sharedMcpCall: () => mcpSession.call(), + sharedCliRun: (invocation) => { + if ( + invocation.input !== undefined || + invocation.timeoutMs !== undefined + ) + return cli.run(invocation); + return cliCommandSession.run({ + arguments: invocation.arguments, + cwd: invocation.cwd ?? process.cwd(), + environment: definedCliEnvironment( + cliProcessEnvironment(invocation.environment, cliCompileCache), + ), + }); + }, }); }, }); diff --git a/tests/support/cli/compiledCliSession.ts b/tests/support/cli/compiledCliSession.ts new file mode 100644 index 0000000000..086efa6806 --- /dev/null +++ b/tests/support/cli/compiledCliSession.ts @@ -0,0 +1,444 @@ +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +import { z } from "zod"; + +import { waitForExit } from "../process/processFixture.js"; + +const START_TIMEOUT_MS = 20_000; +const REQUEST_TIMEOUT_MS = 20_000; +const STOP_TIMEOUT_MS = 1_000; +const STDERR_TAIL_CHARS = 4_000; +const workerPath = fileURLToPath( + new URL("./compiledCliWorker.mjs", import.meta.url), +); +const readyMessage = z.object({ ready: z.literal(true) }).strict(); +const responseMessage = z + .object({ + id: z.number(), + exitCode: z.number(), + stdout: z.string(), + error: z.string().optional(), + }) + .strict(); + +type CliResponse = z.infer; + +interface PendingResponse { + readonly resolve: (response: CliResponse) => void; + readonly reject: (cause: unknown) => void; +} + +interface OwnedStop { + readonly child: ChildProcessWithoutNullStreams; + attempt: Promise | undefined; +} + +interface SessionState { + child: ChildProcessWithoutNullStreams | undefined; + starting: Promise | undefined; + buffer: string; + nextId: number; + readonly pending: Map; + readonly stopping: OwnedStop[]; + queue: Promise; + stderr: string; + closed: boolean; + closing: Promise | undefined; + settleStart: ((cause?: Error) => void) | undefined; + ready: boolean; + readonly requestTimeoutMs: number; + readonly stopTimeoutMs: number; + readonly spawnWorker: () => ChildProcessWithoutNullStreams; +} + +/** One warm compiled CLI process serving production JSON commands. */ +export interface CompiledCliSession { + run(args: readonly string[]): Promise; + close(): Promise; +} + +/** Limits and process factory for session tests. Production callers omit them. */ +export interface CompiledCliSessionOptions { + readonly spawnWorker?: () => ChildProcessWithoutNullStreams; + readonly requestTimeoutMs?: number; + readonly stopTimeoutMs?: number; +} + +/** Start a session that loads the compiled CLI once and reuses that process. */ +export const startCompiledCliSession = ( + options: CompiledCliSessionOptions = {}, +): CompiledCliSession => { + const state = createSessionState(options); + return { + run: (args) => enqueue(state, () => dispatch(state, args)), + close: () => closeSession(state), + }; +}; + +const positiveMilliseconds = ( + value: number | undefined, + fallback: number, +): number => + value !== undefined && Number.isSafeInteger(value) && value > 0 + ? value + : fallback; + +const defaultSpawnWorker = (): ChildProcessWithoutNullStreams => + spawn(process.execPath, [workerPath], { + cwd: process.cwd(), + env: { ...process.env, REA_LOG_LEVEL: "silent" }, + stdio: ["pipe", "pipe", "pipe"], + }); + +const createSessionState = ( + options: CompiledCliSessionOptions, +): SessionState => ({ + child: undefined, + starting: undefined, + buffer: "", + nextId: 1, + pending: new Map(), + stopping: [], + queue: Promise.resolve(), + stderr: "", + closed: false, + closing: undefined, + settleStart: undefined, + ready: false, + requestTimeoutMs: positiveMilliseconds( + options.requestTimeoutMs, + REQUEST_TIMEOUT_MS, + ), + stopTimeoutMs: positiveMilliseconds(options.stopTimeoutMs, STOP_TIMEOUT_MS), + spawnWorker: options.spawnWorker ?? defaultSpawnWorker, +}); + +const enqueue = ( + state: SessionState, + task: () => Promise, +): Promise => { + const run = state.queue.then(task); + state.queue = run.then( + () => undefined, + () => undefined, + ); + return run; +}; + +const dispatch = async ( + state: SessionState, + args: readonly string[], +): Promise => { + await ensureStarted(state); + const child = state.child; + if (child === undefined) + throw new Error("Compiled CLI session is not running"); + const id = state.nextId; + state.nextId += 1; + const response = await request(state, child, id, args); + if (response.exitCode !== 0) throw new Error(failureText(state, response)); + return JSON.parse(response.stdout) as unknown; +}; + +const request = ( + state: SessionState, + child: ChildProcessWithoutNullStreams, + id: number, + args: readonly string[], +): Promise => + new Promise((resolve, reject) => { + let settled = false; + let timer: ReturnType | undefined; + const settle = (finish: () => void): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + state.pending.delete(id); + finish(); + }; + timer = setTimeout(() => { + settle(() => { + reject( + new Error( + `Compiled CLI command did not respond within ${String(state.requestTimeoutMs)} ms`, + ), + ); + retireChild(state, child); + }); + }, state.requestTimeoutMs); + timer.unref(); + state.pending.set(id, { + resolve: (response) => { + settle(() => { + resolve(response); + }); + }, + reject: (cause) => { + settle(() => { + reject(cause); + }); + }, + }); + child.stdin.write(`${JSON.stringify({ id, args })}\n`, (cause) => { + if (cause === undefined || cause === null) return; + settle(() => { + reject(cause); + }); + }); + }); + +const failureText = (state: SessionState, response: CliResponse): string => { + const detail = response.error ?? response.stdout.slice(0, 500); + const stderr = state.stderr.trim(); + return stderr.length === 0 + ? `Compiled CLI command failed with status ${String(response.exitCode)}: ${detail}` + : `Compiled CLI command failed with status ${String(response.exitCode)}: ${detail}\n${stderr}`; +}; + +const ensureStarted = async (state: SessionState): Promise => { + if (state.closed) throw sessionClosed(); + await releaseRetired(state); + if (state.closed) throw sessionClosed(); + if (state.ready && state.child !== undefined) return; + state.starting ??= launch(state); + await state.starting; + if (state.closed) throw sessionClosed(); +}; + +const launch = (state: SessionState): Promise => + new Promise((resolve, reject) => { + const child = state.spawnWorker(); + state.child = child; + let settled = false; + let timer: ReturnType | undefined; + const finish = (cause?: Error): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (state.settleStart === finish) state.settleStart = undefined; + if (cause === undefined) resolve(); + else reject(cause); + }; + timer = setTimeout(() => { + signalChild(child, "SIGKILL"); + finish( + new Error( + `Compiled CLI session did not start within ${String(START_TIMEOUT_MS)} ms\n${state.stderr}`, + ), + ); + }, START_TIMEOUT_MS); + timer.unref(); + state.settleStart = finish; + const fail = (cause: unknown): void => { + finish(cause instanceof Error ? cause : new Error(String(cause))); + }; + const currentChild = (): boolean => state.child === child; + child.once("error", fail); + child.once("exit", (code, signal) => { + if (!currentChild()) return; + state.child = undefined; + state.ready = false; + state.starting = undefined; + state.buffer = ""; + const cause = new Error( + `Compiled CLI session exited (${String(code)}, ${String(signal)})\n${state.stderr}`, + ); + fail(cause); + rejectPending(state, cause); + }); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + if (!currentChild()) return; + state.stderr = `${state.stderr}${chunk}`.slice(-STDERR_TAIL_CHARS); + }); + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + if (!currentChild()) return; + state.buffer += chunk; + drainLines(state, () => { + state.ready = true; + finish(); + }); + }); + }); + +const hasExited = (child: ChildProcessWithoutNullStreams): boolean => + child.exitCode !== null || child.signalCode !== null; + +const signalChild = ( + child: ChildProcessWithoutNullStreams, + signal: NodeJS.Signals, +): void => { + try { + child.kill(signal); + } catch (error) { + const code = + error instanceof Error && "code" in error ? error.code : undefined; + if (code !== "ESRCH") throw error; + } +}; + +const detachChild = ( + state: SessionState, + child: ChildProcessWithoutNullStreams, +): void => { + if (state.child !== child) return; + state.child = undefined; + state.ready = false; + state.starting = undefined; + state.buffer = ""; +}; + +const stopOwned = (state: SessionState, entry: OwnedStop): Promise => { + entry.attempt ??= (async () => { + if (hasExited(entry.child)) return true; + signalChild(entry.child, "SIGKILL"); + return waitForExit(entry.child, state.stopTimeoutMs); + })().finally(() => { + entry.attempt = undefined; + }); + return entry.attempt; +}; + +const trackRetired = ( + state: SessionState, + child: ChildProcessWithoutNullStreams, +): OwnedStop => { + const existing = state.stopping.find((entry) => entry.child === child); + if (existing !== undefined) return existing; + const entry: OwnedStop = { child, attempt: undefined }; + state.stopping.push(entry); + if (!hasExited(child)) void stopOwned(state, entry).catch(() => undefined); + return entry; +}; + +const releaseRetired = async (state: SessionState): Promise => { + let failed = false; + for (const entry of [...state.stopping]) { + if (hasExited(entry.child)) continue; + const stopped = await stopOwned(state, entry).catch(() => false); + if (!stopped || !hasExited(entry.child)) failed = true; + } + const retained = state.stopping.filter((entry) => !hasExited(entry.child)); + state.stopping.length = 0; + state.stopping.push(...retained); + if (failed) + throw new Error("Compiled CLI session stayed running after SIGKILL"); +}; + +const retireChild = ( + state: SessionState, + child: ChildProcessWithoutNullStreams, +): void => { + detachChild(state, child); + if (hasExited(child)) return; + trackRetired(state, child); +}; + +const drainLines = (state: SessionState, onReady: () => void): void => { + let newline = state.buffer.indexOf("\n"); + while (newline !== -1) { + const line = state.buffer.slice(0, newline); + state.buffer = state.buffer.slice(newline + 1); + if (line.length > 0) acceptLine(state, line, onReady); + newline = state.buffer.indexOf("\n"); + } +}; + +const acceptLine = ( + state: SessionState, + line: string, + onReady: () => void, +): void => { + let message: unknown; + try { + message = JSON.parse(line) as unknown; + } catch (cause) { + rejectPending( + state, + cause instanceof Error ? cause : new Error(String(cause)), + ); + return; + } + if (readyMessage.safeParse(message).success) { + onReady(); + return; + } + const parsed = responseMessage.safeParse(message); + if (!parsed.success) { + rejectPending( + state, + new Error("Compiled CLI session returned an unexpected message"), + ); + return; + } + const pending = state.pending.get(parsed.data.id); + state.pending.delete(parsed.data.id); + if (pending === undefined) { + rejectPending( + state, + new Error( + `Compiled CLI session returned unexpected id ${String(parsed.data.id)}`, + ), + ); + return; + } + pending.resolve(parsed.data); +}; + +const rejectPending = (state: SessionState, cause: Error): void => { + for (const pending of state.pending.values()) pending.reject(cause); + state.pending.clear(); +}; + +const closeSession = (state: SessionState): Promise => { + state.closed = true; + if (state.closing !== undefined) return state.closing; + let attempt: Promise; + attempt = shutdown(state).catch((error: unknown) => { + if (state.closing === attempt) state.closing = undefined; + throw error; + }); + state.closing = attempt; + return attempt; +}; + +const shutdown = async (state: SessionState): Promise => { + const child = state.child; + let forced = false; + const stillRunning: OwnedStop[] = []; + if (child !== undefined) { + rejectPending(state, sessionClosed()); + state.settleStart?.(sessionClosed()); + detachChild(state, child); + const stopped = await stopGracefully(state, child); + if (stopped === "forced") forced = true; + if (stopped === "stuck" && !hasExited(child)) + stillRunning.push({ child, attempt: undefined }); + } + try { + await releaseRetired(state); + } finally { + state.stopping.push(...stillRunning); + } + if (state.stopping.length > 0) + throw new Error("Compiled CLI session stayed running after SIGKILL"); + if (forced) + throw new Error("Compiled CLI session did not exit after stdin closed"); +}; + +const stopGracefully = async ( + state: SessionState, + child: ChildProcessWithoutNullStreams, +): Promise<"clean" | "forced" | "stuck"> => { + child.stdin.end(); + if (await waitForExit(child, state.stopTimeoutMs)) return "clean"; + signalChild(child, "SIGTERM"); + if (await waitForExit(child, state.stopTimeoutMs)) return "forced"; + signalChild(child, "SIGKILL"); + if (await waitForExit(child, state.stopTimeoutMs)) return "forced"; + return "stuck"; +}; + +const sessionClosed = (): Error => new Error("Compiled CLI session is closed"); diff --git a/tests/support/cli/compiledCliWorker.mjs b/tests/support/cli/compiledCliWorker.mjs new file mode 100644 index 0000000000..e8012c7510 --- /dev/null +++ b/tests/support/cli/compiledCliWorker.mjs @@ -0,0 +1,139 @@ +import { Writable } from "node:stream"; +import { createInterface } from "node:readline"; + +const { cliConfigurationError, createCli } = + await import("../../../dist/cli-runtime/cli.js"); +const { createCliResultOutput } = + await import("../../../dist/cli-runtime/cliResultOutput.js"); +const { + normalizeCliArguments, + renderCliOutputArgumentError, + renderEmptyFilteredCliOutput, + validateCliOutputArguments, +} = await import("../../../dist/cli-runtime/cliOutput.js"); + +const MAX_STDOUT_CHARS = 16 * 1024 * 1024; + +const writeMessage = (message) => { + process.stdout.write(`${JSON.stringify(message)}\n`); +}; + +// One CLI instance serves every command. A fresh instance per command retains +// its command registrations after the call returns; production exits instead. +const outputParts = []; +let outputCharacters = 0; +const destination = new Writable({ + write(chunk, _encoding, callback) { + const text = Buffer.isBuffer(chunk) + ? chunk.toString("utf8") + : String(chunk); + outputCharacters += text.length; + if (outputCharacters > MAX_STDOUT_CHARS) { + callback(new Error("Compiled CLI stdout exceeded 16 MiB")); + return; + } + outputParts.push(text); + callback(); + }, +}); +Object.defineProperty(destination, "isTTY", { value: false }); +const resultOutput = createCliResultOutput(["--json"], destination); +const cli = createCli(process.env, resultOutput); + +const resetOutput = () => { + outputParts.length = 0; + outputCharacters = 0; +}; + +const runCommand = async (args) => { + const previousExitCode = process.exitCode; + process.exitCode = undefined; + resetOutput(); + try { + const cliArgs = normalizeCliArguments(args); + const outputArguments = validateCliOutputArguments(cliArgs); + if (!outputArguments.ok) + return { + exitCode: 1, + stdout: renderCliOutputArgumentError(outputArguments), + }; + const configurationError = cliConfigurationError(process.env); + if (configurationError !== undefined) + return { exitCode: 1, stdout: "", error: configurationError }; + + let wroteOutput = false; + let explicitExit; + try { + await cli.serve(cliArgs, { + stdout: (output) => { + if (resultOutput?.handled) return; + if (output.length === 0) return; + wroteOutput = true; + destination.write(output); + }, + exit: (code) => { + explicitExit = code; + }, + }); + } catch (cause) { + return { + exitCode: 1, + stdout: outputParts.join(""), + error: + cause instanceof Error + ? (cause.stack ?? cause.message) + : String(cause), + }; + } + if (!wroteOutput && !resultOutput?.handled) { + const empty = renderEmptyFilteredCliOutput(cliArgs); + if (empty !== undefined) destination.write(empty); + } + // logCliCommand reports some failures only through process.exitCode. + return { + exitCode: + explicitExit ?? + (typeof process.exitCode === "number" ? process.exitCode : 0), + stdout: outputParts.join(""), + }; + } finally { + process.exitCode = previousExitCode; + } +}; + +const acceptRequest = async (line) => { + let request; + try { + request = JSON.parse(line); + } catch (cause) { + writeMessage({ + id: -1, + exitCode: 1, + stdout: "", + error: cause instanceof Error ? cause.message : String(cause), + }); + return; + } + const args = request?.args; + if ( + typeof request?.id !== "number" || + !Array.isArray(args) || + args.some((argument) => typeof argument !== "string") + ) { + writeMessage({ + id: typeof request?.id === "number" ? request.id : -1, + exitCode: 1, + stdout: "", + error: "Compiled CLI session expected an id and string arguments", + }); + return; + } + writeMessage({ id: request.id, ...(await runCommand(args)) }); +}; + +const lines = createInterface({ input: process.stdin, crlfDelay: Infinity }); +writeMessage({ ready: true }); +for await (const line of lines) { + if (line.trim().length === 0) continue; + await acceptRequest(line); +} diff --git a/tests/support/cli/javascriptApplication.ts b/tests/support/cli/javascriptApplication.ts index 6a04b9592c..186953cb91 100644 --- a/tests/support/cli/javascriptApplication.ts +++ b/tests/support/cli/javascriptApplication.ts @@ -35,7 +35,11 @@ export const resolveJavaScriptApplicationMcpCall = async ( return (await connectLocalToolsMcp()).call; }; -/** Analyze a directory; a supplied MCP caller reuses its test-owned connection. */ +/** + * Analyze a directory. A supplied MCP caller reuses its test-owned connection. + * CLI uses the driver's file-scoped process when present, and a fresh process + * otherwise. + */ export const analyzeJavaScriptDirectory = async ( surface: "CLI" | "stdio MCP", cli: TestCli, @@ -50,7 +54,8 @@ export const analyzeJavaScriptDirectory = async ( ) => { let document: unknown; if (surface === "CLI") { - const response = await cli.run({ + const run = cli.sharedCliRun ?? ((invocation) => cli.run(invocation)); + const response = await run({ arguments: [ "analyze-javascript-application", root,