OpenSource/reverse-engineering-anything
1
0 Fork 0
mirror of https://github.com/morluto/rea.git synced 2026-10-29 19:05:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: merge main into environment state writes

Browse Source
This commit is contained in:
morluto committed 2026-10-21 18:30:47 +08:00
2 parents 73c0e9b50a c324d31b45
commit b2244622e3
325 files changed
+27593 -2752
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.agents/skills/reverse-engineer-anything/SKILL.md
+15 -3
View File
@@ -91,9 +91,15 @@ explicit path or endpoint and do not need it.
- ASAR or extracted JavaScript/Electron tree:
`analyze_javascript_application`.
- Archive/package member inventory (ZIP/APK/IPA/MSIX/AppX or DMG):
`open_binary` with the supplied local path; use `inspect_artifact` when its
graph and findings help answer the question.
- Archive/package member inventory (ZIP/APK/IPA/MSIX/AppX, DMG, or a PyInstaller
executable): `open_binary` with the supplied local path; use `inspect_artifact`
when its graph and findings help answer the question. A PyInstaller CArchive
is the application layer; the bootloader is only the native prefix. Start with
`detail: "summary"` for large inventories, then use retained inventory views
for selected member fields/pages. A complete small inventory can cost less
than extra calls and schemas. Runtime options are metadata, not files.
Compact delivery still inventories and hashes members. See the
[Python packaged applications guide](https://github.com/morluto/rea/blob/main/docs/python-packaged-applications.md).
- Apple app plist, Interface Builder, asset catalog, or keyed archive resources:
`open_bundle_resources`, then the selected resource tool. This does not
require `CFBundleExecutable` or a deep provider. Resource sessions have no
@@ -201,6 +207,12 @@ explicit path or endpoint and do not need it.
package order, dependency indices, and the section file range. A missing
`.dep-v0` section is null metadata, not proof the binary has no dependencies.
A rustc commit hash is an inference from a read-only `/rustc/<40-hex>/` path.
For selected crates, use `view: {collection: "packages", names: [...],
fields: ["name", "version", "source"]}` on the first call; explicitly include
`dependencies` for direct index references. Use `detail: "summary"` for counts
and ranges, or retained `inspect_analysis_view` Rust package/edge pages for
subsequent filters. Keep the complete parent Evidence accessible; CLI compact
delivery requires `--evidence_output`.
See the
[Rust binary guide](https://github.com/morluto/rea/blob/main/docs/rust-binaries.md).
.github/workflows/ci.yml
+18 -4
View File
@@ -409,7 +409,9 @@ jobs:
with:
save: ${{ github.event_name != 'pull_request' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }}
- run: npm run test:prepare
- name: Find available trusted full-suite timing history
# Fast and full shards measure different file costs. Download only the
# matching history so one tier cannot pack the other with its weights.
- name: Find available trusted timing history
id: history
continue-on-error: true
env:
@@ -417,13 +419,14 @@ jobs:
CI_REPOSITORY: ${{ github.repository }}
CI_REPOSITORY_ID: ${{ github.repository_id }}
CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
CI_TIMING_TIER: ${{ needs.changes.outputs.tier }}
run: node scripts/ci/test-history.mjs
- name: Download previous timing report
if: steps.history.outputs.run-id != ''
continue-on-error: true
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: test-timings
name: ${{ steps.history.outputs.artifact-name }}
run-id: ${{ steps.history.outputs.run-id }}
github-token: ${{ github.token }}
path: .cache/ci-test-history
@@ -555,11 +558,22 @@ jobs:
include-hidden-files: true
if-no-files-found: error
retention-days: 30
# Default-branch runs only. Pull-request artifacts share this name and
# the lookup reads one page, so publishing them would hide main.
- name: Publish successful timing history
if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'full' && success() && needs.test-shard.result == 'success'
if: >-
success() &&
needs.changes.outputs.code == 'true' &&
needs.test-shard.result == 'success' &&
(
needs.changes.outputs.tier == 'full' ||
(needs.changes.outputs.tier == 'fast' && needs.test-source.result == 'success')
) &&
github.ref == format('refs/heads/{0}', github.event.repository.default_branch) &&
(github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: test-timings
name: test-timings-${{ needs.changes.outputs.tier }}
path: .cache/ci-test-history/test-results.json
include-hidden-files: true
if-no-files-found: error
README.md
+30 -29
View File
@@ -174,35 +174,36 @@ CLI commands use the same workflows.
REA requires Node.js 22.x (>=22.19), 24.x (>=24.11), or 26+, plus npm.
Additional tools and host support depend on the target:
| Target | What REA returns | Requirements and guide |
| ---------------------- | ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- |
| Desktop installations | Runtime candidates, supporting files, component markers and next probes | Built-in bounded directory/header inspection; [desktop applications](docs/desktop-applications.md) |
| Native binaries | Pseudocode, assembly, strings, symbols, calls and references | Hopper, Ghidra or IDA; [native analysis](docs/native-investigation.md) |
| Offline ELF layout | Sections, segments, original symbols/relocations and static mitigation candidates | Caller-supplied pwntools on Linux x64; [binary diagnostics](docs/binary-diagnostics.md) |
| Go binaries | Embedded compiler version, modules, replacements and build settings | Built-in static ELF/PE/thin Mach-O inspection; no Go installation required |
| Model artifacts | GGUF / safetensors metadata, tensor declarations and derived totals | Built-in header inspection; optional full-file hash; [model files](docs/model-artifacts.md) |
| IL2CPP metadata | Unity global-metadata.dat assembly images, type and method names and index ranges | Built-in static decoding of versions 24, 27, 29, 31 and 39; no Unity required |
| Godot PCK | Archive inventory and extraction, original header/directory facts and selected content identities | Built-in static formats 2, 3 and 4; no Godot required |
| Rust binaries | Embedded cargo-auditable crate names, versions, sources and dependency edges | Built-in static ELF/PE/thin Mach-O inspection; [Rust binaries](docs/rust-binaries.md) |
| WASM artifact | Validation, sections, exact import/export WAT forms and decoded WAT | Caller-supplied WABT 1.0.42; `inspect_wasm_artifact` |
| EVM bytecode | Dispatch selectors, byte offsets, inferred arguments and mutability | Local raw/hex carrier on Linux x64; [offline EVM guide](docs/evm-bytecode.md) |
| SQLite databases | Tables, columns, indexes, views, triggers and selected ordinary-table rows | Local database/WAL snapshot; Node.js 24.x >=24.15 or 26+ with native SQLite limits |
| Recorded Linux crashes | Raw notes, every recorded thread's registers/signals and optional mapping candidates | Caller-supplied pwntools on Linux x64; optional GDB/pwndbg; [recorded crashes](docs/recorded-crashes.md) |
| JavaScript / Electron | Modules, imports, source maps, routes, IPC and native add-on relationships | Node.js and npm; Linux: procps-compatible `ps`; [application analysis](docs/javascript-application-workflows.md) |
| Websites | Page structure, scripts, network observations and requested screenshots | A Chrome-family browser; [browser analysis](docs/browser-observation.md) |
| Saved network captures | Requests, responses, exposed payloads and source locations | HAR or Fiddler Everywhere SAZ; mitmdump on Linux for native mitmproxy; [capture guide](docs/web-network-captures.md) |
| .NET assemblies | Metadata, CIL instructions, declared native dependencies and build comparisons | None (static only); [managed-code guide](docs/managed-code-analysis.md) |
| Android APKs | Manifest declarations, classes, decompiled methods and references | Headless JADX and a full JDK on Linux/macOS/Windows x64; [Android guide](docs/android-analysis.md) |
| Android devices | Devices, packages, processes, logs, dumps, screen captures, and APK/file transfers | Caller-supplied adb (emulator or hardware); [ADB guide](docs/adb-device-analysis.md) |
| JEB projects | Engine-backed project units, artifact digests and type/method pseudo-code | A caller-started JEB client serving MCP, selected with REA_JEB_MCP_URL; [JEB guide](docs/jeb-analysis.md) |
| Android resources | Decoded manifest, string tables, locales and version facts | Caller-supplied Apktool launcher; [resource guide](docs/apktool-resource-analysis.md) |
| Flutter apps | Dart snapshot hash, per-ABI library digests and engine build provenance | None (static APK parsing); [Flutter guide](docs/flutter-analysis.md) |
| Firmware | Regions, extraction results and native-analysis handoffs | Binwalk / Unblob on Linux; [firmware guide](docs/firmware-analysis.md) |
| Recorded packets | Selected fields, repeated values, filters and decode-as rules | Linux: TShark 4.x + prlimit; [capture guide](docs/packet-captures.md) |
| Packet summaries | Endpoints, DNS, TLS SNI, protocol distribution and HTTP requests | Caller-supplied tshark and capinfos (Linux-verified); [pcap guide](docs/pcap-analysis.md) |
| Recorded logic signals | Decoder annotations, sample ranges and diagnostics | Linux: sigrok-cli 0.7.2 + prlimit; [signal recordings](docs/signal-recordings.md) |
| Packages and resources | File inventories, digests, plists, Apple bundle anatomy and extracted resources | [Artifact and JavaScript guide](docs/javascript-artifact-reconstruction.md), [Apple applications](docs/apple-application-analysis.md) |
| Process behavior | Terminal output, interactions, exit and filesystem observations, and run comparisons | Linux/macOS with a native PTY; [process capture](docs/process-capture.md) |
| Target | What REA returns | Requirements and guide |
| ----------------------- | ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- |
| Desktop installations | Runtime candidates, supporting files, component markers and next probes | Built-in bounded directory/header inspection; [desktop applications](docs/desktop-applications.md) |
| Native binaries | Pseudocode, assembly, strings, symbols, calls and references | Hopper, Ghidra or IDA; [native analysis](docs/native-investigation.md) |
| Offline ELF layout | Sections, segments, original symbols/relocations and static mitigation candidates | Caller-supplied pwntools on Linux x64; [binary diagnostics](docs/binary-diagnostics.md) |
| Go binaries | Embedded compiler version, modules, replacements and build settings | Built-in static ELF/PE/thin Mach-O inspection; no Go installation required |
| Model artifacts | GGUF / safetensors metadata, tensor declarations and derived totals | Built-in header inspection; optional full-file hash; [model files](docs/model-artifacts.md) |
| IL2CPP metadata | Unity global-metadata.dat assembly images, type and method names and index ranges | Built-in static decoding of versions 24, 27, 29, 31 and 39; no Unity required |
| Godot PCK | Archive inventory and extraction, original header/directory facts and selected content identities | Built-in static formats 2, 3 and 4; no Godot required |
| Rust binaries | Embedded cargo-auditable crate names, versions, sources and dependency edges | Built-in static ELF/PE/thin Mach-O inspection; [Rust binaries](docs/rust-binaries.md) |
| WASM artifact | Validation, sections, exact import/export WAT forms and decoded WAT | Caller-supplied WABT 1.0.42; `inspect_wasm_artifact` |
| EVM bytecode | Dispatch selectors, byte offsets, inferred arguments and mutability | Local raw/hex carrier on Linux x64; [offline EVM guide](docs/evm-bytecode.md) |
| SQLite databases | Tables, columns, indexes, views, triggers and selected ordinary-table rows | Local database/WAL snapshot; Node.js 24.x >=24.15 or 26+ with native SQLite limits |
| Recorded Linux crashes | Raw notes, every recorded thread's registers/signals and optional mapping candidates | Caller-supplied pwntools on Linux x64; optional GDB/pwndbg; [recorded crashes](docs/recorded-crashes.md) |
| JavaScript / Electron | Modules, imports, source maps, routes, IPC and native add-on relationships | Node.js and npm; Linux: procps-compatible `ps`; [application analysis](docs/javascript-application-workflows.md) |
| Websites | Page structure, scripts, network observations and requested screenshots | A Chrome-family browser; [browser analysis](docs/browser-observation.md) |
| Saved network captures | Requests, responses, exposed payloads and source locations | HAR or Fiddler Everywhere SAZ; mitmdump on Linux for native mitmproxy; [capture guide](docs/web-network-captures.md) |
| .NET assemblies | Metadata, CIL instructions, declared native dependencies and build comparisons | None (static only); [managed-code guide](docs/managed-code-analysis.md) |
| Android APKs | Manifest declarations, classes, decompiled methods and references | Headless JADX and a full JDK on Linux/macOS/Windows x64; [Android guide](docs/android-analysis.md) |
| Android devices | Devices, packages, processes, logs, dumps, screen captures, and APK/file transfers | Caller-supplied adb (emulator or hardware); [ADB guide](docs/adb-device-analysis.md) |
| JEB projects | Engine-backed project units, artifact digests and type/method pseudo-code | A caller-started JEB client serving MCP, selected with REA_JEB_MCP_URL; [JEB guide](docs/jeb-analysis.md) |
| Android resources | Decoded manifest, string tables, locales and version facts | Caller-supplied Apktool launcher; [resource guide](docs/apktool-resource-analysis.md) |
| Flutter apps | Dart snapshot hash, per-ABI library digests and engine build provenance | None (static APK parsing); [Flutter guide](docs/flutter-analysis.md) |
| Firmware | Regions, extraction results and native-analysis handoffs | Binwalk / Unblob on Linux; [firmware guide](docs/firmware-analysis.md) |
| Recorded packets | Selected fields, repeated values, filters and decode-as rules | Linux: TShark 4.x + prlimit; [capture guide](docs/packet-captures.md) |
| Packet summaries | Endpoints, DNS, TLS SNI, protocol distribution and HTTP requests | Caller-supplied tshark and capinfos (Linux-verified); [pcap guide](docs/pcap-analysis.md) |
| Recorded logic signals | Decoder annotations, sample ranges and diagnostics | Linux: sigrok-cli 0.7.2 + prlimit; [signal recordings](docs/signal-recordings.md) |
| Packages and resources | File inventories, digests, plists, Apple bundle anatomy and extracted resources | [Artifact and JavaScript guide](docs/javascript-artifact-reconstruction.md), [Apple applications](docs/apple-application-analysis.md) |
| Process behavior | Terminal output, interactions, exit and filesystem observations, and run comparisons | Linux/macOS with a native PTY; [process capture](docs/process-capture.md) |
| PyInstaller executables | CArchive members, Python version, entry scripts and extracted bytecode | None (static only); [Python packaged applications](docs/python-packaged-applications.md) |
Static JavaScript and .NET inspection read the supplied files without running
the application. Runtime capture runs or interacts with the selected target
docs/.vitepress/config.ts
+4
View File
@@ -50,6 +50,10 @@ export default defineConfig({
{ text: "Electron fuses", link: "/electron-fuses" },
{ text: "Process capture", link: "/process-capture" },
{ text: ".NET assemblies", link: "/managed-code-analysis" },
{
text: "Python packaged applications",
link: "/python-packaged-applications",
},
{ text: "Android applications", link: "/android-analysis" },
{ text: "Apple applications", link: "/apple-application-analysis" },
{ text: "Desktop installations", link: "/desktop-applications" },
docs/apple-application-analysis.md
+29 -12
View File
@@ -213,20 +213,37 @@ Resolution follows dyld:
Each edge lists every candidate path tried, with its outcome: `resolved`,
`absent`, `not-mach-o`, `malformed`, `unsupported` (recognized Mach-O outside
parser coverage, such as big-endian images), `architecture-missing`,
`outside-target`, `escapes-target`, or `undetermined`. Slices follow dyld's
graded architectures, so an `x86_64h` process also loads `x86_64`. The edge's `resolution` is one of:
`not-loadable`, `platform-mismatch`, `outside-target`, `escapes-target`, or
`undetermined`. `not-loadable` means a
matching-architecture Mach-O image was found but it is not a dylib and cannot
satisfy a dylib dependency. `platform-mismatch` means a matching-architecture
dylib was found but its observed platform is incompatible with the process.
These outcomes reject that candidate; dyld search may continue and resolve a
later candidate. Candidate outcomes describe individual search paths, while
`resolution.status` describes the final edge. Slices follow dyld's graded
architectures, so an `x86_64h` process also loads `x86_64`. The edge's
`resolution` is one of:
| Status | Meaning |
| -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `resolved` | An image inside the analyzed root loads, and no earlier candidate was undeterminable. |
| `conditional` | An image inside the root resolves, but an earlier candidate outside the root (for example `/usr/lib/swift`) would win if it exists on the running system. |
| `unresolved` | Every candidate is definitively absent or unusable inside the root. |
| `undetermined` | No candidate resolves inside the root, and some candidate lies outside it or depends on the environment. |
| Status | Meaning |
| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `resolved` | An image inside the analyzed root loads, and no earlier candidate was undeterminable. |
| `conditional` | An in-root image may load, but an earlier outside candidate, unmodeled environment input, or unknown platform compatibility can change which image dyld loads. |
| `unresolved` | Every candidate is definitively absent or unusable inside the root. |
| `undetermined` | No candidate resolves inside the root, and some candidate lies outside it or depends on the environment. |
A `conditional` image is still traced, because it is the image the analyzed
root supplies. Every edge below it has `loader_conditional: true`, and a
request that reuses it by install name is `conditional` too. Findings on those
edges say that they apply only if the conditional image loads.
A `conditional` image is still traced, because it is an in-root candidate
that may load. Every edge below it has `loader_conditional: true`, and a
request that reuses it by install name is `conditional` too. An in-root dylib
with no platform load command remains unknown unless the root is an executable
with observed macOS platform and the candidate’s CPU type is x86_64, i386, or arm64. In that macOS context,
[Apple's dyld](https://github.com/apple-oss-distributions/dyld/blob/fd8d0c4d52320ebf64db34f3cb280310d905c5ae/mach_o/UnsafeHeader.cpp#L1137-L1145) supplies an implicit macOS platform to legacy dylibs on those CPU
types. The candidate records `platform_basis: "macos-dyld-implicit"`; the
reported slice fields `platform: null` and `platforms: []` remain the parsed
facts. For other process contexts the platform remains unknown, coverage is
partial, and the candidate is not treated as incompatible. If a later known
candidate resolves, it remains the traced image with conditional resolution;
the trace does not claim the unknown candidate's dependency graph. Findings on
conditional edges say that they apply only if the conditional image loads.
Absolute install names that name a path beneath the analyzed root are checked
within that root, with symlinks and `..` resolved by the same bounded tree
docs/cli.md
+6 -4
View File
@@ -512,10 +512,12 @@ error and exit `1`, as does a listener that fails after binding.
pair refusals, a non-empty or file-occupied directory, a missing package
tarball or a name that is not a kebab-case label or is a Windows device name,
writing nothing in each case.
A write that fails after the first file (a permission or disk-space failure
on a later path) propagates as the host's failure and leaves the partial
folder in place for inspection; remove it before retrying, since the next
attempt refuses the non-empty directory.
A write that fails after the first file (a permission, disk-space or
pathname-length failure on a later path) propagates as the host's failure,
naming its cause and, when it failed on one bundle file or the package
tarball, that file. A failure while syncing the finished folder names only its
cause. Either way the partial folder stays in place for inspection; remove it
before retrying, since the next attempt refuses the non-empty directory.
A `cancelled` result does not by itself determine the exit status; it depends on
the command's signal handling. Commands routed through the shared
docs/ida-provider.md
+9
View File
@@ -41,6 +41,15 @@ rea function /absolute/path/to/program main --json
rea decompile /absolute/path/to/program main --json
```
If the registration cannot be read or validated, provider diagnostics retain its
selected `registration_path` and the configuration owner's setting/constraint
facts in `configuration_details.settings`, plus `configuration_remediation`.
`configured` means a path was selected, not that it is valid or an IDA connection
was established; failed registrations remain unavailable. Correct
`REA_IDA_MCP_CONFIG` in the next CLI invocation's environment or the MCP server's
launch environment. Restart/reconnect that server after changing its environment;
an export in another shell does not update a running server.
Open the **original input binary** in REA, not an `.idb` or `.i64` database. Its SHA-256 must match IDA's recorded input hash. REA checks the attached identity and image base around each analysis request. A mismatch or target switch fails with recovery guidance. The original input must remain readable by REA; WSL can use its mounted alias while the GUI runs on Windows. Attached profiles, including native mode, refuse universal Mach-O inputs at admission: every slice shares the recorded whole-file hash, so REA cannot verify which slice the GUI loaded. Open a thin file extracted with `lipo -thin` in both IDA and REA, or use headless mode, which analyzes the selected slice.
REA's close operation releases its MCP connection or proxy. It never saves or closes the existing GUI database. External edits remain visible because results are live and are not replayed from snapshots. These checks do not lock the GUI or prove that the database is unpatched.
docs/index.md
+13 -13
View File
@@ -47,19 +47,19 @@ For runtime instrumentation, see the [Frida guide](./frida-instrumentation.md).
## Choose a guide
| Task | Guides |
| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| Identify a desktop installation | [Runtime candidates, evidence and follow-up files](desktop-applications.md) |
| Inspect native functions, references, or types | [Native binaries](native-investigation.md), [Go metadata](go-binaries.md), [Rust dependencies](rust-binaries.md), [offline ELF layout](binary-diagnostics.md), [recorded crashes](recorded-crashes.md) |
| Select a native provider or import format | [Ghidra semantics](ghidra-provider.md), [Windows](windows-ghidra-p0.md), [DOS](ghidra-dos.md), [raw bytes](raw-binary.md), [MIPS](ghidra-mips.md), [PSP](ghidra-psp.md), [NativeAOT](ghidra-nativeaot.md), [IDA](ida-provider.md), [Kuna](kuna-provider.md), [JEB](jeb-analysis.md) |
| Analyze JavaScript or Electron artifacts | [Static analysis](javascript-artifact-reconstruction.md), [tracing and comparisons](javascript-application-workflows.md), [application graph](javascript-application-graph.md), [source recovery](javascript-recovery.md) |
| Compare static JavaScript with runtime observations | [Runtime attribution](javascript-runtime-observation.md), [reconciliation](javascript-runtime-reconciliation.md), [Electron observation](electron-observation.md), [Electron fuses](electron-fuses.md) |
| Inspect assemblies, packages, or firmware | [Godot PCK](godot-pck.md), [.NET](managed-code-analysis.md), [Android](android-analysis.md), [Apktool resources](apktool-resource-analysis.md), [ADB acquisition](adb-device-analysis.md), [Apple](apple-application-analysis.md), [Harmony](harmony-analysis.md), [firmware](firmware-analysis.md), [WASM](wasm-artifacts.md), [EVM](evm-bytecode.md) |
| Observe or compare a browser scenario | [CDP observation](browser-observation.md), [scenarios](browser-scenario-contract.md), [runtime evidence](web-runtime.md), [historical captures](web-network-captures.md) |
| Serve recorded behavior as an agent environment | [Agent environments](agent-environments.md) |
| Trace captured website source | [Source locations](web-source-location.md), [module traces](website-module-trace.md), [script export](website-script-export.md) |
| Capture and compare a command | [Process capture](process-capture.md) |
| Use CLI/MCP and retain evidence | [CLI](cli.md), [MCP contracts](mcp-contracts.md), [agent prompts](mcp-prompts.md), [obligation ledgers](reconstruction-obligation-ledgers.md) |
| Task | Guides |
| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Identify a desktop installation | [Runtime candidates, evidence and follow-up files](desktop-applications.md) |
| Inspect native functions, references, or types | [Native binaries](native-investigation.md), [Go metadata](go-binaries.md), [Rust dependencies](rust-binaries.md), [offline ELF layout](binary-diagnostics.md), [recorded crashes](recorded-crashes.md) |
| Select a native provider or import format | [Ghidra semantics](ghidra-provider.md), [Windows](windows-ghidra-p0.md), [DOS](ghidra-dos.md), [raw bytes](raw-binary.md), [MIPS](ghidra-mips.md), [PSP](ghidra-psp.md), [NativeAOT](ghidra-nativeaot.md), [IDA](ida-provider.md), [Kuna](kuna-provider.md), [JEB](jeb-analysis.md) |
| Analyze JavaScript or Electron artifacts | [Static analysis](javascript-artifact-reconstruction.md), [tracing and comparisons](javascript-application-workflows.md), [application graph](javascript-application-graph.md), [source recovery](javascript-recovery.md) |
| Compare static JavaScript with runtime observations | [Runtime attribution](javascript-runtime-observation.md), [reconciliation](javascript-runtime-reconciliation.md), [Electron observation](electron-observation.md), [Electron fuses](electron-fuses.md) |
| Inspect assemblies, packages, or firmware | [Godot PCK](godot-pck.md), [.NET](managed-code-analysis.md), [Python packages](python-packaged-applications.md), [Android](android-analysis.md), [Apktool resources](apktool-resource-analysis.md), [ADB acquisition](adb-device-analysis.md), [Apple](apple-application-analysis.md), [Harmony](harmony-analysis.md), [firmware](firmware-analysis.md), [WASM](wasm-artifacts.md), [EVM](evm-bytecode.md) |
| Observe or compare a browser scenario | [CDP observation](browser-observation.md), [scenarios](browser-scenario-contract.md), [runtime evidence](web-runtime.md), [historical captures](web-network-captures.md) |
| Serve recorded behavior as an agent environment | [Agent environments](agent-environments.md) |
| Trace captured website source | [Source locations](web-source-location.md), [module traces](website-module-trace.md), [script export](website-script-export.md) |
| Capture and compare a command | [Process capture](process-capture.md) |
| Use CLI/MCP and retain evidence | [CLI](cli.md), [MCP contracts](mcp-contracts.md), [agent prompts](mcp-prompts.md), [obligation ledgers](reconstruction-obligation-ledgers.md) |
For development, use [testing](testing.md), [client verification](client-verification.md),
[tool design](tool-design.md), and [releasing](releasing.md).
docs/model-artifacts.md
+5
View File
@@ -5,6 +5,11 @@ GGUF v2/v3 or safetensors file. Detection uses content, including GGUF byte
order, rather than its extension. REA does not load a model, interpret weights,
run inference, import Python, open external-data references or use the network.
Unsupported GGUF versions remain refused. Their diagnostic preserves the four
version bytes and both UINT32 interpretations with byte order unknown, rather
than inferring big-endian from an unrecognized little-endian value. Little-endian
v2/v3 and big-endian v3 remain supported; big-endian v2 remains refused.
```bash
rea inspect-model-artifact /absolute/path/model.gguf --json
rea inspect-model-artifact /absolute/path/model.safetensors --hash full --json
docs/native-investigation.md
+22 -2
View File
@@ -298,13 +298,20 @@ markup, including UTF-8 and BOM-marked UTF-16 sources. Unidentified text and
OpenStep plists, including a data-only value such as `<deadbeef>`, report
`unknown` even when plutil decodes their values through an XML conversion.
Binary plist inspection reads the original bytes through an owned file descriptor,
so conversion cannot erase control characters, fractional dates, negative zero,
or the high word of a 128-bit integer. It accepts at most 64 MiB of input and
reserves an estimated 64 MiB decode budget before parsing; this estimate is not
a hard process-memory bound. Binary-source provenance records the `file`
classification without claiming a `plutil` conversion.
A plist explicitly selected through the path input or as the active plist target
reports an unreadable or malformed selection as `invalid_request`. A missing or
unreadable default plist of another target reports `capability_unavailable`.
Permission failures remain `access_denied`; output and cleanup failures retain
their own error categories.
A plist decode refusal includes plutil's diagnostic from either output stream.
A conversion refusal includes plutil's diagnostic from either output stream.
`details.captured_output` retains bounded stdout/stderr excerpts, observed byte
totals and exit status. `truncated: true` means those excerpts are incomplete.
@@ -444,6 +451,13 @@ complete observation.
an unfixed zero in a nullable ABI field is treated as null. Incomplete or
conflicting fixup streams keep affected pointers unresolved; coverage reports
parsing and resource limits while retaining validated prefix facts.
Conformance records preserve the raw three-bit `type_reference_kind`. Kinds
0 and 1 identify direct and indirect nominal descriptors; kind 2 preserves
the inline Objective-C class name and its source location. Kind 3 preserves
an imported class binding or the local class-object address when available,
but class metadata/name recovery requiring the Objective-C runtime remains
partial. Unknown kinds retain the raw kind and reference location without
being treated as absent or assigned a guessed type name.
So an external superclass resolves from its `_OBJC_CLASS_$_` bind (for
example `NSObject`). The `pointer_fixups` coverage facet names the mechanism.
@@ -714,7 +728,13 @@ leap seconds) and out-of-range dates remain explicit unknowns. REA does not
emulate CoreFoundation's normalization of overflowing calendar fields. Invalid
dates returned by other decoders retain `{ "$plist_type": "date", "iso8601": null }`
with a counted limitation. `inspect_plist` and Interface Builder decoding use
the same projection.
the same projection. Binary plist decoding refuses a reachable date whose marker
is not `0x33`; date and numeric payloads must end before the offset table.
Valid overlapping object records retain their independent source values. A valid `0x33` date is
reported with an ISO-8601 timestamp only when its float64 seconds are exactly
the canonical encoding of that millisecond instant. Any other finite binary
date uses `{ "$plist_type": "date", "iso8601": null }` and the counted
limitation; the archive bytes remain the source of the instant.
A negative-zero real is reported as `{ "$plist_type": "real", "value": "-0" }`
because a JSON number cannot carry the sign of zero. `inspect_plist` and
docs/pcap-analysis.md
+5 -1
View File
@@ -67,7 +67,11 @@ Addresses count once per packet, including loopback and nested IP occurrences.
DNS addresses are associated only with one question in the same response packet;
additional records may be included, so this does not prove final resolution.
Ambiguous addresses remain in a separate inventory. Protocol distribution counts
exact `frame.protocols` stack strings. TLS SNI is recorded handshake metadata,
exact nonempty `frame.protocols` stack strings. Frames for which TShark reports
an empty stack, including zero-length packets, still contribute to the packet
count without adding a protocol row. An all-empty stack inventory can have
complete examined coverage; it does not establish the absence of network traffic.
TLS SNI is recorded handshake metadata,
not process identity or proof of the destination's authenticity.
All examined inventories are retained within the resource budget; no fixed
docs/process-capture.md
+27 -1
View File
@@ -461,7 +461,33 @@ selected roots; it is not a per-file allowance. File contents are read in
Ownership inspections bind run-token reads to native start identities and retain
related generations through reparenting, group changes, omitted rows, and cleanup
retries. The captured run keeps one evidence owner through settlement and cleanup.
retries. A host without native identity reads still token-checks process-table
rows that appear only in a launcher-exit refresh before reporting that group
clean or publishing an empty lineage. A matching token authorizes a signal only
for a group whose leader carries the same token. A matching token whose leader
is absent, or an unreadable token for a still-live new row, stays unresolved. A
readable foreign token is not signaled. An unreadable row that has exited by
the confirmation read does not hide a new row in that confirmation table, and a
group proven while revalidating another owned group is signaled in the same
cleanup. A matched row that changes parent or group on the confirmation read
is token-checked again. A PID whose token was already read is read again
on the next confirmation even when its parent and group return to the
pre-exit values. A PID whose first token was foreign is read again
on that table, because the PID alone does not identify the process. A PID
from the pre-exit table is read again when its parent or group changes, and
an unchanged pre-existing row that was never token-read stays unread. Lineage
publishes one row for each PID. Confirmation reads are bounded; a table that keeps producing unreadable
replacements stays unresolved. That refresh does not retain a ledger
for a later attempt that never observes the launcher. Lineage reads a row again
when its parent, group, or launcher-tree membership changes, and it reads a
token again on the next refresh even when those fields are unchanged. The
launcher is read on every lineage refresh; a missing, unowned, or
relation-changed launcher PID is not published as a verified lineage. A
descendant absent from the latest refresh is omitted. An
ancestor move cannot leave a foreign descendant inside a verified result. A
group that reappears after cleanup already processed it is revalidated once;
another appearance stays unresolved. The captured run keeps
one evidence owner through settlement and cleanup.
An absent or changed token leaves a previously related generation unresolved;
current token proof is still required before signaling. Swept cleanup verifies the
remaining family after its group signals. Coverage is bounded by the native
docs/python-packaged-applications.md
+128
View File
@@ -0,0 +1,128 @@
# Python packaged applications
REA statically inventories PyInstaller executables with an appended CArchive.
`inspect_artifact` and `extract_artifact` share the same reader through CLI and
MCP. They read archive bytes without launching the application. Bytecode
source decompilation, encrypted PYZ recovery, zipapp and PEX analysis are outside
this capability.
## Facts and focused queries
The root occurrence's `format_metadata` records the CArchive layout (2.0 or
2.1), archive and cookie offsets, table extent, entry count, raw Python version,
normalized major/minor when known, library name and encryption marker. Member
metadata retains the original name, table index, type, compression flag,
relative offset and known directory record extent. Logical paths use archive
path rules; the source name retains its spelling, including backslashes.
`relative_offset` is relative to the CArchive start for CArchive rows, or to
the decoded PYZ image for PYZ rows. A CArchive directory record starts at
`archive_start + toc_offset + directory_record_offset`; its size is
`directory_record_size`. PYZ row directory extents remain unknown.
Namespace packages remain directories. Their `relative_offset` and
`compressed_size` preserve any payload declared by older producers, while
`declared_size` is zero because no bytecode is materialized for a namespace.
Every declared span must end before the PYZ table of contents.
PYZ `compression_flag` is derived from storage: nonempty payloads are compressed,
including legacy namespaces; an empty namespace has no compressed payload.
CArchive type `o` entries are runtime options. They are metadata occurrences,
with no content artifact or extractable file. Their query paths are
`__pyinstaller__/runtime-options/<table index>`; the option text is
`format_metadata.source_name`. Repeated declarations retain distinct occurrence
identities. These paths do not claim output filesystem names.
Type `d` entries are multipackage dependencies. They are also metadata, under
`__pyinstaller__/dependencies/<table index>`, and never materialize an empty file.
The reader preserves the raw reference and splits its first colon into
`dependency_archive` and `dependency_member`. `dependency_status` remains
`unresolved`, or `invalid-reference` with unknown parts for malformed references.
`external_dependency_count` summarizes these declarations. REA does not open
another executable or claim its contents are available. A real file with the
same logical path as metadata remains independently extractable.
Both declaration types require empty payloads and no compression. Their query
paths do not participate in filesystem collision checks, directory ancestry or
source-map target lookup; their parent identifies the declaring container.
Complete authenticated inventory page sets preserve producer identities in any
merge order, including metadata/file aliases. Malformed TOC diagnostics name the
table index; original names remain available in the supplied artifact.
Path failures use bounded escaped previews and report the original length;
accepted paths and retained source names are not shortened.
Request `detail: "summary"` for a compact inspection, then query retained
Evidence with `inspect_analysis_view`. An inventory view can select
`collection: "occurrences"`, `entry_kinds: ["metadata"]`, exact logical paths or
occurrence IDs, requested `fields`, and `offset`/`limit`. Filters intersect;
identities, parent Evidence and coverage survive projection. Full inventory
Evidence remains available for export. A compact reply reduces returned context;
it still performs the complete member inventory and hashing work.
Focused views add calls and tool-schema costs. A complete small inventory can
cost less overall; prefer compact delivery when the full member graph would
overwhelm the answer.
```bash
rea inspect-artifact /absolute/path/hello --detail summary --evidence_output /absolute/new/evidence.json --json
rea extract-artifact /absolute/path/hello --json
```
## Decoding and evidence limits
Inventory hashes members individually through bounded streams. Stored PYZ
images use descriptor ranges; outer-compressed images stream to private files,
which the reader owns through cleanup and any cleanup retry. Extraction binds
the selected carrier to its inventoried digest and checks source identity and
state before and after reading. It does not retain expanded module bodies.
Opaque outer-compressed images reuse that verified private image when opened,
instead of inflating the carrier again. Direct source spans describe stored
CArchive bytes; decompressed content has no direct span even when its encoded
and decoded sizes coincide. Original offsets and compression remain metadata.
Only the PYZ header and marshal directory are loaded for listing. PYZ tables
have a 16 MiB budget; CArchive tables have a 32 MiB budget. Parsed facts have a
32 MiB serialized budget. Projected entries share a 64 MiB retained budget,
including metadata, bookkeeping and conservative reservations for namespace
objects and possible case-collision annotations at each path component. These
reservations happen before normalization or trie allocation.
Outer-compressed PYZ images have a separate per-reader capacity of 128 private
files, bounding retained inodes and image identities. The complete image count
is admitted before any temporary file is created; excess images return a
capacity failure. This is a resource policy, not a PyInstaller format limit.
Only a zero-byte read establishes EOF; short successful descriptor reads fill
the bounded range. Compressed members must consume their complete declared
input span; trailing junk and concatenated streams are refused.
Plaintext expansion has an aggregate archive-derived budget. Exceeding a budget
returns a resource failure.
Supported final Python bytecode magic values cover Python 3.7 through 3.14.
Supported marshal directory values are lists, tuples, strings, integers and
bytes. An unsupported magic or directory stays an opaque member with a
limitation and `pyz_status: "opaque"`; its module contents remain unknown.
Any opaque PYZ makes producer traversal coverage unknown, including retained
summaries and exhausted queries. Per-archive reasons remain on the opaque
occurrences; the coverage observation summarizes their count without repeating
every limitation. An empty result does not establish module absence.
Observed `pyz_python_magic` bytes remain available even when their Python version
is unknown.
The directory declares compressed module lengths, so uncompressed declared
sizes and source spans inside decoded data remain unknown.
PYZ modules receive a derived 16-byte `.pyc` header from their own supported
magic. CArchive bytecode receives that header only when every sibling PYZ magic
and the cookie's Python version agree. Otherwise it retains its original bytes
and `bytecode_header: "unknown"`. `bytecode_magic` and graph derivation edges
identify synthesized headers; member digests cover the resulting derived files.
These headers do not establish successful execution or decompilation.
The nonempty CArchive Python module `pyimod00_crypto_key` (`m`/`M`) is recorded
as an encryption marker; a same-named data file or script is not. Affected
PYZ blobs are reported without decoding; extraction refuses encrypted entries.
The raw PYZ container remains readable and hashable, including when its module
table is opaque. Its marker stays in metadata and opaque child coverage stays
unknown; extracting the container does not decrypt its payloads.
The marker also applies to nonempty legacy namespace payloads. Empty namespaces
have no payload to encrypt and retain a null encryption source.
Detection searches the last 16 MiB for a cookie, allowing a trailing macOS code
signature within that bound. Member offsets are relative to the archive start:
cookie position plus cookie size minus package length. Native bootloader
analysis remains available through the native tools.
docs/readme/README_ar.md
+1
View File
@@ -152,6 +152,7 @@ npx rea-agents@latest setup
| تسجيلات الشبكة المحفوظة | الطلبات، والاستجابات، ومحتويات الحمولة المتاحة، ومواضعها في المصدر | HAR؛ وmitmdump على Linux لتسجيلات mitmproxy بصيغتها الأصلية؛ [دليل تسجيلات الشبكة](../web-network-captures.md) |
| التقاط الحزم | نقاط النهاية وDNS وTLS SNI وتوزيع البروتوكولات وطلبات HTTP | tshark وcapinfos يوفّرهما المُستدعي؛ [دليل pcap](../pcap-analysis.md) |
| تجميعات .NET | البيانات الوصفية، وتعليمات CIL، واعتماديات الشيفرة الأصلية المعلنة، ومقارنات البناء | فحص ثابت؛ [دليل الشيفرة المُدارة](../managed-code-analysis.md) |
| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) |
| حزم Android APK | تصريحات ملف manifest، والأصناف، والدوال الناتجة عن فك الترجمة، والمراجع | JADX دون واجهة رسومية وJDK كامل على Linux/macOS/Windows x64؛ [دليل Android](../android-analysis.md) |
| أجهزة Android | الأجهزة والحزم والعمليات والسجلات والدامبات ولقطات الشاشة ونقل APK/الملفات | adb يوفّره المُستدعي (محاكي أو جهاز)؛ [دليل ADB](../adb-device-analysis.md) |
| مشاريع JEB | وحدات المشروع من المحرك وبصمات المصنّفات والكود الزائف للأنواع/الدوال | عميل JEB يبدأه المُستدعي ويقدّم MCP، ويُختار عبر REA_JEB_MCP_URL؛ [دليل JEB](../jeb-analysis.md) |
docs/readme/README_de.md
+1
View File
@@ -152,6 +152,7 @@ REA benötigt Node.js 22.x (>=22.19), 24.x (>=24.11) oder 26+ sowie npm. Zusätz
| Gespeicherte Netzwerkaufzeichnungen | Anfragen, Antworten, zugängliche Nutzdaten und Quellpositionen | HAR; mitmdump unter Linux für Aufzeichnungen im nativen mitmproxy-Format; [Leitfaden zu Netzwerkaufzeichnungen](../web-network-captures.md) |
| Paketerfassungen | Endpunkte, DNS, TLS SNI, Protokollverteilung und HTTP-Anfragen | Vom Aufrufer bereitgestellte tshark- und capinfos-Binaries; [pcap-Leitfaden](../pcap-analysis.md) |
| .NET-Assemblies | Metadaten, CIL-Anweisungen, deklarierte native Abhängigkeiten und Build-Vergleiche | Statische Untersuchung; [Leitfaden für verwalteten Code](../managed-code-analysis.md) |
| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) |
| Android-APKs | Manifest-Deklarationen, Klassen, dekompilierte Methoden und Verweise | JADX ohne grafische Oberfläche und ein vollständiges JDK unter Linux/macOS/Windows x64; [Android-Leitfaden](../android-analysis.md) |
| Android-Geräte | Geräte, Pakete, Prozesse, Logs, Dumps, Bildschirmaufnahmen sowie APK- und Dateiübertragungen | Vom Aufrufer bereitgestelltes adb (Emulator oder Gerät); [ADB-Leitfaden](../adb-device-analysis.md) |
| JEB-Projekte | Engine-gestützte Projekteinheiten, Artefakt-Prüfsummen und Pseudokode von Typen/Methoden | Ein vom Aufrufer gestarteter JEB-Client mit MCP, ausgewählt über REA_JEB_MCP_URL; [JEB-Leitfaden](../jeb-analysis.md) |
docs/readme/README_el.md
+1
View File
@@ -158,6 +158,7 @@ npx rea-agents@latest setup
| Ιστότοποι | Δομή σελίδων, scripts, παρατηρήσεις δικτύου και ζητούμενα στιγμιότυπα οθόνης | Πρόγραμμα περιήγησης της οικογένειας Chrome· [ανάλυση μέσω προγράμματος περιήγησης](../browser-observation.md) |
| Αποθηκευμένες καταγραφές δικτύου | Αιτήματα, αποκρίσεις, διαθέσιμα δεδομένα φορτίου και θέσεις στην πηγή | HAR ή SAZ του Fiddler Everywhere· mitmdump σε Linux για την εγγενή μορφή mitmproxy· [οδηγός καταγραφών](../web-network-captures.md) |
| Assemblies .NET | Μεταδεδομένα, εντολές CIL, δηλωμένες εξαρτήσεις από κώδικα μηχανής και συγκρίσεις builds | Καμία πρόσθετη απαίτηση (μόνο στατική ανάλυση)· [οδηγός διαχειριζόμενου κώδικα](../managed-code-analysis.md) |
| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) |
| APK Android | Δηλώσεις manifest, κλάσεις, απομεταγλωττισμένες μέθοδοι και αναφορές | JADX χωρίς γραφικό περιβάλλον και πλήρες JDK σε Linux/macOS/Windows x64· [οδηγός Android](../android-analysis.md) |
| Συσκευές Android | Συσκευές, πακέτα, διεργασίες, logs, dumps, στιγμιότυπα οθόνης και μεταφορές APK/αρχείων | adb που παρέχει ο χρήστης (εξομοιωτής ή φυσική συσκευή)· [οδηγός ADB](../adb-device-analysis.md) |
| Έργα JEB | Μονάδες έργου από τη μηχανή ανάλυσης, ψηφιακά αποτυπώματα αρχείων και ψευδοκώδικα τύπων/μεθόδων | Client JEB που έχει ξεκινήσει ο χρήστης και παρέχει MCP, επιλεγμένος μέσω REA_JEB_MCP_URL· [οδηγός JEB](../jeb-analysis.md) |
docs/readme/README_es.md
+1
View File
@@ -152,6 +152,7 @@ REA requiere Node.js 22.x (>=22.19), 24.x (>=24.11) o 26+, además de npm. Las h
| Capturas de red guardadas | Solicitudes, respuestas, contenido accesible de las cargas útiles y ubicaciones de origen | HAR; mitmdump en Linux para capturas en formato nativo de mitmproxy; [guía de capturas](../web-network-captures.md) |
| Capturas de paquetes | Endpoints, DNS, TLS SNI, distribución de protocolos y solicitudes HTTP | tshark y capinfos suministrados por el autor de la llamada; [guía pcap](../pcap-analysis.md) |
| Ensamblados .NET | Metadatos, instrucciones CIL, dependencias nativas declaradas y comparaciones de compilaciones | Inspección estática; [guía de código administrado](../managed-code-analysis.md) |
| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) |
| APK de Android | Declaraciones del manifiesto, clases, métodos descompilados y referencias | JADX sin interfaz gráfica y un JDK completo en Linux/macOS/Windows x64; [guía de Android](../android-analysis.md) |
| Dispositivos Android | Dispositivos, paquetes, procesos, registros, volcados, capturas de pantalla y transferencias de APK/archivos | adb suministrado por el autor de la llamada (emulador o dispositivo); [guía de ADB](../adb-device-analysis.md) |
| Proyectos JEB | Unidades de proyecto del motor, resúmenes de artefactos y pseudocódigo de tipos/métodos | Un cliente JEB iniciado por el autor de la llamada que sirve MCP, seleccionado con REA_JEB_MCP_URL; [guía de JEB](../jeb-analysis.md) |
docs/readme/README_fa.md
+1
View File
@@ -143,6 +143,7 @@ REA به Node.js 22.x (>=22.19)، 24.x (>=24.11) یا 26+، به‌همراه np
| وب‌سایت‌ها | ساختار صفحه، اسکریپت‌ها، مشاهدات شبکه و اسکرین‌شات‌های درخواستی | مرورگر مبتنی بر Chrome؛ [تحلیل مرورگر](../browser-observation.md) |
| داده‌های ضبط‌شدهٔ شبکه | درخواست‌ها، پاسخ‌ها، payloadهای آشکارشده و مکان آن‌ها در منبع | HAR؛ ابزار mitmdump در لینوکس برای ضبط‌های بومی mitmproxy؛ [راهنمای ضبط شبکه](../web-network-captures.md) |
| اسمبلی‌های .NET | فراداده، دستورالعمل‌های CIL، وابستگی‌های بومی اعلام‌شده و مقایسهٔ buildها | بررسی ایستا؛ [راهنمای کد مدیریت‌شده](../managed-code-analysis.md) |
| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) |
| فایل‌های APK اندروید | تعاریف Manifest، کلاس‌ها، متدهای دیکامپایل‌شده و ارجاعات | JADX بدون رابط گرافیکی و JDK کامل در Linux/macOS/Windows x64؛ [راهنمای اندروید](../android-analysis.md) |
| دستگاه‌های اندروید | دستگاه‌ها، بسته‌ها، فرایندها، لاگ‌ها، دامپ‌ها، اسکرین‌شات‌ها و انتقال APK/فایل | adb تأمین‌شده توسط فراخوان‌کننده (شبیه‌ساز یا دستگاه)؛ [راهنمای ADB](../adb-device-analysis.md) |
| پروژه‌های JEB | واحدهای پروژه از موتور، هش‌های مصنوع و شبه‌کدِ نوع/متد | یک کلاینت JEB که توسط فراخوان‌کننده اجرا شده و MCP ارائه می‌دهد و با REA_JEB_MCP_URL انتخاب می‌شود؛ [راهنمای JEB](../jeb-analysis.md) |
docs/readme/README_fr.md
+1
View File
@@ -156,6 +156,7 @@ REA nécessite Node.js 22.x (>=22.19), 24.x (>=24.11) ou 26+, ainsi que npm. Les
| Captures réseau enregistrées | Requêtes, réponses, payloads exposés et emplacements dans le code source | HAR ; mitmdump sous Linux pour les captures mitmproxy natives ; [guide des captures](../web-network-captures.md) |
| Captures de paquets | Points de terminaison, DNS, TLS SNI, distribution des protocoles et requêtes HTTP | tshark et capinfos fournis par l'appelant ; [guide pcap](../pcap-analysis.md) |
| Assemblys .NET | Métadonnées, instructions CIL, dépendances natives déclarées et comparaisons de builds | Inspection statique ; [guide du code managé](../managed-code-analysis.md) |
| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) |
| APK Android | Déclarations du manifeste, classes, méthodes décompilées et références | JADX headless et un JDK complet sous Linux/macOS/Windows x64 ; [guide Android](../android-analysis.md) |
| Appareils Android | Appareils, paquets, processus, journaux, dumps, captures d'écran et transferts d'APK/fichiers | adb fourni par l'appelant (émulateur ou appareil) ; [guide ADB](../adb-device-analysis.md) |
| Projets JEB | Unités de projet fournies par le moteur, condensés d'artefacts et pseudo-code de types/méthodes | Un client JEB démarré par l'appelant servant MCP, sélectionné via REA_JEB_MCP_URL ; [guide JEB](../jeb-analysis.md) |
docs/readme/README_id.md
+1
View File
@@ -159,6 +159,7 @@ Alat tambahan dan dukungan sistem host bergantung pada target yang dianalisis:
| Rekaman jaringan tersimpan | Permintaan, respons, payload yang terekspos, dan lokasi sumber | HAR; mitmdump di Linux untuk rekaman native mitmproxy; [panduan rekaman](../web-network-captures.md) |
| Capturan paket | Endpoint, DNS, TLS SNI, distribusi protokol, dan permintaan HTTP | tshark dan capinfos buatan pemanggil; [panduan pcap](../pcap-analysis.md) |
| Assembly .NET | Metadata, instruksi CIL, dependensi native yang dideklarasikan, dan perbandingan build | Tidak ada (hanya analisis statis); [panduan kode terkelola](../managed-code-analysis.md) |
| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) |
| APK Android | Deklarasi manifest, kelas, metode hasil dekompilasi, dan referensi | JADX mode headless dan JDK lengkap di Linux/macOS/Windows x64; [panduan Android](../android-analysis.md) |
| Perangkat Android | Perangkat, paket, proses, log, dump, tangkapan layar, serta transfer APK/berkas | adb buatan pemanggil (emulator atau perangkat); [panduan ADB](../adb-device-analysis.md) |
| Proyek JEB | Unit proyek bertenaga mesin, hash artefak, dan pseudokode tipe/metode | Klien JEB yang dijalankan pemanggil dan menyajikan MCP, dipilih dengan REA_JEB_MCP_URL; [panduan JEB](../jeb-analysis.md) |
docs/readme/README_it.md
+1
View File
@@ -156,6 +156,7 @@ REA richiede Node.js 22.x (>=22.19), 24.x (>=24.11) o 26+, oltre a npm. Strument
| Acquisizioni di rete salvate | Richieste, risposte, payload esposti e posizioni sorgente | HAR; mitmdump su Linux per acquisizioni mitmproxy native; [guida alle acquisizioni](../web-network-captures.md) |
| Captured di pacchetti | Endpoint, DNS, TLS SNI, distribuzione dei protocolli e richieste HTTP | tshark e capinfos forniti dal chiamante; [guida pcap](../pcap-analysis.md) |
| Assembly .NET | Metadati, istruzioni CIL, dipendenze native dichiarate e confronti di build | Nessuno (solo statico); [guida managed-code](../managed-code-analysis.md) |
| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) |
| APK Android | Dichiarazioni manifest, classi, metodi decompilati e riferimenti | JADX headless e un JDK completo su Linux/macOS/Windows x64; [guida Android](../android-analysis.md) |
| Dispositivi Android | Dispositivi, pacchetti, processi, log, dump, screenshot e trasferimenti APK/file | adb fornito dal chiamante (emulatore o hardware); [guida ADB](../adb-device-analysis.md) |
| Progetti JEB | Unità di progetto dal motore, digest degli artifact e pseudocodice di tipi/metodi | Client JEB avviato dal chiamante via MCP, selezionato con REA_JEB_MCP_URL; [guida JEB](../jeb-analysis.md) |
docs/readme/README_ja.md
+1
View File
@@ -152,6 +152,7 @@ REA には Node.js 22.x（>=22.19）、24.x（>=24.11）、または 26+ と npm
| 保存済みのネットワークキャプチャ | リクエスト、レスポンス、取得可能なペイロード、ソース内の位置 | HAR；mitmproxy ネイティブ形式のキャプチャには Linux 上の mitmdump；[キャプチャガイド](../web-network-captures.md) |
| パケットキャプチャ | エンドポイント、DNS、TLS SNI、プロトコル分布、HTTPリクエスト | 呼び出し側が用意した tshark と capinfos；[pcapガイド](../pcap-analysis.md) |
| .NET アセンブリ | メタデータ、CIL 命令、宣言されたネイティブ依存関係、ビルドの比較 | 静的な検査；[マネージドコードのガイド](../managed-code-analysis.md) |
| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) |
| Android APK | マニフェストの宣言、クラス、逆コンパイルされたメソッド、参照 | Linux/macOS/Windows x64 上のヘッドレス JADX と完全な JDK；[Android ガイド](../android-analysis.md) |
| Android デバイス | デバイス、パッケージ、プロセス、ログ、ダンプ、スクリーンキャプチャ、APK/ファイル転送 | 呼び出し側が用意した adb（エミュレーターまたは実機）；[ADB ガイド](../adb-device-analysis.md) |
| JEBプロジェクト | エンジンによるプロジェクトユニット、アーティファクトのダイジェスト、型/メソッドの擬似コード | 呼び出し側が起動し MCP を提供する JEB クライアント（REA_JEB_MCP_URL で選択）；[JEB ガイド](../jeb-analysis.md) |
docs/readme/README_ko.md
+1
View File
@@ -173,6 +173,7 @@ REA에는 Node.js 22.x(>=22.19), 24.x(>=24.11) 또는 26+와 npm이 필요합니
| 기록된 논리 신호 | 디코더 주석, 샘플 범위, 진단 정보 | Linux: sigrok-cli 0.7.2 + prlimit; [신호 기록 가이드](../signal-recordings.md) |
| 패키지 및 리소스 | 파일 목록, 다이제스트, plist, Apple 번들 구조, 추출한 리소스 | [아티팩트 및 JavaScript 가이드](../javascript-artifact-reconstruction.md), [Apple 애플리케이션](../apple-application-analysis.md) |
| 프로세스 동작 | 터미널 출력, 상호작용, 종료 및 파일 시스템 관찰 결과, 실행 비교 | 네이티브 PTY를 지원하는 Linux/macOS; [프로세스 캡처](../process-capture.md) |
| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) |
정적 JavaScript 및 .NET 검사는 제공된 파일을 읽으며 애플리케이션을 실행하지 않습니다. 런타임 캡처는 사용자의 권한으로 선택한 대상을 실행하거나 대상과 상호작용합니다. 각 런타임 가이드에서 그 영향을 설명합니다.
docs/readme/README_ml.md
+1
View File
@@ -160,6 +160,7 @@ REA-യ്ക്ക് Node.js 22.x (>=22.19), 24.x (>=24.11), അല്ലെ
| Websites | Page structure, scripts, network observations, ആവശ്യപ്പെട്ട screenshots | Chrome-family browser; [browser analysis](../browser-observation.md) |
| Saved network captures | Requests, responses, പുറത്തുകാണുന്ന payload-ുകൾ, source locations | HAR അല്ലെങ്കിൽ Fiddler Everywhere SAZ; native mitmproxy-നായി Linux-ൽ mitmdump; [capture guide](../web-network-captures.md) |
| .NET assemblies | Metadata, CIL instructions, പ്രഖ്യാപിച്ച native dependencies, build comparisons | ഒന്നും ആവശ്യമില്ല (static മാത്രം); [managed-code guide](../managed-code-analysis.md) |
| Python packaged apps | CArchive/PYZ entries, Python version, bytecode, resources and runtime options | Static PyInstaller parsing; [Python guide](../python-packaged-applications.md) |
| Android APKs | Manifest declarations, classes, decompiled methods, references | Linux/macOS/Windows x64-ൽ Headless JADX-ഉം full JDK-ഉം; [Android guide](../android-analysis.md) |
| Android devices | Devices, packages, processes, logs, dumps, screen captures, APK/file transfers | Caller നൽകുന്ന adb (emulator അല്ലെങ്കിൽ hardware); [ADB guide](../adb-device-analysis.md) |
| JEB projects | Engine-backed project units, artifact digests, type/method pseudo-code | MCP സേവനം നൽകുന്ന caller ആരംഭിച്ച JEB client; REA_JEB_MCP_URL ഉപയോഗിച്ച് തിരഞ്ഞെടുക്കുക; [JEB guide](../jeb-analysis.md) |
docs/readme/README_pl.md
+1
View File
@@ -152,6 +152,7 @@ REA wymaga Node.js 22.x (>=22.19), 24.x (>=24.11) lub 26+ oraz npm. Dodatkowe na
| Zapisane przechwycenia ruchu sieciowego | Żądania, odpowiedzi, dostępne dane przesyłane i lokalizacje źródłowe | HAR; mitmdump w systemie Linux dla natywnych przechwyceń mitmproxy; [przewodnik przechwytywania](../web-network-captures.md) |
| Przechwytywanie pakietów | Punkty końcowe, DNS, TLS SNI, dystrybucja protokołów i żądania HTTP | Dostarczone przez wywołującego tshark i capinfos; [przewodnik pcap](../pcap-analysis.md) |
| Zestawy .NET | Metadane, instrukcje CIL, zadeklarowane zależności natywne i porównania kompilacji | Analiza statyczna; [przewodnik kodu zarządzanego](../managed-code-analysis.md) |
| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) |
| Pakiety APK dla Androida | Deklaracje manifestu, klasy, zdekompilowane metody i odwołania | JADX bez interfejsu graficznego i pełny JDK w systemach Linux/macOS/Windows x64; [przewodnik Androida](../android-analysis.md) |
| Urządzenia z Androidem | Urządzenia, pakiety, procesy, logi, zrzuty, zrzuty ekranu oraz transfery APK/plików | adb dostarczony przez wywołującego (emulator lub urządzenie); [przewodnik ADB](../adb-device-analysis.md) |
| Projekty JEB | Jednostki projektu obsługiwane przez silnik, skróty artefaktów i pseudokod typów/metod | Uruchomiony przez wywołującego klient JEB udostępniający MCP, wybrany przez REA_JEB_MCP_URL; [przewodnik JEB](../jeb-analysis.md) |
docs/readme/README_pt-BR.md
+1
View File
@@ -152,6 +152,7 @@ O REA requer Node.js 22.x (>=22.19), 24.x (>=24.11) ou 26+, além do npm. As fer
| Capturas de rede salvas | Requisições, respostas, payloads acessíveis e localizações de origem | HAR; mitmdump no Linux para capturas no formato nativo do mitmproxy; [guia de capturas](../web-network-captures.md) |
| Capturas de pacotes | Endpoints, DNS, TLS SNI, distribuição de protocolos e requisições HTTP | tshark e capinfos fornecidos pelo chamador; [guia pcap](../pcap-analysis.md) |
| Assemblies .NET | Metadados, instruções CIL, dependências nativas declaradas e comparações de builds | Inspeção estática; [guia de código gerenciado](../managed-code-analysis.md) |
| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) |
| APKs Android | Declarações do manifesto, classes, métodos descompilados e referências | JADX sem interface gráfica e um JDK completo no Linux/macOS/Windows x64; [guia de Android](../android-analysis.md) |
| Dispositivos Android | Dispositivos, pacotes, processos, logs, despejos, capturas de tela e transferências de APK/arquivos | adb fornecido pelo chamador (emulador ou aparelho); [guia de ADB](../adb-device-analysis.md) |
| Projetos JEB | Unidades de projeto do motor, hashes de artefatos e pseudocódigo de tipos/métodos | Um cliente JEB iniciado pelo chamador servindo MCP, selecionado com REA_JEB_MCP_URL; [guia de JEB](../jeb-analysis.md) |
docs/readme/README_ru.md
+1
View File
@@ -162,6 +162,7 @@ REA требует Node.js 22.x (>=22.19), 24.x (>=24.11) или 26+, а так
| Записанные логические сигналы | Аннотации декодера, диапазоны отсчётов и диагностика | Linux: sigrok-cli 0.7.2 + prlimit; [руководство по сигналам](../signal-recordings.md) |
| Пакеты и ресурсы | Списки файлов, хеши, plist, структура пакетов Apple и извлечённые ресурсы | [Руководство по артефактам и JavaScript](../javascript-artifact-reconstruction.md), [приложения Apple](../apple-application-analysis.md) |
| Поведение процессов | Вывод терминала, взаимодействия, наблюдения за завершением и файловой системой, сравнения запусков | Linux/macOS с нативным PTY; [захват процессов](../process-capture.md) |
| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) |
Статическое исследование JavaScript и .NET читает предоставленные файлы, не запуская приложение. Захват во время выполнения запускает выбранную цель или взаимодействует с ней с правами вашего пользователя; соответствующие руководства описывают последствия.
docs/readme/README_th.md
+1
View File
@@ -152,6 +152,7 @@ REA ต้องใช้ Node.js 22.x (>=22.19), 24.x (>=24.11) หรือ 26
| ข้อมูลเครือข่ายที่บันทึกไว้ | คำขอ คำตอบ เพย์โหลดที่ปรากฏ และตำแหน่งต้นทาง | HAR; mitmdump บน Linux สำหรับข้อมูลที่บันทึกในรูปแบบเนทีฟของ mitmproxy; [คู่มือการบันทึกข้อมูล](../web-network-captures.md) |
| การจับแพ็กเก็ต | จุดเชื่อมต่อ, DNS, TLS SNI, การกระจายโปรโตคอล และคำขอ HTTP | tshark และ capinfos ที่ผู้เรียกจัดเตรียม; [คู่มือ pcap](../pcap-analysis.md) |
| แอสเซมบลี .NET | เมทาดาทา คำสั่ง CIL การพึ่งพาเนทีฟที่ประกาศไว้ และการเปรียบเทียบผลการบิลด์ | การตรวจสอบแบบสถิต; [คู่มือโค้ดแบบจัดการ](../managed-code-analysis.md) |
| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) |
| APK ของ Android | การประกาศใน manifest คลาส เมธอดที่ดีคอมไพล์ และการอ้างอิง | JADX แบบไม่มี GUI และ JDK แบบเต็มบน Linux/macOS/Windows x64; [คู่มือ Android](../android-analysis.md) |
| อุปกรณ์ Android | อุปกรณ์ แพ็กเกจ โพรเซส บันทึก ดัมพ์ ภาพหน้าจอ และการถ่ายโอน APK/ไฟล์ | adb ที่ผู้เรียกจัดเตรียม (โปรแกรมจำลองหรืออุปกรณ์จริง); [คู่มือ ADB](../adb-device-analysis.md) |
| โปรเจกต์ JEB | หน่วยโปรเจกต์จากเอนจิน, แฮชของอาร์ติแฟกต์ และซูโดโค้ดของไทป์/เมท็อด | ไคลเอนต์ JEB ที่ผู้เรียกเริ่มและให้บริการ MCP เลือกด้วย REA_JEB_MCP_URL; [คู่มือ JEB](../jeb-analysis.md) |
docs/readme/README_tr.md
+1
View File
@@ -156,6 +156,7 @@ REA; Node.js 22.x (>=22.19), 24.x (>=24.11) veya 26+ ve npm gerektirir. Ek araç
| Web siteleri | Sayfa yapısı, betikler, ağ gözlemleri ve istenen ekran görüntüleri | Chrome ailesinden bir tarayıcı; [tarayıcı analizi](../browser-observation.md) |
| Kaydedilmiş ağ yakalamaları | İstekler, yanıtlar, erişilebilir yük içerikleri ve kaynak konumları | HAR veya Fiddler Everywhere SAZ; Linux'ta native mitmproxy için mitmdump; [ağ yakalama rehberi](../web-network-captures.md) |
| .NET derlemeleri | Meta veriler, CIL talimatları, bildirilmiş yerel bağımlılıklar ve derleme karşılaştırmaları | Statik inceleme; [yönetilen kod rehberi](../managed-code-analysis.md) |
| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) |
| Android APK'leri | Manifest bildirimleri, sınıflar, tersine derlenmiş metotlar ve referanslar | Linux/macOS/Windows x64 üzerinde arayüzsüz JADX ve tam bir JDK; [Android rehberi](../android-analysis.md) |
| Android cihazları | Cihazlar, paketler, süreçler, günlükler, dökümler, ekran görüntüleri ve APK/dosya aktarımları | Çağıranın sağladığı adb (emülatör veya cihaz); [ADB rehberi](../adb-device-analysis.md) |
| JEB projeleri | Motor destekli proje birimleri, artefakt özetleri ve tür/metot sözdekodu | Çağıranın başlattığı, MCP sunan bir JEB istemcisi, REA_JEB_MCP_URL ile seçilir; [JEB rehberi](../jeb-analysis.md) |
docs/readme/README_uk.md
+1
View File
@@ -152,6 +152,7 @@ REA потребує Node.js 22.x (>=22.19), 24.x (>=24.11) або 26+, а та
| Збережені мережеві захоплення | Запити, відповіді, доступні корисні дані та місця в джерелі | HAR; mitmdump у Linux для захоплень у власному форматі mitmproxy; [посібник мережевих захоплень](../web-network-captures.md) |
| Захоплення пакетів | Кінцеві точки, DNS, TLS SNI, розподіл протоколів та HTTP-запити | Надані викликаючою стороною tshark та capinfos; [підручник pcap](../pcap-analysis.md) |
| Збірки .NET | Метадані, інструкції CIL, оголошені нативні залежності й порівняння збірок | Статична перевірка; [посібник керованого коду](../managed-code-analysis.md) |
| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) |
| Android APK | Оголошення маніфесту, класи, декомпільовані методи й посилання | JADX без графічного інтерфейсу й повний JDK у Linux/macOS/Windows x64; [посібник Android](../android-analysis.md) |
| Пристрої Android | Пристрої, пакети, процеси, журнали, дампи, знімки екрана та передавання APK/файлів | adb, наданий викликаючою стороною (емулятор або пристрій); [підручник ADB](../adb-device-analysis.md) |
| Проєкти JEB | Єдиниці проєкту від рушія, дайджести артефактів і псевдокод типів/методів | Запущений викликаною стороною клієнт JEB з MCP, обирається через REA_JEB_MCP_URL; [посібник JEB](../jeb-analysis.md) |
docs/readme/README_vi.md
+1
View File
@@ -162,6 +162,7 @@ REA yêu cầu Node.js 22.x (>=22.19), 24.x (>=24.11) hoặc 26+, cùng với np
| Tín hiệu logic đã ghi | Chú thích bộ giải mã, phạm vi mẫu và chẩn đoán | Linux: sigrok-cli 0.7.2 + prlimit; [hướng dẫn tín hiệu](../signal-recordings.md) |
| Gói và tài nguyên | Danh sách tệp, giá trị băm, plist, cấu trúc bundle Apple và tài nguyên được trích xuất | [Hướng dẫn hiện vật phần mềm và JavaScript](../javascript-artifact-reconstruction.md), [ứng dụng Apple](../apple-application-analysis.md) |
| Hành vi tiến trình | Đầu ra terminal, tương tác, quan sát khi thoát và hệ thống tệp, cùng so sánh các lần chạy | Linux/macOS có PTY gốc; [ghi lại tiến trình](../process-capture.md) |
| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) |
Kiểm tra tĩnh JavaScript và .NET đọc các tệp được cung cấp mà không chạy ứng dụng. Ghi lại khi chạy sẽ khởi chạy hoặc tương tác với mục tiêu đã chọn bằng quyền người dùng của bạn; từng hướng dẫn mô tả tác động của nó.
docs/readme/README_zh-TW.md
+1
View File
@@ -156,6 +156,7 @@ REA 需要 Node.js 22.x（>=22.19）、24.x（>=24.11）或 26+，以及 npm。
| 已保存的網路擷取 | 請求、回應、可訪問的酬載和來源位置 | HAR；原生 mitmproxy 擷取需要 Linux 上的 mitmdump；[擷取指南](../web-network-captures.md) |
| 封包擷取 | 端點、DNS、TLS SNI、通訊協定分佈和 HTTP 請求 | 呼叫端提供的 tshark 和 capinfos；[pcap 指南](../pcap-analysis.md) |
| .NET 程式集 | 中繼資料、CIL 指令、聲明的原生依賴和建置對比 | 靜態檢查；[托管程式碼指南](../managed-code-analysis.md) |
| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) |
| Android APK | 清單聲明、類別、反編譯的方法和引用 | Linux/macOS/Windows x64 上的無界面 JADX 和完整 JDK；[Android 指南](../android-analysis.md) |
| Android 裝置 | 裝置、套件、程序、日誌、傾印、螢幕擷取以及 APK/檔案傳輸 | 呼叫端提供的 adb（模擬器或實體裝置）；[ADB 指南](../adb-device-analysis.md) |
| JEB 專案 | 引擎支援的專案單元、成品摘要和型別/方法的虛擬碼 | 由呼叫端啟動、提供 MCP 的 JEB 用戶端，透過 REA_JEB_MCP_URL 選擇；[JEB 指南](../jeb-analysis.md) |
docs/readme/README_zh.md
+1
View File
@@ -156,6 +156,7 @@ REA 需要 Node.js 22.x（>=22.19）、24.x（>=24.11）或 26+，以及 npm。
| 已保存的网络捕获 | 请求、响应、可访问的载荷和来源位置 | HAR；原生 mitmproxy 捕获需要 Linux 上的 mitmdump；[捕获指南](../web-network-captures.md) |
| 数据包捕获 | 端点、DNS、TLS SNI、协议分布和 HTTP 请求 | 调用方提供的 tshark 和 capinfos；[pcap 指南](../pcap-analysis.md) |
| .NET 程序集 | 元数据、CIL 指令、声明的原生依赖和构建对比 | 静态检查；[托管代码指南](../managed-code-analysis.md) |
| PyInstaller | CArchive, PYZ, Python bytecode | Static inspection; [Python packaged applications](../python-packaged-applications.md) |
| Android APK | 清单声明、类、反编译的方法和引用 | Linux/macOS/Windows x64 上的无界面 JADX 和完整 JDK；[Android 指南](../android-analysis.md) |
| Android 设备 | 设备、软件包、进程、日志、转储、屏幕截图以及 APK/文件传输 | 调用方提供的 adb（模拟器或实体设备）；[ADB 指南](../adb-device-analysis.md) |
| JEB 项目 | 引擎支持的项目单元、工件摘要和类型/方法的伪代码 | 由调用方启动、提供 MCP 的 JEB 客户端，通过 REA_JEB_MCP_URL 选择；[JEB 指南](../jeb-analysis.md) |
docs/rust-binaries.md
+61
View File
@@ -21,6 +21,67 @@ requires an absolute filesystem path and preserves that exact selection:
## Observations
Complete delivery remains the default. For a dependency question, select the
needed records on the first call instead of returning the complete graph:
```json
{
"name": "inspect_rust_binary",
"arguments": {
"path": "/artifacts/application",
"view": {
"collection": "packages",
"names": ["serde", "serde_derive"],
"fields": ["name", "version", "source", "dependencies"]
}
}
}
```
`detail: "summary"` returns section ranges, root index, package/edge/cycle counts
and compiler-hash count. A `view` selects compact delivery automatically;
combining it with explicit `detail: "complete"` is an input error. The complete
parent Evidence is retained in the MCP session and remains exportable. Subsequent
`inspect_analysis_view` calls accept that `parent_evidence_id`, or its portable
Evidence file, without reopening the binary:
```json
{
"source": {
"kind": "retained-evidence",
"evidence_id": "ev_<parent digest>"
},
"view": {
"kind": "rust",
"collection": "edges",
"from_indices": [12],
"limit": 20
}
}
```
Package filters accept exact `names`, `versions` and original `indices`; edge
filters accept original edge `indices`, `from_indices` and `to_indices`.
Filters intersect, and an empty filter array selects nothing. `offset` counts
matching records; `limit` defaults to 64. Pages retain original indices scoped
to the parent Evidence, selected fields, examined coverage and the next offset.
Default package fields omit dependency vectors; explicitly select
`dependencies` to obtain direct references. No dependency path is traversed.
The CLI uses the same views and requires an exclusive output bundle for compact
delivery so the complete Evidence survives process exit:
```bash
rea inspect-rust-binary /artifacts/application --json \
--view '{"collection":"packages","names":["serde"],"fields":["name","version"]}' \
--evidence_output /artifacts/rust-evidence.json
```
Compact results distinguish complete embedded-record coverage from unknown
runtime dependency coverage. A missing section remains unknown rather than an
empty dependency set. Retained views describe the captured artifact digest;
they do not assert that the current file or running application is unchanged.
The inline Evidence result contains the artifact path, SHA-256, byte length,
container format, architecture, word size, and byte order. When a `.dep-v0`
section is present, `auditable_metadata` contains:
docs/specs/agent-environments/prefix-variance/plan.md
+18
View File
@@ -0,0 +1,18 @@
# Plan
1. Move tool naming (`parameterNames`, `pathTemplate`, `toolName`,
`compareTools`) from `environmentManifestRules.ts` to a new
`environmentManifestNames.ts`, so the rules module has room for the rule.
2. Add `prefix-variance` to `TEMPLATE_RULES`. A `prefix-variance` parameter
records the rule it shares (`sharedRule`), which names it and keys it.
3. Add the pass after `sibling-variance`: walk positions in order, index the
opt-in parameters at each position by their preceding parts, template
qualifying literals, then rebucket by naming key, keeping a tool's own rule
for each merged position.
4. Share the 32-segment split between the three comparing passes and list the
segment-limited rules once for the limitation.
5. Document the rule in the view schema description and in
`docs/web-network-captures.md`.
6. Tests: shared positions across methods and depths, the spelling condition,
merges that keep a tool's own rule, pairs, format-rule positions and the
segment limit.
docs/specs/agent-environments/prefix-variance/spec.md
+59
View File
@@ -0,0 +1,59 @@
# Share templated positions across tools
Issue: https://github.com/morluto/rea/issues/4933
## Outcome
A session usually edits one scratch entity: it creates it, then updates and
deletes it at a fixed path such as `/api/v1/collections/0RZ576QZQA5KG`. The
manifest templates a segment only when sibling routes of the same method vary
there, so those writes stay literal tools (`patch_collections_0rz576qzqa5kg`)
even though the reads of the same collection were templated as
`/api/v1/collections/{collection_id}`. A literal write tool addresses one
entity that may not exist in a later state, and nothing can model its effect.
A new opt-in template rule, `prefix-variance`, templates such a literal with
the parameter another tool already uses at that position, so the write becomes
`patch_collections_by_collection_id` on the same template as the read.
## Requirements
- `prefix-variance` applies only when listed, after every other opt-in rule, so
it can share the parameters they produce.
- A literal at position `p` is templated when another tool on the same origin,
of any method and any segment count, has a parameter at `p` from one of the
other opt-in rules, after the same preceding parts. Parameters in the
preceding parts compare by the rule that names them, so a position templated
by `prefix-variance` matches the parameter it shares.
- The literal qualifies when it is one of the values such a parameter
observed at that position, when it carries a digit and has the length and
character classes (digits, lower case, upper case, each other character) of
one of them, or when every value the parameter observed, one or more, has one
length of eight or more characters, as generated identifiers often do, and
the literal has that length and uses only their character classes, so an
identifier that happens to hold no digit still qualifies. An empty literal, a
stop word or a `v1`-style version never qualifies. Each parameter's spelling
is summarised once, so a literal is checked without rescanning its values,
and a value longer than 128 characters is compared only exactly, so a huge
segment costs no classification.
- Positions templated by the four format rules are never shared, so
`/api/teams/{numeric}` still absorbs no literal.
- Values stay with the parameter that observed them: when parameters of
different rules share a position, the literal shares the one that observed
it, else the earliest one it is spelled like.
- The parameter takes the name of the parameter it shares (`collection_id`,
`uid`, a pair's `_namespace` and `_name`) and is labelled `prefix-variance`.
- Tools whose routes now match, by method and by the rule that names each
parameter, merge. A merged parameter keeps the rule of a tool that templated
that position itself, when one did.
- Like the sibling rules, it compares routes of at most 32 segments and says so
in a limitation when a longer route exists.
## Non-goals
- No change to the default rules or to any other rule: manifests that do not
list `prefix-variance` are byte-for-byte unchanged.
- No templating from response bodies, such as an id a create returned; the
manifest reads request lines only.
- No change to which groups are excluded: a write whose response has no body
or content type outside `/api/` is still excluded as `non-json`.
docs/specs/agent-environments/prefix-variance/tasks.md
+10
View File
@@ -0,0 +1,10 @@
# Tasks
- [x] Record the outcome, requirements and non-goals.
- [x] Move tool naming into its own module.
- [x] Add the `prefix-variance` rule, its names, its merge and its limitation.
- [x] Document the rule in the schema and the capture guide.
- [x] Add rule tests.
- [x] Run static, formatting, duplication, documentation and test gates.
- [x] Project the 51 corpus captures with and without the rule and record the results.
- [x] Rebuild five randomly drawn environments end to end and compare them with their published versions and the live applications.
docs/specs/agent-environments/prefix-variance/verification.md
+149
View File
@@ -0,0 +1,149 @@
# Verification
Verified base: 33b0643e on main 1629266e (the rule, the alias removal `knip`
asked for, and the fixed-length clause the end-to-end runs below led to). Executed on macOS with Node v22.23.1 (the repository pins
24.18.0 in `.nvmrc`, which was not installed).
## Gates
- `npm run build:cached`, `npm run check:fast` (typecheck and lint),
`npm run format:check`, `npx knip`, `npm run scan:todos`,
`npm run docs:check`, `npm run verify:test-discovery` and
`npm run mcp-catalog:check`: passed.
- `npm run jscpd`: 3.49% duplicated lines over 3561 files with 3067 clones;
main measures 3.49% with 3068, since the shared segment-limit split removes
one clone.
- oxlint reports no new finding on the changed files. The manifest accumulator
keeps main's three warnings (file length, and the complexity and parameter
count of its record append), one line longer.
- `npx vitest run --project domain --project services`: 7119 passed, 5
pre-existing skips.
- `npm run test:prepare`, then `npm run test:focused --` the analysis-view MCP
and CLI boundaries, the compact input schema and tool schema validity
boundaries, and the environment serve and export boundaries:
7 files, 76 passed.
New tests in `environmentManifestView.test.ts` cover:
- a literal shared across methods and depths, and the same capture without the rule;
- the spelling condition: same classes and length, case, missing digit, a colon word, a version;
- a digit-free member of a fixed-length identifier family, against a shorter word, a hyphenated
value and a family shorter than eight characters;
- merges that keep a tool's own rule, including a sibling-variance parameter under a shared one;
- both halves of a templated pair;
- format-rule positions and the segment limit.
## The showcase captures
The 50 showcase captures (45 HAR inspections and 5 `capture_browser_scenario`
captures) were projected with main's build and this branch's.
- Without the rule, both builds give identical results for every capture: the
manifest with no rules, the manifest with `after-uid-literal`,
`opaque-token`, `sibling-variance` and `sibling-pair-variance`, and the
environment-state view. 150 of 150 outputs hash equal.
- With `prefix-variance` added to `after-uid-literal`, `opaque-token` and
`sibling-variance`, 69 literal tools in 10 captures are templated, 25 of them
writes:
- Komga: collection and readlist `PATCH` and `DELETE`, book pages.
- n8n: workflow `PATCH`, `DELETE` and archive, and the reads of a workflow
whose id holds no digit.
- Memos: memo `PATCH` and `DELETE`, user settings reads, and the reads of
two memos whose ids hold no digit.
- Navidrome: playlist `PUT` and `DELETE`.
- Mealie: recipe `PATCH` and `DELETE`.
- Grist: document `PATCH` and `DELETE`, and table record writes.
- Redpanda Console: subject and topic writes.
- Sentry: release `PUT` and `DELETE`.
- Shlink: short URL `PATCH`.
- Mattermost: `me` reads, which `sibling-variance` already folds into the
user parameter.
Tools go from 1591 to 1578, because Redpanda Console's per-subject reads and
the per-memo and per-workflow reads merge.
- Literals the rule leaves alone, as intended, include Espo CRM `layout`,
Federal Register `facets`, Ghost `slug`, Shlink `stats`, Lychee
`Metrics::photo` and Vault's path names.
- The environment-state view succeeds on every capture with the rule. Its types
move to the templated collections (531 become 526 as literal ones merge); the
one fewer item view is the Redpanda Console merge.
With the write plans of #5600 (its branch with this commit and main merged in,
locally):
- Served write tools rise from 33 to 39: Komga collection create, update and
delete; n8n workflow delete; Sentry release update and delete.
- Write tools with no state target fall from 102 to 87.
- The newly modelled tools that stay unserved are limited by the capture:
- no read after the write: Komga readlists, Mealie, Navidrome, Redpanda Console;
- n8n's item read shows an expanded `shared` relation its create response
never returned;
- Shlink returns `tags` sorted while the update echoes the request's order.
## Five environments end to end
Five of the ten environments the rule changes were drawn at random, redrawing
until the draw held one whose new writes are served: Komga, Mealie, n8n,
Sentry and Shlink. Each was rebuilt on its own Linux VM with `npm pack` of
#5600, main and this branch merged locally, from the same scoped HAR as its
published version: manifest with `prefix-variance`, exported bundle, smoke
task, the generated retrieval tasks for `gpt-oss-120b` and `GLM-5.3-Flash`, and
a dry-run dataset. Where writes are served, hand-written write tasks ask for a
change in plain language and are graded from the environment's state through
`data/get`. The first n8n run found that a digit-free workflow id stayed
literal; the fixed-length clause fixed it, and n8n was rebuilt.
| Environment | Tools (published → rebuilt) | Served from state (reads / writes) | Retrieval gpt-oss | Retrieval GLM | Writes gpt-oss | Writes GLM |
| ----------- | --------------------------- | ---------------------------------- | ----------------- | ------------- | -------------- | ---------- |
| Komga | 42 → 46 | 26 / 3 | 12/12 (was 11/12) | 12/12 (11/12) | 3/4 | 4/4 |
| Mealie | 39 → 39 | 36 / 0 | 12/12 (12/12) | 12/12 (12/12) | – | – |
| n8n | 30 → 33 | 17 / 1 | 9/9 (9/9) | 9/9 (9/9) | 2/2 | 2/2 |
| Sentry | 34 → 34 | 26 / 3 | 12/12 (12/12) | 11/12 (12/12) | 2/4, rerun 0/4 | 4/4 |
| Shlink | 17 → 17 | 5 / 0 | 5/7, rerun 7/7 | 7/7 (7/7) | – | – |
Every smoke task passed. The misses:
- Sentry writes with gpt-oss: every failed episode passed `id: "sentry"` for
the `/api/0/` segment the tool schema describes as `observed: 0`, so its calls
addressed `/api/sentry/…`; every passing episode, and every GLM episode, passed
`0`. The served tools work; the format-rule parameter for a segment that only
ever held `0` invites the guess.
- Komga update with gpt-oss and Shlink's first gpt-oss run: the model provider
rejected tool calls the model generated (`null` and `undefined` for optional
query arguments) before the environment saw them.
- Sentry `qa-03` with GLM: the model never called the alert-rules tool.
The same calls were also replayed against the live applications, seeded by
the same scripts (ids differ, so each side maps its ids to names):
- Komga 1.29.1, 9 of 12 steps equal: create, update and delete of a
collection and the reads after them agree except for the generated id. The
collection list after a create omits the new collection, because that list
keeps its recorded membership, and the 404 for a deleted collection replays
the recorded body with its path and timestamp.
- n8n 1.123.85, 3 of 8 steps equal: n8n refuses to delete a workflow that is
not archived (400) and the environment deletes it; the workflow list replays
its recorded response, so a delete does not show in it; archiving is not
served. The read of the digit-free workflow failed in the first build (the
state held only its list fields) and agrees in the second, apart from ids the
seed generates.
## After the review round
The branch merged main at 52c84f6f, then kept each shared position's values
with the parameter that observed them (one donor per naming rule, the literal
taking the donor that observed it, else the earliest it is spelled like),
summarised each donor's spelling once, and let one observation form a
fixed-length family, as the requirement states. Two new tests, a mixed-rule
position and a single observed workflow id, fail on the previous head and pass
here. The gates above passed again (domain and services 7255 passed with 5
pre-existing skips; the focused boundaries 76; `jscpd` 3.49%). Over the 50
showcase captures the tools with and without `prefix-variance` are unchanged,
name for name: 69 literal tools templated, 25 of them writes.
A second review round bounded the spelling summary: characters are classified
in one pass, and a value longer than 128 characters is compared only exactly.
Its test fails on the previous head, the gates passed again (domain and
services 7257, focused boundaries 76, `jscpd` 3.49%), and the census over the 50
captures is again unchanged.
docs/sqlite-databases.md
+23 -2
View File
@@ -102,8 +102,29 @@ bounds simultaneous input buffering and private snapshot/worker counts.
The database and WAL together are limited to 256 MiB. SQLite values/records,
SQL statements and the complete worker JSON reply each have a 16 MiB limit.
The owned worker has a 256 MiB JavaScript heap limit and a 30-second inspection
deadline. SQLite native allocations are separate from the JavaScript heap;
The owned worker has a 256 MiB JavaScript heap limit and defaults to a 30,000 ms
deadline. Select optional MCP `worker_timeout_ms`, or CLI
`--worker-timeout-ms <milliseconds>`, to change the budget for that call. Values
must be integer milliseconds from 1 to 2,147,483,647 (the runtime timer range).
It covers worker startup, queries and reply writing, **not** DB/WAL snapshot
acquisition, final reply reads, cleanup or delivery. Allow corresponding headroom
in the MCP client's whole-call deadline; increasing only that deadline does not
change the worker budget. No server restart is needed.
The result's `worker_timeout_ms` reports the actual parent-supervised budget,
including the default. Historical Evidence without this field has unknown
execution settings, not an inferred default. Explicit input selection is also
retained in Evidence parameters; omission stays omitted. Completed observations
retained under cancellation or cleanup failure keep the same budget facts.
Timeout errors name the selected milliseconds, exact CLI/MCP controls, and
cleanup-before-retry requirements. Incomplete cleanup remains a blocker even
when a larger budget is available.
For example, `rea inspect-sqlite-database ./application.sqlite
--worker-timeout-ms 60000 --json` selects 60 seconds; equivalent MCP input is
`{"path":"/artifacts/application.sqlite","worker_timeout_ms":60000}`.
SQLite native allocations are separate from the JavaScript heap;
overall native heap and total resident memory are not capped. Row limits do not
override these byte and resource limits. Resource-limit failures do not return
an incomplete schema as successful complete evidence.
docs/testing.md
+63 -10
View File
@@ -156,7 +156,27 @@ rejection without a reason. That cleanup stays owned until a later close or
call releases it. Another server does not open while the cleanup remains, and
a cleanup that already succeeded is not run again. Pass
`mcpCall` when a case needs its own server, environment, or lifecycle. CLI
cases still start a fresh process.
cases for that helper share one compiled CLI process per child environment.
The first case pays process startup; later cases reuse the loaded runtime.
The process restarts when the environment changes, after 32 commands, or when
the worker reports a resident set of at least 1 GiB. A fatal worker failure
or a broken session frame retires that process and waits for it to exit before
the next command starts.
`cli.run` still starts a fresh process, including stdin and explicit deadlines.
`runFileCli` uses the file session for commands that have neither. Go binary,
IL2CPP metadata, and dylib-resolution commands use that session. File teardown
stops the shared CLI process, including startup still in flight, and does not
open another after close.
`exportedFunctionBindingsCli.test.ts` keeps its own compiled CLI parity
process for the file and still compares that process's JSON command results
with the MCP session. A command that does not respond is rejected. That
process stays owned until it exits, and another process does not start while
it remains. A process that stays running after SIGKILL remains owned so a
later close can retry. Closing the session reports a failure when the process
does not exit after stdin closes, once the child has been stopped.
`applicationWorkflowCli.test.ts` keeps one compiled CLI process for its JSON
commands, including a non-zero status whose stdout is still a JSON document,
and closes that process after the file.
The Apple artifact verifier step shares a job-temporary bytecode cache among
its fresh CLI/MCP children via `REA_ARTIFACT_NODE_COMPILE_CACHE`. The artifact
@@ -272,7 +292,23 @@ termination. Simulated Windows controls do not establish native Windows behavior
Native process-ownership controls distinguish start identity, zombie lifecycle,
current parent/group relation, and token-read evidence. Retain generation-bound
candidates through smaller lineage observations, table omissions, reparenting,
group changes, and failed cleanup retries. A readable token with unavailable
group changes, and failed cleanup retries. Identityless exit-refresh controls
cover a replacement row that is not linked to the prior descendant set: a
matching token on its own group is signaled, a foreign or absent token is not,
an unreadable live token stays unresolved, a replacement that appears only in
the confirmation table is still token-checked, a group discovered during
revalidation is signaled before cleanup succeeds, a member that changes groups
during confirmation is signaled in its new group, a PID first seen with a
foreign token is read again on the confirmation table, a PID whose token was
already read is read again even when its parent and group return to the
pre-exit values, a pre-existing PID is read again when its parent or group
changes, an unchanged pre-existing row that was never token-read stays unread,
an unbounded confirmation
chain stays unresolved, lineage reads a row again when its launcher-tree
membership changes, a cached lineage token is read again on the next refresh, an initial descendant whose token already matched is read again on a later refresh, the launcher is read again on that refresh and a missing, unowned, or relation-changed launcher PID stays unavailable, a descendant absent from the latest refresh is omitted,
a group that reappears after it was processed is revalidated once, a further
reappearance stays unresolved, lineage publishes one row per PID, and lineage does not publish a
verified empty descendant list while that row remains unexamined. A readable token with unavailable
identity stays unresolved. Native UID facts, rather than an earlier process-table
row, determine whether a token denial concerns another user. Darwin tests compile
the helper and exercise snapshot growth, absence, lifecycle, and UID observations;
@@ -485,9 +521,15 @@ reports even when a shard fails, uploads available diagnostics, and writes the
slowest files to the workflow summary. Partial reports are marked diagnostic
only. Failed, cancelled, or missing selected shards still fail the required gate,
and aggregate coverage thresholds remain enforced in full qualification.
Only successful complete full-tier merges publish the small `test-timings` artifact, with
checkout-relative paths; the build job downloads that artifact for the sequencer
instead of the coverage/JUnit bundle. Timing weights come from Vitest module
Only a successful complete merge on the default branch publishes shard history,
and only from a push, schedule, or manual run. Fast runs publish
`test-timings-fast`. Full runs publish `test-timings-full`. Pull requests do not
publish it: the lookup reads one unpaginated artifact page, and pull-request
artifacts would crowd out the default branch. The build job downloads the
artifact that matches its tier, so fast shards use durations measured without
coverage and full shards keep coverage-weighted durations. A tier with no
qualifying artifact leaves history unset. The download is that small
checkout-relative timing file, not the coverage/JUnit bundle. Timing weights come from Vitest module
diagnostics and include tests and hooks, worker preparation, environment,
imports, and setup. Import timings already include transform waits, which are
not added again. The weights describe file costs under the measured worker and
@@ -1354,11 +1396,22 @@ catalog, and managed-conformance metadata. Each shard still installs the locked
dependencies and runs its assigned tests; full qualification enables coverage. It no longer
regenerates the same metadata independently.
The build job also finds an available `test-timings` artifact from a successful
trusted default-branch scheduled or manual qualification and includes that same
report in each shard's snapshot. The bounded lookup verifies the artifact's
producer; newer schedules that reuse qualification without producing timings do
not hide the last full run's artifact.
The build job finds the newest trusted timing artifact for its tier
(`test-timings-fast` or `test-timings-full`) from a successful default-branch
push, schedule, or manual CI run, and includes that same report in each shard's
snapshot. The bounded lookup verifies the producer. Pull-request runs and other
branches are not producers. A tier with no qualifying artifact leaves history
unset, and Vitest uses its own assignment. A later schedule that reuses
qualification without uploading timings leaves the previous artifact in place.
On 2026-10-21 the newest full history was run
`39545047294`, recorded before compiled-CLI session reuse. Fast run
`39552964873` packed its eight built shards from that history, and those shard
jobs executed for 319–591 seconds. Replaying that run's file durations through
the sequencer's two-worker schedule balances every shard at about 404 seconds
of file time. The same replay with the stale history weights leaves the slowest
shard near 547 seconds and the fastest near 293. That spread is file-schedule
time, not runner queue, which is why each tier keeps its own history.
CI assigns discovered files by historical duration, with a deterministic
fallback weight for new files, rather than dividing only by file count.
Each file keeps its full duration on one worker. Extra workers overlap other
docs/wasm-artifacts.md
+32 -1
View File
@@ -26,6 +26,17 @@ execution validates tool files and version banners. Host process-ownership and
private-workspace controls must also be available. The real-tool profile has
been exercised on macOS arm64; other hosts require their own verification.
Configuration failures do not establish anything about the selected WASM bytes.
For a rejected version probe, `details.diagnostics` identifies the executable,
tool, expected version and configuration key; `details.captured_output` preserves
its actual stdout and stderr, including empty banners or warnings. Fix the
reported directory, executable access or release rather than changing the module
or repeating an unchanged call. All three tools must report 1.0.42 on stdout with
empty stderr. For CLI calls, set the environment before the next invocation. For
MCP, change the server's launch environment and restart/reconnect that server;
exporting a variable in a different shell does not update a running server.
REA never installs tools or relaxes version admission to recover.
```bash
REA_WABT_BIN_DIRECTORY=/absolute/wabt/bin rea inspect-wasm-artifact \
/artifacts/module.wasm --glue_paths /artifacts/glue.js \
@@ -64,7 +75,27 @@ The tool profile preserves upstream release/source identity, resolved executable
paths, executable digests, exact banners and arguments. Executable digests are
checked again after each command. Caller-supplied tools are trusted programs;
version and digest recording are not publisher authentication or an OS sandbox.
Each subprocess has a 30-second deadline and 32 MiB combined output budget.
Each WABT version/analysis command defaults to a 30,000 ms deadline. Set optional
MCP `command_timeout_ms`, or CLI `--command-timeout-ms <milliseconds>`, to select a
positive integer up to 2,147,483,647 ms (the runtime timer range) for that call.
The selected deadline is retained as `tool_profile.timeout_ms` and, when supplied,
in Evidence parameters. It applies separately to each command, not the whole
inspection, file admission or cleanup. The MCP client must allow time for all
commands, file checks, cleanup and delivery. A provider timeout reports the
actual selected deadline and these recovery controls; increasing only the client
deadline does not change the provider budget. No server restart is required to
change this per-call setting. Resolve incomplete cleanup before another attempt.
For example, `rea inspect-wasm-artifact /artifacts/module.wasm
--command-timeout-ms 60000 --format json` uses a 60-second per-command deadline;
the equivalent MCP input is `{"path":"/artifacts/module.wasm",
"command_timeout_ms":60000,"detail":"summary"}`. Summary delivery retains the
complete Evidence but does not reduce WABT analysis work.
The 8 MiB input allowance and 32 MiB combined command-output budget remain fixed;
increasing `command_timeout_ms`, host memory or the client deadline does not raise
them. Do not truncate required module bytes to fit. Select another suitable
complete artifact only if it answers the question, otherwise report the limit.
Inspection output past the budget fails with `resource_constraint`; an oversized
`--version` probe still reports `unreadable_output`. Both streams share the
budget, so a stream that starts after the other has filled it retains nothing.
docs/web-network-captures.md
+18 -7
View File
@@ -449,12 +449,13 @@ template and `params` carry the four format rules with names `id`, `uuid`,
only when listed, each at most once (a repeated rule is an `invalid_request`),
in this order:
| Rule | Applies to | Parameter |
| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------- |
| `after-uid-literal` | The literal segment right after a literal `uid`, unless it is a stop word | `uid` |
| `opaque-token` | A lowercase alphanumeric segment of twelve or more characters mixing letters and digits that is not a `v1`-style version | `uid` |
| `sibling-pair-variance` | Groups equal in method, origin and segment count that differ in two adjacent literals right after a plural literal at index 2 or later and followed by a literal, with at least three distinct pairs and both positions varying (`/repos/{owner}/{repo}/issues`); stop words and `v1`-style versions never qualify | singular plural literal plus `_namespace`, then `_name` |
| `sibling-variance` | Groups equal in method, origin and segment count that differ in one literal at index 2 or later; the siblings whose literal is identifier-like (a digit or `:`, or three or more hyphenated names under a plural literal) merge, never a stop word, and the other literals stay their own tools | singular previous literal plus `_id`, else `id` |
| Rule | Applies to | Parameter |
| ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------- |
| `after-uid-literal` | The literal segment right after a literal `uid`, unless it is a stop word | `uid` |
| `opaque-token` | A lowercase alphanumeric segment of twelve or more characters mixing letters and digits that is not a `v1`-style version | `uid` |
| `sibling-pair-variance` | Groups equal in method, origin and segment count that differ in two adjacent literals right after a plural literal at index 2 or later and followed by a literal, with at least three distinct pairs and both positions varying (`/repos/{owner}/{repo}/issues`); stop words and `v1`-style versions never qualify | singular plural literal plus `_namespace`, then `_name` |
| `sibling-variance` | Groups equal in method, origin and segment count that differ in one literal at index 2 or later; the siblings whose literal is identifier-like (a digit or `:`, or three or more hyphenated names under a plural literal) merge, never a stop word, and the other literals stay their own tools | singular previous literal plus `_id`, else `id` |
| `prefix-variance` | A literal at a position that another tool, of any method or segment count, templates by one of the rules above after the same preceding segments, when the literal is one of that parameter's observed values, carries a digit and has the length and character classes (digits, lower case, upper case, each other character) of one, or has the single length of eight or more characters every observed value has and only their character classes; stop words and `v1`-style versions never qualify, and tools that now match merge | the name of the parameter it shares |
With `sibling-variance` selected, literal siblings within its comparison scope
fold into existing opt-in parameters as observed values unless they are stop words.
@@ -471,7 +472,17 @@ cut to their first label, `_by_<params>` appended when parameters exist,
sanitized to `[a-z0-9_]`, at most 60 characters and made unique with numeric
suffixes: `get_dashboards_by_uid`, `get_plugins_settings_by_plugin_id`.
`sibling-variance` and `sibling-pair-variance` compare routes of at most 32 segments; longer routes keep
`prefix-variance` runs last so it can share the parameters the other rules
produce: with `sibling-variance`, a `PATCH` or `DELETE` of the one entity a
session edited, `/api/v1/collections/0RZ576QZQA5KG`, becomes
`/api/v1/collections/{collection_id}` beside the reads that varied there, and
so do deeper routes such as `/api/v1/collections/{collection_id}/books`. Its
parameter is labelled `prefix-variance`, unless a merged tool templated the
position by its own rule. When parameters of different rules share a position,
a literal shares the one that observed it, else the earliest one it is spelled
like.
`sibling-variance`, `sibling-pair-variance` and `prefix-variance` compare routes of at most 32 segments; longer routes keep
their grouped templates and the manifest says so in a limitation. `body_shape`
is derived for `POST`, `PUT` and `PATCH` from the tool's earliest record when it
carries an `application/json` or `application/*+json` request body: top-level
docs/web-runtime.md
+3 -1
View File
@@ -32,7 +32,9 @@ A document replacement ends the window; final coverage then remains unavailable.
Same-document navigation such as `history.pushState()` remains valid when the frame and known loader identity stay stable and the current origin is allowed. Unknown loader identity retains a conservative URL check; newly reported or replaced loaders are rejected. Malformed producer events end an armed window immediately and trigger cleanup, independently of the selected duration.
Target detachment ends the window even when the browser's root connection stays open. When the closed session cannot acknowledge cleanup, the operation returns a failure with the selected target, failed commands and original producer reason; it does not claim successful cleanup.
Target detachment ends the window with `session_detached`, even when the browser's root connection stays open. An `Inspector.targetCrashed` event ends it with `target_terminated`. A lost transport records `transport_disconnected` in browser observation; execution capture returns the typed transport failure. These outcomes retain the selected target and the observed lifecycle fact.
When an ended session cannot acknowledge cleanup, the operation returns a failure with the failed commands and original typed producer reason. A coverage-stop failure also retains the observed window's end reason in its diagnostics; cleanup remains unconfirmed until the owning session can resolve it.
Child frames, workers, service workers, WebAssembly, chronological execution order and framework-specific causal interpretation are outside this operation. Script ownership comes from frame/context metadata, never URL equality. Repeated URLs and synthetic names therefore retain separate identities.
experiments/analysis-efficiency/README.md
+12
View File
@@ -1,5 +1,17 @@
# Analysis efficiency measurements
The [2026-10-21 coverage reconciliation](coverage-review-20261021.md) maps all
46 source directories to concrete workflow evidence, including original feature
PRs, historical no-benefit results and remaining verification boundaries.
The [GGUF validation checkpoint](model-utf8-validation.md) removes decoding of
omitted strings while retaining full UTF-8 validation. It records real-model
phase gains, unchanged task token costs and the higher main-task memory peak.
The [WAT framing checkpoint](wasm-string-framing.md) measures structural scanning
on real SQLite and VS Code components, with unchanged Evidence and token costs,
small whole-task timing gains and no memory improvement.
The [native inventory view checkpoint](native-inventory-views.md) compares
complete delivery with summaries and selected rows on real `true` and jq
inventories, including oversized-response recovery, portable old Evidence,
experiments/analysis-efficiency/corpus.json
+32
View File
@@ -433,6 +433,38 @@
"members": 315,
"expanded_bytes": 1301927
}
},
{
"id": "alpine-ripgrep-rust",
"file": "ripgrep",
"bytes": 3416168,
"sha256": "78387bbe12b2110df8d2ce364390a00c190f4543a2f40b8279e49f6c5c85af9c",
"provenance": {
"project": "ripgrep",
"version": "14.1.1-r0",
"upstreamFile": "ripgrep-14.1.1-r0.apk",
"url": "https://dl-cdn.alpinelinux.org/alpine/v3.21/community/x86_64/ripgrep-14.1.1-r0.apk",
"transport_sha256": "213dd0b047163f6627cc2fdd6fbfe205009476eb4fc48c352a3f5b9ab647f6d4",
"transport_bytes": 1422830,
"aports_revision": "4e899d119e4c983a938436d6f397a509521225d1",
"derivation": "selected regular APK tar member usr/bin/rg; read-only static inspection; never executed"
}
},
{
"id": "alpine-zellij-rust",
"file": "zellij",
"bytes": 17151864,
"sha256": "9a86e9ca47902a869fdf1886f4cf17714f4fa5655427128beadff498cfb392db",
"provenance": {
"project": "zellij",
"version": "0.40.0-r0",
"upstreamFile": "zellij-0.40.0-r0.apk",
"url": "https://dl-cdn.alpinelinux.org/alpine/v3.21/community/x86_64/zellij-0.40.0-r0.apk",
"transport_sha256": "575075945415b221857563e19c5f4c661af9f98bbba54f0994c7385c0e432432",
"transport_bytes": 6722129,
"aports_revision": "3be26a3a26e76e1bf3eff784bffe3956d29d77a0",
"derivation": "selected regular APK tar member usr/bin/zellij; read-only static inspection; never executed"
}
}
]
}
experiments/analysis-efficiency/coverage-review-20261021.md
+59
View File
@@ -0,0 +1,59 @@
# Workflow coverage reconciliation, 2026-10-21
The checked snapshot is main `264862bf2`, with **46 first-level `src/`
directories and 212 production stdio MCP tools**. Directory counts and tool
counts are different inventories: one application workflow can cross several
directories, and one provider can expose many independently costly operations.
This is a source/history review, not a new runtime qualification of all tools.
The search includes this experiment directory, owning contracts and source
history, original feature PRs and related open/closed issues. Missing experiment
files do not establish missing audits. In particular, the model and IL2CPP
feature PRs contain substantial real-task evidence absent from this directory.
Historical measurements retain their original revisions, platforms and limits.
| Directories / workflow | Evidence already present | Remaining coverage boundary |
| ----------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `kuna`, `ghidra` | Earlier native entry work and [Ghidra projection](ghidra-evidence-projection.md) | Selected questions only; database reuse is a separate [research issue #3136](https://github.com/morluto/rea/issues/3136). |
| `ida` | [Windows native correctness and completed-task costs](ida-native-windows.md), including real `whoami.exe` | Warm selected-task gains do not establish cold analysis, all string formats, all operations or other IDA platforms. |
| `hopper`, `jeb` | [Hopper string records](hopper-string-records.md), [Linux runtime](hopper-linux-python-runtime.md), [real JEB CE](jeb-linux-native.md) | Preserve negative whole-task results; other editions, platforms and workflows are separate scopes. |
| `rizin`, `cutter` | Existing command/session adapters and lifecycle fixes | No comparable dedicated real completed-task performance record located in the searched records. Actual engine/GUI availability must be established before selecting a task. |
| `gdb`, `objdump` | [Real Bash command observations](command-observation-views.md) | Selected read-only disassembly is measured; other commands, large outputs and running-target observation are separate tasks. |
| `adb`, `frida`, `inspector` | [Frida native status/history experiment](frida-status-views.md) | Frida status does not qualify arbitrary scripts, ADB/device transfers, browser Inspector events or live target freshness. |
| `go`, `rust` | [Go build/dependency selection on real binaries](go-build-views.md); Rust cargo-auditable support [#2456](https://github.com/morluto/rea/pull/2456) | Go is already measured. No comparable dedicated Rust dependency-investigation cost record located in this review. |
| `javascript`, `browser`, `electron` | [Agent fact workflow](javascript-facts-agent-history.json), lifecycle experiments and [#4415](https://github.com/morluto/rea/issues/4415) | Real Joplin preparation remains exploratory. Individual JS features and browser/runtime workflows need their own evidence. |
| `android`, `apktool` | [Package summary](android-package-summary.md), [three-APK shared resource decoding](apktool-string-tables.md) | These do not qualify all JADX class/method/reference operations or physical-device workflows. #4777 is already fixed. |
| `dotnet`, `il2cpp` | [Managed summaries/pages](managed-artifact-summary.md), IL2CPP metadata [#3807](https://github.com/morluto/rea/pull/3807) and native links/seeds [#5327](https://github.com/morluto/rea/pull/5327) | IL2CPP already has real Windows/Android row comparisons, task costs and Ghidra import. Unsupported versions/generics/platforms remain separate correctness and coverage work. |
| `model` | GGUF/safetensors [#5603](https://github.com/morluto/rea/pull/5603): real models, independent readers and complete-task costs | Existing delivery is already audited. [#5774](https://github.com/morluto/rea/issues/5774) isolates omitted-string validation cost within that producer. |
| `native`, `artifacts` | [Native inventory](native-inventory-views.md), [entry views](native-entry-views.md); real NIB output reductions [#1460](https://github.com/morluto/rea/pull/1460), [#1477](https://github.com/morluto/rea/pull/1477) | Inventory does not qualify Apple UI/resource, signatures, dyld or runtime traces. NIB byte reductions alone do not establish token, time, CPU or memory savings for a completed question. |
| `firmware` | Region pilot plus real full OpenWrt extraction/publication [#4602](https://github.com/morluto/rea/pull/4602) | Extraction is already measured; noisy timings establish no general speedup. Other image layouts, extraction trees and publication filesystems remain unqualified. |
| `flutter`, `sqlite`, `wasm`, `evm`, `pcap`, `signals` | [Dart AOT](flutter-aot-summary.md), [SQLite controls](sqlite-existing-controls.md), [WAT framing](wasm-string-framing.md), [EVM](evm-interface-summary.md), [PCAP](pcap-small-captures.md), [signals](signal-selection-workflows.md) | Extend a specific question. Preserve cases where existing controls suffice and small tasks regress; these records do not qualify each format or tool exhaustively. |
| `application`, `domain`, `server`, `cli`, `contracts` | [Mixed retained investigations](retained-investigation-workflows.md), [immutable parent validation](immutable-parent-validation.md), [capture validation](retained-capture-validation.md) | Shared selected-query paths are already optimized. Reconstruction/environment composition and new fact families require their own end-to-end questions. |
| `browser` / recorded captures | [SAZ selection](saz-selection-workflows.md) and [real qlog producer/retained reads](retained-capture-validation.md) | [#3133](https://github.com/morluto/rea/issues/3133) still includes actual model/task questions; producer measurements do not close those requirements. |
| `filesystem`, `process`, `reference`, `providers`, `captures`, `windows`, `main`, `composition`, `config`, `evaluation` | Cross-workflow resource/cleanup checks, [process traces](process-trace-validation.md), [MCP profile](mcp-profile-identity.md) and Windows lifecycle checkpoints | Infrastructure coverage follows the exercised workflow. No blanket startup/resource claim; reference imports and `compare_source_to_bundle` need a dedicated completed source-tree question. |
Every one of the 46 directories appears above; repeated `browser` highlights a
distinct workflow, not another directory. The table records located evidence,
not proof that every historical PR was inspected. Open issue state by itself
does not prove missing implementation or absent measurements.
## Three bounded follow-up questions
1. **Rust dependency identity:** inspect a public cargo-auditable binary, find
exact dependency/version edges and retain complete source Evidence. Compare
existing delivery first; independently decode its auditable section. Include
a small binary and account for all discovery, queries and serialization.
2. **Apple UI graph lookup:** on macOS, locate a specific control/outlet in a
real Transmission/XLD NIB graph using existing summary/document pages.
Compare equivalent complete questions and independently check archive
records. Historical byte reductions alone are not the completion criterion.
3. **Reference/source comparison:** import a pinned public source tree and
answer one source-to-bundle identity question through existing workflows.
Independently check file manifests, symlink/unknown semantics and selected
matches; measure repeated traversal and retained representation before
proposing an index or cache.
Each lane stops with a measured benefit, an evidenced no-change decision, or an
explicit unavailable prerequisite. Rizin/Cutter and device/live workflows stay
visible as additional gaps; no new execution, injection or mutation authority is
implied by this review. No all-provider run or new API is justified by counts.
experiments/analysis-efficiency/model-utf8-validation.json
+435
View File
@@ -0,0 +1,435 @@
{
"baseline_source": "264862bf2e3612c9cb6fcfb861843cbba4e1893f",
"candidate_code": "c2550343173c5abd8f34e925eac1bcc12b7ee6a2",
"toolchain": {
"node": "24.18.0",
"npm": "11.16.0"
},
"runtime_filetables_sha256": {
"baseline": "81c293ff8997017cfe9db2a65a8b70b188e3e6d20cef07902599c69ade5d9fac",
"candidate": "b4b654a7151f6affbb3ab8541bff74b643d9f78073d338b8feb2b095dd12b6b5"
},
"lock_sha256": "699a14749e3825c62430faeb18224190fee1c9347760494e38e334e180bea6ec",
"gguf_source_sha256": {
"baseline": "9c387d5fba10db7830e74ec74a73f4b3d2e1347cca665e948b7891881b186007",
"candidate": "c43e5f8eebe04c0f82e0a970606f20ddf409c7a5fd0e298c88b09bcd0e7864a9"
},
"driver_sha256": "9732f45b8b550eef47c1fc993b435129108934f3ec3501a34df1784e6ee37b44",
"resource_guard_sha256": {
"resource-run.py": "7a9642ab86ba34290564f636eb080cacf38b4185a254028680b238bff172993b",
"resource_payload.py": "6f089e645459eba80c5d73e39c21606c78173a44daf49fabc3b88c4631ce51c6"
},
"oracle": {
"gguf": "0.19.0",
"safetensors": "0.8.0",
"receipts": [
{
"file": "stories15M-q4_0.gguf",
"sha256": "66967fbece6dbe97886593fdbb73589584927e29119ec31f08090732d1861739",
"metadata": 20,
"tensors": 57,
"status": "matched"
},
{
"file": "stories260K.gguf",
"sha256": "270cba1bd5109f42d03350f60406024560464db173c0e387d91f0426d3bd256d",
"metadata": 19,
"tensors": 48,
"status": "matched"
},
{
"file": "tiny-bert.safetensors",
"sha256": "965f02b6a7e5520fc12f710e4e3b6132f697f1c8f648819553c5ade86752d2de",
"metadata": 2,
"tensors": 100,
"status": "matched"
}
]
},
"tokenizer": "tiktoken 0.12.0 cl100k_base; independent compact JSON chunks, catalog once + each named request/reply; not actual model usage or initialization/framing",
"cases": [
{
"artifact": {
"repo": "ggml-org/models",
"revision": "499bc8821c6b12b4e53c5bffcb21ec206f212d81",
"member": "tinyllamas/stories15M-q4_0.gguf",
"file": "stories15M-q4_0.gguf",
"url": "https://huggingface.co/ggml-org/models/resolve/499bc8821c6b12b4e53c5bffcb21ec206f212d81/tinyllamas/stories15M-q4_0.gguf",
"bytes": 19077344
},
"normalized_sha256": "a894614da7ce9486b07eb2ede05ce11307d4726089722278bc4c95043feb6b66",
"answers_sha256": "64277f01f43440bc99a94ab0787a4ddde22866ed20140d87d3942b1723cbb067",
"catalog_sha256": "859b1433ef7c45ea23ba4073715d25ed7a76fd9df6d74285d20a5ea39e8e2898",
"mcp": [
{
"pair": 1,
"side": "baseline",
"wall_seconds": 6.667,
"cpu_seconds": 6.480601,
"peak_family_mib": 769.34,
"catalog_plus_calls_tokens": 895583,
"calls_ms": [367.272, 1936.653, 162.868, 12.072, 19.236]
},
{
"pair": 1,
"side": "candidate",
"wall_seconds": 6.867,
"cpu_seconds": 6.690645,
"peak_family_mib": 816.34,
"catalog_plus_calls_tokens": 895583,
"calls_ms": [324.806, 1822.885, 153.167, 11.988, 18.093]
},
{
"pair": 2,
"side": "baseline",
"wall_seconds": 6.77,
"cpu_seconds": 6.555891,
"peak_family_mib": 798.98,
"catalog_plus_calls_tokens": 895583,
"calls_ms": [456.467, 1823.786, 160.038, 11.296, 18.607]
},
{
"pair": 2,
"side": "candidate",
"wall_seconds": 6.565,
"cpu_seconds": 6.3467,
"peak_family_mib": 813.566,
"catalog_plus_calls_tokens": 895583,
"calls_ms": [329.862, 1767.086, 152.299, 10.948, 17.523]
},
{
"pair": 3,
"side": "baseline",
"wall_seconds": 6.666,
"cpu_seconds": 6.49779,
"peak_family_mib": 771.02,
"catalog_plus_calls_tokens": 895583,
"calls_ms": [345.895, 1854.963, 156.895, 11.476, 15.852]
},
{
"pair": 3,
"side": "candidate",
"wall_seconds": 6.567,
"cpu_seconds": 6.386103,
"peak_family_mib": 818.891,
"catalog_plus_calls_tokens": 895583,
"calls_ms": [342.734, 1827.835, 157.363, 10.122, 17.6]
}
],
"phase": [
{
"pair": 1,
"side": "baseline",
"first_ms": 154.027,
"repeat_median_ms": 56.14,
"wall_seconds": 1.028,
"cpu_seconds": 0.87953,
"peak_family_mib": 75.867
},
{
"pair": 1,
"side": "candidate",
"first_ms": 121.41,
"repeat_median_ms": 35.429,
"wall_seconds": 0.925,
"cpu_seconds": 0.717987,
"peak_family_mib": 77.945
},
{
"pair": 2,
"side": "baseline",
"first_ms": 159.766,
"repeat_median_ms": 53.575,
"wall_seconds": 1.027,
"cpu_seconds": 0.854739,
"peak_family_mib": 75.586
},
{
"pair": 2,
"side": "candidate",
"first_ms": 121.383,
"repeat_median_ms": 38.773,
"wall_seconds": 0.925,
"cpu_seconds": 0.733397,
"peak_family_mib": 75.125
},
{
"pair": 3,
"side": "baseline",
"first_ms": 148.447,
"repeat_median_ms": 53.58,
"wall_seconds": 1.026,
"cpu_seconds": 0.848173,
"peak_family_mib": 75.363
},
{
"pair": 3,
"side": "candidate",
"first_ms": 121.712,
"repeat_median_ms": 37.191,
"wall_seconds": 0.926,
"cpu_seconds": 0.728712,
"peak_family_mib": 77.887
}
]
},
{
"artifact": {
"repo": "ggml-org/models",
"revision": "499bc8821c6b12b4e53c5bffcb21ec206f212d81",
"member": "tinyllamas/stories260K.gguf",
"file": "stories260K.gguf",
"url": "https://huggingface.co/ggml-org/models/resolve/499bc8821c6b12b4e53c5bffcb21ec206f212d81/tinyllamas/stories260K.gguf",
"bytes": 1185376
},
"normalized_sha256": "15f4dbdc30dd9bd936747057031c6391c38b4c1717d56ae33ed2f0cba4184405",
"answers_sha256": "99257428a01aa513b435c95ca7217c69f4641d9021372a6333e7cb9e2cb35d63",
"catalog_sha256": "859b1433ef7c45ea23ba4073715d25ed7a76fd9df6d74285d20a5ea39e8e2898",
"mcp": [
{
"pair": 1,
"side": "baseline",
"wall_seconds": 6.568,
"cpu_seconds": 6.375324,
"peak_family_mib": 743.887,
"catalog_plus_calls_tokens": 895421,
"calls_ms": [223.11, 1938.73, 102.018, 10.759, 16.11]
},
{
"pair": 1,
"side": "candidate",
"wall_seconds": 6.566,
"cpu_seconds": 6.338327,
"peak_family_mib": 744.852,
"catalog_plus_calls_tokens": 895421,
"calls_ms": [248.979, 1886.809, 105.801, 10.738, 15.653]
},
{
"pair": 2,
"side": "baseline",
"wall_seconds": 6.365,
"cpu_seconds": 6.206216,
"peak_family_mib": 783.84,
"catalog_plus_calls_tokens": 895421,
"calls_ms": [248.196, 1845.737, 112.874, 10.724, 17.594]
},
{
"pair": 2,
"side": "candidate",
"wall_seconds": 7.383,
"cpu_seconds": 7.211823,
"peak_family_mib": 742.281,
"catalog_plus_calls_tokens": 895421,
"calls_ms": [233.155, 1909.721, 104.803, 11.603, 19.357]
},
{
"pair": 3,
"side": "baseline",
"wall_seconds": 6.565,
"cpu_seconds": 6.323702,
"peak_family_mib": 740.84,
"catalog_plus_calls_tokens": 895421,
"calls_ms": [233.664, 1918.301, 104.389, 14.561, 15.227]
},
{
"pair": 3,
"side": "candidate",
"wall_seconds": 6.566,
"cpu_seconds": 6.315785,
"peak_family_mib": 778.133,
"catalog_plus_calls_tokens": 895421,
"calls_ms": [232.37, 1851.929, 78.318, 21.857, 16.438]
}
],
"phase": [
{
"pair": 1,
"side": "baseline",
"first_ms": 35.366,
"repeat_median_ms": 7.784,
"wall_seconds": 0.625,
"cpu_seconds": 0.429065,
"peak_family_mib": 62.922
},
{
"pair": 1,
"side": "candidate",
"first_ms": 30.009,
"repeat_median_ms": 7.934,
"wall_seconds": 0.624,
"cpu_seconds": 0.392848,
"peak_family_mib": 59.664
},
{
"pair": 2,
"side": "baseline",
"first_ms": 32.0,
"repeat_median_ms": 8.772,
"wall_seconds": 0.624,
"cpu_seconds": 0.412059,
"peak_family_mib": 60.422
},
{
"pair": 2,
"side": "candidate",
"first_ms": 32.032,
"repeat_median_ms": 9.36,
"wall_seconds": 0.624,
"cpu_seconds": 0.412412,
"peak_family_mib": 59.441
},
{
"pair": 3,
"side": "baseline",
"first_ms": 33.628,
"repeat_median_ms": 9.496,
"wall_seconds": 0.624,
"cpu_seconds": 0.396232,
"peak_family_mib": 60.41
},
{
"pair": 3,
"side": "candidate",
"first_ms": 30.506,
"repeat_median_ms": 8.172,
"wall_seconds": 0.622,
"cpu_seconds": 0.404398,
"peak_family_mib": 61.672
}
]
},
{
"artifact": {
"repo": "hf-internal-testing/tiny-random-bert",
"revision": "f171d7baecaf37b5da5a3616d8833b9969753535",
"member": "model.safetensors",
"file": "tiny-bert.safetensors",
"url": "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/f171d7baecaf37b5da5a3616d8833b9969753535/model.safetensors",
"bytes": 520212
},
"normalized_sha256": "8658d90261cdbaaf1f5f8d2e9408ad301b40ad3e6b7aab7ac9c5d4be71ca36d5",
"answers_sha256": "82072404a0aa24e3b42979397421f455da2589c2ea649f28ba7279d43e6c3bc8",
"catalog_sha256": "859b1433ef7c45ea23ba4073715d25ed7a76fd9df6d74285d20a5ea39e8e2898",
"mcp": [
{
"pair": 1,
"side": "baseline",
"wall_seconds": 6.667,
"cpu_seconds": 6.46146,
"peak_family_mib": 741.602,
"catalog_plus_calls_tokens": 893755,
"calls_ms": [251.201, 1965.785, 114.055, 10.837, 28.862]
},
{
"pair": 1,
"side": "candidate",
"wall_seconds": 6.767,
"cpu_seconds": 6.519606,
"peak_family_mib": 736.367,
"catalog_plus_calls_tokens": 893755,
"calls_ms": [255.557, 1979.227, 86.726, 10.093, 24.531]
},
{
"pair": 2,
"side": "baseline",
"wall_seconds": 6.565,
"cpu_seconds": 6.379249,
"peak_family_mib": 730.273,
"catalog_plus_calls_tokens": 893755,
"calls_ms": [245.517, 1900.802, 94.351, 10.0, 25.635]
},
{
"pair": 2,
"side": "candidate",
"wall_seconds": 6.569,
"cpu_seconds": 6.388131,
"peak_family_mib": 785.746,
"catalog_plus_calls_tokens": 893755,
"calls_ms": [250.715, 1846.568, 103.314, 21.618, 27.261]
},
{
"pair": 3,
"side": "baseline",
"wall_seconds": 6.771,
"cpu_seconds": 6.530512,
"peak_family_mib": 745.098,
"catalog_plus_calls_tokens": 893755,
"calls_ms": [255.441, 1949.189, 110.897, 10.679, 24.345]
},
{
"pair": 3,
"side": "candidate",
"wall_seconds": 6.466,
"cpu_seconds": 6.296937,
"peak_family_mib": 741.348,
"catalog_plus_calls_tokens": 893755,
"calls_ms": [243.919, 1895.978, 113.995, 10.66, 27.269]
}
],
"phase": [
{
"pair": 1,
"side": "baseline",
"first_ms": 27.648,
"repeat_median_ms": 8.11,
"wall_seconds": 0.623,
"cpu_seconds": 0.394405,
"peak_family_mib": 57.281
},
{
"pair": 1,
"side": "candidate",
"first_ms": 27.003,
"repeat_median_ms": 8.082,
"wall_seconds": 0.523,
"cpu_seconds": 0.383678,
"peak_family_mib": 57.266
},
{
"pair": 2,
"side": "baseline",
"first_ms": 26.941,
"repeat_median_ms": 8.04,
"wall_seconds": 0.626,
"cpu_seconds": 0.401321,
"peak_family_mib": 59.504
},
{
"pair": 2,
"side": "candidate",
"first_ms": 28.012,
"repeat_median_ms": 8.006,
"wall_seconds": 0.625,
"cpu_seconds": 0.392541,
"peak_family_mib": 59.012
},
{
"pair": 3,
"side": "baseline",
"first_ms": 28.734,
"repeat_median_ms": 8.499,
"wall_seconds": 0.624,
"cpu_seconds": 0.411118,
"peak_family_mib": 59.23
},
{
"pair": 3,
"side": "candidate",
"first_ms": 29.961,
"repeat_median_ms": 8.632,
"wall_seconds": 0.625,
"cpu_seconds": 0.41875,
"peak_family_mib": 57.5
}
]
}
],
"published_equivalent_source": "2f091ccc734d7bd945bd61432d41cf5ba68be12b",
"candidate_reproduction": "Use baseline_source plus the byte-identical src/model/GgufHeader.ts from published_equivalent_source; driver SHA-256 is pinned separately. Measured local candidate predates the integration rebase.",
"post_review_receipt_fix": {
"measured_driver_version": "2f091ccc734d7bd945bd61432d41cf5ba68be12b",
"driver_sha256": "449860fb973418f2e3e51183a81be1e3111ffa6453de8208a4b175d3d410322f",
"publication_owner_sha256": "901092ae6dc7fda1a8869d62362a0a9f560e03d5ef5667a5c6ed660db000b9a2",
"validation": "3 new fault-injection regressions reject original driver and pass current; 11 shared receipt regressions pass. Real stories15M MCP and seven-inspection phase reruns qualify; original paired samples retain their measured driver hash."
}
}
experiments/analysis-efficiency/model-utf8-validation.md
+147
View File
@@ -0,0 +1,147 @@
# Omitted GGUF string validation
[Issue #5774](https://github.com/morluto/rea/issues/5774) follows the existing
real-model and completed-task audit in [#5603](https://github.com/morluto/rea/pull/5603).
The [coverage reconciliation](coverage-review-20261021.md) explains why neither
model inspection nor IL2CPP, firmware extraction or IDA should be described as
wholly unaudited. This experiment changes only one model producer hot path.
## Owner and preserved semantics
GGUF arrays retain eight preview values. Previously, each omitted string still
constructed a fatal UTF-8 decoder and materialized discarded text. `GgufHeader`
now validates an omitted value of at most one existing 65,536-byte chunk with
Node's `isUtf8`. Longer values retain the streaming fatal decoder, including
multi-byte characters split across chunks. Retained values use their original
decoder, preserving BOMs and exact text.
Every admitted byte is still read, hashed and validated. Header bounds,
representation admission, cancellation checks, array-loop fairness, source
offsets, partial-progress semantics, preview omission and complete retained
Evidence are unchanged. There is no cache, new contract, selector or default.
CLI and MCP share this producer through the existing application workflow.
## Corpus and correctness
The [curated receipts](model-utf8-validation.json) pin source commits, runtime
file tables, driver/resource sources, immutable public URLs, artifact digests,
complete report/answer fingerprints and all final samples. Baseline is main
`264862bf2e3612c9cb6fcfb861843cbba4e1893f`; measured candidate code is
`c2550343173c5abd8f34e925eac1bcc12b7ee6a2`. Later documentation/integration
commits do not relabel that cohort. That measured local commit predates the
rebase: its production file is byte-identical to published
`2f091ccc734d7bd945bd61432d41cf5ba68be12b`, with both source hashes recorded.
| Artifact | Bytes | Metadata / tensors | Independent reader |
| ------------------------------------------------ | ---------: | -----------------: | ------------------ |
| ggml-org stories15M Q4_0 GGUF | 19,077,344 | 20 / 57 | gguf 0.19.0 |
| ggml-org stories260K GGUF | 1,185,376 | 19 / 48 | gguf 0.19.0 |
| hf-internal-testing tiny-random-bert safetensors | 520,212 | 2 / 100 | safetensors 0.8.0 |
Official independent readers match all exposed metadata values/array previews
and counts, tensor names, types, shapes and parameter counts. GGUF reader
offsets/lengths match every tensor; safetensors reader geometry and independent
Python JSON offsets match its file ranges. Python byte hashing checks every
header digest. No weights are executed or loaded into a model framework.
The main GGUF has 32,000 vocabulary entries; the smaller GGUF and safetensors
are controls, not evidence for larger vocabularies or other GGUF versions.
## Completed public tasks
Linux x64, Node 24.18.0/npm 11.16.0. Three alternating baseline/candidate,
candidate/baseline, baseline/candidate pairs per artifact run serially. Each
fresh production stdio MCP task discovers all 212 tools, directly selects two
metadata records, retrieves two exact tensors from retained Evidence, repeats
both calls, exports complete Evidence, validates answers and closes the
transport. OS caches are unflushed; the controller reads inputs for hashing.
Fresh process does not mean cold disk or cold model-engine analysis.
Medians of three whole tasks per side, including SDK/catalog/schema validation,
all five calls, source checks, export and teardown:
| Artifact | Wall seconds, before → after | Family CPU seconds, before → after | Peak family MiB, before → after |
| ----------- | ---------------------------: | ---------------------------------: | ------------------------------: |
| stories15M | 6.667 → 6.567 | 6.498 → 6.386 | 771.0 → 816.3 |
| stories260K | 6.565 → 6.566 | 6.324 → 6.338 | 743.9 → 744.9 |
| tiny BERT | 6.667 → 6.569 | 6.461 → 6.388 | 741.6 → 741.3 |
The primary task's observed wall reduction is about 1.5%, but the unchanged
safetensors control has similar variation. No stable whole-task speedup is
established. Main-model peak memory rises: ranges 769.3–799.0 → 813.6–818.9
MiB; its cause is unmeasured. No peak-memory improvement is claimed.
Separate fresh-process provider cohorts perform one first inspection and six
repeated inspections, comparing the complete report after every operation:
| Artifact | First / repeat median ms, before | First / repeat median ms, after |
| ----------- | -------------------------------: | ------------------------------: |
| stories15M | 154.03 / 53.58 | 121.41 / 37.19 |
| stories260K | 33.63 / 8.77 | 30.51 / 8.17 |
| tiny BERT | 27.65 / 8.11 | 28.01 / 8.08 |
For stories15M this is about 21% faster first inspection and 31% faster repeated
inspection. Median CPU for the seven-inspection process is 0.855 → 0.729 s;
peak family memory is 75.6 → 77.9 MiB. Phase gains do not include discovery
or substitute for the complete-task results above.
One catalog plus all named request/reply JSON chunks estimates median
895,583 / 895,421 / 893,755 `cl100k_base` tokens for the three artifacts in
both implementations, using tiktoken 0.12.0. This includes serialized output
schemas, not merely named input schemas. It is an SDK serialization proxy;
actual model usage, repeated host injection, negotiation and framing are
unmeasured. No token saving is claimed.
All 18 public tasks and 18 provider processes completed under one CPU, 4 GiB
hard / 3 GiB early-stop family memory, no swap and 192 tasks. Payload exit,
counters and owned cleanup are verified; no limit/OOM events occurred. Kernel
cgroup peak includes file cache. Terminal whole-scope counters after scope
removal remain unknown. Raw inputs, exports, profiles and transcripts stay
outside Git.
## Reproduction and validation boundaries
Build the baseline and the same baseline with the published GGUF source file
using the pinned toolchain; this reconstructs the measured local candidate.
Capture a complete
baseline normalized report from existing CLI/MCP Evidence using the same
absolute input path. Run `model-utf8-validation.mjs` with absolute `--runtime`,
`--artifact`, `--expected` and a new `--output` directory under `resource-run.py
--observe-payload-exit`. Add `--phase` for the seven-inspection provider probe.
The driver validates advertised named schemas through the actual SDK and
compares full exported parents, preview facts and tensor selections. A failing
receipt or cleanup invalidates a task; it is not silently retried as success.
Five new producer regressions cover empty omitted strings, exact-chunk strings,
split multi-byte characters, both byte orders, retained BOMs and rejection of
short/long malformed omitted values. Existing malformed-header, admission,
cancellation, partial-progress and public CLI/MCP tests remain required.
An initial new-test assertion incorrectly hashed alignment padding; the
corrected oracle uses the actual declared header, matching existing semantics.
The first test launcher refused built projects; the exact built-project gate
was then used. Neither attempt is a passing check. The initial resource-wrapper
copy failed source-hash lookup before launching a build; corrected guarded
builds qualified before measurement.
Integrated onto main `71b696582`: all 68 tests pass again, and three real public
CLI complete reports plus three production SDK tasks match the original
normalized reports. Typecheck, lint/module boundaries, formatting, metadata,
runtime build, docs checks (five tests) and separate architecture guards pass.
`npm run check` does **not** pass: Knip reports an unlisted `g++` invocation in
the Windows test added by main's #5772. An independent main source checkout,
with matching compiled worker artifacts and an uncached Knip run, reproduces
only that finding. It is outside this change and is left unchanged. Initial
control attempts lacked resolvable worker artifacts and are not clean controls.
Development builds/checks used separate 6/8 GiB hard guards; runtime experiments
retain their 4 GiB limit. No guard stop or OOM occurred.
Greptile identified a receipt defect in the phase driver, not the model parser:
failed inspections/assertions were not recorded before cleanup, and a close
failure could suppress publication. Each attempted phase now records its cost
and primary failure before cleanup. Phase and MCP paths share one atomic receipt
publisher; cleanup failures are independent facts and cannot replace the primary
failure. Three new fault-injection tests fail on the original driver and pass
after the correction; all 11 existing shared receipt tests pass. These stand-ins
qualify failure accounting only. Real stories15M MCP and seven-inspection phase
reruns also pass with verified counters/cleanup. The original 36 paired samples
keep their measured driver hash; the receipt records the post-review driver and
publication-owner hashes separately. No timing cohort is relabeled.
experiments/analysis-efficiency/model-utf8-validation.mjs
+243
View File
@@ -0,0 +1,243 @@
/** Equivalent model tasks; inputs, expected reports and raw receipts stay outside Git. */
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { createWriteStream } from "node:fs";
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { finished } from "node:stream/promises";
import { pathToFileURL } from "node:url";
import { parseArgs } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { createRetainedQueryCaller } from "./retained-query-calls.mjs";
import {
finalizeRetainedQueryReceipt,
publishRetainedQueryReceipt,
} from "./retained-query-finalization.mjs";
const { values } = parseArgs({
options: {
...Object.fromEntries(
["runtime", "artifact", "expected", "output"].map((name) => [
name,
{ type: "string" },
]),
),
phase: { type: "boolean", default: false },
},
});
for (const name of ["runtime", "artifact", "expected", "output"])
assert.ok(values[name] && values[name] === resolve(values[name]), name);
const hash = (value) => createHash("sha256").update(value).digest("hex");
const expected = JSON.parse(await readFile(values.expected, "utf8"));
assert.equal(expected.artifact.path, values.artifact);
await mkdir(values.output, { mode: 0o700 });
const report = {
artifact_sha256: hash(await readFile(values.artifact)),
normalized_sha256: hash(JSON.stringify(expected)),
calls: [],
answers: [],
};
if (values.phase) {
const { ModelArtifactProvider } = await import(
pathToFileURL(join(values.runtime, "dist/model/ModelArtifactProvider.js"))
);
const provider = new ModelArtifactProvider();
try {
for (let index = 0; index < 7; index++) {
const sample = {
phase: index === 0 ? "fresh" : "repeat",
is_error: true,
};
const start = performance.now();
try {
const result = await provider.inspect({
path: values.artifact,
hash: "header",
});
sample.elapsed_ms = performance.now() - start;
assert.ok(result.ok, JSON.stringify(result));
assert.deepEqual(result.value, expected);
sample.is_error = false;
} catch (cause) {
sample.elapsed_ms ??= performance.now() - start;
sample.failure = String(cause?.stack ?? cause);
throw cause;
} finally {
report.calls.push(sample);
}
}
report.status = "passed";
} catch (cause) {
report.status = "failed";
report.failure = String(cause?.stack ?? cause);
process.exitCode = 1;
} finally {
report.cleanup_failures = [];
try {
await provider.close();
} catch (cause) {
report.cleanup_failures.push(`provider: ${String(cause)}`);
report.status = "failed";
}
if (!(await publishRetainedQueryReceipt({ report, output: values.output })))
process.exitCode = 1;
}
} else {
await runMcpTask();
}
async function runMcpTask() {
const account = await mkdtemp(join(tmpdir(), "rea-model-utf8-"));
const client = new Client({ name: "model-utf8-validation", version: "1" });
const transport = new StdioClientTransport({
command: process.execPath,
args: [join(values.runtime, "scripts/rea.mjs"), "mcp"],
env: {
HOME: account,
PATH: process.env.PATH,
NODE_OPTIONS: "--max-old-space-size=2048",
UV_THREADPOOL_SIZE: "1",
REA_LOG_LEVEL: "silent",
TMPDIR: account,
},
stderr: "pipe",
});
const serverLog = createWriteStream(join(values.output, "server.log"));
const serverLogFinished = finished(serverLog);
serverLogFinished.catch(() => {});
transport.stderr?.pipe(serverLog);
const transcript = [];
const call = createRetainedQueryCaller(client, (receipt, event) => {
report.calls.push(receipt);
transcript.push(event);
});
const observe = async (name, input, phase) => {
const reply = await call(name, input, phase);
assert.notEqual(reply.isError, true, JSON.stringify(reply));
assert.ok(reply.structuredContent);
return reply.structuredContent;
};
try {
await client.connect(transport);
const catalog = await client.listTools();
assert.equal(catalog.nextCursor, undefined);
await writeFile(
join(values.output, "catalog.json"),
JSON.stringify(catalog),
);
report.catalog_sha256 = hash(JSON.stringify(catalog));
report.catalog_tools = catalog.tools.length;
const definitions = new Map(catalog.tools.map((tool) => [tool.name, tool]));
const originalCall = client.callTool.bind(client);
client.callTool = (input, options) =>
originalCall(input, {
...options,
toolDefinition: definitions.get(input.name),
});
const keys = [
...new Set(expected.metadata.slice(0, 2).map((row) => row.key)),
];
const names = expected.tensors.slice(0, 2).map((row) => row.name);
const fields = [
"key",
"type",
"value",
"element_count",
"preview_complete",
];
const tensorFields = [
"name",
"type",
"shape",
"file_offset",
"byte_length",
];
const select = (rows, selectedFields) =>
rows.map((row) =>
Object.fromEntries(
["index", ...selectedFields].map((key) => [key, row[key]]),
),
);
const observations = [];
for (const phase of ["fresh", "repeat"]) {
const metadata = await observe(
"inspect_model_artifact",
{
path: values.artifact,
view: { collection: "metadata", keys, fields },
},
phase,
);
const parentId = metadata.normalized_result.parent_evidence_id;
assert.equal(typeof parentId, "string");
assert.deepEqual(
metadata.normalized_result.items,
select(
expected.metadata.filter((row) => keys.includes(row.key)),
fields,
),
);
const tensors = await observe(
"inspect_analysis_view",
{
source: { kind: "retained-evidence", evidence_id: parentId },
view: {
kind: "model-artifact",
collection: "tensors",
names,
fields: tensorFields,
},
},
`${phase}:tensors`,
);
assert.equal(tensors.normalized_result.parent_evidence_id, parentId);
assert.deepEqual(
tensors.normalized_result.items,
select(expected.tensors.slice(0, 2), tensorFields),
);
observations.push(parentId);
report.answers.push({
parent_evidence_id: parentId,
metadata: metadata.normalized_result.items,
tensors: tensors.normalized_result.items,
});
}
await observe(
"export_evidence_bundle",
{ path: join(values.output, "evidence.json") },
"export",
);
const bundle = JSON.parse(
await readFile(join(values.output, "evidence.json"), "utf8"),
);
for (const id of observations) {
const parent = bundle.records.find((record) => record.evidence_id === id);
assert.ok(parent);
assert.deepEqual(parent.normalized_result, expected);
}
assert.deepEqual(report.answers[0], report.answers[1]);
assert.equal(hash(await readFile(values.artifact)), report.artifact_sha256);
report.status = "passed";
} catch (cause) {
report.status = "failed";
report.failure = String(cause?.stack ?? cause);
process.exitCode = 1;
} finally {
if (
!(await finalizeRetainedQueryReceipt({
report,
client,
transport,
account,
serverLog,
serverLogFinished,
output: values.output,
transcript,
}))
)
process.exitCode = 1;
}
}
experiments/analysis-efficiency/retained-capture-validation.md
+1 -1
View File
@@ -18,7 +18,7 @@ Current history and the existing evidence support these narrower conclusions:
| Apktool | [#4777](https://github.com/morluto/rea/issues/4777) is closed. Shared decoding and optional primary-string omission have [three-APK evidence](apktool-string-tables.md). Do not implement another decode cache from the old issue list. |
| Hopper | [#5017](https://github.com/morluto/rea/pull/5017) already reuses decoded string records. Its [checkpoint](hopper-string-records.md) preserves faster warm queries and the negative whole-task result. |
| JEB | The [Linux CE audit](jeb-linux-native.md) verifies selected real CLI/MCP reads. It does not justify another wrapper cache; discovery and other platforms remain separate coverage gaps. |
| IDA | Existing [native Windows verification](ida-native-windows.md) qualifies selected correctness/authority paths. It does not establish a comparable full performance audit. |
| IDA | Existing [native Windows verification](ida-native-windows.md) includes selected real completed-task performance and correctness/authority checks. It does not qualify every operation or platform. |
| JavaScript/browser/Electron | Prior experiments and retained pages already exist. [#4415](https://github.com/morluto/rea/issues/4415) remains exploratory; no safe general Joplin preparation speedup is established. |
| Other static formats | Managed, Flutter, SQLite, WASM, EVM, PCAP, signals and SAZ already have checkpoints in this directory. Extend a specific measured question rather than create duplicate selectors or caches. |
| Other providers, languages and infrastructure | Rizin/Cutter/GDB/objdump, Go/Rust, ADB/Frida/inspector and startup/resource owners still need bounded real-workflow questions. Historical local improvements do not prove complete coverage or establish another bottleneck. |
experiments/analysis-efficiency/retained-query-finalization.mjs
+10
View File
@@ -58,6 +58,16 @@ export const finalizeRetainedQueryReceipt = async ({
);
if (report.cleanup_failures.length > 0) report.retained_workspace = account;
return publishRetainedQueryReceipt({ report, output, transcript, stdout });
};
/** Share atomic publication and failure accounting with non-MCP phase probes. */
export const publishRetainedQueryReceipt = async ({
report,
output,
transcript,
stdout = process.stdout,
}) => {
report.receipt_failures = [];
if (transcript !== undefined)
await attempt(report.receipt_failures, "transcript-file", () =>
experiments/analysis-efficiency/rust-dependency-costs.py
+79
View File
@@ -0,0 +1,79 @@
"""Count completed Rust questions; tokenizer estimates are not model usage."""
import argparse
import hashlib
import json
from pathlib import Path
import tiktoken
def compact(value):
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False)
def inspect(path, encoding):
report = json.loads((path / "report.json").read_text())
catalog = json.loads((path / "catalog.json").read_text())
transcript = json.loads((path / "transcript.json").read_text())
log = path.with_suffix(".log")
scope = next(json.loads(line) for line in reversed(log.read_text().splitlines())
if line.startswith('{"resource_scope":'))
observed = scope.get("payload_resource_observation") or {}
events = observed.get("resource_events", {})
memory = events.get("memory.events", {})
assert (report["status"] == "passed" and not report["cleanup_failures"]
and not report["receipt_failures"] and scope["abort"] is None
and scope.get("payload_resource_observation_verified") is True
and observed.get("payload_cleanup_verified") is True
and observed.get("payload_exit_code") == 0
and all(memory.get(key) == 0 for key in ["high", "max", "oom", "oom_kill"])
and events.get("pids.events", {}).get("max") == 0), path
assert [event["name"] for event in transcript] == [
"inspect_rust_binary", "export_evidence_bundle"]
count = lambda value: len(encoding.encode(value, disallowed_special=()))
schema_fields = ["name", "description", "inputSchema"]
tool_tokens = lambda tools: count(compact([
{key: tool[key] for key in schema_fields} for tool in tools]))
inputs = tool_tokens(catalog["tools"])
selected = tool_tokens([tool for tool in catalog["tools"] if tool["name"] in [
"inspect_rust_binary", "export_evidence_bundle"]])
arguments = sum(count(compact({"name": event["name"], "arguments": event["input"]}))
for event in transcript)
replies = sum(count(block["text"]) for event in transcript
for block in event["reply"]["content"] if block["type"] == "text")
return {"case": path.name, "lane": report["lane"],
"answer": report["answer"], "parent_evidence_id": report["parent_evidence_id"],
"artifact_sha256": report["artifact_sha256"],
"complete_result_sha256": report["normalized_sha256"],
"catalog_sha256": report["catalog_sha256"],
"catalog_tools": report["catalog_tools"], "calls": report["calls"],
"wall_seconds": scope["elapsed_seconds"],
"cpu_seconds": observed["cpu_stat"]["usage_usec"] / 1e6,
"kernel_family_peak_bytes": observed["memory_peak_bytes"],
"events_at_payload_exit": events,
"final_whole_scope_events": scope.get("resource_events_final"),
"completed_with_verified_cleanup": True,
"token_proxy": {"input_catalog_once": inputs, "arguments": arguments,
"text_replies": replies, "combined": inputs + arguments + replies,
"only_two_selected_tools": selected + arguments + replies,
"complete_transport_catalog": count(compact(catalog))},
"sha256": {file.name: hashlib.sha256(file.read_bytes()).hexdigest()
for file in [path / "report.json", path / "catalog.json",
path / "transcript.json", log]}}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("cases", type=Path, nargs="+")
args = parser.parse_args()
encoding = tiktoken.get_encoding("cl100k_base")
receipt = {"tiktoken": tiktoken.__version__, "encoding": encoding.name,
"actual_model_usage": None,
"projection": "All discovered tool descriptions/input schemas once, plus every call argument and text reply including export. Selected-two-tool and complete transport catalog costs are separate. No model, host injection, history replay or retry observed.",
"recordings": [inspect(path, encoding) for path in args.cases]}
args.output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n")
if __name__ == "__main__":
main()
experiments/analysis-efficiency/rust-dependency-views.json
+578
View File
@@ -0,0 +1,578 @@
{
"baseline_source": "d672cc23998b6cbb21c485f118595ebfb82fed33",
"candidate_code": "cae2571e7e64a7364b94bb92c2630cba80ec4478",
"runtime_filetables": {
"baseline": {
"sha256": "6497f70d4864489e948b6a6f9611cf1bd9e6d7f60722fcdd41c0984ab7c48d53",
"files": 1917
},
"candidate": {
"sha256": "078e6fd0f837ca252265d12d3a95e346f836762d2474117c51f4c9f7ae629b21",
"files": 1920
}
},
"lock_sha256": "699a14749e3825c62430faeb18224190fee1c9347760494e38e334e180bea6ec",
"source_sha256": {
".agents/skills/reverse-engineer-anything/SKILL.md": "4baf850678f0781cbb989a7ac1452c4cbc5d98e4125bc059b5381c336cc8bc07",
"docs/rust-binaries.md": "3317a494376c3dde0c81a848ea678375db4af61fa85ba8d505b7d89b319c8371",
"experiments/analysis-efficiency/corpus.json": "81c223028c7c6308949e508413dea6aef79b691f6dc7fe02eb6ff81e6d52b0cc",
"experiments/analysis-efficiency/rust-dependency-costs.py": "c7eb7a9bc696e06c4cfba3b0aa730bf8cb9f2bcdec3a371eae3849a47c5f3135",
"experiments/analysis-efficiency/rust-dependency-views.mjs": "7c59302fe00c8a00e4cea06bca8f73974e40a618143ea4f1c7c5e275c9c07df5",
"src/application/rust/RustBinaryService.test.ts": "9b424569f52ed6c5e4629b3bda4fb8a8dc805adc0881e82f1bf39f38618c57a3",
"src/application/rust/RustBinaryService.ts": "caa089f62cdf24ee8a4ceb6eb7bdbf9df90fc724f921197f9e69b680366a98a7",
"src/cli/rustCommands.ts": "18c87736f20437f178d2094f0b46392d167ff510fa8a08a6b121610538fc257b",
"src/contracts/analysisViewToolContracts.ts": "149cd98c2ed22444ac9e59fa9144b6840c3fd13bd1613ddf8da23310766829f2",
"src/contracts/rust/rustToolContracts.ts": "5433f15f581618b3184147c04ccd21e04bc4a3a880b7ea0db6dbc38e5825efab",
"src/domain/analysisView/analysisView.ts": "8122d62bf795f3abae4bd903a641e336f0c2650cab98836051839bfaec4e1d65",
"src/domain/analysisView/goBinaryView.ts": "c434f7d7c198481d2fab2c2f7d5672f0db34256e9c8700a52c65c55def05df0b",
"src/domain/analysisView/rustBinaryView.test.ts": "3c53131b5643ed874214c0ba90e66960d2acdf9d22a881d283ef44aba671dfe4",
"src/domain/analysisView/rustBinaryView.ts": "8c330cf31b3ccb37e3b12a40e8157c50a9d43ac42cf6477dcca969398e0f79ad",
"src/domain/analysisView/selectViewRows.ts": "6eb486166e991045bf0a74cf874c80d2542862147b7ba095e45a61ec26afd1e5",
"src/domain/rust/rustBinary.ts": "90fc6fc20faa0f663dc14e08e8735804ae77abaa7fdb2e918162b358efb42462",
"src/domain/rust/rustBinaryQuery.ts": "ecb2ad8a97e3f3d232f94f5969baaef62143392362d18b4d46ab6babcfe96da2",
"src/server/createServer.ts": "b005d3d71edc98b9eb3422d83b8ddeddcc7db0b1982a1e93ce4578d588dbd8e6",
"src/server/registerRustTools.ts": "e57723a66916f3e2d9208e7202ac50b28536234874fbc3752e9259e0145299ae",
"tests/acceptance/rustBinaryCli.test.ts": "85962f740bd1d2f1fc99601337ab7e7ca3f4e659553d41297aa216e8928c0af4",
"tests/acceptance/rustDependencyViews.test.ts": "b1505497e9dbfb1ce855725a8af501f1ebc82bb2c528b8992e9b33cf8628ef7f",
"tests/boundary/mcp/rustBinaryMcp.test.ts": "4d2c5f172ebcd3407265b195adc7c5bc5dd590d97a443560a6d8c6afda60d08a"
},
"resource_guard_sha256": {
"resource-run.py": "7a9642ab86ba34290564f636eb080cacf38b4185a254028680b238bff172993b",
"resource_payload.py": "6f089e645459eba80c5d73e39c21606c78173a44daf49fabc3b88c4631ce51c6"
},
"host": {
"platform": "Linux-6.1.0-54-cloud-amd64-x86_64-with-glibc2.36",
"machine": "x86_64",
"affinity": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
"node": "v24.18.0",
"npm": "11.16.0",
"objcopy": "GNU objcopy (GNU Binutils for Debian) 2.40",
"readelf": "GNU readelf (GNU Binutils for Debian) 2.40",
"python": "3.11.2"
},
"artifacts": [
{
"package": "ripgrep-14.1.1-r0.apk",
"url": "https://dl-cdn.alpinelinux.org/alpine/v3.21/community/x86_64/ripgrep-14.1.1-r0.apk",
"package_sha256": "213dd0b047163f6627cc2fdd6fbfe205009476eb4fc48c352a3f5b9ab647f6d4",
"package_bytes": 1422830,
"member": "usr/bin/rg",
"bytes": 3416168,
"sha256": "78387bbe12b2110df8d2ce364390a00c190f4543a2f40b8279e49f6c5c85af9c",
"aports_revision": "4e899d119e4c983a938436d6f397a509521225d1"
},
{
"package": "zellij-0.40.0-r0.apk",
"url": "https://dl-cdn.alpinelinux.org/alpine/v3.21/community/x86_64/zellij-0.40.0-r0.apk",
"package_sha256": "575075945415b221857563e19c5f4c661af9f98bbba54f0994c7385c0e432432",
"package_bytes": 6722129,
"member": "usr/bin/zellij",
"bytes": 17151864,
"sha256": "9a86e9ca47902a869fdf1886f4cf17714f4fa5655427128beadff498cfb392db",
"aports_revision": "3be26a3a26e76e1bf3eff784bffe3956d29d77a0"
}
],
"oracle": [
{
"file": "ripgrep",
"packages": 42,
"edges": 88,
"reply_bytes": 10605,
"root": 33,
"upstream_json_sha256": "ea478aa80f91aa8f5159aba648ddfdda642fa2c98759e54e65623a19065634be",
"evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4",
"matched": true
},
{
"file": "zellij",
"packages": 422,
"edges": 1063,
"reply_bytes": 86805,
"root": 417,
"upstream_json_sha256": "9bcf8d694440a7e064189616aa40b69046e32ee1c70683ef2e0778150b9163ef",
"evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb",
"matched": true
}
],
"root_cycle_oracle": [
{
"file": "ripgrep",
"root_index_independently_matched": 33,
"independent_topological_visit_count": 42,
"acyclic": true
},
{
"file": "zellij",
"root_index_independently_matched": 417,
"independent_topological_visit_count": 422,
"acyclic": true
}
],
"tokenizer": {
"tiktoken": "0.12.0",
"encoding": "cl100k_base",
"actual_model_usage": null,
"projection": "All discovered tool descriptions/input schemas once, plus every call argument and text reply including export. Selected-two-tool and complete transport catalog costs are separate. No model, host injection, history replay or retry observed."
},
"samples": [
{
"case": "r1-zellij-before",
"wall_seconds": 5.77,
"cpu_seconds": 5.626476,
"kernel_family_peak_bytes": 644210688,
"completed_with_verified_cleanup": true,
"sha256": {
"report.json": "f2bd86aedd299c4fe16cca470eeaf135b6553c1bfe345018def503424c5661a6",
"catalog.json": "411381fb2417945f8a29eb8db330f82b44d8790d3f1e88c7f766b532c528008f",
"transcript.json": "bca9b5474a2124a99535c4aca0c38d9c77146d508822904aae4adc458b05b10e",
"r1-zellij-before.log": "79acdc6c966345d59ba88aafd22b5fb11cdd759c62611e720a47aef308ef070c"
},
"calls_ms": [397.233, 103.9],
"reply_bytes": [185833, 310],
"combined_tokens": 223755
},
{
"case": "r2-zellij-before",
"wall_seconds": 4.957,
"cpu_seconds": 4.781552,
"kernel_family_peak_bytes": 636317696,
"completed_with_verified_cleanup": true,
"sha256": {
"report.json": "f3f58187dc63116f02586b2b3403848a8a88f6f72c5e322d0be00c9fcb2b7710",
"catalog.json": "411381fb2417945f8a29eb8db330f82b44d8790d3f1e88c7f766b532c528008f",
"transcript.json": "718aa02bf3fd74099b44418ac220e639719495b3e718357fb4e34748e5499e41",
"r2-zellij-before.log": "3e36a14c0b7e8d4d6e6f8b4b024c090133773f2699a8737d17dc5352b5f490ac"
},
"calls_ms": [380.788, 40.185],
"reply_bytes": [185833, 310],
"combined_tokens": 223755
},
{
"case": "r3-zellij-before",
"wall_seconds": 4.854,
"cpu_seconds": 4.693446,
"kernel_family_peak_bytes": 639963136,
"completed_with_verified_cleanup": true,
"sha256": {
"report.json": "75d54eb72935201b8946d83a3bef98d93dc1219022bf735935d42934069b7dea",
"catalog.json": "411381fb2417945f8a29eb8db330f82b44d8790d3f1e88c7f766b532c528008f",
"transcript.json": "940eb7899a07a1115402fbc7b89010b463723b26e129f62a51263b9c9e598b1f",
"r3-zellij-before.log": "0f266f87cf5f2984ac10964f3255ebb76a706695256683c7f80c071b84d1f3a6"
},
"calls_ms": [340.588, 30.985],
"reply_bytes": [185833, 310],
"combined_tokens": 223755
},
{
"case": "r1-zellij-after",
"wall_seconds": 6.277,
"cpu_seconds": 6.056786,
"kernel_family_peak_bytes": 637652992,
"completed_with_verified_cleanup": true,
"sha256": {
"report.json": "8d34bbdba7cfedc58e281efc372cf6969def10bc99b022e4bc786d447daf4709",
"catalog.json": "ba99f10270e8ab53d765f124856921f59156482d603fa883240e1b4068eab921",
"transcript.json": "e09cc44b0ce78a74be6cc9d7c90a78e2173514e5a4cc1dd9f02e5a728c880600",
"r1-zellij-after.log": "a90915676741794e52568dd30604a4e306ebfec77df37982bef6ea3d4411e405"
},
"calls_ms": [517.341, 43.422],
"reply_bytes": [9261, 308],
"combined_tokens": 199584
},
{
"case": "r2-zellij-after",
"wall_seconds": 5.671,
"cpu_seconds": 5.441365,
"kernel_family_peak_bytes": 633688064,
"completed_with_verified_cleanup": true,
"sha256": {
"report.json": "425d70965ad49048d01a88473c1fb4790a7ef6f4ceb1c5374a652fc239fa96e5",
"catalog.json": "ba99f10270e8ab53d765f124856921f59156482d603fa883240e1b4068eab921",
"transcript.json": "96d3bc1798136135beed8e7f2b4eac626124655fb01ca1cdfe1040786ddf9efa",
"r2-zellij-after.log": "74fed3f25bd859ce4f6d4f4e7a6d3aaf31c9f079f730427fffa489a003adcfaa"
},
"calls_ms": [473.788, 28.945],
"reply_bytes": [9261, 308],
"combined_tokens": 199584
},
{
"case": "r3-zellij-after",
"wall_seconds": 5.054,
"cpu_seconds": 4.816254,
"kernel_family_peak_bytes": 622559232,
"completed_with_verified_cleanup": true,
"sha256": {
"report.json": "349360e9b1c19faaa796106a70a26fd5711b804af807fc1b9c98a6e012530f9e",
"catalog.json": "ba99f10270e8ab53d765f124856921f59156482d603fa883240e1b4068eab921",
"transcript.json": "12fc240871df42621b8e80efdb6451136ea317ae364f7c6741aed348a12a73a9",
"r3-zellij-after.log": "18d9edcae6ac46c4034b958ba2e607a524aaf80dfbc671a5fd654f6390ee5c3f"
},
"calls_ms": [359.933, 34.275],
"reply_bytes": [9261, 308],
"combined_tokens": 199584
},
{
"case": "r1-ripgrep-before",
"wall_seconds": 4.859,
"cpu_seconds": 4.643315,
"kernel_family_peak_bytes": 599961600,
"completed_with_verified_cleanup": true,
"sha256": {
"report.json": "2918f88999120cfba046ad5dc615d12bcc98c034c767a00edb3631a2b8bd622a",
"catalog.json": "411381fb2417945f8a29eb8db330f82b44d8790d3f1e88c7f766b532c528008f",
"transcript.json": "aa98de6fb6cbd24672e3a5b8aaf22ff1f977276d00c14ebd32e90501bd9eb4aa",
"r1-ripgrep-before.log": "a3963e991e1286e84a03f69f2d7e8a7e313ea262123269777599d93b7e529153"
},
"calls_ms": [181.636, 30.005],
"reply_bytes": [22643, 312],
"combined_tokens": 200603
},
{
"case": "r2-ripgrep-before",
"wall_seconds": 5.057,
"cpu_seconds": 4.853134,
"kernel_family_peak_bytes": 596680704,
"completed_with_verified_cleanup": true,
"sha256": {
"report.json": "35f8de985a6201efc40f629195a9a801d2f333d605a46220bd4ae313c35f2abd",
"catalog.json": "411381fb2417945f8a29eb8db330f82b44d8790d3f1e88c7f766b532c528008f",
"transcript.json": "bed19b9d031b3416bc79870fb8d94ed0a2376004dfb9b69b78c56bd4a38912ff",
"r2-ripgrep-before.log": "ec0a5b2933307131af7435d737dde4a72ad3c92f3923e46aaaae7b0b7b7b1fd3"
},
"calls_ms": [189.36, 29.044],
"reply_bytes": [22643, 312],
"combined_tokens": 200603
},
{
"case": "r3-ripgrep-before",
"wall_seconds": 4.753,
"cpu_seconds": 4.542499,
"kernel_family_peak_bytes": 589565952,
"completed_with_verified_cleanup": true,
"sha256": {
"report.json": "daaf848260488ef347afc4c600fe89c2902d6a68fa65085c0fcae7a20d7c6d17",
"catalog.json": "411381fb2417945f8a29eb8db330f82b44d8790d3f1e88c7f766b532c528008f",
"transcript.json": "43e570e7421f14aaaf7dc92b8c4a69d12a221851c7e813587eda8ca73e0b1d7f",
"r3-ripgrep-before.log": "6f88e1abbe8ebb8a68cf3a5d678c227516c8506029b0114094b1a42bfd056971"
},
"calls_ms": [201.092, 30.174],
"reply_bytes": [22643, 312],
"combined_tokens": 200603
},
{
"case": "r1-ripgrep-after",
"wall_seconds": 5.159,
"cpu_seconds": 4.945401,
"kernel_family_peak_bytes": 609128448,
"completed_with_verified_cleanup": true,
"sha256": {
"report.json": "14294e20d39e6413e1dac9836a01dd498b7a669c4fa0a9708fd131452a0b6881",
"catalog.json": "ba99f10270e8ab53d765f124856921f59156482d603fa883240e1b4068eab921",
"transcript.json": "dd32475dc2205d3574a0496730a9c60751464c2fcc4a955051e889f1b0547c0d",
"r1-ripgrep-after.log": "dd22e6f0e9eac449e0485a67ae79839958109e5331f4872aa9a527a51aae1766"
},
"calls_ms": [244.153, 30.492],
"reply_bytes": [9207, 310],
"combined_tokens": 199584
},
{
"case": "r2-ripgrep-after",
"wall_seconds": 4.757,
"cpu_seconds": 4.528198,
"kernel_family_peak_bytes": 615854080,
"completed_with_verified_cleanup": true,
"sha256": {
"report.json": "d45dc890ea69f370b47ccefc2b454dc1d0c3baf2beafd0fbe36ffb91c890e8e3",
"catalog.json": "ba99f10270e8ab53d765f124856921f59156482d603fa883240e1b4068eab921",
"transcript.json": "2b060caea34407a0c5c18fac3cad43a4dd048eee54bb760dd56ca95665b6a492",
"r2-ripgrep-after.log": "36cdb1176e1fdeef6e00f360c1a36ddb97506fcd31b426ee9be746da9ece7535"
},
"calls_ms": [249.866, 28.919],
"reply_bytes": [9207, 310],
"combined_tokens": 199584
},
{
"case": "r3-ripgrep-after",
"wall_seconds": 4.758,
"cpu_seconds": 4.512774,
"kernel_family_peak_bytes": 615874560,
"completed_with_verified_cleanup": true,
"sha256": {
"report.json": "61ec33fe49c943a71177562212d255780ee6cf4cce273c1733a9ae6c138c6167",
"catalog.json": "ba99f10270e8ab53d765f124856921f59156482d603fa883240e1b4068eab921",
"transcript.json": "eb48ed154e646e3aa1ea9cb1aae99e363bb68ac122777ba4f3ac63d84547db37",
"r3-ripgrep-after.log": "34fd034def0ab7e0c22e9aa89aaaef5f5cc58a9125606400d8e866023578432b"
},
"calls_ms": [224.772, 27.335],
"reply_bytes": [9207, 310],
"combined_tokens": 199584
}
],
"resource_limits": {
"cpu_quota_percent": 100,
"affinity": [15],
"hard_memory_gib": 4,
"early_stop_gib": 3,
"swap_bytes": 0,
"tasks": 192,
"host_preflight_gib": 8,
"host_stop_gib": 4
},
"cache_state": "Fresh process/session every task; unflushed OS caches, controller hashes inputs first; not cold disk.",
"failed_attempts": [
{
"attempt": "baseline runtime build and cached retry",
"outcome": "Commands exited 0; cleanup counters unqualified due extra observed child; no qualified build resource claim. Actual CLI and SDK tasks separately qualify."
},
{
"attempt": "prototype source typechecks",
"outcome": "Failed on new optional subject and expanded result-union typings; fixed and passed final check."
},
{
"attempt": "prototype static gate",
"outcome": "New acceptance function exceeded line limit; refactored and passed."
},
{
"attempt": "first one-CPU acceptance cohort",
"outcome": "Existing cross-Go/Rust acceptance test timed out at its unchanged 30s limit. All nine CLI acceptance tests passed with a bounded two-CPU/4-GiB cohort; no timeout change."
}
],
"rejected_prototype": {
"change": "Replace native JSON.stringify report sizing with shared JS byte counter",
"provider_first_repeat_ms": {
"provider-ripgrep-candidate": {
"first_ms": 76.985,
"repeat_median_ms": 20.225
},
"provider-zellij-baseline": {
"first_ms": 181.069,
"repeat_median_ms": 92.804
},
"provider-ripgrep-baseline": {
"first_ms": 60.221,
"repeat_median_ms": 18.323
},
"provider-zellij-candidate": {
"first_ms": 183.659,
"repeat_median_ms": 76.214
}
},
"decision": "No consistent CPU or memory advantage established across both real artifacts; reverted. Final provider file is byte-identical to baseline."
},
"diagnostic_phase_controls": {
"driver_sha256": "eeca9644b4b7199e45e49447bf83f7b33ac60c9ce1c38908b06f59d72e3b7485",
"method": "Same task driver plus performance.now around client.connect and listTools; baseline complete, candidate complete and candidate selected in forward/reverse order. Private instrumented driver, not the primary cohort.",
"samples": [
{
"case": "control1-baseline",
"connect_ms": 2515.583,
"catalog_ms": 1676.58,
"calls_ms": [431.356, 57.797],
"wall_seconds": 5.46,
"cpu_seconds": 5.243091,
"kernel_family_peak_bytes": 644329472,
"qualified": true,
"report_sha256": "badff5397d47a240272feb4bd2348e3122d5482eea116933b29eddd740768955",
"log_sha256": "9b0eb1bfe51e932a4e8978d165fc26f213b003ee4452f704da1f9ce813830850"
},
{
"case": "control1-candidate-complete",
"connect_ms": 3086.124,
"catalog_ms": 1726.06,
"calls_ms": [485.865, 30.696],
"wall_seconds": 6.164,
"cpu_seconds": 5.94611,
"kernel_family_peak_bytes": 634658816,
"qualified": true,
"report_sha256": "ac65641dcf7dd556017348860484b2a0485628803ae2f9e619cee40007fe8aff",
"log_sha256": "457a2dee6f38d2f57b4616c5b1138abc26c18235844fa4589e89b4f15ca4abf8"
},
{
"case": "control1-candidate-selected",
"connect_ms": 3057.187,
"catalog_ms": 1677.082,
"calls_ms": [431.301, 32.986],
"wall_seconds": 6.063,
"cpu_seconds": 5.798686,
"kernel_family_peak_bytes": 639565824,
"qualified": true,
"report_sha256": "ef8e46c78e81f13c84f1bf0a8a8284078252d5af16302a716144149c136f89f5",
"log_sha256": "8d72501364d0a95fb250e2d0486b9b34076688309748b8dc054a75681e8543e0"
},
{
"case": "control2-baseline",
"connect_ms": 2678.85,
"catalog_ms": 1467.674,
"calls_ms": [413.185, 36.964],
"wall_seconds": 5.364,
"cpu_seconds": 5.183688,
"kernel_family_peak_bytes": 643944448,
"qualified": true,
"report_sha256": "d7e3318331528e1201fd1efc7cdc4edb0624a32e6c6db501504cc77f49374157",
"log_sha256": "d61f437e5c88b2eb994e1f716c05915f0955d5877b1d34207fd39e3d36f85970"
},
{
"case": "control2-candidate-complete",
"connect_ms": 2527.714,
"catalog_ms": 1545.887,
"calls_ms": [436.068, 30.776],
"wall_seconds": 5.36,
"cpu_seconds": 5.159006,
"kernel_family_peak_bytes": 635162624,
"qualified": true,
"report_sha256": "fb432b0f5eeedc74e66144c136954e580f5efb5b86a921ace3bf29477e90f0aa",
"log_sha256": "4e7485b8b2a5bb0619f0e84df96b68e02827e2afee11533e0f12532c12fd84f7"
},
{
"case": "control2-candidate-selected",
"connect_ms": 2642.283,
"catalog_ms": 1465.879,
"calls_ms": [424.082, 32.29],
"wall_seconds": 5.362,
"cpu_seconds": 5.110599,
"kernel_family_peak_bytes": 637669376,
"qualified": true,
"report_sha256": "2d4b794902c9fd76b11a2f7ae87bb06be3cc9effadfb161f6f4debb9044ebcfd",
"log_sha256": "e98cc393c717ae19bb0c880d609a05e2f2d4249701f245c19b2f030b8be305fb"
}
]
},
"cohort_facts": {
"zellij-complete": {
"answer": {
"packages": [
{
"index": 294,
"name": "serde",
"version": "1.0.137",
"source": "crates.io",
"dependencies": [296]
},
{
"index": 296,
"name": "serde_derive",
"version": "1.0.137",
"source": "crates.io",
"dependencies": [256, 264, 331]
}
],
"direct_edges": [
{
"from_index": 294,
"to_index": 296
}
]
},
"parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb",
"artifact_sha256": "9a86e9ca47902a869fdf1886f4cf17714f4fa5655427128beadff498cfb392db",
"complete_result_sha256": "a9ce92f47aaa92fb0edc1c288d06c298ae623d731499fd4cedb86ab9c2bbb483",
"catalog_sha256": "411381fb2417945f8a29eb8db330f82b44d8790d3f1e88c7f766b532c528008f",
"catalog_tools": 212,
"token_proxy": {
"input_catalog_once": 197359,
"arguments": 58,
"text_replies": 26338,
"combined": 223755,
"only_two_selected_tools": 26850,
"complete_transport_catalog": 886882
}
},
"zellij-selected": {
"answer": {
"packages": [
{
"name": "serde",
"version": "1.0.137",
"source": "crates.io",
"dependencies": [296],
"index": 294
},
{
"name": "serde_derive",
"version": "1.0.137",
"source": "crates.io",
"dependencies": [256, 264, 331],
"index": 296
}
],
"direct_edges": [
{
"from_index": 294,
"to_index": 296
}
]
},
"parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb",
"artifact_sha256": "9a86e9ca47902a869fdf1886f4cf17714f4fa5655427128beadff498cfb392db",
"complete_result_sha256": "a9ce92f47aaa92fb0edc1c288d06c298ae623d731499fd4cedb86ab9c2bbb483",
"catalog_sha256": "ba99f10270e8ab53d765f124856921f59156482d603fa883240e1b4068eab921",
"catalog_tools": 212,
"token_proxy": {
"input_catalog_once": 198278,
"arguments": 83,
"text_replies": 1223,
"combined": 199584,
"only_two_selected_tools": 2256,
"complete_transport_catalog": 891294
}
},
"ripgrep-complete": {
"answer": {
"packages": [
{
"index": 36,
"name": "serde",
"version": "1.0.210",
"source": "crates.io",
"dependencies": []
}
],
"direct_edges": []
},
"parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4",
"artifact_sha256": "78387bbe12b2110df8d2ce364390a00c190f4543a2f40b8279e49f6c5c85af9c",
"complete_result_sha256": "9eb75332f423bcb94f627694cde96fb3f39ce7d47313468e027e8ccf03c38f72",
"catalog_sha256": "411381fb2417945f8a29eb8db330f82b44d8790d3f1e88c7f766b532c528008f",
"catalog_tools": 212,
"token_proxy": {
"input_catalog_once": 197359,
"arguments": 62,
"text_replies": 3182,
"combined": 200603,
"only_two_selected_tools": 3698,
"complete_transport_catalog": 886882
}
},
"ripgrep-selected": {
"answer": {
"packages": [
{
"name": "serde",
"version": "1.0.210",
"source": "crates.io",
"dependencies": [],
"index": 36
}
],
"direct_edges": []
},
"parent_evidence_id": "ev_be5662081d3c5b609a5e65a78c732674d2fcc160e5b1173ca0d083316673eaa4",
"artifact_sha256": "78387bbe12b2110df8d2ce364390a00c190f4543a2f40b8279e49f6c5c85af9c",
"complete_result_sha256": "9eb75332f423bcb94f627694cde96fb3f39ce7d47313468e027e8ccf03c38f72",
"catalog_sha256": "ba99f10270e8ab53d765f124856921f59156482d603fa883240e1b4068eab921",
"catalog_tools": 212,
"token_proxy": {
"input_catalog_once": 198278,
"arguments": 87,
"text_replies": 1219,
"combined": 199584,
"only_two_selected_tools": 2256,
"complete_transport_catalog": 891294
}
}
},
"accounting": "Kernel cgroup CPU/peak memory at payload exit while an observer remains alive, with verified exclusive observer and cleanup. Final whole-scope counters unavailable; observer overhead is included. All qualified samples have zero high/max/OOM/pid events.",
"published_equivalent_source": "cc83fd1ee3827861ea94d0e8f325e4521323bc89",
"integration_base": "a5f982de8dbe4b092cc20ab89d492b7e7028b758",
"candidate_reproduction": "Primary paired cohort uses candidate_code on baseline_source. Integration rebase production files are byte-identical to published_equivalent_source; original measured commit was pushed before rebase. Driver/counter/resource hashes and runtime tables are pinned separately."
}
experiments/analysis-efficiency/rust-dependency-views.md
+176
View File
@@ -0,0 +1,176 @@
# Rust dependency fact views
Issue [#5802](https://github.com/morluto/rea/issues/5802) asks for exact embedded
crate versions, sources and selected direct relationships without receiving the
entire dependency report. The existing path-only operation returned every
package, dependency vector and edge. This work adds opt-in first-call selection,
summary delivery and retained package/edge queries to the existing CLI/MCP
contracts. Complete delivery remains the default.
The producer still reads and validates the complete admitted file and report.
This is a context optimization; it does not establish faster binary analysis or
lower peak memory. It adds no tool, path cache, dependency walker or graph index.
Only successful interpretation of an authenticated immutable Evidence result is
reused, keyed by that exact object in a WeakMap for its lifetime. Mutable imports
are revalidated, and every view checks the artifact subject again.
## Pinned artifacts and independent checks
[Curated receipts](rust-dependency-views.json) pin source commits, runtime file
tables, source/driver/resource hashes, package URLs and all measured samples.
Baseline is `d672cc23998b6cbb21c485f118595ebfb82fed33`; measured candidate is
`cae2571e7e64a7364b94bb92c2630cba80ec4478`, pushed before the integration rebase.
The production files are byte-identical to published
`cc83fd1ee` after rebasing onto `a5f982de8`. Later integration measurements do
not relabel the original cohort.
Official Alpine v3.21/community/x86_64 APKs, not locally rebuilt test applications:
| Artifact | File bytes | Packages / edges | Complete CLI reply bytes |
| ---------------------------------- | ---------: | ---------------: | -----------------------: |
| ripgrep 14.1.1-r0, `usr/bin/rg` | 3,416,168 | 42 / 88 | 10,605 |
| Zellij 0.40.0-r0, `usr/bin/zellij` | 17,151,864 | 422 / 1,063 | 86,805 |
The [corpus manifest](corpus.json) records APK and extracted-file SHA-256,
transport size and aports revision. Acquisition selected only the named regular
tar member, with bounded download/extraction. Neither binary was executed.
HTTPS provenance and recorded digests are not an APK-signature verification.
GNU objcopy/readelf 2.40 independently extracted `.dep-v0` into private output
files without overwriting the input. Python 3.11.2 zlib/JSON parsing, following
the [upstream parsing format](https://github.com/rust-secure-code/cargo-auditable/blob/master/PARSING.md),
matched every exposed package field and dependency edge, omitted revision,
root index, compressed/inflated length, exact section range and full artifact
digest. An independent topological pass visited all 42/422 packages and matched
the empty cyclic sets. This qualifies these two ELF artifacts; PE/Mach-O view
coverage below uses constructed format fixtures, not real shipped applications.
## Equivalent completed SDK tasks
Linux x64, Node 24.18.0/npm 11.16.0. Three alternating before/after,
after/before, before/after pairs per artifact run serially. Each fresh production
stdio SDK session discovers all 212 tools and uses their named output schemas.
OS caches are unflushed; the controller hashes inputs first. These are fresh
process/session runs, not cold-disk runs.
The fixed question is: which versions and sources of `serde` and `serde_derive`
are embedded, and is there a direct edge between those selected packages?
Both lanes make exactly two RPCs: inspect once, then export complete Evidence.
Baseline derives the answer from the complete reply. Candidate selects both
names and `name/version/source/dependencies` in the initial inspection. There
are no extra lookup calls. All runs compare the answer and exported complete
parent against the independently checked report, including identical parent
Evidence IDs and source SHA-256. Zellij has the direct edge 294 → 296;
ripgrep has only `serde`, with no such edge within the embedded records.
Medians of three whole tasks, including startup, discovery, SDK validation,
both calls, answer/export checks and teardown:
| Artifact | Wall seconds, before → after | Family CPU seconds, before → after | Peak family MiB, before → after |
| -------- | ---------------------------: | ---------------------------------: | ------------------------------: |
| Zellij | 4.957 → 5.671 | 4.782 → 5.441 | 610.3 → 604.3 |
| ripgrep | 4.859 → 4.758 | 4.643 → 4.528 | 569.0 → 587.3 |
The main cohort's Zellij elapsed/CPU medians increase about 14%; do not read the
context improvement as a whole-task speedup. Two further forward/reverse
diagnostic controls added clocks around connect and discovery, comparing
baseline complete, candidate complete and candidate selected. In the first
control these took 5.460/6.164/6.063 seconds, with connect times
2.516/3.086/3.057 seconds. In the reverse control all took 5.360–5.364 seconds,
with connect times 2.528–2.679 seconds. Candidate complete also varied, so those
controls do not isolate a persistent regression to compact projection. They
do not erase the original slower cohort. No stable speedup, peak-memory
improvement or exact cause of startup variation is established.
Initial inspector RPC medians were 380.8 → 473.8 ms for Zellij and
189.4 → 244.2 ms for ripgrep. These include first-use named SDK schema validation,
retention, projection and serialization; they are not isolated parser timings.
Extra retained reads need no source reopen, as the CLI/MCP regressions demonstrate;
their warm latency was not benchmarked.
## Context accounting and limits
Offline tiktoken 0.12.0, `cl100k_base`, counts all discovered tool descriptions
and input schemas once, every call argument and every text reply, including
export. It counts the whole scripted task rather than only the selected reply:
| Artifact | All 212 input definitions + calls, before → after | Only the two used definitions + calls, before → after |
| -------- | ------------------------------------------------: | ----------------------------------------------------: |
| Zellij | 223,755 → 199,584 (−10.8%) | 26,850 → 2,256 (−91.6%) |
| ripgrep | 200,603 → 199,584 (−0.5%) | 3,698 → 2,256 (−39.0%) |
The Zellij inspect response, including both MCP text and structured content,
shrinks from 185,833 to 9,261 bytes; ripgrep shrinks from 22,643 to 9,207.
Those byte counts alone are not token measurements. Full transport catalogs,
including all output schemas, grow from 886,882 to 891,294 estimated tokens.
If all of that transport metadata is charged once along with calls, totals are
913,278 → 892,600 for Zellij and 890,126 → 892,600 for ripgrep: the small case
then costs more. Schema expansion has a cost even when Rust is not called.
No AI inference, actual model usage/billing, host schema injection, prompt-history
replays or analyst answer quality was measured. The two-definition column is a
discovery scenario, not an observed routed host. Do not extrapolate these
proxies to a host that repeats schemas or truncates full replies. Opt-in
delivery preserves caller choice and avoids forcing the small-case tradeoff.
## Correctness, resource bounds and rejected changes
Original-index facts, all matching versions, exact intersecting batch filters,
empty selections, selected fields and matching-row pagination share one domain
projector through the existing application layer. Package defaults omit
dependency vectors; callers explicitly request them for direct references.
Edges support both endpoint filters. Complete parent Evidence stays available
for export, and portable/retained queries still work after deleting the source.
Missing metadata remains unknown, cycles are counted without walking paths,
compiler hashes remain inferred, and embedded coverage never implies complete
runtime dependencies. Defaults preserve complete Evidence identity.
Executed tests cover real CLI/stdio SDK schemas, ELF/PE/thin-Mach-O fixtures,
duplicate package names/versions, retained and portable imports, malformed
edge imports, subject mismatch, unknown metadata, contradictory delivery,
retention requirements, bounded warnings and immutable-cache trust. Existing
Go, malformed Rust, cancellation, capacity and cleanup coverage is retained.
Full typecheck/lint/format/knip/module/architecture checks passed before the
integration rebase; final integration checks are recorded in the PR.
Runtime tasks use one pinned CPU, 4-GiB hard/3-GiB early-stop family limits,
no swap, 192 tasks and host-memory preflight/stop guards. All 12 primary and
six diagnostic samples have verified payload exit/cleanup and zero high/max/
OOM/pid events. CPU and peak memory use cgroup counters at payload exit while
the observer remains alive; final whole-scope counters are unavailable.
Build/check guards use 6/8-GiB hard bounds respectively. An existing 30-second
mixed Go/Rust test timed out in the initial single-CPU cohort; all nine CLI
acceptance tests passed with a bounded two-CPU/4-GiB rerun, without changing
the test's timeout. Other corrected prototype failures remain in the receipts.
A proposed JS JSON-size counter replacement had inconsistent provider timing
and no demonstrated memory benefit on these artifacts. It was reverted; the
final Rust provider is unchanged. Full-file input allocation, complete report
construction, ledger retention and schema duplication remain costs. This issue
does not qualify every Rust workflow, larger graphs, real PE/Mach-O applications,
WASM support or current runtime dependency state.
## Reproduction
Acquire the exact public APK URLs from the manifest and extract only the named
regular binary. Verify both SHA-256 values before inspection. Use objcopy
`--dump-section .dep-v0=SECTION INPUT PRIVATE_COPY`, then Python zlib/JSON to
compare the normalized package/edge records and section bytes. Save the verified
complete CLI `normalized_result` as `EXPECTED`.
Build isolated baseline/candidate runtimes with the pinned toolchain. Keep raw
outputs outside Git. For each artifact, alternate three serial pairs:
```bash
python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \
node experiments/analysis-efficiency/rust-dependency-views.mjs \
--runtime /absolute/runtime --artifact /absolute/binary \
--expected /absolute/expected-report.json --output /absolute/new-case \
--lane complete
# Candidate uses --lane selected. Both lanes include complete Evidence export.
```
With private `tiktoken==0.12.0`, run
`rust-dependency-costs.py --output FILE CASE...`; each `CASE.log` must contain
qualified guard observations. The counter rejects failed tasks, missing
cleanup and resource events. Guard-stopped attempts are not completed samples.
experiments/analysis-efficiency/rust-dependency-views.mjs
+172
View File
@@ -0,0 +1,172 @@
/** Equivalent selected dependency questions; raw artifacts/transcripts stay outside Git. */
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { createWriteStream } from "node:fs";
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { finished } from "node:stream/promises";
import { parseArgs } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { createRetainedQueryCaller } from "./retained-query-calls.mjs";
import { finalizeRetainedQueryReceipt } from "./retained-query-finalization.mjs";
const { values } = parseArgs({
options: {
...Object.fromEntries(
["runtime", "artifact", "expected", "output"].map((name) => [
name,
{ type: "string" },
]),
),
lane: { type: "string" },
},
});
for (const name of ["runtime", "artifact", "expected", "output"])
assert.ok(values[name] && values[name] === resolve(values[name]), name);
assert.ok(["complete", "selected"].includes(values.lane));
const hash = (value) => createHash("sha256").update(value).digest("hex");
const expected = JSON.parse(await readFile(values.expected, "utf8"));
assert.equal(expected.artifact.path, values.artifact);
const names = ["serde", "serde_derive"];
const fields = ["name", "version", "source", "dependencies"];
const choose = (report) =>
(report.auditable_metadata?.packages ?? []).flatMap((row, index) =>
names.includes(row.name)
? [
{
index,
...Object.fromEntries(fields.map((field) => [field, row[field]])),
},
]
: [],
);
const answer = (rows) => {
const targets = new Set(
rows.filter((row) => row.name === "serde_derive").map((row) => row.index),
);
return {
packages: rows,
direct_edges: rows
.filter((row) => row.name === "serde")
.flatMap((row) =>
row.dependencies
.filter((index) => targets.has(index))
.map((to_index) => ({ from_index: row.index, to_index })),
),
};
};
const expectedRows = choose(expected);
await mkdir(values.output, { mode: 0o700 });
const account = await mkdtemp(join(tmpdir(), "rea-rust-views-"));
const client = new Client({ name: "rust-dependency-views", version: "1" });
const transport = new StdioClientTransport({
command: process.execPath,
args: [join(values.runtime, "scripts/rea.mjs"), "mcp"],
env: {
HOME: account,
PATH: process.env.PATH,
NODE_OPTIONS: "--max-old-space-size=2048",
UV_THREADPOOL_SIZE: "1",
REA_LOG_LEVEL: "silent",
TMPDIR: account,
},
stderr: "pipe",
});
const serverLog = createWriteStream(join(values.output, "server.log"));
const serverLogFinished = finished(serverLog);
serverLogFinished.catch(() => {});
transport.stderr?.pipe(serverLog);
const report = {
lane: values.lane,
artifact_sha256: hash(await readFile(values.artifact)),
normalized_sha256: hash(JSON.stringify(expected)),
calls: [],
};
const transcript = [];
const call = createRetainedQueryCaller(client, (receipt, event) => {
report.calls.push(receipt);
transcript.push(event);
});
const observe = async (name, input, phase) => {
const reply = await call(name, input, phase);
assert.notEqual(reply.isError, true, JSON.stringify(reply));
assert.ok(reply.structuredContent);
return reply.structuredContent;
};
try {
await client.connect(transport);
const catalog = await client.listTools();
assert.equal(catalog.nextCursor, undefined);
report.catalog_tools = catalog.tools.length;
report.catalog_sha256 = hash(JSON.stringify(catalog));
await writeFile(join(values.output, "catalog.json"), JSON.stringify(catalog));
const definitions = new Map(catalog.tools.map((tool) => [tool.name, tool]));
const original = client.callTool.bind(client);
client.callTool = (input, options) =>
original(input, {
...options,
toolDefinition: definitions.get(input.name),
});
const selected = values.lane === "selected";
const reply = await observe(
"inspect_rust_binary",
{
path: values.artifact,
...(selected ? { view: { collection: "packages", names, fields } } : {}),
},
"question",
);
const parentId = selected
? reply.normalized_result.parent_evidence_id
: reply.evidence_id;
const rows = selected
? reply.normalized_result.page.rows
: choose(reply.normalized_result);
assert.deepEqual(rows, expectedRows);
report.answer = answer(rows);
assert.deepEqual(report.answer, answer(expectedRows));
report.parent_evidence_id = parentId;
await observe(
"export_evidence_bundle",
{ path: join(values.output, "evidence.json") },
"export",
);
const bundle = JSON.parse(
await readFile(join(values.output, "evidence.json"), "utf8"),
);
const parent = bundle.records.find(
(record) => record.evidence_id === parentId,
);
assert.ok(parent);
assert.deepEqual(parent.normalized_result, expected);
assert.equal(parent.subject.digest.sha256, report.artifact_sha256);
if (selected)
assert.deepEqual(reply.normalized_result.source_coverage, {
auditable_metadata_available: expected.auditable_metadata !== null,
embedded_records:
expected.auditable_metadata === null ? "unknown" : "complete",
runtime_dependencies: "unknown",
});
assert.equal(hash(await readFile(values.artifact)), report.artifact_sha256);
report.status = "passed";
} catch (cause) {
report.status = "failed";
report.failure = String(cause?.stack ?? cause);
process.exitCode = 1;
} finally {
if (
!(await finalizeRetainedQueryReceipt({
report,
client,
transport,
account,
serverLog,
serverLogFinished,
output: values.output,
transcript,
}))
)
process.exitCode = 1;
}
experiments/analysis-efficiency/test_model_utf8_validation.py
+68
View File
@@ -0,0 +1,68 @@
"""Phase receipt fault injection only; these stand-ins do not qualify a model provider."""
import json
from pathlib import Path
import subprocess
import tempfile
import unittest
class ModelPhaseReceipts(unittest.TestCase):
def run_phase(self, inspect_body, close_body):
driver = Path(__file__).with_name('model-utf8-validation.mjs').resolve()
with tempfile.TemporaryDirectory(prefix='rea-model-phase-test-') as directory:
root = Path(directory)
artifact = root / 'input'
artifact.write_bytes(b'receipt fixture, not a model')
expected = {'artifact': {'path': str(artifact)}}
(root / 'expected.json').write_text(json.dumps(expected))
(root / 'package.json').write_text('{"type":"module"}')
module = root / 'dist' / 'model'
module.mkdir(parents=True)
(module / 'ModelArtifactProvider.js').write_text(
'export class ModelArtifactProvider {\n'
'async inspect() {' + inspect_body.replace('EXPECTED', json.dumps(expected)) + '}\n'
'async close() {' + close_body + '}\n}\n')
result = subprocess.run([
'node', str(driver), '--phase', '--runtime', str(root),
'--artifact', str(artifact), '--expected', str(root / 'expected.json'),
'--output', str(root / 'output'),
], capture_output=True, text=True, timeout=15)
self.assertEqual(result.returncode, 1, result.stderr)
report = json.loads((root / 'output' / 'report.json').read_text())
self.assertEqual(json.loads(result.stdout), report)
self.assertEqual(report['status'], 'failed')
self.assertEqual(report['receipt_failures'], [])
return report
def test_rejected_inspection_keeps_cost_and_primary_failure_after_close_failure(self):
report = self.run_phase(
'throw new Error("inspection failed");',
'throw new Error("cleanup failed");')
self.assertEqual(len(report['calls']), 1)
sample = report['calls'][0]
self.assertEqual(sample['phase'], 'fresh')
self.assertTrue(sample['is_error'])
self.assertGreaterEqual(sample['elapsed_ms'], 0)
self.assertIn('inspection failed', sample['failure'])
self.assertIn('inspection failed', report['failure'])
self.assertEqual(report['cleanup_failures'], ['provider: Error: cleanup failed'])
def test_mismatch_keeps_inspection_time_and_failure(self):
report = self.run_phase('return {ok:true,value:{wrong:true}};', '')
self.assertEqual(len(report['calls']), 1)
self.assertTrue(report['calls'][0]['is_error'])
self.assertGreaterEqual(report['calls'][0]['elapsed_ms'], 0)
self.assertIn('AssertionError', report['failure'])
self.assertEqual(report['cleanup_failures'], [])
def test_close_failure_keeps_all_successful_samples_and_publishes_failed_receipt(self):
report = self.run_phase('return {ok:true,value:EXPECTED};',
'throw new Error("cleanup failed");')
self.assertEqual(len(report['calls']), 7)
self.assertTrue(all(not sample['is_error'] for sample in report['calls']))
self.assertNotIn('failure', report)
self.assertEqual(report['cleanup_failures'], ['provider: Error: cleanup failed'])
if __name__ == '__main__':
unittest.main()
experiments/analysis-efficiency/wasm-string-framing.json
+561
View File
@@ -0,0 +1,561 @@
{
"schema_version": 1,
"issue": 5760,
"baseline_source": "d0067ba34a39ea47dd7e11bb2e6169bc0938cc3f",
"candidate_code_source": "9776d78f6c799cba8d8cf5c9b3139a86449eaebd",
"integration_base": "335c042d166b2c8b2c0600935cf77607ae11edbc",
"toolchain": {
"node": "24.18.0",
"npm": "11.16.0",
"wabt": "1.0.42",
"tokenizer": "0.12.0",
"encoding": "cl100k_base"
},
"catalog_sha256": "4586abd317171cbb307295cc687c24f7f45e603959538643325895b546a38242",
"catalog_tools": 212,
"package_lock_sha256": "699a14749e3825c62430faeb18224190fee1c9347760494e38e334e180bea6ec",
"runtime_mcp_sha256": {
"baseline": "fa6831f05d97b6b28642b709b5497e351d2ee632dd1abc0d2f87a648057afe87",
"candidate": "e4af82d847fe3d9816f4f7ae1f584ef129ba151bc5a6f95080a7519fd08be2cf"
},
"sources_sha256": {
"WabtString.ts": "398f3cd98c087740a553768d10c21912d28f33e13bdab589eec760da4ed0b9fd",
"WabtWat.ts": "70fb9acfbe747be4fa5d8be235012a88f94485bc0baf0fbba1fe22f18b87f44b",
"WabtInterface.ts": "1a63c834a46268bbaf97e6eb39a99b3ccd3f2893527749d65ee6f5026bc2782a"
},
"native_tools_sha256": {
"wasm-validate": "6b722ed0df68204cfc7a8effdf1fc2ddeac2d184bd211cdf7fdfa5f9d49d3eb5",
"wasm-objdump": "dad264def1a3b7755859c5bf48227f01259a95fa82b7c38358b12b9e70b27a41",
"wasm2wat": "4322d652f650c1ee117b549c9722f42fc93eb4676ff046d60ba033b5639697a5"
},
"artifacts": {
"small": {
"sha256": "f399acbd52258f950f24dd5ca4a82674f1f26cea60e7be7c5daaade568768fda",
"bytes": 80,
"wat_sha256": "918ecd8a14daf20b58707d0879efda8b38df37f219279688b3aa3a812ae5646a",
"wat_bytes": 270,
"answer_normalized_sha256": "8eb8aedcf17bac4a98d34f5a004b5541f671350f94918575e2fc4eee9fd2db29",
"parent_evidence_id": "ev_88b35d9f926672b9e42fc4e1514090eaf2f99bb918439f1270dd53f7fe684e74"
},
"sqlite": {
"sha256": "2ee8f3dab694532afc8840e07703127287662d08b74e6ff50491ce63f00d5752",
"bytes": 868907,
"wat_sha256": "7b1f8cd4fcbf310f4203ce64e15b4eb068ecdf29838551137245efecb7283f81",
"wat_bytes": 15382128,
"answer_normalized_sha256": "330b84e01541f70a964c826754da3e5f49a1bb1bd535ae1d1f00fb44a1157ff6",
"parent_evidence_id": "ev_899ecb126cd0a283169641085fa10e46460504bf8bc8eba69580fa6d641e431e"
},
"css": {
"sha256": "8a23977fe271357cce6f254ef88c9bebf3854602d8046605aef6a45c02135c59",
"bytes": 128668,
"wat_sha256": "c53273cb7da0867ac52490d083c7952fcc538b3848c6d0d1f2e2621e2ded23c0",
"wat_bytes": 13599679,
"answer_normalized_sha256": "3d301feaa48a646c4d9f2a026353544519cc4694c9840b1c764d81ec9ba7fb65",
"parent_evidence_id": "ev_27108db83ae4d0a9289c23dace9c0e9e6d573f72f1e982b490c7f45394ba702c"
}
},
"samples": [
{
"case": "small",
"lane": "baseline",
"round": 1,
"wall_seconds": 12.417,
"cpu_seconds": 12.114979,
"memory_peak_bytes": 823410688,
"transport_proxy_tokens": 894936,
"fresh_inspect_ms": 1781.0228269999998,
"repeat_inspect_ms": 1793.3802260000011,
"reply_bytes": 36180,
"raw_guard_log_sha256": "f4ab763c21c4dd22f07ab978690966fd1efc6961216e41e195316e7330c269d6"
},
{
"case": "small",
"lane": "candidate",
"round": 1,
"wall_seconds": 10.191,
"cpu_seconds": 9.868799,
"memory_peak_bytes": 829722624,
"transport_proxy_tokens": 894936,
"fresh_inspect_ms": 1298.6234909999998,
"repeat_inspect_ms": 1917.7533390000008,
"reply_bytes": 36182,
"raw_guard_log_sha256": "5cfd93018a2cabbe5b9fa4b9062fbed8e6da638ad5672fd566f5cf76182fecd5"
},
{
"case": "sqlite",
"lane": "baseline",
"round": 1,
"wall_seconds": 14.317,
"cpu_seconds": 13.845753,
"memory_peak_bytes": 1032982528,
"transport_proxy_tokens": 895044,
"fresh_inspect_ms": 3405.006531,
"repeat_inspect_ms": 3345.840587999999,
"reply_bytes": 37248,
"raw_guard_log_sha256": "46b1b2e9f02e41be7a7cbae8a311b799ab52148a150ec5c0c683e0745c44ad03"
},
{
"case": "sqlite",
"lane": "candidate",
"round": 1,
"wall_seconds": 13.311,
"cpu_seconds": 12.956772,
"memory_peak_bytes": 1029627904,
"transport_proxy_tokens": 895044,
"fresh_inspect_ms": 2838.048675,
"repeat_inspect_ms": 3233.226218,
"reply_bytes": 37250,
"raw_guard_log_sha256": "8933144b3f6fd100de72b1c1f22a791b7c4bef74c355bb9211be4c9e9383fe4b"
},
{
"case": "css",
"lane": "baseline",
"round": 1,
"wall_seconds": 13.211,
"cpu_seconds": 12.79919,
"memory_peak_bytes": 1099599872,
"transport_proxy_tokens": 895430,
"fresh_inspect_ms": 2823.246057,
"repeat_inspect_ms": 3302.541784000001,
"reply_bytes": 37426,
"raw_guard_log_sha256": "9f4b7dfcd97faf8c85463a4c17f0612b6c37b7e29843f5e52b43df27aaae0ac6"
},
{
"case": "css",
"lane": "candidate",
"round": 1,
"wall_seconds": 12.912,
"cpu_seconds": 12.46982,
"memory_peak_bytes": 1110261760,
"transport_proxy_tokens": 895430,
"fresh_inspect_ms": 2839.796418,
"repeat_inspect_ms": 3002.9903570000006,
"reply_bytes": 37428,
"raw_guard_log_sha256": "dc2ed426cd1191ca779a4e736bceb3d0001a5b8344dbba01544ecd91a6bb73b6"
},
{
"case": "small",
"lane": "candidate",
"round": 2,
"wall_seconds": 10.295,
"cpu_seconds": 9.986392,
"memory_peak_bytes": 807264256,
"transport_proxy_tokens": 894936,
"fresh_inspect_ms": 1687.949067,
"repeat_inspect_ms": 1907.0628659999993,
"reply_bytes": 36182,
"raw_guard_log_sha256": "376199e896af92b6351d2dde7c765b5aa4b1feac0e9664da934e24732eaa22ec"
},
{
"case": "small",
"lane": "baseline",
"round": 2,
"wall_seconds": 10.192,
"cpu_seconds": 9.86319,
"memory_peak_bytes": 826806272,
"transport_proxy_tokens": 894936,
"fresh_inspect_ms": 1621.7653269999996,
"repeat_inspect_ms": 1937.1286520000012,
"reply_bytes": 36180,
"raw_guard_log_sha256": "2d40d7309203eefcb7cbb5a5e81640e728839eb30f29cad017584738ea50616f"
},
{
"case": "sqlite",
"lane": "candidate",
"round": 2,
"wall_seconds": 13.52,
"cpu_seconds": 13.133355,
"memory_peak_bytes": 1211944960,
"transport_proxy_tokens": 895044,
"fresh_inspect_ms": 3167.13338,
"repeat_inspect_ms": 3425.1366179999986,
"reply_bytes": 37250,
"raw_guard_log_sha256": "a962200a36a2530b044548b475b28d7234f73b875ffbc3fe16ef4b36e61ae49a"
},
{
"case": "sqlite",
"lane": "baseline",
"round": 2,
"wall_seconds": 13.515,
"cpu_seconds": 13.170183,
"memory_peak_bytes": 1017561088,
"transport_proxy_tokens": 895044,
"fresh_inspect_ms": 3224.859665,
"repeat_inspect_ms": 3230.52117,
"reply_bytes": 37248,
"raw_guard_log_sha256": "0452dae8c26a023bd6220bca03dfd7a58425ef579f8e7964f04b4cb944723f84"
},
{
"case": "css",
"lane": "candidate",
"round": 2,
"wall_seconds": 12.112,
"cpu_seconds": 11.727334,
"memory_peak_bytes": 1186643968,
"transport_proxy_tokens": 895430,
"fresh_inspect_ms": 2662.82945,
"repeat_inspect_ms": 2489.199032999999,
"reply_bytes": 37428,
"raw_guard_log_sha256": "9d27bfa9720f4291324aa6fc47e44ffdee8b6fbc63ea29a2c82d297d0ba1f0c6"
},
{
"case": "css",
"lane": "baseline",
"round": 2,
"wall_seconds": 12.409,
"cpu_seconds": 12.038676,
"memory_peak_bytes": 1079783424,
"transport_proxy_tokens": 895430,
"fresh_inspect_ms": 2793.9973870000003,
"repeat_inspect_ms": 2541.021634999999,
"reply_bytes": 37426,
"raw_guard_log_sha256": "ddc43e4517a2ed634d5a969104b9051ac4e654485d3552f19d4c141a5834b205"
},
{
"case": "small",
"lane": "baseline",
"round": 3,
"wall_seconds": 9.797,
"cpu_seconds": 9.449418,
"memory_peak_bytes": 822263808,
"transport_proxy_tokens": 894936,
"fresh_inspect_ms": 1612.1414539999996,
"repeat_inspect_ms": 1833.922163000001,
"reply_bytes": 36180,
"raw_guard_log_sha256": "92aa46fe57caeae60cb3105b2ac370a8f8463afed4a7f666f46be885cad0ca4d"
},
{
"case": "small",
"lane": "candidate",
"round": 3,
"wall_seconds": 10.192,
"cpu_seconds": 9.868713,
"memory_peak_bytes": 829575168,
"transport_proxy_tokens": 894936,
"fresh_inspect_ms": 1629.0064629999997,
"repeat_inspect_ms": 1962.5034050000004,
"reply_bytes": 36182,
"raw_guard_log_sha256": "0451dacd244ced0ac56ceb0f73b39e1b86fa1517a0ec88391a3ab37d45b9bd20"
},
{
"case": "sqlite",
"lane": "baseline",
"round": 3,
"wall_seconds": 13.714,
"cpu_seconds": 13.319143,
"memory_peak_bytes": 1196531712,
"transport_proxy_tokens": 895044,
"fresh_inspect_ms": 3228.7134460000007,
"repeat_inspect_ms": 3397.271211000001,
"reply_bytes": 37248,
"raw_guard_log_sha256": "d3f7e1e572a4c3251ca479a3c018d2f024b80103e443a8daeb125bd896165d2a"
},
{
"case": "sqlite",
"lane": "candidate",
"round": 3,
"wall_seconds": 12.709,
"cpu_seconds": 12.349818,
"memory_peak_bytes": 1205280768,
"transport_proxy_tokens": 895044,
"fresh_inspect_ms": 3071.7333470000003,
"repeat_inspect_ms": 3024.3819120000007,
"reply_bytes": 37250,
"raw_guard_log_sha256": "57b405e4a4e8a9acbb4d9e4bb4663d931ad7ffec52a2b6a2969a7362287db892"
},
{
"case": "css",
"lane": "baseline",
"round": 3,
"wall_seconds": 12.315,
"cpu_seconds": 11.921752,
"memory_peak_bytes": 1081831424,
"transport_proxy_tokens": 895430,
"fresh_inspect_ms": 2798.227023,
"repeat_inspect_ms": 2563.7297770000005,
"reply_bytes": 37426,
"raw_guard_log_sha256": "2f6fca011f6718111b455d8104121ff6880131f58e0a82df069efef9445f5c07"
},
{
"case": "css",
"lane": "candidate",
"round": 3,
"wall_seconds": 11.901,
"cpu_seconds": 11.54646,
"memory_peak_bytes": 1123659776,
"transport_proxy_tokens": 895430,
"fresh_inspect_ms": 2600.1125399999996,
"repeat_inspect_ms": 2398.082381,
"reply_bytes": 37428,
"raw_guard_log_sha256": "180726dbb90af48c442109f75ccc50bf243ebcc8392908f9322eb91100a5f179"
}
],
"parser_samples": [
{
"case": "small.decoded",
"lane": "baseline",
"round": 1,
"elapsed_ms": [
2.9900399999999934, 1.9184900000000056, 0.3164599999999922,
0.622399999999999, 0.6818110000000104, 1.7410099999999886
],
"result_sha256": "beb5c7e4e8da08e3c104753b768c8ca40559d517dba80c483253991721b39c7f",
"wall_seconds": 0.422,
"cpu_seconds": 0.169929,
"memory_peak_bytes": 28078080
},
{
"case": "small.decoded",
"lane": "candidate",
"round": 1,
"elapsed_ms": [
3.8333299999999895, 1.5887500000000045, 0.4467700000000008,
0.3875499999999903, 0.3238100000000088, 0.21590999999999383
],
"result_sha256": "beb5c7e4e8da08e3c104753b768c8ca40559d517dba80c483253991721b39c7f",
"wall_seconds": 0.323,
"cpu_seconds": 0.154998,
"memory_peak_bytes": 27508736
},
{
"case": "sqlite",
"lane": "baseline",
"round": 1,
"elapsed_ms": [
248.049268, 224.29203599999994, 176.49019599999997, 155.392424,
162.08057399999996, 172.70000600000003
],
"result_sha256": "74eb7108ddd459dfa7bfb7a1fe653c8862eb43acd89d91ae033e7ac9158587ce",
"wall_seconds": 1.532,
"cpu_seconds": 1.326985,
"memory_peak_bytes": 71315456
},
{
"case": "sqlite",
"lane": "candidate",
"round": 1,
"elapsed_ms": [
103.56140300000001, 103.344674, 56.359200999999985, 43.188242,
44.79720200000003, 44.11639200000002
],
"result_sha256": "74eb7108ddd459dfa7bfb7a1fe653c8862eb43acd89d91ae033e7ac9158587ce",
"wall_seconds": 0.827,
"cpu_seconds": 0.598442,
"memory_peak_bytes": 71557120
},
{
"case": "tree-sitter-css",
"lane": "baseline",
"round": 1,
"elapsed_ms": [
133.568665, 122.39741400000003, 110.63244400000002, 111.27637299999998,
111.761034, 111.34842400000002
],
"result_sha256": "b59e5bdc32939c215443da6f3ce48e61fe2bfd86ff75b7a85d3c10a93b6fae7e",
"wall_seconds": 1.127,
"cpu_seconds": 0.882723,
"memory_peak_bytes": 55656448
},
{
"case": "tree-sitter-css",
"lane": "candidate",
"round": 1,
"elapsed_ms": [
15.454230999999993, 15.1781, 17.94840099999999, 15.499510999999984,
15.028279999999995, 2.9674699999999916
],
"result_sha256": "b59e5bdc32939c215443da6f3ce48e61fe2bfd86ff75b7a85d3c10a93b6fae7e",
"wall_seconds": 0.424,
"cpu_seconds": 0.255557,
"memory_peak_bytes": 54427648
},
{
"case": "small.decoded",
"lane": "candidate",
"round": 2,
"elapsed_ms": [
2.492229000000009, 1.6617090000000019, 0.4688189999999963,
0.40376000000000545, 0.18850899999999626, 0.19684000000000879
],
"result_sha256": "beb5c7e4e8da08e3c104753b768c8ca40559d517dba80c483253991721b39c7f",
"wall_seconds": 0.324,
"cpu_seconds": 0.14932,
"memory_peak_bytes": 27750400
},
{
"case": "small.decoded",
"lane": "baseline",
"round": 2,
"elapsed_ms": [
3.0103999999999758, 2.4378910000000076, 0.30259000000000924,
0.5248499999999865, 0.6185799999999801, 1.3529399999999896
],
"result_sha256": "beb5c7e4e8da08e3c104753b768c8ca40559d517dba80c483253991721b39c7f",
"wall_seconds": 0.428,
"cpu_seconds": 0.219561,
"memory_peak_bytes": 30007296
},
{
"case": "sqlite",
"lane": "candidate",
"round": 2,
"elapsed_ms": [
150.102906, 136.73758599999996, 85.33841300000006, 42.61153200000001,
45.540752, 43.07188099999996
],
"result_sha256": "74eb7108ddd459dfa7bfb7a1fe653c8862eb43acd89d91ae033e7ac9158587ce",
"wall_seconds": 0.959,
"cpu_seconds": 0.738843,
"memory_peak_bytes": 68399104
},
{
"case": "sqlite",
"lane": "baseline",
"round": 2,
"elapsed_ms": [
236.412329, 219.61998900000003, 163.37286700000004, 152.177506,
153.764226, 153.57296699999984
],
"result_sha256": "74eb7108ddd459dfa7bfb7a1fe653c8862eb43acd89d91ae033e7ac9158587ce",
"wall_seconds": 1.532,
"cpu_seconds": 1.317278,
"memory_peak_bytes": 70189056
},
{
"case": "tree-sitter-css",
"lane": "candidate",
"round": 2,
"elapsed_ms": [
15.074011000000013, 22.501490000000018, 23.890242, 9.095690999999988,
25.85492100000002, 16.470331000000016
],
"result_sha256": "b59e5bdc32939c215443da6f3ce48e61fe2bfd86ff75b7a85d3c10a93b6fae7e",
"wall_seconds": 0.524,
"cpu_seconds": 0.297941,
"memory_peak_bytes": 53497856
},
{
"case": "tree-sitter-css",
"lane": "baseline",
"round": 2,
"elapsed_ms": [
135.69754600000002, 128.91107599999998, 115.180316, 115.23454600000002,
117.18869600000005, 103.74374499999999
],
"result_sha256": "b59e5bdc32939c215443da6f3ce48e61fe2bfd86ff75b7a85d3c10a93b6fae7e",
"wall_seconds": 1.13,
"cpu_seconds": 0.916205,
"memory_peak_bytes": 57880576
},
{
"case": "small.decoded",
"lane": "baseline",
"round": 3,
"elapsed_ms": [
2.7721499999999963, 2.0805500000000023, 0.2946500000000043,
0.4885200000000083, 0.37317999999999074, 1.052209999999988
],
"result_sha256": "beb5c7e4e8da08e3c104753b768c8ca40559d517dba80c483253991721b39c7f",
"wall_seconds": 0.321,
"cpu_seconds": 0.161694,
"memory_peak_bytes": 28037120
},
{
"case": "small.decoded",
"lane": "candidate",
"round": 3,
"elapsed_ms": [
2.4845909999999947, 1.5559899999999942, 1.9349599999999896,
0.6082500000000124, 0.23478000000000065, 0.22989100000000917
],
"result_sha256": "beb5c7e4e8da08e3c104753b768c8ca40559d517dba80c483253991721b39c7f",
"wall_seconds": 0.321,
"cpu_seconds": 0.154815,
"memory_peak_bytes": 27426816
},
{
"case": "sqlite",
"lane": "baseline",
"round": 3,
"elapsed_ms": [
219.92179099999998, 204.47974999999997, 159.57980799999996,
152.3218579999999, 150.58105799999998, 149.3406970000001
],
"result_sha256": "74eb7108ddd459dfa7bfb7a1fe653c8862eb43acd89d91ae033e7ac9158587ce",
"wall_seconds": 1.428,
"cpu_seconds": 1.22534,
"memory_peak_bytes": 71987200
},
{
"case": "sqlite",
"lane": "candidate",
"round": 3,
"elapsed_ms": [
103.150995, 103.714395, 58.592934000000014, 47.94820199999998,
47.04763299999996, 47.239012
],
"result_sha256": "74eb7108ddd459dfa7bfb7a1fe653c8862eb43acd89d91ae033e7ac9158587ce",
"wall_seconds": 0.828,
"cpu_seconds": 0.60423,
"memory_peak_bytes": 72187904
},
{
"case": "tree-sitter-css",
"lane": "baseline",
"round": 3,
"elapsed_ms": [
126.407417, 121.22317700000002, 109.654766, 108.04800599999999,
109.459836, 100.50628600000005
],
"result_sha256": "b59e5bdc32939c215443da6f3ce48e61fe2bfd86ff75b7a85d3c10a93b6fae7e",
"wall_seconds": 1.029,
"cpu_seconds": 0.856826,
"memory_peak_bytes": 58101760
},
{
"case": "tree-sitter-css",
"lane": "candidate",
"round": 3,
"elapsed_ms": [
14.889070999999987, 15.635390999999998, 17.013621, 15.000391000000008,
17.56236100000001, 2.893010000000004
],
"result_sha256": "b59e5bdc32939c215443da6f3ce48e61fe2bfd86ff75b7a85d3c10a93b6fae7e",
"wall_seconds": 0.422,
"cpu_seconds": 0.269288,
"memory_peak_bytes": 57966592
}
],
"limits": {
"cpu_affinity": [15],
"cpu_quota_percent": 100,
"hard_memory_bytes": 4294967296,
"early_stop_bytes": 3221225472,
"swap_bytes": 0,
"task_ceiling": 192
},
"limitations": [
"Fresh server sessions; OS file caches warm/unflushed. Parser samples use one fresh and five repeated in-memory parses per process.",
"SDK transport proxy counts one complete catalog plus five named request/reply envelopes. Initialization/framing and actual model usage are unmeasured. No token saving is claimed.",
"Memory is kernel process-family memory.peak including file cache; final counters after transient scope removal remain unknown.",
"First string-only 18-task development cohort, one invalid-source-kind harness attempt, and the cache-hit build with unverified payload cleanup are preserved outside Git and excluded from final benefit claims.",
"Independent native WebAssembly.Module import/export oracle compiles without instantiating or executing target code. It does not prove every typed signature or unknown future proposal."
],
"resource_qualification": {
"completed_mcp_tasks": 18,
"completed_parser_processes": 18,
"all_payload_exit_codes_zero": true,
"all_payload_cleanup_verified": true,
"all_payload_counters_verified": true,
"all_memory_high_hard_oom_and_task_limit_events_zero": true,
"final_whole_scope_counters": "unknown"
},
"call_sequence": [
"inspect_wasm_artifact(summary):fresh",
"inspect_analysis_view(exports,first4)",
"inspect_wasm_artifact(summary):repeat",
"inspect_analysis_view(exports,first4)",
"export_evidence_bundle"
],
"transport_proxy_scope": "One complete SDK catalog and all five named request/reply envelopes, compact UTF-8 JSON encoded with cl100k_base; not actual model input."
}
experiments/analysis-efficiency/wasm-string-framing.md
+107
View File
@@ -0,0 +1,107 @@
# WAT framing costs on shipped WASM
[Issue #5760](https://github.com/morluto/rea/issues/5760) follows the current
WASM summary workflow; no new tool, selector, cache or default is needed.
The [curated measurements](wasm-string-framing.json) pin source, runtime,
artifact, producer and answer fingerprints, including every final sample.
## Owner and preserved behavior
WABT's WAT framing scanned instruction text character by character. Interface
tokenization also decoded quoted bytes merely to obtain an end position.
`WabtString` now owns a shared quoted-string boundary operation, preserving
escaped-quote/backslash parity. The top-level framer searches for structural
ASCII delimiters and skips string payloads. Name and custom-payload consumers
still decode their bytes; unsupported typed signatures remain unknown.
Nested/line comments, exact source forms and full retained WAT remain available.
Data forms were already excluded from interface parsing; this does not add a
new data filter or reinterpret raw WASM outside WABT.
Baseline source is `d0067ba34a39`; candidate code is
`9776d78f6c799cba8d8cf5c9b3139a86449eaebd`, based on latest main
`335c042d166b` (the intervening main change is testing documentation).
Linux x64, Node 24.18.0/npm 11.16.0, native WABT 1.0.42. Three alternating
baseline/candidate, candidate/baseline, baseline/candidate pairs ran serially.
Each task starts a fresh production stdio MCP server, discovers all 212 tools,
inspects the selected artifact twice with existing summary delivery, reads the
first four exact exports after each inspection, exports complete Evidence and
closes. OS file caches are warm/unflushed; no cold-disk claim is made.
The public SQLite Wasm 3.53.4-build1 and VS Code tree-sitter-wasm 0.3.1 CSS
components match the pinned [corpus](corpus.json) archive and member digests.
An 80-byte source-assembled module supplies the small control, with an escaped
import name. Native `WebAssembly.Module` import/export metadata independently
checks all returned names/kinds without instantiating or executing target code.
A separate native `wasm2wat --enable-all` output checks exact full WAT and its
SHA-256. Every before/after original Evidence ID, normalized report digest,
source byte digest, selected export and catalog digest agrees. This does not
independently prove every typed signature or future WASM proposal.
## Completed-task costs
Medians of three complete tasks per side; kernel process-family memory includes
file cache. Wall time includes startup, discovery, all five investigation RPCs,
client validation, independent checks, export and transport cleanup.
| Artifact | Wall seconds, before → after | CPU seconds, before → after | Peak MiB, before → after |
| ------------- | ---------------------------: | --------------------------: | -----------------------: |
| Small control | 10.192 → 10.192 | 9.863 → 9.869 | 785.3 → 791.1 |
| SQLite | 13.714 → 13.311 | 13.319 → 12.957 | 985.1 → 1149.4 |
| CSS grammar | 12.409 → 12.112 | 12.039 → 11.727 | 1031.7 → 1071.6 |
Whole-task median wall reductions are about 2.9%/2.4% for SQLite/CSS; small-task
time is unchanged. Three pairs are descriptive observations, not statistical
or cross-provider guarantees. Memory does not improve: SQLite's median is
higher, with overlapping 970–1141 versus 982–1156 MiB ranges. No heap-retention
cause or memory-saving claim is established.
Separate fresh-process parser cohorts use one first parse and five repeated
in-memory parses, checking the complete parsed-result digest after each parse.
Median first/warm parse milliseconds: SQLite 236.41/153.76 → 103.56/45.54;
CSS 133.57/111.35 → 15.07/15.64; small 2.99/0.62 → 2.49/0.40. These phase
improvements do not include native WABT production or complete task costs.
The unchanged catalog and all five named request/reply envelopes cost median
894,936 / 895,044 / 895,430 `cl100k_base` tokens for small/SQLite/CSS in both
implementations. This is a single serialized SDK transport proxy using
tiktoken 0.12.0; actual model input/usage, repeated model namespaces and protocol
initialization/framing are unmeasured. No token saving is claimed.
All 18 final MCP tasks and 18 parser processes completed under one CPU,
4 GiB hard/3 GiB early-stop memory, no swap and 192 tasks, with payload-exit
counters and cleanup verified and no limit/OOM events. Final whole-scope counters
after transient scope removal remain unknown. Raw binaries, exports,
transcripts and logs stay outside Git. The earlier string-only 18-task
cohort, an invalid source-kind harness attempt and a cache-hit build with
unverified payload cleanup are retained and excluded from final benefit claims.
## Reproduction
Build/freeze each runtime with the pinned toolchain; keep the selected artifact,
WABT directory and independently produced WAT paths identical across lanes:
```sh
python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \
node experiments/analysis-efficiency/wasm-string-framing.mjs \
--runtime /absolute/runtime --artifact /absolute/input.wasm \
--wat /absolute/native-output.wat --tools /absolute/wabt/bin \
--output /absolute/new-output-directory
```
For parser-only cohorts, import `dist/wasm/WabtWat.js` from each frozen runtime,
load the same native WAT once, and call `parseWabtWat(text)` six times. Measure
each call with `performance.now()`; outside that timer, hash
`JSON.stringify(result)` and require the same digest for every call and lane.
Keep each artifact in a separate fresh guarded process.
Source verification passed 76 tests across nine files, including malformed
strings, slash parity, comments, invalid names, capacity/partial facts,
cancellation, cleanup retry and shared application behavior. `npm run check`
passed typecheck, lint, formatting, Knip, metadata and architecture guards.
Four compiled CLI/MCP/acceptance files passed another 14 tests. The native WABT
artifact verifier passed 51 CLI/MCP cases; the native interface verifier passed
nine cases, including all admitted kinds, escaped/BOM names, padded LEB128,
signature changes, duplicate imports and unsupported reference semantics.
The build used a separate 6 GiB hard/5 GiB early-stop guard; the full check used
8/7 GiB, peaking at 5.37 GiB without resource events. These are build/check
budgets; runtime tasks retain the smaller bounds above.
experiments/analysis-efficiency/wasm-string-framing.mjs
+184
View File
@@ -0,0 +1,184 @@
/** Compare unchanged public WASM tasks; raw inputs, exports and transcripts stay outside Git. */
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { createWriteStream } from "node:fs";
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { tmpdir } from "node:os";
import { finished } from "node:stream/promises";
import { parseArgs } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { createRetainedQueryCaller } from "./retained-query-calls.mjs";
import { finalizeRetainedQueryReceipt } from "./retained-query-finalization.mjs";
const { values } = parseArgs({
options: Object.fromEntries(
["runtime", "artifact", "wat", "tools", "output"].map((name) => [
name,
{ type: "string" },
]),
),
});
for (const name of ["runtime", "artifact", "wat", "tools", "output"])
assert.ok(
values[name] && values[name] === resolve(values[name]),
`${name} must be absolute`,
);
const hash = (value) => createHash("sha256").update(value).digest("hex");
const bytes = await readFile(values.artifact);
const nativeWat = await readFile(values.wat, "utf8");
// Native compilation validates the independent name inventory; no instance or target code runs.
const module = new WebAssembly.Module(bytes);
const nativeImports = WebAssembly.Module.imports(module);
const nativeExports = WebAssembly.Module.exports(module);
await mkdir(values.output, { mode: 0o700 });
const account = await mkdtemp(join(tmpdir(), "rea-wat-framing-"));
const client = new Client({ name: "wasm-string-framing", version: "1" });
const transport = new StdioClientTransport({
command: process.execPath,
args: [join(values.runtime, "scripts/rea.mjs"), "mcp"],
env: {
HOME: account,
PATH: process.env.PATH,
NODE_OPTIONS: "--max-old-space-size=2048",
UV_THREADPOOL_SIZE: "1",
REA_LOG_LEVEL: "silent",
REA_WABT_BIN_DIRECTORY: values.tools,
TMPDIR: account,
},
stderr: "pipe",
});
const serverLog = createWriteStream(join(values.output, "server.log"));
const serverLogFinished = finished(serverLog);
serverLogFinished.catch(() => {});
transport.stderr?.pipe(serverLog);
const report = {
artifact_sha256: hash(bytes),
artifact_bytes: bytes.length,
calls: [],
answers: [],
};
const transcript = [];
const call = createRetainedQueryCaller(client, (receipt, event) => {
report.calls.push(receipt);
transcript.push(event);
});
const observe = async (name, input, phase) => {
const reply = await call(name, input, phase);
assert.notEqual(reply.isError, true, JSON.stringify(reply));
assert.ok(reply.structuredContent);
return reply.structuredContent;
};
const started = performance.now();
try {
await client.connect(transport);
const catalog = await client.listTools();
assert.equal(catalog.nextCursor, undefined);
await writeFile(join(values.output, "catalog.json"), JSON.stringify(catalog));
report.catalog_sha256 = hash(JSON.stringify(catalog));
report.catalog_tools = catalog.tools.length;
const definitions = new Map(
catalog.tools.map((definition) => [definition.name, definition]),
);
const originalCall = client.callTool.bind(client);
client.callTool = (input, options) =>
originalCall(input, {
...options,
toolDefinition: definitions.get(input.name),
});
const observations = [];
for (const phase of ["fresh", "repeat"]) {
const summary = await observe(
"inspect_wasm_artifact",
{ path: values.artifact, detail: "summary" },
phase,
);
const parentId = summary.normalized_result.parent_evidence_id;
const exports = await observe(
"inspect_analysis_view",
{
source: { kind: "retained-evidence", evidence_id: parentId },
view: { kind: "wasm", facet: "exports", offset: 0, limit: 4 },
},
`${phase}:exports`,
);
assert.equal(exports.normalized_result.parent_evidence_id, parentId);
observations.push({ summary, exports, parentId });
}
await observe(
"export_evidence_bundle",
{ path: join(values.output, "evidence.json") },
"export",
);
const bundle = JSON.parse(
await readFile(join(values.output, "evidence.json"), "utf8"),
);
const kind = (value) => (value === "func" ? "function" : value);
for (const { summary, exports, parentId } of observations) {
const parent = bundle.records.find(
(record) => record.evidence_id === parentId,
);
assert.ok(parent);
assert.equal(parent.subject.digest.sha256, report.artifact_sha256);
const facts = parent.normalized_result;
assert.equal(facts.validation, "valid");
assert.equal(facts.wat.text, nativeWat);
assert.equal(facts.wat.sha256, hash(nativeWat));
assert.equal(facts.interface.state, "complete");
assert.deepEqual(
facts.interface.entries
.filter((entry) => entry.direction === "import")
.map((entry) => ({
module: entry.module,
name: entry.name,
kind: kind(entry.kind),
})),
nativeImports,
);
assert.deepEqual(
facts.interface.entries
.filter((entry) => entry.direction === "export")
.map((entry) => ({ name: entry.name, kind: kind(entry.kind) })),
nativeExports,
);
assert.deepEqual(exports.normalized_result.item, facts.exports.slice(0, 4));
assert.equal(
summary.normalized_result.summary.available_counts.imports,
nativeImports.length,
);
assert.equal(
summary.normalized_result.summary.available_counts.exports,
nativeExports.length,
);
assert.equal(facts.runtime_execution, "not-performed");
assert.equal(facts.network_fetch, "not-performed");
report.answers.push({
parent_evidence_id: parentId,
normalized_sha256: hash(JSON.stringify(facts)),
exports: exports.normalized_result.item,
});
}
assert.deepEqual(report.answers[0], report.answers[1]);
assert.equal(hash(await readFile(values.artifact)), report.artifact_sha256);
report.status = "passed";
} catch (cause) {
report.status = "failed";
report.failure = String(cause?.stack ?? cause);
process.exitCode = 1;
} finally {
report.elapsed_ms_before_cleanup = performance.now() - started;
if (
!(await finalizeRetainedQueryReceipt({
report,
client,
transport,
account,
serverLog,
serverLogFinished,
output: values.output,
transcript,
}))
)
process.exitCode = 1;
}
knip.json
+1 -1
View File
@@ -18,7 +18,7 @@
"vitepress": {
"entry": ["docs/.vitepress/config.ts"]
},
"ignoreBinaries": ["mkfifo"],
"ignoreBinaries": ["^g\\+\\+$", "mkfifo"],
"ignoreDependencies": [
"@stryker-mutator/command-runner",
"electron",
native/windows/README.md
+15 -2
View File
@@ -111,8 +111,21 @@ automatically. Alternatively, build it using the Linux/MinGW lane above.
- Cleanup walks locked directory objects, removes files by handle, and unlinks
reparse entries without traversing their targets. Failed removal retains the
root and ancestor leases for retry; a partially removed runtime accepts only
cleanup, and its registry entry remains until removal succeeds. The runtime
owner retries only native close failures with Win32 sharing/lock codes 32/33,
cleanup, and its registry entry remains until removal succeeds. `CloseHandle`
is checked. A value that is still open, or whose liveness cannot be read,
stays with its runtime, process, or file owner and is not passed to
`CloseHandle` again. A process pipe left in that state is treated as ended
and is not read or written again. An open liveness probe does not prove that the object
still belongs to that owner. A value that is no longer in the process handle table is
dropped and is not closed again. An incomplete snapshot whose writer cannot
be closed stays on the runtime instead of being forgotten; source handles
that remain open after a published snapshot are closed with that runtime. An
admitted file whose close fails stays on the private runtime and is closed
again before that runtime's tree walk. After the root directory handle is
released, a later close resumes with the remaining parent handles and does
not walk from that released root. A process whose job close succeeded and
whose later handle close failed is retried with process close alone. The
runtime owner retries only native close failures with Win32 sharing/lock codes 32/33,
for at most ten attempts with a 100 ms delay between attempts. Other native
errors are returned immediately; exhausting the retry budget preserves the
runtime owner for a later explicit close. These cleanup retries do not use an
native/windows/src/authority.hpp
+133 -14
View File
@@ -7,11 +7,14 @@
#include <bcrypt.h>
#include <node_api.h>
#include "handle_close.hpp"
#include <algorithm>
#include <array>
#include <atomic>
#include <cstdint>
#include <memory>
#include <mutex>
#include <stdexcept>
#include <string>
#include <utility>
@@ -54,24 +57,134 @@ inline void require(bool condition, const char* constraint, const std::wstring&
}
class Handle {
HANDLE value_ = INVALID_HANDLE_VALUE;
public:
Handle() = default;
explicit Handle(HANDLE value) : value_(value) {}
~Handle() { reset(); }
OwnedHandle<HANDLE> owned_;
public:
Handle() noexcept : owned_(INVALID_HANDLE_VALUE) {}
explicit Handle(HANDLE value) noexcept : owned_(INVALID_HANDLE_VALUE) { adopt(value, false); }
~Handle() noexcept;
Handle(const Handle&) = delete;
Handle& operator=(const Handle&) = delete;
Handle(Handle&& other) noexcept : value_(other.release()) {}
Handle& operator=(Handle&& other) noexcept { reset(other.release()); return *this; }
HANDLE get() const { return value_; }
bool valid() const { return value_ != INVALID_HANDLE_VALUE && value_ != nullptr; }
HANDLE release() { return std::exchange(value_, INVALID_HANDLE_VALUE); }
void reset(HANDLE next = INVALID_HANDLE_VALUE) {
if (valid()) CloseHandle(value_);
value_ = next;
}
Handle(Handle&& other) noexcept : owned_(std::move(other.owned_)) {}
Handle& operator=(Handle&& other) noexcept;
HANDLE get() const noexcept { return owned_.get(); }
bool valid() const noexcept { return owned_.valid(); }
bool fullyReleased() const noexcept { return owned_.fullyReleased(); }
// A failed close leaves the numeric value blocked. It must not be used for
// I/O or closed again; only a not-in-table probe may drop it.
bool closeBlocked() const noexcept { return owned_.closeBlocked(); }
void adopt(HANDLE value, bool blockClose) noexcept;
HandleCloseDecision release() noexcept;
void reset(HANDLE next = INVALID_HANDLE_VALUE) noexcept;
};
struct RetainedNativeHandle {
HANDLE value;
bool blockClose;
};
struct RetainedNativeHandles {
std::mutex mutex;
std::vector<RetainedNativeHandle> handles;
};
// Leaked for the process lifetime. Handle destructors can run during exit,
// after an ordinary function-local static would already have been destroyed.
inline RetainedNativeHandles& retainedNativeHandles() {
static auto* const retained = new RetainedNativeHandles();
return *retained;
}
inline HANDLE normalizedNativeHandle(HANDLE value) noexcept {
return value == nullptr ? INVALID_HANDLE_VALUE : value;
}
inline CloseAttempt closeNativeHandle(HANDLE value) noexcept {
if (CloseHandle(value) != FALSE) return {true, 0};
return {false, GetLastError()};
}
inline LivenessProbe probeNativeHandle(HANDLE value) noexcept {
DWORD flags = 0;
if (GetHandleInformation(value, &flags) != FALSE) return {HandleLiveness::Open, 0};
const DWORD error = GetLastError();
if (error == ERROR_INVALID_HANDLE) return {HandleLiveness::NotInTable, error};
return {HandleLiveness::Unavailable, error};
}
// Last owner for a value whose C++ handle is dying. A failed record keeps the
// kernel object until process exit instead of closing an unconfirmed number.
inline void retainNativeHandle(HANDLE value, bool blockClose) noexcept {
value = normalizedNativeHandle(value);
if (value == INVALID_HANDLE_VALUE) return;
try {
auto& retained = retainedNativeHandles();
const std::lock_guard<std::mutex> lock(retained.mutex);
retained.handles.push_back(RetainedNativeHandle{value, blockClose});
} catch (...) {
// The number stays open until process exit. Closing it here could reuse it.
}
}
inline void Handle::adopt(HANDLE value, bool blockClose) noexcept {
value = normalizedNativeHandle(value);
if (value == INVALID_HANDLE_VALUE) return;
if (!owned_.park(value, blockClose)) retainNativeHandle(value, blockClose);
}
inline HandleCloseDecision Handle::release() noexcept {
return owned_.release(&closeNativeHandle, &probeNativeHandle);
}
inline void Handle::reset(HANDLE next) noexcept {
owned_.reset(normalizedNativeHandle(next), &closeNativeHandle, &probeNativeHandle,
&retainNativeHandle);
}
inline Handle::~Handle() noexcept {
owned_.abandon(&closeNativeHandle, &probeNativeHandle, &retainNativeHandle);
}
inline Handle& Handle::operator=(Handle&& other) noexcept {
if (this == &other) return *this;
const auto incoming = other.owned_.take();
owned_.release(&closeNativeHandle, &probeNativeHandle);
const auto keep = [&](HANDLE value, bool blockClose) noexcept {
if (value == INVALID_HANDLE_VALUE || value == nullptr) return;
if (!owned_.park(value, blockClose)) retainNativeHandle(value, blockClose);
};
keep(incoming.value, incoming.blockValue);
keep(incoming.extra, incoming.blockExtra);
return *this;
}
inline void requireReleased(Handle& handle, const char* constraint,
const std::wstring& path = L"") {
const HandleCloseDecision decision = handle.release();
if (handle.fullyReleased()) return;
const DWORD code = decision.closeError != 0 ? decision.closeError
: decision.evidenceError != 0 ? decision.evidenceError
: ERROR_GEN_FAILURE;
throw Failure(constraint, path, code);
}
// Closes values detached by a dying owner. Survivors stay in the retainer.
inline void retryRetainedNativeHandles() noexcept {
std::vector<RetainedNativeHandle> pending;
try {
auto& retained = retainedNativeHandles();
const std::lock_guard<std::mutex> lock(retained.mutex);
pending.swap(retained.handles);
} catch (...) {
return;
}
for (const RetainedNativeHandle& item : pending) {
// The destructor releases the value once and parks it again on failure.
Handle handle;
handle.adopt(item.value, item.blockClose);
}
}
enum class Kind { File, Runtime, Process };
struct Resource {
const Kind kind;
@@ -91,7 +204,13 @@ struct Runtime : Resource {
std::vector<Handle> parents;
Handle directory;
std::vector<Handle> immutableFiles;
// Handles whose close stayed unconfirmed after snapshot or owner teardown.
// runtime close retries these before deleting the private tree.
std::vector<Handle> unreleased;
bool closing = false;
// The private tree and root directory handle have been released. Later
// closes resume with the parent handles and do not duplicate that root.
bool treeReleased = false;
// Main-thread admission is single-flight; only the accepted copy owns the
// atomic cancellation flag consumed by its async worker. ERROR_BUSY rejects
// another copy before it can reset that flag or replace pending ownership.
native/windows/src/filesystem.cc
+52 -4
View File
@@ -482,14 +482,31 @@ static void dispose(HANDLE handle, const std::wstring& path) {
require(SetFileInformationByHandle(handle, FileDispositionInfo, &deletion, sizeof(deletion)), "Delete owned object by handle failed", path);
}
static void parkUnresolved(Runtime& root, Handle& handle) noexcept {
if (handle.fullyReleased()) return;
handle.release();
if (handle.fullyReleased()) return;
try {
root.unreleased.push_back(std::move(handle));
} catch (...) {
// The owning Handle remains live. Its destructor retains the same value.
}
}
static void parkUnresolved(Runtime& root, std::vector<Handle>& handles) noexcept {
for (auto& handle : handles) parkUnresolved(root, handle);
handles.clear();
}
static void disposeSnapshot(Runtime& root, Handle& file, const std::wstring& path) {
// Snapshot readers do not share DELETE access. Release the writer only for
// failed-copy cleanup, with its parent pinned and its identity recorded.
// A failed close keeps this writer; opening a second handle would drop it.
auto parent = runtimeDirectory(root, path.substr(0, path.find_last_of(L'\\')));
FILE_ID_INFO expected{};
require(GetFileInformationByHandleEx(file.get(), FileIdInfo, &expected, sizeof(expected)),
"Read incomplete snapshot identity failed", path);
file.reset();
requireReleased(file, "Close incomplete snapshot writer failed", path);
auto cleanup = openRelative(parent->get(), path.substr(path.find_last_of(L'\\') + 1),
DELETE | GENERIC_READ | FILE_WRITE_ATTRIBUTES,
FILE_SHARE_READ, FILE_NON_DIRECTORY_FILE, path);
@@ -505,12 +522,24 @@ static void disposeSnapshot(Runtime& root, Handle& file, const std::wstring& pat
}
void closeRuntime(Runtime& root) {
retryRetainedNativeHandles();
if (root.closed) return;
root.closing = true;
// Windows keeps the bearer descriptor and snapshot leases until this owner
// cleanup. POSIX removes its consumed descriptor earlier. Abrupt owner death
// closes process jobs but cannot run this walk, so private files can remain.
for (auto& handle : root.unreleased)
requireReleased(handle, "Close retained runtime handle failed", root.path);
root.unreleased.clear();
// Writers share only FILE_SHARE_READ, which blocks a later delete open.
// Release them before the walk. A failed close stays on this owner and does
// not open a second delete handle.
for (auto& handle : root.immutableFiles)
requireReleased(handle, "Close retained runtime file failed", root.path);
root.immutableFiles.clear();
// Once the root directory handle is released, another walk would duplicate
// an invalid handle and never reach surviving parents.
if (!root.treeReleased) {
// Enumerate and open children through the retained directory handles. Paths
// below are diagnostic labels only: renames and ancestor junction replacement
// cannot redirect cleanup into an unrelated tree.
@@ -567,6 +596,7 @@ void closeRuntime(Runtime& root) {
std::wstring name;
if (!frame.next(name)) {
dispose(frame.directory.get(), frame.path);
requireReleased(frame.directory, "Close runtime directory handle failed", frame.path);
frames.pop_back();
continue;
}
@@ -578,10 +608,17 @@ void closeRuntime(Runtime& root) {
require(GetFileInformationByHandleEx(child.get(), FileAttributeTagInfo, &info, sizeof(info)), "Read cleanup object tag failed", path);
if ((info.FileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0 && (info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) == 0)
frames.push_back(std::make_unique<Frame>(std::move(child), path));
else dispose(child.get(), path);
else {
dispose(child.get(), path);
requireReleased(child, "Close runtime child handle failed", path);
}
}
// Failed walks retain the root's identity leases for a later cleanup retry.
root.directory.reset();
requireReleased(root.directory, "Close runtime root handle failed", root.path);
root.treeReleased = true;
}
for (auto& parent : root.parents)
requireReleased(parent, "Close runtime parent handle failed", root.path);
root.parents.clear();
root.closed = true;
}
@@ -657,12 +694,17 @@ static void completeSnapshot(napi_env env, napi_status status, void* pointer) {
set(env, result, "source", identity(env, work->source->get(), work->source->requestedPath));
set(env, result, "snapshot", identity(env, work->destination.get(), work->path));
work->root->immutableFiles.push_back(std::move(work->destination));
// The digest is the byte outcome. Source handles that fail close stay on
// the runtime and are retried by runtime close; they do not erase the copy.
if (work->source != nullptr) parkUnresolved(*work->root, work->source->handles);
check(napi_resolve_deferred(env, work->deferred, result));
} catch (const Failure& failure) {
reject(failure);
} catch (const std::exception& failure) {
reject(Failure(failure.what(), work->path, ERROR_GEN_FAILURE));
}
if (work->source != nullptr) parkUnresolved(*work->root, work->source->handles);
parkUnresolved(*work->root, work->destination);
napi_delete_reference(env, work->rootReference);
napi_delete_async_work(env, work->work);
}
@@ -707,8 +749,14 @@ napi_value filesystemCall(napi_env env, const std::wstring& operation,
return result;
}
if (operation == L"close") {
retryRetainedNativeHandles();
auto& file = static_cast<File&>(resource(env, args[0], Kind::File));
file.handles.clear(); file.closed = true;
// Leave a failed handle on this File. The private runtime keeps that
// wrapper and retries this close before runtime_close.
for (auto& handle : file.handles)
requireReleased(handle, "Close admitted file handle failed", file.path);
file.handles.clear();
file.closed = true;
return null(env);
}
if (operation == L"read") {
native/windows/src/handle_close.hpp
+206
View File
@@ -0,0 +1,206 @@
#pragma once
#include <cstdint>
#include <utility>
namespace rea {
// A failed CloseHandle and a later liveness probe are not one atomic fact.
// Another thread can reuse the numeric value before GetHandleInformation
// runs, so an Open result does not prove the probed object is still ours.
// After a failed close, drop the value only when the probe says it is gone.
// Otherwise keep it and do not call CloseHandle again.
enum class HandleLiveness { Open, NotInTable, Unavailable };
enum class HandleCloseDisposition {
Released,
TerminalInvalid,
Unconfirmed,
};
struct CloseAttempt {
bool succeeded = false;
std::uint32_t error = 0;
};
struct LivenessProbe {
HandleLiveness liveness = HandleLiveness::Unavailable;
std::uint32_t error = 0;
};
struct HandleCloseDecision {
HandleCloseDisposition disposition = HandleCloseDisposition::Released;
bool clearValue = false;
bool retryClose = false;
std::uint32_t closeError = 0;
std::uint32_t evidenceError = 0;
};
inline HandleCloseDecision decideOwnedHandleClose(bool closeSucceeded,
std::uint32_t closeError,
LivenessProbe evidence) {
if (closeSucceeded)
return {HandleCloseDisposition::Released, true, false, 0, 0};
switch (evidence.liveness) {
case HandleLiveness::Open:
case HandleLiveness::Unavailable:
return {HandleCloseDisposition::Unconfirmed, false, false, closeError,
evidence.error};
case HandleLiveness::NotInTable:
return {HandleCloseDisposition::TerminalInvalid, true, false, closeError,
evidence.error};
}
return {HandleCloseDisposition::Unconfirmed, false, false, closeError,
evidence.error};
}
// One current value plus at most one parked replacement. Close and liveness
// are injected so this gate can be tested without a kernel handle table.
template <class T>
class OwnedHandle {
public:
explicit OwnedHandle(T invalid) noexcept
: invalid_(invalid), value_(invalid), extra_(invalid) {}
OwnedHandle(const OwnedHandle&) = delete;
OwnedHandle& operator=(const OwnedHandle&) = delete;
OwnedHandle(OwnedHandle&& other) noexcept
: invalid_(other.invalid_),
value_(std::exchange(other.value_, other.invalid_)),
extra_(std::exchange(other.extra_, other.invalid_)),
blockValue_(std::exchange(other.blockValue_, false)),
blockExtra_(std::exchange(other.blockExtra_, false)) {}
OwnedHandle& operator=(OwnedHandle&&) = delete;
T get() const noexcept { return value_; }
T invalid() const noexcept { return invalid_; }
bool valid() const noexcept { return value_ != invalid_; }
bool fullyReleased() const noexcept {
return value_ == invalid_ && extra_ == invalid_;
}
bool closeBlocked() const noexcept { return blockValue_; }
// Install a value into an empty slot. A live owner uses reset.
bool park(T value, bool blockClose) noexcept {
if (value == invalid_) return true;
if (value_ == invalid_) {
value_ = value;
blockValue_ = blockClose;
return true;
}
if (extra_ == invalid_) {
extra_ = value;
blockExtra_ = blockClose;
return true;
}
return false;
}
struct Slots {
T value;
T extra;
bool blockValue;
bool blockExtra;
};
Slots take() noexcept {
Slots slots{value_, extra_, blockValue_, blockExtra_};
value_ = invalid_;
extra_ = invalid_;
blockValue_ = false;
blockExtra_ = false;
return slots;
}
template <class Close, class Probe>
HandleCloseDecision release(Close&& close, Probe&& probe) noexcept {
HandleCloseDecision last{HandleCloseDisposition::Released, true, false, 0, 0};
for (;;) {
if (value_ == invalid_) {
if (extra_ == invalid_) return last;
value_ = extra_;
blockValue_ = blockExtra_;
extra_ = invalid_;
blockExtra_ = false;
}
last = finalizeSlot(value_, blockValue_, close, probe);
if (value_ != invalid_) return last;
}
}
// Close the current owner. On an unconfirmed close, keep it and park `next`
// instead of overwriting the numeric value. `sink` receives a value that
// does not fit in the parked slot; it must retain that exact value.
template <class Close, class Probe, class Sink>
void reset(T next, Close&& close, Probe&& probe, Sink&& sink) noexcept {
release(close, probe);
if (!fullyReleased()) {
if (next != invalid_ && next != value_ && next != extra_ && !park(next, false))
sink(next, false);
return;
}
value_ = next;
blockValue_ = false;
}
// Last-resort detach. Release once, then hand any surviving values to `sink`
// without a second close of a value whose liveness is not Open.
template <class Close, class Probe, class Sink>
void abandon(Close&& close, Probe&& probe, Sink&& sink) noexcept {
release(close, probe);
if (value_ != invalid_) sink(value_, blockValue_);
if (extra_ != invalid_) sink(extra_, blockExtra_);
value_ = invalid_;
extra_ = invalid_;
blockValue_ = false;
blockExtra_ = false;
}
private:
T invalid_;
T value_;
T extra_;
bool blockValue_ = false;
bool blockExtra_ = false;
template <class Close, class Probe>
HandleCloseDecision finalizeSlot(T& slot, bool& block, Close&& close,
Probe&& probe) noexcept {
if (slot == invalid_)
return {HandleCloseDisposition::Released, true, false, 0, 0};
if (block) {
const LivenessProbe evidence = probe(slot);
if (evidence.liveness == HandleLiveness::NotInTable) {
slot = invalid_;
block = false;
return {HandleCloseDisposition::TerminalInvalid, true, false, 0,
evidence.error};
}
// Open or unreadable: the number stays close-blocked. Closing it could
// release an object that reused the value.
return {HandleCloseDisposition::Unconfirmed, false, false, 0,
evidence.error};
}
const CloseAttempt attempt = close(slot);
if (attempt.succeeded) {
slot = invalid_;
block = false;
return {HandleCloseDisposition::Released, true, false, 0, 0};
}
const LivenessProbe evidence = probe(slot);
const HandleCloseDecision decision =
decideOwnedHandleClose(false, attempt.error, evidence);
if (decision.clearValue) {
slot = invalid_;
block = false;
} else {
block = !decision.retryClose;
}
return decision;
}
};
} // namespace rea
native/windows/src/handle_close_test.cc
+254
View File
@@ -0,0 +1,254 @@
#include "handle_close.hpp"
#include <cstdlib>
#include <iostream>
#include <string>
#include <utility>
#include <vector>
namespace {
int failures = 0;
void expect(bool condition, const char* description) {
if (condition) return;
++failures;
std::cerr << "failed: " << description << '\n';
}
rea::CloseAttempt fail(std::uint32_t error) {
return {false, error};
}
rea::LivenessProbe live(rea::HandleLiveness liveness, std::uint32_t error = 0) {
return {liveness, error};
}
void testDecisionMatrix() {
const auto released = rea::decideOwnedHandleClose(true, 5, live(rea::HandleLiveness::Open, 9));
expect(released.disposition == rea::HandleCloseDisposition::Released && released.clearValue &&
!released.retryClose && released.closeError == 0,
"a successful close clears the value and does not retry");
const auto still = rea::decideOwnedHandleClose(false, 6, live(rea::HandleLiveness::Open));
expect(still.disposition == rea::HandleCloseDisposition::Unconfirmed && !still.clearValue &&
!still.retryClose && still.closeError == 6,
"an open probe after a failed close does not authorize another close");
const auto gone = rea::decideOwnedHandleClose(false, 6, live(rea::HandleLiveness::NotInTable, 6));
expect(gone.disposition == rea::HandleCloseDisposition::TerminalInvalid && gone.clearValue &&
!gone.retryClose && gone.evidenceError == 6,
"a value missing from the handle table is dropped and never closed again");
const auto unknown =
rea::decideOwnedHandleClose(false, 5, live(rea::HandleLiveness::Unavailable, 12));
expect(unknown.disposition == rea::HandleCloseDisposition::Unconfirmed && !unknown.clearValue &&
!unknown.retryClose && unknown.closeError == 5 && unknown.evidenceError == 12,
"unavailable liveness keeps the value and blocks another close");
}
void testReleaseRetainsAndRetriesOnlyLiveValues() {
rea::OwnedHandle<int> handle(-1);
expect(handle.park(4, false), "adopt the first value");
int closes = 0;
int probes = 0;
const auto decision = handle.release(
[&](int) {
++closes;
return fail(6);
},
[&](int) {
++probes;
return live(rea::HandleLiveness::Open);
});
expect(decision.disposition == rea::HandleCloseDisposition::Unconfirmed && handle.get() == 4 &&
closes == 1 && probes == 1 && handle.closeBlocked(),
"a failed close of a value that still looks open keeps that value blocked");
const auto retried = handle.release(
[&](int) {
++closes;
return rea::CloseAttempt{true, 0};
},
[&](int) {
++probes;
return live(rea::HandleLiveness::Open);
});
expect(retried.disposition == rea::HandleCloseDisposition::Unconfirmed && handle.get() == 4 &&
handle.closeBlocked() && closes == 1 && probes == 2,
"a later open probe does not close a value whose previous close failed");
}
void testUnconfirmedCloseDoesNotCallCloseAgain() {
rea::OwnedHandle<int> handle(-1);
expect(handle.park(8, false), "adopt the unconfirmed value");
int closes = 0;
int probes = 0;
handle.release(
[&](int) {
++closes;
return fail(5);
},
[&](int) {
++probes;
return live(rea::HandleLiveness::Unavailable, 12);
});
expect(handle.get() == 8 && handle.closeBlocked() && closes == 1 && probes == 1,
"the first failed close records unconfirmed liveness");
handle.release(
[&](int) {
++closes;
return fail(5);
},
[&](int) {
++probes;
return live(rea::HandleLiveness::Unavailable, 12);
});
expect(closes == 1 && probes == 2 && handle.get() == 8 && handle.closeBlocked(),
"unconfirmed liveness does not call close again");
handle.release(
[&](int) {
++closes;
return rea::CloseAttempt{true, 0};
},
[&](int) {
++probes;
return live(rea::HandleLiveness::Open);
});
expect(handle.get() == 8 && handle.closeBlocked() && closes == 1 && probes == 3,
"a later open probe still does not close the unconfirmed value");
}
void testTerminalInvalidityDropsTheValue() {
rea::OwnedHandle<int> handle(-1);
expect(handle.park(3, false), "adopt the invalid value");
int closes = 0;
const auto decision = handle.release(
[&](int) {
++closes;
return fail(6);
},
[&](int) { return live(rea::HandleLiveness::NotInTable, 6); });
expect(decision.disposition == rea::HandleCloseDisposition::TerminalInvalid &&
handle.fullyReleased() && closes == 1,
"terminal invalidity clears the owner");
handle.release(
[&](int) {
++closes;
return fail(6);
},
[&](int) { return live(rea::HandleLiveness::Open); });
expect(closes == 1 && handle.fullyReleased(),
"a dropped value is not passed to close again");
}
void testBlockedProbeCanDiscoverTerminalInvalidityWithoutClosing() {
rea::OwnedHandle<int> handle(-1);
expect(handle.park(9, true), "install a close-blocked value");
int closes = 0;
const auto decision = handle.release(
[&](int) {
++closes;
return fail(1);
},
[&](int) { return live(rea::HandleLiveness::NotInTable, 6); });
expect(decision.disposition == rea::HandleCloseDisposition::TerminalInvalid &&
handle.fullyReleased() && closes == 0,
"a blocked value that is no longer in the table is not closed");
}
void testResetParksTheReplacementAndOverflowsTheThirdValue() {
rea::OwnedHandle<int> handle(-1);
expect(handle.park(1, false), "adopt the current value");
std::vector<std::pair<int, bool>> sunk;
handle.reset(
2,
[&](int) { return fail(6); },
[&](int) { return live(rea::HandleLiveness::Open); },
[&](int value, bool block) { sunk.emplace_back(value, block); });
expect(handle.get() == 1 && sunk.empty(), "a failed close does not overwrite the live value");
int closes = 0;
handle.reset(
3,
[&](int value) {
++closes;
return value == 1 ? fail(6) : rea::CloseAttempt{true, 0};
},
[&](int) { return live(rea::HandleLiveness::Open); },
[&](int value, bool block) { sunk.emplace_back(value, block); });
expect(handle.get() == 1 && handle.closeBlocked() && sunk.size() == 1 && sunk[0].first == 3 &&
!sunk[0].second && closes == 0,
"a blocked value is not closed again and a third value is retained by the sink");
}
void testReleaseClosesTheParkedReplacementAfterTheCurrentValue() {
rea::OwnedHandle<int> handle(-1);
expect(handle.park(1, false), "adopt the current value");
expect(handle.park(2, false), "park the replacement");
std::vector<int> order;
const auto decision = handle.release(
[&](int value) {
order.push_back(value);
return rea::CloseAttempt{true, 0};
},
[&](int) { return live(rea::HandleLiveness::Unavailable, 1); });
expect(decision.disposition == rea::HandleCloseDisposition::Released && handle.fullyReleased() &&
order.size() == 2 && order[0] == 1 && order[1] == 2,
"release closes the current value and then the parked replacement");
}
void testAbandonHandsSurvivingValuesToTheSinkOnce() {
rea::OwnedHandle<int> handle(-1);
expect(handle.park(7, false), "adopt the abandoned value");
expect(handle.park(8, false), "park a second abandoned value");
int closes = 0;
std::vector<std::pair<int, bool>> sunk;
handle.abandon(
[&](int) {
++closes;
return fail(5);
},
[&](int) { return live(rea::HandleLiveness::Unavailable, 11); },
[&](int value, bool block) { sunk.emplace_back(value, block); });
expect(handle.fullyReleased() && closes == 1 && sunk.size() == 2 && sunk[0] == std::make_pair(7, true) &&
sunk[1] == std::make_pair(8, false),
"abandon retains every surviving value and does not close the unconfirmed one twice");
}
void testMovePreservesTheCloseGate() {
rea::OwnedHandle<int> handle(-1);
expect(handle.park(4, true), "adopt a blocked value");
rea::OwnedHandle<int> moved(std::move(handle));
expect(handle.fullyReleased() && moved.get() == 4 && moved.closeBlocked(),
"move transfers the value and its close gate");
int closes = 0;
moved.release([&](int) {
++closes;
return fail(1);
},
[&](int) { return live(rea::HandleLiveness::Unavailable, 2); });
expect(closes == 0 && moved.get() == 4 && moved.closeBlocked(),
"a moved blocked value is probed and not closed while liveness is unknown");
}
} // namespace
int main() {
testDecisionMatrix();
testReleaseRetainsAndRetriesOnlyLiveValues();
testUnconfirmedCloseDoesNotCallCloseAgain();
testTerminalInvalidityDropsTheValue();
testBlockedProbeCanDiscoverTerminalInvalidityWithoutClosing();
testResetParksTheReplacementAndOverflowsTheThirdValue();
testReleaseClosesTheParkedReplacementAfterTheCurrentValue();
testAbandonHandsSurvivingValuesToTheSinkOnce();
testMovePreservesTheCloseGate();
if (failures != 0) {
std::cerr << failures << " assertion(s) failed\n";
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
native/windows/src/process.cc
+21 -6
View File
@@ -217,14 +217,19 @@ static napi_value readOutput(napi_env env, Handle& read, bool& ended) {
constexpr size_t kMaximumReadPerPoll = 1024 * 1024;
std::array<unsigned char, 65536> chunk;
std::vector<unsigned char> bytes;
if (!read.valid()) ended = true;
// A close-blocked pipe is EOF. Its number may have been reused, so this
// poll must not pass it to PeekNamedPipe or ReadFile. process_close still
// owns that value.
if (!read.valid() || read.closeBlocked()) ended = true;
else {
while (bytes.size() < kMaximumReadPerPoll) {
DWORD available = 0;
if (!PeekNamedPipe(read.get(), nullptr, 0, nullptr, &available, nullptr)) {
require(GetLastError() == ERROR_BROKEN_PIPE,
"Observe owned process output failed");
read.reset();
// Broken pipe is EOF for the bytes already queued. A failed close keeps
// this handle for process_close instead of dropping those bytes.
read.release();
ended = true;
break;
}
@@ -350,7 +355,8 @@ static void completeInput(napi_env env, napi_status status, void* pointer) {
}
static napi_value inputWrite(napi_env env, napi_value owner, Process& process, napi_value input) {
require(process.stdinWrite.valid(), "Owned process has no writable input", L"", ERROR_BROKEN_PIPE);
require(process.stdinWrite.valid() && !process.stdinWrite.closeBlocked(),
"Owned process has no writable input", L"", ERROR_BROKEN_PIPE);
require(!process.inputPending, "Owned process input already has a pending write", L"", ERROR_BUSY);
auto work = std::make_unique<InputWork>(process);
void* bytes = nullptr;
@@ -411,7 +417,7 @@ napi_value processCall(napi_env env, const std::wstring& operation, const std::v
require(args.size() == 1, "Wrong native process handle argument count", L"", ERROR_INVALID_PARAMETER);
auto& process = static_cast<Process&>(resource(env, args[0], Kind::Process));
if (operation == L"process_stdin_close") {
process.stdinWrite.reset();
requireReleased(process.stdinWrite, "Close owned process stdin failed");
return null(env);
}
if (operation == L"process_poll") {
@@ -446,8 +452,17 @@ napi_value processCall(napi_env env, const std::wstring& operation, const std::v
}
if (operation == L"process_close") {
// Closing the job is the crash-safe authority; no PID enumeration is used.
process.job.reset(); process.process.reset(); process.stdoutRead.reset(); process.stderrRead.reset(); process.stdinWrite.reset();
process.notifications.reset();
// A failed close leaves that handle on this process so a later close can
// retry it. Handles that already closed stay released, so the retry
// resumes at the first handle still owned. Callers that have observed
// settlement must retry this close alone; kill and poll still need the job.
retryRetainedNativeHandles();
requireReleased(process.job, "Close owned Job Object failed");
requireReleased(process.process, "Close owned process failed");
requireReleased(process.stdoutRead, "Close owned process stdout failed");
requireReleased(process.stderrRead, "Close owned process stderr failed");
requireReleased(process.stdinWrite, "Close owned process stdin failed");
requireReleased(process.notifications, "Close owned process notifications failed");
process.closed = true;
return null(env);
}
scripts/ci/test-history.mjs
+91 -64
View File
@@ -4,21 +4,38 @@ import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { normalizeWorkflowRun } from "./workflow-run.mjs";
const ARTIFACT_NAME = "test-timings";
const TIER_EVENTS = ["push", "schedule", "workflow_dispatch"];
const TIER_PROFILES = {
fast: { artifactName: "test-timings-fast", events: TIER_EVENTS },
full: { artifactName: "test-timings-full", events: TIER_EVENTS },
};
const MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
const MAX_CANDIDATES = 5;
const EMPTY_SELECTION = { runId: "", artifactName: "" };
const positiveId = (value) =>
(typeof value === "string" || typeof value === "number") &&
/^[1-9]\d*$/.test(String(value)) &&
Number.isSafeInteger(Number(value));
function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) {
function findTimingRun({
repository,
branch,
repositoryId,
tier,
now = Date.now(),
}) {
if (
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository ?? "") ||
!branch ||
!positiveId(repositoryId)
)
return "";
return EMPTY_SELECTION;
const profile = Object.hasOwn(TIER_PROFILES, tier)
? TIER_PROFILES[tier]
: null;
// Reused and other tiers do not publish shard history. Missing history uses
// Vitest's own assignment rather than another tier's artifact.
if (!profile) return EMPTY_SELECTION;
const deadline = Date.now() + 20_000;
const api = (...arguments_) => {
const remaining = deadline - Date.now();
@@ -33,83 +50,93 @@ function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) {
}),
);
};
const { artifacts } = api(
`repos/${repository}/actions/artifacts`,
"-f",
`name=${ARTIFACT_NAME}`,
"-F",
"per_page=100",
);
if (!Array.isArray(artifacts))
throw new Error("Missing timing artifact inventory");
const candidates = artifacts
.filter((artifact) => {
const lineage = artifact?.workflow_run;
const age = now - Date.parse(artifact?.created_at);
return (
artifact?.name === ARTIFACT_NAME &&
artifact?.expired === false &&
typeof artifact?.created_at === "string" &&
Number.isFinite(age) &&
age >= 0 &&
age < MAX_AGE_MS &&
positiveId(lineage?.id) &&
positiveId(lineage?.repository_id) &&
positiveId(lineage?.head_repository_id) &&
lineage.repository_id === lineage.head_repository_id &&
lineage.repository_id === Number(repositoryId) &&
lineage.head_branch === branch
);
})
.sort(
(left, right) =>
Date.parse(right.created_at) - Date.parse(left.created_at),
)
.slice(0, MAX_CANDIDATES);
for (const artifact of candidates) {
const run = normalizeWorkflowRun(
api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`),
const search = ({ artifactName, events }) => {
const { artifacts } = api(
`repos/${repository}/actions/artifacts`,
"-f",
`name=${artifactName}`,
"-F",
"per_page=100",
);
const runAge = now - Date.parse(run?.created_at);
if (
typeof run?.created_at === "string" &&
Number.isFinite(runAge) &&
runAge >= 0 &&
runAge < MAX_AGE_MS &&
run.id === artifact.workflow_run.id &&
run.status === "completed" &&
run.conclusion === "success" &&
run.workflowPath?.file === ".github/workflows/ci.yml" &&
["schedule", "workflow_dispatch"].includes(run.event) &&
run.head_branch === branch &&
run.head_repository?.full_name === repository
)
return String(run.id);
}
return "";
if (!Array.isArray(artifacts))
throw new Error("Missing timing artifact inventory");
const candidates = artifacts
.filter((artifact) => {
const lineage = artifact?.workflow_run;
const age = now - Date.parse(artifact?.created_at);
return (
artifact?.name === artifactName &&
artifact?.expired === false &&
typeof artifact?.created_at === "string" &&
Number.isFinite(age) &&
age >= 0 &&
age < MAX_AGE_MS &&
positiveId(lineage?.id) &&
positiveId(lineage?.repository_id) &&
positiveId(lineage?.head_repository_id) &&
lineage.repository_id === lineage.head_repository_id &&
lineage.repository_id === Number(repositoryId) &&
lineage.head_branch === branch
);
})
.sort(
(left, right) =>
Date.parse(right.created_at) - Date.parse(left.created_at),
)
.slice(0, MAX_CANDIDATES);
for (const artifact of candidates) {
const run = normalizeWorkflowRun(
api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`),
);
const runAge = now - Date.parse(run?.created_at);
if (
typeof run?.created_at === "string" &&
Number.isFinite(runAge) &&
runAge >= 0 &&
runAge < MAX_AGE_MS &&
run.id === artifact.workflow_run.id &&
run.status === "completed" &&
run.conclusion === "success" &&
run.workflowPath?.file === ".github/workflows/ci.yml" &&
events.includes(run.event) &&
run.head_branch === branch &&
run.head_repository?.full_name === repository
)
return String(run.id);
}
return "";
};
const runId = search(profile);
return runId
? { runId, artifactName: profile.artifactName }
: EMPTY_SELECTION;
}
function write(runId) {
function write({ runId, artifactName }) {
if (process.env.GITHUB_OUTPUT)
appendFileSync(process.env.GITHUB_OUTPUT, `run-id=${runId}\n`);
console.log(JSON.stringify({ run_id: runId }));
appendFileSync(
process.env.GITHUB_OUTPUT,
`run-id=${runId}\nartifact-name=${artifactName}\n`,
);
console.log(JSON.stringify({ run_id: runId, artifact_name: artifactName }));
}
if (
process.argv[1] &&
fileURLToPath(import.meta.url) === resolve(process.argv[1])
) {
let runId = "";
let selection = EMPTY_SELECTION;
try {
runId = findTimingRun({
selection = findTimingRun({
repository: process.env.CI_REPOSITORY,
branch: process.env.CI_DEFAULT_BRANCH,
repositoryId: process.env.CI_REPOSITORY_ID,
tier: process.env.CI_TIMING_TIER,
});
} catch {
// History improves shard ordering but must never block CI.
}
write(runId);
write(selection);
}
scripts/lib/browser-runtime-termination-e2e.mjs
+10 -1
View File
@@ -1,6 +1,6 @@
import assert from "node:assert/strict";
import { CdpConnection } from "../../dist/browser/CdpConnection.js";
import { mcpTextValue } from "./mcp-verifier-results.mjs";
import { mcpTextValue, requireMcpToolError } from "./mcp-verifier-results.mjs";
import { closeVerifierResources } from "./verifier-cleanup.mjs";
/** Close a separately owned fixture page after actual MCP arming and inspect native cleanup diagnostics. */
@@ -60,17 +60,26 @@ export async function verifyRuntimeTargetTermination(client, input, origin) {
"Detached instrumentation cannot claim confirmed cleanup",
);
const failure = mcpTextValue(response);
const projected = requireMcpToolError(response);
assert.match(failure, /cleanup.*confirmed/u);
assert.ok(
failure.includes(targetId),
"Failure must retain the selected target identity",
);
assert.match(failure, /Profiler\.stopPreciseCoverage/u);
assert.equal(projected.code, "cleanup_incomplete");
assert.equal(
projected.details?.diagnostics?.primary_error?.details?.diagnostics
?.observed_window?.end_reason,
"session_detached",
"Target.closeTarget detaches this CDP session; it does not prove the target crashed",
);
return {
target_id: targetId,
page_closed_after_arming: true,
failure_inline: true,
cleanup: "unconfirmed_reported",
observed_window_end_reason: "session_detached",
};
} catch (cause) {
primaryError = cause;
scripts/verify-real-browser.mjs
+5 -4
View File
@@ -546,13 +546,14 @@ async function verifyPageScopedTransport(provider, proxy, origin) {
const disconnected = await disconnectedPromise;
if (
!disconnected.ok ||
disconnected.value.window.end_reason !== "target_terminated" ||
disconnected.value.window.end_reason !== "transport_disconnected" ||
!disconnected.value.timeline.some(
({ type }) => type === "target_terminated",
)
({ type }) => type === "transport_disconnected",
) ||
disconnected.value.timeline.some(({ type }) => type === "target_terminated")
)
throw new Error(
"Page-scoped CDP disconnect was not reported as target_terminated",
"Page-scoped CDP disconnect must report transport loss without claiming a target crash",
);
}
scripts/verify-wasm-artifact.mjs
+23 -1
View File
@@ -118,6 +118,9 @@ try {
entrypoint,
"inspect-wasm-artifact",
input.path,
...(input.command_timeout_ms === undefined
? []
: ["--command-timeout-ms", String(input.command_timeout_ms)]),
...input.glue_paths.flatMap((path) => ["--glue_paths", path]),
...input.candidate_paths.flatMap((path) => [
"--candidate_paths",
@@ -238,7 +241,12 @@ try {
);
for (const [index, gluePath] of gluePaths.entries())
await writeFile(gluePath, glueSources[index]);
const input = { path, glue_paths: gluePaths, candidate_paths: [candidate] };
const input = {
path,
glue_paths: gluePaths,
candidate_paths: [candidate],
...(index === 0 ? { command_timeout_ms: 60_000 } : {}),
};
assert.ok(ajv.validate(advertised.inputSchema, input));
const cliValue = await inspectCli(input);
assert.equal(cliValue.category, undefined, JSON.stringify(cliValue));
@@ -256,6 +264,11 @@ try {
assert.equal(cli.evidence_id, mcp.evidence_id);
assert.deepEqual(cli, mcp);
const report = mcp.normalized_result;
assert.equal(
report.tool_profile.timeout_ms,
input.command_timeout_ms ?? 30_000,
);
assert.equal(mcp.parameters.command_timeout_ms, input.command_timeout_ms);
assert.equal(report.artifact.sha256, sha256(original));
assert.equal(report.artifact.bytes, original.length);
assert.equal(report.wat.text, wat);
@@ -359,6 +372,7 @@ try {
path: invalidPath,
glue_paths: [],
candidate_paths: [],
...(index === 0 ? { command_timeout_ms: 42_000 } : {}),
};
const cliError = await inspectCli(invalidInput);
assert.equal(
@@ -391,6 +405,14 @@ try {
),
);
assert.equal(failed.subject.digest.sha256, sha256(invalidBytes));
assert.equal(
failed.normalized_result.tool_profile.timeout_ms,
invalidInput.command_timeout_ms ?? 30_000,
);
assert.equal(
failed.normalized_result.tool_profile.output_budget_bytes,
32 * 1024 * 1024,
);
assert.equal(
failed.normalized_result.validation,
"rejected-by-selected-profile",
src/application/AppBundleExecutable.ts
+1 -1
View File
@@ -4,7 +4,7 @@ import { join } from "node:path";
import { parseBinary } from "plist";
import { z } from "zod";
import { preflightPropertyListDecode } from "../artifacts/apple/InterfaceBuilderDecodeBudget.js";
import { preflightPropertyListDecode } from "../artifacts/apple/PropertyListDecodeBudget.js";
import { BinaryTargetError } from "../domain/configurationErrors.js";
import { isPathWithinRoot } from "../domain/localPath.js";
import {
src/application/BinaryTargetResolver.ts
+36 -6
View File
@@ -90,11 +90,15 @@ export const parseBinaryTarget = async (
const hostArchitecture = options.hostArchitecture ?? process.arch;
const { targetKind, formatHint, signal, architecture, entryPoint } = options;
const candidate = absoluteCallerPath(input, { cwd });
// Failures before canonicalization concern the selected path itself; later
// failures concern the canonical path opened with symbolic links rejected.
let stage: TargetFilesystemStage = "resolve";
try {
throwIfTargetResolutionCancelled(signal);
await access(candidate, constants.R_OK);
throwIfTargetResolutionCancelled(signal);
const canonical = await realpath(candidate);
stage = "read";
// Windows native admission must inspect the caller-selected reparse components.
const sourcePath = process.platform === "win32" ? candidate : canonical;
throwIfTargetResolutionCancelled(signal);
@@ -135,6 +139,7 @@ export const parseBinaryTarget = async (
return ok({
path,
sourcePath,
...bundle,
sha256: await sha256Handle(handle, path, metadata, signal),
kind: "executable",
format: "raw-binary",
@@ -174,6 +179,7 @@ export const parseBinaryTarget = async (
return ok({
path,
sourcePath,
...bundle,
sha256: await sha256Handle(handle, path, metadata, signal),
kind: "executable",
format: "dos-com",
@@ -245,24 +251,39 @@ export const parseBinaryTarget = async (
);
if (cause instanceof BinaryTargetError) return err(cause);
if (!(cause instanceof Error)) throw cause;
const failure = classifyBinaryTargetFilesystemFailure(candidate, cause);
const failure = classifyTargetFilesystemFailure(candidate, cause, stage);
if (failure === undefined) throw cause;
return err(failure);
}
};
/**
* `resolve` covers the caller-selected path before canonicalization; `read`
* covers a canonical path opened with symbolic links rejected.
*/
type TargetFilesystemStage = "resolve" | "read";
/** Classify a filesystem failure while reading a previously selected target. */
export const classifyBinaryTargetFilesystemFailure = (
path: string,
cause: unknown,
): BinaryTargetError | undefined =>
classifyTargetFilesystemFailure(path, cause, "read");
const classifyTargetFilesystemFailure = (
path: string,
cause: unknown,
stage: TargetFilesystemStage,
): BinaryTargetError | undefined => {
if (!(cause instanceof Error)) return undefined;
const code = filesystemErrorCode(cause);
return code === undefined
? undefined
: new BinaryTargetError(path, targetFailureReason(code, cause.message), {
cause,
});
: new BinaryTargetError(
path,
targetFailureReason(code, cause.message, stage),
{ cause },
);
};
const filesystemErrorCode = (cause: Error): string | undefined => {
@@ -270,7 +291,11 @@ const filesystemErrorCode = (cause: Error): string | undefined => {
return typeof code === "string" ? code : undefined;
};
const targetFailureReason = (code: string, detail: string): string => {
const targetFailureReason = (
code: string,
detail: string,
stage: TargetFilesystemStage,
): string => {
switch (code) {
case "ENOENT":
case "ENOTDIR":
@@ -283,7 +308,12 @@ const targetFailureReason = (code: string, detail: string): string => {
case "ENOTFILE":
return "target is not a regular file";
case "ELOOP":
return `target changed while being opened: ${detail}`;
// A canonical path has no symbolic links left, so a link rejected while
// reading it appeared after resolution. Before that, ELOOP is a loop or
// an over-deep link chain in the selected path itself.
return stage === "resolve"
? `target path could not be resolved: too many symbolic links or a symbolic link loop: ${detail}`
: `target changed while being opened: ${detail}`;
default:
return `target could not be read (${code}): ${detail}`;
}
src/application/CapabilityInventory.ts
+9
View File
@@ -685,6 +685,7 @@ const targetAvailability = ({
targetKind,
targetFormat,
targetOpen,
descriptors,
}: AvailabilityContext): Availability | null => {
if (FRIDA_OPERATION_NAMES.has(name)) return null;
if (!targetOpen && ACTIVE_TARGET_KINDS.has(kind))
@@ -692,6 +693,14 @@ const targetAvailability = ({
reason: "target_required",
remediation: "Call open_binary with a supported local target.",
};
const descriptor = descriptors.get(name);
if (
targetOpen &&
descriptor?.available === false &&
(descriptor.availability_code === "target_kind_unsupported" ||
descriptor.availability_code === "target_format_unsupported")
)
return { reason: "target_unsupported", remediation: descriptor.reason };
if (
targetKind !== undefined &&
targetKind !== "executable" &&
src/application/binary/BinarySessionStatus.test.ts
+33 -1
View File
@@ -1,3 +1,4 @@
import type { BinaryTarget } from "../../domain/binaryTargetTypes.js";
import { describe, expect, it } from "vitest";
import type {
@@ -124,6 +125,7 @@ const project = async (
string,
{ readonly reason: string }
> = new Map(),
target?: BinaryTarget,
) => {
const route: SessionProviderRoute = {
identity: providerA,
@@ -142,7 +144,7 @@ const project = async (
};
return projectedStatusSchema.parse(
await binarySessionStatus({
target: undefined,
target,
route,
router,
runtimeUnavailability,
@@ -412,3 +414,33 @@ describe("binary session provider capability projection", () => {
]);
});
});
it.each(["elf", "pe"] as const)(
"reports format rejection for %s dylib discovery",
async (format) => {
const target: BinaryTarget = {
kind: "executable",
format,
path: "/fixture",
sha256: "a".repeat(64),
architecture: "x86_64",
availableArchitectures: ["x86_64"],
...(format === "pe"
? { executableRole: "application" as const, managed: false }
: {}),
} as BinaryTarget;
const status = await project(
[descriptor(providerA, "trace_dylib_resolution", [])],
[],
new Map(),
target,
);
expect(expandCanonicalCapabilities(status.capabilities)).toMatchObject([
{
operation: "trace_dylib_resolution",
available: false,
availability_code: "target_format_unsupported",
},
]);
},
);
src/application/binary/BinarySessionStatus.ts
+12
View File
@@ -1,3 +1,4 @@
import { artifactTargetRefusal } from "../../domain/apple/artifactTargetApplicability.js";
import type { BinaryTarget } from "../../domain/binaryTargetTypes.js";
import type { ResolvedAppBundleResourceRoot } from "../AppBundleExecutable.js";
import { binaryTargetSummary } from "./BinaryTargetSummary.js";
@@ -62,6 +63,17 @@ export const binarySessionStatus = async ({
compareUnicodeCodePoints(left.operation, right.operation),
)
.map((descriptor) => {
const targetRefusal =
target === undefined
? undefined
: artifactTargetRefusal(descriptor.operation, target);
if (targetRefusal !== undefined && descriptor.available)
return {
...descriptor,
available: false as const,
availabilityCode: targetRefusal.code,
reason: targetRefusal.reason,
};
const unavailable = runtimeUnavailability.get(descriptor.operation);
return unavailable === undefined
? descriptor
src/application/environment/environmentBundleWrite.test.ts
+110
View File
@@ -19,6 +19,10 @@ import {
} from "../../../tests/fixtures/environmentServe.js";
import { SafeOutputTree } from "../../artifacts/SafeOutputTree.js";
import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js";
import {
AnalysisAccessDeniedError,
AnalysisOutputError,
} from "../../domain/analysisErrorCore.js";
import {
exportEnvironmentBundle,
readEnvironmentExportInputs,
@@ -186,6 +190,7 @@ it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
error: {
_tag: "AnalysisAccessDeniedError",
operation: "environment-export",
diagnostics: { output_directory: directory },
},
});
expect(await readdir(directory)).toEqual([]);
@@ -195,6 +200,111 @@ it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
},
);
it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
"names the nested path when write permission is lost after earlier files",
async () => {
const root = await createTestTempDirectory("rea-bundle-later-denied-");
const files = await writeEnvironmentServeFiles(root);
const inputs = await readEnvironmentExportInputs(
files.manifestPath,
files.capturePath,
);
if (!inputs.ok) throw inputs.error;
const directory = join(root, "bundle");
const original = SafeOutputTree.prototype.write;
let writes = 0;
// The real host refuses the second file once the directory is read-only.
const writer = vi
.spyOn(SafeOutputTree.prototype, "write")
.mockImplementation(async function (this: SafeOutputTree, ...args) {
writes += 1;
if (writes === 2) await chmod(directory, 0o500);
return original.apply(this, args);
});
try {
const result = await exportEnvironmentBundle({
...inputs.value,
source: environmentServeSource(),
directory,
});
if (result.ok) throw new Error("expected a permission refusal");
expect(result.error).toMatchObject({
_tag: "AnalysisAccessDeniedError",
diagnostics: { output_directory: directory },
});
if (!(result.error instanceof AnalysisAccessDeniedError))
throw result.error;
expect(result.error.path.startsWith(`${directory}/`)).toBe(true);
expect(await readdir(directory)).not.toEqual([]);
} finally {
writer.mockRestore();
await chmod(directory, 0o700);
}
},
);
it.each([
{
failure: "pathless descriptor failure",
// A disk-full `FileHandle.write()` reports a code and syscall, no path.
error: () =>
Object.assign(new Error("ENOSPC: no space left on device, write"), {
code: "ENOSPC",
syscall: "write",
}),
},
{
failure: "integrity refusal",
error: () =>
new ArtifactReaderFailure(
"integrity",
"Extracted content disagrees with inventory",
),
},
] as const)(
"reports a later $failure with its own recovery",
async ({ failure, error }) => {
const root = await createTestTempDirectory("rea-bundle-later-failure-");
const files = await writeEnvironmentServeFiles(root);
const inputs = await readEnvironmentExportInputs(
files.manifestPath,
files.capturePath,
);
if (!inputs.ok) throw inputs.error;
const directory = join(root, "bundle");
const original = SafeOutputTree.prototype.write;
const targets: string[] = [];
const writer = vi
.spyOn(SafeOutputTree.prototype, "write")
.mockImplementation(async function (this: SafeOutputTree, ...args) {
targets.push(args[0]);
if (targets.length === 2) throw error();
return original.apply(this, args);
});
try {
const result = await exportEnvironmentBundle({
...inputs.value,
source: environmentServeSource(),
directory,
});
if (result.ok || !(result.error instanceof AnalysisOutputError))
throw new Error("expected an output failure");
expect(result.error.diagnostics).toEqual({
output_directory: directory,
});
if (failure === "pathless descriptor failure") {
expect(result.error.reason).toContain(
join(directory, targets[1] ?? ""),
);
expect(result.error.remediationAction).toContain("free disk space");
} else expect(result.error.remediationAction).toBeUndefined();
expect(await readdir(directory)).not.toEqual([]);
} finally {
writer.mockRestore();
}
},
);
it
.skipIf(process.platform === "win32")
.each(["replace-root", "foreign-entry", "symlink-parent"] as const)(
src/application/environment/environmentBundleWrite.ts
+112 -17
View File
@@ -1,6 +1,6 @@
import { createHash } from "node:crypto";
import { mkdir, type FileHandle } from "node:fs/promises";
import { dirname } from "node:path";
import { dirname, join } from "node:path";
import { Readable } from "node:stream";
import {
@@ -12,6 +12,7 @@ import { ArtifactFileCleanupFailure } from "../../artifacts/ArtifactFileOperatio
import { RegularFileCleanupFailure } from "../../filesystem/OwnedRegularFile.js";
import { readFileHandleChunks } from "../../filesystem/readFileHandleChunks.js";
import { streamChunkToBuffer } from "../../artifacts/StreamBytes.js";
import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js";
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
import { SafeOutputTree } from "../../artifacts/SafeOutputTree.js";
import { SafeOutputTreeCreationFailure } from "../../artifacts/SafeOutputTreeCreationFailure.js";
@@ -26,15 +27,69 @@ import type {
WrittenBundleFile,
} from "./environmentExport.js";
/**
* A host failure on one file the bundle writer was reading or writing.
* Descriptor operations such as `read`, `write` and `fsync` fail without a
* path, so this names the file and the direction of the failed operation.
*/
export class BundleFileHostFailure extends Error {
constructor(
readonly path: string,
readonly operation: "reading" | "writing",
cause: Error,
) {
super(`${cause.message}, ${operation} ${path}`, { cause });
}
}
/** An untyped host system failure, which carries an errno-style code. */
const isHostSystemFailure = (cause: unknown): cause is Error =>
cause instanceof Error &&
!(cause instanceof ArtifactReaderFailure) &&
typeof Reflect.get(cause, "code") === "string";
/** Attribute a pathless host failure to the bundle file being written. */
const writingBundleFile = async <Result>(
directory: string,
relative: string,
write: () => Promise<Result>,
): Promise<Result> => {
try {
return await write();
} catch (cause: unknown) {
if (isHostSystemFailure(cause) && Reflect.get(cause, "path") === undefined)
throw new BundleFileHostFailure(
join(directory, relative),
"writing",
cause,
);
throw cause;
}
};
/** Read the admitted tarball, attributing host failures to the source. */
async function* tarballChunks(source: BundleTarball): AsyncIterable<unknown> {
try {
yield* readFileHandleChunks(source.handle);
} catch (cause: unknown) {
if (isHostSystemFailure(cause))
throw new BundleFileHostFailure(source.path, "reading", cause);
throw cause;
}
}
const writeBytes = async (
tree: SafeOutputTree,
directory: string,
relative: string,
bytes: Buffer,
): Promise<WrittenBundleFile> => {
const written = await tree.write(relative, Readable.from([bytes]), {
bytes: bytes.length,
sha256: createHash("sha256").update(bytes).digest("hex"),
});
const written = await writingBundleFile(directory, relative, () =>
tree.write(relative, Readable.from([bytes]), {
bytes: bytes.length,
sha256: createHash("sha256").update(bytes).digest("hex"),
}),
);
return { path: written.relativePath, bytes: written.bytesWritten };
};
@@ -47,29 +102,50 @@ interface BundleTarball {
/** Hash and copy only the instance retained before destination admission. */
const writeTarball = async (
tree: SafeOutputTree,
directory: string,
relative: string,
source: BundleTarball,
): Promise<WrittenBundleFile> => {
const hash = createHash("sha256");
let observed = 0;
for await (const chunk of readFileHandleChunks(source.handle)) {
for await (const chunk of tarballChunks(source)) {
const bytes = streamChunkToBuffer(chunk);
observed += bytes.length;
if (observed > source.size) throw new RegularFileChangedError(source.path);
hash.update(bytes);
}
if (observed !== source.size) throw new RegularFileChangedError(source.path);
const written = await tree.write(
relative,
readFileHandleChunks(source.handle),
{
const sha256 = hash.digest("hex");
const written = await writingBundleFile(directory, relative, () =>
tree.write(relative, Readable.from(tarballChunks(source)), {
bytes: source.size,
sha256: hash.digest("hex"),
},
sha256,
}),
);
return { path: written.relativePath, bytes: written.bytesWritten };
};
/**
* Only destination admission can blame the selected directory. Path failures
* after admission are host write failures on bundle files, not argument refusals.
*/
const directoryRefusal = (cause: unknown): unknown =>
cause instanceof ArtifactReaderFailure &&
cause.reason === "path" &&
cause.cleanup === undefined
? new AnalysisInputError(
ENVIRONMENT_EXPORT_OPERATION,
{ cause, userMessage: cause.message },
[
{
path: ["directory"],
reason: "invalid_value",
message: cause.message,
},
],
)
: cause;
/** The output owner anchors every create and verifies the complete tree before publication. */
const writeBundleFiles = async (
input: EnvironmentExportInput,
@@ -79,6 +155,7 @@ const writeBundleFiles = async (
const resources = new ArtifactResourceScope();
let tree: SafeOutputTree | undefined;
let written: WrittenBundleFile[] | undefined;
let admitted = false;
let outcome:
| {
readonly kind: "completed";
@@ -95,19 +172,37 @@ const writeBundleFiles = async (
if (cause instanceof SafeOutputTreeCreationFailure) tree = cause.tree;
throw cause;
}
admitted = true;
written = [];
for (const file of plan.files)
written.push(
await writeBytes(tree, file.path, Buffer.from(file.content)),
await writeBytes(
tree,
input.directory,
file.path,
Buffer.from(file.content),
),
);
written.push(
await writeBytes(tree, plan.copies.manifest, input.manifest.bytes),
await writeBytes(
tree,
input.directory,
plan.copies.manifest,
input.manifest.bytes,
),
);
written.push(
await writeBytes(tree, plan.copies.evidence, input.evidence.bytes),
await writeBytes(
tree,
input.directory,
plan.copies.evidence,
input.evidence.bytes,
),
);
if (plan.copies.tarball !== null && tarball !== undefined)
written.push(await writeTarball(tree, plan.copies.tarball, tarball));
written.push(
await writeTarball(tree, input.directory, plan.copies.tarball, tarball),
);
await tree.commit();
outcome = { kind: "completed", files: written };
} catch (cause: unknown) {
@@ -125,7 +220,7 @@ const writeBundleFiles = async (
}
if (outcome.kind === "failed") {
if (owner?.published === true && written !== undefined) return written;
throw outcome.cause;
throw admitted ? outcome.cause : directoryRefusal(outcome.cause);
}
return outcome.files;
};
src/application/environment/environmentExport.test.ts
+1 -100
View File
@@ -1,8 +1,7 @@
import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
import { join, sep } from "node:path";
import * as v8 from "node:v8";
import { describe, expect, it, vi } from "vitest";
import { describe, expect, it } from "vitest";
import {
environmentServeFixture,
@@ -12,17 +11,11 @@ import {
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
import { PACKAGE_METADATA } from "../../generatedPackageMetadata.js";
import { RegularFileChangedError } from "../../filesystem/RegularFile.js";
import { nativeJsonParseRequiredBytes } from "../JsonInputFile.js";
import * as regularFileRead from "../RegularFileRead.js";
import {
exportEnvironmentBundle,
readEnvironmentExportInputs,
} from "./environmentExport.js";
vi.mock("node:v8", { spy: true });
vi.mock("../RegularFileRead.js", { spy: true });
const listFiles = async (root: string): Promise<string[]> =>
(await readdir(root, { recursive: true, withFileTypes: true }))
.filter((entry) => entry.isFile())
@@ -52,41 +45,6 @@ const prepare = async (prefix: string) => {
};
describe("readEnvironmentExportInputs", () => {
it.each(["manifest", "evidence"] as const)(
"returns a tagged input error when the %s snapshot changes",
async (role) => {
const root = await createTestTempDirectory(
"rea-environment-input-changed-",
);
const files = await writeEnvironmentServeFiles(root);
const changedPath =
role === "manifest" ? files.manifestPath : files.capturePath;
const actual = await vi.importActual<typeof regularFileRead>(
"../RegularFileRead.js",
);
const reader = vi.mocked(regularFileRead.readRegularFile);
reader.mockImplementation(async (path, options) => {
if (path === changedPath) throw new RegularFileChangedError(path);
return actual.readRegularFile(path, options);
});
try {
const result = await readEnvironmentExportInputs(
files.manifestPath,
files.capturePath,
);
if (result.ok) throw new Error("Expected changed input refusal");
expect(projectAnalysisError(result.error)).toMatchObject({
code: "invalid_request",
details: {
operation: "environment-export",
issues: [{ path: [role], reason: "invalid_value" }],
},
});
} finally {
reader.mockImplementation(actual.readRegularFile);
}
},
);
it("reads each file once as the bytes on disk and the value they parse to", async () => {
const root = await createTestTempDirectory("rea-environment-inputs-");
const fixture = environmentServeFixture();
@@ -107,32 +65,6 @@ describe("readEnvironmentExportInputs", () => {
expect(inputs.value.evidence.value).toEqual(fixture.capture);
});
it("refuses an input whose whole read and parse would not fit live heap headroom, before reading it", async () => {
const root = await createTestTempDirectory("rea-environment-export-heap-");
const files = await writeEnvironmentServeFiles(
root,
environmentServeFixture(),
);
vi.mocked(v8.getHeapStatistics).mockReturnValueOnce({
...v8.getHeapStatistics(),
total_available_size: 1024,
});
const refused = await readEnvironmentExportInputs(
files.manifestPath,
files.capturePath,
);
expect(refused.ok).toBe(false);
if (refused.ok) return;
expect(refused.error).toMatchObject({
_tag: "AnalysisResourceConstraintError",
resource: "memory",
reportedLimits: {
input_path: files.manifestPath,
available_heap_bytes: 1024,
},
});
});
it("refuses inline JSON, a directory and a file that is not JSON by role", async () => {
const root = await createTestTempDirectory("rea-environment-inputs-bad-");
const files = await writeEnvironmentServeFiles(root);
@@ -316,34 +248,3 @@ describe("exportEnvironmentBundle file modes", () => {
},
);
});
describe("readEnvironmentExportInputs heap admission", () => {
it("reserves headroom for snapshot sealing after the native parse", async () => {
const root = await createTestTempDirectory(
"rea-environment-export-copies-",
);
const files = await writeEnvironmentServeFiles(
root,
environmentServeFixture(),
);
const { size } = await stat(files.manifestPath);
// Room for parsing, but not for the owned snapshot traversal that follows it.
vi.mocked(v8.getHeapStatistics).mockReturnValueOnce({
...v8.getHeapStatistics(),
total_available_size: nativeJsonParseRequiredBytes(size) * 1.5,
});
const refused = await readEnvironmentExportInputs(
files.manifestPath,
files.capturePath,
);
expect(refused.ok).toBe(false);
if (refused.ok) return;
expect(refused.error).toMatchObject({
_tag: "AnalysisResourceConstraintError",
reportedLimits: {
input_path: files.manifestPath,
required_heap_bytes: nativeJsonParseRequiredBytes(size, true),
},
});
});
});
src/application/environment/environmentExport.ts
+74 -16
View File
@@ -1,3 +1,4 @@
import { isAbsolute } from "node:path";
import { getHeapStatistics } from "node:v8";
import { AnalysisError } from "../../domain/analysisErrorBase.js";
@@ -21,7 +22,10 @@ import { AnalysisCleanupLease } from "../../artifacts/AnalysisCleanupLease.js";
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
import { RegularFileCleanupFailure } from "../../filesystem/OwnedRegularFile.js";
import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js";
import { writeEnvironmentBundle } from "./environmentBundleWrite.js";
import {
BundleFileHostFailure,
writeEnvironmentBundle,
} from "./environmentBundleWrite.js";
import { sealOwnedJsonInput } from "../OwnedJsonInput.js";
import { nativeJsonParseRequiredBytes } from "../JsonInputFile.js";
import { readRegularFile } from "../RegularFileRead.js";
@@ -94,9 +98,36 @@ const systemCode = (cause: unknown): string | undefined =>
? cause.code
: undefined;
/** Recovery for a host write failure that may leave partial output behind. */
const hostWriteFailure = (directory: string) => ({
userMessage: `Writing the environment bundle to ${directory} failed on the host; the details name the cause and, when it failed on one bundle file, that file.`,
remediationAction: `Resolve the reported host failure (for example, choose a shorter destination path or free disk space). Inspect ${directory} and remove any partial output this export wrote before retrying, since a retry refuses a non-empty directory.`,
diagnostics: { output_directory: directory },
});
/** Recovery for a host failure reading the package tarball during the copy. */
const hostReadFailure = (source: string, directory: string) => ({
userMessage: `Reading ${source} failed on the host while writing the environment bundle to ${directory}; the details name the cause.`,
remediationAction: `Resolve the reported host failure reading ${source}. Inspect ${directory} and remove any partial output this export wrote before retrying, since a retry refuses a non-empty directory.`,
diagnostics: { output_directory: directory },
});
/** The host failure beneath a wrapper, whose message names its code and absolute path. */
const hostSystemError = (cause: Error): Error | undefined => {
const seen = new Set<unknown>([cause]);
let current = cause.cause;
while (current instanceof Error && !seen.has(current)) {
if (systemCode(current) !== undefined) return current;
seen.add(current);
current = current.cause;
}
return undefined;
};
const accessDeniedError = (
cause: unknown,
path: string,
diagnostics?: Readonly<Record<string, JsonValue>>,
): AnalysisAccessDeniedError | undefined => {
const seen = new Set<Error>();
let current = cause;
@@ -110,6 +141,7 @@ const accessDeniedError = (
code,
{
cause,
...(diagnostics === undefined ? {} : { diagnostics }),
...(cause instanceof ArtifactReaderFailure &&
cause.cleanup !== undefined
? { cleanup: cause.cleanup }
@@ -124,6 +156,21 @@ const accessDeniedError = (
return undefined;
};
/** The absolute path a host failure beneath wrappers names, if it names one. */
const hostFailurePath = (cause: unknown): string | undefined => {
const seen = new Set<unknown>();
let current = cause;
while (current instanceof Error && !seen.has(current)) {
seen.add(current);
if (current instanceof BundleFileHostFailure) return current.path;
const path = Reflect.get(current, "path");
if (systemCode(current) !== undefined && typeof path === "string")
return isAbsolute(path) ? path : undefined;
current = current.cause;
}
return undefined;
};
/** Host permission failures retain their typed cause; unexpected failures propagate. */
const rethrowUnlessDenied = (cause: unknown, path: string): never => {
throw accessDeniedError(cause, path) ?? cause;
@@ -301,7 +348,13 @@ export const exportEnvironmentBundle = async (
files = await writeEnvironmentBundle(input, plan.value);
} catch (cause: unknown) {
if (cause instanceof AnalysisError) return err(cause);
const denied = accessDeniedError(cause, input.directory);
// A refused bundle write names the path the host refused, which may be a
// nested file after earlier files were written into the directory.
const denied = accessDeniedError(
cause,
hostFailurePath(cause) ?? input.directory,
{ output_directory: input.directory },
);
if (denied !== undefined) return err(denied);
if (cause instanceof RegularFileChangedError)
return err(
@@ -313,32 +366,37 @@ export const exportEnvironmentBundle = async (
),
);
if (cause instanceof ArtifactReaderFailure) {
if (cause.reason === "path" && cause.cleanup === undefined)
return err(
inputError(
{
path: ["directory"],
reason: "invalid_value",
message: cause.message,
},
cause.message,
),
);
const host = hostSystemError(cause);
const error = new AnalysisOutputError(
ENVIRONMENT_EXPORT_OPERATION,
cause.message,
host === undefined
? cause.message
: `${cause.message}: ${host.message}`,
{
cause,
...(cause.cleanup === undefined ? {} : { cleanup: cause.cleanup }),
// Only a host failure gets host recovery; an integrity or other
// refusal keeps its own message without disk-space advice.
...(cause.cleanup !== undefined
? { cleanup: cause.cleanup }
: host === undefined
? { diagnostics: { output_directory: input.directory } }
: hostWriteFailure(input.directory)),
},
);
bindArtifactFileCleanup(cause, error);
return err(error);
}
if (cause instanceof Error && systemCode(cause) !== undefined)
if (
cause instanceof BundleFileHostFailure ||
(cause instanceof Error && systemCode(cause) !== undefined)
)
return err(
new AnalysisOutputError(ENVIRONMENT_EXPORT_OPERATION, cause.message, {
cause,
...(cause instanceof BundleFileHostFailure &&
cause.operation === "reading"
? hostReadFailure(cause.path, input.directory)
: hostWriteFailure(input.directory)),
}),
);
throw cause;
src/application/rust/RustBinaryService.test.ts
+69 -2
View File
@@ -4,7 +4,7 @@ import {
AnalysisInputError,
AnalysisOutputError,
} from "../../domain/analysisErrorCore.js";
import { parseEvidence } from "../../domain/evidence.js";
import { parseEvidence, type Evidence } from "../../domain/evidence.js";
import type { RustBinary } from "../../domain/rust/rustBinary.js";
import { ok } from "../../domain/result.js";
import { RustBinaryService } from "./RustBinaryService.js";
@@ -59,7 +59,7 @@ it("rejects a relative path and extra arguments before reading", async () => {
const relative = await service.inspect({ path: "program" });
const extra = await service.inspect({
path: report.artifact.path,
detail: "complete",
unrecognized: true,
});
expect(relative.ok).toBe(false);
expect(extra.ok).toBe(false);
@@ -221,3 +221,70 @@ it.each([
expect(reason.cleanupIncomplete).toBe(true);
},
);
it("requires retention and rejects contradictory detail before invoking the reader", async () => {
let calls = 0;
const service = new RustBinaryService({
identity,
inspect: async () => {
calls++;
return ok(report);
},
});
const missing = await service.inspect({
path: report.artifact.path,
detail: "summary",
});
expect(missing.ok).toBe(false);
const conflict = await service.inspect({
path: report.artifact.path,
detail: "complete",
view: { collection: "packages" },
});
if (conflict.ok) throw new Error("Contradictory delivery was accepted");
expect(conflict.error).toBeInstanceOf(AnalysisInputError);
expect(calls).toBe(0);
});
it("retains the unchanged parent and keeps delivery selectors out of provider inputs", async () => {
const inputs: unknown[] = [];
const service = new RustBinaryService({
identity,
inspect: async (input) => {
inputs.push(input);
return ok(report);
},
});
const first = await service.inspect({ path: report.artifact.path });
const explicit = await service.inspect({
path: report.artifact.path,
detail: "complete",
});
if (!first.ok || !explicit.ok) throw new Error("Complete delivery failed");
expect(explicit.value).toEqual(first.value);
const records = new Map<string, Evidence>();
const selected = await service.inspect(
{
path: report.artifact.path,
view: { collection: "packages", names: ["sample"], fields: ["name"] },
},
undefined,
{
recordEvidence: (evidence) => {
records.set(evidence.evidence_id, evidence);
return ok("added" as const);
},
evidenceById: (id) => records.get(id),
},
);
if (!selected.ok) throw selected.error;
expect(selected.value.normalized_result).toMatchObject({
kind: "rust",
parent_evidence_id: first.value.evidence_id,
page: { collection: "packages", rows: [{ index: 0, name: "sample" }] },
});
expect(records.get(first.value.evidence_id)).toEqual(first.value);
expect(inputs).toEqual(
Array.from({ length: 3 }, () => ({ path: report.artifact.path })),
);
});
src/application/rust/RustBinaryService.ts
+29 -20
View File
@@ -8,30 +8,22 @@ import {
AnalysisOutputError,
} from "../../domain/analysisErrorCore.js";
import { createEvidence, type Evidence } from "../../domain/evidence.js";
import { BINARY_ARCHITECTURES } from "../../domain/binaryTargetTypes.js";
import {
inspectRustBinaryInputSchema,
rustArchitectureForSubject,
rustBinarySchema,
} from "../../domain/rust/rustBinary.js";
import { inspectRustBinaryRequestSchema } from "../../domain/rust/rustBinaryQuery.js";
import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js";
import { err, ok, type Result } from "../../domain/result.js";
import { completeProviderEvidence } from "../ProviderOperationExecution.js";
import {
requireAnalysisViewRetention,
projectRetainedAnalysis,
type AnalysisViewRetention,
} from "../analysisView/AnalysisViewService.js";
const OPERATION = "inspect_rust_binary";
const architectureForSubject = (architecture: string) => {
switch (architecture) {
case "386":
return "x86";
case "amd64":
return "x86_64";
default:
return BINARY_ARCHITECTURES.find(
(supported) => supported === architecture,
);
}
};
/** Shared exact input validation and artifact-bound Evidence for CLI and MCP. */
export class RustBinaryService {
constructor(readonly provider: RustBinaryPort) {}
@@ -44,12 +36,16 @@ export class RustBinaryService {
async inspect(
rawInput: unknown,
options?: ExecutionOptions,
retention: AnalysisViewRetention = {
recordEvidence: undefined,
evidenceById: undefined,
},
): Promise<Result<Evidence, AnalysisError>> {
if (options?.signal?.aborted)
return err(
new AnalysisCancelledError(OPERATION, { cause: options.signal.reason }),
);
const input = inspectRustBinaryInputSchema.safeParse(rawInput);
const input = inspectRustBinaryRequestSchema.safeParse(rawInput);
if (!input.success)
return err(
analysisInputErrorFromIssues(OPERATION, input.error.issues, rawInput, {
@@ -66,7 +62,13 @@ export class RustBinaryService {
},
]),
);
const inspected = await this.provider.inspect(input.data, options);
const { detail, view, ...parameters } = input.data;
const compact = detail === "summary" || view !== undefined;
if (compact) {
const ready = requireAnalysisViewRetention(OPERATION, retention);
if (!ready.ok) return ready;
}
const inspected = await this.provider.inspect(parameters, options);
if (!inspected.ok) return inspected;
const report = rustBinarySchema.safeParse(inspected.value);
if (!report.success || report.data.artifact.path !== input.data.path)
@@ -76,10 +78,17 @@ export class RustBinaryService {
"Rust dependency reader returned malformed metadata or changed the selected artifact identity.",
),
);
return completeProviderEvidence(
evidenceFor(this.provider.identity, input.data, report.data),
const completed = completeProviderEvidence(
evidenceFor(this.provider.identity, parameters, report.data),
options?.signal,
);
return compact && completed.ok
? projectRetainedAnalysis(
completed.value,
retention,
view === undefined ? { kind: "summary" } : { kind: "rust", ...view },
)
: completed;
}
}
@@ -88,7 +97,7 @@ const evidenceFor = (
parameters: { readonly path: string },
value: ReturnType<typeof rustBinarySchema.parse>,
): Evidence => {
const architecture = architectureForSubject(value.architecture);
const architecture = rustArchitectureForSubject(value.architecture);
const section = value.auditable_metadata;
return createEvidence(
{
src/application/sqlite/SqliteDatabaseService.test.ts
+70
View File
@@ -4,11 +4,13 @@ import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCo
import { err } from "../../domain/result.js";
import { ok } from "../../domain/result.js";
import { createAnalysisExecution } from "../AnalysisProvider.js";
import { sqliteDatabaseSchema } from "../../domain/sqlite/sqliteDatabase.js";
const fixture = () => ({
artifact: { path: "/selected.db", sha256: "a".repeat(64), bytes: 4096 },
wal: null,
engine: { name: "SQLite", version: "3.50.2" },
worker_timeout_ms: 30_000,
database_encoding: "UTF-8",
schema: {
scope: "inventory",
@@ -43,6 +45,10 @@ it.each([
table: "data",
row_limit: Number.MAX_SAFE_INTEGER + 1,
},
...[0, -1, 1.5, 2_147_483_648, "1000", null].map((worker_timeout_ms) => ({
path: "/selected.db",
worker_timeout_ms,
})),
])(
"rejects invalid SQLite selection before provider effects: %j",
async (input) => {
@@ -58,6 +64,70 @@ it.each([
},
);
it("preserves the selected worker budget in provider input and Evidence", async () => {
const report = { ...fixture(), worker_timeout_ms: 1234 };
const service = new SqliteDatabaseService({
inspect: async (input) => {
expect(input.worker_timeout_ms).toBe(1234);
return ok(execution(report));
},
});
const result = await service.inspect({
path: "/selected.db",
worker_timeout_ms: 1234,
});
if (!result.ok) throw result.error;
expect(result.value.parameters).toMatchObject({ worker_timeout_ms: 1234 });
expect(result.value.normalized_result).toMatchObject({
worker_timeout_ms: 1234,
});
});
it.each([
{ selected: undefined, reported: undefined },
{ selected: undefined, reported: 1234 },
{ selected: 1234, reported: undefined },
{ selected: 1234, reported: 30_000 },
])(
"rejects a missing or changed fresh producer budget: %j",
async ({ selected, reported }) => {
const { worker_timeout_ms: _default, ...report } = fixture();
const service = new SqliteDatabaseService({
inspect: async () =>
ok(
execution({
...report,
...(reported === undefined ? {} : { worker_timeout_ms: reported }),
}),
),
});
expect(
await service.inspect({
path: "/selected.db",
...(selected === undefined ? {} : { worker_timeout_ms: selected }),
}),
).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } });
},
);
it("retains the actual default without inventing an explicit caller selection", async () => {
const service = new SqliteDatabaseService({
inspect: async () => ok(execution(fixture())),
});
const result = await service.inspect({ path: "/selected.db" });
if (!result.ok) throw result.error;
expect(result.value.normalized_result).toMatchObject({
worker_timeout_ms: 30_000,
});
expect(result.value.parameters).not.toHaveProperty("worker_timeout_ms");
});
it("parses historical report omissions as unknown without supplying a default", () => {
const { worker_timeout_ms: _default, ...historical } = fixture();
const parsed = sqliteDatabaseSchema.parse(historical);
expect(parsed).not.toHaveProperty("worker_timeout_ms");
});
it("preserves producer failures and rejects pre-cancelled requests before effects", async () => {
const failure = new AnalysisCapabilityUnavailableError(
"rea-sqlite",
src/application/sqlite/SqliteDatabaseService.ts
+5 -1
View File
@@ -18,6 +18,7 @@ import {
type InspectSqliteDatabaseInput,
type SqliteDatabase,
SQLITE_ROW_LIMIT_DEFAULT,
DEFAULT_SQLITE_WORKER_TIMEOUT_MS,
sqliteSchemaScope,
sqliteInspectionParameters,
} from "../../domain/sqlite/sqliteDatabase.js";
@@ -36,6 +37,9 @@ const matchesSelectedDatabase = (
subject.path === input.path &&
subject.sha256 === report.artifact.sha256 &&
subject.format === "file" &&
// Fresh executions must report known settings; only historical parsing permits omission.
report.worker_timeout_ms ===
(input.worker_timeout_ms ?? DEFAULT_SQLITE_WORKER_TIMEOUT_MS) &&
(report.wal === null || report.wal.path === `${input.path}-wal`) &&
report.schema.scope === sqliteSchemaScope(input) &&
(report.schema.scope !== "selected_table" ||
@@ -105,7 +109,7 @@ export class SqliteDatabaseService {
return err(
new AnalysisOutputError(
OPERATION,
"SQLite adapter returned malformed data or changed the selected database identity/table/row limit.",
"SQLite adapter returned malformed data or changed the selected database identity/table/row limit/worker budget.",
),
);
return ok(
src/application/wasm/WasmArtifactService.test.ts
+37
View File
@@ -10,6 +10,10 @@ it.each([
{ path: "/module", glue_paths: ["relative"] },
{ path: "/module", candidate_paths: ["relative"] },
{ path: "/module", fetch: true },
...[0, -1, 1.5, 2_147_483_648, "1000", null].map((command_timeout_ms) => ({
path: "/module",
command_timeout_ms,
})),
])("rejects invalid selection before effects: %j", async (input) => {
const service = new WasmArtifactService({
close: () => Promise.resolve(),
@@ -22,6 +26,39 @@ it.each([
error: { _tag: "AnalysisInputError" },
});
});
it("preserves the selected command budget in provider input and Evidence", async () => {
const value = wasmArtifactFixture();
value.tool_profile.timeout_ms = 42_000;
const service = new WasmArtifactService({
close: () => Promise.resolve(),
inspect: async (input) => {
expect(input).toMatchObject({ command_timeout_ms: 42_000 });
return ok(wasmArtifactExecution(value));
},
});
const result = await service.inspect({
path: value.artifact.path,
command_timeout_ms: 42_000,
});
if (!result.ok) throw result.error;
expect(result.value.parameters).toMatchObject({ command_timeout_ms: 42_000 });
expect(result.value.normalized_result).toEqual(value);
});
it("rejects a provider report that silently changes the selected command budget", async () => {
const value = wasmArtifactFixture();
const service = new WasmArtifactService({
close: () => Promise.resolve(),
inspect: async () => ok(wasmArtifactExecution(value)),
});
expect(
await service.inspect({
path: value.artifact.path,
command_timeout_ms: 42_000,
}),
).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } });
});
it.each(["path", "digest", "wat", "candidates", "glue"])(
"rejects mismatched upstream observation: %s",
async (problem) => {
src/application/wasm/WasmArtifactService.ts
+3
View File
@@ -13,6 +13,7 @@ import {
inspectWasmArtifactInputSchema,
wasmArtifactSchema,
wasmArtifactMatchesSubject,
DEFAULT_WASM_COMMAND_TIMEOUT_MS,
} from "../../domain/wasm/wasmArtifact.js";
import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js";
import { err, type Result } from "../../domain/result.js";
@@ -73,6 +74,8 @@ export class WasmArtifactService {
if (
!report.success ||
report.data.artifact.path !== input.data.path ||
report.data.tool_profile.timeout_ms !==
(input.data.command_timeout_ms ?? DEFAULT_WASM_COMMAND_TIMEOUT_MS) ||
!wasmArtifactMatchesSubject(report.data.artifact, execution.subject) ||
JSON.stringify(report.data.glue.map((item) => item.artifact.path)) !==
JSON.stringify(input.data.glue_paths) ||
src/artifacts/ArtifactOperationFailure.ts
+5 -3
View File
@@ -5,6 +5,7 @@ import {
AnalysisUnsupportedTargetError,
} from "../domain/analysisErrorCore.js";
import { ArtifactOperationError } from "../domain/artifactOperationError.js";
import { BinaryTargetError } from "../domain/configurationErrors.js";
import { ArtifactReaderFailure } from "./ArtifactReader.js";
import {
artifactFileCancellationCause,
@@ -36,12 +37,13 @@ export const translateArtifactFailure = (
bindArtifactFileCleanup(cause, error);
return error;
}
// Caller-selection and unsupported-target failures are already typed; keep
// their correction details instead of reducing them to an I/O failure.
// Caller-selection, target and unsupported-target failures are already typed;
// keep their correction details instead of reducing them to an I/O failure.
if (
cause instanceof AnalysisInputError ||
cause instanceof AnalysisCancelledError ||
cause instanceof AnalysisUnsupportedTargetError
cause instanceof AnalysisUnsupportedTargetError ||
cause instanceof BinaryTargetError
)
return cause;
return new ArtifactOperationError(operation, "io");
src/artifacts/ArtifactPaths.test.ts
+31
View File
@@ -4,13 +4,44 @@ import {
ArtifactPathRegistry,
annotateEnglishUnicodeCaseCollisions,
ENGLISH_UNICODE_CASE_INVENTORY_LIMITATION,
normalizeArtifactPath,
} from "./ArtifactPaths.js";
const NOTE =
"a case-insensitive destination cannot store both. This comparison is not a filesystem observation.";
it.each(["drive", "escaped", "normalization", "prefix", "collision"])(
"bounds path failure diagnostics at the owning boundary: %s",
(kind) => {
const name = "x".repeat(1024 * 1024);
const registry = new ArtifactPathRegistry();
const fail = () => {
if (kind === "drive") normalizeArtifactPath("C:" + name);
else if (kind === "escaped")
normalizeArtifactPath("\0".repeat(name.length));
else if (kind === "normalization")
normalizeArtifactPath(name + "/../child");
else {
registry.add(name, "file");
registry.add(kind === "prefix" ? name + "/child" : name, "directory");
}
};
const failure = (() => {
try {
fail();
} catch (cause: unknown) {
return cause;
}
return undefined;
})();
expect(failure).toMatchObject({ reason: "path" });
expect((failure as Error).message.length).toBeLessThan(1024);
},
);
const annotate = (paths: readonly string[]) => {
const occurrences = paths.map((logical_path) => ({
entry_kind: "file" as const,
logical_path,
limitations: [] as string[],
}));
src/artifacts/ArtifactPaths.ts
+30 -10
View File
@@ -2,10 +2,25 @@ import { posix } from "node:path";
import { logicalPathNfc } from "../domain/artifactIdentity.js";
import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
import { ArtifactReaderFailure } from "./ArtifactReader.js";
import { ArtifactReaderFailure, type ArtifactEntry } from "./ArtifactReader.js";
const DRIVE_PREFIX = /^[A-Za-z]:/u;
// Bound diagnostic JSON expansion to 768 characters before formatting. This is
// a presentation budget, never a limit on accepted paths or retained facts.
const PATH_DIAGNOSTIC_CODE_UNITS = 128;
const pathDiagnostic = (path: string): string =>
path.length <= PATH_DIAGNOSTIC_CODE_UNITS
? JSON.stringify(path)
: `${JSON.stringify(path.slice(0, PATH_DIAGNOSTIC_CODE_UNITS))}… (${String(path.length)} UTF-16 code units)`;
type DestinationEntryKind = Exclude<ArtifactEntry["kind"], "metadata">;
/** Declarations have query identities, but never claim a destination path. */
export const hasDestinationPath = (
kind: ArtifactEntry["kind"],
): kind is DestinationEntryKind => kind !== "metadata";
/**
* English Unicode case folding used only to describe logical names a
* case-insensitive destination might collapse. It is not a filesystem case table.
@@ -25,7 +40,7 @@ export const normalizeArtifactPath = (input: string): string => {
if (input.includes("\0") || input.startsWith("/") || DRIVE_PREFIX.test(input))
throw new ArtifactReaderFailure(
"path",
`Artifact path is absolute or unsafe: ${JSON.stringify(input)}`,
`Artifact path is absolute or unsafe: ${pathDiagnostic(input)}`,
);
const normalized = logicalPathNfc(input).replace(/\/+$/u, "");
const parts = normalized.split("/");
@@ -36,7 +51,7 @@ export const normalizeArtifactPath = (input: string): string => {
)
throw new ArtifactReaderFailure(
"path",
`Artifact path is not normalized: ${JSON.stringify(input)}`,
`Artifact path is not normalized: ${pathDiagnostic(input)}`,
);
return normalized;
};
@@ -45,14 +60,15 @@ export const normalizeArtifactPath = (input: string): string => {
export class ArtifactPathRegistry {
readonly #root: PathTrieNode = { kind: undefined, children: new Map() };
add(path: string, kind: "file" | "directory" | "symlink" | "slice"): void {
add(path: string, kind: ArtifactEntry["kind"]): void {
if (!hasDestinationPath(kind)) return;
const parts = path.split("/");
let node = this.#root;
for (const [index, part] of parts.entries()) {
if (node.kind !== undefined && node.kind !== "directory")
throw new ArtifactReaderFailure(
"path",
`Artifact prefix conflict: ${path}`,
`Artifact prefix conflict: ${pathDiagnostic(path)}`,
);
let child = node.children.get(part);
if (child === undefined) {
@@ -64,12 +80,12 @@ export class ArtifactPathRegistry {
if (node.kind !== undefined && node.kind !== kind)
throw new ArtifactReaderFailure(
"path",
`Artifact path collision: ${path}`,
`Artifact path collision: ${pathDiagnostic(path)}`,
);
if (kind !== "directory" && node.children.size > 0)
throw new ArtifactReaderFailure(
"path",
`Artifact prefix conflict: ${path}`,
`Artifact prefix conflict: ${pathDiagnostic(path)}`,
);
}
}
@@ -78,7 +94,7 @@ export class ArtifactPathRegistry {
}
interface PathTrieNode {
kind: "file" | "directory" | "symlink" | "slice" | undefined;
kind: DestinationEntryKind | undefined;
readonly children: Map<string, PathTrieNode>;
}
@@ -103,15 +119,19 @@ const englishCaseFold = (part: string): string =>
*/
export const annotateEnglishUnicodeCaseCollisions = (
occurrences: readonly {
readonly entry_kind: ArtifactEntry["kind"];
readonly logical_path: string;
limitations: string[];
}[],
): string | undefined => {
const destinations = occurrences.filter(({ entry_kind }) =>
hasDestinationPath(entry_kind),
);
const collisions = englishUnicodeCaseCollisions(
occurrences.map(({ logical_path }) => logical_path),
destinations.map(({ logical_path }) => logical_path),
);
if (collisions.size === 0) return undefined;
for (const occurrence of occurrences)
for (const occurrence of destinations)
for (const collision of collisions.get(occurrence.logical_path) ?? [])
occurrence.limitations.push(
caseCollisionLimitation(occurrence.logical_path, collision),
src/artifacts/ArtifactProvider.ts
+8 -38
View File
@@ -1,3 +1,4 @@
import { artifactTargetRefusal } from "../domain/apple/artifactTargetApplicability.js";
import { bindArtifactFileCleanup } from "./ArtifactFileOperation.js";
import type { AnalysisError } from "../domain/analysisErrorBase.js";
import { translateArtifactFailure } from "./ArtifactOperationFailure.js";
@@ -24,10 +25,7 @@ import {
artifactExtractionExecutionSchema,
type ArtifactAnalysisOperation,
} from "../contracts/artifactToolContracts.js";
import {
isMachOFormat,
type BinaryTarget,
} from "../domain/binaryTargetTypes.js";
import type { BinaryTarget } from "../domain/binaryTargetTypes.js";
import {
AnalysisCapabilityUnavailableError,
AnalysisInputError,
@@ -215,23 +213,17 @@ class ArtifactClient implements AnalysisClient {
),
);
try {
const targetRefusal = artifactTargetRefusal(operation, this.target);
if (targetRefusal !== undefined)
throw this.unsupportedTarget(operation, targetRefusal.reason);
if (operation === "inspect_artifact") {
const inspected = await this.inspectArtifact(parameters, options);
return inspected;
}
if (operation === "decode_interface_builder") {
if (
this.target.kind !== "executable" ||
this.target.sourcePath === undefined ||
!this.target.sourcePath.toLowerCase().endsWith(".app")
)
throw this.unsupportedTarget(
operation,
"decode_interface_builder requires an active .app bundle target",
);
const limits = interfaceBuilderLimitsSchema.parse(parameters);
const result = await analyzeInterfaceBuilderBundle({
bundlePath: this.target.sourcePath,
bundlePath: this.target.sourcePath ?? this.target.path,
targetSha256: this.target.sha256,
limits,
...(options?.signal === undefined ? {} : { signal: options.signal }),
@@ -266,15 +258,7 @@ class ArtifactClient implements AnalysisClient {
]);
const bundlePath = standalone
? dirname(this.target.path)
: this.target.sourcePath;
if (
bundlePath === undefined ||
(!standalone && !bundlePath.toLowerCase().endsWith(".app"))
)
throw this.unsupportedTarget(
operation,
"inspect_keyed_archive requires an active plist or .app bundle",
);
: (this.target.sourcePath ?? this.target.path);
const result = await inspectBundleKeyedArchive({
bundlePath,
targetSha256: this.target.sha256,
@@ -386,16 +370,7 @@ class ArtifactClient implements AnalysisClient {
parameters: Readonly<Record<string, JsonValue>>,
options?: ExecutionOptions,
) {
const bundlePath = this.target.sourcePath;
if (
this.target.kind !== "executable" ||
bundlePath === undefined ||
!bundlePath.toLowerCase().endsWith(".app")
)
throw this.unsupportedTarget(
"inspect_asset_catalog",
"inspect_asset_catalog requires an active .app bundle target",
);
const bundlePath = this.target.sourcePath ?? this.target.path;
const result = await analyzeAppleAssetCatalogs({
environment: this.environment,
bundlePath,
@@ -418,11 +393,6 @@ class ArtifactClient implements AnalysisClient {
parameters: Readonly<Record<string, JsonValue>>,
options?: ExecutionOptions,
) {
if (this.target.kind !== "executable" || !isMachOFormat(this.target.format))
throw this.unsupportedTarget(
"trace_dylib_resolution",
"trace_dylib_resolution requires an active Mach-O or .app bundle target",
);
// Only a target opened from an app bundle directory carries its Info.plist;
// a regular file whose name ends in .app is a standalone image.
const bundle =
src/artifacts/ArtifactReader.ts
+2 -1
View File
@@ -33,7 +33,7 @@ export interface ArtifactReaderFailureOptions extends ErrorOptions {
/** Archive-neutral entry metadata. Reader adapters never choose output paths. */
export interface ArtifactEntry {
readonly path: string;
readonly kind: "file" | "directory" | "symlink" | "slice";
readonly kind: "file" | "directory" | "symlink" | "slice" | "metadata";
readonly declaredSize: number | null;
readonly compressedSize: number | null;
readonly executable: boolean;
@@ -167,6 +167,7 @@ export interface ArtifactReader {
| ZipPackageFormat
| "asar"
| "godot-pck"
| "pyinstaller"
| "file";
entries(
signal?: AbortSignal,
src/artifacts/DesktopApplicationReader.ts
+39 -1
View File
@@ -2,6 +2,7 @@ import { desktopPathConventions } from "../domain/nativeConvention.js";
import { isAbsolute } from "node:path";
import {
DirectoryArtifactReader,
DirectoryArtifactRootFailure,
DirectoryArtifactTraversalLimit,
type DirectoryArtifactEntry,
} from "./DirectoryArtifactReader.js";
@@ -33,7 +34,15 @@ import { digestCanonicalValue } from "../domain/canonicalDigest.js";
import { err, ok } from "../domain/result.js";
import { createDesktopApplicationEvidence } from "../domain/desktopApplicationEvidence.js";
import { ARTIFACT_GRAPH_PROVIDER } from "../application/InvestigationProviders.js";
import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js";
import {
AnalysisResourceConstraintError,
AnalysisUnsupportedTargetError,
} from "../domain/analysisErrorCore.js";
import type {
AnalysisError,
AnalysisPartialObservation,
} from "../domain/analysisErrorBase.js";
import { BinaryTargetError } from "../domain/configurationErrors.js";
const RETAINED_METADATA_BYTES = 32 * 1024 * 1024;
@@ -141,6 +150,8 @@ export class DesktopApplicationReader implements DesktopApplicationPort {
throw this.#resources.cleanupFailure(outcome, [closed], partial);
if (outcome.kind === "failed") {
const cause = outcome.cause;
if (cause instanceof DirectoryArtifactRootFailure)
throw selectedRootFailure(input.path, cause, partial);
if (!(cause instanceof ArtifactReaderFailure) || partial === undefined)
throw cause;
throw new ArtifactReaderFailure(
@@ -167,6 +178,33 @@ export class DesktopApplicationReader implements DesktopApplicationPort {
}
}
/**
* The root is the caller's selection: a non-directory is an unsupported
* target, and host failures keep the shared target-unavailable/access-denied
* classification instead of becoming artifact I/O faults.
*/
const selectedRootFailure = (
path: string,
failure: DirectoryArtifactRootFailure,
partialObservation: AnalysisPartialObservation | undefined,
): AnalysisError =>
failure.reason === "path"
? new AnalysisUnsupportedTargetError(
"inspect_desktop_application",
path,
failure.message,
{
cause: failure,
partialObservation,
remediationAction:
"Select the application's local installation directory, such as a macOS .app bundle or the directory containing its executable.",
},
)
: new BinaryTargetError(path, failure.message, {
cause: failure.cause,
partialObservation,
});
const scanDirectory = async (
reader: DirectoryArtifactReader,
input: InspectDesktopApplicationInput,
src/artifacts/DirectoryArtifactReader.ts
+23 -4
View File
@@ -60,6 +60,12 @@ export class DirectoryArtifactTraversalLimit extends ArtifactReaderFailure {
}
}
/**
* The selected root could not be admitted or opened. The consumer that owns
* the caller's selection classifies it; `cause` retains the host failure.
*/
export class DirectoryArtifactRootFailure extends ArtifactReaderFailure {}
/** Non-symlink-following directory reader. */
export class DirectoryArtifactReader implements ArtifactReader {
readonly format = "directory" as const;
@@ -133,7 +139,7 @@ export class DirectoryArtifactReader implements ArtifactReader {
"Artifact root changed during traversal",
);
if (!metadata.isDirectory())
throw new ArtifactReaderFailure(
throw new DirectoryArtifactRootFailure(
"path",
"Artifact root must be a directory",
);
@@ -143,7 +149,12 @@ export class DirectoryArtifactReader implements ArtifactReader {
});
})().catch((cause: unknown) => {
if (cause instanceof ArtifactReaderFailure) throw cause;
throw directoryIoFailure("resolve artifact root", this.#rootPath, cause);
throw directoryIoFailure(
"resolve artifact root",
this.#rootPath,
cause,
DirectoryArtifactRootFailure,
);
}));
}
@@ -158,7 +169,14 @@ export class DirectoryArtifactReader implements ArtifactReader {
const root = admittedRoot.canonicalPath;
const handle = new OwnedDirectoryHandle(
await opendir(directory).catch((cause: unknown) => {
throw directoryIoFailure("open directory", directory, cause);
throw directoryIoFailure(
"open directory",
directory,
cause,
directory === root
? DirectoryArtifactRootFailure
: ArtifactReaderFailure,
);
}),
);
this.#directories.add(handle);
@@ -484,6 +502,7 @@ const directoryIoFailure = (
operation: string,
path: string,
cause: unknown,
Failure: typeof ArtifactReaderFailure = ArtifactReaderFailure,
): ArtifactReaderFailure => {
if (cause instanceof ArtifactReaderFailure) return cause;
const code =
@@ -494,7 +513,7 @@ const directoryIoFailure = (
? ` (${cause.code})`
: "";
const detail = failureMessage(cause);
return new ArtifactReaderFailure(
return new Failure(
"io",
`Could not ${operation} at ${path}${code}: ${detail}`,
{ cause },
src/artifacts/PyInstallerArtifactReader.ts
+608
View File
@@ -0,0 +1,608 @@
import type { Readable } from "node:stream";
import { finished } from "node:stream/promises";
import { setImmediate } from "node:timers/promises";
import { jsonValueByteLength } from "../domain/jsonValueByteLength.js";
import type { JsonValue } from "../domain/jsonValue.js";
import { normalizeArtifactPath } from "./ArtifactPaths.js";
import { lstat, type FileHandle } from "node:fs/promises";
import type { PyInstallerMetadata } from "../domain/artifactFormatMetadata.js";
import {
cookieMetadata,
scriptPythonMagic,
} from "./pyinstaller/PyInstallerMetadata.js";
import { artifactExpansionBudgetBytes } from "../domain/artifactExpansionBudget.js";
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
import {
RegularFileAdmissionFailure,
openRegularFile,
sameRegularFileState,
type StableRegularFileDescriptor,
} from "../filesystem/RegularFile.js";
import {
ArtifactReaderFailure,
sameArtifactEntry,
type ArtifactEntry,
type ArtifactReader,
type ArtifactReaderTraversal,
type ArtifactTraversalOptions,
} from "./ArtifactReader.js";
import { abortIfNeeded, hashReadable } from "./ArtifactHash.js";
import { readArtifactFileChunks } from "./readArtifactFileChunks.js";
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
import { artifactIntegrityDetails } from "../domain/artifactIntegrity.js";
import { PyInstallerPyzStore } from "./pyinstaller/PyInstallerPyzStore.js";
import { pyInstallerMemberStream } from "./pyinstaller/PyInstallerMemberStream.js";
import {
findPyInstallerCookie,
readCArchiveToc,
type CArchiveEntry,
type PyInstallerCookie,
} from "./pyinstaller/CArchiveLayout.js";
import {
hasEncryptionMarker,
projectPyzContainer,
projectPyzMember,
projectStoredMember,
sealMember,
type PyInstallerMember,
type PyzImage,
} from "./pyinstaller/CArchiveMembers.js";
import { readPyzArchive } from "./pyinstaller/PyzArchive.js";
import type { PyInstallerFailure } from "./pyinstaller/PyInstallerFailure.js";
import { pyInstallerFileAdmissionFailure } from "./pyinstaller/PyInstallerFileAdmission.js";
const MAX_PARSED_FACT_BYTES = 32 * 1024 * 1024;
const MAX_RETAINED_ENTRY_BYTES = 64 * 1024 * 1024;
const RECORD_OWNERSHIP_RESERVATION = 256;
// Conservative allowance for registry/case trie nodes, maps and traversal state.
// Repeated prefixes are charged again rather than underestimating retained work.
const PATH_COMPONENT_RESERVATION = 1024;
interface LoadedArchive {
readonly members: readonly PyInstallerMember[];
readonly traversal: ArtifactReaderTraversal;
}
/**
* Read-only inventory of a PyInstaller CArchive appended to a native executable.
* Encrypted PYZ entries are reported and not decoded. Code members are not decompiled.
*/
export class PyInstallerArtifactReader implements ArtifactReader {
readonly format = "pyinstaller" as const;
#source: StableRegularFileDescriptor | undefined;
#opening: Promise<FileHandle> | undefined;
#closed = false;
readonly #scope = new ArtifactResourceScope();
readonly #cleanup = new ArtifactResourceScope();
readonly #lifecycle = new AbortController();
#boundSha256: string | undefined;
#loaded: Promise<LoadedArchive> | undefined;
#budget = 0;
#charged = 0;
#parsedFactBytes = 0;
#retainedEntryBytes = 0;
#metadata: PyInstallerMetadata | undefined;
readonly #pyzStore = new PyInstallerPyzStore();
readonly #members = new Map<string, PyInstallerMember>();
readonly #openedPyzSizes = new Map<string, number>();
constructor(
private readonly path: string,
source?: StableRegularFileDescriptor,
) {
this.#source = source;
this.#cleanup.retain({
kind: "lease",
resource: "PyInstaller private PYZ images",
lease: { release: () => this.#pyzStore.close() },
});
}
formatMetadata(): PyInstallerMetadata | undefined {
return this.#metadata === undefined ? undefined : { ...this.#metadata };
}
async prepareContainer(
expectedSha256: string,
signal?: AbortSignal,
): Promise<void> {
const active = this.#signal(signal);
await this.#scope.run(async () => {
const handle = await this.#openHandle(active);
await this.#verifySource();
const observed =
this.#boundSha256 ??
(
await hashReadable(
readArtifactFileChunks(handle, { signal: active }),
active,
)
).sha256;
await this.#verifySource();
abortIfNeeded(active);
if (observed !== expectedSha256)
throw new ArtifactReaderFailure(
"integrity",
"PyInstaller carrier changed after inventory",
undefined,
artifactIntegrityDetails(
".",
{ algorithm: "sha256", digest: expectedSha256 },
observed,
false,
),
);
this.#boundSha256 = observed;
});
}
async *entries(
signal?: AbortSignal,
options?: ArtifactTraversalOptions,
): AsyncIterable<ArtifactEntry> {
const active = this.#signal(signal);
const loaded = await this.#scope.run(async () => {
const value = await this.#load(active);
await this.#verifySource();
this.#members.clear();
return value;
});
options?.onCoverage?.(loaded.traversal);
for (const member of loaded.members) {
abortIfNeeded(active);
const sealed = sealMember(member);
this.#members.set(sealed.entry.adapterKey, sealed);
yield sealed.entry;
}
}
async open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable> {
const active = this.#signal(signal);
return this.#scope.run(async () => {
const stored = this.#members.get(entry.adapterKey);
if (stored === undefined || !sameArtifactEntry(entry, stored.entry))
throw new ArtifactReaderFailure(
"integrity",
`PyInstaller entry was not produced by this reader: ${entry.path}`,
);
if (entry.kind !== "file")
throw new ArtifactReaderFailure(
"format",
`PyInstaller entry is not a regular file: ${entry.path}`,
);
if (entry.encrypted)
throw new ArtifactReaderFailure(
"unavailable",
`PYZ entry is encrypted and was not decoded: ${entry.path}`,
);
const read = stored.read;
if (read === undefined)
throw new ArtifactReaderFailure(
"format",
`PyInstaller entry has no readable bytes: ${entry.path}`,
);
await this.#verifySource();
const knownSize =
read.kind === "pyz"
? (read.uncompressedSize ??
this.#openedPyzSizes.get(entry.adapterKey))
: undefined;
if (read.kind === "pyz" && knownSize === undefined)
this.#charge(read.header?.length ?? 0);
const parts = await this.#pyzStore.memberSource(
read,
this.#source?.handle,
active,
);
if (active.aborted) {
parts.source.destroy();
await finished(parts.source, { cleanup: true }).catch(() => undefined);
abortIfNeeded(active);
}
// Keep one close attempt in this read. Failed owners stay in the store
// for reader.close() to retry, without losing the original failure.
let closeAttempt: Promise<void> | undefined;
const closeSource = (primary?: unknown) =>
(closeAttempt ??= parts.close?.(primary));
const stream = pyInstallerMemberStream(parts.source, {
sourceBytes:
read.kind === "carchive" ? read.source.compressedSize : read.length,
compressed:
read.kind === "pyz"
? read.compressed
: read.source.compressionFlag === 1,
header: read.header,
expectedBytes:
read.kind === "carchive" ? read.source.uncompressedSize : knownSize,
maxBytes:
read.kind === "carchive"
? read.source.uncompressedSize
: (knownSize ?? this.#budget - this.#charged),
signal: active,
...(read.kind === "pyz" && knownSize === undefined
? { charge: (bytes: number) => this.#charge(bytes) }
: {}),
complete: async (bytes) => {
try {
await parts.verify?.();
await this.#verifySource();
abortIfNeeded(active);
} catch (cause: unknown) {
await closeSource(cause);
throw cause;
}
await closeSource();
abortIfNeeded(active);
if (read.kind === "pyz" && knownSize === undefined)
this.#openedPyzSizes.set(entry.adapterKey, bytes);
},
});
void this.#scope
.run(async () => {
let primary: unknown;
try {
await stream.completion;
} catch (cause: unknown) {
primary = cause;
}
await closeSource(primary);
if (primary !== undefined) throw primary;
})
.catch((cause: unknown) =>
stream.output.destroy(
cause instanceof Error
? cause
: new Error("PyInstaller stream failed", { cause }),
),
);
return stream.output;
});
}
provenance(): readonly [] {
return [];
}
async close(): Promise<void> {
this.#closed = true;
const closing = new ArtifactReaderFailure(
"unavailable",
"PyInstaller reader closed",
);
this.#lifecycle.abort(closing);
await this.#scope.close();
await this.#cleanup.close();
this.#source = undefined;
this.#closed = true;
this.#loaded = undefined;
this.#members.clear();
this.#openedPyzSizes.clear();
}
#signal(signal?: AbortSignal): AbortSignal {
abortIfNeeded(signal);
if (this.#closed)
throw new ArtifactReaderFailure(
"unavailable",
"PyInstaller archive is closed",
);
return signal === undefined
? this.#lifecycle.signal
: AbortSignal.any([signal, this.#lifecycle.signal]);
}
async #load(signal?: AbortSignal): Promise<LoadedArchive> {
this.#loaded ??= this.#parse(signal);
return this.#loaded;
}
async #parse(signal?: AbortSignal): Promise<LoadedArchive> {
const handle = await this.#openHandle(signal);
await this.#verifySource();
const size = (await handle.stat()).size;
const cookie = await this.#cookie(handle, size, signal);
this.#budget = artifactExpansionBudgetBytes(cookie.packageLength);
const toc = await this.#toc(handle, cookie, signal);
for (const entry of toc) {
this.#charge(entry.uncompressedSize);
this.#reserveMetadata({ ...entry }, false);
}
const images = await this.#pyzImages(handle, cookie, toc, signal);
const magic = scriptPythonMagic(
cookie,
[...images.values()].map((image) => image.archive),
);
const encrypted = hasEncryptionMarker(toc);
const members = await this.#project(
toc,
cookie,
images,
magic,
encrypted,
signal,
);
for (const member of members)
if (member.read?.kind === "carchive")
this.#charge(member.read.header?.length ?? 0);
await this.#verifySource();
this.#metadata = cookieMetadata(cookie, toc, encrypted);
let opaqueImages = 0;
for (const image of images.values())
if (image.archive.kind === "opaque") opaqueImages++;
return {
members,
traversal: {
status: opaqueImages === 0 ? "complete" : "unknown",
limitations:
opaqueImages === 0
? []
: [
`${String(opaqueImages)} PYZ archives remain opaque; module traversal is unknown. See opaque PYZ occurrences for source magic and per-archive limitations.`,
],
},
};
}
async #openHandle(signal?: AbortSignal): Promise<FileHandle> {
abortIfNeeded(signal);
if (this.#closed)
throw new ArtifactReaderFailure(
"unavailable",
"PyInstaller archive is closed",
);
if (this.#source !== undefined) return this.#source.handle;
// Admission belongs to the reader. A departing caller cannot cancel a peer
// or replace a late descriptor owner; every caller joins the same admission.
this.#opening ??= this.#admit();
let handle: FileHandle;
try {
handle = await this.#opening;
} catch (cause: unknown) {
if (
cause instanceof ArtifactReaderFailure &&
cause.reason === "cancelled" &&
signal?.aborted
)
throw new ArtifactReaderFailure("cancelled", cause.message, {
cause: signal.reason,
cleanup: cause.cleanup,
cleanupLeases: cause.cleanupLeases,
});
throw cause;
}
abortIfNeeded(signal);
return handle;
}
async #admit(): Promise<FileHandle> {
try {
const handle = await openRegularFile(this.path, {
symlinks: "reject",
signal: this.#lifecycle.signal,
});
this.#retainCarrier(new OwnedFileHandle(handle));
this.#source = { handle, initial: await handle.stat() };
return handle;
} catch (cause: unknown) {
if (cause instanceof RegularFileAdmissionFailure)
this.#retainCarrier(cause.owner);
throw pyInstallerFileAdmissionFailure(this.path, cause);
}
}
#retainCarrier(handle: OwnedFileHandle): void {
this.#cleanup.retain({ kind: "file-handle", resource: this.path, handle });
}
async #cookie(
handle: FileHandle,
size: number,
signal?: AbortSignal,
): Promise<PyInstallerCookie> {
const found = await findPyInstallerCookie(handle, size, signal);
if (found === undefined)
throw new ArtifactReaderFailure(
"format",
"PyInstaller CArchive cookie was not found",
);
if (!found.ok) throw failure(found.error);
return found.value;
}
async #toc(
handle: FileHandle,
cookie: PyInstallerCookie,
signal?: AbortSignal,
): Promise<readonly CArchiveEntry[]> {
const toc = await readCArchiveToc(handle, cookie, signal);
if (!toc.ok) throw failure(toc.error);
return toc.value;
}
async #pyzImages(
handle: FileHandle,
cookie: PyInstallerCookie,
toc: readonly CArchiveEntry[],
signal: AbortSignal | undefined,
): Promise<ReadonlyMap<CArchiveEntry, PyzImage>> {
const images = new Map<CArchiveEntry, PyzImage>();
// Admit the complete filesystem footprint before the first private write.
this.#pyzStore.reserveImages(
toc.reduce(
(count, entry) =>
count +
(entry.typeCode === "z" && entry.compressionFlag === 1 ? 1 : 0),
0,
),
);
for (const entry of toc) {
if (entry.typeCode !== "z") continue;
abortIfNeeded(signal);
const spooled =
entry.compressionFlag === 0
? undefined
: await this.#pyzStore.spool(
handle,
cookie.archiveStart,
entry,
signal,
);
const source = spooled ?? {
handle,
start: cookie.archiveStart + entry.offset,
};
const parsed = await readPyzArchive(
source.handle,
source.start,
entry.uncompressedSize,
signal,
);
if (!parsed.ok) throw failure(parsed.error);
if (parsed.value.kind === "entries")
for (const item of parsed.value.entries)
this.#reserveMetadata({ ...item, container: entry.name }, false);
if (spooled !== undefined) await this.#pyzStore.release(spooled.handle);
images.set(entry, {
archive: parsed.value,
handle: source.handle,
start: source.start,
});
}
return images;
}
async #project(
toc: readonly CArchiveEntry[],
cookie: PyInstallerCookie,
images: ReadonlyMap<CArchiveEntry, PyzImage>,
magic: Buffer | undefined,
encrypted: boolean,
signal: AbortSignal | undefined,
): Promise<PyInstallerMember[]> {
const members: PyInstallerMember[] = [];
const append = (member: PyInstallerMember): void => {
this.#reservePathComponents(member.entry.path);
const entry = {
...member.entry,
path: normalizeArtifactPath(member.entry.path),
};
this.#reserveMetadata(
{
path: entry.path,
adapter_key: entry.adapterKey,
format_metadata: entry.formatMetadata ?? null,
limitations: [...entry.limitations],
},
true,
);
members.push({ ...member, entry });
};
for (const entry of toc) {
if (members.length % 256 === 0) await setImmediate();
abortIfNeeded(signal);
const image = entry.typeCode === "z" ? images.get(entry) : undefined;
if (image === undefined) {
append(projectStoredMember(entry, cookie.archiveStart, magic));
continue;
}
append(projectPyzContainer(entry, cookie.archiveStart, image, encrypted));
if (image.archive.kind !== "entries") continue;
for (const item of image.archive.entries) {
if (members.length % 256 === 0) await setImmediate();
abortIfNeeded(signal);
append(
projectPyzMember(
entry,
item,
encrypted,
image.archive.pythonMagic,
item.namespace || encrypted
? undefined
: {
handle: image.handle,
offset: image.start + item.offset,
length: item.length,
},
),
);
}
}
return members;
}
async #verifySource(): Promise<void> {
const source = this.#source;
if (source === undefined)
throw new ArtifactReaderFailure(
"unavailable",
"PyInstaller archive is closed",
);
const [opened, currentPath] = await Promise.all([
source.handle.stat(),
lstat(this.path),
]);
if (
!sameRegularFileState(source.initial, opened) ||
!sameRegularFileState(source.initial, currentPath)
)
throw new ArtifactReaderFailure(
"integrity",
`PyInstaller source changed during reading: ${this.path}`,
);
}
#reservePathComponents(path: string): void {
const remaining = MAX_RETAINED_ENTRY_BYTES - this.#retainedEntryBytes;
// Case annotations can carry the path and two prefixes at every component.
// Reserve escaped text growth as well as registry/case trie nodes and maps.
const componentCost =
PATH_COMPONENT_RESERVATION + 6 * jsonValueByteLength(path);
let reservation = componentCost;
for (
let index = 0;
index < path.length && reservation <= remaining;
index++
)
if (path.charCodeAt(index) === 47) reservation += componentCost;
if (reservation > remaining)
throw new ArtifactReaderFailure(
"limit",
"PyInstaller paths exceed their retained namespace budget",
{ boundary: "memory" },
);
this.#retainedEntryBytes += reservation;
}
#reserveMetadata(value: JsonValue, projected: boolean): void {
const used = projected ? this.#retainedEntryBytes : this.#parsedFactBytes;
const limit = projected ? MAX_RETAINED_ENTRY_BYTES : MAX_PARSED_FACT_BYTES;
const remaining = limit - used - RECORD_OWNERSHIP_RESERVATION;
const bytes = remaining < 0 ? 0 : jsonValueByteLength(value, remaining);
if (remaining < 0 || bytes > remaining)
throw new ArtifactReaderFailure(
"limit",
"PyInstaller directory facts exceed their retained metadata budget",
{ boundary: "memory" },
);
if (projected)
this.#retainedEntryBytes += bytes + RECORD_OWNERSHIP_RESERVATION;
else this.#parsedFactBytes += bytes + RECORD_OWNERSHIP_RESERVATION;
}
#charge(bytes: number): void {
if (
!Number.isSafeInteger(bytes) ||
bytes < 0 ||
this.#charged + bytes > this.#budget
)
throw new ArtifactReaderFailure(
"limit",
`PyInstaller members declare ${String(this.#charged + Math.max(0, bytes))} uncompressed bytes, exceeding the decompression budget`,
);
this.#charged += bytes;
}
}
const failure = (error: PyInstallerFailure): ArtifactReaderFailure =>
new ArtifactReaderFailure(error.reason, error.message);
src/artifacts/apple/InterfaceBuilderAnalysis.ts
+38 -10
View File
@@ -10,6 +10,7 @@ import {
import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js";
import {
projectPlistValue,
binaryDatePrecisionOverlapLimitation,
unknownDateLimitation,
unknownScalarLimitation,
} from "../../domain/apple/plistValue.js";
@@ -18,8 +19,9 @@ import {
parseXmlPropertyList,
} from "../../domain/propertyListKeys.js";
import { decodeNibArchive, type NibArchiveDocument } from "./NibArchive.js";
import { preflightPropertyListDecode } from "./PropertyListDecodeBudget.js";
import {
preflightPropertyListDecode,
interfaceBuilderDecodeBudgetFailure,
InterfaceBuilderDecodeBudget,
InterfaceBuilderDecodeBudgetExceeded,
} from "./InterfaceBuilderDecodeBudget.js";
@@ -39,9 +41,10 @@ import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"
import { decodeXmlPlistText } from "../../domain/propertyListXmlText.js";
import {
decodeBinaryPlistNumbers,
scanBinaryPlistNumberLiterals,
} from "./PlistNumberLiterals.js";
decodeBinaryPlistScalars,
scanBinaryPlistScalars,
type BinaryPlistScalarDecodePlan,
} from "./PlistScalars.js";
import type { PlistNumberLiterals } from "../../domain/apple/plistNumbers.js";
import {
hasBinaryPlistSignature,
@@ -70,6 +73,7 @@ export const analyzeInterfaceBuilderBundle = async (input: {
const unresolvedConnections = new Map<string, number>();
const prototypeKeyOmissions = new Map<string, number | null>();
const unknownDates = new Map<string, number>();
const datePrecisionOverlaps = new Map<string, number>();
const unknownScalars = new Map<string, number>();
let omitted = 0;
let attempted = 0;
@@ -118,6 +122,14 @@ export const analyzeInterfaceBuilderBundle = async (input: {
let nib: ReturnType<typeof projectNibArchive> | null = null;
let xmlText: string | undefined;
let binaryPrototypeKeyObserved = false;
let scalarPlan: BinaryPlistScalarDecodePlan = {
reachableDateOffsets: [],
overlappingDateOffsets: [],
reachableWideIntegerOffsets: [],
overlappingWideIntegerOffsets: [],
reachableNumericOffsets: [],
};
let datePrecisionOverlapCount = 0;
if (isNibArchive) {
const documentBudget = new InterfaceBuilderDecodeBudget(
decodeBudget.remainingBytes,
@@ -138,7 +150,11 @@ export const analyzeInterfaceBuilderBundle = async (input: {
bytes,
decodeBudget.remainingBytes,
xmlText,
{ numberProjection: true },
{
numberProjection: true,
budget: "the aggregate Interface Builder decode budget",
fail: interfaceBuilderDecodeBudgetFailure,
},
);
decodeBudget.reserve(
decodedReservation.estimatedBytes,
@@ -147,6 +163,9 @@ export const analyzeInterfaceBuilderBundle = async (input: {
plistReservation = decodedReservation.estimatedBytes;
binaryPrototypeKeyObserved =
decodedReservation.binaryPrototypeKeyObserved;
scalarPlan = decodedReservation;
datePrecisionOverlapCount =
decodedReservation.datePrecisionOverlapCount;
}
const {
value: raw,
@@ -155,7 +174,7 @@ export const analyzeInterfaceBuilderBundle = async (input: {
unknownScalarCount,
numberLiterals,
} = nib === null
? decodePlist(bytes, xmlText, binaryPrototypeKeyObserved)
? decodePlist(bytes, xmlText, binaryPrototypeKeyObserved, scalarPlan)
: {
value: nib.raw,
omittedPrototypeKeys: 0,
@@ -171,6 +190,8 @@ export const analyzeInterfaceBuilderBundle = async (input: {
prototypeKeyOmissions.set(entry.path, omittedPrototypeKeys);
if (unknownDateCount !== 0)
unknownDates.set(entry.path, unknownDateCount);
if (datePrecisionOverlapCount > 0)
datePrecisionOverlaps.set(entry.path, datePrecisionOverlapCount);
if (unknownScalarCount !== 0)
unknownScalars.set(entry.path, unknownScalarCount);
const documentHash = createHash("sha256").update(bytes).digest("hex");
@@ -218,6 +239,7 @@ export const analyzeInterfaceBuilderBundle = async (input: {
unresolvedConnections,
prototypeKeyOmissions,
unknownDates,
datePrecisionOverlaps,
unknownScalars,
invalid,
omittedArchives,
@@ -288,6 +310,7 @@ const finalizeHierarchyCoverage = (
unresolvedConnections: ReadonlyMap<string, number>;
prototypeKeyOmissions: ReadonlyMap<string, number | null>;
unknownDates: ReadonlyMap<string, number>;
datePrecisionOverlaps: ReadonlyMap<string, number>;
unknownScalars: ReadonlyMap<string, number>;
invalid: readonly string[];
omittedArchives: readonly { path: string; reason: string }[];
@@ -301,6 +324,7 @@ const finalizeHierarchyCoverage = (
unresolvedConnections,
prototypeKeyOmissions,
unknownDates,
datePrecisionOverlaps,
unknownScalars,
invalid,
omittedArchives,
@@ -390,6 +414,9 @@ const finalizeHierarchyCoverage = (
...[...unknownDates].map(
([path, count]) => `${path}: ${unknownDateLimitation(count)}`,
),
...[...datePrecisionOverlaps].map(([path, count]) => {
return `${path}: ${binaryDatePrecisionOverlapLimitation(count)}`;
}),
...[...unknownScalars].map(
([path, count]) => `${path}: ${unknownScalarLimitation(count)}`,
),
@@ -759,6 +786,7 @@ const decodePlist = (
bytes: Buffer,
xmlText: string | undefined,
binaryPrototypeKeyObserved: boolean,
scalarPlan: BinaryPlistScalarDecodePlan,
): {
readonly value: JsonValue;
readonly omittedPrototypeKeys: number | null;
@@ -767,16 +795,16 @@ const decodePlist = (
readonly numberLiterals: PlistNumberLiterals;
} => {
const binary = hasBinaryPlistSignature(bytes);
const binaryNumberMetadata = binary
? scanBinaryPlistNumberLiterals(bytes)
const binaryScalarMetadata = binary
? scanBinaryPlistScalars(bytes, scalarPlan)
: undefined;
const { value, omittedPrototypeKeys, numberLiterals } = binary
? {
value: parseBinary(
decodeBinaryPlistNumbers(bytes, binaryNumberMetadata!),
decodeBinaryPlistScalars(bytes, binaryScalarMetadata!, scalarPlan),
),
omittedPrototypeKeys: binaryPrototypeKeyObserved ? null : 0,
numberLiterals: binaryNumberMetadata!.literals,
numberLiterals: binaryScalarMetadata!.literals,
}
: parseXmlWithinDecodeBudget(xmlText ?? decodeXmlPlistText(bytes));
const projected = projectPlistValue(value);
src/artifacts/apple/InterfaceBuilderDecodeBudget.test.ts
+125 -4
View File
@@ -6,9 +6,14 @@ import { buildBinary } from "plist";
import { describe, expect, it } from "vitest";
import { analyzeInterfaceBuilderBundle } from "./InterfaceBuilderAnalysis.js";
import { preflightPropertyListDecode } from "./InterfaceBuilderDecodeBudget.js";
import { interfaceBuilderDecodeBudgetFailure } from "./InterfaceBuilderDecodeBudget.js";
import { preflightPropertyListDecode } from "./PropertyListDecodeBudget.js";
import { unknownDateLimitation } from "../../domain/apple/plistValue.js";
import { parseXmlPropertyList } from "../../domain/propertyListKeys.js";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { scanBinaryPlistScalars } from "./PlistScalars.js";
import { binaryPlistWithOverlappingDateAndData } from "../../../tests/fixtures/binaryPlistObjectTable.js";
import { embedSubmillisecondDateInWideIntegerPayload } from "../../../tests/fixtures/binaryPlistScalarAlias.js";
const example = "<!DOCTYPE document [ harmless example ]>";
@@ -33,8 +38,12 @@ it.each([
undefined,
{ numberProjection: true },
).estimatedBytes;
if (_format === "binary")
expect(projected).toBeGreaterThanOrEqual(ordinary + bytes.length);
if (_format === "binary") {
const objectCount = Number(bytes.readBigUInt64BE(bytes.length - 24));
expect(projected).toBeGreaterThanOrEqual(
ordinary + bytes.length + objectCount * 96,
);
}
expect(() =>
preflightPropertyListDecode(bytes, ordinary, undefined, {
numberProjection: true,
@@ -112,7 +121,15 @@ describe("inert XML declaration examples in Interface Builder archives", () => {
'<!DOCTYPE plist [<!ENTITY harmless "value">]><plist',
);
expect(() =>
preflightPropertyListDecode(Buffer.from(xml), 16 * 1024 * 1024),
preflightPropertyListDecode(
Buffer.from(xml),
16 * 1024 * 1024,
undefined,
{
budget: "the aggregate Interface Builder decode budget",
fail: interfaceBuilderDecodeBudgetFailure,
},
),
).toThrow(
"XML plist internal entity expansion exceeds the aggregate Interface Builder decode budget",
);
@@ -141,3 +158,107 @@ describe("inert XML declaration examples in Interface Builder archives", () => {
).toMatchObject({ binaryPrototypeKeyObserved: false });
});
});
it("retains explicit date uncertainty for an overlapping binary plist object", async () => {
const root = await createTestTempDirectory("rea-ib-binary-date-overlap-");
const bundle = join(root, "Example.app");
await mkdir(bundle);
const bytes = binaryPlistWithOverlappingDateAndData();
await writeFile(join(bundle, "Panel.nib"), bytes);
const result = await analyzeInterfaceBuilderBundle({
bundlePath: bundle,
targetSha256: "a".repeat(64),
});
expect(result.limitations).toContainEqual(
expect.stringContaining(
"Panel.nib: 1 date value(s) are reported with unknown timestamps because their sub-millisecond source records overlap",
),
);
});
it("preserves aliased wide-integer precision and marks the date unknown in bundle output", async () => {
const root = await createTestTempDirectory("rea-ib-binary-scalar-alias-");
const bundle = join(root, "Example.app");
await mkdir(bundle);
const source = Buffer.from(
buildBinary({
"com.apple.ibtool.document.objects": {
button: {
customClass: "UIButton",
wide: Buffer.alloc(16),
date: new Date(0),
},
},
"com.apple.ibtool.document.connections": {},
"com.apple.ibtool.document.hierarchy": [],
"com.apple.ibtool.document.classes": {},
}),
);
const { bytes, integerDecimal } =
embedSubmillisecondDateInWideIntegerPayload(source);
await writeFile(join(bundle, "Panel.nib"), bytes);
const result = await analyzeInterfaceBuilderBundle({
bundlePath: bundle,
targetSha256: "a".repeat(64),
});
const button = result.graph.nodes.find(
({ attributes }) => attributes.interface_builder_object_id === "button",
);
expect(button?.attributes).toMatchObject({
date: { $plist_type: "date", iso8601: null },
wide: { $plist_type: "integer", decimal: integerDecimal },
});
});
describe("binary Interface Builder plist dates", () => {
const control = Buffer.from(
"62706c6973743030d4010203040506090c5924617263686976657258246f626a656374735424746f70582476657273696f6e5f100f4e534b657965644172636869766572a2070855246e756c6c333fb999999999999ad10a0b54726f6f74800112000186a008111b24293244474d56595e600000000000000101000000000000000d00000000000000000000000000000065",
"hex",
);
const marker = Buffer.alloc(9);
marker[0] = 0x33;
marker.writeDoubleBE(0.1, 1);
const nib = async (bytes: Buffer, name: string) => {
const root = await createTestTempDirectory(`rea-ib-date-${name}-`);
const bundle = join(root, "Example.app");
await mkdir(bundle);
await writeFile(join(bundle, "Panel.nib"), bytes);
return analyzeInterfaceBuilderBundle({
bundlePath: bundle,
targetSha256: "a".repeat(64),
});
};
it("refuses an invalid reachable date marker without projecting a timestamp", async () => {
const bytes = Buffer.from(control);
const offset = bytes.indexOf(marker);
bytes[offset] = 0x30;
const result = await nib(bytes, "marker");
expect(JSON.stringify(result)).not.toContain("2001-01-01T00:00:00.100Z");
expect(result.limitations.join("\n")).toContain(
"binary plist date marker 0x30 is invalid",
);
});
it("counts a sub-millisecond date as unknown and keeps the canonical control known", async () => {
const submillisecond = Buffer.from(control);
const offset = submillisecond.indexOf(marker);
submillisecond.writeDoubleBE(0.1005, offset + 1);
const unknown = await nib(submillisecond, "subms");
expect(unknown.limitations).toContain(
`Panel.nib: ${unknownDateLimitation(1)}`,
);
expect(JSON.stringify(unknown)).not.toContain("2001-01-01T00:00:00.100Z");
const known = await nib(Buffer.from(control), "known");
expect(known.limitations.join("\n")).not.toContain("date value(s)");
expect(known.limitations.join("\n")).not.toContain(
"binary plist date marker",
);
});
});
src/artifacts/apple/InterfaceBuilderDecodeBudget.ts
+10 -311
View File
@@ -1,6 +1,3 @@
import { decodeXmlPlistText } from "../../domain/propertyListXmlText.js";
import { hasBinaryPlistSignature } from "../../domain/apple/appleArchiveFormat.js";
/** Reason reported when an archive would exceed the shared decoding budget. */
export type InterfaceBuilderBudgetReason = "aggregate_decode_budget_exhausted";
@@ -15,6 +12,16 @@ export class InterfaceBuilderDecodeBudgetExceeded extends Error {
}
}
/** Map shared plist preflight limits to Interface Builder's public reason. */
export const interfaceBuilderDecodeBudgetFailure = (
_kind: "cycle" | "depth" | "expansion",
message: string,
): Error =>
new InterfaceBuilderDecodeBudgetExceeded(
"aggregate_decode_budget_exhausted",
message,
);
/** Shared byte budget for decoder records and projected archive data. */
export class InterfaceBuilderDecodeBudget {
readonly #initialBytes: number;
@@ -96,311 +103,3 @@ export class InterfaceBuilderDecodeBudget {
return true;
}
}
/** Preflight the representation created by the pinned plist decoder. */
export const preflightPropertyListDecode = (
bytes: Buffer,
maxBytes: number,
xmlText?: string,
options: Pick<BinaryPlistExpansionOptions, "numberProjection"> = {},
): PlistDecodePreflight => {
const { numberProjection = false } = options;
const expansion = hasBinaryPlistSignature(bytes)
? estimateBinaryPlistExpansion(bytes, maxBytes, { numberProjection })
: {
estimatedBytes: estimateXmlPlistExpansion(
bytes,
maxBytes,
xmlText,
numberProjection,
),
binaryPrototypeKeyObserved: false,
};
if (expansion.estimatedBytes > maxBytes)
throw new InterfaceBuilderDecodeBudgetExceeded(
"aggregate_decode_budget_exhausted",
"plist representation exceeds the aggregate Interface Builder decode budget",
);
return expansion;
};
const estimateXmlPlistExpansion = (
bytes: Buffer,
maxBytes: number,
xmlText?: string,
numberProjection = false,
): number => {
const xml = xmlText ?? decodeXmlPlistText(bytes);
const decodedSourceBytes = xml.length * 2;
let elementCount = 0;
// Reserve room for numeric source metadata and the projected attribute copy.
const elementBytes = numberProjection ? 256 : 128;
// Value nesting is bounded by the shared plist parser, which counts plist
// value depth; raw element depth would also count the envelope and keys.
for (const token of xmlPreflightTokens(xml)) {
if (token.doctype) {
// Only a declaration outside opaque text can introduce an internal
// subset. Brackets in quoted external identifiers are literal text.
if (token.internalSubset)
throw new InterfaceBuilderDecodeBudgetExceeded(
"aggregate_decode_budget_exhausted",
"XML plist internal entity expansion exceeds the aggregate Interface Builder decode budget",
);
continue;
}
elementCount += 1;
if (decodedSourceBytes * 2 + elementCount * elementBytes > maxBytes)
return maxBytes + 1;
}
return decodedSourceBytes * 2 + elementCount * elementBytes;
};
/** Consume each XML span once; malformed tokens cannot restart suffix scans. */
function* xmlPreflightTokens(xml: string) {
let cursor = 0;
while (cursor < xml.length) {
const start = xml.indexOf("<", cursor);
if (start < 0) return;
const opaqueEnd = xml.startsWith("<!--", start)
? "-->"
: xml.startsWith("<![CDATA[", start)
? "]]>"
: xml.startsWith("<?", start)
? "?>"
: undefined;
if (opaqueEnd !== undefined) {
const end = xml.indexOf(opaqueEnd, start + 2);
if (end < 0) return;
cursor = end + opaqueEnd.length;
continue;
}
const doctype = /^<!DOCTYPE\b/iu.test(xml.slice(start, start + 10));
const element = /^<\/?[A-Za-z]/u.test(xml.slice(start, start + 4));
let quote: string | undefined;
let internalSubset = false;
let end = start + 1;
for (; end < xml.length; end += 1) {
const character = xml[end];
if (quote !== undefined) {
if (character === quote) quote = undefined;
} else if (character === '"' || character === "'") quote = character;
else if (character === ">") break;
else if (doctype && character === "[") internalSubset = true;
}
if (end === xml.length) return;
cursor = end + 1;
if (doctype || element) yield { doctype, internalSubset };
}
}
/** Resource a binary plist preflight found over its budget, or a malformed reference cycle. */
export type BinaryPlistExpansionFailure = "cycle" | "depth" | "expansion";
const interfaceBuilderBudgetFailure = (
_kind: BinaryPlistExpansionFailure,
message: string,
): Error =>
new InterfaceBuilderDecodeBudgetExceeded(
"aggregate_decode_budget_exhausted",
message,
);
/** How a caller names and reports binary plist expansion failures. */
export interface BinaryPlistExpansionOptions {
/** Include original number metadata and exact-integer attribute projection. */
readonly numberProjection?: boolean;
/** Budget named in failure messages. */
readonly budget?: string;
/** Error for budget, depth, and reference-cycle failures. */
readonly fail?: (kind: BinaryPlistExpansionFailure, message: string) => Error;
/**
* Structural bounds: "strict" keeps objects and the offset table before the
* trailer; "decoder" follows the decoder's object-table boundary policy
* while still rejecting structures whose coercion would lose plist facts.
*/
readonly bounds?: "strict" | "decoder";
}
/** Preflight size and binary-key observations; XML keys are inspected by decode. */
export interface PlistDecodePreflight {
readonly estimatedBytes: number;
readonly binaryPrototypeKeyObserved: boolean;
}
const PROTOTYPE_KEY = "__proto__";
/**
* Count every expanded binary-plist reference before calling plist.parseBinary,
* which recurses on each reference and copies shared containers. Malformed
* structure throws TypeError; budget, depth, and cycle failures use `fail`.
*/
export const estimateBinaryPlistExpansion = (
bytes: Buffer,
maxBytes: number,
{
budget = "the aggregate Interface Builder decode budget",
fail = interfaceBuilderBudgetFailure,
bounds = "strict",
numberProjection = false,
}: BinaryPlistExpansionOptions = {},
): PlistDecodePreflight => {
if (bytes.length < 40)
throw new TypeError("binary plist trailer is truncated");
const trailer = bytes.length - 32;
const offsetSize = bytes[trailer + 6] ?? 0;
const referenceSize = bytes[trailer + 7] ?? 0;
if (
![1, 2, 4, 8].includes(offsetSize) ||
![1, 2, 4, 8].includes(referenceSize)
)
throw new TypeError("binary plist trailer integer sizes are invalid");
const readInteger = (offset: number, size: number): number => {
if (offset < 0 || size < 1 || offset + size > bytes.length) return -1;
let value = 0;
for (let index = 0; index < size; index += 1)
value = value * 256 + (bytes[offset + index] ?? 0);
return Number.isSafeInteger(value) ? value : -1;
};
const objectCount = readInteger(trailer + 8, 8);
const topObject = readInteger(trailer + 16, 8);
const offsetTable = readInteger(trailer + 24, 8);
const tableEnd = bounds === "strict" ? trailer : bytes.length;
const objectsEnd = bounds === "strict" ? offsetTable : bytes.length;
if (
objectCount < 1 ||
topObject < 0 ||
topObject >= objectCount ||
offsetTable < 8 ||
offsetTable + objectCount * offsetSize > tableEnd
)
throw new TypeError("binary plist trailer or object table is invalid");
// Metadata covers the full object table; projected values may repeat through references.
const objectBytes = numberProjection ? 144 : 16;
const expandedNodeBytes = numberProjection ? 224 : 96;
// Number projection may require a same-sized parser-compatible input copy
// for wide integers. Reserve for it before the caller allocates that copy.
let estimate =
bytes.length * (numberProjection ? 3 : 2) + objectCount * objectBytes;
let binaryPrototypeKeyObserved = false;
if (estimate > maxBytes)
throw fail("expansion", `binary plist object table exceeds ${budget}`);
const active = new Set<number>();
const pending: Array<{
object: number;
depth: number;
exit: boolean;
dictionaryKey: boolean;
}> = [{ object: topObject, depth: 0, exit: false, dictionaryKey: false }];
let pendingObjectCount = 1;
while (pending.length > 0) {
const entry = pending.pop();
if (entry === undefined) break;
if (entry.exit) {
active.delete(entry.object);
continue;
}
pendingObjectCount -= 1;
if (entry.depth > 128)
throw fail("depth", `binary plist nesting exceeds ${budget}`);
if (active.has(entry.object))
throw fail("cycle", `binary plist reference cycle exceeds ${budget}`);
estimate += expandedNodeBytes;
if (estimate > maxBytes)
throw fail(
"expansion",
`binary plist reference expansion exceeds ${budget}`,
);
const offset = readInteger(
offsetTable + entry.object * offsetSize,
offsetSize,
);
if (offset < 8 || offset >= objectsEnd)
throw new TypeError("binary plist object offset is invalid");
const marker = bytes[offset] ?? 0;
const type = marker >> 4;
if (entry.dictionaryKey && type !== 5 && type !== 6)
throw new TypeError("binary plist dictionary key is not a string");
let size = marker & 0x0f;
let cursor = offset + 1;
if (size === 0x0f && type !== 0 && type !== 8) {
const extMarker = bytes[cursor] ?? 0;
if (extMarker >> 4 !== 1 || (extMarker & 0x0f) > 3)
throw new TypeError("binary plist extended object size is invalid");
cursor += 1;
const integerSize = 1 << (extMarker & 0x0f);
size = readInteger(cursor, integerSize);
cursor += integerSize;
if (size < 0 || cursor > objectsEnd)
throw new TypeError("binary plist extended object size is invalid");
}
if (type === 4 || type === 5 || type === 6) {
const byteLength = type === 6 ? size * 2 : size;
if (cursor + byteLength > objectsEnd)
throw new TypeError("binary plist scalar object is truncated");
if (
!binaryPrototypeKeyObserved &&
entry.dictionaryKey &&
(type === 5 || type === 6) &&
size === PROTOTYPE_KEY.length
) {
let matches = true;
for (let index = 0; index < PROTOTYPE_KEY.length; index += 1) {
const codeUnit =
type === 5
? (bytes[cursor + index] ?? 0)
: ((bytes[cursor + index * 2] ?? 0) << 8) |
(bytes[cursor + index * 2 + 1] ?? 0);
if (codeUnit !== PROTOTYPE_KEY.charCodeAt(index)) {
matches = false;
break;
}
}
if (matches) binaryPrototypeKeyObserved = true;
}
estimate += type === 4 ? size + 4 * Math.ceil(size / 3) : byteLength * 2;
if (estimate > maxBytes)
throw fail(
"expansion",
`binary plist scalar expansion exceeds ${budget}`,
);
continue;
}
if (type !== 10 && type !== 13) continue;
const childCount = type === 13 ? size * 2 : size;
const refsEnd = cursor + childCount * referenceSize;
if (!Number.isSafeInteger(refsEnd) || refsEnd > objectsEnd)
throw new TypeError("binary plist reference table is truncated");
const availableNodes = Math.floor(
(maxBytes - estimate) / expandedNodeBytes,
);
if (childCount > availableNodes - pendingObjectCount)
throw fail(
"expansion",
`binary plist reference expansion exceeds ${budget}`,
);
active.add(entry.object);
pending.push({
object: entry.object,
depth: entry.depth,
exit: true,
dictionaryKey: false,
});
for (let index = childCount - 1; index >= 0; index -= 1) {
const reference = readInteger(
cursor + index * referenceSize,
referenceSize,
);
if (reference < 0 || reference >= objectCount)
throw new TypeError("binary plist reference is invalid");
pending.push({
object: reference,
depth: entry.depth + 1,
exit: false,
dictionaryKey: type === 13 && index < size,
});
}
pendingObjectCount += childCount;
}
return { estimatedBytes: estimate, binaryPrototypeKeyObserved };
};
src/artifacts/apple/KeyedArchiveReader.test.ts
+375 -1
View File
@@ -1,7 +1,24 @@
import { createHash } from "node:crypto";
import { readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { build, buildBinary, type PlistValue } from "plist";
import { decodeKeyedArchiveBytes } from "./KeyedArchiveReader.js";
import { estimateBinaryPlistExpansion } from "./PropertyListDecodeBudget.js";
import {
decodeKeyedArchiveBytes,
inspectBundleKeyedArchive,
} from "./KeyedArchiveReader.js";
import { unknownDateLimitation } from "../../domain/apple/plistValue.js";
import { deeplyNestedXmlPlist } from "../../../tests/fixtures/keyedArchiveXmlNesting.js";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { scanBinaryPlistScalars } from "./PlistScalars.js";
import { embedSubmillisecondDateInWideIntegerPayload } from "../../../tests/fixtures/binaryPlistScalarAlias.js";
import {
readBinaryPlistObjectOffset,
readBinaryPlistObjectTableLayout,
writeBinaryPlistObjectOffset,
} from "../../../tests/fixtures/binaryPlistObjectTable.js";
const archive = {
$archiver: "NSKeyedArchiver",
@@ -22,6 +39,146 @@ const archive = {
],
$top: { root: { UID: 1 }, other: { UID: 2 } },
};
const binaryArchiveWithDate = (): Buffer =>
Buffer.from(
buildBinary({
...archive,
$objects: ["$null", new Date("2001-01-01T00:00:00.100Z")],
$top: { root: { UID: 1 } },
}),
);
const binaryArchiveWithOverlappingDate = (): Buffer => {
const bytes = Buffer.from(
buildBinary({
...archive,
$objects: [
"$null",
new Date("2001-01-01T00:00:00.000Z"),
Buffer.alloc(0),
],
$top: { root: { UID: 1 } },
}),
);
const date = scanBinaryPlistScalars(bytes).dates[0];
if (date === undefined)
throw new Error("binary archive fixture lost its date");
bytes.writeDoubleBE(0.0005, date.payloadOffset);
const objectTable = readBinaryPlistObjectTableLayout(bytes);
for (let objectId = 0; objectId < objectTable.objectCount; objectId += 1) {
const offset = readBinaryPlistObjectOffset(bytes, objectTable, objectId);
if (offset !== date.offset && bytes[offset] === 0x40) {
writeBinaryPlistObjectOffset(
bytes,
objectTable,
objectId,
date.payloadOffset + 1,
);
return bytes;
}
}
throw new Error("binary archive fixture lost its empty data object");
};
const binaryArchiveDateCrossingOffsetTable = (): Buffer => {
const bytes = Buffer.from(binaryArchiveWithDate());
const date = scanBinaryPlistScalars(bytes).dates[0];
if (date === undefined)
throw new Error("binary archive fixture lost its date");
const objectTable = readBinaryPlistObjectTableLayout(bytes);
let dateObjectId = -1;
for (let objectId = 0; objectId < objectTable.objectCount; objectId += 1) {
if (
readBinaryPlistObjectOffset(bytes, objectTable, objectId) === date.offset
)
dateObjectId = objectId;
}
if (dateObjectId < 0)
throw new Error("binary archive fixture lost its date object-table entry");
const crossingOffset = objectTable.tableOffset - 1;
bytes[crossingOffset] = 0x33;
writeBinaryPlistObjectOffset(
bytes,
objectTable,
dateObjectId,
crossingOffset,
);
return bytes;
};
describe("keyed archive scalar projection", () => {
it("preserves the exact integer and unknown date through archive projection", () => {
const source = Buffer.from(
buildBinary({
...archive,
$objects: ["$null", Buffer.alloc(16), new Date(0)],
$top: { root: { UID: 1 } },
}),
);
const { bytes, integerDecimal } =
embedSubmillisecondDateInWideIntegerPayload(source);
const graph = decodeKeyedArchiveBytes(bytes, {
root: "root",
offset: 1,
limit: 2,
});
expect(graph.objects).toContainEqual(
expect.objectContaining({
id: 1,
value: { $plist_type: "integer", decimal: integerDecimal },
}),
);
expect(graph.objects).toContainEqual(
expect.objectContaining({
id: 2,
value: { $plist_type: "date", iso8601: null },
}),
);
});
it("rejects a reachable date whose payload enters the offset table", () => {
expect(() =>
decodeKeyedArchiveBytes(binaryArchiveDateCrossingOffsetTable(), {
root: "root",
offset: 0,
limit: 2,
}),
).toThrow(
expect.objectContaining({
message: "binary plist date object is truncated",
}),
);
});
it("keeps overlapping date precision unknown without changing its sibling object", () => {
const graph = decodeKeyedArchiveBytes(binaryArchiveWithOverlappingDate(), {
root: "root",
offset: 0,
limit: 4,
});
expect(graph.objects).toContainEqual(
expect.objectContaining({
id: 1,
value: { $plist_type: "date", iso8601: null },
}),
);
expect(graph.objects).toContainEqual(
expect.objectContaining({
id: 2,
value: { $plist_type: "data", base64: "" },
}),
);
expect(graph.limitations).toContainEqual(
expect.stringContaining("sub-millisecond source records overlap"),
);
});
});
describe("inert keyed archive decoding", () => {
it.each(["bplist00", "NIBArchive"])(
"rejects high-bit mutations of %s as malformed bytes rather than another archive kind",
@@ -375,6 +532,223 @@ describe("binary plist reference preflight", () => {
});
});
/**
* Python `plistlib` binary NSKeyedArchiver with `$objects = ["$null", date]`
* and `$top.root` as UID 1. The date is marker `0x33` and float64 `0.1`
* seconds after the 2001 epoch (`2001-01-01T00:00:00.100Z`).
*/
const pythonBinaryDateControl = Buffer.from(
"62706c6973743030d4010203040506090c5924617263686976657258246f626a656374735424746f70582476657273696f6e5f100f4e534b657965644172636869766572a2070855246e756c6c333fb999999999999ad10a0b54726f6f74800112000186a008111b24293244474d56595e600000000000000101000000000000000d00000000000000000000000000000065",
"hex",
);
const canonicalDateMarker = Buffer.alloc(9);
canonicalDateMarker[0] = 0x33;
canonicalDateMarker.writeDoubleBE(0.1, 1);
/** Copy the control and mutate its single canonical date marker. */
const mutateCanonicalDate = (
mutate: (markerOffset: number, bytes: Buffer) => void,
): Buffer => {
const bytes = Buffer.from(pythonBinaryDateControl);
const offset = bytes.indexOf(canonicalDateMarker);
if (offset < 0 || bytes.indexOf(canonicalDateMarker, offset + 1) !== -1)
throw new Error("expected one canonical binary date in the control");
mutate(offset, bytes);
return bytes;
};
const dateScalar = (iso8601: string | null) => ({
id: 1,
kind: "scalar" as const,
status: "observed" as const,
value: { $plist_type: "date", iso8601 },
});
describe("binary keyed archive dates", () => {
const selection = { offset: 0, limit: 10 };
it("keeps the canonical millisecond control as an observed timestamp", async () => {
const bytes = Buffer.from(pythonBinaryDateControl);
const graph = decodeKeyedArchiveBytes(bytes, selection);
expect(graph.objects[1]).toMatchObject(
dateScalar("2001-01-01T00:00:00.100Z"),
);
expect(graph.limitations).not.toContain(unknownDateLimitation(1));
expect(bytes).toEqual(pythonBinaryDateControl);
expect(createHash("sha256").update(bytes).digest("hex")).toBe(
"39f08db8341bd74220352dfd559485515cce1e6035e08f27a59ab8f856374a30",
);
const root = await createTestTempDirectory("rea-keyed-date-control-");
await writeFile(join(root, "archive.plist"), bytes);
const inspected = await inspectBundleKeyedArchive({
bundlePath: root,
targetSha256: createHash("sha256").update(bytes).digest("hex"),
parameters: { path: "archive.plist" },
});
expect(inspected.objects[1]).toMatchObject(
dateScalar("2001-01-01T00:00:00.100Z"),
);
expect(inspected.limitations).not.toContain(unknownDateLimitation(1));
expect(await readFile(join(root, "archive.plist"))).toEqual(bytes);
});
it.each(
[0x30, 0x31, 0x32, 0x34].map(
(marker) => [marker.toString(16), marker] as const,
),
)(
"refuses reachable binary date marker 0x%s instead of projecting a timestamp",
async (_hex, marker) => {
const bytes = mutateCanonicalDate((offset, copy) => {
copy[offset] = marker;
});
const original = Buffer.from(bytes);
expect(() => decodeKeyedArchiveBytes(bytes, selection)).toThrow(
new TypeError(
`binary plist date marker 0x${marker.toString(16)} is invalid`,
),
);
expect(bytes).toEqual(original);
if (marker === 0x30)
expect(createHash("sha256").update(bytes).digest("hex")).toBe(
"ac5696b866c312feb53c2e82edd8fea79dbeb572eb1b07e4291603faa41e8c0e",
);
const root = await createTestTempDirectory("rea-keyed-date-marker-");
const path = join(root, "archive.plist");
await writeFile(path, bytes);
await expect(
inspectBundleKeyedArchive({
bundlePath: root,
targetSha256: null,
parameters: { path: "archive.plist" },
}),
).rejects.toMatchObject({
name: "ArtifactReaderFailure",
reason: "format",
message: `Cannot decode selected Foundation keyed archive: binary plist date marker 0x${marker.toString(16)} is invalid`,
});
expect(await readFile(path)).toEqual(bytes);
},
);
it("keeps a sub-millisecond instant unknown without changing the canonical sibling", async () => {
const submillisecond = mutateCanonicalDate((offset, copy) => {
copy.writeDoubleBE(0.1005, offset + 1);
});
const original = Buffer.from(submillisecond);
const graph = decodeKeyedArchiveBytes(submillisecond, selection);
expect(graph.objects[1]).toMatchObject(dateScalar(null));
expect(JSON.stringify(graph.objects[1]?.value)).not.toContain(
"2001-01-01T00:00:00.100Z",
);
expect(graph.limitations).toContain(unknownDateLimitation(1));
expect(graph.limitations.join("\n")).not.toContain(
"number metadata could not be completely observed",
);
expect(submillisecond).toEqual(original);
expect(createHash("sha256").update(submillisecond).digest("hex")).toBe(
"ab69ba6e8034c6f60b828ee323a5a3e81bae3e7657477e0ebfbfdb93d53ed2ce",
);
const root = await createTestTempDirectory("rea-keyed-date-subms-");
await writeFile(join(root, "archive.plist"), submillisecond);
const inspected = await inspectBundleKeyedArchive({
bundlePath: root,
targetSha256: createHash("sha256").update(submillisecond).digest("hex"),
parameters: { path: "archive.plist" },
});
expect(inspected.objects[1]).toMatchObject(dateScalar(null));
expect(inspected.limitations).toContain(unknownDateLimitation(1));
expect(await readFile(join(root, "archive.plist"))).toEqual(submillisecond);
const mixed = Buffer.from(
buildBinary({
$archiver: "NSKeyedArchiver",
$version: 100000,
$objects: [
"$null",
new Date("2001-01-01T00:00:00.100Z"),
new Date("2001-01-01T00:00:00.100Z"),
new Date("2001-01-01T00:00:00.100Z"),
],
$top: { root: { UID: 1 } },
}),
);
const pattern = Buffer.from(canonicalDateMarker);
const offsets: number[] = [];
for (
let offset = mixed.indexOf(pattern);
offset !== -1;
offset = mixed.indexOf(pattern, offset + pattern.length)
)
offsets.push(offset);
expect(offsets).toHaveLength(3);
// 0.1004 and 0.1005 both round to the same millisecond as the control.
mixed.writeDoubleBE(0.1004, (offsets[1] ?? 0) + 1);
mixed.writeDoubleBE(0.1005, (offsets[2] ?? 0) + 1);
const mixedOriginal = Buffer.from(mixed);
const mixedGraph = decodeKeyedArchiveBytes(mixed, selection);
expect(mixedGraph.objects.slice(1).map(({ value }) => value)).toEqual([
{ $plist_type: "date", iso8601: "2001-01-01T00:00:00.100Z" },
{ $plist_type: "date", iso8601: null },
{ $plist_type: "date", iso8601: null },
]);
expect(mixedGraph.limitations).toContain(unknownDateLimitation(2));
expect(mixed).toEqual(mixedOriginal);
});
});
describe("binary plist date object region", () => {
it("refuses a binary date that extends past the object region", () => {
// Strict bounds end objects at the offset table. Four payload bytes
// remain before that table, short of the required eight.
const bytes = Buffer.alloc(8 + 5 + 1 + 32);
bytes.write("bplist00");
bytes[8] = 0x33;
const offsetTable = 13;
bytes[offsetTable] = 8;
const trailer = bytes.length - 32;
bytes[trailer + 6] = 1;
bytes[trailer + 7] = 1;
bytes.writeBigUInt64BE(1n, trailer + 8);
bytes.writeBigUInt64BE(0n, trailer + 16);
bytes.writeBigUInt64BE(BigInt(offsetTable), trailer + 24);
expect(() =>
estimateBinaryPlistExpansion(bytes, 1024 * 1024, { bounds: "strict" }),
).toThrow(new TypeError("binary plist date object is truncated"));
// Decoder bounds extend other objects to the end of the file. The date
// payload still has to end at the offset table. The keyed-archive reader
// uses those bounds and must refuse the same payload.
expect(() =>
estimateBinaryPlistExpansion(bytes, 1024 * 1024, { bounds: "decoder" }),
).toThrow(new TypeError("binary plist date object is truncated"));
expect(() =>
decodeKeyedArchiveBytes(bytes, { offset: 0, limit: 1 }),
).toThrow(new TypeError("binary plist date object is truncated"));
});
it("refuses a decoder-bounds date whose marker sits past the offset table", () => {
// The one-byte offset table at byte 8 names an object at byte 9. That
// 0x33 marker and its eight payload bytes fit in the file, which decoder
// bounds would otherwise accept, but they are outside the object region.
const bytes = Buffer.alloc(64);
bytes.write("bplist00");
bytes[8] = 9;
bytes[9] = 0x33;
const trailer = bytes.length - 32;
bytes[trailer + 6] = 1;
bytes[trailer + 7] = 1;
bytes.writeBigUInt64BE(1n, trailer + 8);
bytes.writeBigUInt64BE(0n, trailer + 16);
bytes.writeBigUInt64BE(8n, trailer + 24);
expect(() =>
estimateBinaryPlistExpansion(bytes, 1024 * 1024, { bounds: "decoder" }),
).toThrow(new TypeError("binary plist date object is truncated"));
expect(() =>
decodeKeyedArchiveBytes(bytes, { offset: 0, limit: 1 }),
).toThrow(new TypeError("binary plist date object is truncated"));
});
});
describe("XML plist nesting preflight", () => {
const selection = { offset: 0, limit: 1 };
src/artifacts/apple/KeyedArchiveReader.ts
+21 -8
View File
@@ -7,6 +7,7 @@ import {
AnalysisUnsupportedTargetError,
} from "../../domain/analysisErrorCore.js";
import {
binaryDatePrecisionOverlapLimitation,
negativeZeroRealLimitation,
nonFiniteRealLimitation,
unknownDateLimitation,
@@ -14,9 +15,9 @@ import {
} from "../../domain/apple/plistValue.js";
import { createPlistNumberProjection } from "../../domain/apple/plistNumbers.js";
import {
decodeBinaryPlistNumbers,
scanBinaryPlistNumberLiterals,
} from "./PlistNumberLiterals.js";
decodeBinaryPlistScalars,
scanBinaryPlistScalars,
} from "./PlistScalars.js";
import {
omittedPrototypeKeysLimitation,
parseXmlPropertyList,
@@ -36,7 +37,7 @@ import {
hasBinaryPlistSignature,
hasNibArchiveSignature,
} from "../../domain/apple/appleArchiveFormat.js";
import { estimateBinaryPlistExpansion } from "./InterfaceBuilderDecodeBudget.js";
import { estimateBinaryPlistExpansion } from "./PropertyListDecodeBudget.js";
const MAX_BYTES = 64 * 1024 * 1024;
/** Decoded representation budget, the same 8:1 ratio Interface Builder uses. */
@@ -72,16 +73,20 @@ export const decodeKeyedArchiveBytes = (
numberProjection: true,
})
: undefined;
const binaryNumberMetadata = binary
? scanBinaryPlistNumberLiterals(bytes)
const binaryScalarMetadata = binary
? scanBinaryPlistScalars(bytes, binaryExpansion)
: undefined;
const parsed =
binaryExpansion !== undefined
? {
value: parseBinary(
decodeBinaryPlistNumbers(bytes, binaryNumberMetadata!),
decodeBinaryPlistScalars(
bytes,
binaryScalarMetadata!,
binaryExpansion!,
),
),
numberLiterals: binaryNumberMetadata!.literals,
numberLiterals: binaryScalarMetadata!.literals,
omittedPrototypeKeys: binaryExpansion.binaryPrototypeKeyObserved
? null
: 0,
@@ -127,6 +132,14 @@ export const decodeKeyedArchiveBytes = (
...(unknownDateCount === 0
? []
: [unknownDateLimitation(unknownDateCount)]),
...(binaryExpansion === undefined ||
binaryExpansion.datePrecisionOverlapCount === 0
? []
: [
binaryDatePrecisionOverlapLimitation(
binaryExpansion.datePrecisionOverlapCount,
),
]),
...(unknownScalarCount === 0
? []
: [unknownScalarLimitation(unknownScalarCount)]),
src/artifacts/apple/MachoLoadCommandReader.test.ts
+106
View File
@@ -142,6 +142,112 @@ it("uses legacy simulator and device platform facts when resolving dylibs", asyn
);
});
it.each([false, true] as const)(
"applies macOS dyld's implicit platform to a platform-less dylib and traces its required child present=%s",
async (childPresent) => {
const root = await createTestTempDirectory("rea-dylib-platform-unknown-");
const executablePath = "Contents/MacOS/probe";
const dylibPath = "Contents/Frameworks/libA.dylib";
const fallbackPath = "Contents/Fallbacks/libA.dylib";
const childPath = "Contents/Frameworks/libMissing.dylib";
const fallbackChildPath = "Contents/Fallbacks/libFallbackOnly.dylib";
const executable = machoImage({
cpu: CPU.arm64,
commands: [
buildVersionCommand(1),
rpathCommand("@executable_path/../Frameworks"),
rpathCommand("@executable_path/../Fallbacks"),
dylibCommand(LC.LOAD_DYLIB, "@rpath/libA.dylib"),
],
});
const dylib = machoImage({
cpu: CPU.arm64,
fileType: FILE_TYPE.dylib,
commands: [
rpathCommand("@loader_path"),
dylibCommand(LC.LOAD_DYLIB, "@rpath/libMissing.dylib"),
],
});
const child = machoImage({
cpu: CPU.arm64,
fileType: FILE_TYPE.dylib,
commands: [buildVersionCommand(1)],
});
const fallback = machoImage({
cpu: CPU.arm64,
fileType: FILE_TYPE.dylib,
commands: [
rpathCommand("@loader_path"),
dylibCommand(LC.LOAD_DYLIB, "@rpath/libFallbackOnly.dylib"),
],
});
const fallbackChild = machoImage({
cpu: CPU.arm64,
fileType: FILE_TYPE.dylib,
commands: [buildVersionCommand(1)],
});
await mkdir(join(root, "Contents/MacOS"), { recursive: true });
await mkdir(join(root, "Contents/Frameworks"), { recursive: true });
await mkdir(join(root, "Contents/Fallbacks"), { recursive: true });
await writeFile(join(root, executablePath), executable);
await writeFile(join(root, dylibPath), dylib);
await writeFile(join(root, fallbackPath), fallback);
await writeFile(join(root, fallbackChildPath), fallbackChild);
if (childPresent) await writeFile(join(root, childPath), child);
const trace = await traceDylibResolution({
rootPath: root,
targetPath: join(root, executablePath),
targetSha256: createHash("sha256").update(executable).digest("hex"),
enumerateRoots: false,
parameters: { roots: [executablePath] },
});
expect(
trace.images.find(({ path }) => path === executablePath)?.slices[0],
).toMatchObject({ platform: 1, platforms: [1] });
expect(
trace.images.find(({ path }) => path === dylibPath)?.slices[0],
).toMatchObject({ platform: null, platforms: [] });
expect(trace.edges[0]).toMatchObject({
loader: executablePath,
install_name: "@rpath/libA.dylib",
candidates: [
{
outcome: "resolved",
resolved_path: dylibPath,
platform_basis: "macos-dyld-implicit",
},
],
resolution: { status: "resolved", image: dylibPath },
});
expect(trace.edges[1]).toMatchObject({
loader: dylibPath,
install_name: "@rpath/libMissing.dylib",
resolution: childPresent
? { status: "resolved", image: childPath }
: { status: "unresolved", image: null },
loader_conditional: false,
});
expect(trace.images.map(({ path }) => path)).toEqual(
childPresent
? [executablePath, childPath, dylibPath].sort()
: [executablePath, dylibPath].sort(),
);
expect(trace.images.map(({ path }) => path)).not.toContain(fallbackPath);
expect(trace.images.map(({ path }) => path)).not.toContain(
fallbackChildPath,
);
expect(trace.coverage.status).toBe("complete");
expect(trace.limitations).toContainEqual(
expect.stringContaining(dylibPath),
);
expect(trace.findings.map(({ kind }) => kind)).toEqual(
childPresent ? [] : ["required-load-unresolved"],
);
},
);
describe("absolute dyld candidate paths at the filesystem reader boundary", () => {
it.each(["install-name", "rpath"] as const)(
"resolves an absolute in-root %s while preserving its source path",
src/artifacts/apple/PlistNumberLiterals.test.ts
-40
View File
@@ -1,40 +0,0 @@
import { describe, expect, it } from "vitest";
import {
decodeBinaryPlistNumbers,
scanBinaryPlistNumberLiterals,
} from "./PlistNumberLiterals.js";
const dataObjectWithUnusedOverlappingInteger = (): Buffer => {
const bytes = Buffer.alloc(66);
Buffer.from("bplist00", "ascii").copy(bytes);
bytes[8] = 0xa1; // One-element array referencing data object 1.
bytes[9] = 1;
bytes[10] = 0x4f; // Data object with an extended length of 18 bytes.
bytes[11] = 0x10;
bytes[12] = 18;
bytes[13] = 0x14; // An unused object-table entry points into this data.
bytes.fill(0xff, 14, 30);
bytes[30] = 0;
bytes.set([8, 10, 13], 31); // object offsets, including the overlapping entry
bytes[40] = 1; // offset width
bytes[41] = 1; // reference width
bytes.writeBigUInt64BE(3n, 42);
bytes.writeBigUInt64BE(0n, 50);
bytes.writeBigUInt64BE(31n, 58);
return bytes;
};
describe("binary plist number rewriting", () => {
it("does not rewrite an integer marker inside another object's data", () => {
const bytes = dataObjectWithUnusedOverlappingInteger();
const metadata = scanBinaryPlistNumberLiterals(bytes);
expect(metadata.literals.incomplete).toBe(true);
expect(metadata.wideIntegers).toEqual([]);
expect(decodeBinaryPlistNumbers(bytes, metadata)).toBe(bytes);
expect(bytes.subarray(13, 30)).toEqual(
Buffer.from([0x14, ...Array<number>(16).fill(0xff)]),
);
});
});
Loaded 100 of 325 files, more files were not shown because too many files have changed in this diff. Show more