fix(ci): integrate main and repair inherited static checks

This commit is contained in:
hhang committed 2026-10-30 09:20:41 +08:00
commit afd4d571f6
543 files changed
+39569 -12128

No files matched your search

@@ -2,7 +2,7 @@
name: reverse-engineer-anything
description: Reverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.
metadata:
version: "53"
version: "55"
---
# REA
@@ -94,6 +94,9 @@ explicit path or endpoint and do not need it.
than extra calls and schemas. Runtime options are metadata, not files.
Compact delivery still inventories and hashes members. See the
[Python packaged applications guide](https://github.com/morluto/rea/blob/main/docs/python-packaged-applications.md).
- Bun `--compile` executable: `open_binary`, then `inspect_artifact`; the Bun
module graph is the application layer. Extract stored modules and analyze the
JavaScript. See [Bun standalone executables](https://github.com/morluto/rea/blob/main/docs/bun-standalone.md).
- Apple app plist, Interface Builder, asset catalog, or keyed archive resources:
`open_bundle_resources`, then the selected resource tool. This does not
require `CFBundleExecutable` or a deep provider. Resource sessions have no
@@ -124,9 +124,11 @@ shape, with the application's own locators, paths and headers:
}
```
Give every step a `timeout_ms`. A step without one waits until the scenario
is cancelled, so a locator that never matches hangs the capture, and the
client with it. Match `wait_for` locators to exactly one element.
Give each step that accepts `timeout_ms` a deadline. Without one, a step waits
until it completes or the scenario is cancelled, so a locator that never
matches hangs the capture and the client with it. The `wait_for_timeout` pause
instead uses `duration_ms` and does not accept `timeout_ms`. Match `wait_for`
locators to exactly one element.
1. Sign in through the UI: `fill` the user and password fields (the password
as a secret reference), submit, then `wait_for` an element only a signed-in
@@ -153,13 +155,17 @@ client with it. Match `wait_for` locators to exactly one element.
with varied free text. After each write, and before the next one, read the
object back by id and then its list: create 1, read 1, list, create 2,
read 2, list; never several writes before their reads.
4. Leave more than a second between a create and its first edit
(`wait_for_timeout` with `duration_ms: 1100`).
4. Leave more than a second between two creates and between a create and its
first edit (`wait_for_timeout` with `duration_ms: 1100`), so lists sorted
by creation time show them in order.
5. Update both objects and read each back, changing the fields a person
would change (a name, an email address, a status, a due date) rather than
only a description: a served update accepts only the members a recorded
update stored. Delete both and read each back (expect 404, or the object
marked deleted) and the list.
update stored. Run an action that turns something on or off (star,
bookmark, archive) twice in a row on one object, reading it back after
each: the second call shows whether it sets or flips the field, and the
environment serves the action only once a call has shown it. Delete both
and read each back (expect 404, or the object marked deleted) and the list.
6. Vary what the two creates send, however you write them: the first every
field the application takes, the second only what a person must give (its
title or name), with the other fields left out or empty. A served create
@@ -695,3 +695,20 @@ refusals. ESP8266 images are refused, and flash-encrypted images are not
recognized. Segments are not disassembled, and RISC-V or Xtensa segments
cannot be opened as raw-binary. See
[ESP images](https://github.com/morluto/rea/blob/main/docs/esp-image.md).
## Bun standalone executables
A `bun build --compile` executable stores its module graph in Mach-O
`__BUN,__bun`, a PE or ELF `.bun` section, or an appended trailer.
`open_binary` then `inspect_artifact` reports root format `bun-standalone` with
the carrier, structural layout, module count, entry point, raw flags and
compile exec-argv; each module keeps its `/$bunfs/...` source name, loader,
encoding, sizes and SHA-256, with sourcemap, bytecode and module-info regions as
separate children. Use summary root metadata `entry_point_logical_path` as the
exact inventory `logical_paths` selector, then read the selected content node by
its artifact ID; the reader owns virtual-name normalization and indexed fallback. `extract-artifact` writes stored contents through the safe
output tree; route extracted JavaScript to `analyze_javascript_application`.
Layout labels describe record shapes, not a detected compiler version; unknown
or ambiguous layouts are unsupported. Bytecode is retained, not decoded, and the
executable is never run. See
[Bun standalone executables](https://github.com/morluto/rea/blob/main/docs/bun-standalone.md).
+4
View File
@@ -144,6 +144,9 @@ jobs:
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version-file: .nvmrc
- name: Validate source architecture before dependency installation
if: needs.changes.outputs.code == 'true'
run: node scripts/verify/architecture-guards.mjs
- name: Cache npm package downloads
id: npm-cache
uses: ./.github/actions/npm-cache
@@ -773,6 +776,7 @@ jobs:
tests/boundary/providers/native/nativeSignatureTargetBinding.test.ts
tests/boundary/cli/nativeSignatureCli.test.ts
tests/boundary/cli/nativeMachoArm64_32Cli.test.ts
tests/boundary/cli/nativeMachoEncryptionCli.test.ts
tests/boundary/mcp/nativeSignatureTargetBindingMcp.test.ts
src/process/capture/FilesystemSnapshot.test.ts
tests/process-global/filesystemSnapshotFallbackCancellation.test.ts
+161 -5
View File
@@ -22,6 +22,9 @@ HOPPER_SYMBOL_NAME_UTF16_UNITS = 1024
# that does not fit is null, not an older descriptor path.
# 536870887 is Node's 536870888-character string ceiling minus the framing LF.
# The ceiling is the encoded payload; JSON expansion is measured before it.
# Encoded JSON allowance, in bytes, shared by one instruction list, one
# native-caller list, and one caller-scan page. This release has no runtime override.
CALLER_SCAN_ALLOWANCE_BYTES = 512 * 1024
_BRIDGE_MAX_REQUEST_BYTES_SETTING = "REA_BRIDGE_MAX_REQUEST_BYTES"
DEFAULT_BRIDGE_MAX_REQUEST_BYTES = 16 * 1024 * 1024
MINIMUM_BRIDGE_MAX_REQUEST_BYTES = 7464
@@ -232,6 +235,9 @@ EXHAUSTIVE_ANALYSIS_METHODS = frozenset(
"procedure_assembly",
"procedure_callees",
"procedure_callers",
"rea_caller_scan_page",
"rea_caller_scan_target",
"rea_caller_base",
"procedure_info",
"procedure_pseudo_code",
"procedure_references",
@@ -485,19 +491,30 @@ def _instruction_addresses(procedure):
return result
def _native_calls(procedure, direction):
def _iter_native_calls(procedure, direction):
references = procedure.getAllCallees() if direction == "outgoing" else procedure.getAllCallers()
kinds = {0: "none", 1: "unknown", 2: "direct", 3: "objective_c"}
calls = {}
unresolved = []
for reference in references:
source, target, kind = reference.fromAddress(), reference.toAddress(), reference.type()
if source in BAD_ADDRESSES:
yield None, None, None
continue
if target in BAD_ADDRESSES:
unresolved.append({"address": _hex(source), "reason": "Native CallReference has no resolved target (type %s)" % kind})
yield source, None, {"address": _hex(source), "reason": "Native CallReference has no resolved target (type %s)" % kind}
continue
calls[(source, target)] = {"classification": kinds.get(kind, "unknown"), "provider_type": kind, "provenance": "hopper-public-python-api:CallReference"}
yield source, target, {"classification": kinds.get(kind, "unknown"), "provider_type": kind, "provenance": "hopper-public-python-api:CallReference"}
def _native_calls(procedure, direction):
calls = {}
unresolved = []
for source, target, detail in _iter_native_calls(procedure, direction):
if detail is None:
continue
if target is None:
unresolved.append(detail)
else:
calls[(source, target)] = detail
return calls, sorted(unresolved, key=lambda item: int(item["address"], 16))
@@ -547,6 +564,136 @@ def _procedure_map(document):
return result
def _caller_target_data(procedure):
addresses = []
seen = set()
size = 0
for block in procedure.basicBlockIterator():
for address in _block_instruction_addresses(procedure, block):
if address in seen:
continue
encoded = _hex(address)
size += len(encoded) + 4
if size > CALLER_SCAN_ALLOWANCE_BYTES:
return {"target": _procedure_identity(procedure), "instructions": addresses, "instructions_complete": False}
addresses.append(encoded)
seen.add(address)
return {"target": _procedure_identity(procedure), "instructions": addresses, "instructions_complete": True}
def _caller_scan_target(document, params):
return _caller_target_data(_procedure(document, params.get("procedure")))
def _caller_base(document, params):
procedure = _procedure(document, params.get("procedure"))
callers = []
size = 0
complete = True
for item in procedure.getAllCallerProcedures():
address = _hex(item.getEntryPoint())
size += len(address) + 4
if size > CALLER_SCAN_ALLOWANCE_BYTES:
complete = False
break
callers.append(address)
return {
**_caller_target_data(procedure),
"callers": sorted(callers, key=lambda value: int(value, 16)),
"callers_complete": complete,
}
def _caller_scan_page(document, params):
"""Observe forward relationships in one bounded, stateless bridge request."""
start = params.get("start")
if isinstance(start, bool) or not isinstance(start, int) or start < 0:
raise InvalidRequestError("Caller scan start must be a non-negative integer")
segments = document.getSegmentsList()
counts = [segment.getProcedureCount() for segment in segments]
total = sum(counts)
end = min(start + 64, total)
if start > total:
raise InvalidRequestError("Caller scan start exceeds the procedure inventory")
items = []
item_bytes = 0
page_limit = CALLER_SCAN_ALLOWANCE_BYTES
oversized = False
identities = {}
containing = {}
def identity(procedure):
entry = procedure.getEntryPoint()
if entry not in identities:
identities[entry] = _procedure_identity(procedure)
return identities[entry]
def owner(address):
if address not in containing:
procedure, _ = _containing_procedure(document, address)
containing[address] = identity(procedure) if procedure is not None else None
return containing[address]
segment_start = 0
for segment, count in zip(segments, counts):
begin, stop = max(start - segment_start, 0), min(end - segment_start, count)
segment_start += count
for index in range(begin, max(begin, stop)):
candidate = segment.getProcedureAtIndex(index)
item = {"caller": identity(candidate), "callees": [], "references": []}
size = len(json.dumps(item, ensure_ascii=True).encode("utf-8")) + 1
remaining = page_limit - item_bytes
too_large = size > remaining
if not too_large:
for callee in candidate.getAllCalleeProcedures():
address = _hex(callee.getEntryPoint())
size += len(address) + 4
if size > remaining:
too_large = True
break
item["callees"].append(address)
if not too_large:
callee_entries = set(item["callees"])
for source, destination, call in _iter_native_calls(candidate, "outgoing"):
# Account for every native record, even when its target cannot
# be associated with an observed procedure.
size += 32
if size > remaining:
too_large = True
break
if destination is None:
continue
source_procedure = item["caller"] if _hex(source) == item["caller"]["address"] else owner(source)
target_address = _hex(destination)
if target_address in callee_entries:
target_entry = target_address
else:
target_procedure = owner(destination)
if target_procedure is None:
continue
target_entry = target_procedure["address"]
edge = {
"source_address": _hex(source), "target_address": _hex(destination),
"source_procedure": "caller" if source_procedure is not None and source_procedure["address"] == item["caller"]["address"] else source_procedure,
"target_procedure": target_entry,
"call": call,
}
size += len(json.dumps(edge, ensure_ascii=True).encode("utf-8")) + 1
if size > remaining:
too_large = True
break
item["references"].append(edge)
if too_large:
oversized = not items
break
items.append(item)
item_bytes += size
else:
continue
break
return {"start": start, "next": start + len(items), "total": total, "items": items, "oversized": oversized}
def _string_record(segment, address, display):
"""Read one native typed object, retaining its shortened display separately."""
length = segment.getObjectLength(address)
@@ -887,6 +1034,15 @@ def _dispatch(method, params):
if method in EXHAUSTIVE_ANALYSIS_METHODS:
_api().require_analysis_complete(document, method)
if method == "rea_caller_scan_target":
return _caller_scan_target(document, params)
if method == "rea_caller_base":
return _caller_base(document, params)
if method == "rea_caller_scan_page":
return _caller_scan_page(document, params)
if method == "analyze_function":
return _analyze_function(document, params)
if method == "read_function_instructions":
+4
View File
@@ -103,6 +103,10 @@ existing analysis families directly, for example
| `start_adb_activity` | **mutates device** | Starts an intent (`am start -a` with optional URI, component, and up to 16 typed extras); preserves the activity manager's verdict |
| `stop_adb_app` | **mutates device** | Force-stops every process of a package, ending a dynamic-analysis run |
Screen capture validates the PNG signature, IHDR framing, dimensions, and IHDR
checksum before publishing. It does not validate encoding fields or decode and
validate image contents; that limitation is retained in Evidence.
When `push_adb_file` receives an existing directory, adb places the local file
inside it under the local basename; Evidence reports that effective file path
while retaining the caller's directory path in the input parameters. The
+11 -9
View File
@@ -739,14 +739,14 @@ tools are not served, so no call claims a change it does not make; each one's
plan states why in its `reason`. A served write changes the live state, which
every MCP session and `data/get` then see:
| Effect | Call |
| ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `create` | Adds an entity to the collection the path names, built from the observed entity with each ruled field taken from the call or generated, at the plan's `insert` position. A plan tied to its exemplar's collection writes only there. |
| `update` | Sets the sent members on the entity the path names, constants for members accepted only with their recorded value, and the clock, nested clock, increment, slug, stamp, derived, composed, reference, unique and renewed changes the plan models. |
| `update` (action) | A `POST`, `PUT` or `PATCH` to an item path plus an action segment (`/{type}/{id}/archive`) sets, on the entity before the segment, each scalar the recorded calls showed the action set (`set`), with any sent members; it is served only when a recorded call found those fields already set, so a toggle never is. |
| `delete` | Removes the entity the path names. |
| bulk `update` or `delete` | A `DELETE`, `PUT` or `PATCH` at a collection, or a `POST` to it plus `delete`, `remove` or `destroy`, applies its effect to every entity the body names, as recorded: an array of ids or of objects carrying the id field, at the root or as the body's one array member (`{"IDs": […], "Read": true}`). |
| `document` | Merges the body into the plan's document object, or replaces it for `PUT`. |
| Effect | Call |
| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `create` | Adds an entity to the collection the path names, built from the observed entity with each ruled field taken from the call or generated, at the plan's `insert` position. A plan tied to its exemplar's collection writes only there. |
| `update` | Sets the sent members on the entity the path names, constants for members accepted only with their recorded value, and the clock, nested clock, increment, slug, stamp, derived, composed, reference, unique and renewed changes the plan models. |
| `update` (action) | A `POST`, `PUT` or `PATCH` to an item path plus an action segment (`/{type}/{id}/archive`) sets, on the entity before the segment, each scalar the recorded calls showed the action set (`set`), or flips each boolean they showed it flipping both ways (`toggle`), with any sent members; it is served only when a recorded call found what it sets already set, or flipped back what it toggles. |
| `delete` | Removes the entity the path names. |
| bulk `update` or `delete` | A `DELETE`, `PUT` or `PATCH` at a collection, or a `POST` to it plus `delete`, `remove` or `destroy`, applies its effect to every entity the body names, as recorded: an array of ids or of objects carrying the id field, at the root or as the body's one array member (`{"IDs": […], "Read": true}`). |
| `document` | Merges the body into the plan's document object, or replaces it for `PUT`. |
An update's unsent changes follow the plan's rules. A `held-clock` is a clock
the client sends back as it read it and the application stamps anew (Ghost's
@@ -757,7 +757,9 @@ the entity already names under its `created…by` counterpart into a
`modified…by` or `updated…by` field. A `derived` field takes a value showing
the one sent member's text that the entity stores nowhere (a field Ghost
derives from `html`), only with that member's recorded value. An action's
`set` rule sets a scalar to the value the recorded calls left it at.
`set` rule sets a scalar to the value the recorded calls left it at, and its
`toggle` rule flips a boolean, refusing a call on an entity that holds none
there (Miniflux's `PUT /v1/entries/{id}/bookmark` stars and unstars).
Its answer has the status and content type of the plan's recorded response,
with the written entity or its identity rendered in, no body, or the recorded
+14 -10
View File
@@ -179,22 +179,26 @@ share the application workflow, projection, retention and publication owners.
- `schemes_agree_on_signer` (observed): every present block scheme presents
the same _set_ of signer certificates. An extra signer in one scheme is
disagreement; a signer without a certificate makes it `null` (unknown).
Build-tools 33+ rotate through v3.1 by default, so a rotated app usually
reads `false` here and `true` below.
A single present block scheme with known signers agrees with itself;
v1 is compared separately below.
- `signer_change_declared_by_rotation` (declared): the signers differ, but
each appears in a lineage the APK declares. Read the two together: a
rotated APK shows `false` / `true`; signers that differ with no declared
lineage containing them show `false` / `false`. It is `null` when a
signer's identity is unknown, or when the signers differ and a declared
lineage could not be decoded.
each appears in a lineage the APK declares. Read the two together: block
schemes with different signers covered by a declared lineage show `false`
/ `true`; differing signers without such a lineage show `false` / `false`.
A lineage alone does not establish a change between block schemes. It is
`null` when a signer's identity is unknown, or when the signers differ and
a declared lineage could not be decoded.
- `v1_signers_match_block_signers` (observed): the v1 signer certificates
equal what the lowest block scheme that has signers (v2, else v3, else
v3.1) expects v1 to carry. That is its signer set, except that a v3/v3.1
signer declaring a rotation lineage expects the lineage's first
(original) certificate, as apksig requires. `false` is notable: the JAR
signature names a different key than the block signature. A key rotation
keeps v1 (and v2) on the original key, so it reads `true` while
`schemes_agree_on_signer` reads `false`, with or without v2. `null` when
signature names a different key than the block signature. When v1 uses
the lineage's original key and v3 its newer key, this comparison reads
`true`. If v3 is the only present block scheme, agreement is `true` and
declared inter-block signer change is `false` even though rotation is
present. If v2 also uses the original key, agreement is `false` and the
declared inter-block change is `true`. `null` when
v1 or a populated block scheme is absent, any signer's identity is unknown (a
`decode_error`, a `null` certificate or a `subjectKeyIdentifier`
`SignerInfo`), or the lineage's original certificate cannot be
+62
View File
@@ -0,0 +1,62 @@
# Bun standalone executables
`inspect-artifact` inventories embedded Bun module graphs in Mach-O `__BUN,__bun`,
PE `.bun`, ELF `.bun`, and appended trailers. `extract-artifact` writes their stored
module contents through the existing safe output tree. Neither operation executes
the executable or decompiles bytecode.
```sh
rea inspect-artifact ./hello --json
rea extract-artifact ./hello --json
```
In MCP, open the executable with `open_binary`, then call `inspect_artifact`.
Use `detail: "summary"` for large graphs and `inspect_analysis_view` to page retained
Evidence. The root metadata's `entry_point_logical_path` is the reader-owned
location of its entry module. Use it as an inventory `logical_paths` filter,
then read selected node fields with the returned content identity. This also
works for names mapped to indexed extraction paths. The complete Evidence stays
retained; no source-name normalization is needed in the caller.
The root reports the carrier, layout, module count, entry point, raw flags and
compile exec-argv. Each module retains its source name, loader and encoding values,
entry-point status, sizes and SHA-256. Sourcemaps, bytecode and module-info regions
are separate children. Source names such as `/$bunfs/root/main.js` remain in
metadata; extraction uses a relative path. Unsafe or non-UTF-8 names receive an
indexed path and retain their source spelling or bytes. Virtual names deeper
than 256 logical path parts also use an indexed path with an explicit limitation,
preventing unbounded path-tree work while retaining the source fact. Duplicate names remain
separate occurrences; ordinary extraction collision rules apply.
The recognized record shapes are the 36-byte Bun 1.3.3 layout and the 52-byte
Bun 1.3.9 layout. These labels identify structural layouts, not a detected compiler
version. Bun does not store an explicit revision identifier: unknown table sizes
and ambiguous known layouts are unsupported. A future layout that is byte-for-byte
compatible cannot be distinguished. Invalid pointers and truncated claimed sections
are malformed. An absent trailer without a named section does not establish a Bun
graph. Unknown enum values remain raw, with a null name.
Sourcemaps are stored Bun regions, not necessarily standalone JSON source maps.
Extracted contents retain their original encoding. Use JavaScript analysis only on
compatible source files; bytecode is retained as evidence, not decoded.
Module parsing reads exact metadata ranges rather than loading stored asset
bodies or whole pages around small pointers. A cache retains at most 256 KiB
and 1,024 ranges for one reader preparation. Cumulative uncached metadata reads
are limited to 256 MiB, checked before allocation or I/O. The module table,
decoded strings and combined logical path tree have separate admission checks;
at most 250,000 distinct path prefixes are admitted before inventory construction.
Stored bodies are range-streamed for hashing and extraction. Header/suffix observations are reused
only for the same admitted file state, with selected-path checks before publication.
See [limits and recovery](design/limits-and-recovery.md) for admission ceilings.
Synthetic tests cover the carriers and malformed declarations. Run
`node scripts/verify-bun-standalone.mjs` after `npm run build:cached` for local Bun
verification, and add `--docker` to compile a Linux fixture with `oven/bun:1.4.0`.
Bun and Docker are fixture producers only and are not installed by REA.
Layout references: [Bun 1.3.3 serializer](https://github.com/oven-sh/bun/blob/bun-v1.3.3/src/StandaloneModuleGraph.zig)
and [Bun 1.3.9 serializer](https://github.com/oven-sh/bun/blob/bun-v1.3.9/src/StandaloneModuleGraph.zig).
The current 52-byte record order can also be checked in the
[Bun 1.4.4 Rust serializer](https://github.com/oven-sh/bun/blob/bun-v1.4.4/src/standalone_graph/StandaloneModuleGraph.rs).
+5
View File
@@ -1256,3 +1256,8 @@ rea inspect-pe-load-config /absolute/path/to/application.exe --json
Reports TLS callbacks and size-covered load-config fields without execution.
Use `--max-callbacks` and `--max-file-bytes` to set read budgets. See
[PE TLS and load configuration](pe-load-config.md).
Bun standalone executables use the existing `inspect-artifact` and
`extract-artifact` commands to inventory and materialize embedded source,
sourcemaps and bytecode without running the executable. See
[Bun standalone executables](bun-standalone.md) for layout coverage and limits.
+3 -2
View File
@@ -283,8 +283,9 @@ patterns. A provider being unavailable does not discard its selected paths;
doctor reports its readiness separately. Explicit selections override discovered
defaults, including when setup runs again with a changed selection. Unrelated
environment variables, process ownership tokens and one-shot target paths are not
copied. Existing client environment overrides remain intact unless setup supplies
the same key.
copied. For an explicitly configured startup target, see the
[initial-target settings](mcp-contracts.md#initial-target-at-startup). Existing
client environment overrides remain intact unless setup supplies the same key.
You can edit `startupTimeoutMs` and `toolTimeoutMs` there; doctor keeps positive
caller-selected values aligned and reports a non-blocking advisory when a tool
timeout is shorter than the known provider allowance.
+14
View File
@@ -378,3 +378,17 @@ The [verification report](../verification/runtime-analysis-settings.md) separate
public MCP checks, simulated provider coverage, real Ghidra execution, and
measured task costs. It does not establish real-provider behavior for every
editable setting or active worker telemetry.
## Bun standalone module graphs
`MAX_BUN_MODULE_COUNT` admits at most 100,000 module records.
`MAX_BUN_RETAINED_BYTES` admits at most 256 MiB of serialized graph payload and
separately bounds retained entry metadata. Physical metadata read work is also
limited to 256 MiB per preparation, including repeated uncached ranges. A
256 KiB / 1,024-entry exact-range cache avoids full-page amplification. The
combined logical path tree admits at most 250,000 distinct prefixes; this is
checked before adding nodes, with shared prefixes counted once. The shared
artifact expansion budget also bounds the sum of referenced member lengths. Refusals name the boundary.
There is no runtime override; additional host memory or `NODE_OPTIONS` does not
raise these ceilings. Use a smaller producer fixture or a separately reviewed
reader for larger graphs. No partial extraction is claimed on admission failure.
+110
View File
@@ -0,0 +1,110 @@
# .NET single-file bundles
Use `rea inspect-artifact /absolute/path/host --json` to inventory an ordinary
.NET single-file host and `rea extract-artifact /absolute/path/host --json` to
materialize its members into a fresh, REA-owned output directory. With a host
selected through `open_binary`, the corresponding MCP tools are
`inspect_artifact` and `extract_artifact`. No .NET installation is needed for
inspection or extraction.
For a compact inventory, call `inspect_artifact` with `detail: "summary"`,
then use `inspect_analysis_view` with the returned `parent_evidence_id`.
Inventory occurrence pages accept `logical_paths`, `occurrence_ids`, selected
`fields`, `offset`, and `limit`; node pages accept batches of `artifact_ids`
to retrieve expanded-byte digests. Include `format_metadata` and
`source_location` when member declarations and stored extents matter. Identity
fields survive projection, and the complete inspection Evidence stays retained.
REA detects the apphost marker and its preceding header offset from PE, ELF,
and thin Mach-O bytes, regardless of filename. An unbundled apphost's zero
offset remains an ordinary native artifact. Universal Mach-O slice inventory
keeps its existing behavior; bundled contents inside slices are not expanded.
When a native file has a recognized PyInstaller CArchive, apphost-marker
scanning stops before its payload. A .NET helper stored inside that archive
does not replace the outer container's format; an outer .NET host can still
contain a PyInstaller helper as a member.
## Supported layout
| Header | Producer generation | Member encoding |
| ------ | ---------------------------------- | --------------------------- |
| 1.0 | .NET Core 3.x | Uncompressed |
| 2.0 | .NET 5 | Uncompressed |
| 6.0 | .NET 6 and later using this layout | Uncompressed or raw DEFLATE |
Versions, minor versions, and flags outside this set are refused explicitly;
they do not silently become an ordinary PE or a successful empty inventory.
Flags 0 and the v2+ compatibility flag 1 are admitted. Unknown file-type bytes
are retained with a limitation, rather than treated as executable authority.
Encrypted/protected wrappers and recovering their payloads are outside this
reader's scope.
The host keeps its whole-file SHA-256. Each embedded file gets its expanded-byte
SHA-256, path, declared size, compressed size, and `format_metadata` declaration
(`major_version`, `minor_version`, `file_type`). Its `source_location` describes
the stored host extent, including the compressed length when applicable.
The root manifest's `root_format` is `dotnet-bundle`; the root content node
still records its native PE/ELF/Mach-O format. A file-type declaration or `.dll`
suffix does not establish a valid assembly. Inspect an extracted component
explicitly with the managed tools to establish PE/CLI facts.
## Admission and containment
Metadata is range-read with a 64 KiB cache. Policy budgets are 16 MiB of
manifest metadata, 100,000 files, 64 KiB per UTF-8 string, 512 MiB of stored or
expanded bytes per file, 256 path parts per member, 250,000 distinct in-memory
path-tree nodes, and 2 GiB of aggregate expanded bytes. The node budget counts
shared directory prefixes only once. These are REA
resource policies, not limits imposed by Microsoft's format. Marker scanning
streams 64 KiB chunks and reuses completed probe facts during extraction;
ordinary native files without a marker incur one extra
read pass before host hashing.
Extraction hashes the whole host again through the same descriptor used to read
members and compares it with the inventory's root digest before creating an
output directory. This adds a full-host read pass to extraction, including when
the embedded members themselves are small.
Malformed/truncated headers, unsafe or conflicting paths, overlapping member
ranges, members crossing the manifest, contradictory configuration extents,
invalid UTF-8, corrupt/trailing compressed data, and incorrect expanded sizes
fail before output publication. Source descriptor/path identity is checked
during reading; extraction verifies the host digest and member bytes against
the inventory. Existing output roots are never overwritten. Cancellation and
failures release owned streams/descriptors through the shared extraction
rollback scope.
## Reproduce producer fixtures
The source-owned generator uses the installed .NET 8 SDK's **Microsoft
HostModel** producer to emit x64 v1, v2, plain v6, and compressed v6 bundles
for Windows, Linux, or macOS. It does not run generated hosts or download runtime packs. Supply an
SDK apphost template from
`packs/Microsoft.NETCore.App.Host.<rid>/<version>/runtimes/<rid>/native/apphost`
(`apphost.exe` on Windows). Pass `windows` (the default), `linux`, or `osx`
as the optional third generator argument to match that template.
```bash
dotnet build scripts/fixtures/dotnet-bundle/BundleFixture.csproj -c Release --configfile scripts/fixtures/dotnet-bundle/NuGet.Config -o /absolute/path/producer -p:BaseIntermediateOutputPath=/absolute/path/producer-obj/
dotnet /absolute/path/producer/BundleFixture.dll /absolute/path/apphost.exe /absolute/path/absent-bundles-directory
rea inspect-artifact /absolute/path/absent-bundles-directory/net8-compressed/Probe.exe --json
rea extract-artifact /absolute/path/absent-bundles-directory/net8-compressed/Probe.exe --json
```
The generator writes `oracle.json` beside each host. An independent C#
`BinaryReader`/`DeflateStream` pass records version, member types and stored
extents, then compares every expanded member digest with its source file.
Compare those records with inspection Evidence and the extracted bytes.
This verifies Microsoft's compression writer separately from synthetic parser
fixtures; it is not evidence that the bundled host executes on a target runtime.
Deterministic boundary and public-entrypoint coverage:
```bash
npm run test:focused -- tests/boundary/filesystem/dotnetBundleArtifacts.test.ts tests/acceptance/dotnetBundleJourney.test.ts
```
Layout references: Microsoft's [manifest writer](https://github.com/dotnet/runtime/blob/v8.0.22/src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs),
[file entry writer](https://github.com/dotnet/runtime/blob/v8.0.22/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs),
[apphost marker handling](https://github.com/dotnet/runtime/blob/v8.0.22/src/installer/managed/Microsoft.NET.HostModel/AppHost/HostWriter.cs),
and [compression producer](https://github.com/dotnet/runtime/blob/v8.0.22/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs).
+5
View File
@@ -82,6 +82,11 @@ Published file `relative_path` preserves producer spelling for lineage; `path`
uses the writer's actual normalized filename. An extracted filename that cannot
be represented losslessly as UTF-8 fails publication explicitly with its raw
name bytes in hexadecimal instead of silently omitting the entry.
Unpublished symlinks retain the extracted tree's target bytes in lowercase
`link_target_hex`; `link_target` is null when those bytes are not valid UTF-8.
Older retained Evidence may omit the hexadecimal field. Unblob may have rewritten
the target before REA observed it, so these bytes are not necessarily those stored
in the firmware image.
## CLI and MCP
+6 -1
View File
@@ -928,7 +928,12 @@ selected only when it is the arm64 family's only candidate. REA checks the
entire source's SHA-256 while copying the slice; source identity and reported
offsets still refer to the original container. Any other ambiguous architecture
subtype requires an explicitly extracted thin image; REA does not guess.
Configured loader arguments remain explicit overrides.
Set `HOPPER_LOADER_ARGS_JSON` in REA's launch environment to a JSON array of
strings to override the derived Hopper loader arguments, for example
`["-l","Mach-O"]`. Empty string elements are preserved; NUL bytes are not
accepted. An omitted setting or empty array uses the derived arguments. `rea
setup` forwards an explicitly selected setting to supported registrations;
restart the MCP server to apply a change. These arguments are Hopper-specific.
FAT64 is distinct from the CPU architecture: FAT32 containers can contain 64-bit
executable images. This preparation avoids the Raw Binary loader dialog observed
with native FAT64 loading on Hopper 6.1.0-demo.
+19
View File
@@ -224,6 +224,25 @@ resulting value. Deferred callback bodies do not become inputs to the enclosing
assignment. Computed method keys belong to the enclosing callable; parameter
defaults and method bodies belong to the method. Uncertain later property reads
remain opt-in through `include_ambiguous_dynamic_edges`.
Binary, logical, unary, template, conditional, and sequence expressions retain
`influences` edges from supported immediate inputs to the result, then to a
direct declaration, assignment, call argument, or return. These are static
dependencies, not aliases or proof that the expression executed; conditional
operands and logical-assignment writes carry `conditional: true`. Unsupported
operand dependencies, unmodeled receiving containers, dynamic call results,
and sequence side effects remain unknown. An applicable
unknown makes the trace status `partial` even when a seed was found. Literal
operator operands, direct call arguments and directly returned scalar literals
can also be seeds. Older graphs that only recorded a call-site literal remain
searchable. The offered per-file node capacity also bounds the producer's
expression and operand collections before graph projection.
Expression nodes and unknowns share the existing per-file graph admission
budgets; omitted facts make graph coverage truncated and the trace partial.
Consumers that previously treated `found` as compatible with a relevant
unknown should handle `partial` and inspect the returned unknowns. Retained
graphs without `influences` edges remain readable but do not gain edges until
the application is analyzed again.
Global timer, `fetch`, `WebSocket`, `JSON.parse`, and coercion calls receive
these built-in effects only when the file has no visible possible write to that
global, including a computed write through a global object. The check does not
+6 -2
View File
@@ -243,13 +243,17 @@ authenticates the saved bundle; the full source is retained on disk.
The canonical parser admits PE/CLI bytes and their metadata/CIL without loading
an assembly. It reports observed implementation markers and unavailable facts;
it does not unpack .NET single-file hosts or infer a NativeAOT identity from an
ordinary native PE. It does not decode IL2CPP; use
it does not itself unpack .NET single-file hosts or infer a NativeAOT identity
from an ordinary native PE. It does not decode IL2CPP; use
[IL2CPP metadata](il2cpp-metadata.md) for the image, type and method names in a
Unity `global-metadata.dat`. Inputs without admitted CLI metadata do not
become managed assemblies through naming or routing guesses.
Inspect separately obtained components explicitly and preserve their identities.
For ordinary .NET single-file hosts, use the existing artifact tools to
[inventory and extract bundle members](dotnet-bundles.md), then inspect the
extracted assembly explicitly with the managed tools.
A valid implementation marker establishes a candidate native boundary, not
recovered native semantics or runtime behavior.
+35 -10
View File
@@ -177,8 +177,13 @@ if retention is unavailable or fails, the diagnostic reports that reason.
SDK-generated input validation failures are normalized into the same canonical
`invalid_request` text when the bounded diagnostic fits the response budget.
The complete field/type diagnostic is preserved when normalization fits the
response budget, including request-specific rejected fields and reasons. Oversized
SDK diagnostics keep the existing retained-Evidence recovery path. An unknown
response budget, including request-specific rejected fields and reasons. Zod
input issues retain their field paths and available type and range constraints.
SDK diagnostic parts may be distributed between field issues and the main
message so each retained explanation appears once.
If their conservative projection allowance is exhausted, the response states
that field-level detail was omitted and retains the original SDK diagnostic.
Oversized SDK diagnostics keep the existing retained-Evidence recovery path. An unknown
tool remains an MCP JSON-RPC `-32602` error, but its message directs the caller
to `tools/list` and an advertised operation instead of repeating the unknown
name.
@@ -435,10 +440,12 @@ result's `_meta`, whether the call succeeded or failed:
```
The marker is declared once per tool contract (`returnsTargetContent`) and is
on by default: only `export_evidence_bundle` and `record_unknown`, whose
results and errors contain only REA session state or caller-supplied values,
omit it. Tools whose results or errors can include provider, launcher, or
device output stay marked, including `binary_session` (provider failure
on by default: `export_evidence_bundle`, `record_unknown`, `search_guidance`,
and `read_guidance_section` omit it. The first two contain only REA session
state or caller-supplied values; the guidance tools return REA's own installed
documentation, not text from an analysis target. Tools whose results or errors
can include provider, launcher, or device output stay marked, including
`binary_session` (provider failure
reasons and availability diagnostics), `close_binary` (provider cleanup
diagnostics with captured launcher output), `inspect_adb_client` and
`inspect_apktool_client` (version output and stderr excerpts), and
@@ -590,10 +597,12 @@ reach WABT or change the original source Evidence ID.
Completed `inspect_macho` Evidence supports a summary and
`view: {kind: "macho", facet: "exports", offset: 0, limit: 64}`. Facets are
`entrypoints`, `architectures`, `build_metadata`, `load_commands`,
`dependencies`, `imports`, `exports`, `segments`, and `provenance`; pages return
the original rows. The summary keeps the selected slice, header fields and UUID,
and replaces each collection with its returned count, reported total and
`exhaustive` flag. Each page repeats the paged collection's `source_total` and
`dependencies`, `imports`, `exports`, `segments`, `encryption`, and
`provenance`; pages return the original rows. Evidence retained before
`encryption` was reported pages no encryption rows and reports that collection
as non-exhaustive with an unknown status. The summary keeps the selected
slice, header fields and UUID, and replaces each collection with its returned
count, reported total and `exhaustive` flag. Each page repeats the paged collection's `source_total` and
`source_exhaustive`; exhausting a non-exhaustive collection does not establish
that unreturned symbols are absent. Large frameworks return megabytes of
symbols (Electron Framework: 2.5 MB of imports and exports, 3.3 KB as a
@@ -791,6 +800,22 @@ inspection fails or is cancelled before a provider client exists. If no client
remains, no provider run or health observation is available; follow the cleanup
error's reported resources.
## Initial target at startup
MCP normally starts without a target; use `open_binary` to select one during the
session. To open a target before serving MCP, set `HOPPER_TARGET_PATH` to the
selected path in REA's launch environment. `HOPPER_TARGET_KIND` accepts
`executable` (the default) or `database`. Despite the names, these are initial
session settings; the configured provider still determines how the target is
opened. If the initial open fails, REA reports the failure and exits with code 1
without serving MCP. Shutdown during startup cancels and cleans up the open as
described below.
The source-checkout helper `node scripts/print-mcp-config.mjs /path/to/target`
prints a registration containing these settings; `--kind database` selects the
database kind. The helper is not part of the installed npm package. `rea setup`
does not copy these one-shot target settings into generated registrations.
## Transports
The eager stdio runtime owns SIGINT, SIGTERM and stdin EOF before opening an
+20
View File
@@ -18,6 +18,26 @@ non-string values are rejected with field paths. `action: "list"` accepts no
workflow identifier or arguments. The MCP adapter respects the same
`REA_MCP_TOOL_FAMILIES` scope as native prompts, including render requests.
To find a section in the installed guidance bundle by analyst question, use
`search_guidance` and pass a returned locator to `read_guidance_section`. The
CLI equivalent is `rea guidance --query "exact JavaScript module"`, followed
by `rea guidance --action read --locator LOCATOR`. Search is limited to the
packaged managed skill files, works without providers or a target, and returns
deterministically paged matches. Every non-generic query word must occur in the
section; generic-only queries return no matches. Results are documentation; locators
are bound to the installed bundle and report stale or unavailable guidance without
returning cached section content. A no-match result says nothing about analysis
capability availability.
Search returns at most 20 matches per page with 500-code-unit context excerpts.
Read retrieves the complete selected section. Admission permits 1 MiB per file,
4 MiB per bundle, 4,096 sections, and 25,165,824 expanded UTF-16 code units before
section text is materialized. Unreadable or oversized bundles return
`bundle_unavailable`, with no partial index or inferred absence. Cleanup failures
remain errors with their retained retry owner. The bundle and complete section
are bound by fixed-size hashes in locators, so a long heading does not make its own
locator unreadable.
## Prompt inventory
| Prompt | Purpose |
+63
View File
@@ -343,6 +343,27 @@ Hopper regex mode.
## Bounded static traces
Hopper caller queries, incoming procedure references and function dossiers
reconcile reported reverse relationships with forward call observations. Each
standalone query or composed workflow scans the current document in pages of at
most 64 procedures and retains at most 4 MiB of encoded scan results. A composed
workflow reuses its own index and clears it when the workflow finishes. The
default cooperative scan budget is
2,000 ms; set `REA_HOPPER_CALLER_SCAN_BUDGET_MS` to a positive integer number of
milliseconds before starting REA to change it. The default limits additional
work on the serial Hopper bridge; larger documents may need a higher setting.
Results report the examined procedure count and any budget interruption.
An individual page, native caller list, and target instruction list each have
a 512 KiB encoded allowance. Hitting one reports the incomplete observation.
Cancellation is checked between pages, but cannot interrupt a Hopper API call
already executing; Hopper's native API itself materializes its reference arrays
before REA can bound their copies. Hopper exposes no document revision token, so scans are not
reused across workflows and coverage and consistency during a live scan remain
unknown even when every procedure was examined.
Hopper observations are re-observed rather than replayed from snapshots because
the provider has no document revision token. Safe observations may still be
retained as Evidence.
`trace_native_ui_action` / `rea trace-native-ui-action <target> <seed>` accepts
one authored selector/object ID, native symbol or exact function address. It
joins UI wiring to uniquely matched class/selector implementations and follows
@@ -568,6 +589,40 @@ segments do not produce file ranges. When the selected slice's container offset
is unknown, those mapped locations are omitted with an explicit limitation;
architecture inventory locations remain available.
`inspect_macho.encryption` reports the `LC_ENCRYPTION_INFO(_64)` state of every
slice, not only the selected one. It decodes only the FAT table, slice headers
and load commands. Like every Mach-O container read, a `0xcafebabe` input of at
most 16 MiB is first read whole to rule out a JVM class file, so that read
includes code bytes; larger inputs and thin images read only the structures
above. A slice is `encrypted` when its single encryption command has a nonzero
`cryptid`, `not-encrypted` when `cryptid` is zero, and `absent` only when every
load command was read and none was an encryption command. A command whose width
or `cmdsize` contradicts the image, more than one such command, or a range past
the slice is `malformed`; a slice whose commands could not be fully read is
`malformed` or `unknown` with a reason, keeping any commands seen before the
failure. A slice with more than two encryption commands keeps the first two
and states the count in its reason. Range mapping in all slices shares one
budget of segment and section records (16,384, a sixteenth of the native
output budget); a slice past it keeps its status and reports its mapping as
`unknown` with that reason. Framing caps retained segment commands before decoding.
Only a successful mapping consumes the shared allowance; failed mappings leave
it available to later slices. Each slice's `range` is end-exclusive in slice and input-file
coordinates and maps to file-backed segment fragments, VM addresses and the
overlapping nonempty sections; bytes outside every segment remain unmapped
fragments. A segment with no VM size keeps its file bytes as fragments with
null VM addresses, and a segment whose file size exceeds its VM size or whose
VM range leaves the address space makes the mapping `unknown` with a reason
while the slice keeps its encryption status. `architecture` is the lipo name of the record at the same offset,
size and CPU identity, and is null when no single record matches. A ciphertext
range adds a limitation naming its input-file bytes, because disassembly,
decompilation, strings and references derived from them do not describe
program code. Encryption is read through the same regular-file descriptor
that `inspect_macho` binds for its load-command reads, whose SHA-256 must match
the session target before and after the native tools run, or the call reports
`artifact_changed`. A target that cannot be opened, or a symlinked, non-regular
or changing target, fails the call instead of reporting tool facts without an
encryption status.
Hopper's `address_to_file_offset` combines the native image-relative mapping with
original-file coordinates only after checking the source against the session's
SHA-256. FAT tables and embedded Mach-O headers come from that same hashing pass.
@@ -604,6 +659,9 @@ complete observation.
undecodable filenames and skipped symlinks, as partial coverage. Known
executable roots remain available; unexamined names and symlink targets
cannot establish an empty root set or the absence of other roots.
A dependency symlink whose target is not valid UTF-8 is undetermined, with
its stored target bytes in the limitation and partial coverage; a selected
root crossing such a symlink is refused with the same bytes in its diagnostic.
- `inspect_asset_catalog` / `rea inspect-asset-catalog <app>` reads compiled
`Assets.car` metadata through macOS `assetutil --info`. Catalog digests, raw
rendition fields, pagination and exact UI resource-name matches are returned.
@@ -757,6 +815,11 @@ complete observation.
files and universal slices before metadata decoding. CPU subtypes and
capability bits remain source facts; an `arm64` request may select an
`arm64e` image under the existing slice preference policy.
Objective-C class, protocol and category lists require eight-byte entries;
Swift type and conformance sections require four-byte entries. Misaligned
section lengths and unsupported non-PowerPC big-endian CPU declarations return
`unsupported_target` with the selected path and precise reason. Recovery
selects a well-formed or supported artifact and reopens that target.
The byte reader holds the whole selected file in memory, so it admits files up
to 64 MiB / 67108864 bytes by default. Select a larger budget with
`max_input_bytes` (CLI `--max-input-bytes`), at most 1 GiB / 1073741824
@@ -0,0 +1,11 @@
# Implementation plan
1. **Sorted creates.** Placed creates whose confirming read was a whole listing in collection order may be placed
again; narrowing keeps the sorts a later listing follows and adds those its items show, the type's id field last.
2. **Listed references.** `referenceRule` reads a sent list of ids; `many` on the reference rule and its schema;
`referencedValue` maps each id; the create merge lets an empty reference agree and frees the members it fixed.
3. **Hidden bodies.** Scenario records give a request with a body Playwright did not expose, and no `Content-Type`, a
`bodySize` of 0.
4. **Toggles.** `toggledRules` turns a plan's `set` and a later call's opposite `set` into `toggle`; the rule's schema,
serving, later-call checks and the action verdict.
5. **Skill, tests and docs.**
@@ -0,0 +1,41 @@
# Serve sorted creates, listed references and toggles
Issue: https://github.com/morluto/rea/issues/4933
## Outcome
Over 148 recorded captures (the 130 measured before and 18 from a fresh recording round), #7539 withholds 302 of 694
write plans. A short diagnosis of the largest clusters found three gaps REA could close:
- **A create's sort learned too late.** Grafana lists folders by title. A recording's first scratch folder sorted
after the one seeded folder, so the plan placed creates last; the second sorted before the first, the list after it
contradicted the plan, and the update and delete after that went unconfirmed.
- **Lists of references.** Homebox creates an item from `parentId` and `tagIds` and answers with the parent and tag
objects. REA modelled a parent but not a list of tags, and a create that sent `null` and `[]` disagreed with one that
named both.
- **Toggles.** Miniflux stars and unstars an entry with a body-less `PUT /v1/entries/{id}/bookmark`. Chromium exposes
no body for it, which REA read as unusable, and an action flipping a flag back was taken to contradict the call that
set it.
## Behaviour
- **Sorted creates.** A create a whole listing in collection order placed last is placed again by a field a later such
listing shows it sorted by, the type's id field last. A sort an earlier such listing did not follow stays ruled out,
though the entity that showed it is gone. A create another read placed keeps its place.
- **Listed references.** A field holding one object per id a sent list names, in that order, is a `reference` with
`many`. A create that sent the member as `null` or `[]` and stored the field so agrees with one that named entities,
whichever create first showed the reference;
serving stores the entities a call names, or the empty value as that create did. A `null` id stays refused where
no recorded create sent one, and a call leaving the member out stores no field where the observed create showed
none.
- **Hidden bodies.** A scenario request whose body Playwright did not expose and that names no `Content-Type` sent no
body.
- **Toggles.** An item action a later call shows flipping back a boolean an earlier call set, when no call found it
already set, has a `toggle` rule: serving flips the field, refusing an entity that holds no boolean there.
- **Skill.** Recordings run an action twice in a row on one object, reading it back after each (skill 55).
Plans and views that need none of this are unchanged.
## Acceptance
- No write of the 148 captures goes from served to withheld, and no view changes.
@@ -0,0 +1,11 @@
# Tasks
- [x] Place a create again by the sort a later whole listing shows.
- [x] Learn and serve references a sent list of ids names, and let an empty reference agree.
- [x] Treat a hidden request body without a `Content-Type` as absent.
- [x] Learn, check and serve toggles.
- [x] Ask recordings to run an action twice on one object.
- [x] Add view, serving, record and unit tests.
- [x] Document the rules.
- [x] Measure the recorded captures.
- [x] Run static, formatting, documentation and test gates.
@@ -0,0 +1,88 @@
# Verification
Executed on macOS with Node v24.22.1 on this branch and on #7539 at `30e2a582`; the census and the base reproduction
were repeated after merging #7539 at `2817665d`, `174a0ff2`, `5691f97f` (which merges main with #7332)
and `3b800a10`.
## Diagnosis
Over 148 recorded captures, #7539 withholds 302 of 694 write plans. The largest clusters: Homebox item writes (36),
unconfirmed deletes (23), Grafana's feature-flag evaluation calls (15, which change no state), Linkding renumbering
bundles after a delete (7) and Miniflux's bookmark toggle (5).
- **Grafana.** A fresh recording created "Scratch Pier 3", which the list after it showed after the one seeded folder,
then "Scratch Meadow Notes 4", which the list after it showed before the first, by title. The create plan had settled
on `last` when the first list confirmed it, so the second list contradicted it, and the folder update and delete
after it went unconfirmed.
- **Homebox.** Item creates send `parentId` and `tagIds` and answer with `parent` and `tags` objects. The recording
that created one item with neither (`null` and `[]`) and one with both was withheld: `tags` had no rule, and the
first create fixed both members to their empty values.
- **Miniflux.** `PUT /v1/entries/{id}/bookmark` sends no body and answers `204`. Chromium exposes no body for it and
the request names no `Content-Type`; REA read the body as unusable. Across the captures, every `POST`, `PUT` or
`PATCH` whose body was captured names `content-type` among its request header names (835 of 835).
## Gates
- `npm run -s check:fast`, `format:check`, `knip`, `scan:todos`, `docs:check`, `verify:test-discovery`,
`mcp-catalog:check` and `jscpd`: passed.
- `npx vitest run --project domain --project services`: 11,886 passed, 6 skipped, merged with `3b800a10`. An earlier run of the suite failed a
`SetupRegistrationPrewarm` runner test whose child process outran its 5-second timeout; alone it passes on this
branch and on #7539 (3.0 to 4.7 seconds).
- `npm run -s test:focused --` the environment build, export and serve CLI boundaries, the analysis-view CLI and MCP
boundaries and the environment serve MCP boundary: 7 files, 81 passed. `outputSchemaProfile`, `toolSchemaValidity`,
`mcpSchemaValidation`, `outputSchemaSemanticConstraints` and `analysisViewMcp`: 5 files, 54 passed; the compact
catalog stays under its frame.
## Tests
- View: a create placed last, then a create a later whole listing shows before it by title, learns `title` ascending
and its delete is served; an item create naming its parent and tags by id, after one naming none, learns
`reference` rules for `parent` and for `tags` (`many`) and is served; a body-less action two calls show starring
then unstarring an entry learns `toggle` and is served, and is not when its calls flip the flag one way only or a
later call leaves it unchanged.
- Serving: a create stores the parent and the tags its ids name, in the order sent; a create leaving out both, or
sending them as `null` and `[]`, stores `null` and `[]`; a tag the environment does not hold is refused, and so is a
`null` shelf where every recorded create named one; an entry the recording never starred flips on each call.
- Records: a hidden request body is absent when the request names no `Content-Type`, and stays unusable when it names
one.
- The nine behaviour tests fail on #7539 at `30e2a582`, `2817665d`, `174a0ff2`, `5691f97f` and `3b800a10`; the three that guard toggles and `null` ids
pass there too.
## Write plans and views over recorded captures
The 148 captures, built by #7539 and by this branch: 694 plans, 392 served and 399, both at `30e2a582` and after
merging `2817665d`, `174a0ff2`, `5691f97f` and
`3b800a10`.
- No write goes from served to withheld, and no served plan changes. Newly served: Grafana's folder create, update and
delete in a fresh recording; Homebox's item create in another; Miniflux's bookmark toggle in three recordings.
- Five withheld plans explain more. Two Miniflux recordings learn the toggle but stay withheld: a list of starred
entries with no `order` parameter lists them by `published_at`, which REA answers in state order. Linkding's
unarchive learns its body-less call and stays `unconfirmed`, seen once. Two Homebox item writes learn their tag
references and stay withheld for other members.
- Views are unchanged: 3,188 views across the captures.
## Review round
Greptile's review of `4e777d28` raised five findings. Two were in this branch's code and are fixed:
- A sort an earlier whole listing ruled out could come back from a later listing once the entity that ruled it out was
deleted, and a create's retry could adopt it. Each create now keeps the sorts its whole listings did not follow.
- A create sending a reference's member as `null` after a later create added the reference, the first having shown no
such field, was withheld: the empty-reference agreement now applies there too.
Both tests fail on `4e777d28` and on #7539 at `174a0ff2`; the census is unchanged. The other three were in #7332's and
#7539's code: `308aebf0` (#7332), already here, covers the made-object count, and #7539's `174a0ff2`, merged here,
fixes the update default and the read clock spelling.
Greptile's review of `f7c1bfe9` raised one more, fixed: once a later create sent a reference's member as `null`, a
served create leaving the member out stored `null`, though the create the plan observes left it out and stored no such
field. Serving now leaves the field out there, and still stores `null` for a call that sends it; the serving test fails
on `f7c1bfe9`. The census is unchanged.
## Not verified
- No environment was rebuilt and run with agents for this branch, and no recording was made with skill 55.
- The order an application gives a list read with no sort parameter (Miniflux's `published_at`) when it is not state
order.
- The rest of the Homebox cluster: `assetId` numbering and item updates; Linkding renumbering bundles after a delete.
@@ -0,0 +1,9 @@
# Implementation plan
1. **Update members.** `refinedBody` keeps only the required members a later update also sent.
2. **Numbered unless sent.** `sent` on sequence rules and their schema; the create merge adopts it for an echoed member
and a sequence from a call that left the member out; serving and replay take the sent value when there is one.
3. **Cleared by a PUT.** `explainUpdate` records a `PUT`'s left-out member cleared to an empty value as a default;
update refinement keeps defaults from later calls; `updatedFields` applies them to members a call leaves out.
4. **Update clocks.** `restampedFields` respells clocks from the first later read as it does nested clocks.
5. **Skill, tests and docs.**
@@ -0,0 +1,56 @@
# Serve writes whose calls send different members
Issue: https://github.com/morluto/rea/issues/4933
## Outcome
A fresh recording round of seven applications, main against #7332, showed the recording skill now varies what the two
creates and two updates of a type send, as it asks. Where the second call left out a member the first sent, REA
withheld the write:
- **Updates.** An update plan required every member its first call sent and never narrowed that, so EspoCRM's second
contact update, which changed only an email address, was refused in both recordings of the branch.
- **Members numbered when left out.** Linkding stores a bundle's `order` as sent, or one past the others when a call
leaves it out. One create showed the first and another the second, which the create merge took as a disagreement.
The same recordings and the 130 earlier captures showed two more update gaps:
- **Whole-object PUTs.** Paperless clears a tag's, correspondent's or document type's `owner` when a `PUT` leaves it
out, and Homebox an item's `assetId`, so those updates were `unexplained-change`.
- **Update clocks read at another precision.** Vikunja answers a task update with `done_at` to the nanosecond while its
reads spell it to the second, so the read after the update contradicted it.
The round also found two creates of EspoCRM contacts in one second, which a list sorted newest first shows in an order
the application breaks ties in. The skill now spaces creates.
## Behaviour
- **Update members.** A later update narrows the plan's required members to those every call sent.
- **Numbered unless sent.** A member one create sent and stored as sent, and another left out and the application
numbered one past the others, is a `sequence` with `sent`, in either order: serving stores the value a call sends,
else the next number, and a later call that sends it must have stored what it sent. The next number follows the
largest value any call took, a deleted entity's included, in replay as in serving.
- **Cleared by a PUT.** A field a `PUT` left out that the application cleared to an empty value (`""`, `null`, `false`,
`[]` or `{}`) is that member's default (`body.defaults`) when every call that left it out stored that value, as its
answer shows it or, where the answer leaves the member out, the read after it: serving stores it for an update that
leaves the member out. Calls that left it out and stored it otherwise withhold the plan.
- **Update clocks.** The state holds an update's clock as the first read of the entity after it spells the same instant
at a coarser precision in the same zone and separator, as it already does for nested clocks: replay and serving store
every update's clocks in the spelling of the read the plan keeps as `observation.read`, so a later update whose read
spells them otherwise contradicts the plan, while answers keep the update's own. A create's clocks stay as answered,
as serving makes them.
- **Skill.** Recordings leave more than a second between two creates, as between a create and its first edit.
Plans and views that need none of this are unchanged.
## Not covered
- Vikunja numbers tasks per project. Creates into projects the recording did not list cannot be numbered, so its task
creates stay withheld.
- Linkding renumbers bundles' `order` after a delete, which no rule models.
- Homebox item creates expand sent tag ids into tag objects and pin `parentId`; they stay withheld.
## Acceptance
- No write tool of the recorded captures goes from served to withheld.
- The writes the diagnosis named are served where their recordings show nothing else against them.
@@ -0,0 +1,11 @@
# Tasks
- [x] Narrow an update plan's required members to those every call sent.
- [x] Serve a member the value a call sends, else the next number, when calls showed both.
- [x] Learn and serve the value a `PUT` stores for a member it leaves out.
- [x] Hold an update's clock as the read after it spells it.
- [x] Ask recordings to space creates.
- [x] Add view, serving and unit tests.
- [x] Document the rules.
- [x] Measure the recorded captures.
- [x] Run static, formatting, documentation and test gates.
@@ -0,0 +1,95 @@
# Verification
Executed on macOS with Node v24.22.1 on this branch, on #7332 at `7d01a5d5` (main `5aa23c3d`), and on Ubuntu 24.04 VMs
with snap Chromium for the recording round.
## Diagnosis
Claude Code (`claude-sonnet-5-5`) built an environment of each of seven applications from the same prompt, once with
REA main `5aa23c3d` (skill 50) and once with #7332 at `7d01a5d5` (skill 53). Each bundle's write tasks ran with
`fireworks_ai/gpt-oss-120b` and `fireworks_ai/glm-5p3-flash`, which scored alike:
| Application | Write tasks | Main | #7332 |
| ----------- | ----------- | ------- | ------- |
| Miniflux | 3 | 3/3 | 3/3 |
| Grafana | 2 | 2/2 | 2/2 |
| Vikunja | 3 + 2 | 0/3 0/2 | 1/3 2/2 |
| Paperless | 3 | 1/3 | 2/3 |
| Linkding | 2 | 0/2 | 2/2 |
| Homebox | 2 | 2/2 | 0/2 |
| EspoCRM | 3 | 1/3 | 0/3 |
| Total | 20 | 9/20 | 12/20 |
Served writes were 14 and 25. Recorded again, Homebox and EspoCRM gave main 2/2 and 1/3 and #7332 2/2 and 3/3: the
first losses were the agents' runs, not REA. Built with both versions, none of the 18 recordings served fewer writes
with #7332 (44 and 53 in all).
Replaying #7332's recordings with failing checks and refusals logged gave the causes this branch takes: EspoCRM's
second contact update left out `addressCity` and was refused for lacking a required member (both recordings); Linkding's
second bundle create left `order` to the application, which the merge took as disagreeing with the first create's
echo; Vikunja's task update stamped `done_at` to the nanosecond where the read after it showed seconds. Over the
earlier captures, 14 withheld updates were `PUT`s whose application cleared a member they left out (Paperless `owner`,
Homebox `assetId`).
## Gates
- `npm run -s check:fast`, `format:check`, `scan:todos`, `docs:check`, `verify:test-discovery`, `mcp-catalog:check` and
`jscpd`: passed. `npx knip` reports `staticFileUrlPath`, `ObjcClassImportBindingCoverage` and
`AndroidManifestComponentInventory`, as main `5aa23c3d` does.
- `npx vitest run --project domain --project services`: 11,694 passed, 8 failed (`appleApplication.test.ts`), which fail
the same way on main `5aa23c3d`.
- `npm run -s test:focused --` the environment build, export and serve CLI boundaries, the analysis-view CLI and MCP
boundaries and the environment serve MCP boundary: 80 passed. `outputSchemaProfile`, `toolSchemaValidity`,
`mcpSchemaValidation`, `outputSchemaSemanticConstraints` and `analysisViewMcp`: 54 passed; the compact catalog stays
under its frame.
- The seven new view and serving tests fail on #7332 at `7d01a5d5` and pass here.
## Write plans over recorded captures
The 130 captures #7332 measured and the round's 18 recordings, built by #7332 at `7d01a5d5` and by this branch: 694
plans, 386 served and 398. No write goes from served to withheld and every view is unchanged. Newly served:
- Paperless correspondent, document type and tag updates in four recordings, whose `PUT` clears `owner`.
- EspoCRM's contact update and Linkding's bundle create in #7332's recordings of the round.
- Grafana's folder update and delete in a fresh recording whose second update sent fewer members.
- Homebox's item update in the published capture, whose `PUT` clears `assetId`.
Vikunja's task update reads back confirmed, but stays withheld: the task creates before it are not modelled (per-project
numbers), so their entities carry the creates' answer spellings into later reads.
## Review round
- Greptile's first review: a `sequence` with `sent` now accepts a value a later create sent, whichever create came
first; a default must agree with every update that left its member out, else the plan is withheld; and serving
stores an update's clocks as the update plan's read-back spells them (`observation.read`). Their four tests fail on
`6300ec80`. #7332's made-object clock check, merged in, has its test there.
- Gates as above: `domain` + `services` 11,699 passed, 8 failed (main's `appleApplication.test.ts`); focused boundaries
80 passed; the five MCP schema files 54 passed. The census stays at 694 plans and 398 served, with no plan or view
changing.
Greptile's review after main `21b19270` was merged in (`467ac78f`) raised three more, fixed:
- A member a call left out and the application stored a fixed value for (`body.defaults`) is no longer turned into a
`sequence` with `sent` by a later call that left it out and got the next number.
- Serving stores an update's clock at a coarser read's precision from the answer's own digits, so an answer to the
nanosecond read back to the microsecond keeps its microseconds.
- A `PUT` that names a `reference` field's entity by its member supplies that field: it is not checked against, or
given, the default a `PUT` that left the field out showed, whichever of the two calls came first.
Their tests fail on `467ac78f`, and the reference case with the clearing call first on `7030ca4d`. Gates pass but
`knip`, whose one finding main `21b19270` reports too; `domain` + `services` 12,046 passed; focused boundaries 82 passed; the five MCP schema files 55 passed. Over the 148 captures, main
serves 380 writes and this branch 392, with no plan or view changing from `467ac78f`.
Greptile's review of `b56f8924` raised two more on numbered members, fixed: a member a call sent empty (`""`) is no
longer filled in for a call that leaves it out when a `sequence` with `sent` numbers it; and a later create the
application numbered at or below a value an earlier create sent (9 sent and deleted, then 2) no longer turns the echo
into a numbered `sequence`, since serving counts past every value a call sent. Their tests fail on `b56f8924`; gates
pass but `knip` (main's finding), `domain` + `services` 12,048 passed, and the census is unchanged. Greptile's next
review found that check counting values sent in other collections, which withheld a create numbered within its own
collection; the check now counts them in the sequence's scope, and its test fails on `d4922b3d`.
## Not verified
- No recording round ran with this branch, so its write tasks are not measured.
- The skill's spacing of creates.
- Vikunja per-project task numbers, Linkding's renumbering after a delete and Homebox item creates, which stay withheld.
+4
View File
@@ -1741,6 +1741,10 @@ The Apple artifact CLI/MCP lane uses one `macos-15` job to avoid adding demand
for scarce macOS runner capacity. It runs the artifact-format and metadata
verifiers, followed by one consolidated focused invocation containing all
existing native and process-boundary regression paths.
The real native provenance verifier checks CLI/MCP agreement on complete
per-slice encryption absence. The encryption CLI regression builds a universal
image, marks its owned arm64 command with `cryptid=1`, and compares the reported
range with `otool`; this verifies declared metadata, not actual FairPlay data.
The Linux source-test shards download a shared runtime and generated test
metadata snapshot produced by the build job with `npm run test:prepare`. The
+42 -11
View File
@@ -740,7 +740,9 @@ one-element array member) that a later item read of a type first shows (served
only with the path values its recorded call sent, unless it creates into the
type's own collection, whose path has no parameters, and a `path` rule stores
each of its path values), and otherwise `none` with reason `no-state-target`. A plan is built from the tool's first successful call: its
request body must be a usable JSON object (as response bodies are usable), and
request body must be a usable JSON object (as response bodies are usable), or
absent for an item action (a scenario request is absent when Playwright exposed
no body and it names no `Content-Type`, which the bodies browsers send carry), and
the written entity must show in that call's response or in a later read before
the next write. Each field of a created entity gets the first rule that
explains it: generated `identity` (a sequence, UUID or token, by the recorded
@@ -780,15 +782,19 @@ with the entity leaves out a field only later reads showed, which no initial
entity holds, when the entity does not hold it, as Vikunja's reads of tasks it
did not create show no `subscription`. A `reference` is an object
holding fields of the
entity a sent member names, as the state holds it (a parent beside `parentId`):
entity a sent member names, as the state holds it (a parent beside `parentId`),
or a list of them, one per id a sent list names, in its order (`many`, tags
beside `tagIds`):
the member is accepted with any value, serving looks the entity up and refuses a
call naming one the environment does not hold or holds in more than one
collection, and the field is absent, from the entity and the create's answer,
when the call does not send the member. An id naming entities of one type in
several collections teaches no reference. Each later create widens its plan
when, explained on its own, it agrees with it: both calls show the same fields
but for a reference whose member a call without it did not send (one an earlier
create added stays), every field both rule they rule alike (an echo one showed
but for a reference whose member a call without it did not send, or sent as
`null` or `[]` and stored so (one an earlier create added stays, and a call may
send the member empty as that create did; one leaving it out stores no such field
when the create the plan observes showed none), every field both rule they rule alike (an echo one showed
sorted stays sorted, unless a call stored its strings out of order as sent),
and a field neither rules holds one value, unless the later call's rule for it
spells the earlier calls' value too: a `composed` or `template` string, or a
@@ -798,7 +804,14 @@ clocks and own members, and a count only the later call shows starts at the
first object's value; a later create's made object must hold clocks its own
request stamped, in the rule's format. Members either call sent are accepted, only those every call sent are
required, and an echoed member a call left out takes the value that call stored
(`body.defaults`). A member a create sent empty (`""`, `null`, `false`, `[]` or
(`body.defaults`). A member one call sent and stored as sent, and another left
out and the application numbered one past the others, takes the value a call
sends, else that number (a `sequence` with `sent`), whichever call came first,
unless a call that left it out stored a fixed value (its `body.defaults`) or
the number a later call was given is not above every value an earlier create
sent in the sequence's scope (the type, or the later call's collection); an empty value a call sent is not applied to a call that leaves it out;
a later call that sends it must have stored what it sent, and the next number
follows the largest value any call took, a deleted entity's included. A member a create sent empty (`""`, `null`, `false`, `[]` or
`{}`), as a form sends a field left as it was, is not required: a call that
leaves it out is applied as sending that value (`body.blank`), unless a call
that left it out showed a default, which it then keeps. A call may send it as any
@@ -861,13 +874,28 @@ is unchanged) must be a clock, a nested clock, an increment, a slug, a fresh `un
value, a `stamp` of the acting user, a value `derived` from the one member it
stores nowhere, a `reference` to the entity a sent member names (kept by a call that names the entity the field already repeats), a `composed` string whose parts read members it stored as sent
or fields it left as they were and spell the value before the call too, or for
an item action a `set` of a scalar to the value it showed, or the plan is
an item action a `set` of a scalar to the value it showed (a `toggle` once a
later call flips back the boolean an earlier one set, when no call found it set
already), or, for a `PUT`, a
member it left out that the application cleared to an empty value, which calls
that leave it out then store (`body.defaults`) as long as every call that left
it out stored that value (a call naming a `reference` field's entity by its
member does not leave the field out), or the plan is
`unexplained-change`; an increment, slug, stamp, derived
value or set must hold for every call of the tool, and an action is served
only once a call found what it sets already set. A write is served only
value, set or toggle must hold for every call of the tool, and an action is
served only once a call found what it sets already set, or flipped back what it
toggles. A write is served only
with the query its first call sent, which serving applies to a call that
leaves a parameter out and the catalog advertises as required. Every member the first call sent is required, except by `PATCH` and
`POST` updates and members a create sent empty.
leaves a parameter out and the catalog advertises as required. Every member every call sent is required, except by `PATCH` and
`POST` updates and members a create sent empty. The state holds an update's
clock as the first read of the entity after it spells the same instant at a
coarser precision in the same zone and separator: replay and serving store every
update's clocks in the spelling of the read the plan keeps as
`observation.read`, so a later update whose read spells them otherwise
contradicts the plan, while answers keep the update's own; a create's clocks
stay as answered. A `PUT` default must agree with what every call that left the
member out stored, as its answer shows it or, where the answer leaves it out,
the read after the call.
A read on a line `query` answers is checked against the query evaluated over
the replayed state. It covers the collection only when every parameter sorts or
@@ -900,7 +928,10 @@ contradicted it; otherwise its `reason` is the first of its build reason
(`no-call`, `body-unusable`, `no-observation`, `unexplained-change`),
`contradicted` (with `contradicted_by`), `side-effect`, `unconfirmed` and `incomplete-entities` (an update or delete that answers with the entity, when no initial entity holds every field its recorded answer shows but those only later reads showed, counting for an update the fields its rules write, so it could answer only for entities the environment created). An otherwise served plan whose learned rules generate clock values is withheld with `unknown-clock` when no global clock start is established; its learned rules and confirming reads remain retained.
`checked_record_ordinals` lists the confirming reads, `insert` places created
entities (sorted by a field the collection's recorded order shows, else last),
entities (sorted by a field the collection's recorded order shows, else last;
a create a whole listing in collection order placed last is placed again by a
field a later such listing shows it sorted by and no earlier one ruled out, the
type's id field last),
and `write_clock` is the end of the last timed record, to the next second, when
all request completions are known; otherwise it is null. An
update that writes the field its type's creates are sorted by moves its entity
+2
View File
@@ -184,6 +184,7 @@
"typecheck": "node scripts/run-typescript.mjs --noEmit",
"verify:test-temp-hygiene": "npm run test:prepare && node scripts/verify-test-temp-hygiene.mjs",
"verify:hopper": "npm run verify:fixtures && npm run build:cached && node scripts/verify-real-hopper.mjs",
"verify:hopper:callers": "npm run build:cached && node scripts/verify-real-hopper-callers.mjs",
"verify:hopper:deadlines": "npm run verify:fixtures && npm run build:cached && node scripts/verify-real-hopper-deadlines.mjs",
"verify:hopper:fat": "npm run build:cached && node scripts/verify-real-hopper-fat.mjs",
"verify:hopper:fat64": "npm run build:cached && node scripts/verify-real-hopper-fat.mjs --fat64",
@@ -309,6 +310,7 @@
"verify:agentenv:environment": "npm run build:cached && node scripts/verify-agentenv-environment.mjs",
"verify:web:http3": "npm run build:cached && node scripts/verify-http3-network-captures.mjs",
"verify:binary:layout": "npm run build:cached && node scripts/verify-binary-layout.mjs",
"verify:bun:standalone": "npm run build:cached && node scripts/verify-bun-standalone.mjs",
"verify:go:binary": "npm run build:cached && node scripts/verify-go-binary.mjs",
"verify:il2cpp:metadata": "npm run build:cached && node scripts/verify-il2cpp-metadata.mjs",
"verify:wasm:package": "npm run build:cached && node scripts/verify-packaged-wasm-artifact.mjs",
+1
View File
@@ -107,6 +107,7 @@ export const SCOPES = {
"tests/boundary/providers/native/nativeSignatureTargetBinding.test.ts",
"tests/boundary/cli/nativeSignatureCli.test.ts",
"tests/boundary/cli/nativeMachoArm64_32Cli.test.ts",
"tests/boundary/cli/nativeMachoEncryptionCli.test.ts",
"tests/boundary/mcp/nativeSignatureTargetBindingMcp.test.ts",
"src/process/capture/FilesystemSnapshot.test.ts",
"tests/process-global/filesystemSnapshotFallbackCancellation.test.ts",
@@ -0,0 +1,13 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<DebugType>none</DebugType>
</PropertyGroup>
<ItemGroup>
<Reference Include="Microsoft.NET.HostModel">
<HintPath>$(MSBuildSDKsPath)/../Microsoft.NET.HostModel.dll</HintPath>
</Reference>
</ItemGroup>
</Project>
@@ -0,0 +1,2 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration><packageSources><clear /></packageSources></configuration>
+92
View File
@@ -0,0 +1,92 @@
using System.Runtime.InteropServices;
using System.Buffers.Binary;
using System.IO.Compression;
using System.Security.Cryptography;
using System.Text.Json;
using Microsoft.NET.HostModel.AppHost;
using Microsoft.NET.HostModel.Bundle;
// Trusted fixture generator only. Generated hosts are inspected, never executed.
if (args.Length is < 2 or > 3)
throw new ArgumentException("Expected apphost template, absent output directory, and optional windows/linux/osx target");
var targetOS = (args.ElementAtOrDefault(2) ?? "windows") switch
{
"windows" => OSPlatform.Windows,
"linux" => OSPlatform.Linux,
"osx" => OSPlatform.OSX,
_ => throw new ArgumentException("Target OS must be windows, linux or osx"),
};
var hostName = targetOS == OSPlatform.Windows ? "Probe.exe" : "Probe";
var template = Path.GetFullPath(args[0]);
var output = Path.GetFullPath(args[1]);
if (Directory.Exists(output)) throw new IOException("Output directory must be absent");
Directory.CreateDirectory(output);
var assembly = typeof(Program).Assembly.Location;
var resource = Path.Combine(output, "resource.txt");
File.WriteAllText(resource, string.Concat(Enumerable.Repeat("owned compressed fixture content\n", 4096)));
foreach (var (version, compress) in new[] { (new Version(3, 1), false), (new Version(5, 0), false), (new Version(8, 0), false), (new Version(8, 0), true) })
{
var label = $"net{version.Major}-{(compress ? "compressed" : "plain")}";
var directory = Path.Combine(output, label);
Directory.CreateDirectory(directory);
var host = Path.Combine(directory, targetOS == OSPlatform.Windows ? "template.exe" : "template");
HostWriter.CreateAppHost(template, host, "BundleFixture.dll");
var bundler = new Bundler(hostName, directory,
BundleOptions.BundleOtherFiles | (compress ? BundleOptions.EnableCompression : BundleOptions.None),
targetOS, Architecture.X64, version, appAssemblyName: "BundleFixture");
var inputs = new Dictionary<string, string>
{
["BundleFixture.dll"] = assembly,
["BundleFixture.deps.json"] = Path.ChangeExtension(assembly, ".deps.json"),
["BundleFixture.runtimeconfig.json"] = Path.ChangeExtension(assembly, ".runtimeconfig.json"),
["data/resource.txt"] = resource,
};
var generated = bundler.GenerateBundle(new[] {
new FileSpec(host, hostName),
new FileSpec(assembly, "BundleFixture.dll"),
new FileSpec(Path.ChangeExtension(assembly, ".deps.json"), "BundleFixture.deps.json"),
new FileSpec(Path.ChangeExtension(assembly, ".runtimeconfig.json"), "BundleFixture.runtimeconfig.json"),
new FileSpec(resource, "data/resource.txt"),
});
File.WriteAllText(Path.Combine(directory, "oracle.json"), JsonSerializer.Serialize(Observe(generated, inputs), new JsonSerializerOptions { WriteIndented = true }));
Console.WriteLine(generated);
}
// Independent positive-fixture oracle: .NET's BinaryReader owns its string/number framing.
static object Observe(string path, Dictionary<string, string> inputs)
{
var bytes = File.ReadAllBytes(path);
var signature = Convert.FromHexString("8b1202b96a612038727b930214d7a03213f5b9e6efae3318ee3b2dce24b36aae");
var marker = bytes.AsSpan().IndexOf(signature);
if (marker < 8) throw new InvalidDataException("Generated host has no bundle marker");
var header = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(marker - 8, 8));
using var stream = new MemoryStream(bytes);
stream.Position = header;
using var reader = new BinaryReader(stream);
var major = reader.ReadUInt32();
var minor = reader.ReadUInt32();
var count = reader.ReadInt32();
var bundleId = reader.ReadString();
if (major >= 2) stream.Position += 40;
var rows = new List<object>();
var found = new HashSet<string>();
for (var index = 0; index < count; index++)
{
var offset = reader.ReadInt64();
var size = reader.ReadInt64();
var compressed = major == 6 ? reader.ReadInt64() : 0;
var kind = reader.ReadByte();
var name = reader.ReadString();
if (!inputs.TryGetValue(name, out var input) || !found.Add(name))
throw new InvalidDataException("Unexpected generated member: " + name);
using var stored = new MemoryStream(bytes, checked((int)offset), checked((int)(compressed == 0 ? size : compressed)));
using var expanded = compressed == 0 ? null : new DeflateStream(stored, CompressionMode.Decompress);
var digest = Convert.ToHexString(SHA256.HashData((Stream)expanded ?? stored)).ToLowerInvariant();
var expected = File.ReadAllBytes(input);
if (expected.LongLength != size || digest != Convert.ToHexString(SHA256.HashData(expected)).ToLowerInvariant())
throw new InvalidDataException("Generated member bytes differ: " + name);
rows.Add(new { name, offset, size, compressed_size = compressed, file_type = kind, sha256 = digest });
}
if (!found.SetEquals(inputs.Keys)) throw new InvalidDataException("Generated bundle omitted an input");
return new { source = "Microsoft.NET.HostModel + independent BinaryReader/DeflateStream", host = path, major, minor, header_offset = header, bundle_id = bundleId, sha256 = Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(), files = rows };
}
+5
View File
@@ -0,0 +1,5 @@
#include <stdio.h>
__attribute__((noinline)) int target_fn(int x) { return x * 5; }
__attribute__((noinline)) int call_through(int (*fp)(int), int x) { return fp(x); }
__attribute__((noinline)) int call_direct(int x) { return target_fn(x); }
int main(void) { printf("%d %d\n", call_through(target_fn, 3), call_direct(4)); return 0; }
+5
View File
@@ -38,6 +38,11 @@ export const parseDevelopmentTestRequest = (mode, arguments_) => {
// Test-only outputs are declared beside the test planner rather than becoming
// runtime or typecheck prerequisites. Add a consumer here when it reads one.
const artifactConsumers = {
"artifacts:skills": [
"tests/boundary/cli/guidanceCli.test.ts",
"tests/boundary/guidanceSearch.test.ts",
"tests/boundary/mcp/registerGuidanceTools.test.ts",
],
"artifacts:mcp-catalog": [
"tests/boundary/mcp/deferredRuntime.test.ts",
"tests/boundary/mcp/contractPresentation.test.ts",
+18
View File
@@ -0,0 +1,18 @@
/** One static import or re-export specifier and its source line. */
export interface ImportSpecifierFact {
readonly specifier: string;
readonly line: number;
}
/** Static string specifier on one import or re-export node, when it has one. */
export function importSpecifier(node: {
readonly type: string;
}): string | undefined;
/**
* Static import and re-export specifiers in source order.
* Comments and string literals are not imports.
*/
export function readImportSpecifiers(
source: string,
): readonly ImportSpecifierFact[];
+43
View File
@@ -0,0 +1,43 @@
import { parse } from "@babel/parser";
import { traverseFast } from "@babel/types";
const literalValue = (node) => {
if (node?.type === "StringLiteral") return node.value;
if (node?.type === "TemplateLiteral" && node.expressions.length === 0)
return node.quasis[0]?.value.cooked ?? undefined;
return undefined;
};
/** Static string specifier on one import or re-export node, when it has one. */
export const importSpecifier = (node) => {
switch (node.type) {
case "ImportDeclaration":
case "ExportNamedDeclaration":
case "ExportAllDeclaration":
case "ImportExpression":
case "TSImportType":
return literalValue(node.source);
case "TSImportEqualsDeclaration":
return node.moduleReference.type === "TSExternalModuleReference"
? literalValue(node.moduleReference.expression)
: undefined;
default:
return undefined;
}
};
/**
* Static import and re-export specifiers in source order.
* Comments and string literals are not imports. Non-string dynamic imports
* are omitted. Callers decide which specifiers are in-repo edges.
*/
export const readImportSpecifiers = (source) => {
const specifiers = [];
const tree = parse(source, { sourceType: "module", plugins: ["typescript"] });
traverseFast(tree, (node) => {
const specifier = importSpecifier(node);
if (specifier === undefined) return;
specifiers.push({ specifier, line: node.loc?.start.line ?? 1 });
});
return specifiers;
};
+6 -42
View File
@@ -1,8 +1,8 @@
import { parse } from "@babel/parser";
import { traverseFast } from "@babel/types";
import { isAbsolute, relative, resolve } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { readImportSpecifiers } from "./import-specifiers.mjs";
const PURE_LAYERS = new Set(["domain", "contracts"]);
const MIGRATED_PROVIDER_ROOTS = new Set([
"sqlite",
@@ -27,30 +27,6 @@ const MIGRATED_PROVIDER_ROOTS = new Set([
const repositoryPath = (root, path) =>
relative(root, path).replaceAll("\\", "/");
const literalValue = (node) => {
if (node?.type === "StringLiteral") return node.value;
if (node?.type === "TemplateLiteral" && node.expressions.length === 0)
return node.quasis[0]?.value.cooked ?? undefined;
return undefined;
};
const importSpecifier = (node) => {
switch (node.type) {
case "ImportDeclaration":
case "ExportNamedDeclaration":
case "ExportAllDeclaration":
case "ImportExpression":
case "TSImportType":
return literalValue(node.source);
case "TSImportEqualsDeclaration":
return node.moduleReference.type === "TSExternalModuleReference"
? literalValue(node.moduleReference.expression)
: undefined;
default:
return undefined;
}
};
const sourceTarget = (file, specifier) => {
if (specifier.startsWith("file:")) return fileURLToPath(specifier);
if (isAbsolute(specifier)) return resolve(specifier);
@@ -115,26 +91,14 @@ const failedBoundary = (file, target) => {
export const inspectModuleBoundaries = (file, source, root) => {
const absoluteFile = resolve(root, file);
const importer = repositoryPath(root, absoluteFile);
const tree = parse(source, {
sourceType: "module",
plugins: ["typescript"],
});
const violations = [];
traverseFast(tree, (node) => {
const specifier = importSpecifier(node);
if (specifier === undefined) return;
for (const { specifier, line } of readImportSpecifiers(source)) {
const resolved = sourceTarget(absoluteFile, specifier);
if (resolved === undefined) return;
if (resolved === undefined) continue;
const target = repositoryPath(root, resolved);
const boundary = failedBoundary(importer, target);
if (boundary !== undefined)
violations.push({
file: importer,
line: node.loc?.start.line ?? 1,
specifier,
target,
boundary,
});
});
violations.push({ file: importer, line, specifier, target, boundary });
}
return violations;
};
+5 -5
View File
@@ -128,7 +128,7 @@ if (!(await compiledRuntimeExists(runtimeFiles))) {
);
const configurationError = cliConfigurationError(process.env);
let dispatchFailure;
let closeCliJavaScriptResources;
let closeCliResources;
// Incur can exit from the callback below before this function returns.
// That callback owns the status, including a closed-pipe exit of 141.
let exiting = false;
@@ -150,7 +150,7 @@ if (!(await compiledRuntimeExists(runtimeFiles))) {
writeStdout(versionOutput);
} else {
const cliRuntime = await import("../dist/cli-runtime/cli.js");
closeCliJavaScriptResources = cliRuntime.closeCliJavaScriptResources;
closeCliResources = cliRuntime.closeCliResources;
const { createCli } = cliRuntime;
const { createCliResultOutput } =
await import("../dist/cli-runtime/cliResultOutput.js");
@@ -176,10 +176,10 @@ if (!(await compiledRuntimeExists(runtimeFiles))) {
void (async () => {
let status = code;
try {
await closeCliJavaScriptResources();
await closeCliResources();
} catch (cleanup) {
process.stderr.write(
`Failed to close the JavaScript analysis worker: ${cleanupFailureText(cleanup)}\n`,
`Failed to close CLI resources: ${cleanupFailureText(cleanup)}\n`,
);
if (status === 0) status = 1;
}
@@ -225,7 +225,7 @@ if (!(await compiledRuntimeExists(runtimeFiles))) {
// Serve and output rendering can throw after a worker is parked. Stop it
// on that path too, and keep the dispatch failure when both fail.
try {
await closeCliJavaScriptResources?.();
await closeCliResources?.();
} catch (cleanup) {
// The exit callback reports this same shared failure and then exits.
if (!exiting) {
+4 -2
View File
@@ -61,8 +61,10 @@ try {
process.execPath,
resolve("scripts/run-turbo.mjs"),
"run",
...(plan.needsBuild ? ["artifacts:skills"] : []),
...plan.artifactTasks,
...new Set([
...(plan.needsBuild ? ["artifacts:skills"] : []),
...plan.artifactTasks,
]),
]);
if (prepared.code !== 0) process.exitCode = prepared.code;
}
+113
View File
@@ -0,0 +1,113 @@
#!/usr/bin/env node
import assert from "node:assert/strict";
import { readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
import { parseEvidence } from "../dist/domain/evidence.js";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
import { createVerifierCommands } from "./lib/verify-commands.mjs";
const root = await PrivateRuntimeRoot.create({ prefix: "rea-bun-verifier-" });
const commands = await createVerifierCommands();
const run = (command, args, options = {}) =>
commands.exec(command, args, {
timeout: 120_000,
maxBuffer: 16 * 1024 * 1024,
...options,
});
// The root holds the compiled target and child temporary files. Release it
// only after every owned command family has settled.
let commandsClosed = false;
await withVerifierCleanup(async () => {
const source = join(root.path, "hello.ts");
await writeFile(source, 'console.log("REA_BUN_VERIFICATION");\n');
const docker = process.argv.includes("--docker");
const binary = join(root.path, "hello");
const compiler = docker
? "oven/bun:1.4.0"
: (await run("bun", ["--version"])).stdout.trim();
if (docker) {
await run("docker", [
"run",
"--rm",
"--network=none",
"-v",
`${root.path}:/work`,
"-w",
"/work",
"oven/bun:1.4.0",
"build",
"--compile",
"hello.ts",
"--outfile",
"hello",
]);
} else {
await run("bun", ["build", "--compile", source, "--outfile", binary]);
}
const cli = async (command) =>
JSON.parse(
(
await run(
process.execPath,
["scripts/rea.mjs", command, binary, "--json"],
{
env: {
...process.env,
TMPDIR: root.path,
TMP: root.path,
TEMP: root.path,
},
},
)
).stdout,
);
const inspected = parseEvidence(
await cli("inspect-artifact"),
).normalized_result;
const graph = inspected.substeps.find(
(step) => step.operation === "inventory_artifact",
)?.evidence.normalized_result;
assert.equal(graph.manifest.root_format, "bun-standalone");
const module = graph.occurrences.find(
(row) =>
row.format_metadata?.kind === "bun-standalone-entry" &&
row.format_metadata.region === "contents" &&
row.format_metadata.entry_point,
);
assert.ok(module);
const extracted = parseEvidence(
await cli("extract-artifact"),
).normalized_result;
assert.equal(extracted.containment_verified, true);
assert.match(
await readFile(join(extracted.output_root, module.logical_path), "utf8"),
/REA_BUN_VERIFICATION/u,
);
console.log(
JSON.stringify(
{
compiler,
host: `${process.platform}-${process.arch}`,
docker,
root_format: graph.manifest.root_format,
entry_point: module.logical_path,
sha256: graph.nodes.find(
(node) => node.artifact_id === module.artifact_id,
)?.sha256,
extraction_verified: true,
target_executed: false,
},
null,
2,
),
);
}, [
async () => {
await commands.close();
commandsClosed = true;
},
async () => {
if (commandsClosed) await root.close();
},
]);
+309
View File
@@ -0,0 +1,309 @@
// Run from the repository root after `npm run build:cached` on macOS arm64,
// with Hopper and the Xcode command-line tools installed. No target is executed.
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { constants } from "node:fs";
import { access, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { createRequire } from "node:module";
import { join } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { promisify } from "node:util";
import { artifactMcpCall } from "./lib/artifact-e2e.mjs";
import { snapshotHopperRuntime } from "./lib/real-hopper-cleanup.mjs";
import { HOPPER_TARGET_LEASE_DIRECTORY } from "../dist/process/HopperLease.js";
assert.equal(process.platform, "darwin", "requires macOS");
assert.equal(
process.arch,
"arm64",
"requires macOS arm64 (fixture tail branch is ARM64)",
);
const root = process.cwd();
const require = createRequire(join(root, "package.json"));
const { Client } = await import(
pathToFileURL(require.resolve("@modelcontextprotocol/client"))
);
const { StdioClientTransport } = await import(
pathToFileURL(require.resolve("@modelcontextprotocol/client/stdio"))
);
const { parseConfig } = await import(
pathToFileURL(join(root, "dist/config/parseConfig.js"))
);
const config = parseConfig(process.env);
if (!config.ok) throw config.error;
await access(config.value.hopperLauncherPath, constants.X_OK);
const run = promisify(execFile);
const clang = (await run("xcrun", ["--find", "clang"])).stdout.trim();
const sdk = (
await run("xcrun", ["--sdk", "macosx", "--show-sdk-path"])
).stdout.trim();
const directory = await mkdtemp(join("/tmp", "rea-hopper-callers-verifier-"));
const fixture = join(directory, "tail-call");
const large = join(directory, "long-scan");
const source = fileURLToPath(
new URL("./fixtures/hopper-tail-call.c", import.meta.url),
);
const digest = async (path) =>
createHash("sha256")
.update(await readFile(path))
.digest("hex");
const sessionsBefore = await snapshotHopperRuntime(
"/tmp",
HOPPER_TARGET_LEASE_DIRECTORY,
);
const ownedProcessIds = new Set();
const client = new Client({ name: "hopper-callers-verifier", version: "1" });
const transport = new StdioClientTransport({
command: process.execPath,
args: ["dist/main.js"],
cwd: root,
env: {
...process.env,
REA_ANALYSIS_PROVIDER: "hopper",
REA_LOG_LEVEL: "silent",
REA_HOPPER_CALLER_SCAN_BUDGET_MS: "5000",
},
stderr: "pipe",
});
let stderr = "";
transport.stderr?.on("data", (chunk) => {
stderr += chunk.toString();
});
const call = async (name, args, options = {}) => {
process.stderr.write(`calling ${name}\n`);
const result = await artifactMcpCall(
client,
{ name, arguments: args },
{ timeout: 120000, ...options },
);
assert.notEqual(result.isError, true, `${name}: ${JSON.stringify(result)}`);
return (
result.structuredContent ??
JSON.parse(result.content.find((x) => x.type === "text").text)
);
};
const address = (result) => {
const match = result.normalized_result.match(/^(0x[0-9a-f]+):/mu);
assert(match, "Hopper assembly has no instruction address");
return match[1];
};
const matchingRef = (ref, site, from, to) =>
ref.source_address === site &&
ref.source_procedure?.address === from &&
ref.target_procedure?.address === to &&
ref.call?.classification === "direct";
try {
await run(clang, [
"-isysroot",
sdk,
"-arch",
"arm64",
"-O1",
source,
"-o",
fixture,
]);
const lines = ["#include <stdio.h>"];
for (let i = 0; i < 4000; i++) {
const id = String(i).padStart(4, "0");
lines.push(
`__attribute__((noinline, used)) int rea_progress_${id}(void) { puts("REA_PROGRESS_${id}"); return ${i}; }`,
);
}
lines.push("int main(void) { return rea_progress_0000(); }");
const largeSource = join(directory, "long-scan.c");
await writeFile(largeSource, lines.join("\n"));
await run(clang, [
"-isysroot",
sdk,
"-arch",
"arm64",
"-O1",
largeSource,
"-o",
large,
]);
const before = await digest(fixture),
beforeLarge = await digest(large);
await client.connect(transport);
if (transport.pid === null)
throw new Error("MCP process identity unavailable");
ownedProcessIds.add(transport.pid);
let notifications = 0;
const original = transport.onmessage;
transport.onmessage = (message) => {
if (message.method === "notifications/progress") notifications++;
original?.(message);
};
await call("open_binary", { path: fixture });
const assembly = await call("procedure_assembly", {
procedure: "_call_direct",
});
const branch = assembly.normalized_result.match(
/^(0x[0-9a-f]+):\s+b\s+_target_fn\s*$/mu,
);
assert(
branch,
`compiler did not produce expected tail branch: ${assembly.normalized_result}`,
);
const site = branch[1],
from = address(assembly);
const to = address(
await call("procedure_assembly", { procedure: "_target_fn" }),
);
assert(
(
await call("procedure_callees", { procedure: from })
).normalized_result.includes(to),
);
const callers = await call("procedure_callers", {
procedure: to,
detail: "complete",
});
assert(callers.normalized_result.includes(from), "tail caller missing");
assert.equal(callers.relationship_coverage.status, "unknown");
assert(
callers.raw_result.forward_observations.callee_inventory_callers.some(
(x) => x.address === from,
),
"forward caller provenance missing",
);
const refs = (
await call("procedure_references", { procedure: to, direction: "incoming" })
).normalized_result.references;
assert(
refs.some((x) => matchingRef(x, site, from, to)),
"exact incoming tail reference missing",
);
const dossier = (
await call("analyze_function", { procedure: to, detail: "complete" })
).normalized_result;
assert(
dossier.callers.some((x) => x.address === from),
"dossier caller missing",
);
assert(
dossier.incoming_references.some((x) => matchingRef(x, site, from, to)),
"dossier reference missing",
);
const graph = (
await call("get_call_graph", {
address: to,
direction: "backward",
detail: "complete",
})
).normalized_result.graph;
assert(
Object.values(graph)
.flat()
.some((x) => x.address === to && x.calls.includes(from)),
"backward graph edge missing",
);
const path = (
await call("trace_call_path", {
start: to,
goal: from,
direction: "backward",
})
).normalized_result;
assert.equal(path.goal_status, "reached");
assert(
path.edges.some(
(x) => x.source_address === from && x.target_address === to,
),
);
await call("close_binary", {});
assert.equal(await digest(fixture), before);
console.log(
JSON.stringify({
event: "tail_call_verified",
from,
site,
to,
sha256: before,
}),
);
await call("open_binary", { path: large });
await call("procedure_assembly", { procedure: "_rea_progress_0000" });
const query = {
address: "_rea_progress_0000",
direction: "backward",
detail: "complete",
};
const count = notifications;
await call("get_call_graph", query);
assert.equal(notifications, count, "unsolicited progress without token");
const progress = [];
await call("get_call_graph", query, { onprogress: (x) => progress.push(x) });
assert(
progress.every((x, i) => i === 0 || x.progress > progress[i - 1].progress),
"progress not monotonic",
);
assert(
progress.some((x) =>
x.message?.includes("hopper_caller_reconciliation: Examined"),
),
"missing examined-count progress",
);
const controller = new AbortController();
let partial = false;
let resolved = false;
try {
await artifactMcpCall(
client,
{ name: "get_call_graph", arguments: query },
{
timeout: 120000,
signal: controller.signal,
onprogress: (x) => {
const match = x.message?.match(
/hopper_caller_reconciliation: Examined (\d+) of (\d+) procedures/u,
);
if (match && Number(match[1]) < Number(match[2])) {
partial = true;
controller.abort("cancel during real scan");
}
},
},
);
resolved = true;
} catch (error) {
if (!controller.signal.aborted) throw error;
}
assert(
partial && !resolved,
"did not observe cancellation after partial scan progress; cancellation coverage unavailable on this host",
);
await client.ping();
const retry = await call("get_call_graph", query);
assert(retry.normalized_result.graph, "retry did not return graph");
await call("close_binary", {});
assert.equal(await digest(large), beforeLarge);
console.log(
JSON.stringify({
event: "progress_cancel_retry_verified",
progress,
sha256: beforeLarge,
}),
);
} finally {
await client.close();
await transport.close();
await new Promise((resolve) => setTimeout(resolve, 500));
const leaked = [
...(await snapshotHopperRuntime(
"/tmp",
HOPPER_TARGET_LEASE_DIRECTORY,
ownedProcessIds,
)),
].filter((path) => !sessionsBefore.has(path));
if (leaked.length === 0)
await rm(directory, { recursive: true, force: true });
if (stderr) process.stderr.write(stderr);
assert.deepEqual(
leaked,
[],
`Hopper resources remain: ${leaked.join(", ")}; fixture retained at ${directory}`,
);
}
+72
View File
@@ -10,10 +10,14 @@ import { ArtifactResourceScope } from "../../../dist/artifacts/ArtifactResourceS
import { inspectAppleDispatchMetadata } from "../../../dist/native/AppleDispatchMetadata.js";
import {
artifactCliEvidence,
artifactMcpCall,
artifactMcpResult,
withArtifactMcp,
} from "../../lib/artifact-e2e.mjs";
import { nativeDispatchMetadataResultSchema } from "../../../dist/domain/native/objcSwiftMetadata.js";
import { analysisErrorProjectionSchema } from "../../../dist/contracts/errorSchemas.js";
import { cliOperationFailurePayload } from "../../../dist/cliLogging.js";
import { requireMcpToolError } from "../../lib/mcp-verifier-results.mjs";
if (process.platform !== "darwin")
throw new Error(
@@ -344,6 +348,74 @@ try {
commands,
target_sha256: cliResult.target_sha256,
});
for (const { section, size, reason } of [
{
section: "__objc_classlist",
size: 12,
reason: "Misaligned Objective-C class-list size",
},
{
section: "__swift5_types",
size: 6,
reason: "Misaligned Swift type section",
},
]) {
const bytes = Buffer.alloc(4096);
bytes.writeUInt32LE(0xfeedfacf);
bytes.writeUInt32LE(0x0100000c, 4);
bytes.writeUInt32LE(2, 12);
bytes.writeUInt32LE(1, 16);
bytes.writeUInt32LE(152, 20);
bytes.writeUInt32LE(0x19, 32); // LC_SEGMENT_64
bytes.writeUInt32LE(152, 36);
bytes.write("__DATA", 40);
bytes.writeBigUInt64LE(0x100000000n, 56);
bytes.writeBigUInt64LE(4096n, 64);
bytes.writeBigUInt64LE(4096n, 80);
bytes.writeUInt32LE(7, 88);
bytes.writeUInt32LE(5, 92);
bytes.writeUInt32LE(1, 96);
bytes.write(section, 104);
bytes.write("__DATA", 120);
bytes.writeBigUInt64LE(0x100000200n, 136);
bytes.writeBigUInt64LE(BigInt(size), 144);
bytes.writeUInt32LE(512, 152);
const path = join(root, `malformed-${section}.macho`);
await writeFile(path, bytes);
const target = await parseBinaryTarget(path);
assert.ok(
target.ok,
"Malformed section fixture must pass target admission",
);
let cliFailure;
try {
await artifactCliEvidence("inspect-native-dispatch-metadata", path);
assert.fail("Malformed metadata unexpectedly succeeded");
} catch (cause) {
assert.equal(cause.code, 1);
cliFailure = analysisErrorProjectionSchema.parse(
cliOperationFailurePayload(JSON.parse(cause.stdout)),
);
}
assert.equal(cliFailure.code, "unsupported_target");
assert.equal(cliFailure.category, "unsupported_target");
assert.deepEqual(cliFailure.details, {
operation: "inspect_native_dispatch_metadata",
path: target.value.path,
reason,
});
await withArtifactMcp(path, async (client) => {
const failure = requireMcpToolError(
await artifactMcpCall(client, {
name: "inspect_native_dispatch_metadata",
arguments: {},
}),
);
assert.equal(failure.code, cliFailure.code);
assert.deepEqual(failure.details, cliFailure.details);
});
results.push({ variant: `malformed-${section}`, reason });
}
for (const mode of LINK_MODES) {
const objcPath = join(root, `fixture-${mode.mode}`);
await exec("/usr/bin/xcrun", [
@@ -134,6 +134,15 @@ export async function verifyNativeProvenance(app, executable) {
]);
if (operation === "inspect_macho") {
const encryption = cli.normalized_result.encryption;
assert.equal(encryption.exhaustive, true);
assert.ok(encryption.total > 0);
assert.equal(encryption.items.length, encryption.total);
for (const slice of encryption.items) {
assert.equal(slice.status, "absent");
assert.deepEqual(slice.commands, []);
assert.equal(slice.range, null);
}
const delayed = cli.normalized_result.load_commands.items.find(
({ fields }) => fields.name === "@rpath/libdelay.dylib (offset 28)",
);
+3
View File
@@ -156,6 +156,9 @@ check(
files,
/`[^`\n]*\\0/,
[
// Bundle fingerprint: fixed packaged paths (no NUL/newline) and 64 hex
// digest characters per record, never guide contents. Preserve v1 locators.
"src/application/GuidanceAccess.ts",
"src/application/javascript/ElectronBoundaryAnalysis.ts",
"src/application/javascript/ElectronBoundaryGraphContext.ts",
"src/application/javascript/JavaScriptArtifactGraphContext.ts",
+135
View File
@@ -0,0 +1,135 @@
import {
createAnalysisExecution,
type AnalysisExecution,
} from "../application/AnalysisProvider.js";
import { AnalysisProtocolError } from "../domain/analysisErrorCore.js";
import { err, ok, type Result } from "../domain/result.js";
import type { AnalysisError } from "../domain/analysisErrorBase.js";
import { adbArguments, runAdb, type AdbRunLimits } from "./AdbCommand.js";
import type { AdbBinarySelection } from "./AdbConfiguration.js";
import {
parseAdbDevicesOutput,
parseGetpropOutput,
} from "./AdbDeviceOutput.js";
import type { AdbRequest } from "../domain/adb/adbDeviceAnalysis.js";
import type {
AdbClientResult,
AdbProviderIdentity,
} from "./AdbOperationResults.js";
const DEVICES_TIMEOUT_MS = 20_000;
const DEVICES_MAX_OUTPUT_BYTES = 1024 * 1024;
const SHELL_TIMEOUT_MS = 30_000;
const GETPROP_MAX_OUTPUT_BYTES = 2 * 1024 * 1024;
export interface AdbCommandContext {
readonly operation: AdbRequest["operation"];
readonly request: AdbRequest;
readonly selection: AdbBinarySelection;
readonly client: AdbClientResult;
readonly provider: AdbProviderIdentity;
readonly signal: AbortSignal | undefined;
readonly shell: (
arguments_: readonly string[],
limits: AdbRunLimits,
) => ReturnType<typeof runAdb>;
}
export const inspectAdbClientOperation = (
context: AdbCommandContext,
): Result<AnalysisExecution, AnalysisError> =>
ok(
createAnalysisExecution(
{ client: context.client, contacted_adb_server: false },
context.provider,
),
);
export const listAdbDevicesOperation = async (
context: AdbCommandContext,
): Promise<Result<AnalysisExecution, AnalysisError>> => {
const ran = await runAdb(
context.operation,
{ device: "adb-devices" },
context.selection,
["devices", "-l"],
{
timeoutMs: DEVICES_TIMEOUT_MS,
maxOutputBytes: DEVICES_MAX_OUTPUT_BYTES,
volatileOutput: true,
},
context.signal,
);
if (!ran.ok) return ran;
const { devices, unparsedLines, daemonStarted } = parseAdbDevicesOutput(
ran.value.stdout,
ran.value.stderr,
);
if (unparsedLines.total > 0)
return err(
new AnalysisProtocolError(
`adb devices reported an unrecognized line: ${unparsedLines.sample[0]!}`,
),
);
return ok(
createAnalysisExecution(
{
client: context.client,
devices,
may_have_started_adb_server: daemonStarted,
},
context.provider,
{
limitations: [
"Device kind is inferred from the emulator serial prefix and USB transport metadata, not from device attestation",
"Kind stays unknown for TCP transports unless the serial is an emulator port",
],
},
),
);
};
export const inspectAdbDeviceOperation = async (
context: AdbCommandContext,
): Promise<Result<AnalysisExecution, AnalysisError>> => {
const { request } = context;
if (request.operation !== "inspect_adb_device")
return err(
new AnalysisProtocolError(
`adb device inspection does not accept ${request.operation}`,
),
);
const serial = request.input.serial;
const ran = await runAdb(
context.operation,
{ device: serial },
context.selection,
adbArguments(serial, ["shell", "getprop"]),
{
timeoutMs: SHELL_TIMEOUT_MS,
maxOutputBytes: GETPROP_MAX_OUTPUT_BYTES,
volatileOutput: true,
},
context.signal,
);
if (!ran.ok) return ran;
const { properties, emulatorObserved } = parseGetpropOutput(ran.value.stdout);
return ok(
createAnalysisExecution(
{
client: context.client,
serial,
properties,
property_count: properties.length,
emulator_observed: emulatorObserved,
},
context.provider,
{
limitations: [
"Properties are a fixed whitelist of build identity facts, not the full getprop dump",
"emulator_observed is null when the build does not report ro.kernel.qemu",
],
},
),
);
};
+53
View File
@@ -0,0 +1,53 @@
import { createAnalysisExecution } from "../application/AnalysisProvider.js";
import type { AdbUnparsedLines } from "./AdbUnparsedLines.js";
import type {
AdbBinarySelection,
AdbClientIdentity,
} from "./AdbConfiguration.js";
export const clientIdentityResult = (
selection: AdbBinarySelection,
identity: AdbClientIdentity,
) => ({
binary_path: selection.binary,
path_source: selection.pathSource,
version: identity.version,
installed_path: identity.installedPath,
});
export type AdbClientResult = ReturnType<typeof clientIdentityResult>;
export type AdbProviderIdentity = {
id: string;
name: string;
version: string | null;
};
/**
* Build a parsed-listing execution whose `unparsed_lines`, coverage, and
* truncation limitation all derive from the parser's unrecognized-line facts,
* so the limitation can never describe a field the result does not carry.
*/
export const parsedListingExecution = (
result: Readonly<Record<string, unknown>>,
unparsed: AdbUnparsedLines,
provider: AdbProviderIdentity,
limitations: readonly string[] = [],
) =>
createAnalysisExecution(
{
...result,
unparsed_lines: unparsed.sample,
coverage: unparsed.total > 0 ? "partial" : "complete",
},
provider,
{
limitations: [
...limitations,
...(unparsed.total > unparsed.sample.length
? [
`unparsed_lines reports the first ${String(unparsed.sample.length)} of ${String(unparsed.total)} unrecognized lines`,
]
: []),
],
},
);
+42 -5
View File
@@ -38,6 +38,11 @@ import type { AdbRequest } from "../domain/adb/adbDeviceAnalysis.js";
const it = test.skipIf(process.platform === "win32");
const SCREEN_PNG = Buffer.from(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC",
"base64",
);
const VERSION_BLOCK =
"printf 'Android Debug Bridge version 1.0.41\\nVersion 34.0.5-stub\\nInstalled as %s\\n' \"$0\"";
@@ -1340,6 +1345,36 @@ exit 1`,
expect(closeAttempts).toBe(2);
});
it("verifies the device-side digest for a long device path", async () => {
const { binary, root } = await writeAdbStub(
() => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi
if [ "$1" = "-s" ] && [ "$4" = "test" ]; then exit 1; fi
if [ "$1" = "-s" ] && [ "$3" = "push" ]; then exit 0; fi
if [ "$1" = "-s" ] && [ "$4" = "sha256sum" ]; then printf '%s %s\\n' "8748b5d17d1ebcec7d46674fa6a8b92c0f95598a552f27c1c8a132eb58c04043" "$5"; exit 0; fi
exit 1`,
);
const localFile = join(root, "payload.bin");
await writeFile(localFile, "push-payload");
const digest = createHash("sha256").update("push-payload").digest("hex");
const pushed = await evidenceResult(providerOver(binary), {
operation: "push_adb_file",
input: {
serial: "emulator-5554",
local_path: localFile,
// Each component stays within the usual 255-byte limit; the whole
// path still makes the digest line exceed the former 1 KiB budget.
device_path: `/data/local/tmp/${Array.from({ length: 5 }, (_, index) =>
String(index).padEnd(220, "d"),
).join("/")}/payload.bin`,
overwrite: false,
},
});
expect(pushed).toMatchObject({
device_sha256: digest,
device_digest_source: "device_sha256sum",
});
});
it("reports a digest mismatch after push as a changed artifact", async () => {
const { binary, root } = await writeAdbStub(
() => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi
@@ -1453,19 +1488,20 @@ exit 1`,
it("captures the screen as a digested PNG with its dimensions", async () => {
const { binary, root } = await writeAdbStub(
() => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi
(root) => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi
if [ "$1" = "-s" ] && [ "$3" = "exec-out" ]; then
printf '\\211PNG\\r\\n\\032\\n\\0\\0\\0\\rIHDR\\0\\0\\0\\004\\0\\0\\0\\003'
cat "${join(root, "screen-fixture.png")}"
exit 0
fi
exit 1`,
);
await writeFile(join(root, "screen-fixture.png"), SCREEN_PNG);
const output = join(root, "captures");
const result = await evidenceResult(providerOver(binary), {
operation: "capture_adb_screen",
input: { serial: "emulator-5554", output_directory: output },
});
expect(result).toMatchObject({ width: 4, height: 3 });
expect(result).toMatchObject({ width: 1, height: 1 });
expect(
await readFile(join(output, "screen.png")).then(
(bytes) => bytes.subarray(1, 4).toString("latin1") === "PNG",
@@ -1476,13 +1512,14 @@ exit 1`,
it("stops a screen capture aborted after adb returns but before publication", async () => {
const { binary, root } = await writeAdbStub(
() => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi
(root) => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi
if [ "$1" = "-s" ] && [ "$3" = "exec-out" ]; then
printf '\\211PNG\\r\\n\\032\\n\\0\\0\\0\\rIHDR\\0\\0\\0\\004\\0\\0\\0\\003'
cat "${join(root, "screen-fixture.png")}"
exit 0
fi
exit 1`,
);
await writeFile(join(root, "screen-fixture.png"), SCREEN_PNG);
const output = join(root, "cancelled-capture");
const controller = new AbortController();
adbTestHooks.afterScreencap = () => controller.abort();
+60 -156
View File
@@ -66,6 +66,7 @@ import {
} from "../domain/analysisErrorCore.js";
import { AnalysisError } from "../domain/analysisErrorBase.js";
import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js";
import { readPngHeaderDimensions } from "../domain/pngHeader.js";
import { err, ok, type Result } from "../domain/result.js";
import type { JsonValue } from "../domain/jsonValue.js";
import { jsonObjectSchema } from "../domain/jsonValue.js";
@@ -111,8 +112,6 @@ import {
import { snapshotEnvironment } from "../process/snapshotEnvironment.js";
import {
displayDevicePath,
parseAdbDevicesOutput,
parseGetpropOutput,
parsePackageListOutput,
parsePmPathOutput,
} from "./AdbDeviceOutput.js";
@@ -132,7 +131,6 @@ import {
parseFeatureListOutput,
parsePackageChangeOutput,
parseProcessListOutput,
parsePngDimensions,
parseResolveActivityOutput,
parseServiceListOutput,
parseSettingsGetOutput,
@@ -141,7 +139,18 @@ import {
parseWmDensityOutput,
parseWmSizeOutput,
} from "./AdbShellOutput.js";
import type { AdbUnparsedLines } from "./AdbUnparsedLines.js";
import {
clientIdentityResult,
parsedListingExecution,
type AdbClientResult,
type AdbProviderIdentity,
} from "./AdbOperationResults.js";
import {
inspectAdbClientOperation,
inspectAdbDeviceOperation,
listAdbDevicesOperation,
type AdbCommandContext,
} from "./AdbDeviceOperations.js";
/** Stable identity for the ADB device-acquisition provider. */
export const ADB_PROVIDER_IDENTITY = {
@@ -150,10 +159,7 @@ export const ADB_PROVIDER_IDENTITY = {
version: null,
} as const;
const DEVICES_TIMEOUT_MS = 20_000;
const DEVICES_MAX_OUTPUT_BYTES = 1024 * 1024;
const SHELL_TIMEOUT_MS = 30_000;
const GETPROP_MAX_OUTPUT_BYTES = 2 * 1024 * 1024;
const PACKAGE_LIST_TIMEOUT_MS = 60_000;
const PACKAGE_LIST_MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
const PM_PATH_TIMEOUT_MS = 30_000;
@@ -403,49 +409,6 @@ const syncStagedAdbFile = async (
if (primary !== undefined) throw primary;
};
const clientIdentityResult = (
selection: AdbBinarySelection,
identity: AdbClientIdentity,
) => ({
binary_path: selection.binary,
path_source: selection.pathSource,
version: identity.version,
installed_path: identity.installedPath,
});
type ClientIdentity = ReturnType<typeof clientIdentityResult>;
type ProviderIdentity = { id: string; name: string; version: string | null };
/**
* Build a parsed-listing execution whose `unparsed_lines`, coverage, and
* truncation limitation all derive from the parser's unrecognized-line facts,
* so the limitation can never describe a field the result does not carry.
*/
const parsedListingExecution = (
result: Readonly<Record<string, unknown>>,
unparsed: AdbUnparsedLines,
provider: ProviderIdentity,
limitations: readonly string[] = [],
) =>
createAnalysisExecution(
{
...result,
unparsed_lines: unparsed.sample,
coverage: unparsed.total > 0 ? "partial" : "complete",
},
provider,
{
limitations: [
...limitations,
...(unparsed.total > unparsed.sample.length
? [
`unparsed_lines reports the first ${String(unparsed.sample.length)} of ${String(unparsed.total)} unrecognized lines`,
]
: []),
],
},
);
/** Materialize the caller's output directory, refusing existing files inside. */
const prepareOutputDirectory = async (
operation: string,
@@ -1376,7 +1339,7 @@ export class AdbProvider {
);
}
const client = clientIdentityResult(selection, identity);
const provider: ProviderIdentity = {
const provider: AdbProviderIdentity = {
...ADB_PROVIDER_IDENTITY,
version: identity.version,
};
@@ -1395,87 +1358,22 @@ export class AdbProvider {
{ ...limits, volatileOutput: true },
signal,
);
const context: AdbCommandContext = {
operation,
request,
selection,
client,
provider,
signal,
shell,
};
switch (operation) {
case "inspect_adb_client":
return ok(
createAnalysisExecution(
{ client, contacted_adb_server: false },
provider,
),
);
case "list_adb_devices": {
const ran = await runAdb(
operation,
{ device: "adb-devices" },
selection,
["devices", "-l"],
{
timeoutMs: DEVICES_TIMEOUT_MS,
maxOutputBytes: DEVICES_MAX_OUTPUT_BYTES,
volatileOutput: true,
},
signal,
);
if (!ran.ok) return ran;
const { devices, unparsedLines, daemonStarted } = parseAdbDevicesOutput(
ran.value.stdout,
ran.value.stderr,
);
if (unparsedLines.total > 0)
return err(
new AnalysisProtocolError(
`adb devices reported an unrecognized line: ${unparsedLines.sample[0]!}`,
),
);
return ok(
createAnalysisExecution(
{ client, devices, may_have_started_adb_server: daemonStarted },
provider,
{
limitations: [
"Device kind is inferred from the emulator serial prefix and USB transport metadata, not from device attestation",
"Kind stays unknown for TCP transports unless the serial is an emulator port",
],
},
),
);
}
case "inspect_adb_device": {
const ran = await runAdb(
operation,
{ device: request.input.serial },
selection,
adbArguments(request.input.serial, ["shell", "getprop"]),
{
timeoutMs: SHELL_TIMEOUT_MS,
maxOutputBytes: GETPROP_MAX_OUTPUT_BYTES,
volatileOutput: true,
},
signal,
);
if (!ran.ok) return ran;
const { properties, emulatorObserved } = parseGetpropOutput(
ran.value.stdout,
);
return ok(
createAnalysisExecution(
{
client,
serial: request.input.serial,
properties,
property_count: properties.length,
emulator_observed: emulatorObserved,
},
provider,
{
limitations: [
"Properties are a fixed whitelist of build identity facts, not the full getprop dump",
"emulator_observed is null when the build does not report ro.kernel.qemu",
],
},
),
);
}
return inspectAdbClientOperation(context);
case "list_adb_devices":
return listAdbDevicesOperation(context);
case "inspect_adb_device":
return inspectAdbDeviceOperation(context);
case "list_adb_packages": {
const scopeFlag =
request.input.scope === "third_party"
@@ -1976,8 +1874,8 @@ export class AdbProvider {
private async pullPackage(
operation: "pull_adb_package",
selection: AdbBinarySelection,
client: ClientIdentity,
provider: ProviderIdentity,
client: AdbClientResult,
provider: AdbProviderIdentity,
input: {
readonly serial: string;
readonly package: string;
@@ -2373,8 +2271,8 @@ export class AdbProvider {
private async pullFile(
operation: "pull_adb_file",
selection: AdbBinarySelection,
client: ClientIdentity,
provider: ProviderIdentity,
client: AdbClientResult,
provider: AdbProviderIdentity,
input: {
readonly serial: string;
readonly device_path: string;
@@ -2470,8 +2368,8 @@ export class AdbProvider {
private async pushFile(
operation: "push_adb_file",
selection: AdbBinarySelection,
client: ClientIdentity,
provider: ProviderIdentity,
client: AdbClientResult,
provider: AdbProviderIdentity,
input: {
readonly serial: string;
readonly local_path: string;
@@ -2585,7 +2483,12 @@ export class AdbProvider {
{ device: input.serial, path: devicePath },
selection,
adbArguments(input.serial, ["shell", "sha256sum", devicePath]),
{ timeoutMs: SHELL_TIMEOUT_MS, maxOutputBytes: 1024 },
{
timeoutMs: SHELL_TIMEOUT_MS,
// sha256sum echoes the path, escaping special characters, so a
// long path must not exhaust the budget for the digest line.
maxOutputBytes: 1024 + 2 * Buffer.byteLength(devicePath),
},
signal,
);
if (!deviceDigest.ok) {
@@ -2674,8 +2577,8 @@ export class AdbProvider {
private async installPackage(
operation: "install_adb_package",
selection: AdbBinarySelection,
client: ClientIdentity,
provider: ProviderIdentity,
client: AdbClientResult,
provider: AdbProviderIdentity,
input: {
readonly serial: string;
readonly apk_path: string;
@@ -2775,8 +2678,8 @@ export class AdbProvider {
private async uninstallPackage(
operation: "uninstall_adb_package",
selection: AdbBinarySelection,
client: ClientIdentity,
provider: ProviderIdentity,
client: AdbClientResult,
provider: AdbProviderIdentity,
input: { readonly serial: string; readonly package: string },
signal: AbortSignal | undefined,
): Promise<Result<AnalysisExecution, AnalysisError>> {
@@ -2842,8 +2745,8 @@ export class AdbProvider {
private async startApp(
operation: "start_adb_app",
selection: AdbBinarySelection,
client: ClientIdentity,
provider: ProviderIdentity,
client: AdbClientResult,
provider: AdbProviderIdentity,
input: { readonly serial: string; readonly package: string },
signal: AbortSignal | undefined,
): Promise<Result<AnalysisExecution, AnalysisError>> {
@@ -2898,8 +2801,8 @@ export class AdbProvider {
private async startActivity(
operation: "start_adb_activity",
selection: AdbBinarySelection,
client: ClientIdentity,
provider: ProviderIdentity,
client: AdbClientResult,
provider: AdbProviderIdentity,
input: {
readonly serial: string;
readonly action: string;
@@ -2960,8 +2863,8 @@ export class AdbProvider {
private async runAmStart(
operation: "start_adb_app" | "start_adb_activity",
selection: AdbBinarySelection,
client: ClientIdentity,
provider: ProviderIdentity,
client: AdbClientResult,
provider: AdbProviderIdentity,
serial: string,
arguments_: readonly string[],
resultFields: Record<string, unknown>,
@@ -3050,8 +2953,8 @@ export class AdbProvider {
private async stopApp(
operation: "stop_adb_app",
selection: AdbBinarySelection,
client: ClientIdentity,
provider: ProviderIdentity,
client: AdbClientResult,
provider: AdbProviderIdentity,
input: { readonly serial: string; readonly package: string },
signal: AbortSignal | undefined,
): Promise<Result<AnalysisExecution, AnalysisError>> {
@@ -3084,8 +2987,8 @@ export class AdbProvider {
private async captureScreen(
operation: "capture_adb_screen",
selection: AdbBinarySelection,
client: ClientIdentity,
provider: ProviderIdentity,
client: AdbClientResult,
provider: AdbProviderIdentity,
input: { readonly serial: string; readonly output_directory: string },
signal: AbortSignal | undefined,
): Promise<Result<AnalysisExecution, AnalysisError>> {
@@ -3114,11 +3017,11 @@ export class AdbProvider {
return err(
new AnalysisCancelledError(operation, { cause: signal.reason }),
);
const dimensions = parsePngDimensions(captured.value.stdout);
if (dimensions.width === null)
const dimensions = readPngHeaderDimensions(captured.value.stdout);
if (dimensions === undefined)
return err(
new AnalysisProtocolError(
"screencap did not return a PNG payload; the device may not expose a display",
"screencap returned an invalid PNG header; the device may not expose a display",
),
);
return this.#withRuntimeRoot(
@@ -3182,6 +3085,7 @@ export class AdbProvider {
{
limitations: [
"The capture is one frame at read time; no recording or interaction occurred",
"PNG signature, IHDR framing, checksum, and dimensions are validated; encoding and image contents are not validated",
...(usedExclusiveFallback
? [
"The exclusive-create fallback makes the output path visible while its bytes are being written",
@@ -3209,8 +3113,8 @@ export class AdbProvider {
private async collectBugreport(
operation: "collect_adb_bugreport",
selection: AdbBinarySelection,
client: ClientIdentity,
provider: ProviderIdentity,
client: AdbClientResult,
provider: AdbProviderIdentity,
input: { readonly serial: string; readonly output_directory: string },
signal: AbortSignal | undefined,
): Promise<Result<AnalysisExecution, AnalysisError>> {
-23
View File
@@ -609,26 +609,3 @@ export const parseAmStartOutput = (
(text === "" ? "am start returned no activity" : text.slice(0, 512)),
};
};
const PNG_SIGNATURE = Buffer.from([
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
]);
/** Read image dimensions from a PNG header, or null for other payloads. */
export const parsePngDimensions = (
bytes: Uint8Array,
): {
width: number | null;
height: number | null;
} => {
if (
bytes.length < 24 ||
!Buffer.from(bytes.subarray(0, 8)).equals(PNG_SIGNATURE)
)
return { width: null, height: null };
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
return {
width: view.getUint32(16),
height: view.getUint32(20),
};
};
+31 -4
View File
@@ -8,6 +8,7 @@ import { AnalysisOutputError } from "../domain/analysisErrorCore.js";
import type { JsonValue } from "../domain/jsonValue.js";
import { analysisBookmarkSchema } from "../domain/native/nativeInventory.js";
import { parseContainingProcedureResolution } from "../domain/native/nativeObservation.js";
import { nativeAddressCoordinateInputSchema } from "../domain/native/nativeAddress.js";
import { resolveAnalysisDocument } from "./AnalysisDocument.js";
import { err, ok, type Result } from "../domain/result.js";
@@ -176,8 +177,34 @@ const matchingBookmarks = (value: JsonValue, address: string): Facet => {
remediation:
"Retry list_bookmarks directly and report the provider output shape if it remains malformed.",
};
return {
state: "available",
value: parsed.data.filter((bookmark) => bookmark.address === address),
};
const target = nativeAddressCoordinateInputSchema.safeParse(address);
const matching = [];
for (const bookmark of parsed.data) {
if (bookmark.address === address) {
matching.push(bookmark);
continue;
}
const candidate = nativeAddressCoordinateInputSchema.safeParse(
bookmark.address,
);
if (
!target.success ||
!candidate.success ||
(target.data.offset === candidate.data.offset &&
target.data.space !== candidate.data.space)
)
return {
state: "unavailable",
reason:
"Bookmark address identity could not be compared with the selected address.",
remediation:
"Call list_bookmarks directly to inspect the provider addresses.",
};
if (
target.data.offset === candidate.data.offset &&
target.data.space === candidate.data.space
)
matching.push(bookmark);
}
return { state: "available", value: matching };
};
+3
View File
@@ -1,3 +1,4 @@
import type { AnalysisWorkScope } from "./AnalysisWorkScope.js";
import type { AnalysisObservation } from "../domain/analysisObservation.js";
import type { ProviderInspectionScope } from "./ProviderInspectionScope.js";
import type { BinaryTarget } from "../domain/binaryTargetTypes.js";
@@ -26,6 +27,8 @@ import {
} from "../domain/relationshipCoverage.js";
export interface ExecutionOptions {
/** Ephemeral state shared only by one composed workflow. */
readonly workScope?: AnalysisWorkScope;
readonly signal?: AbortSignal;
readonly progress?: ProgressReporter;
/**
+18
View File
@@ -0,0 +1,18 @@
/** In-memory, provider-owned intermediate facts confined to one composed task. */
export class AnalysisWorkScope {
readonly #entries = new Map<object, unknown>();
#closed = false;
get<T>(owner: object): T | undefined {
return this.#entries.get(owner) as T | undefined;
}
set<T>(owner: object, value: T): void {
if (!this.#closed) this.#entries.set(owner, value);
}
clear(): void {
this.#closed = true;
this.#entries.clear();
}
}
@@ -1,3 +1,4 @@
import { sqliteDatabaseCapability } from "./sqlite/SqliteDatabaseCapability.js";
import { persistentDebuggerHostSupported } from "../process/PersistentDebuggerHost.js";
import { pcapCaptureHostSupported } from "../domain/pcap/pcapAnalysis.js";
import { describe, expect, it } from "vitest";
@@ -100,6 +101,77 @@ const entry = (
return found;
};
it("reports the active resource session's dispatch policy without limiting independent tools", () => {
const resourceStatus = {
...status({ open: true }),
session_kind: "bundle_resources",
capabilities: [
{
provider: { id: "bundle-resources", name: "Resources", version: "1" },
shared_limitations: [],
groups: [
{
operations: [
"inspect_plist",
"decode_interface_builder",
"inspect_asset_catalog",
"inspect_keyed_archive",
],
effects: [],
limitations: [],
available: true,
reason: null,
availability_code: null,
},
],
},
],
};
const entries = buildCapabilityInventory(resourceStatus, enabledPolicy);
for (const name of [
"inspect_artifact",
"extract_artifact",
"trace_dylib_resolution",
"get_navigation_context",
])
expect(entries.find((item) => item.name === name)).toMatchObject({
available: false,
reason: "target_unsupported",
remediation: expect.stringContaining("open_binary"),
});
for (const name of [
"inspect_plist",
"decode_interface_builder",
"inspect_asset_catalog",
"inspect_keyed_archive",
"inspect_pe_imports",
"inspect_desktop_application",
"close_binary",
])
expect(entries.find((item) => item.name === name)).toMatchObject({
available: true,
});
expect(
entries.find((item) => item.name === "inspect_apk_signature"),
).toMatchObject({
available: true,
reason: "available",
remediation: null,
});
const sqlite = sqliteDatabaseCapability(process.versions.node);
expect(
entries.find((item) => item.name === "inspect_sqlite_database"),
).toMatchObject(
sqlite.available
? { available: true }
: {
available: false,
reason: "provider_unavailable",
remediation: sqlite.reason,
},
);
});
describe("ADB and Apktool host availability", () => {
it("uses only their selected owner probes", () => {
const ready = {
+18 -1
View File
@@ -42,6 +42,7 @@ import type { JebAnalysisAvailability } from "./jeb/JebAnalysisPort.js";
import { FRIDA_TOOL_CONTRACTS } from "../contracts/fridaToolContracts.js";
import { TOOL_CONTRACTS } from "../contracts/toolContracts.js";
import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
import { isBundleResourceOperation } from "./binary/BundleResourceSession.js";
const FRIDA_OPERATION_NAMES: ReadonlySet<string> = new Set(
FRIDA_TOOL_CONTRACTS.map(({ name }) => name),
@@ -49,6 +50,7 @@ const FRIDA_OPERATION_NAMES: ReadonlySet<string> = new Set(
const statusSchema = z.object({
open: z.boolean(),
session_kind: z.literal("bundle_resources").optional(),
kind: z.enum(["executable", "database", "archive", "artifact"]).optional(),
format: z.string().optional(),
capabilities: z
@@ -128,6 +130,7 @@ type AvailabilityContext = {
readonly name: string;
readonly kind: ToolKind;
readonly targetOpen: boolean;
readonly resourceSession: boolean;
readonly requiresActiveTarget: boolean;
readonly targetKind:
| "executable"
@@ -260,6 +263,7 @@ export const buildCapabilityInventory = (
name: contract.name,
kind: contract.kind,
targetOpen: status.open,
resourceSession: status.session_kind === "bundle_resources",
requiresActiveTarget:
"requiresActiveTarget" in contract &&
contract.requiresActiveTarget === true,
@@ -380,6 +384,19 @@ export function compactToolAvailability(
}
const availabilityFor = (context: AvailabilityContext): Availability => {
const workflowAvailability = workflowAvailabilityFor(context);
if (
context.targetOpen &&
context.resourceSession &&
context.requiresActiveTarget &&
!isBundleResourceOperation(context.name) &&
(workflowAvailability === null || context.kind === "session")
)
return {
reason: "target_unsupported",
remediation:
"This resource-only session cannot execute this operation. Call open_binary with an executable analysis target.",
};
// These built-in workflows accept a caller-selected endpoint; they do not load the optional passive provider.
if (
context.name === "inspect_byte_entropy" ||
@@ -403,7 +420,6 @@ const availabilityFor = (context: AvailabilityContext): Availability => {
reason: "provider_unavailable",
remediation: `Adapter ${failure.providerId} could not load: ${failure.reason}. Repair the adapter installation and restart REA; other capabilities remain available.`,
};
const workflowAvailability = workflowAvailabilityFor(context);
if (workflowAvailability !== null) return workflowAvailability;
const hostDecision = hostAvailability(context);
if (hostDecision !== null) return hostDecision;
@@ -453,6 +469,7 @@ const workflowAvailabilityFor = ({
"Run persistent debugger sessions on Linux with /proc descriptor readback.",
);
if (
name === "inspect_apk_signature" ||
name === "inspect_desktop_application" ||
name === "inspect_go_binary" ||
name === "inspect_il2cpp_metadata" ||
@@ -0,0 +1,139 @@
import { expect, it } from "vitest";
import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
import { readGuidanceSectionInputSchema } from "../contracts/guidanceContracts.js";
import { MANAGED_SKILL_FILES } from "./SetupSkill.js";
import {
readGuidanceSectionWithReader,
searchGuidanceWithReader,
} from "./GuidanceAccess.js";
const reader = (text: string) => async (path: string) =>
path === MANAGED_SKILL_FILES[0] ? text : "# Other\n";
it("does not substitute a common domain word for an unknown query subject", async () => {
const resources = new ArtifactResourceScope();
expect(
await searchGuidanceWithReader(
{ query: "how do I inspect module frobnicate qzxv" },
reader("# Modules\nInspect a module with retained facts.\n"),
resources,
),
).toMatchObject({ ok: true, value: { status: "no_matches" } });
await resources.close();
});
it.each([
["## C#", "C#"],
["## C# ###", "C#"],
["## C\\#", "C\\#"],
[" ## C#", "C#"],
["## C# \t ### \t", "C#"],
["## ###", ""],
])("preserves the source title in %s", async (line, heading) => {
const resources = new ArtifactResourceScope();
const source = reader(`${line}\nneedle instructions\n`);
const result = await searchGuidanceWithReader(
{ query: "needle" },
source,
resources,
);
if (!result.ok || result.value.status !== "matches")
throw new Error("Expected matching documentation");
expect(result.value.matches[0]?.heading).toBe(heading);
const value = await readGuidanceSectionWithReader(
{ locator: result.value.matches[0]?.locator },
source,
resources,
);
expect(value).toMatchObject({
ok: true,
value: { status: "section", heading },
});
await resources.close();
});
it("issues readable locators independently of heading length", async () => {
const resources = new ArtifactResourceScope();
const heading = `needle ${"x".repeat(3000)}`;
const source = reader(`## ${heading}\nInstructions\n`);
const result = await searchGuidanceWithReader(
{ query: "needle" },
source,
resources,
);
if (!result.ok || result.value.status !== "matches")
throw new Error("Expected match");
const locator = result.value.matches[0]?.locator;
expect(readGuidanceSectionInputSchema.safeParse({ locator }).success).toBe(
true,
);
expect(
await readGuidanceSectionWithReader({ locator }, source, resources),
).toMatchObject({
ok: true,
value: { status: "section", heading },
});
await resources.close();
});
it.each([2048, 2049])(
"bounds the combined index across guides with %i sections each",
async (count) => {
const resources = new ArtifactResourceScope();
const result = await searchGuidanceWithReader(
{ query: "needle", limit: 1 },
async (path) =>
path === MANAGED_SKILL_FILES[0] || path === MANAGED_SKILL_FILES[1]
? "# needle\n".repeat(count)
: "",
resources,
);
expect(result).toMatchObject({
ok: true,
value: { status: count === 2048 ? "matches" : "bundle_unavailable" },
});
await resources.close();
},
);
it("preserves retry cleanup errors and their live owner in both operations", async () => {
const resources = new ArtifactResourceScope();
const source = reader("# needle\nInstructions\n");
const initial = await searchGuidanceWithReader(
{ query: "needle" },
source,
resources,
);
if (!initial.ok || initial.value.status !== "matches")
throw new Error("Expected match");
const locator = initial.value.matches[0]?.locator;
let blocked = true;
resources.retain({
kind: "lease",
resource: "guidance descriptor",
lease: {
release: async () => {
if (blocked) throw new Error("still owned");
},
},
});
const retryingReader = (path: string) => resources.run(() => source(path));
const errors = [];
for (const result of [
await searchGuidanceWithReader(
{ query: "needle" },
retryingReader,
resources,
),
await readGuidanceSectionWithReader({ locator }, retryingReader, resources),
]) {
if (result.ok) throw new Error("Cleanup failures must remain errors");
expect(result.error).toMatchObject({ cleanupIncomplete: true });
expect(result.error.cleanupResources).toContain("guidance descriptor");
errors.push(result.error);
}
blocked = false;
await resources.close();
expect(resources.hasPending).toBe(false);
for (const error of errors) expect(error.cleanupIncomplete).toBe(false);
});
@@ -0,0 +1,183 @@
import { expect, it } from "vitest";
import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js";
import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
import { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js";
import { RetainedRegularFileCleanupFailure } from "./RegularFileRead.js";
import { MANAGED_SKILL_FILES } from "./SetupSkill.js";
import {
readGuidanceSectionWithReader,
searchGuidanceWithReader,
} from "./GuidanceAccess.js";
const gate = () => {
let release = () => {};
const promise = new Promise<void>((resolve) => {
release = resolve;
});
return { promise, release };
};
const content = (path: string) =>
path === MANAGED_SKILL_FILES[0] ? "# needle\nInstructions\n" : "# Other\n";
it.each(["search", "read"] as const)(
"finishes every guide in an admitted %s while shutdown waits",
async (action) => {
const resources = new ArtifactResourceScope();
const reached = gate();
const resume = gate();
const paths: string[] = [];
const reader = (path: string, admitted = resources) =>
admitted.run(async () => {
paths.push(path);
if (path === MANAGED_SKILL_FILES[1]) {
reached.release();
await resume.promise;
}
return content(path);
});
const launched: Promise<unknown>[] = [];
try {
const initial = await searchGuidanceWithReader(
{ query: "needle" },
async (path) => content(path),
resources,
);
if (!initial.ok || initial.value.status !== "matches")
throw new Error("Expected match");
const operation =
action === "search"
? searchGuidanceWithReader({ query: "needle" }, reader, resources)
: readGuidanceSectionWithReader(
{ locator: initial.value.matches[0]?.locator },
reader,
resources,
);
launched.push(operation);
await awaitAcknowledgement(
reached.promise,
operation,
"second guidance read",
);
let closed = false;
const closing = resources.close().then(() => {
closed = true;
});
launched.push(closing);
await Promise.resolve();
expect(closed).toBe(false);
resume.release();
expect(await operation).toMatchObject({
ok: true,
value: { status: action === "search" ? "matches" : "section" },
});
await closing;
expect(paths).toEqual(MANAGED_SKILL_FILES);
expect(closed).toBe(true);
expect(resources.hasPending).toBe(false);
} finally {
resume.release();
await Promise.allSettled(launched);
await resources.close();
}
},
);
it("retains a failed child while another admitted request reaches later reads", async () => {
const resources = new ArtifactResourceScope();
const reached = gate();
const resume = gate();
const paths: string[] = [];
let blocked = true;
const failure = new RetainedRegularFileCleanupFailure(
new RegularFileCleanupFailure({
path: "owned guidance descriptor",
owner: {
close: async () => {
if (blocked) throw new Error("close denied");
},
} as never,
outcome: { kind: "failed", cause: new Error("read failed") },
cleanupCause: new Error("close denied"),
}),
);
const launched: Promise<unknown>[] = [];
try {
const active = searchGuidanceWithReader(
{ query: "needle" },
(path, child) =>
child.run(async () => {
paths.push(path);
if (path === MANAGED_SKILL_FILES[1]) {
reached.release();
await resume.promise;
}
return content(path);
}),
resources,
);
launched.push(active);
await awaitAcknowledgement(
reached.promise,
active,
"overlapping guidance read",
);
const failing = searchGuidanceWithReader(
{ query: "needle" },
async (_path, child) => {
child.retain(failure.cleanupOwner);
throw failure;
},
resources,
);
launched.push(failing);
const failed = await failing;
if (failed.ok) throw new Error("Expected cleanup failure");
expect(failed.error.cleanupResources).toEqual([
"owned guidance descriptor",
]);
expect(failed.error.cleanupIncomplete).toBe(true);
const closing = resources.close().then(
() => undefined,
(cause: unknown) => cause,
);
launched.push(closing);
resume.release();
expect(await active).toMatchObject({
ok: true,
value: { status: "matches" },
});
expect(paths).toEqual(MANAGED_SKILL_FILES);
expect(await closing).toMatchObject({
cleanup: { resources: ["owned guidance descriptor"] },
});
expect(resources.hasPending).toBe(true);
expect(failed.error.cleanupIncomplete).toBe(true);
blocked = false;
await resources.close();
expect(resources.hasPending).toBe(false);
expect(failed.error.cleanupIncomplete).toBe(false);
} finally {
resume.release();
blocked = false;
await Promise.allSettled(launched);
await resources.close();
}
});
it("refuses new work after shutdown without reporting a missing bundle", async () => {
const resources = new ArtifactResourceScope();
await resources.close();
let reads = 0;
await expect(
searchGuidanceWithReader(
{ query: "needle" },
async (path) => {
reads += 1;
return content(path);
},
resources,
),
).rejects.toMatchObject({ reason: "unavailable" });
expect(reads).toBe(0);
});
+320
View File
@@ -0,0 +1,320 @@
import { expect, it } from "vitest";
import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
import { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js";
import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
import { MANAGED_SKILL_FILES } from "./SetupSkill.js";
import { searchGuidanceOutputSchema } from "../contracts/guidanceContracts.js";
import { RetainedRegularFileCleanupFailure } from "./RegularFileRead.js";
import {
readGuidanceSectionWithReader,
searchGuidanceWithReader,
} from "./GuidanceAccess.js";
const moduleGuide = `# JavaScript applications\n\n## Read a module without repeating analysis\n\nFor an exact JavaScript module (javascript-module), select its source path and module identity:\n\n\`\`\`json\n{"node_id":"module:main.js","kind":"javascript-module"}\n\`\`\`\n`;
const guideFixture = async (path: string): Promise<string> =>
path === "references/javascript-applications.md"
? moduleGuide
: `# ${path}\n\nGeneral module guidance for pagination.\n`;
const valueOf = async (
request: ReturnType<typeof searchGuidanceWithReader>,
) => {
const result = await request;
if (!result.ok) throw result.error;
return searchGuidanceOutputSchema.parse(result.value);
};
it("ranks a focused JavaScript module section and retrieves it with its locator", async () => {
const resources = new ArtifactResourceScope();
const result = await valueOf(
searchGuidanceWithReader(
{ query: "exact JavaScript module" },
guideFixture,
resources,
),
);
expect(result.status).toBe("matches");
if (result.status !== "matches") throw new Error("Expected guide matches");
expect(result.matches[0]).toMatchObject({
guide: "references/javascript-applications.md",
heading: "Read a module without repeating analysis",
matched_terms: ["exact", "javascript", "module"],
context: expect.stringContaining("javascript-module"),
});
const selected = result.matches[0];
if (selected === undefined) throw new Error("Expected selected guidance");
const retrieved = await readGuidanceSectionWithReader(
{ locator: selected.locator },
guideFixture,
resources,
);
if (!retrieved.ok) throw retrieved.error;
expect(retrieved.value).toMatchObject({
status: "section",
source: "installed_rea_guidance",
guide: selected.guide,
heading: selected.heading,
locator: selected.locator,
content: expect.stringContaining('"node_id":"module:main.js"'),
});
});
it("returns an honest no-match result and rejects empty queries and invalid bounds", async () => {
const resources = new ArtifactResourceScope();
expect(
await valueOf(
searchGuidanceWithReader(
{ query: "unlikely qzxv term" },
guideFixture,
resources,
),
),
).toMatchObject({
status: "no_matches",
total_matches: 0,
message: expect.stringContaining("does not indicate"),
});
expect(
await valueOf(
searchGuidanceWithReader(
{ query: "how do I frobnicate qzxv" },
guideFixture,
resources,
),
),
).toMatchObject({
status: "no_matches",
total_matches: 0,
});
expect(
await searchGuidanceWithReader({ query: " " }, guideFixture, resources),
).toMatchObject({ ok: false });
expect(
await searchGuidanceWithReader(
{ query: "x".repeat(2049) },
guideFixture,
resources,
),
).toMatchObject({ ok: false });
expect(
await searchGuidanceWithReader(
{ query: "module", limit: 21 },
guideFixture,
resources,
),
).toMatchObject({ ok: false });
});
it("does not treat Markdown headings inside fenced code as section boundaries", async () => {
const result = await valueOf(
searchGuidanceWithReader(
{ query: "phantom-heading-token" },
async (path) =>
`${await guideFixture(path)}\n\n\`\`\`markdown\n# phantom-heading-token\n\`\`\`\n`,
new ArtifactResourceScope(),
),
);
expect(result.status).toBe("matches");
if (result.status !== "matches") throw new Error("Expected guide matches");
expect(result.matches.map(({ heading }) => heading)).not.toContain(
"phantom-heading-token",
);
});
it("refuses an oversized guide bundle instead of indexing partial content", async () => {
const result = await searchGuidanceWithReader(
{ query: "module" },
async (path) =>
path === "SKILL.md" ? "x".repeat(4_194_305) : guideFixture(path),
new ArtifactResourceScope(),
);
expect(result).toMatchObject({
ok: true,
value: { status: "bundle_unavailable" },
});
});
it("paginates deterministically without losing, duplicating, or reordering matches", async () => {
const readGuide = guideFixture;
const first = await valueOf(
searchGuidanceWithReader(
{ query: "module", limit: 2 },
readGuide,
new ArtifactResourceScope(),
),
);
const repeat = await valueOf(
searchGuidanceWithReader(
{ query: "module", limit: 2 },
readGuide,
new ArtifactResourceScope(),
),
);
expect(first).toEqual(repeat);
if (first.status !== "matches") throw new Error("Expected guide matches");
expect(first.has_more).toBe(true);
const all = first.matches.map(({ locator }) => locator);
let cursor = first.next_cursor;
while (cursor !== undefined) {
const page = await valueOf(
searchGuidanceWithReader(
{ query: "module", limit: 2, cursor },
readGuide,
new ArtifactResourceScope(),
),
);
if (page.status !== "matches") throw new Error("Expected guide matches");
all.push(...page.matches.map(({ locator }) => locator));
cursor = page.next_cursor;
}
const complete = await valueOf(
searchGuidanceWithReader(
{ query: "module", limit: 20 },
readGuide,
new ArtifactResourceScope(),
),
);
if (complete.status !== "matches") throw new Error("Expected guide matches");
expect(new Set(all).size).toBe(all.length);
expect(all).toEqual(complete.matches.map(({ locator }) => locator));
});
it("rejects malformed locators and reports changed or missing installed guides", async () => {
const resources = new ArtifactResourceScope();
const search = await valueOf(
searchGuidanceWithReader(
{ query: "exact JavaScript module" },
guideFixture,
resources,
),
);
if (search.status !== "matches") throw new Error("Expected guide matches");
expect(
await readGuidanceSectionWithReader(
{ locator: "../../outside.md" },
guideFixture,
resources,
),
).toMatchObject({ ok: false });
const changedReader = async (path: string) =>
`${await guideFixture(path)}\nChanged after search.\n`;
const stale = await readGuidanceSectionWithReader(
{ locator: search.matches[0]?.locator },
changedReader,
resources,
);
expect(stale).toMatchObject({ ok: true, value: { status: "stale_locator" } });
const firstPage = await valueOf(
searchGuidanceWithReader(
{ query: "module", limit: 1 },
guideFixture,
resources,
),
);
if (firstPage.status !== "matches" || firstPage.next_cursor === undefined)
throw new Error("Expected a continuation token");
expect(
await searchGuidanceWithReader(
{ query: "module", limit: 1, cursor: firstPage.next_cursor },
changedReader,
resources,
),
).toMatchObject({ ok: true, value: { status: "stale_continuation" } });
expect(
await searchGuidanceWithReader(
{ query: "different query", cursor: firstPage.next_cursor },
guideFixture,
resources,
),
).toMatchObject({ ok: false });
expect(
await searchGuidanceWithReader(
{ query: "module", cursor: "garbage" },
guideFixture,
resources,
),
).toMatchObject({ ok: false });
const missingReader = async (): Promise<string> => {
throw Object.assign(new Error("missing"), { code: "ENOENT" });
};
expect(
await searchGuidanceWithReader(
{ query: "module" },
missingReader,
resources,
),
).toMatchObject({ ok: true, value: { status: "bundle_unavailable" } });
expect(
await readGuidanceSectionWithReader(
{ locator: search.matches[0]?.locator },
missingReader,
resources,
),
).toMatchObject({ ok: true, value: { status: "bundle_unavailable" } });
});
it("returns cleanup failures separately and retains their owner until retry succeeds", async () => {
const resources = new ArtifactResourceScope();
let failRelease = true;
const failure = new RegularFileCleanupFailure({
path: "guidance file handle",
owner: {
close: async () => {
if (failRelease) throw new Error("descriptor close failed");
},
} as never,
outcome: { kind: "failed", cause: new Error("guide read failed") },
cleanupCause: new Error("descriptor close failed"),
});
const retainedFailure = new RetainedRegularFileCleanupFailure(failure);
resources.retain(retainedFailure.cleanupOwner);
const result = await searchGuidanceWithReader(
{ query: "module" },
async () => {
throw retainedFailure;
},
resources,
);
if (result.ok) throw new Error("Expected a retained cleanup failure");
expect(result.error).toMatchObject({ cleanupIncomplete: true });
expect(result.error.cleanupResources).toEqual(["guidance file handle"]);
expect(resources.hasPending).toBe(true);
failRelease = false;
await resources.close();
expect(resources.hasPending).toBe(false);
expect(result.error.cleanupIncomplete).toBe(false);
expect(result.error.cleanupResources).toEqual([]);
const freshResources = new ArtifactResourceScope();
freshResources.retain({
kind: "lease",
resource: "unrelated pending resource",
lease: { release: async () => undefined },
});
expect(
await searchGuidanceWithReader(
{ query: "module" },
async () => {
throw new Error("guide is unreadable");
},
freshResources,
),
).toMatchObject({
ok: true,
value: { status: "bundle_unavailable" },
});
await freshResources.close();
});
it("uses only the fixed managed guide inventory for injected reads", async () => {
const requested: string[] = [];
await searchGuidanceWithReader(
{ query: "module" },
async (path) => {
requested.push(path);
return guideFixture(path);
},
new ArtifactResourceScope(),
);
expect(requested).toEqual([...MANAGED_SKILL_FILES]);
});
+602
View File
@@ -0,0 +1,602 @@
import { createHash } from "node:crypto";
import { fileURLToPath } from "node:url";
import type { z } from "zod";
import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
import { AnalysisCleanupLease } from "../artifacts/AnalysisCleanupLease.js";
import { MANAGED_SKILL_FILES, readSkillFile } from "./SetupSkill.js";
import { PRODUCT_IDENTITY } from "../identity.js";
import {
readGuidanceSectionInputSchema,
readGuidanceSectionOutputSchema,
type ReadGuidanceSectionOutput,
type SearchGuidanceOutput,
searchGuidanceInputSchema,
searchGuidanceOutputSchema,
} from "../contracts/guidanceContracts.js";
import { AnalysisInputError } from "../domain/analysisErrorCore.js";
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
import {
bindRegularFileCleanup,
RetainedRegularFileCleanupFailure,
} from "./RegularFileRead.js";
import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js";
import { err, ok, type Result } from "../domain/result.js";
import {
parseGuidanceSections,
type GuideSection,
} from "./GuidanceSections.js";
const SKILL_ROOT = `../../skills/${PRODUCT_IDENTITY.skillName}/`;
const SOURCE = "installed_rea_guidance" as const;
const MAX_GUIDE_FILE_BYTES = 1_048_576;
const MAX_GUIDE_BUNDLE_BYTES = 4_194_304;
const sha256 = (value: string | Uint8Array): string =>
createHash("sha256").update(value).digest("hex");
interface GuidanceBundle {
readonly digest: string;
readonly sections: readonly GuideSection[];
}
const readPackagedGuide = (
relativePath: string,
resources: ArtifactResourceScope,
): Promise<string> =>
readSkillFile(
fileURLToPath(new URL(`${SKILL_ROOT}${relativePath}`, import.meta.url)),
resources,
MAX_GUIDE_FILE_BYTES,
);
type GuidanceBundleRead =
| { readonly kind: "available"; readonly bundle: GuidanceBundle }
| { readonly kind: "unavailable"; readonly message: string }
| { readonly kind: "cleanup"; readonly error: ProviderCleanupError };
const guidanceCleanupFailure = (
cause: unknown,
resources: ArtifactResourceScope,
operation: string,
): ProviderCleanupError => {
const observation = ArtifactReaderFailure.cleanupObservation(
cause,
resources.cleanupResources,
);
const error = new ProviderCleanupError(
"rea",
resources.cleanupResources,
{ phase: "managed_guidance_read", reason: observation.reason },
{
operation,
cause,
cleanup: observation,
remediationAction:
"Close REA so it can retry releasing the retained guidance file handle. Do not repeat the request until cleanup completes.",
},
);
bindRegularFileCleanup(cause, error);
return error;
};
const readBundle = async (
readGuide: (relativePath: string) => Promise<string>,
resources: ArtifactResourceScope,
operation: string,
): Promise<GuidanceBundleRead> => {
const files = new Map<string, string>();
let bundleBytes = 0;
try {
for (const relativePath of MANAGED_SKILL_FILES) {
const content = await readGuide(relativePath);
bundleBytes += Buffer.byteLength(content, "utf8");
if (bundleBytes > MAX_GUIDE_BUNDLE_BYTES)
return {
kind: "unavailable",
message:
"Installed REA guidance exceeds the 4 MiB bundle allowance; no partial bundle was indexed.",
};
files.set(relativePath, content.replaceAll("\r\n", "\n"));
}
} catch (cause: unknown) {
return cause instanceof RetainedRegularFileCleanupFailure ||
(cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined)
? {
kind: "cleanup",
error: guidanceCleanupFailure(cause, resources, operation),
}
: {
kind: "unavailable",
message:
"Installed REA guidance is missing or unreadable; no complete bundle could be indexed.",
};
}
const digest = sha256(
[...files]
.map(([path, content]) => `${path}\0${sha256(content)}\n`)
.join(""),
);
const sections = parseGuidanceSections(files);
if (sections === undefined)
return {
kind: "unavailable",
message:
"Installed REA guidance exceeds the 4,096-section or 25,165,824 expanded UTF-16-code-unit index allowance; no partial index was used.",
};
return { kind: "available", bundle: { digest, sections } };
};
const words = (text: string): string[] => [
...new Set(text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? []),
];
const GENERIC_QUERY_WORDS: ReadonlySet<string> = new Set([
"a",
"about",
"an",
"and",
"are",
"as",
"at",
"be",
"been",
"by",
"can",
"could",
"did",
"do",
"does",
"for",
"from",
"how",
"i",
"in",
"is",
"it",
"my",
"of",
"on",
"or",
"should",
"so",
"that",
"the",
"their",
"there",
"they",
"this",
"to",
"was",
"we",
"were",
"what",
"when",
"where",
"which",
"who",
"why",
"will",
"with",
"would",
"you",
"your",
]);
const sectionLocator = (
bundle: GuidanceBundle,
section: GuideSection,
): string => {
const identity = JSON.stringify({
v: 1,
bundle: bundle.digest,
guide: section.guide,
ordinal: section.ordinal,
section: sha256(section.content),
});
return `rea-guide-v1.${Buffer.from(identity).toString("base64url")}`;
};
type GuidanceReader = (
relativePath: string,
resources: ArtifactResourceScope,
) => Promise<string>;
type GuidanceResult<Value> = Result<
Value,
AnalysisInputError | ProviderCleanupError
>;
/** Admit the complete request once; file reads belong to its independent child. */
const runGuidanceOperation = async <Value>(
resources: ArtifactResourceScope,
operationName: string,
operation: (child: ArtifactResourceScope) => Promise<GuidanceResult<Value>>,
): Promise<GuidanceResult<Value>> => {
try {
return await resources.run(async () => {
const child = new ArtifactResourceScope();
try {
return await operation(child);
} finally {
if (child.hasPending)
resources.retain({
kind: "lease",
resource: "managed guidance operation",
lease: new AnalysisCleanupLease(() => child.close(), {
get cleanupResources() {
return child.cleanupResources;
},
}),
});
else await child.close();
}
});
} catch (cause: unknown) {
if (cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined)
return err(guidanceCleanupFailure(cause, resources, operationName));
throw cause;
}
};
/** Reader seams receive the operation child rather than re-entering the root. */
export const searchGuidanceWithReader = (
input: unknown,
readGuide: GuidanceReader,
resources: ArtifactResourceScope,
): Promise<GuidanceResult<SearchGuidanceOutput>> =>
runGuidanceOperation(resources, "search_guidance", (child) =>
searchGuidanceInScope(input, (path) => readGuide(path, child), child),
);
export const readGuidanceSectionWithReader = (
input: unknown,
readGuide: GuidanceReader,
resources: ArtifactResourceScope,
): Promise<GuidanceResult<ReadGuidanceSectionOutput>> =>
runGuidanceOperation(resources, "read_guidance_section", (child) =>
readGuidanceSectionInScope(input, (path) => readGuide(path, child), child),
);
const invalidInput = (
operation: string,
input: unknown,
issues: readonly z.core.$ZodIssue[],
): Result<never, AnalysisInputError> =>
err(analysisInputErrorFromIssues(operation, issues, input));
export const searchGuidance = async (
input: unknown,
resources: ArtifactResourceScope,
): Promise<
Result<SearchGuidanceOutput, AnalysisInputError | ProviderCleanupError>
> => searchGuidanceWithReader(input, readPackagedGuide, resources);
/** Reader seam keeps packaged-guide drift and missing-file behavior testable. */
const searchGuidanceInScope = async (
input: unknown,
readGuide: (relativePath: string) => Promise<string>,
resources: ArtifactResourceScope,
): Promise<
Result<SearchGuidanceOutput, AnalysisInputError | ProviderCleanupError>
> => {
const parsed = searchGuidanceInputSchema.safeParse(input);
if (!parsed.success)
return invalidInput("search_guidance", input, parsed.error.issues);
const cursor =
parsed.data.cursor === undefined
? undefined
: decodeGuidanceCursor(parsed.data.cursor);
if (parsed.data.cursor !== undefined && cursor === undefined)
return invalidInput("search_guidance", input, [
{
code: "custom",
path: ["cursor"],
message: "Invalid guidance continuation token",
},
]);
const queryDigest = sha256(parsed.data.query);
if (cursor !== undefined && cursor.query !== queryDigest)
return invalidInput("search_guidance", input, [
{
code: "custom",
path: ["cursor"],
message: "The continuation token belongs to a different query",
},
]);
const loaded = await readBundle(readGuide, resources, "search_guidance");
if (loaded.kind === "cleanup") return err(loaded.error);
if (loaded.kind === "unavailable")
return ok(
searchGuidanceOutputSchema.parse({
status: "bundle_unavailable",
source: SOURCE,
content_kind: "documentation",
message: loaded.message,
}),
);
const bundle = loaded.bundle;
if (cursor !== undefined && cursor.bundle !== bundle.digest)
return ok(
searchGuidanceOutputSchema.parse({
status: "stale_continuation",
source: SOURCE,
content_kind: "documentation",
message:
"The installed guidance changed since this search page was issued. Restart the search to continue with current content.",
}),
);
const queryWords = words(parsed.data.query);
const specificQueryWords = queryWords.filter(
(word) => !GENERIC_QUERY_WORDS.has(word),
);
const queryWordsToMatch = specificQueryWords;
const ranked = bundle.sections
.flatMap((section) => {
const sectionWords = new Set(words(section.content));
const matchedSpecific = queryWordsToMatch.filter((word) =>
sectionWords.has(word),
);
if (
queryWordsToMatch.length === 0 ||
matchedSpecific.length !== queryWordsToMatch.length
)
return [];
const matched = queryWords.filter((word) => sectionWords.has(word));
const headingWords = new Set(words(section.heading));
const score =
matched.length +
matched.filter((word) => headingWords.has(word)).length * 2;
return {
section,
matched,
score,
wordCount: sectionWords.size,
};
})
.sort(
(left, right) =>
right.score - left.score ||
// Prefer a focused recipe over a long overview when term scores tie.
left.wordCount - right.wordCount ||
compareText(left.section.guide, right.section.guide) ||
left.section.ordinal - right.section.ordinal,
);
if (ranked.length === 0)
return ok(
searchGuidanceOutputSchema.parse({
status: "no_matches",
source: SOURCE,
content_kind: "documentation",
query: parsed.data.query,
message:
"No bundled guidance sections matched this query. This does not indicate whether an analysis capability is available.",
total_matches: 0,
}),
);
const offset = cursor?.offset ?? 0;
if (cursor !== undefined && offset >= ranked.length)
return invalidInput("search_guidance", input, [
{
code: "custom",
path: ["cursor"],
message:
"The continuation token does not select a remaining result page",
},
]);
const page = ranked.slice(offset, offset + parsed.data.limit);
const hasMore = offset + page.length < ranked.length;
return ok(
searchGuidanceOutputSchema.parse({
status: "matches",
source: SOURCE,
content_kind: "documentation",
package: `${PRODUCT_IDENTITY.packageName}@${PRODUCT_IDENTITY.packageVersion}`,
bundle_sha256: bundle.digest,
query: parsed.data.query,
matches: page.map(({ section, matched }) => ({
guide: section.guide,
heading: section.heading,
locator: sectionLocator(bundle, section),
matched_terms: matched,
relevance: `Matched ${matched.join(", ")} in this section.`,
context: contextFor(section.content, matched),
})),
total_matches: ranked.length,
offset,
limit: parsed.data.limit,
has_more: hasMore,
...(hasMore
? {
next_cursor: encodeGuidanceCursor({
v: 1,
bundle: bundle.digest,
query: queryDigest,
offset: offset + page.length,
}),
}
: {}),
}),
);
};
export const readGuidanceSection = async (
input: unknown,
resources: ArtifactResourceScope,
): Promise<
Result<ReadGuidanceSectionOutput, AnalysisInputError | ProviderCleanupError>
> => readGuidanceSectionWithReader(input, readPackagedGuide, resources);
/** Reader seam keeps packaged-guide drift and missing-file behavior testable. */
const readGuidanceSectionInScope = async (
input: unknown,
readGuide: (relativePath: string) => Promise<string>,
resources: ArtifactResourceScope,
): Promise<
Result<ReadGuidanceSectionOutput, AnalysisInputError | ProviderCleanupError>
> => {
const parsed = readGuidanceSectionInputSchema.safeParse(input);
if (!parsed.success)
return invalidInput("read_guidance_section", input, parsed.error.issues);
const identity = decodeLocator(parsed.data.locator);
if (identity === undefined)
return invalidInput("read_guidance_section", input, [
{
code: "custom",
path: ["locator"],
message: "Malformed or unsupported guidance locator",
},
]);
const loaded = await readBundle(
readGuide,
resources,
"read_guidance_section",
);
if (loaded.kind === "cleanup") return err(loaded.error);
if (loaded.kind === "unavailable")
return ok(
readGuidanceSectionOutputSchema.parse({
status: "bundle_unavailable",
source: SOURCE,
content_kind: "documentation",
message: loaded.message,
}),
);
const bundle = loaded.bundle;
if (identity.bundle !== bundle.digest)
return ok(
readGuidanceSectionOutputSchema.parse({
status: "stale_locator",
source: SOURCE,
content_kind: "documentation",
message:
"The installed guidance changed since this locator was issued. Search again for a current locator.",
}),
);
const section = bundle.sections.find(
(candidate) =>
candidate.guide === identity.guide &&
candidate.ordinal === identity.ordinal &&
sha256(candidate.content) === identity.section,
);
if (section === undefined)
return ok(
readGuidanceSectionOutputSchema.parse({
status: "stale_locator",
source: SOURCE,
content_kind: "documentation",
message:
"The selected section is no longer present in the installed guidance. Search again.",
}),
);
return ok(
readGuidanceSectionOutputSchema.parse({
status: "section",
source: SOURCE,
content_kind: "documentation",
package: `${PRODUCT_IDENTITY.packageName}@${PRODUCT_IDENTITY.packageVersion}`,
bundle_sha256: bundle.digest,
guide: section.guide,
heading: section.heading,
locator: parsed.data.locator,
content: section.content,
}),
);
};
interface LocatorIdentity {
readonly v: 1;
readonly bundle: string;
readonly guide: string;
readonly ordinal: number;
readonly section: string;
}
interface GuidanceCursor {
readonly v: 1;
readonly bundle: string;
readonly query: string;
readonly offset: number;
}
const encodeGuidanceCursor = (cursor: GuidanceCursor): string =>
`rea-guidance-page-v1.${Buffer.from(JSON.stringify(cursor)).toString("base64url")}`;
const decodeGuidanceCursor = (cursor: string): GuidanceCursor | undefined => {
const prefix = "rea-guidance-page-v1.";
if (!cursor.startsWith(prefix)) return undefined;
try {
const value: unknown = JSON.parse(
Buffer.from(cursor.slice(prefix.length), "base64url").toString("utf8"),
);
if (typeof value !== "object" || value === null) return undefined;
const item = value as Partial<GuidanceCursor>;
if (
item.v !== 1 ||
typeof item.bundle !== "string" ||
!/^[a-f0-9]{64}$/u.test(item.bundle) ||
typeof item.query !== "string" ||
!/^[a-f0-9]{64}$/u.test(item.query) ||
typeof item.offset !== "number" ||
!Number.isInteger(item.offset) ||
item.offset < 1
)
return undefined;
return item as GuidanceCursor;
} catch {
return undefined;
}
};
const decodeLocator = (locator: string): LocatorIdentity | undefined => {
if (!locator.startsWith("rea-guide-v1.")) return undefined;
try {
const value: unknown = JSON.parse(
Buffer.from(locator.slice("rea-guide-v1.".length), "base64url").toString(
"utf8",
),
);
if (typeof value !== "object" || value === null) return undefined;
const item = value as Partial<LocatorIdentity>;
if (
item.v !== 1 ||
typeof item.bundle !== "string" ||
!/^[a-f0-9]{64}$/u.test(item.bundle) ||
typeof item.guide !== "string" ||
!(MANAGED_SKILL_FILES as readonly string[]).includes(item.guide) ||
!Number.isInteger(item.ordinal) ||
typeof item.section !== "string" ||
!/^[a-f0-9]{64}$/u.test(item.section)
)
return undefined;
return item as LocatorIdentity;
} catch {
return undefined;
}
};
const contextFor = (content: string, matched: readonly string[]): string => {
const lines = content.split("\n");
const heading = lines[0] ?? "";
const terms = new Set(matched);
let bestIndex = -1;
let bestScore = 0;
lines.slice(1).forEach((line, index) => {
const score = words(line).filter((word) => terms.has(word)).length;
if (score > bestScore) {
bestScore = score;
bestIndex = index + 1;
}
});
const excerptStart = bestIndex < 0 ? 1 : Math.max(1, bestIndex - 1);
const excerptEnd = bestIndex < 0 ? 4 : bestIndex + 2;
return [heading, ...lines.slice(excerptStart, excerptEnd)]
.join(" ")
.replaceAll(/\s+/gu, " ")
.slice(0, 500);
};
const compareText = (left: string, right: string): number =>
left < right ? -1 : left > right ? 1 : 0;
+123
View File
@@ -0,0 +1,123 @@
/** Parsed documentation facts; callers never reinterpret Markdown boundaries. */
export interface GuideSection {
readonly guide: string;
readonly heading: string;
readonly content: string;
readonly ordinal: number;
}
// The shipped bundle has tens of headings. These ceilings bound object count
// and UTF-16 text retained by a complete index before allocating section text.
const MAX_SECTIONS = 4096;
// ATX has at most six enclosing levels over the admitted 4 MiB bundle.
const MAX_EXPANDED_CHARACTERS = 6 * 4_194_304;
const blank = (value: string | undefined): boolean =>
value === " " || value === "\t";
/** ATX closing hashes and ASCII whitespace, scanned once from each end. */
const headingTitle = (raw: string): string => {
let start = 0;
let end = raw.length;
while (start < end && blank(raw[start])) start += 1;
while (end > start && blank(raw[end - 1])) end -= 1;
let hashes = end;
while (hashes > start && raw[hashes - 1] === "#") hashes -= 1;
if (hashes < end && (hashes === start || blank(raw[hashes - 1]))) {
end = hashes;
while (end > start && blank(raw[end - 1])) end -= 1;
}
return raw.slice(start, end);
};
interface Heading {
readonly level: number;
readonly title: string;
readonly start: number;
end: number;
}
const atxHeading = (
line: string,
): Pick<Heading, "level" | "title"> | undefined => {
const match = /^ {0,3}(#{1,6})(?:[ \t]+(.*)|$)/u.exec(line);
if (match?.[1] === undefined) return undefined;
const raw = match[2] ?? "";
// Closing hashes need whitespace; literal C# and escaped hashes survive.
const title = headingTitle(raw);
return { level: match[1].length, title };
};
const readHeadings = (
content: string,
remaining: number,
): Heading[] | undefined => {
const headings: Heading[] = [];
const enclosing: Heading[] = [];
let fence: { marker: string; length: number } | undefined;
let start = 0;
while (start < content.length) {
const newline = content.indexOf("\n", start);
const end = newline < 0 ? content.length : newline;
const line = content.slice(start, end);
const fenceMatch = /^ {0,3}(`{3,}|~{3,})(.*)$/u.exec(line);
const marker = fenceMatch?.[1];
if (fence !== undefined) {
if (
marker?.[0] === fence.marker &&
marker.length >= fence.length &&
/^[ \t]*$/u.test(fenceMatch?.[2] ?? "")
)
fence = undefined;
} else if (
marker !== undefined &&
!(marker[0] === "`" && fenceMatch?.[2]?.includes("`"))
) {
fence = { marker: marker[0] ?? "", length: marker.length };
} else {
const parsed = atxHeading(line);
if (parsed !== undefined) {
if (headings.length >= remaining) return undefined;
while ((enclosing.at(-1)?.level ?? 0) >= parsed.level) {
const parent = enclosing.pop();
if (parent !== undefined) parent.end = start;
}
const heading = { ...parsed, start, end: content.length };
headings.push(heading);
enclosing.push(heading);
}
}
start = newline < 0 ? content.length : newline + 1;
}
return headings;
};
/** Bound the whole bundle before materializing an expanded section index. */
export const parseGuidanceSections = (
files: ReadonlyMap<string, string>,
): readonly GuideSection[] | undefined => {
const documents: { guide: string; content: string; headings: Heading[] }[] =
[];
let count = 0;
let expanded = 0;
for (const [guide, content] of files) {
const headings = readHeadings(content, MAX_SECTIONS - count);
if (headings === undefined) return undefined;
count += headings.length;
for (const heading of headings) {
expanded += heading.end - heading.start;
if (expanded > MAX_EXPANDED_CHARACTERS) return undefined;
}
documents.push({ guide, content, headings });
}
const sections: GuideSection[] = [];
for (const { guide, content, headings } of documents)
for (const [ordinal, heading] of headings.entries())
sections.push({
guide,
ordinal,
heading: heading.title,
content: content.slice(heading.start, heading.end).trim(),
});
return sections;
};
+45 -10
View File
@@ -44,6 +44,7 @@ import {
} from "../artifacts/AnalysisCleanupLease.js";
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
import { OwnedStagedOutput } from "../filesystem/OwnedStagedOutput.js";
import { StagedOutputError } from "../filesystem/StagedOutputError.js";
import { absoluteCallerPath } from "../filesystem/callerPath.js";
/** Request control and its owning operation for an interruptible atomic write. */
@@ -399,6 +400,7 @@ const writeTextPartsOwned = async (
path: requestedPath,
cleanup,
published: { ...cause.outcome.value, path: requestedPath },
...stagedOutputDiagnostics(cause.cleanupCause),
}),
cause.lease,
);
@@ -411,16 +413,22 @@ const writeTextPartsOwned = async (
}),
cause.lease,
);
const failure = classifyWriteFailure(cause.outcome.error);
return stagedCleanupFailure(
new EvidenceFileError(
"write",
fileAccessFailureReason(cause.outcome.error),
{
cause,
path: requestedPath,
cleanup,
},
),
new EvidenceFileError("write", failure.reason, {
cause,
path: requestedPath,
cleanup,
...(failure.systemCode === undefined
? {}
: { systemCode: failure.systemCode }),
...stagedOutputDiagnostics(
!cause.outcome.ok &&
cause.outcome.error instanceof StagedOutputError
? cause.outcome.error
: cause.cleanupCause,
),
}),
cause.lease,
);
}
@@ -437,15 +445,33 @@ const writeTextPartsOwned = async (
path: requestedPath,
}),
);
const failure = classifyWriteFailure(cause);
return err(
new EvidenceFileError("write", fileAccessFailureReason(cause), {
new EvidenceFileError("write", failure.reason, {
cause,
path: requestedPath,
...(failure.systemCode === undefined
? {}
: { systemCode: failure.systemCode }),
...stagedOutputDiagnostics(cause),
}),
);
}
};
/** Recognize only host codes that establish a publication resource refusal. */
const classifyWriteFailure = (
cause: unknown,
): {
readonly reason: EvidenceFileError["reason"];
readonly systemCode?: "EFBIG" | "ENOSPC" | "EDQUOT";
} => {
const code = fileErrorCode(cause);
return code === "EFBIG" || code === "ENOSPC" || code === "EDQUOT"
? { reason: "storage-limit", systemCode: code }
: { reason: fileAccessFailureReason(cause) };
};
/** The retained staging lease clears this error only when a retry confirms release. */
const stagedCleanupFailure = <
Failure extends EvidenceFileError | AnalysisCancelledError,
@@ -595,6 +621,15 @@ class StagedOutputCleanupFailure extends Error {
}
}
const stagedOutputDiagnostics = (
cause: unknown,
): {
readonly diagnostics?: Readonly<Record<string, string | number>>;
} => {
if (!(cause instanceof StagedOutputError)) return {};
return { diagnostics: { code: cause.code, ...cause.detail } };
};
const failureMessage = (cause: unknown): string =>
cause instanceof Error ? cause.message || cause.name : String(cause);
@@ -65,6 +65,7 @@ const SELECTED_REGISTRATION_SETTINGS = [
"REA_GHIDRA_PROJECT_PATH",
"REA_GHIDRA_PROGRAM_NAME",
"REA_GHIDRA_NATIVEAOT_JAR",
"REA_HOPPER_CALLER_SCAN_BUDGET_MS",
"REA_HOPPER_STARTUP_TIMEOUT_MS",
"REA_JEB_MCP_URL",
"REA_JEB_MCP_TIMEOUT_MS",
+10 -4
View File
@@ -43,21 +43,27 @@ export const isManagedSkillManifest = (content: string): boolean =>
export const readSkillFile = (
path: string,
resources?: ArtifactResourceScope,
maxBytes = Number.MAX_SAFE_INTEGER,
): Promise<string> =>
readSkillFileSnapshot(path, resources).then(({ bytes }) =>
readSkillFileSnapshot(path, resources, maxBytes).then(({ bytes }) =>
bytes.toString("utf8"),
);
const readSkillFileSnapshot = async (
path: string,
resources?: ArtifactResourceScope,
maxBytes = Number.MAX_SAFE_INTEGER,
): Promise<SkillFileSnapshot> =>
withRegularFile(
path,
async (handle, stats) => {
const bytes = await readBoundedFileBytes(handle, stats.size, {
expectedSize: stats.size,
});
const bytes = await readBoundedFileBytes(
handle,
Math.min(stats.size, maxBytes),
{
expectedSize: stats.size,
},
);
if (bytes === undefined || bytes.length !== stats.size)
throw new RegularFileChangedError(path);
return { bytes, mode: stats.mode & 0o7777, state: stats };
+59 -1
View File
@@ -188,7 +188,8 @@ it("retains unique upstream limits in canonical order across concurrent observat
const analysis: AnalysisOperationPort = {
execute: async (operation, parameters, forwarded) => {
expect(parameters).toEqual({ marker: true });
expect(forwarded).toBe(options);
expect(forwarded).toMatchObject(options);
expect(forwarded?.workScope).toBeDefined();
const call = calls++;
if (call === 0) await gate;
else release();
@@ -621,3 +622,60 @@ it.each([
});
},
);
it("shares temporary provider state only within a single completed workflow", async () => {
const owner = {};
const scopes: Array<
NonNullable<import("./AnalysisProvider.js").ExecutionOptions["workScope"]>
> = [];
const provider = { id: "fixture", name: "Fixture", version: "1" };
const analysis: AnalysisOperationPort = {
execute: async (_operation, _parameters, options) => {
const scope = options?.workScope;
if (scope === undefined) throw new Error("workflow scope missing");
scopes.push(scope);
scope.set(owner, (scope.get<number>(owner) ?? 0) + 1);
return ok(createAnalysisExecution(null, provider));
},
};
const run = () =>
executeComposedWorkflow(analysis, async (observed) => {
await observed.execute("health", {});
await observed.execute("health", {});
expect(scopes.at(-1)?.get(owner)).toBe(2);
return ok(null);
});
await run();
expect(scopes[0]?.get(owner)).toBeUndefined();
scopes[0]?.set(owner, "late completion");
expect(scopes[0]?.get(owner)).toBeUndefined();
await run();
expect(scopes[0]).toBe(scopes[1]);
expect(scopes[0]).not.toBe(scopes[2]);
expect(scopes[2]).toBe(scopes[3]);
});
it("clears workflow state when its callback throws", async () => {
const owner = {};
let captured: import("./AnalysisProvider.js").ExecutionOptions["workScope"];
const analysis: AnalysisOperationPort = {
execute: async (_operation, _parameters, options) => {
captured = options?.workScope;
captured?.set(owner, "in flight");
return ok(
createAnalysisExecution(null, {
id: "fixture",
name: "Fixture",
version: "1",
}),
);
},
};
await expect(
executeComposedWorkflow(analysis, async (observed) => {
await observed.execute("health", {});
throw new Error("failed workflow");
}),
).rejects.toThrow("failed workflow");
expect(captured?.get(owner)).toBeUndefined();
});
+11 -3
View File
@@ -1,3 +1,4 @@
import { AnalysisWorkScope } from "./AnalysisWorkScope.js";
import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
import { jsonStringByteLength } from "../domain/jsonStringByteLength.js";
import { REA_WORKFLOW_PROVIDER } from "../domain/investigationProviders.js";
@@ -91,10 +92,14 @@ export const executeComposedWorkflow = async (
let perItemBytes = 0;
let omittedObservations = 0;
let cachePolicy: "live" | undefined;
const result = await execute({
const workScope = new AnalysisWorkScope();
const observed: AnalysisOperationPort = {
resourceScope: analysis.resourceScope,
execute: async (operation, parameters, options) => {
const execution = await analysis.execute(operation, parameters, options);
const execution = await analysis.execute(operation, parameters, {
...options,
workScope,
});
if (execution.ok && execution.value.evidenceId !== undefined)
evidenceLinks.add(execution.value.evidenceId);
if (execution.ok && execution.value.sourceEvidence !== undefined) {
@@ -139,7 +144,10 @@ export const executeComposedWorkflow = async (
}
return execution;
},
});
};
const result = await Promise.resolve()
.then(() => execute(observed))
.finally(() => workScope.clear());
const sources = sourceLedger.recordsForAnalysis();
if (!result.ok) return withEvidenceSources(result, sources);
for (const source of sources) evidenceLinks.add(source.evidence_id);
@@ -32,6 +32,7 @@ import {
AnalysisSnapshotCache,
SNAPSHOT_CACHE_ENTRY_CEILING,
isSnapshotCacheable,
isSourceEvidenceRetainable,
} from "./AnalysisSnapshotCache.js";
import { EvidenceLedger } from "../investigation/EvidenceLedger.js";
import { enhancedOutputSchemas } from "../../contracts/toolOutputSchemaGroups.js";
@@ -776,6 +777,59 @@ describe("analysis snapshot cache policy", () => {
);
},
);
it("retains caller observations without a document while refusing to replay them", () => {
const descriptor: CapabilityDescriptor = {
operation: "procedure_callers",
provider: ANALYSIS_SNAPSHOT_PROVIDER,
available: true,
reason: null,
effects: {
mutatesArtifact: false,
launchesProcess: false,
mayShowUi: false,
mayAccessNetwork: false,
mayWriteFilesystem: false,
changesPermissions: false,
requiresRoot: false,
},
limitations: [],
};
const parameters = { procedure: "0x1000" };
expect(
isSourceEvidenceRetainable("procedure_callers", descriptor, parameters),
).toBe(true);
expect(
isSnapshotCacheable("procedure_callers", descriptor, parameters),
).toBe(false);
expect(
isSnapshotCacheable("procedure_callers", descriptor, {
...parameters,
document: "fixture",
}),
).toBe(true);
const callees = { ...descriptor, operation: "procedure_callees" as const };
expect(
isSourceEvidenceRetainable("procedure_callees", callees, parameters),
).toBe(true);
const pseudocode = {
...descriptor,
operation: "procedure_pseudo_code" as const,
};
expect(
isSourceEvidenceRetainable(
"procedure_pseudo_code",
pseudocode,
parameters,
),
).toBe(false);
expect(
isSourceEvidenceRetainable("procedure_pseudo_code", pseudocode, {
...parameters,
document: "fixture",
}),
).toBe(true);
});
});
describe("analysis snapshot cache partitioning", () => {
@@ -87,8 +87,28 @@ const DOCUMENT_SCOPED_OPERATIONS: ReadonlySet<string> = new Set(
/** Maximum duplicated result bindings retained across direct and workflow queries. */
export const SNAPSHOT_CACHE_ENTRY_CEILING = 10_000;
/** Whether an operation is immutable and independent of provider UI state. */
export const isSnapshotCacheable = (
const replayDocumentSelected = (
operation: AnalysisOperation,
parameters: Readonly<Record<string, JsonValue>>,
): boolean =>
!DOCUMENT_SCOPED_OPERATIONS.has(operation) ||
typeof parameters.document === "string";
/**
* Caller and callee walks omit `document`. Their observations are still
* Evidence. Other document-scoped reads, including `procedure_pseudo_code`
* from `decompile_all`, already keep each body in the file-backed export.
* Copying those bodies into the in-memory ledger would retain the export.
*/
const DOCUMENTLESS_SOURCE_EVIDENCE_OPERATIONS: ReadonlySet<AnalysisOperation> =
new Set(["procedure_callers", "procedure_callees"]);
/**
* Whether a source observation is safe to retain as Evidence.
* A missing document selector blocks snapshot replay. Only caller and callee
* walks stay in the ledger without one.
*/
export const isSourceEvidenceRetainable = (
operation: AnalysisOperation,
descriptor: CapabilityDescriptor | undefined,
parameters: Readonly<Record<string, JsonValue>>,
@@ -96,15 +116,25 @@ export const isSnapshotCacheable = (
!STATEFUL_OPERATIONS.has(operation) &&
!RESOURCE_SCOPED_OPERATIONS.has(operation) &&
(!DOCUMENT_SCOPED_OPERATIONS.has(operation) ||
typeof parameters.document === "string") &&
typeof parameters.document === "string" ||
DOCUMENTLESS_SOURCE_EVIDENCE_OPERATIONS.has(operation)) &&
(!CURSOR_DEFAULT_OPERATIONS.has(operation) ||
typeof parameters.address === "string") &&
descriptor?.effects.mutatesArtifact === false &&
descriptor.cachePolicy !== "live" &&
(descriptor.cachePolicy === "snapshot" ||
descriptor.effects.mayWriteFilesystem === false) &&
descriptor.effects.changesPermissions === false;
/** Whether an observation can additionally be replayed as a snapshot. */
export const isSnapshotCacheable = (
operation: AnalysisOperation,
descriptor: CapabilityDescriptor | undefined,
parameters: Readonly<Record<string, JsonValue>>,
): descriptor is CapabilityDescriptor =>
isSourceEvidenceRetainable(operation, descriptor, parameters) &&
replayDocumentSelected(operation, parameters) &&
descriptor.cachePolicy !== "live";
/** Windows Ghidra must admit the selected source through its native snapshot owner. */
export const requiresNativeSourceAdmissionBeforeReplay = (
providerId: string | undefined,
+97 -26
View File
@@ -44,6 +44,7 @@ import { InvestigationRecords } from "../investigation/InvestigationRecords.js";
import { recordEvidenceSources } from "../investigation/recordEvidenceSources.js";
import { BinarySessionRecords } from "./BinarySessionRecords.js";
import { binarySessionStatus } from "./BinarySessionStatus.js";
import { isSourceEvidenceRetainable } from "./AnalysisSnapshotCache.js";
import type { BinarySessionDetail } from "../../contracts/sessionStatusContract.js";
import {
resolveSessionOpen,
@@ -59,10 +60,11 @@ import {
} from "./BinarySessionExecution.js";
import { closeAnalysisClient } from "../AnalysisClientCleanup.js";
import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorCleanup.js";
import { analysisErrorWithCancellation } from "../../domain/analysisErrorCancellation.js";
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
import { profilelessNativeDispatchProfile } from "./ProfilelessNativeWorkflowSnapshot.js";
import type { AnalysisSnapshot } from "../../domain/analysisSnapshot.js";
import { AnalysisSnapshotMismatchError } from "../../domain/evidenceErrors.js";
import { ResourceSnapshotMismatchError } from "../../domain/evidenceErrors.js";
import {
parseBinaryTarget,
type BinaryTargetResolverOptions,
@@ -175,6 +177,7 @@ export class BinarySession
| {
readonly run: SessionRun;
readonly retainDocument: boolean;
readonly failures: Set<AnalysisError>;
}
| undefined;
#transition: Promise<void> = Promise.resolve();
@@ -715,7 +718,7 @@ export class BinarySession
_path: string | undefined,
signal: AbortSignal | undefined,
operation: () => Promise<Result<Value, AnalysisError>>,
retirementFailure: (
completionFailure: (
value: Result<Value, AnalysisError>,
error: AnalysisError,
) => AnalysisError,
@@ -749,14 +752,47 @@ export class BinarySession
const recovered = await cleanupRecovery(
this.#externalMutationCleanupFailure,
);
if (!recovered.ok) return recovered;
if (!recovered.ok)
return err(
isAborted(signal)
? analysisErrorWithCancellation(
recovered.error,
new AnalysisCancelledError(operationName, {
cause: signal?.reason,
}),
)
: recovered.error,
);
if (this.#externalMutationCleanupFailure.cleanupIncomplete)
return err(this.#externalMutationCleanupFailure);
return err(
isAborted(signal)
? analysisErrorWithCancellation(
this.#externalMutationCleanupFailure,
new AnalysisCancelledError(operationName, {
cause: signal?.reason,
}),
)
: this.#externalMutationCleanupFailure,
);
this.#externalMutationCleanupFailure = undefined;
}
const pendingCleanup = await this.#retryPendingCleanup();
if (!pendingCleanup.ok) return pendingCleanup;
if (!pendingCleanup.ok)
return err(
isAborted(signal)
? analysisErrorWithCancellation(
pendingCleanup.error,
new AnalysisCancelledError(operationName, {
cause: signal?.reason,
}),
)
: pendingCleanup.error,
);
if (isAborted(signal))
return err(
new AnalysisCancelledError(operationName, { cause: signal?.reason }),
);
const activeAtDispatch = this.#active;
if (activeAtDispatch?.route.binding?.identity.id === "ghidra")
@@ -774,10 +810,39 @@ export class BinarySession
const value = await operation();
this.#externalMutationCleanupFailure =
!value.ok && value.error.cleanupIncomplete ? value.error : undefined;
if (!retireTarget) return ok(value);
const retired = await this.#closeActive({}, false);
if (!retired.ok) return err(retirementFailure(value, retired.error));
if (retireTarget) {
const retired = await this.#closeActive({}, false);
if (!retired.ok) {
const primary = completionFailure(value, retired.error);
const failure =
retired.error.cleanupIncomplete && primary !== retired.error
? analysisErrorWithCleanupFailure(
primary,
retired.error,
operationName,
).bindCleanupDependencies([primary, retired.error])
: primary;
return err(
isAborted(signal)
? analysisErrorWithCancellation(
failure,
new AnalysisCancelledError(operationName, {
cause: signal?.reason,
}),
)
: failure,
);
}
}
if (isAborted(signal))
return err(
completionFailure(
value,
new AnalysisCancelledError(operationName, {
cause: signal?.reason,
}),
),
);
return ok(value);
});
}
@@ -958,6 +1023,7 @@ export class BinarySession
this.#pendingCleanup = {
run,
retainDocument: options.retainDocument === true,
failures: new Set(),
};
return this.#retryPendingCleanup(
options.progress,
@@ -983,10 +1049,19 @@ export class BinarySession
{
...(progress === undefined ? {} : { progress }),
...(pending.retainDocument ? { retainDocument: true } : {}),
...(onCleanupFailure === undefined ? {} : { onCleanupFailure }),
onCleanupFailure: (failure) => {
if (failure.cleanupIncomplete) pending.failures.add(failure);
onCleanupFailure?.(failure);
},
},
);
if (closed.ok) this.#pendingCleanup = undefined;
if (closed.ok) {
for (const failure of pending.failures)
if (!failure.hasCleanupOwner) failure.resolveCleanup();
this.#pendingCleanup = undefined;
} else if (closed.error.cleanupIncomplete) {
pending.failures.add(closed.error);
}
return closed;
}
@@ -1074,21 +1149,13 @@ export class BinarySession
override exportAnalysisSnapshot() {
if (this.#activeBundleResources !== undefined)
return err(
new AnalysisSnapshotMismatchError(
"Analysis snapshots do not apply to a resource-only app session because it has no executable identity or replay binding.",
),
);
return err(new ResourceSnapshotMismatchError("export"));
return super.exportAnalysisSnapshot();
}
override importAnalysisSnapshot(snapshot: AnalysisSnapshot) {
if (this.#activeBundleResources !== undefined)
return err(
new AnalysisSnapshotMismatchError(
"Analysis snapshots do not apply to a resource-only app session because it has no executable identity or replay binding.",
),
);
return err(new ResourceSnapshotMismatchError("import"));
return super.importAnalysisSnapshot(snapshot);
}
@@ -1104,7 +1171,7 @@ export class BinarySession
execute(
name: Parameters<AnalysisOperationPort["execute"]>[0],
arguments_: Readonly<Record<string, JsonValue>>,
options?: Pick<ExecutionOptions, "signal" | "selectorOrigin">,
options?: ExecutionOptions,
): Promise<Result<AnalysisExecution, AnalysisError>> {
const generation = this.#transitionGeneration;
const call = this.#execute(name, arguments_, options, this.#transition);
@@ -1116,7 +1183,7 @@ export class BinarySession
executeIfProviderReady(
name: Parameters<AnalysisOperationPort["execute"]>[0],
arguments_: Readonly<Record<string, JsonValue>>,
options?: Pick<ExecutionOptions, "signal" | "selectorOrigin">,
options?: Pick<ExecutionOptions, "signal" | "selectorOrigin" | "workScope">,
): Promise<Result<AnalysisExecution, AnalysisError> | undefined> {
const active = this.#active;
if (
@@ -1175,7 +1242,7 @@ export class BinarySession
async #execute(
name: Parameters<AnalysisOperationPort["execute"]>[0],
arguments_: Readonly<Record<string, JsonValue>>,
options: Pick<ExecutionOptions, "signal" | "selectorOrigin"> | undefined,
options: ExecutionOptions | undefined,
transition: Promise<void>,
): Promise<Result<AnalysisExecution, AnalysisError>> {
const transitioned = await this.#waitForTransition(
@@ -1255,7 +1322,11 @@ export class BinarySession
);
if (!sourcesRecorded.ok) return sourcesRecorded;
}
if (execution.ok && cacheable && profile !== undefined) {
if (
execution.ok &&
isSourceEvidenceRetainable(name, capability, arguments_) &&
profile !== undefined
) {
const evidence = createAnalysisExecutionEvidence(
name,
arguments_,
@@ -1272,7 +1343,7 @@ export class BinarySession
const recorded = this.recordEvidence(evidence);
if (!recorded.ok) return recorded;
const retained = { ...execution.value, evidenceId: evidence.evidence_id };
if (execution.value.cachePolicy !== "live")
if (cacheable && execution.value.cachePolicy !== "live")
this.recordSnapshot({
target: active.target,
profile,
@@ -14,7 +14,10 @@ import { AnalysisProviderRegistry } from "./AnalysisProviderRegistry.js";
import { BinarySession } from "./BinarySession.js";
import { SessionProviderRouter } from "./SessionProviderRouter.js";
import { bundleResourceRootSchema } from "../../domain/apple/bundleResourceRoot.js";
import { AnalysisSnapshotMismatchError } from "../../domain/evidenceErrors.js";
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
import { analysisErrorRemediationAction } from "../../domain/analysisErrorPresentation.js";
import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
import { ResourceSnapshotMismatchError } from "../../domain/evidenceErrors.js";
import type { AnalysisSnapshot } from "../../domain/analysisSnapshot.js";
import { ProviderCleanupError } from "../../domain/providerCleanupError.js";
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
@@ -36,6 +39,24 @@ const binaryIdentity: ProviderIdentity = {
};
const temporaryDirectories: string[] = [];
const expectResourceSnapshotRefusal = (
error: AnalysisError,
operation: "export" | "import",
): void => {
expect(error).toBeInstanceOf(ResourceSnapshotMismatchError);
expect(error).toMatchObject({
_tag: "EvidenceIntegrityError",
operation,
});
const projected = projectAnalysisError(error);
expect(projected.code).toBe("evidence_integrity_mismatch");
expect(projected.category).toBe("integrity_mismatch");
expect(projected.retryable).toBe(false);
expect(projected.remediation).toEqual({
action: analysisErrorRemediationAction(error),
});
};
afterEach(async () => {
await Promise.all(
temporaryDirectories
@@ -137,13 +158,17 @@ it("opens a resource-only bundle without executable identity and keeps snapshot
resource_root: { canonical_path: opened.value.canonicalPath },
});
const snapshot = session.exportAnalysisSnapshot();
expect(snapshot.ok).toBe(false);
if (!snapshot.ok)
expect(snapshot.error).toBeInstanceOf(AnalysisSnapshotMismatchError);
if (snapshot.ok)
throw new Error("resource session exported a binary snapshot");
expectResourceSnapshotRefusal(snapshot.error, "export");
const imported = session.importAnalysisSnapshot({} as AnalysisSnapshot);
expect(imported.ok).toBe(false);
if (!imported.ok)
expect(imported.error).toBeInstanceOf(AnalysisSnapshotMismatchError);
if (imported.ok)
throw new Error("resource session imported a binary snapshot");
expectResourceSnapshotRefusal(imported.error, "import");
expect(await session.status()).toMatchObject({
open: true,
session_kind: "bundle_resources",
});
const closedResult = await session.close();
expect(closedResult.ok).toBe(true);
expect(closed).toBe(true);
@@ -73,6 +73,47 @@ it("preserves an explicitly unbound provider observation", async () => {
}
});
it("retains live source Evidence without replaying a prior observation", async () => {
const { session, deep } = openSession({ live: true });
try {
expect((await session.open(process.execPath)).ok).toBe(true);
expect(
(
await session.execute("address_name", {
document: "0",
address: "0x9999",
})
).ok,
).toBe(true);
const child = deep.created[0];
if (child === undefined) throw new Error("expected provider child");
let reads = 0;
child.execute = () =>
Promise.resolve(
ok(createAnalysisExecution(`live ${++reads}`, deepIdentity)),
);
const parameters = { document: "0", address: "0x1000" };
const first = await session.execute("address_name", parameters);
const second = await session.execute("address_name", parameters);
expect(first.ok && first.value.result).toBe("live 1");
expect(second.ok && second.value.result).toBe("live 2");
if (!first.ok || !second.ok) return;
expect(first.value.evidenceId).toBeDefined();
expect(second.value.evidenceId).toBeDefined();
expect(first.value.evidenceId).not.toBe(second.value.evidenceId);
expect(session.exportEvidenceBundle().records).toEqual(
expect.arrayContaining([
expect.objectContaining({ normalized_result: "live 1" }),
expect.objectContaining({ normalized_result: "live 2" }),
]),
);
const snapshot = session.exportAnalysisSnapshot();
if (snapshot.ok) expect(snapshot.value.entries).toHaveLength(0);
} finally {
await session.close();
}
});
it("keeps unbound observations out of persisted target snapshots", async () => {
const { session, deep } = openSession();
const directory = await createTestTempDirectory("rea-unbound-snapshot-");
@@ -458,6 +499,7 @@ const openSession = (
failOperationInitially?: boolean;
beforeFirstClose?: () => Promise<void>;
explicitNullSubject?: boolean;
live?: boolean;
} = {},
): {
session: BinarySession;
@@ -493,6 +535,7 @@ const killableDeepProvider = (options: {
failOperationInitially?: boolean;
beforeFirstClose?: () => Promise<void>;
explicitNullSubject?: boolean;
live?: boolean;
}): {
provider: AnalysisProviderCandidate;
created: KillableChild[];
@@ -503,7 +546,12 @@ const killableDeepProvider = (options: {
const withOperationHealth = options.operationHealth !== false;
const provider: AnalysisProviderCandidate = {
identity: () => deepIdentity,
capabilities: () => [capability(deepIdentity, "address_name")],
capabilities: () => [
{
...capability(deepIdentity, "address_name"),
...(options.live ? { cachePolicy: "live" as const } : {}),
},
],
inspectAvailability: (): ProviderAvailability => ({
status: "available",
code: null,
+1 -1
View File
@@ -47,7 +47,7 @@ export interface BinarySessionPort
_path: string | undefined,
signal: AbortSignal | undefined,
operation: () => Promise<Result<Value, AnalysisError>>,
retirementFailure: (
completionFailure: (
value: Result<Value, AnalysisError>,
error: AnalysisError,
) => AnalysisError,
@@ -24,6 +24,13 @@ export const BUNDLE_RESOURCE_OPERATIONS = new Set<AnalysisOperation>([
...BUNDLE_ARTIFACT_RESOURCE_OPERATIONS,
]);
/** Share the dispatch owner's resource-session operation policy with discovery. */
export const isBundleResourceOperation = (name: string): boolean => {
for (const operation of BUNDLE_RESOURCE_OPERATIONS)
if (operation === name) return true;
return false;
};
/** Provider-specific owner of the four read-only app-resource operations. */
export interface BundleResourceClientFactory {
createClient(
@@ -37,6 +37,7 @@ import {
observedEntity,
pathValues,
sortedOrder,
storedUpdate,
unloadable,
heldEntity,
updateTarget,
@@ -210,12 +211,16 @@ export const answerBulk = (
);
if (!answer.ok) return answer;
if (update) {
const stored = written.value.map((entity) => ({
...entity,
fields: storedUpdate(serving, plan, entity.fields),
}));
const updated = serving.store.updateAll(
written.value,
stored,
sortedOrder(
serving,
plan,
written.value.map((after, index) => ({
stored.map((after, index) => ({
before: targets.value[index]!.entity,
after,
})),
@@ -0,0 +1,101 @@
import { describe, expect, it } from "vitest";
import {
ADDED_EMPTY_REFERENCE_ROWS,
BOOKMARK_TOGGLE_ROWS,
ITEM_LOCATIONS,
ITEM_REFERENCES_ROWS,
ITEM_TAGS,
} from "../../../tests/fixtures/environmentStateItemWrites.js";
import { FILED_ROWS } from "../../../tests/fixtures/environmentStateDerivedValues.js";
import { servedFrom } from "../../../tests/fixtures/environmentStateServing.js";
describe("served item writes", () => {
it("answers a create with the parent and tags its ids name, in the order sent", () => {
const items = servedFrom(ITEM_REFERENCES_ROWS);
const body = items.answer("post_items", {
body: {
name: "Snow shovel",
parentId: ITEM_LOCATIONS[0]!["id"]!,
tagIds: [ITEM_TAGS[1]!["id"]!, ITEM_TAGS[0]!["id"]!],
},
}).body;
expect(body).toMatchObject({
name: "Snow shovel",
parent: ITEM_LOCATIONS[0],
tags: [ITEM_TAGS[1], ITEM_TAGS[0]],
});
});
it("answers a create naming no parent and no tags with none", () => {
const items = servedFrom(ITEM_REFERENCES_ROWS);
expect(
items.answer("post_items", { body: { name: "Spare fuse" } }).body,
).toMatchObject({
name: "Spare fuse",
parent: null,
tags: [],
});
});
it("answers a create sending no parent and no tags as a recorded create sent them", () => {
const items = servedFrom(ITEM_REFERENCES_ROWS);
expect(
items.answer("post_items", {
body: { name: "Spare fuse", parentId: null, tagIds: [] },
}).body,
).toMatchObject({ name: "Spare fuse", parent: null, tags: [] });
});
it("answers a create leaving out a parent the first create left out with none, and one sending null with null", () => {
const items = servedFrom(ADDED_EMPTY_REFERENCE_ROWS);
const left = items.answer("post_items", { body: { name: "Spare fuse" } })
.body as Record<string, unknown>;
expect(left).toMatchObject({ name: "Spare fuse" });
expect(Object.hasOwn(left, "parent")).toBe(false);
expect(
items.answer("post_items", {
body: { name: "Spare bulb", parentId: null },
}).body,
).toMatchObject({ name: "Spare bulb", parent: null });
expect(
items.answer("post_items", {
body: { name: "Spare cord", parentId: ITEM_LOCATIONS[0]!["id"]! },
}).body,
).toMatchObject({ name: "Spare cord", parent: ITEM_LOCATIONS[0] });
});
it("refuses a create naming no shelf when every recorded create named one", () => {
const books = servedFrom(FILED_ROWS);
expect(
books.refusal("post_books", { body: { name: "Loose", shelf_id: null } }),
).toMatchObject({ message: expect.stringContaining("shelf_id") });
});
it("refuses a create naming a tag the environment does not hold", () => {
const items = servedFrom(ITEM_REFERENCES_ROWS);
expect(
items.refusal("post_items", {
body: {
name: "Mystery",
tagIds: ["3b9f2a10-0009-4000-8000-000000000009"],
},
}),
).toMatchObject({ message: expect.stringContaining("tagIds") });
});
it("flips an entry's flag on each body-less call", () => {
const entries = servedFrom(BOOKMARK_TOGGLE_ROWS);
const starred = () =>
(
entries.answer("get_entries_by_id", { id: "2" }).body as {
starred: boolean;
}
).starred;
expect(starred()).toBe(false);
entries.answer("put_entries_bookmark_by_id", { id: "2" });
expect(starred()).toBe(true);
entries.answer("put_entries_bookmark_by_id", { id: "2" });
expect(starred()).toBe(false);
});
});
@@ -0,0 +1,123 @@
import { describe, expect, it } from "vitest";
import {
DONE_CLOCK_ROWS,
FEWER_MEMBERS_ROWS,
NUMBERED_THEN_BLANK_ROWS,
NUMBERED_UNLESS_SENT_ROWS,
OWNER_CLEARED_THEN_NAMED_ROWS,
OWNER_NAMED_THEN_CLEARED_ROWS,
PUT_CLEARS_ROWS,
} from "../../../tests/fixtures/environmentStateVariedCalls.js";
import { servedFrom } from "../../../tests/fixtures/environmentStateServing.js";
describe("served writes whose calls send different members", () => {
it("numbers a created bundle that leaves its order out, and keeps one a call sends", () => {
const bundles = servedFrom(NUMBERED_UNLESS_SENT_ROWS);
expect(
bundles.answer("post_bundles", { body: { name: "Echo" } }).body,
).toMatchObject({ name: "Echo", order: 2 });
expect(
bundles.answer("post_bundles", { body: { name: "Foxtrot", order: 9 } })
.body,
).toMatchObject({
name: "Foxtrot",
order: 9,
});
});
it("numbers past an order a call sent, even once that bundle is gone", () => {
const bundles = servedFrom(NUMBERED_UNLESS_SENT_ROWS);
bundles.answer("post_bundles", { body: { name: "Echo" } });
bundles.answer("post_bundles", { body: { name: "Foxtrot", order: 9 } });
bundles.store.removeAll(
bundles.store
.ofType("/api/bundles")
.filter((entity) => entity.fields["order"] === 9),
);
expect(
bundles.answer("post_bundles", { body: { name: "Golf" } }).body,
).toMatchObject({ name: "Golf", order: 10 });
});
it("clears a member a PUT leaves out, as the application did, and keeps one it sends", () => {
const tags = servedFrom(PUT_CLEARS_ROWS);
expect(
tags.answer("put_tags_by_id", { id: "2", body: { name: "Receipts" } })
.body,
).toMatchObject({
name: "Receipts",
owner: null,
});
expect(tags.answer("get_tags_by_id", { id: "2" }).body).toMatchObject({
owner: null,
});
expect(
tags.answer("put_tags_by_id", {
id: "1",
body: { name: "Mail", owner: 3 },
}).body,
).toMatchObject({
name: "Mail",
owner: 3,
});
});
it.each([
["named first", OWNER_NAMED_THEN_CLEARED_ROWS],
["cleared first", OWNER_CLEARED_THEN_NAMED_ROWS],
])(
"clears a referenced owner a PUT leaves out, and stores the one a PUT names by its id: %s",
(_order, rows) => {
const tags = servedFrom(rows);
expect(
tags.answer("put_tags_by_id", { id: "1", body: { name: "Receipts" } })
.body,
).toMatchObject({ name: "Receipts", owner: null });
expect(
tags.answer("put_tags_by_id", {
id: "2",
body: { name: "Mail", ownerId: 4 },
}).body,
).toMatchObject({ name: "Mail", owner: { id: 4, name: "Bo" } });
},
);
it("answers an update with its clock spelled as the recorded answer and stores it as reads spell it", () => {
const tasks = servedFrom(DONE_CLOCK_ROWS);
const answered = tasks.answer("patch_tasks_by_id", {
id: "1",
body: { done: true },
}).body as { done_at: string };
expect(answered.done_at).toMatch(/\.\d{9}Z$/u);
const read = tasks.answer("get_tasks_by_id", { id: "1" }).body as {
done_at: string;
};
expect(read.done_at).toMatch(/:\d{2}Z$/u);
});
it("serves an update that sends only the member every recorded update sent", () => {
const contacts = servedFrom(FEWER_MEMBERS_ROWS);
expect(
contacts.answer("put_contacts_by_id", {
id: "1",
body: { email: "ada.f@example.test" },
}).body,
).toMatchObject({ name: "Ada Finch", email: "ada.f@example.test" });
expect(
contacts.refusal("put_contacts_by_id", {
id: "2",
body: { city: "Kumasi" },
}),
).toMatchObject({
message: expect.stringContaining("email"),
});
});
it("numbers a created bundle that leaves its order out, though a recorded create sent it empty", () => {
const bundles = servedFrom(NUMBERED_THEN_BLANK_ROWS);
expect(
bundles.answer("post_bundles", { body: { name: "Echo" } }).body,
).toMatchObject({ name: "Echo", order: "2" });
});
});
@@ -23,6 +23,7 @@ import {
nestedClockValue,
nextMadeCount,
unenveloped,
readSpelledClocks,
} from "../../domain/analysisView/environmentStateWriteStamps.js";
import {
anyLinkedSource,
@@ -41,6 +42,7 @@ import {
generatedIntegerDigits,
seededInteger,
seededLike,
sequenceNumber,
spelledLike,
startedSequence,
} from "../../domain/analysisView/environmentStateWriteValues.js";
@@ -230,6 +232,12 @@ export const generator = (
): {
readonly forTarget: (target: GeneratedTarget) => GenerateValue;
readonly taken: Map<string, number>;
/** Count a number the call sent for a sequence, as replay does, so later generated ones follow it. */
readonly sentSequence: (
target: string,
rule: Extract<GeneratedRule, { rule: "sequence" }>,
value: JsonValue | undefined,
) => void;
} => {
const { store } = serving;
const write = store.pendingWrite();
@@ -239,6 +247,11 @@ export const generator = (
taken.set(key, value);
return value;
};
const sequenceKey = (
rule: Extract<GeneratedRule, { rule: "sequence" }>,
target: string,
) =>
`${request.tool.name} ${rule.field} ${rule.scope === "collection" ? target : ""}`;
const ofType = store.ofType(request.plan.target ?? "");
// A sequence never reuses a value the state started with, even after its entity was deleted.
const started = store.startedWith(request.plan.target ?? "");
@@ -291,7 +304,7 @@ export const generator = (
spelledLike(
exemplar,
sequence(
`${request.tool.name} ${rule.field} ${rule.scope === "collection" ? target.name : ""}`,
sequenceKey(rule, target.name),
startedSequence(
nextHeld(
rule.scope === "collection"
@@ -361,7 +374,17 @@ export const generator = (
);
}
};
return { forTarget: (target) => (rule) => generate(target, rule), taken };
return {
forTarget: (target) => (rule) => generate(target, rule),
taken,
sentSequence: (target, rule, value) => {
const number = sequenceNumber(value);
if (number === undefined) return;
const key = sequenceKey(rule, target);
// commitWrite keeps the larger of this and the values taken before.
taken.set(key, Math.max(taken.get(key) ?? number, number));
},
};
};
/** A write's answer on its recorded response's status and content type: a rendered body, none, or the recorded one. */
@@ -556,6 +579,17 @@ const answerCreate = (
idField,
);
if (!fields.ok) return err(refused(tool, fields.error));
for (const rule of plan.rules)
if (
rule.rule === "sequence" &&
rule.sent === true &&
Object.hasOwn(call.body ?? {}, rule.field)
)
values.sentSequence(
collection,
rule,
ownMember(fields.value, rule.field),
);
const field = unloadableMember(fields.value);
if (field !== undefined) return err(unloadable(tool, `Field ${field}`));
const id = scalarText(ownMember(fields.value, idField));
@@ -665,7 +699,10 @@ const answerUpdate = (
},
);
if (!answer.ok) return answer;
const after = updatedEntity(entity, written.value);
const after = updatedEntity(
entity,
storedUpdate(serving, plan, written.value),
);
const updated = serving.store.updateAll(
[after],
sortedOrder(serving, plan, [{ before: entity, after }]),
@@ -675,6 +712,20 @@ const answerUpdate = (
return answer;
};
/** An update's fields as the state stores them: its clocks spelled as the plan's read-back spelled them. */
export const storedUpdate = (
serving: StateServing,
plan: StateWritePlan,
written: JsonObject,
): JsonObject =>
readSpelledClocks(
plan.rules,
written,
plan.observation?.read === undefined
? undefined
: recordedObject(serving, plan.observation.read),
);
/** The collection's members in the order a sorted collection holds them after these writes; empty when the plan keeps them in place. */
export const sortedOrder = (
serving: StateServing,
@@ -1,6 +1,7 @@
import type { SemanticFlowProjectionContext } from "./JavaScriptSemanticGraphFlowProjection.js";
import { createSemanticNodeRangeLookup } from "./JavaScriptSemanticGraphProjection.js";
import { sourceRangesEqual } from "../../domain/javascript/javascriptStaticAnalysisHelpers.js";
import { javascriptSourceRangeKey } from "../../domain/javascript/javascriptSourceCoordinates.js";
import {
addSemanticGraphRelation,
addSemanticGraphUnknown,
@@ -13,6 +14,11 @@ export function* projectSemanticAssignments(
): Generator<void, void> {
const readsWithin = createSemanticNodeRangeLookup(context.readNodes);
const writesWithin = createSemanticNodeRangeLookup(context.writeNodes);
const expressions = new Set(
context.ir.expressionFlows.map((flow) =>
javascriptSourceRangeKey(flow.location),
),
);
for (const [index, flow] of context.ir.assignmentFlows.entries()) {
if (index % 64 === 0) yield;
const owner =
@@ -22,9 +28,11 @@ export function* projectSemanticAssignments(
const writes = writesWithin(flow.target).filter((node) =>
sourceRangesEqual(node.identity.source_range, flow.target),
);
const reads = readsWithin(flow.source).filter(
(node) => node.function_node_id === owner,
);
const reads = expressions.has(javascriptSourceRangeKey(flow.source))
? []
: readsWithin(flow.source).filter(
(node) => node.function_node_id === owner,
);
// Compound assignments consume the previous value as well as the RHS.
if (flow.operator !== "=")
reads.push(
@@ -31,7 +31,7 @@ import { getterExpansionSource } from "../../../tests/fixtures/javascriptGetterE
const SHA256 = "a".repeat(64);
it("retains direct call literals and resolves literal queries to their call site", () => {
it("retains direct call literals and resolves literal queries to their argument", () => {
const source =
'api.rpc("proc_0", 7, false, null, dynamic, ...values, `fixed`, -3);';
const graph = graphFor(source);
@@ -57,7 +57,10 @@ it("retains direct call literals and resolves literal queries to their call site
direction: "forward-influence",
}),
);
expect(trace.seed_node_ids).toContain(call.node_id);
expect(
trace.nodes.find((node) => node.node_id === trace.seed_node_ids[0])?.label,
).toBe("argument 0");
expect(trace.nodes.some((node) => node.node_id === call.node_id)).toBe(true);
});
it("reports the primitive byte bound when a direct argument is too large", () => {
@@ -14,9 +14,11 @@ import {
} from "../../domain/javascript/javascriptSemanticGraphSchemas.js";
import type {
JavaScriptSemanticCallArgument,
JavaScriptSemanticCallReturnFlow,
JavaScriptSemanticIr,
} from "../../domain/javascript/javascriptSemanticIr.js";
import { sourceRangesEqual as rangesEqual } from "../../domain/javascript/javascriptStaticAnalysisHelpers.js";
import { javascriptSourceRangeKey } from "../../domain/javascript/javascriptSourceCoordinates.js";
import type { JavaScriptApplicationGraph } from "../../domain/javascript/javascriptApplicationGraph.js";
import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js";
import {
@@ -31,6 +33,7 @@ import {
} from "./JavaScriptSemanticGraphDataProjection.js";
import { projectSemanticResources } from "./JavaScriptSemanticGraphResourceProjection.js";
import { projectSemanticAssignments } from "./JavaScriptSemanticGraphAssignmentProjection.js";
import { projectSemanticExpressionInfluence } from "./JavaScriptSemanticGraphExpressionProjection.js";
import { projectSemanticObjects } from "./JavaScriptSemanticGraphObjectProjection.js";
import { projectSemanticValues } from "./JavaScriptSemanticGraphValueProjection.js";
import {
@@ -935,6 +938,11 @@ function* projectDefinitionsAndReferences(
}
const projectCalls = (context: FileContext): void => {
const expressions = new Set(
context.ir.expressionFlows.map((flow) =>
javascriptSourceRangeKey(flow.location),
),
);
for (const call of context.ir.callSites) {
const callNode = context.callSiteNodes.get(call.callSiteId);
if (callNode === undefined) continue;
@@ -959,7 +967,40 @@ const projectCalls = (context: FileContext): void => {
`${call.callSiteId}\u0000${String(argument.index)}`,
argumentNode,
);
const references = context.referenceNodesWithin(argument.location);
if (!argument.spread && Object.hasOwn(argument, "literal"))
addRelation(context.state, {
source: argumentNode,
target: callNode,
relation: "influences",
resolution: "resolved",
});
if (
call.resolution !== "exact" &&
call.resultUsed &&
argumentNode !== null
)
addSemanticGraphUnknown(context.state, {
node_id: argumentNode.node_id,
family: "data-flow",
relation_kinds: ["influences"],
reason: "dynamic-call",
detail:
"The dynamic call's return dependency on this argument is unknown.",
candidate_node_ids: [],
evidence: unknownSemanticEvidence(context.file, argument.location),
});
const references =
argument.valueExpressionLocation !== null &&
expressions.has(
javascriptSourceRangeKey(argument.valueExpressionLocation),
)
? []
: context
.referenceNodesWithin(argument.location)
.filter(
(reference) =>
reference.function_node_id === callNode.function_node_id,
);
for (const reference of references)
addRelation(context.state, {
source: reference,
@@ -994,20 +1035,46 @@ const projectCalls = (context: FileContext): void => {
resolution: context.callResolutions.get(flow.callSiteId) ?? "candidate",
});
}
for (const flow of context.ir.callReturnFlows)
const returnFlowsByCallSite = new Map<
string,
JavaScriptSemanticCallReturnFlow[]
>();
for (const flow of context.ir.callReturnFlows) {
const grouped = returnFlowsByCallSite.get(flow.callSiteId);
if (grouped === undefined)
returnFlowsByCallSite.set(flow.callSiteId, [flow]);
else grouped.push(flow);
addRelation(context.state, {
source: context.returnSiteNodes.get(flow.returnSiteId),
target: context.callSiteNodes.get(flow.callSiteId),
relation: "returns-to-call",
resolution: context.callResolutions.get(flow.callSiteId) ?? "candidate",
});
for (const flow of context.ir.callResultFlows)
addRelation(context.state, {
source: context.callSiteNodes.get(flow.callSiteId),
target: context.bindingNodes.get(flow.bindingId),
relation: "aliases",
resolution: context.callResolutions.get(flow.callSiteId) ?? "candidate",
});
}
for (const flow of context.ir.callResultFlows) {
const resolution =
context.callResolutions.get(flow.callSiteId) ?? "candidate";
const target = context.bindingNodes.get(flow.bindingId);
const returns = returnFlowsByCallSite.get(flow.callSiteId) ?? [];
// A literal argument may influence the invocation. The assigned value
// comes from modeled returns, so an ignored parameter does not reach it.
if (returns.length === 0) {
addRelation(context.state, {
source: context.callSiteNodes.get(flow.callSiteId),
target,
relation: "aliases",
resolution,
});
continue;
}
for (const item of returns)
addRelation(context.state, {
source: context.returnSiteNodes.get(item.returnSiteId),
target,
relation: "aliases",
resolution,
});
}
projectSemanticReturnValues(context);
};
@@ -1025,6 +1092,14 @@ const createArgumentNode = (
? "spread argument"
: `argument ${String(argument.index)}`,
functionNodeId: callNode.function_node_id,
...(!argument.spread && Object.hasOwn(argument, "literal")
? {
properties: {
literal_argument_value: argument.literal ?? null,
literal_argument_call_site_node_id: callNode.node_id,
},
}
: {}),
});
/** Per-file projection passes, run in this order with a yield between each. */
@@ -1035,6 +1110,7 @@ const FILE_PROJECTION_PASSES: readonly ((
projectSemanticObjects,
projectSemanticAssignments,
projectCalls,
projectSemanticExpressionInfluence,
projectSemanticPromises,
projectSemanticEvents,
projectSemanticTimers,
@@ -0,0 +1,356 @@
import { expect, it } from "vitest";
import { graphForJavaScript } from "../../../tests/fixtures/javascriptSemanticGraph.js";
import { queryJavaScriptSemanticGraph } from "../../domain/javascript/javascriptSemanticQuery.js";
import { resolveJavaScriptSemanticQuerySeeds } from "../../domain/javascript/javascriptSemanticQueryIdentity.js";
import { javaScriptSemanticQueryInputSchema } from "../../domain/javascript/javascriptSemanticQuerySchemas.js";
import { parseJavaScriptSource } from "../../domain/javascript/javascriptSourceParser.js";
import { analyzeParsedJavaScriptSemanticsSteps } from "../../domain/javascript/javascriptSemanticAnalysis.js";
import { completeSemanticSteps } from "../../domain/javascript/javascriptSemanticTraversal.js";
const trace = (body: string) => {
const graph = graphForJavaScript(
`const base = "https://example.test"; ${body}`,
);
return queryJavaScriptSemanticGraph(
graph,
javaScriptSemanticQueryInputSchema.parse({
seed: { kind: "literal", value: "https://example.test" },
direction: "forward-influence",
}),
);
};
it.each([
["declaration", 'const url = base + "/x"; fetch(url);'],
["assignment", 'let url; url = base + "/x"; fetch(url);'],
["nested operator", 'const url = (base + "/x") + "/y"; fetch(url);'],
["call argument", 'fetch(base + "/x");'],
["template", "fetch(`${base}/x`);"],
])("follows transformed input to a %s without opt-in", (_name, body) => {
const result = trace(body);
expect(result.nodes.map((node) => node.label)).toContain("argument 0");
expect(result.relations.some((edge) => edge.relation === "influences")).toBe(
true,
);
});
it("explains an unmodeled dynamic call result at the reached argument", () => {
const result = trace('const url = wrap(base + "/x"); fetch(url);');
expect(result.unknowns.map((unknown) => unknown.detail)).toContain(
"The dynamic call's return dependency on this argument is unknown.",
);
});
it.each([
'const obj = { url: wrap(base + "/x") }; fetch(obj.url);',
'function address() { return wrap(base + "/x"); }',
'consume(wrap(base + "/x"));',
])("keeps nested dynamic call return influence unknown: %s", (body) => {
const result = trace(body);
expect(result.status).not.toBe("found");
expect(result.unknowns.map((unknown) => unknown.detail)).toContain(
"The dynamic call's return dependency on this argument is unknown.",
);
});
it("does not report an unused dynamic call result as a missing value", () => {
const result = trace('consume(base + "/x");');
expect(
result.unknowns.some(
(unknown) =>
unknown.detail ===
"The dynamic call's return dependency on this argument is unknown.",
),
).toBe(false);
});
it("bounds operator production before graph admission and reports truncation", () => {
const expressions = Array.from({ length: 500 }, () => "${base}").join("");
const source = `const base = "value"; const value = \`${expressions}\`;`;
const parsed = parseJavaScriptSource(source, "app.js");
if (parsed === null) throw new Error("Expected parsed source");
const ir = completeSemanticSteps(
analyzeParsedJavaScriptSemanticsSteps(parsed, "import", 4),
);
expect(ir.expressionFlowsTruncated).toBe(true);
expect(ir.expressionFlows.length).toBeLessThanOrEqual(4);
expect(
ir.expressionFlows.reduce((count, flow) => count + flow.operands.length, 0),
).toBeLessThanOrEqual(4);
expect(ir.coverage).toMatchObject({ status: "partial", omittedCount: null });
expect(graphForJavaScript(source, ir).coverage.truncated).toBe(true);
});
it("follows a transformed return expression", () => {
const result = trace('function address() { return base + "/x"; }');
expect(result.nodes.map((node) => node.label)).toContain("return");
});
it("does not treat a deferred callback body as an enclosing call argument value", () => {
const graph = graphForJavaScript(
'const base = "https://example.test"; consume(() => base + "/x");',
);
const base = graph.nodes.find(
(node) => node.kind === "binding" && node.label === "base",
);
const argument = graph.nodes.find(
(node) => node.kind === "expression" && node.label === "argument 0",
);
expect(base).toBeDefined();
expect(argument).toBeDefined();
const result = queryJavaScriptSemanticGraph(
graph,
javaScriptSemanticQueryInputSchema.parse({
seed: { kind: "literal", value: "https://example.test" },
direction: "forward-influence",
include_ambiguous_dynamic_edges: true,
}),
);
expect(result.nodes.map((node) => node.node_id)).not.toContain(
argument?.node_id,
);
});
it.each(['"/unique"', "`/unique`"])(
"seeds an exact literal operand %s and reaches the receiving argument",
(operand) => {
const graph = graphForJavaScript(
`const prefix = "root"; const path = prefix + ${operand}; fetch(path);`,
);
const result = queryJavaScriptSemanticGraph(
graph,
javaScriptSemanticQueryInputSchema.parse({
seed: { kind: "literal", value: "/unique" },
direction: "forward-influence",
}),
);
expect(result.nodes.map((node) => node.label)).toContain("argument 0");
},
);
it("marks a short-circuited operand as conditional without claiming alias identity", () => {
const graph = graphForJavaScript(
'const base = "unique"; const maybe = ready && base; fetch(maybe);',
);
const conditional = graph.relations.filter(
(edge) =>
edge.relation === "influences" && edge.properties.conditional === true,
);
expect(conditional).toHaveLength(1);
expect(
graph.nodes.find((node) => node.node_id === conditional[0]?.source_node_id)
?.label,
).toBe("base");
});
it("reports uncertainty on a read inside an unsupported operand", () => {
const result = trace('const url = format(base) + "/x"; fetch(url);');
expect(result.status).toBe("partial");
expect(
result.unknowns.some((unknown) =>
unknown.detail.includes("unmodeled operator operand"),
),
).toBe(true);
});
it.each([
'function answer() { return "return-only"; }',
'const answer = () => "return-only";',
])("seeds a literal returned directly: %s", (source) => {
const graph = graphForJavaScript(source);
const result = queryJavaScriptSemanticGraph(
graph,
javaScriptSemanticQueryInputSchema.parse({
seed: { kind: "literal", value: "return-only" },
direction: "forward-influence",
}),
);
expect(result.summary.total_seed_matches).toBeGreaterThan(0);
expect(result.nodes.map((node) => node.label)).toContain("return");
});
it("marks a logical assignment's transformed input conditional", () => {
const graph = graphForJavaScript(
'const base = "unique"; let value; value ||= base + "/x";',
);
const conditional = graph.relations.find(
(edge) =>
edge.relation === "influences" &&
edge.properties.conditional === true &&
graph.nodes.find((node) => node.node_id === edge.target_node_id)
?.label === "value",
);
expect(conditional).toBeDefined();
});
it("keeps sequence side effects unknown instead of attributing earlier values directly", () => {
const result = trace('const url = (touch(base), "fixed"); fetch(url);');
expect(result.status).toBe("partial");
expect(
result.unknowns.some((unknown) =>
unknown.detail.includes("unmodeled operator operand"),
),
).toBe(true);
});
it("seeds a direct argument at its occurrence and reaches the call", () => {
const graph = graphForJavaScript('consume("only-direct");');
const result = queryJavaScriptSemanticGraph(
graph,
javaScriptSemanticQueryInputSchema.parse({
seed: { kind: "literal", value: "only-direct" },
direction: "forward-influence",
}),
);
expect(result.summary.total_seed_matches).toBe(1);
expect(
result.nodes.find((node) => node.node_id === result.seed_node_ids[0])
?.label,
).toBe("argument 0");
expect(result.nodes.some((node) => node.kind === "call-site")).toBe(true);
});
it("keeps older call-site literal evidence searchable when argument literals are absent", () => {
const graph = graphForJavaScript('consume("only-direct");');
const legacy = {
...graph,
nodes: graph.nodes.filter(
(node) =>
node.properties.literal_argument_call_site_node_id === undefined,
),
};
const seeds = resolveJavaScriptSemanticQuerySeeds(
legacy,
javaScriptSemanticQueryInputSchema.parse({
seed: { kind: "literal", value: "only-direct" },
direction: "forward-influence",
}),
);
const call = graph.nodes.find((node) => node.kind === "call-site");
expect(seeds).toEqual([call?.node_id]);
});
it("does not treat literals inside a spread or deferred callback as direct arguments", () => {
const graph = graphForJavaScript(
'consume(...["spread-only"], () => "deferred-only");',
);
for (const value of ["spread-only", "deferred-only"]) {
const result = queryJavaScriptSemanticGraph(
graph,
javaScriptSemanticQueryInputSchema.parse({
seed: { kind: "literal", value },
direction: "forward-influence",
}),
);
expect(
result.nodes.filter(
(node) =>
node.properties.literal_argument_call_site_node_id !== undefined,
),
).toEqual([]);
}
});
it.each([
'const obj = { url: base + "/x" }; fetch(obj.url);',
'const values = [base + "/x"]; fetch(values[0]);',
'const url = wrap(base + "/x"); fetch(url);',
])("reports unknown when a consumer boundary is unmodeled: %s", (body) => {
const result = trace(body);
expect(result.status).not.toBe("found");
expect(result.unknowns.length).toBeGreaterThan(0);
});
it("does not reuse an outer literal for the same text in a nested function", () => {
const graph = graphForJavaScript(
'const suffix = "end"; const value = true ? "token" : () => "token" + suffix;',
);
const tokens = graph.nodes.filter(
(node) => node.kind === "literal" && node.properties.value === "token",
);
expect(tokens.length).toBeGreaterThan(1);
const inner = tokens.find((node) => node.function_node_id !== null);
const outer = tokens.find((node) => node.function_node_id === null);
if (inner === undefined || outer === undefined)
throw new Error("Expected separate outer and nested token literals");
expect(inner.identity.source_range).not.toEqual(outer.identity.source_range);
const traced = queryJavaScriptSemanticGraph(
graph,
javaScriptSemanticQueryInputSchema.parse({
seed: { kind: "semantic-node", node_id: outer.node_id },
direction: "forward-influence",
}),
);
expect(traced.nodes.map((node) => node.node_id)).not.toContain(inner.node_id);
});
it("does not send an ignored literal argument to the call result", () => {
const graph = graphForJavaScript(
'function ignore(x) { return "safe"; } const result = ignore("token");',
);
const resultBinding = graph.nodes.find(
(node) => node.kind === "binding" && node.label === "result",
);
if (resultBinding === undefined) throw new Error("Expected result binding");
const forward = queryJavaScriptSemanticGraph(
graph,
javaScriptSemanticQueryInputSchema.parse({
seed: { kind: "literal", value: "token" },
direction: "forward-influence",
}),
);
expect(forward.nodes.some((node) => node.kind === "call-site")).toBe(true);
expect(forward.nodes.map((node) => node.node_id)).not.toContain(
resultBinding.node_id,
);
const backward = queryJavaScriptSemanticGraph(
graph,
javaScriptSemanticQueryInputSchema.parse({
seed: { kind: "semantic-node", node_id: resultBinding.node_id },
direction: "backward-provenance",
}),
);
expect(
backward.nodes.some(
(node) =>
node.properties.value === "token" ||
node.properties.literal_argument_value === "token",
),
).toBe(false);
expect(backward.nodes.some((node) => node.properties.value === "safe")).toBe(
true,
);
});
it("keeps conditional binding candidates distinct from operand occurrences", () => {
const graph = graphForJavaScript('export const choice = flag ? 1 : "1";');
const choices = graph.nodes.filter(
(node) =>
node.kind === "literal" &&
(node.properties.value === 1 || node.properties.value === "1"),
);
expect(choices).toHaveLength(4);
expect(
new Set(
choices.map((node) =>
JSON.stringify([node.properties.value, node.identity.source_range]),
),
).size,
).toBe(4);
});
it("projects repeated operator literals with indexed lookup", () => {
const count = 1_200;
const declarations = Array.from(
{ length: count },
(_, index) => `const item${String(index)} = input + "/item";`,
);
const graph = graphForJavaScript(
`const input = "root";\n${declarations.join("\n")}`,
);
expect(
graph.nodes.filter(
(node) => node.kind === "literal" && node.properties.value === "/item",
),
).toHaveLength(count);
});
@@ -0,0 +1,398 @@
import {
compareJavaScriptSourcePoints,
javascriptSourceRangeContains,
javascriptSourceRangeKey,
} from "../../domain/javascript/javascriptSourceCoordinates.js";
import type { JavaScriptSourceRange } from "../../domain/javascript/javascriptStaticAnalysisTypes.js";
import type { JavaScriptSemanticGraphNode } from "../../domain/javascript/javascriptSemanticGraphSchemas.js";
import { compositeKey } from "../../domain/unicodeCodePointOrder.js";
import type { SemanticFlowProjectionContext } from "./JavaScriptSemanticGraphFlowProjection.js";
import {
addSemanticGraphRelation,
addSemanticGraphUnknown,
retainSemanticGraphNode,
} from "./JavaScriptSemanticGraphConstruction.js";
import { unknownSemanticEvidence } from "./JavaScriptSemanticGraphEvidence.js";
const sameArtifact = (
node: JavaScriptSemanticGraphNode,
context: SemanticFlowProjectionContext,
): boolean =>
node.identity.module_path === context.file.path &&
node.identity.artifact_sha256 === context.file.sha256;
const literalValueKey = (value: string | number | boolean | null): string =>
`${typeof value}:${JSON.stringify(value)}`;
type IndexedOperatorLiteral = {
readonly node: JavaScriptSemanticGraphNode;
readonly range: JavaScriptSourceRange;
readonly order: number;
};
/**
* Literals are remembered by their own source range and value.
* An initializer that merely contains another occurrence, including a
* nested function body, does not reuse the declaration's literal.
*/
const operatorLiteralIndex = (
context: SemanticFlowProjectionContext,
): {
remember(
node: JavaScriptSemanticGraphNode | null,
): JavaScriptSemanticGraphNode | null;
find(
value: string | number | boolean | null,
location: JavaScriptSourceRange,
): JavaScriptSemanticGraphNode | undefined;
} => {
const exact = new Map<string, IndexedOperatorLiteral>();
const byValue = new Map<string, IndexedOperatorLiteral[]>();
const seen = new Set<string>();
let order = 0;
const remember = (
node: JavaScriptSemanticGraphNode | null,
): JavaScriptSemanticGraphNode | null => {
if (
node === null ||
node.kind !== "literal" ||
!sameArtifact(node, context) ||
seen.has(node.node_id)
)
return node;
const range = node.identity.source_range;
const value = node.properties.value;
if (
range === null ||
(typeof value !== "string" &&
typeof value !== "number" &&
typeof value !== "boolean" &&
value !== null)
)
return node;
seen.add(node.node_id);
const entry = { node, range, order: order++ };
const valueKey = literalValueKey(value);
const id = compositeKey([valueKey, javascriptSourceRangeKey(range)]);
const group = byValue.get(valueKey) ?? [];
group.push(entry);
byValue.set(valueKey, group);
if (!exact.has(id)) exact.set(id, entry);
return node;
};
for (const node of context.state.nodes.values()) remember(node);
return {
remember,
find(value, location) {
const valueKey = literalValueKey(value);
const operandKey = javascriptSourceRangeKey(location);
let best: IndexedOperatorLiteral | undefined;
const consider = (entry: IndexedOperatorLiteral | undefined): void => {
if (
entry !== undefined &&
(best === undefined || entry.order < best.order)
)
best = entry;
};
consider(exact.get(compositeKey([valueKey, operandKey])));
const group = byValue.get(valueKey) ?? [];
let low = 0;
let high = group.length;
while (low < high) {
const middle = (low + high) >> 1;
const entry = group[middle];
if (
entry !== undefined &&
compareJavaScriptSourcePoints(entry.range.start, location.start) <= 0
)
low = middle + 1;
else high = middle;
}
const candidate = group[low - 1];
if (
candidate !== undefined &&
javascriptSourceRangeContains(candidate.range, location)
)
consider(candidate);
return best?.node;
},
};
};
/** Project immediate operator dependencies without treating them as aliases. */
export function* projectSemanticExpressionInfluence(
context: SemanticFlowProjectionContext & {
readonly argumentNodes: ReadonlyMap<string, JavaScriptSemanticGraphNode>;
},
): Generator<void, void> {
if (context.ir.expressionFlowsTruncated)
context.state.fileNodesDropped = true;
const key = javascriptSourceRangeKey;
const expressions = new Map<string, JavaScriptSemanticGraphNode>();
const usedExpressions = new Set<string>();
const reads = new Map(
context.readNodes.flatMap((node) =>
node.identity.source_range === null
? []
: [[key(node.identity.source_range), node] as const],
),
);
const writes = new Map(
context.writeNodes.flatMap((node) =>
node.identity.source_range === null
? []
: [[key(node.identity.source_range), node] as const],
),
);
const bindings = new Map(
context.ir.bindings.flatMap((binding) => {
const node = context.bindingNodes.get(binding.bindingId);
return node === undefined
? []
: binding.definitions.map(
(definition) => [key(definition.location), node] as const,
);
}),
);
for (const [index, flow] of context.ir.expressionFlows.entries()) {
if (index % 64 === 0) yield;
const owner =
flow.ownerCallableId === null
? null
: (context.callableNodes.get(flow.ownerCallableId)?.node_id ?? null);
const node = retainSemanticGraphNode(context.state, context.file, {
kind: "expression",
roleKey: `operator:${index}`,
location: flow.location,
label: flow.operator,
functionNodeId: owner,
properties: { operator: flow.operator },
});
if (node !== null) expressions.set(key(flow.location), node);
}
const remainingUnknowns =
context.state.unknownCapacity === null
? context.readNodes.length
: Math.max(
0,
context.state.unknownCapacity - context.state.unknowns.size,
);
let unsupportedReads = 0;
let unsupportedOverflow = false;
const orderedReads = context.readNodes
.flatMap((node) =>
node.identity.source_range === null
? []
: [{ node, location: node.identity.source_range }],
)
.sort((left, right) =>
compareJavaScriptSourcePoints(left.location.start, right.location.start),
);
let processedOperands = 0;
let processedReferences = 0;
const literals = operatorLiteralIndex(context);
for (const [index, flow] of context.ir.expressionFlows.entries()) {
if (index % 64 === 0) yield;
const result = expressions.get(key(flow.location));
for (const operand of flow.operands) {
if (processedOperands % 64 === 0) yield;
processedOperands += 1;
if (operand.kind === "unsupported") {
if (result !== undefined)
addSemanticGraphUnknown(context.state, {
node_id: result.node_id,
family: "data-flow",
relation_kinds: ["influences"],
reason: "unknown-value",
detail: "An operator operand has unmodeled value dependencies.",
candidate_node_ids: [],
evidence: unknownSemanticEvidence(context.file, operand.location),
});
if (!unsupportedOverflow) {
let low = 0,
high = orderedReads.length;
while (low < high) {
const middle = Math.floor((low + high) / 2);
if (
compareJavaScriptSourcePoints(
orderedReads[middle]!.location.start,
operand.location.start,
) < 0
)
low = middle + 1;
else high = middle;
}
for (let offset = low; offset < orderedReads.length; offset++) {
const { node: reference, location } = orderedReads[offset]!;
if (
compareJavaScriptSourcePoints(
location.start,
operand.location.end,
) > 0
)
break;
if (processedReferences % 64 === 0) yield;
processedReferences += 1;
if (processedReferences > remainingUnknowns) {
unsupportedOverflow = true;
break;
}
if (
!javascriptSourceRangeContains(operand.location, location) ||
reference.function_node_id !== result?.function_node_id
)
continue;
if (unsupportedReads >= remainingUnknowns) {
unsupportedOverflow = true;
break;
}
unsupportedReads += 1;
addSemanticGraphUnknown(context.state, {
node_id: reference.node_id,
family: "data-flow",
relation_kinds: ["influences"],
reason: "unknown-value",
detail:
"This read occurs inside an unmodeled operator operand; its influence on the result is unknown.",
candidate_node_ids: [],
evidence: unknownSemanticEvidence(context.file, operand.location),
});
}
}
continue;
}
const source =
operand.kind === "reference"
? reads.get(key(operand.location))
: operand.kind === "expression"
? expressions.get(key(operand.location))
: (literals.find(operand.literal ?? null, operand.location) ??
literals.remember(
retainSemanticGraphNode(context.state, context.file, {
kind: "literal",
roleKey: `operator-literal:${key(operand.location)}:${JSON.stringify(operand.literal ?? null)}`,
location: operand.location,
label: "operator literal",
functionNodeId: result?.function_node_id ?? null,
properties: { value: operand.literal ?? null },
}) ?? null,
));
if (
operand.kind === "expression" &&
source !== undefined &&
result !== undefined
)
usedExpressions.add(key(operand.location));
addSemanticGraphRelation(context.state, {
source,
target: result,
relation: "influences",
resolution: "resolved",
properties: { conditional: operand.conditional },
});
}
}
// Some operands are calls, members or other constructs whose value owner is
// not yet modeled here. Do not claim complete influence for any query that
// may cross one of these unmodeled operator inputs.
if (unsupportedOverflow)
addSemanticGraphUnknown(context.state, {
node_id: null,
family: "data-flow",
relation_kinds: ["influences"],
reason: "resource-limit",
detail:
"The bounded operator-dependency audit omitted some unmodeled operand reads; influence through them remains unknown.",
candidate_node_ids: [],
evidence: unknownSemanticEvidence(context.file, null),
});
const connect = (
source: JavaScriptSemanticGraphNode | undefined,
target: JavaScriptSemanticGraphNode | undefined,
conditional = false,
) => {
if (
source !== undefined &&
target !== undefined &&
source.identity.source_range !== null &&
expressions.get(key(source.identity.source_range)) === source
)
usedExpressions.add(key(source.identity.source_range));
addSemanticGraphRelation(context.state, {
source,
target,
relation: "influences",
resolution: "resolved",
properties: { conditional },
});
};
for (const flow of context.ir.initializerFlows)
connect(expressions.get(key(flow.source)), bindings.get(key(flow.target)));
for (const flow of context.ir.assignmentFlows) {
const source = expressions.get(key(flow.source));
const target = writes.get(key(flow.target));
if (source !== undefined && target !== undefined)
usedExpressions.add(key(flow.source));
addSemanticGraphRelation(context.state, {
source,
target,
relation: "influences",
resolution: "resolved",
properties: {
conditional: ["&&=", "||=", "??="].includes(flow.operator),
assignment_operator: flow.operator,
},
});
}
for (const call of context.ir.callSites)
for (const argument of call.arguments) {
if (argument.valueExpressionLocation === null) continue;
connect(
expressions.get(key(argument.valueExpressionLocation)),
context.argumentNodes.get(
`${call.callSiteId}\u0000${String(argument.index)}`,
),
);
}
const literalReturns = new Map(
context.ir.literalReturns.map((item) => [key(item.location), item]),
);
for (const callable of context.ir.callables)
for (const site of callable.returnSites) {
if (site.valueExpressionLocation !== null) {
const target = context.returnSiteNodes.get(site.returnSiteId);
connect(expressions.get(key(site.valueExpressionLocation)), target);
const literal = literalReturns.get(key(site.valueExpressionLocation));
if (literal !== undefined) {
const source = retainSemanticGraphNode(context.state, context.file, {
kind: "literal",
roleKey: `return-literal:${site.returnSiteId}`,
location: literal.location,
label: "return literal",
functionNodeId: target?.function_node_id ?? null,
properties: { value: literal.value },
});
addSemanticGraphRelation(context.state, {
source,
target,
relation: "aliases",
resolution: "resolved",
});
}
}
}
for (const flow of context.ir.expressionFlows) {
const node = expressions.get(key(flow.location));
if (node === undefined || usedExpressions.has(key(flow.location))) continue;
addSemanticGraphUnknown(context.state, {
node_id: node.node_id,
family: "data-flow",
relation_kinds: ["influences"],
reason: "unknown-value",
detail:
"The receiving expression or container for this operator result is not modeled; further influence is unknown.",
candidate_node_ids: [],
evidence: unknownSemanticEvidence(context.file, flow.location),
});
}
}
@@ -269,10 +269,24 @@ export const semanticObjectReadOccurrence = (
export const projectSemanticReturnValues = (
context: SemanticFlowProjectionContext,
): void => {
const expressions = new Set(
context.ir.expressionFlows.map((flow) =>
javascriptSourceRangeKey(flow.location),
),
);
for (const callable of context.ir.callables)
for (const site of callable.returnSites) {
const returnNode = context.returnSiteNodes.get(site.returnSiteId);
const references = context.referenceNodesWithin(site.location);
const references =
site.valueExpressionLocation !== null &&
expressions.has(javascriptSourceRangeKey(site.valueExpressionLocation))
? []
: context
.referenceNodesWithin(site.location)
.filter(
(reference) =>
reference.function_node_id === returnNode?.function_node_id,
);
for (const reference of references)
addSemanticGraphRelation(context.state, {
source: reference,
@@ -360,7 +360,13 @@ it("does not promote transformed return expressions to resolved aliases", () =>
({ relation, target_node_id }) =>
relation === "aliases" && target_node_id === returnNode.node_id,
),
).toEqual([expect.objectContaining({ resolution: "candidate" })]);
).toEqual([]);
expect(
graph.relations.filter(
({ relation, target_node_id }) =>
relation === "influences" && target_node_id === returnNode.node_id,
),
).toEqual([expect.objectContaining({ resolution: "resolved" })]);
const output = graph.nodes.find(
({ kind, label }) => kind === "binding" && label === "output",
);
@@ -376,10 +382,8 @@ it("does not promote transformed return expressions to resolved aliases", () =>
direction: "backward-provenance",
}),
);
expect(strict.nodes.map(({ node_id }) => node_id)).not.toContain(
input.node_id,
);
expect(strict.status).toBe("ambiguous");
expect(strict.nodes.map(({ node_id }) => node_id)).toContain(input.node_id);
expect(strict.status).toBe("partial");
});
it("uses unavailable evidence when no source produced semantic IR", () => {
@@ -318,3 +318,25 @@ it("applies the same unknown admission in direct and transferred construction",
transferred.finish(digest, application),
);
});
it("bounds reads within unmodeled nested operator operands", () => {
const body = `const base = "token"; const value = ${"wrap(".repeat(30)}base${")".repeat(30)} + "/x";`;
const bounded = fileProjection(source("app.js", body), 200, 1);
expect(bounded.unknowns.length).toBeLessThanOrEqual(1);
expect(bounded.omittedUnknownOccurrences?.length).toBeGreaterThan(0);
const projection = createJavaScriptSemanticGraphProjection({
totalUnknowns: 1,
fileUnknowns: 1,
});
projection.adoptFileProjection(bounded);
const graph = projection.finish(digest, application);
expect(graph.coverage.truncated).toBe(true);
const trace = queryJavaScriptSemanticGraph(
graph,
javaScriptSemanticQueryInputSchema.parse({
seed: { kind: "literal", value: "token" },
direction: "forward-influence",
}),
);
expect(trace.status).not.toBe("found");
});
@@ -1,4 +1,4 @@
import { writeFile } from "node:fs/promises";
import { appendFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect, it, vi } from "vitest";
@@ -6,6 +6,8 @@ import { expect, it, vi } from "vitest";
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
import { withOwnedRegularFile } from "../filesystem/OwnedRegularFile.js";
import { ArtifactEmbeddedFormatProbes } from "./ArtifactEmbeddedFormatProbes.js";
import * as dotnetLayout from "./dotnet/DotnetBundle.js";
import { dotnetBundleFixture } from "./dotnet/DotnetBundle.fixture.js";
import * as pckLocation from "./GodotPckLocation.js";
it("distinguishes examined absence from unexamined formats and honors cancellation on a cached result", async () => {
@@ -70,3 +72,74 @@ it("leaves a failed probe unexamined so a subsequent successful observation can
locate.mockRestore();
}
});
it("reuses a located .NET header only for the same source and honors cached cancellation", async () => {
const root = await createTestTempDirectory("rea-dotnet-probe-facts-");
const path = join(root, "host");
const fixture = dotnetBundleFixture();
await writeFile(path, fixture.bytes);
const locate = vi.spyOn(dotnetLayout, "findDotnetBundleHeader");
try {
const facts = await withOwnedRegularFile(path, {}, async (source) => {
const controller = new AbortController();
const first = new ArtifactEmbeddedFormatProbes(
path,
source,
controller.signal,
);
expect(
Object.hasOwn(first.completedFacts(), "dotnetBundleHeaderOffset"),
).toBe(false);
await expect(first.dotnetBundle()).resolves.toBe(fixture.headerOffset);
const completed = first.completedFacts();
const resumed = new ArtifactEmbeddedFormatProbes(
path,
source,
controller.signal,
completed,
);
await expect(resumed.dotnetBundle()).resolves.toBe(fixture.headerOffset);
expect(locate).toHaveBeenCalledTimes(1);
const reason = new Error("cancel cached .NET observation");
controller.abort(reason);
await expect(resumed.dotnetBundle()).rejects.toMatchObject({
reason: "cancelled",
cause: reason,
});
return completed;
});
await appendFile(path, "changed");
await withOwnedRegularFile(path, {}, async (source) => {
expect(
() => new ArtifactEmbeddedFormatProbes(path, source, undefined, facts),
).toThrow("Root artifact changed before inventory");
});
expect(locate).toHaveBeenCalledTimes(1);
} finally {
locate.mockRestore();
}
});
it("keeps a failed .NET probe unexamined before a successful absence observation", async () => {
const root = await createTestTempDirectory("rea-dotnet-probe-retry-");
const path = join(root, "plain");
await writeFile(path, "plain bytes");
const failure = new Error("injected marker read failure");
const locate = vi
.spyOn(dotnetLayout, "findDotnetBundleHeader")
.mockRejectedValueOnce(failure);
try {
await withOwnedRegularFile(path, {}, async (source) => {
const probes = new ArtifactEmbeddedFormatProbes(path, source);
await expect(probes.dotnetBundle()).rejects.toBe(failure);
expect(
Object.hasOwn(probes.completedFacts(), "dotnetBundleHeaderOffset"),
).toBe(false);
await expect(probes.dotnetBundle()).resolves.toBeUndefined();
expect(probes.completedFacts().dotnetBundleHeaderOffset).toBeNull();
expect(locate).toHaveBeenCalledTimes(2);
});
} finally {
locate.mockRestore();
}
});
@@ -1,3 +1,7 @@
import {
probeBunStandaloneGraph,
type BunStandaloneProbe,
} from "./bun/BunModuleGraphLayout.js";
import type {
StableRegularFileDescriptor,
RegularFileState,
@@ -6,6 +10,7 @@ import {
regularFileState,
sameRegularFileState,
} from "../filesystem/RegularFile.js";
import { findDotnetBundleHeader } from "./dotnet/DotnetBundle.js";
import { ArtifactReaderFailure } from "./ArtifactReader.js";
import { abortIfNeeded } from "./ArtifactHash.js";
import { locateGodotPck, type GodotPckLocation } from "./GodotPckLocation.js";
@@ -22,8 +27,10 @@ import {
/** Completed observations only; omission means unexamined, and null is examined absence. */
export interface ArtifactEmbeddedFormatProbeFacts {
readonly sourceState: RegularFileState;
readonly bunStandalone?: BunStandaloneProbe;
readonly godotPck?: GodotPckLocation | null;
readonly nodeSea?: NodeSeaLocation | null;
readonly dotnetBundleHeaderOffset?: number | null;
readonly pyInstaller?: PyInstallerCookieObservation;
}
@@ -94,6 +101,37 @@ export class ArtifactEmbeddedFormatProbes {
return found.value.state === "located" ? found.value.cookie : undefined;
}
async dotnetBundle(): Promise<number | undefined> {
abortIfNeeded(this.signal);
if (this.#facts.dotnetBundleHeaderOffset !== undefined)
return this.#facts.dotnetBundleHeaderOffset ?? undefined;
// An apphost inside a recognized CArchive cannot claim its outer carrier.
const packaged = await this.pyInstaller();
const header = await findDotnetBundleHeader(
this.source.handle,
this.source.initial.size,
this.signal,
packaged?.archiveStart ?? this.source.initial.size,
);
abortIfNeeded(this.signal);
this.#facts = { ...this.#facts, dotnetBundleHeaderOffset: header ?? null };
return header;
}
async bunStandalone(): Promise<BunStandaloneProbe> {
abortIfNeeded(this.signal);
if (this.#facts.bunStandalone !== undefined)
return this.#facts.bunStandalone;
const observed = await probeBunStandaloneGraph(
this.source.handle,
this.source.initial.size,
this.signal,
);
abortIfNeeded(this.signal);
this.#facts = { ...this.#facts, bunStandalone: observed };
return observed;
}
completedFacts(): ArtifactEmbeddedFormatProbeFacts {
return this.#facts;
}
+10
View File
@@ -187,3 +187,13 @@ it.each([["app.js"], ["app.js", "App.js"], ["other.js"]])(
expect(destinationCaseCollision("res/app.js", names)).toBeUndefined();
},
);
it("reserves prefix capacity atomically without declaring destination kinds", () => {
const registry = new ArtifactPathRegistry({ maxParts: 4, maxNodes: 3 });
registry.reservePath("a/b");
expect(() => registry.reservePath("a/c/d")).toThrow();
expect(() => registry.reservePath("z")).not.toThrow();
expect(() => registry.reservePath("a/b")).not.toThrow();
registry.add("a/b", "file");
expect(() => registry.add("a/b", "directory")).toThrow();
});
+53 -5
View File
@@ -2,7 +2,11 @@ import { posix } from "node:path";
import { logicalPathNfc } from "../domain/artifactIdentity.js";
import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
import { ArtifactReaderFailure, type ArtifactEntry } from "./ArtifactReader.js";
import {
ArtifactReaderFailure,
artifactFixedLimitFailure,
type ArtifactEntry,
} from "./ArtifactReader.js";
const ABSOLUTE_DRIVE_PATH = /^[A-Za-z]:[\\/]/u;
@@ -87,16 +91,59 @@ export const normalizeArtifactPath = (input: string): string => {
return normalized;
};
/** Reject exact and same-spelling prefix collisions. Case-distinct names stay distinct. */
/** Reject prefix collisions and optionally bound the allocated path tree. */
export class ArtifactPathRegistry {
readonly #root: PathTrieNode = { kind: undefined, children: new Map() };
#nodeCount = 0;
constructor(
private readonly budget?: {
readonly maxParts: number;
readonly maxNodes: number;
},
) {}
add(path: string, kind: ArtifactEntry["kind"]): void {
if (!hasDestinationPath(kind)) return;
this.#visit(path, kind);
}
/** Account for distinct prefixes without claiming file/directory validity. */
reservePath(path: string): void {
this.#visit(path);
}
#visit(path: string, kind?: DestinationEntryKind): void {
const parts = path.split("/");
if (this.budget !== undefined && parts.length > this.budget.maxParts)
throw artifactFixedLimitFailure(
"Artifact path depth",
this.budget.maxParts,
"path parts",
);
if (this.budget !== undefined) {
let observed = this.#root;
for (const [index, part] of parts.entries()) {
const existing = observed.children.get(part);
if (existing === undefined) {
if (parts.length - index > this.budget.maxNodes - this.#nodeCount)
throw artifactFixedLimitFailure(
"Artifact path tree",
this.budget.maxNodes,
"path nodes",
);
break;
}
observed = existing;
}
}
let node = this.#root;
for (const [index, part] of parts.entries()) {
if (node.kind !== undefined && node.kind !== "directory")
if (
kind !== undefined &&
node.kind !== undefined &&
node.kind !== "directory"
)
throw new ArtifactReaderFailure(
"path",
`Artifact prefix conflict: ${pathDiagnostic(path)}`,
@@ -105,9 +152,10 @@ export class ArtifactPathRegistry {
if (child === undefined) {
child = { kind: undefined, children: new Map() };
node.children.set(part, child);
if (this.budget !== undefined) this.#nodeCount++;
}
node = child;
if (index === parts.length - 1) {
if (kind !== undefined && index === parts.length - 1) {
if (node.kind !== undefined && node.kind !== kind)
throw new ArtifactReaderFailure(
"path",
@@ -120,7 +168,7 @@ export class ArtifactPathRegistry {
);
}
}
node.kind = kind;
if (kind !== undefined) node.kind = kind;
}
}
+19
View File
@@ -1,3 +1,4 @@
import { fixedAllowanceRemediation } from "../domain/fixedAllowanceRemediation.js";
import type {
ArtifactAdmissionLimit,
ArtifactLimitBoundary,
@@ -46,6 +47,8 @@ export interface ArtifactEntry {
readonly executable: boolean;
readonly encrypted: boolean;
readonly byteOffset: number | null;
/** Stored source extent; expanded size may differ for compressed members. */
readonly byteLength?: number;
readonly declaredIntegrity: ArtifactIntegrityClaim | null;
readonly formatMetadata?: ArtifactFormatMetadata;
readonly unpacked: boolean;
@@ -109,6 +112,7 @@ export const sameArtifactEntry = (
entry.executable === produced.executable &&
entry.encrypted === produced.encrypted &&
entry.byteOffset === produced.byteOffset &&
entry.byteLength === produced.byteLength &&
sameDeclaredIntegrity(entry.declaredIntegrity, produced.declaredIntegrity) &&
sameFormatMetadata(entry.formatMetadata, produced.formatMetadata) &&
entry.unpacked === produced.unpacked &&
@@ -193,6 +197,8 @@ export interface ArtifactReader {
| "unityfs"
| "uimage"
| "pyinstaller"
| "dotnet-bundle"
| "bun-standalone"
| "node-sea"
| "intel-hex"
| "srec"
@@ -333,6 +339,19 @@ export class ArtifactReaderFailure extends Error {
}
}
/** Fixed reader input ceilings share producer-owned recovery across CLI and MCP. */
export const artifactFixedLimitFailure = (
name: string,
maximum: number,
unit: string,
): ArtifactReaderFailure => {
const message = `${fixedAllowanceRemediation(`${name} of ${String(maximum)} ${unit}`, "parser")} This release has no runtime override. Summary delivery and pagination do not raise this input limit.`;
return new ArtifactReaderFailure("limit", message, {
boundary: "capacity",
userMessage: message,
});
};
const combineCleanup = (
primary: AnalysisCleanupObservation | undefined,
cleanup: AnalysisCleanupObservation,
+18
View File
@@ -0,0 +1,18 @@
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
import type { GodotPckAdmission } from "./GodotPckLocation.js";
import type { NodeSeaAdmission } from "./nodeSea/NodeSeaAdmission.js";
import type { PyInstallerAdmission } from "./pyinstaller/PyInstallerAdmission.js";
import type { BunStandaloneAdmission } from "./bun/BunStandaloneAdmission.js";
import type { DotnetBundleAdmission } from "./dotnet/DotnetBundleAdmission.js";
/** Parsed format facts carried with the exact source that established them. */
export type ArtifactFormatAdmission =
| GodotPckAdmission
| NodeSeaAdmission
| PyInstallerAdmission
| DotnetBundleAdmission
| BunStandaloneAdmission;
export type ArtifactReaderAdmission =
| StableRegularFileDescriptor
| ArtifactFormatAdmission;
@@ -0,0 +1,408 @@
import type { BunStandaloneAdmission } from "./bun/BunStandaloneAdmission.js";
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
import { setImmediate } from "node:timers/promises";
import { Readable, Transform, pipeline } from "node:stream";
import { finished } from "node:stream/promises";
import { artifactExpansionBudgetBytes } from "../domain/artifactExpansionBudget.js";
import type {
BunStandaloneEntryMetadata,
BunStandaloneMetadata,
} from "../domain/artifactFormatMetadata.js";
import { jsonValueByteLength } from "../domain/jsonValueByteLength.js";
import type { JsonValue } from "../domain/jsonValue.js";
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
import {
normalizeArtifactPath,
ArtifactPathRegistry,
} from "./ArtifactPaths.js";
import {
ArtifactReaderFailure,
copyArtifactEntry,
sameArtifactEntry,
type ArtifactEntry,
type ArtifactReader,
type ArtifactTraversalOptions,
} from "./ArtifactReader.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
import {
MAX_BUN_RETAINED_BYTES,
bunFormatFailure,
bunMetadataBytesLimit,
probeBunStandaloneGraph,
loadBunStandaloneGraph,
type LocatedBunGraph,
} from "./bun/BunModuleGraphLayout.js";
import {
decodeBunModuleGraph,
type BunDecodedModule,
} from "./bun/BunModuleGraphMembers.js";
const REGIONS = ["contents", "sourcemap", "bytecode", "module-info"] as const;
type BunRegion = (typeof REGIONS)[number];
const MAX_LOGICAL_PATH_PARTS = 256;
const MAX_LOGICAL_PATH_NODES = 250_000;
interface BunModuleDestination {
readonly path: string;
readonly sourcePath?: string;
readonly limitations?: readonly string[];
}
/** One descriptor and bounded reads of an embedded Bun standalone module graph. */
export class BunStandaloneArtifactReader implements ArtifactReader {
readonly format = "bun-standalone" as const;
readonly #scope = new ArtifactResourceScope();
readonly #entries = new Map<string, ArtifactEntry>();
readonly #streams = new Set<Readable>();
readonly #source: ArtifactFileSource;
#admission: BunStandaloneAdmission | undefined;
#prepared: Promise<void> | undefined;
#header: BunStandaloneMetadata | undefined;
#closing = false;
constructor(
path: string,
admitted?: StableRegularFileDescriptor | BunStandaloneAdmission,
) {
if (admitted !== undefined && "kind" in admitted) {
this.#admission = admitted;
this.#source = new ArtifactFileSource(
path,
"Bun standalone",
admitted.source,
);
} else
this.#source = new ArtifactFileSource(path, "Bun standalone", admitted);
}
formatMetadata(): BunStandaloneMetadata | undefined {
return this.#header === undefined ? undefined : { ...this.#header };
}
async prepareContainer(
expectedSha256: string,
signal?: AbortSignal,
): Promise<void> {
await this.#scope.run(async () => {
this.#assertOpen(signal);
await this.#source.bindSelectedPathSha256(expectedSha256, signal);
});
}
async *entries(
signal?: AbortSignal,
_options?: ArtifactTraversalOptions,
): AsyncIterable<ArtifactEntry> {
await this.#scope.run(async () => {
this.#assertOpen(signal);
this.#prepared ??= this.#prepare(signal);
await this.#prepared;
await this.#source.readSelectedPath(signal);
});
let index = 0;
for (const entry of this.#entries.values()) {
if (index++ % 256 === 0) await setImmediate();
this.#assertOpen(signal);
yield copyArtifactEntry(entry);
}
}
async open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable> {
return this.#scope.run(async () => {
this.#assertOpen(signal);
const stored = this.#entries.get(entry.adapterKey);
if (stored === undefined || !sameArtifactEntry(entry, stored))
throw new ArtifactReaderFailure(
"integrity",
"Bun standalone entry does not match its owning inventory",
);
const { handle } = await this.#source.readSelectedPath(signal);
this.#assertOpen(signal);
if (stored.byteOffset === null || stored.declaredSize === null)
throw bunFormatFailure("member has no physical byte range");
const source =
stored.declaredSize === 0
? Readable.from([])
: readFileHandleChunks(handle, {
start: stored.byteOffset,
end: stored.byteOffset + stored.declaredSize - 1,
signal,
cancellationFailure: (reason) =>
new ArtifactReaderFailure(
"cancelled",
"Bun standalone read cancelled",
{ cause: reason },
),
});
const output = memberStream(stored);
output.on("error", () => undefined);
this.#streams.add(output);
output.once("close", () => this.#streams.delete(output));
void this.#scope
.run(
() =>
new Promise<void>((resolve) => {
pipeline(source, output, () => {
resolve();
});
}),
)
.catch((cause: unknown) =>
output.destroy(
cause instanceof Error
? cause
: new Error("Bun standalone stream ownership failed", { cause }),
),
);
return output;
});
}
async close(): Promise<void> {
this.#closing = true;
for (const stream of this.#streams)
stream.destroy(
new ArtifactReaderFailure(
"unavailable",
"Bun standalone reader closed",
),
);
await this.#scope.close();
await Promise.all(
[...this.#streams].map((stream) =>
finished(stream).catch(() => undefined),
),
);
await this.#source.close();
this.#entries.clear();
this.#header = undefined;
this.#admission = undefined;
this.#prepared = undefined;
}
provenance(): readonly [] {
return [];
}
#assertOpen(signal?: AbortSignal): void {
assertArtifactReadActive(signal);
if (this.#closing)
throw new ArtifactReaderFailure(
"unavailable",
"Bun standalone reader is closed",
);
}
async #prepare(signal?: AbortSignal): Promise<void> {
const source = await this.#source.readSelectedPath(signal);
const probe =
this.#admission?.probe ??
(await probeBunStandaloneGraph(
source.handle,
source.initial.size,
signal,
));
if (probe.status === "absent")
throw bunFormatFailure("no Bun section or trailer");
if (probe.failure !== undefined) throw probe.failure;
if (probe.claim === undefined)
throw bunFormatFailure("graph claim was not admitted");
const located = await loadBunStandaloneGraph(
source.handle,
probe.claim,
signal,
);
if (located.status === "absent")
throw bunFormatFailure("no Bun section or trailer");
if (located.failure !== undefined) throw located.failure;
if (located.graph === undefined)
throw bunFormatFailure("graph data was not admitted");
const decoded = await decodeBunModuleGraph(located.graph);
const entryModule = decoded.modules[decoded.entryPointId];
const entryDestination =
entryModule === undefined ? null : logicalModulePath(entryModule);
const header: BunStandaloneMetadata = {
kind: "bun-standalone",
layout_revision: decoded.revision,
carrier: located.graph.carrier,
payload_offset: located.graph.payloadOffset,
payload_length: located.graph.payloadLength,
module_count: decoded.modules.length,
entry_point_id: decoded.entryPointId,
entry_point_source_name: entryModule?.nameText.text ?? null,
entry_point_logical_path: entryDestination?.path ?? null,
compile_exec_argv: decoded.compileExecArgv.text,
compile_exec_argv_hex: decoded.compileExecArgv.hex,
flags: decoded.flags,
};
let retainedBytes = jsonValueByteLength(header, MAX_BUN_RETAINED_BYTES);
if (retainedBytes > MAX_BUN_RETAINED_BYTES)
throw bunMetadataBytesLimit(retainedBytes);
let declaredPlaintext = 0n;
const expansionBudget = BigInt(
artifactExpansionBudgetBytes(located.graph.payloadLength),
);
const paths = new ArtifactPathRegistry({
maxParts: MAX_LOGICAL_PATH_PARTS,
maxNodes: MAX_LOGICAL_PATH_NODES,
});
for (const module of decoded.modules) {
assertArtifactReadActive(signal);
const destination =
module.index === decoded.entryPointId && entryDestination !== null
? entryDestination
: logicalModulePath(module);
for (const region of REGIONS) {
const range = regionRange(module, region);
if (range === null) continue;
declaredPlaintext += BigInt(range.length);
if (declaredPlaintext > expansionBudget)
throw new ArtifactReaderFailure(
"limit",
`Bun standalone readable entries declare ${String(declaredPlaintext)} plaintext bytes, exceeding this graph's ${String(expansionBudget)}-byte expansion budget. This release has no runtime override. Host memory and NODE_OPTIONS do not raise it.`,
{ boundary: "capacity" },
);
const entry = moduleEntry(
located.graph,
module,
region,
range,
destination,
);
paths.reservePath(entry.path);
retainedBytes += jsonValueByteLength(
entryMetadataJson(entry),
Math.max(0, MAX_BUN_RETAINED_BYTES - retainedBytes),
);
if (retainedBytes > MAX_BUN_RETAINED_BYTES)
throw bunMetadataBytesLimit(retainedBytes);
this.#entries.set(entry.adapterKey, entry);
}
}
await this.#source.readSelectedPath(signal);
this.#assertOpen(signal);
this.#header = header;
this.#admission = undefined;
}
}
const regionRange = (
module: BunDecodedModule,
region: BunRegion,
): { readonly offset: number; readonly length: number } | null => {
if (region === "contents") return module.contents;
if (region === "sourcemap") return module.sourcemap;
if (region === "bytecode") return module.bytecode;
return module.moduleInfo;
};
const moduleEntry = (
graph: LocatedBunGraph,
module: BunDecodedModule,
region: BunRegion,
range: { readonly offset: number; readonly length: number },
located: BunModuleDestination,
): ArtifactEntry => {
const suffix =
region === "contents"
? ""
: region === "sourcemap"
? ".bunmap"
: region === "bytecode"
? ".bytecode"
: ".module-info";
const metadata: BunStandaloneEntryMetadata = {
kind: "bun-standalone-entry",
index: module.index,
region,
source_name: module.nameText.text,
source_name_hex: module.nameText.hex,
encoding: module.encoding,
encoding_name: module.encodingName,
loader: module.loader,
loader_name: module.loaderName,
module_format: module.moduleFormat,
module_format_name: module.moduleFormatName,
side: module.side,
side_name: module.sideName,
entry_point: module.index === graph.entryPointId,
sourcemap_bytes: module.sourcemap?.length ?? 0,
bytecode_bytes: module.bytecode?.length ?? 0,
module_info_bytes: module.moduleInfo?.length ?? 0,
bytecode_origin: module.bytecodeOrigin?.text ?? null,
bytecode_origin_hex: module.bytecodeOrigin?.hex ?? null,
};
return {
path: `${located.path}${suffix}`,
...(located.sourcePath === undefined
? {}
: { sourcePath: located.sourcePath }),
kind: "file",
declaredSize: range.length,
compressedSize: null,
executable: false,
encrypted: false,
byteOffset: graph.payloadOffset + range.offset,
declaredIntegrity: null,
unpacked: false,
limitations: [...(located.limitations ?? [])],
adapterKey: `bun:${String(module.index)}:${region}`,
formatMetadata: metadata,
};
};
/**
* Bun virtual names keep a leading slash (`/$bunfs/...`). Strip one slash
* when the remainder is a safe relative path, and keep the raw spelling on
* `sourcePath`. Unsafe, drive, and invalid UTF-8 names fall back to
* `bun-modules/<index>` so inventory can still retain the module.
*/
const logicalModulePath = (module: BunDecodedModule): BunModuleDestination => {
const fallback = `bun-modules/${String(module.index)}`;
const text = module.nameText.text;
if (text === null || text.length === 0) return { path: fallback };
const candidate = text.startsWith("/") ? text.slice(1) : text;
// Preserve virtual source facts without admitting an unbounded logical path tree.
let parts = 1;
for (let index = 0; index < candidate.length; index++)
if (candidate.charCodeAt(index) === 47 && ++parts > MAX_LOGICAL_PATH_PARTS)
return {
path: fallback,
sourcePath: text,
limitations: [
`The virtual name exceeds the fixed ${String(MAX_LOGICAL_PATH_PARTS)}-part logical path allowance; an indexed path is used and the original name is retained.`,
],
};
try {
return {
path: normalizeArtifactPath(candidate),
sourcePath: text,
};
} catch (cause: unknown) {
if (!(cause instanceof ArtifactReaderFailure) || cause.reason !== "path")
throw cause;
return { path: fallback, sourcePath: text };
}
};
const memberStream = (entry: ArtifactEntry): Transform => {
let bytes = 0;
return new Transform({
transform(chunk: Buffer, _encoding, callback) {
bytes += chunk.length;
callback(null, chunk);
},
flush(callback) {
callback(
bytes === entry.declaredSize
? null
: bunFormatFailure(`member ended early: ${entry.path}`),
);
},
});
};
const entryMetadataJson = (entry: ArtifactEntry): JsonValue =>
entry.formatMetadata === undefined ? null : { ...entry.formatMetadata };
+242
View File
@@ -0,0 +1,242 @@
import { Readable } from "node:stream";
import { finished, pipeline } from "node:stream/promises";
import { createInflateRaw } from "node:zlib";
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
import type { DotnetBundleAdmission } from "./dotnet/DotnetBundleAdmission.js";
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
import { abortIfNeeded } from "./ArtifactHash.js";
import {
ArtifactReaderFailure,
copyArtifactEntry,
sameArtifactEntry,
type ArtifactEntry,
type ArtifactReader,
} from "./ArtifactReader.js";
import {
findDotnetBundleHeader,
readDotnetBundle,
type DotnetBundle,
type DotnetBundleFile,
} from "./dotnet/DotnetBundle.js";
/** Stable, range-streamed .NET bundle reader; extraction paths remain owned by SafeOutputTree. */
export class DotnetBundleArtifactReader implements ArtifactReader {
readonly format = "dotnet-bundle" as const;
readonly #source: ArtifactFileSource;
readonly #scope = new ArtifactResourceScope();
#admission: DotnetBundleAdmission | undefined;
#bundle: DotnetBundle | undefined;
#closed = false;
readonly #entries = new Map<
string,
{ entry: ArtifactEntry; file: DotnetBundleFile }
>();
readonly #streams = new Set<Readable>();
constructor(
path: string,
admitted?: StableRegularFileDescriptor | DotnetBundleAdmission,
) {
if (admitted !== undefined && "kind" in admitted) {
this.#admission = admitted;
this.#source = new ArtifactFileSource(
path,
".NET bundle",
admitted.source,
);
} else this.#source = new ArtifactFileSource(path, ".NET bundle", admitted);
}
/** Bind extraction to the whole host that produced its inventory snapshot. */
async prepareContainer(
expectedSha256: string,
signal?: AbortSignal,
): Promise<void> {
await this.#scope.run(async () => {
this.#assertOpen(signal);
await this.#source.bindSelectedPathSha256(expectedSha256, signal);
});
}
async *entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry> {
const bundle = await this.#scope.run(async () => {
this.#assertOpen(signal);
const source = await this.#source.readSelectedPath(signal);
if (this.#bundle === undefined) {
const header =
this.#admission?.headerOffset ??
(await findDotnetBundleHeader(
source.handle,
source.initial.size,
signal,
));
if (header === undefined)
throw new ArtifactReaderFailure(
"format",
"Selected file has no .NET bundle header",
);
this.#bundle = await readDotnetBundle(
source.handle,
source.initial.size,
header,
signal,
);
this.#admission = undefined;
}
await this.#source.readSelectedPath(signal);
return this.#bundle;
});
this.#entries.clear();
for (const [index, file] of bundle.files.entries()) {
this.#assertOpen(signal);
const entry: ArtifactEntry = {
path: file.path,
kind: "file",
declaredSize: file.size,
compressedSize: file.compressedSize || null,
byteOffset: file.offset,
byteLength: file.compressedSize || file.size,
executable: false,
encrypted: false,
declaredIntegrity: null,
unpacked: false,
adapterKey: String(index),
formatMetadata: {
kind: "dotnet-bundle-entry",
major_version: bundle.majorVersion,
minor_version: bundle.minorVersion,
file_type: file.fileType,
},
limitations: [
".NET bundle file types are declarations, not proof of assembly validity or executable authority.",
...(file.fileType > 5
? [
`Unknown .NET bundle file type ${file.fileType}; bytes are retained without interpreting the type.`,
]
: []),
],
};
this.#entries.set(entry.adapterKey, {
entry: copyArtifactEntry(entry),
file,
});
yield entry;
}
await this.#source.readSelectedPath(signal);
}
async open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable> {
return this.#scope.run(async () => {
this.#assertOpen(signal);
const source = await this.#source.readSelectedPath(signal);
this.#assertOpen(signal);
const selected = this.#entries.get(entry.adapterKey);
if (selected === undefined || !sameArtifactEntry(entry, selected.entry))
throw new ArtifactReaderFailure(
"integrity",
"Selected .NET bundle entry differs from its observed manifest",
);
const stream = Readable.from(
this.#readMember(source, selected.file, signal),
{ objectMode: false },
);
this.#streams.add(stream);
stream.once("close", () => this.#streams.delete(stream));
return stream;
});
}
provenance() {
return [];
}
async close(): Promise<void> {
this.#closed = true;
for (const stream of this.#streams) stream.destroy();
await this.#scope.close();
await Promise.all(
[...this.#streams].map((stream) =>
finished(stream).catch(() => undefined),
),
);
await this.#source.close();
this.#entries.clear();
this.#admission = undefined;
this.#bundle = undefined;
}
#assertOpen(signal?: AbortSignal): void {
abortIfNeeded(signal);
if (this.#closed)
throw new ArtifactReaderFailure(
"unavailable",
".NET bundle reader is closed",
);
}
async *#readMember(
source: StableRegularFileDescriptor,
file: DotnetBundleFile,
signal?: AbortSignal,
): AsyncGenerator<Buffer> {
const storedBytes = file.compressedSize || file.size;
const input =
storedBytes === 0
? Readable.from([])
: readFileHandleChunks(source.handle, {
start: file.offset,
end: file.offset + storedBytes - 1,
signal,
});
const inflater = file.compressedSize === 0 ? undefined : createInflateRaw();
const decoded = inflater ?? input;
const pumping =
inflater === undefined ? undefined : pipeline(input, inflater);
// Observe early pipeline rejections while the output iterator is still reading.
void pumping?.catch(() => undefined);
let bytes = 0;
try {
for await (const raw of decoded) {
abortIfNeeded(signal);
const chunk: Buffer = raw;
bytes += chunk.length;
if (bytes > file.size)
throw new ArtifactReaderFailure(
"limit",
`.NET bundle entry expands beyond its declared size: ${file.path}`,
);
yield chunk;
}
await pumping;
if (
bytes !== file.size ||
(inflater !== undefined && inflater.bytesWritten !== storedBytes)
)
throw new ArtifactReaderFailure(
"format",
`.NET bundle member length or compressed extent does not match its declaration: ${file.path}`,
);
await this.#source.readSelectedPath(signal);
} catch (cause) {
abortIfNeeded(signal);
if (cause instanceof ArtifactReaderFailure) throw cause;
const malformedCompression =
inflater !== undefined &&
cause instanceof Error &&
"code" in cause &&
["Z_DATA_ERROR", "Z_BUF_ERROR"].includes(String(cause.code));
throw new ArtifactReaderFailure(
malformedCompression ? "format" : "io",
`Could not read .NET bundle member: ${file.path}`,
{ cause },
);
} finally {
input.destroy();
decoded.destroy();
await pumping?.catch(() => undefined);
}
}
}
+12 -1
View File
@@ -4,6 +4,7 @@ import { isAbsolute, join, relative, sep } from "node:path";
import { AnalysisInputError } from "../../domain/analysisErrorCore.js";
import { resolveTreePath } from "../../domain/apple/dyldPaths.js";
import { observedSymlinkTarget } from "../../domain/symlinkTarget.js";
import {
dylibResolutionInputSchema,
dylibResolutionResultSchema,
@@ -113,7 +114,12 @@ class FilesystemTreeView implements DylibTreeView {
try {
const metadata = await lstat(absolute);
if (metadata.isSymbolicLink()) {
const target = await readlink(absolute);
const observed = observedSymlinkTarget(
await readlink(absolute, { encoding: "buffer" }),
);
if (observed.text === null)
return { kind: "symlink", target: null, targetHex: observed.hex };
const target = observed.text;
// Windows link text separates components with `\`.
return {
kind: "symlink",
@@ -333,6 +339,11 @@ const requireMachoRoots = async (
const lookup = await resolveTreePath(view, path);
if (lookup.kind === "escapes")
throw reject("path", `Root ${path} resolves outside the analyzed root`);
if (lookup.kind === "undetermined")
throw reject(
"path",
`Root ${path} crosses symlink ${lookup.path} with non-UTF-8 target bytes ${lookup.targetHex}`,
);
if (lookup.kind !== "file")
throw reject(
"path",
@@ -0,0 +1,916 @@
import { describe, expect, it } from "vitest";
import { CPU, fatImage, machoImage, readerOf } from "./MachoImage.fixture.js";
import { readMachoEncryption } from "./MachoEncryptionReader.js";
import {
MAX_LOAD_COMMAND_BYTES,
readMachoImage,
} from "./MachoLoadCommandReader.js";
const LC_SEGMENT = 0x1;
const LC_SEGMENT_64 = 0x19;
const LC_ENCRYPTION_INFO = 0x21;
const LC_ENCRYPTION_INFO_64 = 0x2c;
const S_ZEROFILL = 0x1;
const read = (bytes: Uint8Array, maximumMappingRecords = 4096) =>
readMachoEncryption(readerOf(bytes), bytes.length, maximumMappingRecords);
const encryption = (
fields: {
readonly cryptoff: number;
readonly cryptsize: number;
readonly cryptid: number;
},
wide = true,
size = wide ? 24 : 20,
): Uint8Array => {
const bytes = new Uint8Array(size);
const view = new DataView(bytes.buffer);
view.setUint32(0, wide ? LC_ENCRYPTION_INFO_64 : LC_ENCRYPTION_INFO, true);
view.setUint32(4, size, true);
if (size >= 20) {
view.setUint32(8, fields.cryptoff, true);
view.setUint32(12, fields.cryptsize, true);
view.setUint32(16, fields.cryptid, true);
}
return bytes;
};
const segment = (
name: string,
fields: {
readonly vmAddress: bigint;
/** Defaults to the file size, as linkers emit for file-backed segments. */
readonly vmSize?: bigint;
readonly fileOffset: number;
readonly fileSize: number;
readonly sections?: readonly {
readonly name: string;
readonly offset: number;
readonly size: number;
readonly flags?: number;
}[];
},
wide = true,
): Uint8Array => {
const sections = fields.sections ?? [];
const header = wide ? 72 : 56;
const record = wide ? 80 : 68;
const bytes = new Uint8Array(header + sections.length * record);
const view = new DataView(bytes.buffer);
const encoder = new TextEncoder();
view.setUint32(0, wide ? LC_SEGMENT_64 : LC_SEGMENT, true);
view.setUint32(4, bytes.length, true);
bytes.set(encoder.encode(name), 8);
if (wide) {
view.setBigUint64(24, fields.vmAddress, true);
view.setBigUint64(32, fields.vmSize ?? BigInt(fields.fileSize), true);
view.setBigUint64(40, BigInt(fields.fileOffset), true);
view.setBigUint64(48, BigInt(fields.fileSize), true);
view.setUint32(64, sections.length, true);
} else {
view.setUint32(24, Number(fields.vmAddress), true);
view.setUint32(28, Number(fields.vmSize ?? BigInt(fields.fileSize)), true);
view.setUint32(32, fields.fileOffset, true);
view.setUint32(36, fields.fileSize, true);
view.setUint32(48, sections.length, true);
}
sections.forEach((section, index) => {
const at = header + index * record;
bytes.set(encoder.encode(section.name), at);
bytes.set(encoder.encode(name), at + 16);
if (wide) {
view.setBigUint64(at + 40, BigInt(section.size), true);
view.setUint32(at + 48, section.offset, true);
view.setUint32(at + 64, section.flags ?? 0, true);
} else {
view.setUint32(at + 36, section.size, true);
view.setUint32(at + 40, section.offset, true);
view.setUint32(at + 56, section.flags ?? 0, true);
}
});
return bytes;
};
/** A thin image padded so that commands can name ranges past its header. */
const padded = (image: Uint8Array, size = 0x10000): Uint8Array => {
const bytes = new Uint8Array(size);
bytes.set(image);
return bytes;
};
const text = (wide = true) =>
segment(
"__TEXT",
{
vmAddress: wide ? 0x100000000n : 0x1000n,
fileOffset: 0,
fileSize: 0x8000,
sections: [
{ name: "__text", offset: 0x4000, size: 0x100 },
{ name: "__cstring", offset: 0x7f00, size: 0x100 },
{ name: "__bss", offset: 0, size: 0x100, flags: S_ZEROFILL },
],
},
wide,
);
const onlySlice = async (bytes: Uint8Array) => {
const facts = await read(bytes);
if (facts.status !== "parsed" || facts.slices.length !== 1)
throw new Error(`expected one parsed slice, got ${JSON.stringify(facts)}`);
return facts.slices[0]!;
};
describe("Mach-O encryption reader: thin images", () => {
it("reports absent only after reading every load command", async () => {
expect(await onlySlice(machoImage({ commands: [text()] }))).toEqual({
architecture: null,
cpu_type: CPU.arm64.type,
cpu_subtype: CPU.arm64.subtype,
container_file_offset: 0,
size: expect.any(Number),
status: "absent",
reason: null,
commands: [],
range: null,
});
});
it("maps a plaintext range to segments, sections and VM addresses", async () => {
const slice = await onlySlice(
padded(
machoImage({
commands: [
text(),
encryption({ cryptoff: 0x4000, cryptsize: 0x4000, cryptid: 0 }),
],
}),
),
);
expect(slice).toMatchObject({
status: "not-encrypted",
reason: null,
commands: [
{
kind: "LC_ENCRYPTION_INFO_64",
index: 1,
cmdsize: 24,
cryptoff: 0x4000,
cryptsize: 0x4000,
cryptid: 0,
pad: 0,
},
],
range: {
content: "plaintext",
slice_start: 0x4000,
slice_end: 0x8000,
container_start: 0x4000,
container_end: 0x8000,
mapping: {
status: "mapped",
fragments: [
{
slice_start: 0x4000,
slice_end: 0x8000,
segment: "__TEXT",
vm_start: "0x100004000",
vm_end: "0x100008000",
},
],
// Zero-fill sections occupy no file bytes, whatever their offset.
sections: [
{ segment: "__TEXT", section: "__text" },
{ segment: "__TEXT", section: "__cstring" },
],
},
},
});
});
it("marks a nonzero cryptid as ciphertext and keeps unmapped bytes explicit", async () => {
const slice = await onlySlice(
padded(
machoImage({
commands: [
text(),
encryption({ cryptoff: 0x7000, cryptsize: 0x2000, cryptid: 1 }),
],
}),
),
);
expect(slice.status).toBe("encrypted");
expect(slice.range).toMatchObject({
content: "ciphertext",
mapping: {
fragments: [
{ slice_start: 0x7000, slice_end: 0x8000, segment: "__TEXT" },
{
slice_start: 0x8000,
slice_end: 0x9000,
segment: null,
vm_start: null,
vm_end: null,
},
],
sections: [{ segment: "__TEXT", section: "__cstring" }],
},
});
});
});
describe("Mach-O encryption reader: thin image edge cases", () => {
it("keeps an encrypted empty range without claiming any bytes", async () => {
const slice = await onlySlice(
padded(
machoImage({
commands: [
text(),
encryption({ cryptoff: 0x4000, cryptsize: 0, cryptid: 1 }),
],
}),
),
);
expect(slice).toMatchObject({
status: "encrypted",
range: {
slice_start: 0x4000,
slice_end: 0x4000,
mapping: { status: "mapped", fragments: [], sections: [] },
},
});
});
it("reads the 32-bit command form in a 32-bit image", async () => {
const slice = await onlySlice(
padded(
machoImage({
cpu: CPU.i386,
wide: false,
commands: [
text(false),
encryption(
{ cryptoff: 0x1000, cryptsize: 0x1000, cryptid: 1 },
false,
),
],
}),
),
);
expect(slice).toMatchObject({
status: "encrypted",
commands: [{ kind: "LC_ENCRYPTION_INFO", cmdsize: 20, pad: null }],
range: {
mapping: {
fragments: [{ vm_start: "0x2000", vm_end: "0x3000" }],
},
},
});
});
it("reports an unreadable segment mapping as unknown, not unmapped", async () => {
const broken = text();
new DataView(broken.buffer).setUint32(64, 9, true);
const slice = await onlySlice(
padded(
machoImage({
commands: [
broken,
encryption({ cryptoff: 0x4000, cryptsize: 0x100, cryptid: 1 }),
],
}),
),
);
expect(slice).toMatchObject({
status: "encrypted",
range: {
mapping: {
status: "unknown",
reason: "load command 0 section records exceed cmdsize",
},
},
});
});
});
describe("Mach-O encryption reader: segment mapping", () => {
const ciphertext = encryption({
cryptoff: 0x4000,
cryptsize: 0x2000,
cryptid: 1,
});
it("never reports a byte twice when segment file ranges overlap", async () => {
const first = segment("__A", {
vmAddress: 0x1000n,
fileOffset: 0x4000,
fileSize: 0x1800,
});
const second = segment("__B", {
vmAddress: 0x9000n,
fileOffset: 0x5000,
fileSize: 0x1000,
});
const slice = await onlySlice(
padded(machoImage({ commands: [first, second, ciphertext] })),
);
expect(slice.range?.mapping).toMatchObject({
status: "mapped",
fragments: [
{ slice_start: 0x4000, slice_end: 0x5800, segment: "__A" },
{
slice_start: 0x5800,
slice_end: 0x6000,
segment: "__B",
vm_start: "0x9800",
vm_end: "0xa000",
},
],
});
});
it("reports a segment command of the other width as an unknown mapping", async () => {
const slice = await onlySlice(
padded(
machoImage({
cpu: CPU.i386,
wide: false,
commands: [
text(true),
encryption(
{ cryptoff: 0x4000, cryptsize: 0x100, cryptid: 1 },
false,
),
],
}),
),
);
expect(slice).toMatchObject({
status: "encrypted",
range: {
mapping: {
status: "unknown",
reason: "load command 0 LC_SEGMENT_64 appears in a 32-bit image",
},
},
});
});
it("names only sections whose bytes the range and their segment both cover", async () => {
const spanning = segment("__TEXT", {
vmAddress: 0x1000n,
fileOffset: 0x4000,
fileSize: 0x2000,
sections: [
{ name: "__text", offset: 0x3000, size: 0x2000 },
// Declared past the segment's file-backed bytes.
{ name: "__stray", offset: 0x7000, size: 0x1000 },
],
});
const covered = await onlySlice(
padded(
machoImage({
commands: [
spanning,
encryption({ cryptoff: 0x4000, cryptsize: 0x4000, cryptid: 1 }),
],
}),
),
);
expect(covered.range?.mapping).toMatchObject({
sections: [{ segment: "__TEXT", section: "__text" }],
});
const empty = await onlySlice(
padded(
machoImage({
commands: [
spanning,
encryption({ cryptoff: 0x4000, cryptsize: 0, cryptid: 1 }),
],
}),
),
);
expect(empty.range?.mapping).toMatchObject({ fragments: [], sections: [] });
});
it("keeps segment name bytes 0x80-0x9f as their own code points", async () => {
const named = segment("__X", {
vmAddress: 0x1000n,
fileOffset: 0x4000,
fileSize: 0x2000,
});
named[11] = 0x80;
const slice = await onlySlice(
padded(machoImage({ commands: [named, ciphertext] })),
);
expect(slice.range?.mapping).toMatchObject({
fragments: [{ segment: "__X\u0080" }],
});
});
});
describe("Mach-O encryption reader: segment VM extent", () => {
const ciphertext = encryption({
cryptoff: 0x4000,
cryptsize: 0x2000,
cryptid: 1,
});
it("keeps file-only segment bytes without VM addresses", async () => {
const fileOnly = segment("__FILE", {
vmAddress: 0x4000n,
vmSize: 0n,
fileOffset: 0x4000,
fileSize: 0x2000,
});
const slice = await onlySlice(
padded(machoImage({ commands: [fileOnly, ciphertext] })),
);
expect(slice.status).toBe("encrypted");
expect(slice.range?.mapping).toEqual({
status: "mapped",
fragments: [
{
slice_start: 0x4000,
slice_end: 0x6000,
segment: "__FILE",
vm_start: null,
vm_end: null,
},
],
sections: [],
});
});
it.each([
[
"a file size beyond its VM size",
{ vmAddress: 0x1000n, vmSize: 0x1000n },
"segment __TEXT file size 8192 exceeds its VM size 4096",
],
[
"a VM range beyond the address space",
{ vmAddress: 2n ** 64n - 0x1000n, vmSize: 0x2000n },
"segment __TEXT VM range exceeds the 64-bit address space",
],
])(
"reports %s as an unknown mapping without losing the status",
async (_case, vm, reason) => {
const invalid = segment("__TEXT", {
...vm,
fileOffset: 0x4000,
fileSize: 0x2000,
});
const slice = await onlySlice(
padded(machoImage({ commands: [invalid, ciphertext] })),
);
expect(slice.status).toBe("encrypted");
expect(slice.range?.content).toBe("ciphertext");
expect(slice.range?.mapping).toEqual({
status: "unknown",
reason: `load command 0 ${reason}`,
});
},
);
});
describe("Mach-O encryption reader: defects are never absent", () => {
it.each([
{
name: "a 64-bit command in a 32-bit image",
image: () =>
machoImage({
cpu: CPU.i386,
wide: false,
commands: [encryption({ cryptoff: 0, cryptsize: 0, cryptid: 0 })],
}),
reason: "load command 0 LC_ENCRYPTION_INFO_64 appears in a 32-bit image",
},
{
name: "an unexpected cmdsize",
image: () =>
machoImage({
commands: [
encryption({ cryptoff: 0, cryptsize: 0, cryptid: 0 }, true, 32),
],
}),
reason:
"load command 0 LC_ENCRYPTION_INFO_64 has cmdsize 32, expected 24",
},
{
name: "a range beyond the slice",
image: () =>
machoImage({
commands: [
encryption({ cryptoff: 0x4000, cryptsize: 0x4000, cryptid: 1 }),
],
}),
reason: expect.stringMatching(
/^load command 0 encrypted range ends at 32768, beyond the \d+-byte slice$/u,
),
},
{
name: "two encryption commands",
image: () =>
machoImage({
commands: [
encryption({ cryptoff: 0, cryptsize: 0, cryptid: 0 }),
encryption({ cryptoff: 0, cryptsize: 0, cryptid: 1 }),
],
}),
reason: "slice has 2 encryption load commands",
},
])("reports $name as malformed", async ({ image, reason }) => {
const slice = await onlySlice(image());
expect(slice).toMatchObject({ status: "malformed", reason, range: null });
expect(slice.commands.length).toBeGreaterThan(0);
});
it("retains two of many encryption commands and counts the rest", async () => {
const slice = await onlySlice(
machoImage({
commands: Array.from({ length: 1000 }, () =>
encryption({ cryptoff: 0, cryptsize: 0, cryptid: 1 }),
),
}),
);
expect(slice).toMatchObject({
status: "malformed",
reason: "slice has 1000 encryption load commands",
range: null,
});
expect(slice.commands.map(({ index }) => index)).toEqual([0, 1]);
});
it("keeps commands read before a framing defect as partial facts", async () => {
const unsized = new Uint8Array(16);
new DataView(unsized.buffer).setUint32(0, 0x7f000001, true);
const slice = await onlySlice(
machoImage({
commands: [
encryption({ cryptoff: 0, cryptsize: 0, cryptid: 1 }),
unsized,
],
}),
);
expect(slice).toMatchObject({
status: "malformed",
reason: "load command 1 at offset 24 has invalid cmdsize 0",
commands: [{ kind: "LC_ENCRYPTION_INFO_64", cryptid: 1 }],
range: null,
});
});
it("reports the 20-byte form in a 64-bit image as a framing defect", async () => {
const slice = await onlySlice(
machoImage({
commands: [
encryption({ cryptoff: 0, cryptsize: 0, cryptid: 1 }, false),
],
}),
);
expect(slice).toMatchObject({
status: "malformed",
reason: "load command 0 at offset 0 has invalid cmdsize 20",
commands: [],
});
});
it("reports unread load commands as unknown", async () => {
const oversized = machoImage({});
new DataView(oversized.buffer).setUint32(
20,
MAX_LOAD_COMMAND_BYTES + 8,
true,
);
expect(await onlySlice(oversized)).toMatchObject({
status: "unknown",
reason: expect.stringContaining("above the"),
});
});
it("keeps file-level failures distinct from slice status", async () => {
expect(await read(Uint8Array.from([1, 2, 3, 4, 5, 6, 7, 8]))).toEqual({
status: "not-mach-o",
});
expect(
await read(Uint8Array.from([0xfe, 0xed, 0xfa, 0xcf, 0, 0, 0, 0])),
).toEqual({
status: "unsupported",
reason: "big-endian Mach-O images are not supported",
});
});
});
/** A padded image whose __TEXT range 0x4000-0x8000 is ciphertext. */
const encrypted = (
cpu: { readonly type: number; readonly subtype: number } = CPU.arm64,
) =>
padded(
machoImage({
cpu,
commands: [
text(),
encryption({ cryptoff: 0x4000, cryptsize: 0x4000, cryptid: 1 }),
],
}),
);
describe("Mach-O encryption reader: universal files", () => {
it("reports every slice in container coordinates", async () => {
const facts = await read(
fatImage([
{ cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) },
{ cpu: CPU.arm64, bytes: encrypted() },
]),
);
expect(facts).toMatchObject({
status: "parsed",
slices: [
{
cpu_type: CPU.x86_64.type,
container_file_offset: 4096,
status: "absent",
},
{
cpu_type: CPU.arm64.type,
container_file_offset: 8192,
status: "encrypted",
range: {
container_start: 8192 + 0x4000,
container_end: 8192 + 0x8000,
},
},
],
});
});
it("does not let one defective slice hide the others", async () => {
const bigEndian = new Uint8Array(32);
new DataView(bigEndian.buffer).setUint32(0, 0xfeedfacf, false);
const fat = fatImage([
{ cpu: CPU.arm64, bytes: bigEndian },
{ cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.arm64 }) },
{ cpu: CPU.arm64, bytes: machoImage({}) },
{ cpu: CPU.arm64e, bytes: encrypted(CPU.arm64e) },
]);
// The third declaration's range runs past the end of the file.
new DataView(fat.buffer).setUint32(8 + 2 * 20 + 12, fat.length, false);
const facts = await read(fat);
if (facts.status !== "parsed") throw new Error(facts.status);
expect(
facts.slices.map(({ status, reason }) => ({ status, reason })),
).toEqual([
{
status: "unknown",
reason: "big-endian Mach-O slices are not supported",
},
{
status: "malformed",
reason:
"FAT architecture 1 CPU type/subtype disagrees with its Mach-O header",
},
{
status: "malformed",
reason: "FAT architecture 2 slice range extends beyond the file",
},
{ status: "encrypted", reason: null },
]);
expect(facts.slices[2]).toMatchObject({
cpu_type: CPU.arm64.type,
cpu_subtype: CPU.arm64.subtype,
commands: [],
});
});
it("keeps arm64e variants apart by CPU subtype", async () => {
const facts = await read(
fatImage([
{ cpu: CPU.arm64e, bytes: machoImage({ cpu: CPU.arm64e }) },
{ cpu: CPU.arm64eX1, bytes: machoImage({ cpu: CPU.arm64eX1 }) },
]),
);
expect(facts).toMatchObject({
status: "parsed",
slices: [
{ cpu_subtype: CPU.arm64e.subtype, status: "absent" },
{ cpu_subtype: CPU.arm64eX1.subtype, status: "absent" },
],
});
});
it("covers Apple's hidden ARM64 slice after the declared rows", async () => {
const armv7 = { type: 12, subtype: 9 };
const bytes = fatImage([
{ cpu: armv7, bytes: machoImage({ cpu: armv7 }) },
{
cpu: CPU.arm64,
bytes: padded(
machoImage({
commands: [
text(),
encryption({ cryptoff: 0x4000, cryptsize: 0x4000, cryptid: 1 }),
],
}),
),
},
]);
// nfat_arch 1 leaves the ARM64 row hidden in the table-to-slice gap.
new DataView(bytes.buffer, bytes.byteOffset).setUint32(4, 1);
expect(await read(bytes)).toMatchObject({
status: "parsed",
slices: [
{ cpu_type: armv7.type, status: "absent" },
{ cpu_type: CPU.arm64.type, status: "encrypted" },
],
});
});
it("keeps the declared slice when the hidden ARM64 row's range is invalid", async () => {
const armv7 = { type: 12, subtype: 9 };
const bytes = fatImage([
{ cpu: armv7, bytes: machoImage({ cpu: armv7 }) },
{ cpu: CPU.arm64, bytes: padded(machoImage({ commands: [text()] })) },
]);
const view = new DataView(bytes.buffer, bytes.byteOffset);
view.setUint32(4, 1);
// The hidden row's size runs past the end of the file.
view.setUint32(8 + 20 + 12, bytes.byteLength);
expect(await read(bytes)).toMatchObject({
status: "parsed",
slices: [
{ cpu_type: armv7.type, status: "absent" },
{
cpu_type: CPU.arm64.type,
status: "malformed",
reason: "FAT architecture 1 slice range extends beyond the file",
commands: [],
},
],
});
});
});
describe("Mach-O encryption reader: large universal files", () => {
it("keeps a healthy slice when a sibling's range is invalid past the 16 MiB scan limit", async () => {
const slice = padded(
machoImage({
commands: [
text(),
encryption({ cryptoff: 0x4000, cryptsize: 0x4000, cryptid: 1 }),
],
}),
);
const size = MAX_LOAD_COMMAND_BYTES + 0x100000;
const table = new DataView(new ArrayBuffer(8 + 2 * 20));
table.setUint32(0, 0xcafebabe);
table.setUint32(4, 2);
const entry = (index: number, fields: readonly number[]) =>
fields.forEach((value, field) =>
table.setUint32(8 + index * 20 + field * 4, value),
);
entry(0, [CPU.arm64.type, CPU.arm64.subtype, 0x1000, slice.length, 12]);
// Declared past the end of the file.
entry(1, [CPU.x86_64.type, CPU.x86_64.subtype, size, 0x1000, 12]);
const prefix = new Uint8Array(0x1000 + slice.length);
prefix.set(new Uint8Array(table.buffer));
prefix.set(slice, 0x1000);
// The bytes past the slice are zero, so the reader never needs them.
const sparseReadAt = async (offset: number, length: number) => {
const result = new Uint8Array(
Math.max(0, Math.min(length, size - offset)),
);
if (offset < prefix.length)
result.set(prefix.subarray(offset, offset + result.length));
return result;
};
expect(await readMachoEncryption(sparseReadAt, size, 4096)).toMatchObject({
status: "parsed",
slices: [
{ container_file_offset: 0x1000, status: "encrypted" },
{
container_file_offset: size,
status: "malformed",
reason: "FAT architecture 1 slice range extends beyond the file",
},
],
});
// The confirmed container reports the sibling's real fault, not an
// unresolved JVM identity.
expect(await readMachoImage(sparseReadAt, size)).toEqual({
status: "malformed",
reason: "FAT architecture 1 slice range extends beyond the file",
});
});
});
describe("Mach-O encryption reader: mapping budget", () => {
it.each([
"section bytes",
"VM size",
"VM overflow",
"command width",
] as const)(
"releases failed %s mappings before admitting the next slice",
async (failure) => {
const invalid = segment(
"__BROKEN",
{
vmAddress:
failure === "VM overflow" ? 0xffffffffffffffffn : 0x100000000n,
vmSize: failure === "VM size" ? 0xffffn : 0x10000n,
fileOffset: 0,
fileSize: 0x10000,
},
failure !== "command width",
);
if (failure === "section bytes")
new DataView(invalid.buffer).setUint32(64, 1, true);
// Two records for the bad segment and five for text(): the first
// slice fills admission but discards its mapping on validation failure.
const facts = await read(
fatImage([
{
cpu: CPU.x86_64,
bytes: padded(
machoImage({
cpu: CPU.x86_64,
commands: [
invalid,
text(),
encryption({
cryptoff: 0x4000,
cryptsize: 0x4000,
cryptid: 1,
}),
],
}),
),
},
{ cpu: CPU.arm64, bytes: encrypted() },
]),
7,
);
if (facts.status !== "parsed") throw new Error(facts.status);
expect(facts.slices[0]).toMatchObject({
status: "encrypted",
range: { mapping: { status: "unknown", reason: expect.any(String) } },
});
const healthy = await read(encrypted());
if (healthy.status !== "parsed") throw new Error(healthy.status);
expect(facts.slices[1]?.range?.mapping).toEqual(
healthy.slices[0]?.range?.mapping,
);
expect(facts.slices[1]?.range?.mapping.status).toBe("mapped");
},
);
it("leaves the budget to later slices when a slice does not map its range", async () => {
const facts = await read(
fatImage([
{
cpu: CPU.x86_64,
bytes: padded(machoImage({ cpu: CPU.x86_64, commands: [text()] })),
},
{ cpu: CPU.arm64, bytes: encrypted() },
]),
5,
);
if (facts.status !== "parsed") throw new Error(facts.status);
expect(
facts.slices.map(({ status, range }) => [status, range?.mapping.status]),
).toEqual([
["absent", undefined],
["encrypted", "mapped"],
]);
});
it("shares one mapping budget across slices and keeps each status", async () => {
// text() is one segment with three section records: five mapping records.
const facts = await read(
fatImage([
{ cpu: CPU.arm64, bytes: encrypted() },
{ cpu: CPU.arm64e, bytes: encrypted(CPU.arm64e) },
]),
5,
);
if (facts.status !== "parsed") throw new Error(facts.status);
expect(
facts.slices.map(({ status, range }) => ({
status,
mapping: range?.mapping.status,
})),
).toEqual([
{ status: "encrypted", mapping: "mapped" },
{ status: "encrypted", mapping: "unknown" },
]);
expect(facts.slices[1]?.range?.mapping).toEqual({
status: "unknown",
reason:
"mapping needs more than the 5 segment and section records shared by all slices",
});
});
});
@@ -0,0 +1,503 @@
import type {
MachoEncryptedRange,
MachoEncryptionCommand,
MachoEncryptionFacts,
MachoSliceEncryption,
} from "../../domain/apple/machoEncryption.js";
import { validateFatSliceIdentity } from "../../domain/apple/machoContainer.js";
import { readMachoSectionHeader } from "../../domain/apple/machoSectionHeader.js";
import { readMachoSegmentHeader } from "../../domain/apple/machoSegmentHeader.js";
import {
frameLoadCommands,
readMachoContainer,
readSliceCommandTable,
type MachoLoadCommand,
type MachoReadIssue,
type MachoSliceHeader,
type ReadAt,
} from "./MachoLoadCommandReader.js";
const LC_SEGMENT = 0x1;
const LC_SEGMENT_64 = 0x19;
const LC_ENCRYPTION_INFO = 0x21;
const LC_ENCRYPTION_INFO_64 = 0x2c;
// loader.h: encryption_info_command is five 32-bit words; the 64-bit form adds `pad`.
const ENCRYPTION_COMMAND_SIZE = { 32: 20, 64: 24 } as const;
// loader.h section types whose bytes occupy no file range.
const ZEROFILL_SECTION_TYPES = new Set([0x1, 0xc, 0x12]);
// A second encryption command already makes a slice malformed; later ones
// are counted, not retained.
const RETAINED_ENCRYPTION_COMMANDS = 2;
/** Segment and section records left for range mapping across every slice. */
interface MappingBudget {
readonly maximum: number;
remaining: number;
}
/**
* Read each slice's `LC_ENCRYPTION_INFO(_64)` status from structural metadata.
* The shared container recognizer owns the reads needed for JVM disambiguation.
* One slice's defect never hides another slice's status. Range mapping in all
* slices shares `maximumMappingRecords` segment and section records; a slice
* past it keeps its status with an unknown mapping.
*/
export const readMachoEncryption = async (
readAt: ReadAt,
size: number,
maximumMappingRecords: number,
): Promise<MachoEncryptionFacts> => {
const budget: MappingBudget = {
maximum: maximumMappingRecords,
remaining: maximumMappingRecords,
};
const container = await readMachoContainer(readAt, size);
if (container.kind === "not-mach-o") return { status: "not-mach-o" };
if (container.kind === "issue")
return { status: container.status, reason: container.reason };
if (container.kind === "thin")
return {
status: "parsed",
slices: [await readSlice(readAt, { offset: 0, size, fat: null }, budget)],
};
// Apple's hidden ARM64 slice is part of the file's coverage, so an
// undecodable hidden row (a short read) leaves the slice set itself
// unknown before any declared slice is read.
const { hidden } = container;
if (hidden !== null && hidden.status !== "declared")
return { status: hidden.status, reason: hidden.reason };
const slices: MachoSliceEncryption[] = [];
for (const [index, declaration] of container.header.slices.entries()) {
const rangeIssue = container.header.rangeIssues[index] ?? null;
slices.push(
rangeIssue === null
? await readSlice(
readAt,
{
offset: declaration.offset,
size: declaration.size,
fat: declaration,
},
budget,
)
: unread(
declaration,
rangeIssue.category === "unsupported" ? "unknown" : "malformed",
rangeIssue.reason,
),
);
}
if (hidden !== null)
slices.push(
hidden.rangeIssue === null
? await readSlice(
readAt,
{
offset: hidden.declaration.offset,
size: hidden.declaration.size,
fat: hidden.declaration,
},
budget,
)
: unread(hidden.declaration, "malformed", hidden.rangeIssue),
);
return { status: "parsed", slices };
};
type FatDeclaration = Parameters<typeof validateFatSliceIdentity>[0];
const unread = (
declaration: FatDeclaration,
status: "malformed" | "unknown",
reason: string,
): MachoSliceEncryption => ({
architecture: null,
cpu_type: declaration.cpuType,
cpu_subtype: declaration.cpuSubtype,
container_file_offset: declaration.offset,
size: declaration.size,
status,
reason,
commands: [],
range: null,
});
const readSlice = async (
readAt: ReadAt,
slice: {
readonly offset: number;
readonly size: number;
readonly fat: FatDeclaration | null;
},
budget: MappingBudget,
): Promise<MachoSliceEncryption> => {
const table = await readSliceCommandTable(readAt, slice.offset, slice.size);
const base = {
architecture: null,
cpu_type: table.header?.cpuType ?? slice.fat?.cpuType ?? null,
cpu_subtype: table.header?.cpuSubtype ?? slice.fat?.cpuSubtype ?? null,
container_file_offset: slice.offset,
size: slice.size,
};
// Only encryption and segment commands are retained; the rest are framed
// and dropped so a large table of small commands stays one buffer.
const commands: MachoEncryptionCommand[] = [];
let encryptionCommands = 0;
// Null once this slice's segments need more records than the shared
// mapping budget has left; none are retained past that point.
let segments: MachoLoadCommand[] | null = [];
let segmentRecords = 0;
// Commands seen before a failure stay as partial facts; an incomplete walk
// can never prove the command is absent.
const incomplete = ({
status,
reason,
}: MachoReadIssue): MachoSliceEncryption => ({
...base,
status: status === "unsupported" ? "unknown" : "malformed",
reason,
commands,
range: null,
});
if (table.status !== "read") return incomplete(table);
const { header } = table;
const framed = frameLoadCommands(table, (command) => {
if (isEncryptionCommand(command.command)) {
encryptionCommands += 1;
if (commands.length < RETAINED_ENCRYPTION_COMMANDS)
commands.push(decodeEncryptionCommand(command));
} else if (
segments !== null &&
(command.command === LC_SEGMENT || command.command === LC_SEGMENT_64)
) {
segmentRecords += mappingRecords(command);
if (segmentRecords > budget.remaining) segments = null;
else segments.push(command);
}
});
if (framed.status !== "framed") return incomplete(framed);
const identityIssue =
slice.fat === null
? null
: validateFatSliceIdentity(slice.fat, {
cpuType: header.cpuType,
cpuSubtype: header.cpuSubtype,
});
if (identityIssue !== null)
return {
...base,
status: "malformed",
reason: identityIssue,
commands,
range: null,
};
const outcome = classify(commands, encryptionCommands, header, slice.size);
if (outcome.status !== "valid")
return { ...base, ...outcome, commands, range: null };
const range = encryptedRange(
outcome.command,
header,
segments,
slice.offset,
budget.maximum,
);
// Framing caps transient decoding; only a completed mapping retains its
// reservation. Failed mappings discard those records before the next slice.
if (range.mapping.status === "mapped") budget.remaining -= segmentRecords;
return {
...base,
status: outcome.command.cryptid === 0 ? "not-encrypted" : "encrypted",
reason: null,
commands,
range,
};
};
const isEncryptionCommand = (command: number): boolean =>
command === LC_ENCRYPTION_INFO || command === LC_ENCRYPTION_INFO_64;
const decodeEncryptionCommand = ({
command,
body,
index,
}: MachoLoadCommand): MachoEncryptionCommand => {
const view = viewOf(body);
const word = (at: number): number | null =>
body.byteLength >= at + 4 ? view.getUint32(at, true) : null;
const wide = command === LC_ENCRYPTION_INFO_64;
return {
kind: wide ? "LC_ENCRYPTION_INFO_64" : "LC_ENCRYPTION_INFO",
index,
cmdsize: body.byteLength,
cryptoff: word(8),
cryptsize: word(12),
cryptid: word(16),
pad: wide ? word(20) : null,
};
};
type Classification =
| {
readonly status: "valid";
readonly command: MachoEncryptionCommand & {
readonly cryptoff: number;
readonly cryptsize: number;
readonly cryptid: number;
};
}
| { readonly status: "absent"; readonly reason: null }
| { readonly status: "malformed"; readonly reason: string };
const classify = (
commands: readonly MachoEncryptionCommand[],
count: number,
header: MachoSliceHeader,
sliceSize: number,
): Classification => {
if (count === 0) return { status: "absent", reason: null };
const [command] = commands;
if (command === undefined || count > 1)
return {
status: "malformed",
reason: `slice has ${count} encryption load commands`,
};
const width = header.wide ? 64 : 32;
const expectedKind = header.wide
? "LC_ENCRYPTION_INFO_64"
: "LC_ENCRYPTION_INFO";
if (command.kind !== expectedKind)
return {
status: "malformed",
reason: `load command ${command.index} ${command.kind} appears in a ${width}-bit image`,
};
const expectedSize = ENCRYPTION_COMMAND_SIZE[width];
const { cryptoff, cryptsize, cryptid } = command;
if (
command.cmdsize !== expectedSize ||
cryptoff === null ||
cryptsize === null ||
cryptid === null
)
return {
status: "malformed",
reason: `load command ${command.index} ${command.kind} has cmdsize ${command.cmdsize}, expected ${expectedSize}`,
};
// Both fields are 32-bit, so their sum is a safe integer.
if (cryptoff + cryptsize > sliceSize)
return {
status: "malformed",
reason: `load command ${command.index} encrypted range ends at ${cryptoff + cryptsize}, beyond the ${sliceSize}-byte slice`,
};
return {
status: "valid",
command: { ...command, cryptoff, cryptsize, cryptid },
};
};
const encryptedRange = (
command: {
readonly cryptoff: number;
readonly cryptsize: number;
readonly cryptid: number;
},
header: MachoSliceHeader,
segmentCommands: readonly MachoLoadCommand[] | null,
containerOffset: number,
maximumMappingRecords: number,
): MachoEncryptedRange => {
const start = command.cryptoff;
const end = command.cryptoff + command.cryptsize;
return {
content: command.cryptid === 0 ? "plaintext" : "ciphertext",
slice_start: start,
slice_end: end,
container_start: containerOffset + start,
container_end: containerOffset + end,
mapping:
segmentCommands === null
? {
status: "unknown",
reason: `mapping needs more than the ${maximumMappingRecords} segment and section records shared by all slices`,
}
: mapRange(start, end, header, segmentCommands),
};
};
interface SegmentFacts {
readonly index: number;
readonly name: string;
readonly vmAddress: bigint;
readonly vmSize: bigint;
readonly fileOffset: number;
readonly fileSize: number;
readonly sections: readonly {
readonly name: string;
readonly offset: number;
readonly size: number;
readonly zerofill: boolean;
}[];
}
const mapRange = (
start: number,
end: number,
header: MachoSliceHeader,
segmentCommands: readonly MachoLoadCommand[],
): MachoEncryptedRange["mapping"] => {
const segments: SegmentFacts[] = [];
for (const command of segmentCommands) {
if ((command.command === LC_SEGMENT_64) !== header.wide)
return {
status: "unknown",
reason: `load command ${command.index} ${command.command === LC_SEGMENT_64 ? "LC_SEGMENT_64" : "LC_SEGMENT"} appears in a ${header.wide ? 64 : 32}-bit image`,
};
const segment = decodeSegment(command);
if (typeof segment === "string")
return { status: "unknown", reason: segment };
// dyld maps only `vmsize` bytes, so file bytes past it have no address.
if (segment.vmSize > 0n && BigInt(segment.fileSize) > segment.vmSize)
return {
status: "unknown",
reason: `load command ${segment.index} segment ${segment.name} file size ${segment.fileSize} exceeds its VM size ${segment.vmSize}`,
};
if (
segment.vmAddress + segment.vmSize >
(header.wide ? 2n ** 64n : 2n ** 32n)
)
return {
status: "unknown",
reason: `load command ${segment.index} segment ${segment.name} VM range exceeds the ${header.wide ? 64 : 32}-bit address space`,
};
segments.push(segment);
}
const fileBacked = segments
.filter(({ fileSize }) => fileSize > 0)
.sort((left, right) => left.fileOffset - right.fileOffset);
const fragments: Extract<
MachoEncryptedRange["mapping"],
{ status: "mapped" }
>["fragments"] = [];
let cursor = start;
for (const segment of fileBacked) {
// Overlapping segments never report the same byte twice.
const from = Math.max(cursor, segment.fileOffset);
const to = Math.min(end, segment.fileOffset + segment.fileSize);
if (from >= to) continue;
if (from > cursor) fragments.push(unmapped(cursor, from));
// A segment with no VM size keeps file-only bytes that are never mapped.
const mapped = segment.vmSize > 0n;
fragments.push({
slice_start: from,
slice_end: to,
segment: segment.name,
vm_start: mapped
? hex(segment.vmAddress + BigInt(from - segment.fileOffset))
: null,
vm_end: mapped
? hex(segment.vmAddress + BigInt(to - segment.fileOffset))
: null,
});
cursor = Math.max(cursor, to);
}
if (cursor < end) fragments.push(unmapped(cursor, end));
// A section counts only where the range, the section and its segment's
// file-backed bytes all overlap, so names agree with the fragments.
const sections = fileBacked.flatMap((segment) =>
segment.sections
.filter(
({ zerofill, size, offset }) =>
!zerofill &&
Math.max(start, offset, segment.fileOffset) <
Math.min(end, offset + size, segment.fileOffset + segment.fileSize),
)
.map(({ name }) => ({ segment: segment.name, section: name })),
);
return { status: "mapped", fragments, sections };
};
const unmapped = (from: number, to: number) => ({
slice_start: from,
slice_end: to,
segment: null,
vm_start: null,
vm_end: null,
});
/**
* Mapping records a segment command can produce: at most two fragments, and
* each section record that fits in its `cmdsize`, decoded and reported once.
*/
const mappingRecords = ({ command, body }: MachoLoadCommand): number => {
const wide = command === LC_SEGMENT_64;
return (
2 +
Math.max(
0,
Math.floor((body.byteLength - (wide ? 72 : 56)) / (wide ? 80 : 68)),
)
);
};
/** Decode one segment command, or say why its file mapping is unknown. */
const decodeSegment = ({
command,
body,
index,
}: MachoLoadCommand): SegmentFacts | string => {
const wide = command === LC_SEGMENT_64;
const parsed = readMachoSegmentHeader(body, wide, true);
if (!parsed.ok)
return `load command ${index} is too short for a segment command`;
const segment = parsed.value;
const fileOffset = Number(segment.fileOffset);
const fileSize = Number(segment.fileSize);
if (!Number.isSafeInteger(fileOffset + fileSize))
return `load command ${index} segment file range exceeds the safe range`;
const sectionSize = wide ? 80 : 68;
if (
segment.sectionCount >
(body.byteLength - segment.headerSize) / sectionSize
)
return `load command ${index} section records exceed cmdsize`;
const sections: SegmentFacts["sections"][number][] = [];
for (let section = 0; section < segment.sectionCount; section += 1) {
const at = segment.headerSize + section * sectionSize;
const header = readMachoSectionHeader(
body.subarray(at, at + sectionSize),
wide,
true,
);
if (!header.ok)
return `load command ${index} section records exceed cmdsize`;
sections.push({
name: fixedName(header.value.nameBytes),
offset: header.value.offset,
size: Number(header.value.size),
zerofill: ZEROFILL_SECTION_TYPES.has(header.value.flags & 0xff),
});
}
return {
index,
name: fixedName(segment.nameBytes),
vmAddress: segment.address,
vmSize: segment.memorySize,
fileOffset,
fileSize,
sections,
};
};
/** A 16-byte NUL-padded loader.h name, decoded byte for byte. */
const fixedName = (bytes: Uint8Array): string => {
const end = bytes.indexOf(0);
// Node's latin1 maps every byte to one code point; WHATWG "latin1" is
// windows-1252 and would rewrite 0x80-0x9f.
return Buffer.from(end < 0 ? bytes : bytes.subarray(0, end)).toString(
"latin1",
);
};
const hex = (value: bigint): string => `0x${value.toString(16)}`;
const viewOf = (bytes: Uint8Array): DataView =>
new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
@@ -38,6 +38,66 @@ import { javaClassFile } from "../../../tests/fixtures/javaClassFile.js";
const read = (bytes: Uint8Array) =>
readMachoImage(readerOf(bytes), bytes.length);
it.skipIf(process.platform === "win32")(
"keeps non-UTF-8 symlink targets distinct from a real replacement character",
async () => {
const root = await createTestTempDirectory("rea-dylib-raw-symlink-");
const executable = machoImage({
commands: [
buildVersionCommand(1),
dylibCommand(LC.LOAD_DYLIB, "@loader_path/raw-link"),
dylibCommand(LC.LOAD_DYLIB, "@loader_path/unicode-link"),
],
});
await writeFile(join(root, "App"), executable);
await writeFile(
join(root, "lib\ufffd"),
machoImage({
fileType: FILE_TYPE.dylib,
commands: [buildVersionCommand(1)],
}),
);
await symlink(
Buffer.from([0x6c, 0x69, 0x62, 0xff]),
join(root, "raw-link"),
);
await symlink("lib\ufffd", join(root, "unicode-link"));
const trace = await traceDylibResolution({
rootPath: root,
targetPath: join(root, "App"),
targetSha256: createHash("sha256").update(executable).digest("hex"),
enumerateRoots: false,
parameters: { roots: ["App"] },
});
expect(trace.edges.map((edge) => edge.resolution)).toEqual([
{ status: "undetermined", image: null },
{ status: "resolved", image: "lib\ufffd" },
]);
expect(trace.coverage.status).toBe("partial");
expect(trace.limitations).toEqual(
expect.arrayContaining([
expect.stringContaining('"raw-link":"6c6962ff"'),
]),
);
await expect(
traceDylibResolution({
rootPath: root,
targetPath: join(root, "App"),
targetSha256: createHash("sha256").update(executable).digest("hex"),
enumerateRoots: false,
parameters: { roots: ["raw-link"] },
}),
).rejects.toMatchObject({
issues: [
{
path: ["roots", 0],
message: expect.stringContaining("non-UTF-8 target bytes 6c6962ff"),
},
],
});
},
);
it("retains unknown executable-root traversal after a complete callback and keeps known image facts", async () => {
const root = await createTestTempDirectory("rea-dylib-traversal-callback-");
const bytes = machoImage({ commands: [buildVersionCommand(1)] });
@@ -1097,6 +1157,22 @@ describe("LC_LAZY_LOAD_DYLIB_INFO image offsets", () => {
expect(linkedit.status).toBe("malformed");
});
it("does not alias a BOM-prefixed segment name to __LINKEDIT", async () => {
const result = await read(
machoImageWithLazyLoadDylibInfo({
path: "@rpath/libl.dylib",
flagImageOffset: 0x4001,
commandsBefore: [
segmentCommand({
name: "\uFEFF__LINKEDIT",
vmaddr: 0n,
vmsize: 0x100000n,
}),
],
}),
);
expect(result.status).toBe("parsed");
});
it("keeps a short segment parseable until a lazy-info command needs its VM range", async () => {
const short = new Uint8Array(16);
const view = new DataView(short.buffer);
@@ -1882,6 +1958,72 @@ describe("Mach-O load command reader failures", () => {
});
});
describe("Mach-O load command reader defect precedence", () => {
it("keeps defect precedence when one image has several defects", async () => {
const shortRpath = new Uint8Array(8);
const shortView = new DataView(shortRpath.buffer);
shortView.setUint32(0, LC.RPATH, true);
shortView.setUint32(4, 8, true);
const unsized = new Uint8Array(16);
new DataView(unsized.buffer).setUint32(0, LC.RPATH, true);
const decodeFirst = {
status: "malformed",
reason: "load command 0 is too short to hold a string offset",
};
// An earlier command's decode defect wins over a later framing defect,
expect(await read(machoImage({ commands: [shortRpath, unsized] }))).toEqual(
decodeFirst,
);
// over a missing later command,
const missing = machoImage({ commands: [shortRpath] });
new DataView(missing.buffer).setUint32(16, 2, true);
expect(await read(missing)).toEqual(decodeFirst);
// and over unconsumed command bytes.
const unconsumed = machoImage({
commands: [shortRpath, rpathCommand("x")],
});
new DataView(unconsumed.buffer).setUint32(16, 1, true);
expect(await read(unconsumed)).toEqual(decodeFirst);
// The command-size limit is checked before the slice bounds.
const oversized = machoImage({});
new DataView(oversized.buffer).setUint32(
20,
MAX_LOAD_COMMAND_BYTES + 8,
true,
);
expect(await read(oversized)).toMatchObject({ status: "unsupported" });
// A malformed FAT slice wins over an earlier or later unsupported one.
const bigEndian = new Uint8Array(32);
new DataView(bigEndian.buffer).setUint32(0, 0xfeedfacf, false);
const overlong = machoImage({});
new DataView(overlong.buffer).setUint32(20, overlong.byteLength, true);
for (const slices of [
[bigEndian, overlong],
[overlong, bigEndian],
]) {
const fat = fatImage(slices.map((bytes) => ({ cpu: CPU.arm64, bytes })));
expect(await read(fat)).toEqual({
status: "malformed",
reason: "load commands extend beyond the Mach-O slice",
});
}
// The first FAT range issue is reported, whatever its category.
const ranged = fatImage([
{ cpu: CPU.arm64, bytes: machoImage({}) },
{ cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) },
]);
new DataView(ranged.buffer).setUint32(8 + 16, 53, false);
new DataView(ranged.buffer).setUint32(8 + 20 + 12, ranged.length, false);
expect(await read(ranged)).toEqual({
status: "unsupported",
reason: "FAT architecture 0 alignment exponent 53 is unsupported",
});
});
});
it.each([true, false])(
"rejects unconsumed command bytes in wide=%s headers",
async (wide) => {
+368 -207
View File
@@ -36,7 +36,9 @@ import {
readMachoHeaderPrefixFacts,
validateFatSlice,
validateFatSliceIdentity,
type FatContainerHeader,
type FatSliceDeclaration,
type FatSliceRangeIssue,
} from "../../domain/apple/machoContainer.js";
/** Read `length` bytes at `offset`; shorter results mean end of file. */
@@ -100,88 +102,190 @@ class MachoFormatIssue extends Error {
}
}
/** A structural defect, and whether it is malformed or merely unsupported. */
export interface MachoReadIssue {
readonly status: "malformed" | "unsupported";
readonly reason: string;
}
/**
* File-level identity. FAT range issues stay per slice so a tolerant reader
* can still report the slices whose ranges are valid.
*/
export type MachoContainer =
| { readonly kind: "not-mach-o" }
| ({ readonly kind: "issue" } & MachoReadIssue)
| { readonly kind: "thin"; readonly size: number }
| {
readonly kind: "fat";
readonly header: ParsedFatContainerHeader;
readonly hidden: HiddenFatRow | null;
};
/**
* Apple's hidden ARM64 row with its own range defect, or why its slot could
* not be decoded. It stays outside the raw `nfat_arch` declarations.
*/
export type HiddenFatRow =
| {
readonly status: "declared";
readonly declaration: FatSliceDeclaration;
readonly rangeIssue: string | null;
}
| MachoReadIssue;
type ParsedFatContainerHeader = Extract<
FatContainerHeader,
{ readonly state: "parsed" }
>;
/** Classify a file as thin, FAT or not Mach-O without decoding any slice. */
export const readMachoContainer = async (
readAt: ReadAt,
size: number,
): Promise<MachoContainer> => {
const head = await readAt(0, Math.min(size, 8));
if (head.byteLength < 8)
// A recognized magic with no room for a header is a truncated image.
return hasMachoMagic(head)
? {
kind: "issue",
status: "malformed",
reason: "Mach-O header is truncated",
}
: { kind: "not-mach-o" };
const fatPrefix = readFatHeaderPrefix(head);
const mayBeJvmClass =
fatPrefix.state === "parsed" && !fatPrefix.wide && !fatPrefix.littleEndian;
let header: FatContainerHeader;
if (mayBeJvmClass && size <= MAX_JVM_CLASS_IDENTITY_BYTES) {
const complete = await readAt(0, size);
header =
complete.byteLength === size
? readFatContainerHeader(complete, size)
: readFatContainerHeader(head, size);
} else header = readFatContainerHeader(head, size);
if (header.state !== "not-fat") {
if (header.state === "needs-data") {
header = readFatContainerHeader(
await readAt(0, header.requiredBytes),
size,
);
}
if (header.state === "not-fat") return { kind: "not-mach-o" };
if (header.state === "needs-data")
return {
kind: "issue",
status: "malformed",
reason: "FAT table is truncated",
};
if (header.state === "malformed")
return { kind: "issue", status: "malformed", reason: header.reason };
let identity = classifyFatContainerIdentity(header, () => undefined);
if (identity.state === "needs-slice-header") {
const sliceHeaders = new Map<number, number | null>();
for (const offset of identity.offsets)
sliceHeaders.set(offset, await readSliceMagic(readAt, offset));
identity = classifyFatContainerIdentity(header, (offset) =>
sliceHeaders.get(offset),
);
}
if (
identity.state === "not-mach-o" ||
identity.state === "needs-slice-header"
)
return { kind: "not-mach-o" };
// A large CAFEBABE input whose bounded prefix resembles an invalid
// known-CPU FAT table remains unknown until its JVM identity can be
// checked; do not turn the resource limit into a malformed claim. A slice
// with a valid range that starts with a Mach-O header settles identity,
// and its siblings' range faults then stay their own.
if (
identity.state === "malformed" &&
mayBeJvmClass &&
size > MAX_JVM_CLASS_IDENTITY_BYTES &&
!(await observesMachoSlice(readAt, header))
)
return {
kind: "issue",
status: "unsupported",
reason: JVM_CLASS_IDENTITY_LIMIT_REASON,
};
// Aliased declarations are rejected at the declaration owner before any
// slice payload is parsed. A declaration whose own range is already
// defective keeps that fault; it does not turn its neighbours into aliases.
const overlap = findOverlappingFatSlices(rangeValidSlices(header));
if (overlap !== null)
return {
kind: "issue",
status: "malformed",
reason: overlapReason(overlap),
};
// Identity is only malformed through a slice range issue, which the
// returned rangeIssues retain.
return {
kind: "fat",
header,
hidden: await readAppleHiddenArm64Row(readAt, size, header),
};
}
const thin = readMachoHeaderPrefixFacts(head).format;
if (thin?.littleEndian === true) return { kind: "thin", size };
if (thin !== null)
return {
kind: "issue",
status: "unsupported",
reason: "big-endian Mach-O images are not supported",
};
return { kind: "not-mach-o" };
};
/** A slice's first word read big-endian, or null when it is truncated. */
const readSliceMagic = async (
readAt: ReadAt,
offset: number,
): Promise<number | null> => {
const bytes = await readAt(offset, 4);
return bytes.byteLength < 4 ? null : viewOf(bytes).getUint32(0, false);
};
const observesMachoSlice = async (
readAt: ReadAt,
header: ParsedFatContainerHeader,
): Promise<boolean> => {
for (const [index, slice] of header.slices.entries())
if (
header.rangeIssues[index] === null &&
readMachoHeaderPrefixFacts(await readAt(slice.offset, 4)).format !== null
)
return true;
return false;
};
/** Decode every slice's dylib-loading load commands without reading code. */
export const readMachoImage = async (
readAt: ReadAt,
size: number,
): Promise<MachoImageFacts> => {
try {
const head = await readAt(0, Math.min(size, 8));
if (head.byteLength < 8)
// A recognized magic with no room for a header is a truncated image.
return hasMachoMagic(head)
? { status: "malformed", reason: "Mach-O header is truncated" }
: { status: "not-mach-o" };
const fatPrefix = readFatHeaderPrefix(head);
const mayBeJvmClass =
fatPrefix.state === "parsed" &&
!fatPrefix.wide &&
!fatPrefix.littleEndian;
let header: ReturnType<typeof readFatContainerHeader>;
if (mayBeJvmClass && size <= MAX_JVM_CLASS_IDENTITY_BYTES) {
const complete = await readAt(0, size);
header =
complete.byteLength === size
? readFatContainerHeader(complete, size)
: readFatContainerHeader(head, size);
} else header = readFatContainerHeader(head, size);
if (header.state !== "not-fat") {
if (header.state === "needs-data") {
header = readFatContainerHeader(
await readAt(0, header.requiredBytes),
size,
);
}
if (header.state === "not-fat") return { status: "not-mach-o" };
if (header.state === "needs-data")
throw new MachoFormatIssue("malformed", "FAT table is truncated");
if (header.state === "malformed")
throw new MachoFormatIssue("malformed", header.reason);
let identity = classifyFatContainerIdentity(header, () => undefined);
if (identity.state === "needs-slice-header") {
const sliceHeaders = new Map<number, number | null>();
for (const offset of identity.offsets) {
const bytes = await readAt(offset, 4);
sliceHeaders.set(
offset,
bytes.byteLength < 4 ? null : viewOf(bytes).getUint32(0, false),
);
}
identity = classifyFatContainerIdentity(header, (offset) =>
sliceHeaders.get(offset),
);
}
if (identity.state === "not-mach-o") return { status: "not-mach-o" };
if (identity.state === "malformed") {
// A large CAFEBABE input whose bounded prefix resembles an invalid
// known-CPU FAT table remains unknown until its JVM identity can be
// checked; do not turn the resource limit into a malformed claim.
if (mayBeJvmClass && size > MAX_JVM_CLASS_IDENTITY_BYTES)
throw new MachoFormatIssue(
"unsupported",
JVM_CLASS_IDENTITY_LIMIT_REASON,
);
throw new MachoFormatIssue(identity.category, identity.reason);
}
if (identity.state === "needs-slice-header")
return { status: "not-mach-o" };
assertDistinctFatSlices(header.slices);
const hiddenSlice = await readAppleHiddenArm64Slice(readAt, size, header);
return {
status: "parsed",
slices: await readFatSlices(readAt, header.slices),
...(hiddenSlice === null ? {} : { hiddenSlices: [hiddenSlice] }),
};
}
const thin = readMachoHeaderPrefixFacts(head).format;
if (thin?.littleEndian === true)
const container = await readMachoContainer(readAt, size);
if (container.kind === "not-mach-o") return { status: "not-mach-o" };
if (container.kind === "issue")
return { status: container.status, reason: container.reason };
if (container.kind === "thin")
return { status: "parsed", slices: [await readSlice(readAt, 0, size)] };
if (thin !== null)
throw new MachoFormatIssue(
"unsupported",
"big-endian Mach-O images are not supported",
);
return { status: "not-mach-o" };
const { header } = container;
const rangeIssue = header.rangeIssues.find(
(issue): issue is FatSliceRangeIssue => issue !== null,
);
if (rangeIssue !== undefined)
return { status: rangeIssue.category, reason: rangeIssue.reason };
const hiddenSlice = await readHiddenSlice(readAt, container.hidden);
return {
status: "parsed",
slices: await readFatSlices(readAt, header.slices),
...(hiddenSlice === null ? {} : { hiddenSlices: [hiddenSlice] }),
};
} catch (cause: unknown) {
if (cause instanceof MachoFormatIssue)
return { status: cause.status, reason: cause.message };
@@ -189,53 +293,65 @@ export const readMachoImage = async (
}
};
/** Reject aliasing at the declaration owner before parsing any slice payload. */
const assertDistinctFatSlices = (
declarations: readonly FatSliceDeclaration[],
): void => {
const overlap = findOverlappingFatSlices(declarations);
if (overlap !== null)
throw new MachoFormatIssue(
"malformed",
`Mach-O slices ${overlap[0].index} and ${overlap[1].index} overlap.`,
);
};
const rangeValidSlices = (
header: ParsedFatContainerHeader,
): FatSliceDeclaration[] =>
header.slices.filter((_, index) => header.rangeIssues[index] === null);
const overlapReason = (
overlap: NonNullable<ReturnType<typeof findOverlappingFatSlices>>,
): string =>
`Mach-O slices ${overlap[0].index} and ${overlap[1].index} overlap.`;
/**
* Apple lipo stores one hidden ARM64 row after the declared 32-bit FAT table.
* It remains separate from the raw nfat_arch declarations and is only admitted
* from the single table-to-first-slice slot cctools recognizes.
* It is only admitted from the single table-to-first-slice slot cctools
* recognizes.
*/
const readAppleHiddenArm64Slice = async (
const readAppleHiddenArm64Row = async (
readAt: ReadAt,
fileSize: number,
header: Extract<
ReturnType<typeof readFatContainerHeader>,
{ state: "parsed" }
>,
): Promise<MachoSlice | null> => {
header: ParsedFatContainerHeader,
): Promise<HiddenFatRow | null> => {
if (header.wide) return null;
const firstOffset = Math.min(...header.slices.map(({ offset }) => offset));
const end = header.tableEnd + 20;
if (end > fileSize || end > firstOffset) return null;
const bytes = await readAt(header.tableEnd, 20);
if (bytes.byteLength < 20)
throw new MachoFormatIssue(
"malformed",
"Hidden ARM64 FAT row is truncated",
);
return { status: "malformed", reason: "Hidden ARM64 FAT row is truncated" };
const row = readFatSliceDeclarations(bytes, {
count: 1,
wide: false,
littleEndian: header.littleEndian,
});
if (row.status !== "parsed")
throw new MachoFormatIssue("malformed", row.reason);
return { status: "malformed", reason: row.reason };
const declaration = { ...row.slices[0]!, index: header.count };
if (declaration.cpuType !== CPU_TYPE_ARM64) return null;
const rangeIssue = validateFatSlice(declaration, fileSize, end);
const overlap = findOverlappingFatSlices([
...rangeValidSlices(header),
declaration,
]);
return {
status: "declared",
declaration,
rangeIssue:
overlap === null
? validateFatSlice(declaration, fileSize, end)
: overlapReason(overlap),
};
};
const readHiddenSlice = async (
readAt: ReadAt,
row: HiddenFatRow | null,
): Promise<MachoSlice | null> => {
if (row === null) return null;
if (row.status !== "declared")
throw new MachoFormatIssue(row.status, row.reason);
const { declaration, rangeIssue } = row;
if (rangeIssue !== null) throw new MachoFormatIssue("malformed", rangeIssue);
assertDistinctFatSlices([...header.slices, declaration]);
const slice = await readSlice(readAt, declaration.offset, declaration.size, {
fatCpuType: declaration.cpuType,
fatCpuSubtype: declaration.cpuSubtype,
@@ -288,6 +404,127 @@ const readFatSlices = async (
return slices;
};
/** Little-endian `mach_header(_64)` fields that frame one slice's commands. */
export interface MachoSliceHeader {
/** True for `MH_MAGIC_64`; the magic, not the CPU type, sets the width. */
readonly wide: boolean;
readonly headerSize: number;
readonly cpuType: number;
readonly cpuSubtype: number;
readonly fileType: number;
readonly commandCount: number;
readonly commandBytes: number;
}
/** One framed load command; `offset` is relative to the end of the header. */
export interface MachoLoadCommand {
readonly index: number;
readonly command: number;
readonly offset: number;
readonly body: Uint8Array;
}
/** One slice's header and raw load-command table, or why it was not read. */
export type MachoSliceCommandTable =
| {
readonly status: "read";
readonly header: MachoSliceHeader;
readonly bytes: Uint8Array;
}
| (MachoReadIssue & { readonly header: MachoSliceHeader | null });
/** Read one slice's header and load-command bytes without framing them. */
export const readSliceCommandTable = async (
readAt: ReadAt,
base: number,
length: number,
): Promise<MachoSliceCommandTable> => {
const headerBytes = await readAt(base, 28);
const parsed = readMachoHeaderDeclarations(headerBytes, length);
if (!parsed.ok)
return failed("malformed", "Mach-O header is truncated", null);
if (parsed.value === null)
return failed(
"malformed",
`slice at offset ${base} has no Mach-O header`,
null,
);
if (!parsed.value.littleEndian)
return failed(
"unsupported",
"big-endian Mach-O slices are not supported",
null,
);
const { wide, headerSize, commandCount, commandBytes, fileType, identity } =
parsed.value;
const header: MachoSliceHeader = {
wide,
headerSize,
cpuType: identity.cpuType,
cpuSubtype: identity.cpuSubtype,
fileType,
commandCount,
commandBytes,
};
if (header.commandBytes > MAX_LOAD_COMMAND_BYTES)
return failed(
"unsupported",
`load commands occupy ${header.commandBytes} bytes, above the ${MAX_LOAD_COMMAND_BYTES}-byte limit`,
header,
);
if (header.headerSize + header.commandBytes > length)
return failed(
"malformed",
"load commands extend beyond the Mach-O slice",
header,
);
const bytes = await readAt(base + header.headerSize, header.commandBytes);
if (bytes.byteLength < header.commandBytes)
return failed("malformed", "load commands is truncated", header);
return { status: "read", header, bytes };
};
const failed = (
status: MachoReadIssue["status"],
reason: string,
header: MachoSliceHeader | null,
): MachoSliceCommandTable => ({ status, reason, header });
/**
* Hand each load command to `visit` in order without retaining it. A framing
* failure is returned after every command before the failing one was visited.
*/
export const frameLoadCommands = (
{
header,
bytes,
}: { readonly header: MachoSliceHeader; readonly bytes: Uint8Array },
visit: (command: MachoLoadCommand) => void,
): { readonly status: "framed" } | MachoReadIssue => {
const cursor = new MachoLoadCommandCursor({
wide: header.wide,
littleEndian: true,
commandCount: header.commandCount,
commandBytes: bytes.byteLength,
});
while (cursor.nextOffset !== null) {
const frame = cursor.read(
bytes.subarray(cursor.nextOffset, cursor.nextOffset + 8),
);
if (!frame.ok) return { status: "malformed", reason: frame.error };
const { index, command, offset, size } = frame.value;
visit({
index,
command,
offset,
body: bytes.subarray(offset, offset + size),
});
}
const complete = cursor.finish();
if (!complete.ok) return { status: "malformed", reason: complete.error };
return { status: "framed" };
};
const readSlice = async (
readAt: ReadAt,
base: number,
@@ -298,96 +535,19 @@ const readSlice = async (
readonly alignmentExponent: number;
} | null = null,
): Promise<MachoSlice> => {
const header = await readExact(readAt, base, 28, "Mach-O header");
const parsed = readMachoHeaderDeclarations(header, length);
if (!parsed.ok)
throw new MachoFormatIssue("malformed", "Mach-O header is truncated");
if (parsed.value !== null && parsed.value.littleEndian) {
const { wide, headerSize, commandCount, commandBytes, fileType, identity } =
parsed.value;
if (commandBytes > MAX_LOAD_COMMAND_BYTES)
throw new MachoFormatIssue(
"unsupported",
`load commands occupy ${commandBytes} bytes, above the ${MAX_LOAD_COMMAND_BYTES}-byte limit`,
);
if (headerSize + commandBytes > length)
throw new MachoFormatIssue(
"malformed",
"load commands extend beyond the Mach-O slice",
);
const commands = await readExact(
readAt,
base + headerSize,
commandBytes,
"load commands",
);
const decoded = decodeCommands(commands, {
wide,
cpuType: identity.cpuType,
cpuSubtype: identity.cpuSubtype,
fileType,
commandCount,
sliceOffset: base,
sliceSize: length,
fatCpuType: fat?.fatCpuType ?? null,
fatCpuSubtype: fat?.fatCpuSubtype ?? null,
fatAlignmentExponent: fat?.alignmentExponent ?? null,
});
const slice = decoded.slice;
await appendDependencies(
slice,
decoded.dependencies,
readAt,
base,
length,
decoded.maxVmOffset,
);
const distinctPlatforms = [...new Set(slice.platforms)];
slice.platform =
distinctPlatforms.length === 1 ? (distinctPlatforms[0] ?? null) : null;
return slice;
}
throw new MachoFormatIssue(
parsed.value !== null && !parsed.value.littleEndian
? "unsupported"
: "malformed",
parsed.value !== null && !parsed.value.littleEndian
? "big-endian Mach-O slices are not supported"
: `slice at offset ${base} has no Mach-O header`,
);
};
const decodeCommands = (
bytes: Uint8Array,
header: {
readonly wide: boolean;
readonly cpuType: number;
readonly cpuSubtype: number;
readonly fileType: number;
readonly commandCount: number;
readonly sliceOffset: number;
readonly sliceSize: number;
readonly fatCpuType: number | null;
readonly fatCpuSubtype: number | null;
readonly fatAlignmentExponent: number | null;
},
): {
readonly slice: MachoSlice;
readonly dependencies: readonly (
| { readonly kind: "parsed"; readonly value: MachoDependency }
| LazyLoadDylibInfoCommand
)[];
readonly maxVmOffset: bigint;
} => {
const table = await readSliceCommandTable(readAt, base, length);
if (table.status !== "read")
throw new MachoFormatIssue(table.status, table.reason);
const { header } = table;
const slice: MachoSlice = {
architecture: machoArchitectureName(header.cpuType, header.cpuSubtype),
slice_offset: header.sliceOffset,
slice_size: header.sliceSize,
slice_offset: base,
slice_size: length,
cpu_type: header.cpuType,
cpu_subtype: header.cpuSubtype,
fat_cpu_type: header.fatCpuType,
fat_cpu_subtype: header.fatCpuSubtype,
fat_alignment_exponent: header.fatAlignmentExponent,
fat_cpu_type: fat?.fatCpuType ?? null,
fat_cpu_subtype: fat?.fatCpuSubtype ?? null,
fat_alignment_exponent: fat?.alignmentExponent ?? null,
file_type: fileTypeName(header.fileType),
file_type_code: header.fileType,
platform: null,
@@ -404,20 +564,10 @@ const decodeCommands = (
> = [];
const segments: SegmentSpan[] = [];
let unreadableSegment: number | undefined;
let lazyInfo = false;
let lazyInfoCount = 0;
const commands = new MachoLoadCommandCursor({
wide: header.wide,
littleEndian: true,
commandCount: header.commandCount,
commandBytes: bytes.byteLength,
});
while (commands.nextOffset !== null) {
const offset = commands.nextOffset;
const frame = commands.read(bytes.subarray(offset, offset + 8));
if (!frame.ok) throw new MachoFormatIssue("malformed", frame.error);
const { index, command, size } = frame.value;
const body = bytes.subarray(offset, offset + size);
// Each command decodes as it is framed, so a defect in an earlier command
// keeps precedence over a later framing defect.
const framed = frameLoadCommands(table, ({ command, body, index }) => {
if (command === LC_SEGMENT || command === LC_SEGMENT_64) {
const span = readSegmentSpan(body, command === LC_SEGMENT_64);
if (span === undefined) unreadableSegment ??= index;
@@ -430,19 +580,29 @@ const decodeCommands = (
"unsupported",
`LC_LAZY_LOAD_DYLIB_INFO commands exceed the ${MAX_LAZY_INFO_DEPENDENCIES}-dependency limit`,
);
lazyInfo = true;
}
const decoded = decodeCommand(command, body, index, slice);
if (decoded !== undefined) dependencies.push(decoded);
}
const complete = commands.finish();
if (!complete.ok) throw new MachoFormatIssue("malformed", complete.error);
if (lazyInfo && unreadableSegment !== undefined)
});
if (framed.status !== "framed")
throw new MachoFormatIssue(framed.status, framed.reason);
if (lazyInfoCount > 0 && unreadableSegment !== undefined)
throw new MachoFormatIssue(
"malformed",
`load command ${unreadableSegment} is too short to establish the slice VM range for LC_LAZY_LOAD_DYLIB_INFO`,
);
return { slice, dependencies, maxVmOffset: maximumVmOffset(segments) };
await appendDependencies(
slice,
dependencies,
readAt,
base,
length,
maximumVmOffset(segments),
);
const distinctPlatforms = [...new Set(slice.platforms)];
slice.platform =
distinctPlatforms.length === 1 ? (distinctPlatforms[0] ?? null) : null;
return slice;
};
interface LazyLoadDylibInfoCommand {
@@ -485,7 +645,8 @@ const readSegmentSpan = (
const { nameBytes, address: vmaddr, memorySize: vmsize } = parsed.value;
const end = nameBytes.indexOf(0);
return {
name: new TextDecoder("utf-8", { fatal: false }).decode(
// Segment names are fixed-width Mach-O data; a leading BOM is literal name bytes.
name: new TextDecoder("utf-8", { fatal: false, ignoreBOM: true }).decode(
end < 0 ? nameBytes : nameBytes.subarray(0, end),
),
vmaddr,
+78
View File
@@ -0,0 +1,78 @@
import { describe, expect, it } from "vitest";
import { ARTIFACT_FORMAT_IDS } from "../domain/artifactGraph.js";
import {
ZIP_PACKAGE_FORMATS,
type ZipPackageFormat,
} from "../domain/zipPackageFormat.js";
import {
ARTIFACT_FORMATS,
ZIP_FAMILY_CAPABILITIES,
} from "./artifactFormats.js";
import {
createArtifactReader,
type ArtifactReaderFormat,
} from "./createArtifactReader.js";
type Equal<Left, Right> =
(<Value>() => Value extends Left ? 1 : 2) extends <
Value,
>() => Value extends Right ? 1 : 2
? true
: false;
type FormatsWithReaders =
| ZipPackageFormat
| "dotnet-bundle"
| "bun-standalone"
| "dmg"
| "asar"
| "godot-pck"
| "unityfs"
| "uimage"
| "mach-o-universal"
| "directory"
| "pyinstaller"
| "node-sea"
| "intel-hex"
| "srec"
| "esp-image"
| "fit";
const readerFormatsMatch: Equal<ArtifactReaderFormat, FormatsWithReaders> =
true;
const unknownHasNoReader: "unknown" extends ArtifactReaderFormat
? false
: true = true;
describe("artifact format descriptors", () => {
it("derives the reader-format union from descriptors that construct a reader", () => {
expect(readerFormatsMatch).toBe(true);
expect(unknownHasNoReader).toBe(true);
});
it("covers every domain format id and no other id", () => {
expect(Object.keys(ARTIFACT_FORMATS).sort()).toEqual(
[...ARTIFACT_FORMAT_IDS].sort(),
);
});
it("leaves unknown without a reader", () => {
expect("createReader" in ARTIFACT_FORMATS.unknown).toBe(false);
expect(
createArtifactReader("/unused", "unknown", {
environment: {},
unreadableDirectories: "omit",
}),
).toBeUndefined();
});
it("shares one capability row across zip package families", () => {
for (const id of ZIP_PACKAGE_FORMATS)
expect(ARTIFACT_FORMATS[id]).toMatchObject(ZIP_FAMILY_CAPABILITIES);
const zipIds = new Set<ZipPackageFormat>(ZIP_PACKAGE_FORMATS);
for (const id of ARTIFACT_FORMAT_IDS)
if (!zipIds.has(id as ZipPackageFormat))
expect(ARTIFACT_FORMATS[id].zipFamily).toBe(false);
});
});
Loaded 100 of 543 files, more files were not shown because too many files have changed in this diff. Show more