OpenSource/reverse-engineering-anything
1
0 Fork 0
mirror of https://github.com/morluto/rea.git synced 2026-11-12 17:59:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into feat/environment-write-quality

Browse Source
This commit is contained in:
Edgar Arakelyan committed 2026-10-26 22:12:36 -07:00
2 parents 70c9a7c204 5aa23c3d17
commit fa647e8903
621 files changed
+29871 -4569
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.agents/skills/reverse-engineer-anything/references/environments.md
+72 -26
View File
@@ -88,20 +88,37 @@ shape, with the application's own locators, paths and headers:
"path": "/api/folders",
"expected_status": [200]
},
{
"step_id": "new-folder-1",
"action": "click",
"timeout_ms": 15000,
"locator": { "kind": "role", "role": "button", "name": "New folder" }
},
{
"step_id": "folder-title-1",
"action": "fill",
"timeout_ms": 15000,
"locator": { "kind": "css", "selector": "input[name=title]" },
"value": { "source": "literal", "value": "Scratch folder 1" }
},
{
"step_id": "create-folder-1",
"action": "click",
"timeout_ms": 15000,
"locator": { "kind": "role", "role": "button", "name": "Create" },
"save_response": {
"method": "POST",
"path": "/api/folders",
"save": [{ "name": "folder_1", "json_pointer": "/id" }]
}
},
{
"step_id": "read-folder-1",
"action": "request",
"timeout_ms": 15000,
"method": "POST",
"path": "/api/folders",
"json": { "title": "Scratch folder 1" },
"headers": [
{
"name": "X-CSRFToken",
"value": { "source": "cookie", "name": "csrftoken" }
}
],
"expected_status": [200, 201]
"method": "GET",
"path": "/api/folders/{saved:folder_1}",
"expected_status": [200]
}
]
}
@@ -117,13 +134,19 @@ client with it. Match `wait_for` locators to exactly one element.
2. Read with `request` steps: each collection the environment needs, twice
before any write, so fields that change on their own show up; one item of
each.
3. Use `request` steps only when the API lets the caller choose each scratch
object's ID or a prior read already established it. Request paths are fixed
strings; no action substitutes a value from an earlier response. For
server-assigned IDs, create through the UI and open the new object's detail
view to record its read, edit and delete through UI actions. If that journey
cannot be scripted, leave the write unserved and report the limitation;
never guess an ID or create objects with no supported cleanup path.
3. Write the way the application's UI does: fill its forms and click its
buttons, so each request carries the fields the UI sends, including ones
the server would otherwise derive. Give the click or key press that submits
a create `save_response` to keep the new object's ID, then `goto` its page
(`/items/{saved:item_1}`) or read it with a `request` step. Never guess an
ID, and never create a probe object to learn the next one. Write with
`request` steps instead only when the UI cannot reach the operation (an
API-only resource, or a UI that does not call the JSON API), or when coverage
withholds the UI-shaped write as `unexplained-change` or `contradicted`;
that counts as one of the two re-recordings. A `request` step write that
coverage withholds that way usually lacks fields the UI sends: record it
through the UI instead. If a journey cannot be scripted, leave the write
unserved and report the limitation.
Record two numbered scratch objects per write type,
with varied free text. After each write, and before the next one, read the
object back by id and then its list: create 1, read 1, list, create 2,
@@ -143,13 +166,34 @@ About `request` steps:
- `path` is on the start URL's origin; the page must be on that origin.
- Header values are literals, declared secrets, a cookie the browser holds
(`{source: "cookie", name}`), or a local storage entry
(`{source: "local_storage", key, json_pointer}`), with an optional
`prefix` such as `"Bearer "`. Copy the headers the application's own
requests send (a CSRF header, a bearer token). Values read from the page
are redacted from records made after they are read, not from the login
response that returned them, and values shorter than 8 characters are not
redacted at all.
(`{source: "cookie", name}`), a local storage entry
(`{source: "local_storage", key, json_pointer}`), a saved value
(`{source: "saved", name}`), or Basic credentials
(`{source: "basic_credentials", parts: [username, password]}`, sent as the
base64 of `username:password` and redacted), with an optional `prefix` such as
`"Bearer "` or `"Basic "`. Each credential part is a literal, secret, saved
value, cookie or local storage entry, so a header the frontend builds from a
login token (the username and the token) can be copied too; the password is
a secret or saved value, never a literal. Copy the
headers the application's own requests send (a CSRF header such as
`X-CSRFToken` from the `csrftoken` cookie, a bearer token). Values read from the page are redacted from records made after they
are read, not from the login response that returned them, and values
shorter than 8 characters are not redacted at all.
- `save: [{name, json_pointer, secret}]` keeps a string or number from the
step's JSON response, such as the created object's `/id`, for later steps;
`secret: true` redacts it, as for a token the API returns, so mark every
saved token, key or session value secret. This also protects the selected
source response before its body or headers are retained. Compressed or
unknown-size UI responses cannot supply saved values. On a `click` or `press`,
`save_response: {method, path, save}` keeps values from the response to the
new request observed while the step waits (`{*}` in `path` matches one segment).
Matching does not prove UI causality: choose a path distinct from polling;
multiple observed matches fail as ambiguous. The step
fails unless a 2xx JSON response arrives before its `timeout_ms`. Later
steps send a saved value as `{saved:name}` in a request `path` (and its
query) or a `goto` URL; in the `json` body, where a string that is only
`{saved:name}` keeps the saved JSON type; or as `{source: "saved", name}` in
a header or credential part.
- Reading a page value makes later bodies costlier to keep: after a CSRF
token, bodies over about 47 KiB are kept as `unavailable`. When the API
returns larger responses, raise `capture.limits`
@@ -198,8 +242,10 @@ withheld with its hint:
| `contradicted` | Stop: a later read disagrees with the modelled effect. |
| `body-unusable` | Re-record a UI write once with `request` steps (its body was not exposed); else stop. |
Use new scratch names on each attempt; the first attempt's objects may still
exist.
Use new scratch names on each attempt, and delete what an earlier attempt
created before recording again (through the UI, or a `request` step with the
id that capture shows): the environment starts from what the recording first
reads, so a leftover object becomes seeded data that write tasks then find.
New browser scenario captures include request times, so a fully timed capture
can learn a write clock and serve updates that stamp modification times. Older
captures or ones missing a request start keep record order and learn no clock.
bridge/ghidra/ReaGhidraBridge.java
+33 -28
View File
@@ -2099,10 +2099,7 @@ public final class ReaGhidraBridge extends HeadlessScript {
requireKeys(params, Set.of("document", "address"));
requireDocument(params);
Address query = requireAddress(params, "address");
Function function = currentProgram.getFunctionManager().getFunctionAt(query);
if (function == null && query.isMemoryAddress()) {
function = currentProgram.getFunctionManager().getFunctionContaining(query);
}
Function function = functionAtOrContaining(query);
JsonObject result = new JsonObject();
result.addProperty("query_address", canonicalAddress(query));
if (function != null) {
@@ -2118,10 +2115,9 @@ public final class ReaGhidraBridge extends HeadlessScript {
}
private JsonArray search(JsonObject params, boolean procedureSearch) throws Exception {
requireKeys(
params,
Set.of("pattern", "mode", "case_sensitive", "document")
);
Set<String> keys = new HashSet<>(Set.of("pattern", "mode", "case_sensitive", "document"));
if (params.has("literal_ascii_candidate")) keys.add("literal_ascii_candidate");
requireKeys(params, keys);
requireDocument(params);
String expression = requireString(params, "pattern");
String mode = requireString(params, "mode");
@@ -2132,9 +2128,18 @@ public final class ReaGhidraBridge extends HeadlessScript {
List<InventoryItem> inventory = procedureSearch
? procedures().stream().map(FunctionEntry::item).toList()
: strings();
ValueMatcher matcher = mode.equals("literal")
? literalMatcher(expression, caseSensitive)
: regexMatcher(expression, caseSensitive);
ValueMatcher matcher;
if (mode.equals("literal") && !caseSensitive) {
if (!params.has("literal_ascii_candidate"))
throw new RequestFailure("invalid_request", "Insensitive literal search requires an ASCII candidate");
matcher = literalCandidateMatcher(optionalString(params, "literal_ascii_candidate"));
} else {
if (params.has("literal_ascii_candidate"))
throw new RequestFailure("invalid_request", "ASCII candidate applies only to insensitive literals");
matcher = mode.equals("literal")
? literalMatcher(expression)
: regexMatcher(expression, caseSensitive);
}
JsonArray items = new JsonArray();
for (InventoryItem item : inventory) {
monitor.checkCancelled();
@@ -2291,6 +2296,7 @@ public final class ReaGhidraBridge extends HeadlessScript {
return function;
}
/** Exact entries include externals; containment applies only to memory addresses. */
private Function functionAtOrContaining(Address address) {
Function function = currentProgram.getFunctionManager().getFunctionAt(address);
return function == null && address.isMemoryAddress()
@@ -2995,7 +3001,7 @@ public final class ReaGhidraBridge extends HeadlessScript {
if (!currentProgram.getLanguageID().getIdAsString().startsWith("AARCH64:")) {
return null;
}
Function function = containingFunction(jumpTable.getSwitchAddress());
Function function = functionAtOrContaining(jumpTable.getSwitchAddress());
if (function == null) return null;
List<Instruction> instructions = new ArrayList<>();
@@ -3495,7 +3501,7 @@ public final class ReaGhidraBridge extends HeadlessScript {
private boolean unrepresentedCaller(Reference reference, Set<Function> callers) {
if (!reference.getReferenceType().isCall()) return false;
Function caller = containingFunction(reference.getFromAddress());
Function caller = functionAtOrContaining(reference.getFromAddress());
return caller == null || !callers.contains(caller);
}
@@ -3642,7 +3648,7 @@ public final class ReaGhidraBridge extends HeadlessScript {
void observe(Reference reference) {
if (!reference.getReferenceType().isCall()) return;
// Reference identity follows the same containing-function owner as referenceEdge.
Function target = containingFunction(reference.getToAddress());
Function target = functionAtOrContaining(reference.getToAddress());
if (returnedCallees != null && (target == null || !returnedCallees.contains(target)))
unrepresented++;
if (target == null) unresolved(reference.getFromAddress(),
@@ -3812,8 +3818,8 @@ public final class ReaGhidraBridge extends HeadlessScript {
JsonObject result = new JsonObject();
result.addProperty("source_address", canonicalAddress(reference.getFromAddress()));
result.addProperty("target_address", canonicalAddress(reference.getToAddress()));
Function source = containingFunction(reference.getFromAddress());
Function target = containingFunction(reference.getToAddress());
Function source = functionAtOrContaining(reference.getFromAddress());
Function target = functionAtOrContaining(reference.getToAddress());
result.add(
"source_procedure",
source == null ? JsonNull.INSTANCE : procedureIdentity(source)
@@ -3848,14 +3854,6 @@ public final class ReaGhidraBridge extends HeadlessScript {
return result;
}
private Function containingFunction(Address address) {
Function result = currentProgram.getFunctionManager().getFunctionAt(address);
if (result == null && address.isMemoryAddress()) {
result = currentProgram.getFunctionManager().getFunctionContaining(address);
}
return result;
}
private static String procedureName(Function function) {
Symbol symbol = function.getSymbol();
return symbol == null ? function.getName() : symbol.getName(true);
@@ -4707,11 +4705,18 @@ public final class ReaGhidraBridge extends HeadlessScript {
return source.name().toLowerCase(Locale.ROOT);
}
private static ValueMatcher literalMatcher(String pattern, boolean caseSensitive) {
String needle = caseSensitive ? pattern : pattern.toLowerCase(Locale.ROOT);
private static ValueMatcher literalMatcher(String pattern) {
return value -> value.contains(pattern);
}
// REA's host owns Unicode simple folding. Keep all non-ASCII source values
// for its final predicate, and filter ASCII-only values without Java regexes.
private static ValueMatcher literalCandidateMatcher(String asciiPattern) {
if (asciiPattern != null && asciiPattern.chars().anyMatch(character -> character > 127))
throw new RequestFailure("invalid_request", "ASCII candidate contains a non-ASCII character");
return value -> {
String candidate = caseSensitive ? value : value.toLowerCase(Locale.ROOT);
return candidate.contains(needle);
if (value.chars().anyMatch(character -> character > 127)) return true;
return asciiPattern != null && value.toLowerCase(Locale.ROOT).contains(asciiPattern);
};
}
docs/agent-environments.md
+4 -1
View File
@@ -205,7 +205,10 @@ attached and a transaction no record represents, and each transaction URL's
query once. Bodies larger than 4 MiB of UTF-8 text are not
scanned; `credential-scan-incomplete` counts them and lists up to 20 record or
event pointers, including base64 bodies whose decoded scan was cut. An empty name
warning does not establish that these bodies were checked. Names do not establish sensitivity,
warning does not establish that these bodies were checked. A browser scenario's
saved values (`save` and `save_response`) named like credentials but not marked
`secret: true` are recorded unredacted; `credential-named-saved-values` lists up
to 20 of them by parameter pointer. Names do not establish sensitivity,
so the build never refuses on them. Capture
Evidence also records the capture file's absolute path (`subject.local_path`,
docs/android-analysis.md
+58 -2
View File
@@ -30,8 +30,9 @@ code units; follow `coverage.next_offset`. The page preserves `source_status` an
`reported_total_bytes`: exhausting retained text does not make a partial decoded
manifest complete. Complete source Evidence remains exportable under the existing
session lifetime. Servers without retention reject summary before starting JADX.
One-shot CLI output remains complete. Older servers require the existing complete
request after schema discovery. APK authenticity and runtime behavior remain
One-shot CLI output remains complete by default. Select `--detail summary` with
`--evidence_output /local/capture.json` to retain complete Evidence before returning
a linked view. Older servers require the existing complete request after schema discovery. APK authenticity and runtime behavior remain
unverified.
## Manifest component projection
@@ -76,6 +77,61 @@ capacity limitation appears only when an actual collection limit was reached.
The projection does not merge manifests, expand resource references, or
observe runtime enabled/export state.
`enabled` and `grant_uri_permissions` report only literal component-level
`android:enabled` and provider-level `android:grantUriPermissions` declarations.
They are `null` when absent or not a literal boolean; missing attributes are not
filled from application defaults. A nonliteral declaration gets a limitation.
`process` preserves the exact component-level `android:process` spelling (including
empty or symbolic values), or `null` when absent or omitted under the inventory-wide
1 MiB copy budget. Neither field resolves resources, merges manifest values or
establishes the effective runtime process or enabled state. The exact boolean
attributes are retained in `enabled_declaration` and
`grant_uri_permissions_declaration`, including resource references.
`intent_filters` preserves each filter's source ordinal, exact priority spelling,
actions, categories and individual data declarations (scheme, host, port, path,
path prefix/pattern/advanced pattern/suffix and MIME type). These are declarations,
not computed Android matching rules. `meta_data` lists names; values remain in the
complete decoded manifest. Component, filter, data and metadata `location` values
use one-based lines/columns and UTF-16 offsets into that manifest. A component's
`declaration_index` preserves its original declaration ordinal, including gaps.
`declaration_coverage` distinguishes complete, partial and unknown observations.
Historical Evidence omits newly introduced fields and defaults coverage to
`unknown`; omission never establishes absence. Current records contain the fields,
with null for absent or unresolved scalar declarations. The inventory retains at
most 32 filters and 64 nested action/category/data/metadata records per component,
and shares a 1 MiB structured-value copy budget across components. Omitted values
or unsupported filter constructs mark coverage partial and include a limitation;
complete retained manifest text remains accessible.
For a small answer, select `detail: "summary"` with an explicit view:
```json
{
"kind": "android",
"facet": "components",
"actions": ["android.provider.Telephony.SMS_RECEIVED"],
"fields": ["name", "permission", "enabled", "process", "intent_filters"],
"offset": 0,
"limit": 16
}
```
The same JSON works with CLI `--view` or retained `inspect_analysis_view`.
`kinds`, `names` and `actions` use exact matching; names accept either the recorded
declared or resolved name, actions match any requested action, and different
filters combine with AND. `exported` matches only explicit declarations, keeping
`exported_default` separate. Empty selectors match nothing. Every selected row
retains its parent-array `source_index` and `declaration_coverage`; source coverage,
parent identity and limitations remain alongside the page. Follow
`coverage.next_offset`. The 1 MiB projection budget can shorten a page; a single
oversized row requires fewer selected fields. Exhausting a partial source does
not establish complete manifest coverage. `facet: "component-summary"` applies
the same filters and counts kinds, declaration coverage and explicit/default
exported values separately. Both adapters use the same retained projection;
repeated pages do not reload JADX.
The service `foreground_service_type` field preserves the declared attribute,
including an empty value or surrounding spaces; `null` means the attribute is
absent or the component is not a service. This field does not validate service
docs/architecture.mermaid
+2 -2
View File
@@ -21,14 +21,14 @@ graph TD
subgraph "Application Layer"
APP["src/application/<br/>Shared CLI/MCP workflows<br/>enhanced analysis and investigations"]
SESSION["SessionProviderRouter + BinarySession<br/>one immutable deep binding per target<br/>disjoint auxiliary routing"]
REGISTRY["AnalysisProviderRegistry<br/>sorted deep candidates<br/>deterministic selection, no fallback"]
REGISTRY["AnalysisProviderRegistry<br/>sorted deep candidates<br/>deterministic selection, no fallback<br/>ProviderInspectionScope: prerequisite observation lifetime"]
RECORDS["src/application/investigation/<br/>composed InvestigationRecords + narrow ports<br/>EvidenceLedger and Unknown ownership"]
SNAPSHOT["BinarySessionRecords + AnalysisSnapshotCache<br/>binary target/profile snapshots and session file ownership<br/>record coordination and post-commit notifications"]
JS_RECON["src/application/javascript/ + src/artifacts/javascript/<br/>JavaScript artifact reconstruction<br/>directory/ASAR reads<br/>AST-only bundle projection"]
end
subgraph "Provider Layers (selected adapters)"
PROVIDER_OWNERS["src/providers/<br/>shared deferred client and inspection owners<br/>exact cleanup retry and attributed failures"]
PROVIDER_OWNERS["src/providers/<br/>shared deferred client owners<br/>exact cleanup retry and attributed failures"]
PWNTOOLS["src/native/pwntools/ + src/native/pwndbg/<br/>BYO offline ELF/core adapters<br/>owned snapshots/Python and optional core-only GDB<br/>unchanged upstream parsers, ABI structures and mapping API"]
WASM["src/wasm/<br/>caller-supplied WABT; immutable artifact inspection<br/>owned process supervision and retryable workspace cleanup"]
APKSIG["src/apkSignature/<br/>in-process APK Signing Block + v1 reader<br/>bounded single read, no device or execution"]
docs/browser-scenario-contract.md
+173 -1
View File
@@ -51,6 +51,30 @@ covers endpoint discovery only. The following WebSocket handshake and
`Target.attachToTarget` each use `command_timeout_ms`. `capture.limits` controls retained event and network-content
budgets; it does not change those liveness deadlines.
## Validate a scenario before running it
`validate-browser-scenario INPUT_JSON` checks one scenario against the same
`browserScenarioSchema` refinement that `capture_browser_scenario` admits, and
stops there. `INPUT_JSON` may be inline JSON, a JSON file path, or `-` for
standard input.
```bash
rea validate-browser-scenario ./browser-scenario.json --json
```
A valid scenario reports `{"valid":true,"actions":<count>,"secrets":<count>}`
and exits 0. An invalid one exits 1 and reports the same field paths capture
would report, so a correction made here is the correction capture needs:
duplicate or reserved step IDs, duplicate secret identities and
environment-variable names, undeclared secret references, request-step
violations, and unknown fields.
Validation performs no browser discovery, launch, attachment, or navigation and
constructs no provider. It does not read secret environment variables, so a
scenario whose declared variables are unset still validates; the missing-value
failure belongs to capture. A schema-valid scenario does not prove that the
executable or target is available on the host.
## Minimal launch example
Save this as `browser-scenario.json`, replacing the executable path with an
@@ -213,7 +237,8 @@ HTTP(S) URLs may include ordinary query values and fragments directly. Use
ordered structured query entries when a value needs to reference a declared
environment-backed secret. Form, storage, and cookie values are either literal
strings or declared secret references; request step headers can also read a
cookie or local storage value from the page. Raw URL userinfo credentials are rejected.
cookie or local storage value from the page, and request steps and `goto`
destinations can send [saved values](#saved-values). Raw URL userinfo credentials are rejected.
Captured credential-header
values are not retained, and declared secret values are redacted automatically.
A secret may be declared solely to
@@ -406,6 +431,17 @@ data you may change. This scenario creates one item:
by default) get smaller: after a 32-character `X-CSRFToken` value, a body over
about 47 KiB is kept as `unavailable`. Raise `capture.limits` when the
application's responses are larger.
- A header can send Basic credentials, `{"source": "basic_credentials", "parts":
[username, password]}`: the base64 of `username:password`, after a `prefix`
such as `"Basic "` or with none, as some APIs take encoded credentials in
their own header. Each part is a literal, a declared secret, a saved value, a
cookie or a local storage entry, so a header a frontend builds from a login
token can be sent too; the password is never a literal other than empty, since
Evidence records literals as written, and a username with a colon fails the
step, since Basic authentication ends the username at its first colon. The
encoded value is redacted however short, as
`[REDACTED:page:<step_id>.<header>]`, and parts read from the page are
redacted as page values are.
- Validation refuses a header name given twice in a step, header names a
page's `fetch` cannot set, such as `Cookie`, `Origin`, `Host` or `Sec-*`, and
`User-Agent`, which Chrome drops; it refuses line breaks and other ASCII
@@ -431,6 +467,142 @@ Captures with request steps state in `limitations` that the steps sent their
declared requests through the page's fetch. A scenario without request steps
is unchanged.
## Saved values
A step can keep a string or number from a JSON response, by name, and a later
step can send it, so a scenario can read back, edit and delete an object whose
id the application assigns:
```json
{
"browser": {
"mode": "launch",
"executable_path": "/absolute/path/to/chromium"
},
"start_url": { "url": "http://127.0.0.1:3000/items" },
"actions": [
{
"step_id": "save-item-1",
"action": "click",
"timeout_ms": 10000,
"locator": { "kind": "role", "role": "button", "name": "Save" },
"save_response": {
"method": "POST",
"path": "/api/items",
"save": [{ "name": "item_1", "json_pointer": "/id" }]
}
},
{
"step_id": "open-item-1",
"action": "goto",
"timeout_ms": 10000,
"destination": { "url": "http://127.0.0.1:3000/items/{saved:item_1}" },
"wait_until": "load"
},
{
"step_id": "copy-item-1",
"action": "request",
"timeout_ms": 10000,
"method": "POST",
"path": "/api/items/{saved:item_1}/copies",
"json": {
"source_id": "{saved:item_1}",
"title": "Copy of {saved:item_1}"
},
"expected_status": [201]
}
]
}
```
- A `request` step saves with `save: [{name, json_pointer, secret}]`, from its
own response. A `click` or `press` step saves with `save_response: {method,
path, save}`, from a newly observed matching response while the step waits.
The recorder watches before the action and excludes requests already running,
such as an earlier autosave. Method/path matching does not prove that the UI
caused the request: use a path that distinguishes the intended operation from
background polling. Multiple observed matches before binding fail as ambiguous;
requests starting after this observation window cannot be ruled out. It matches the method and a path on the
start origin as the browser spells it, so a path may name `My List` or
`My%20List` (`{*}` matches one segment, and the query is ignored), and fails
the step unless a 2xx JSON response arrives before its `timeout_ms`; one still
being read when the step times out or is cancelled saves nothing. A pointer
that names no string or number fails the step, so later steps never send a
missing value, as does a response over 4,000,000 characters: a request step
stops reading and aborts its fetch there. A UI step reads a body only when
its positive received size is known, at most 12,000,000 bytes, and its
Content-Encoding is absent or identity. Compressed responses and unknown or
zero cache sizes are refused before materializing the body; decoded text
over the character limit also fails. The UI action may already have applied.
- `{saved:name}` sends a saved value in a request step's `path`, including its
query, and in a `goto` destination's `url`, percent-encoded; a destination's
structured `query` values are sent as declared. A saved value that makes a
path segment `.` or `..`, alone or with the text beside it (including `%2e`),
fails the step, since the URL would drop it. In a request step's `json` body,
a string that is only `{saved:name}` becomes the saved value with the JSON
type it was saved with, so a saved number id is sent as a number, and
`{saved:name}` inside a longer string is filled as text. A header value or
Basic credentials part sends one as `{"source": "saved", "name"}`. Other
braces, such as `{id}`, are sent as written, and so is a percent-encoded
`%7Bsaved:name%7D` anywhere but a `goto` URL, whose parsed form spells the
braces that way.
- Validation refuses a saved value that no earlier step saves. A later save of
the same name replaces the value. An integer past 2^53, which JSON parsing
would round to another number, and a number that is not finite, such as
`1e400`, negative zero, or a value that underflows to zero, such as `1e-400`, fail the step
that saves it.
- A scenario keeps at most 4,000,000 characters of saved values at once,
counting distinct retained redaction spellings, including trimmed variants,
of values saved as secret. Replacing a binding releases its live value
charge, while secret spellings remain protected for the scenario lifetime.
A save past the budget fails before changing the binding.
- A value saved with `secret: true` is redacted however short, as a declared
secret is, as `[REDACTED:page:<step_id>.<name>]`. The selected source
response is protected before retaining its body or header values, including
when only headers were requested. If that body cannot be read safely, its
body and header values remain unavailable. Before publishing the capture,
the selected source transaction's earlier URL, request headers and request
body are also protected using the admitted values. Unresolved source values
withhold that transaction's affected metadata/content with an explicit
limitation. Before publication, retained events are rechecked
with the final marked-response values, including immediate follow-up requests
and their response bodies, console/page errors, WebSocket text and event URL/name fields. Learned redaction facts are independent of source-identity readiness. Unknown source values withhold only the selected
source transaction. Other saved values, such as ids, stay visible, since
environments model them. The environment build warns about saved values
named like credentials that are not marked secret
(`credential-named-saved-values`).
- Page-derived cookie/local-storage values and constructed Basic credentials
remain redacted only from observations made after they were read/built.
They do not retroactively rewrite earlier facts. The final event projection
borrows only declared and response-marked patterns; promotion of an already
held page value to an explicit response secret preserves its first marker.
Projection state is session-local, references admitted pattern/byte owners,
and is invalidated on admission or response classification changes.
- Screenshots are withheld as `missing` from the start when the scenario plans
any response save marked `secret: true`, page-derived or Basic authentication
header values, or whenever nonempty sensitive values
are already known. Raw pixels cannot be safely text-redacted, so the provider
does not request pixels in those cases. This also protects earlier steps and
partial failures without retaining an image to scrub later. Other requested
artifacts remain available; affected screenshot/aggregate completeness is
partial and is not eligible for equality.
- Saved URLs and expanded request JSON are admitted before building their expanded strings or
serializing. Its allowance is the smaller of Node's string limit and
one sixteenth of current V8 available heap bytes, allowing for the expanded
tree, serialization, Playwright transport and browser-side copies. Size
accounting includes JSON escaping and repeated references; each distinct
saved value's size is measured once per request. An oversized request fails
before browser dispatch; the complete request envelope, including repeated
header values, is checked as well. URL encoding uses a conservative
nine-characters-per-UTF-16-unit allowance before encoding a saved value.
Reduce the payload/repetitions, or raise the parent
Node heap and restart; this is a conservative allocation estimate, not a
measurement or guarantee of browser/process-family memory.
- Evidence records a request step's empty `save` list as absent, so a scenario
that saves nothing records the same parameters and `scenario_sha256` as
before saved values existed, and REA versions without saved values read its
captures.
## Network evidence
`capture.events: ["network"]` retains request, response, completion, failure,
docs/installation.md
+7
View File
@@ -84,6 +84,13 @@ For an existing configuration, setup and uninstall recovery refuse a detected
replacement or edit and name the recovery backup. Quit clients before editing their configuration;
these checks do not lock out another writer modifying the same file in place,
or exclude pathname changes between the final check and publication.
Setup refreshes an existing `rea` registration only when its launcher proves REA
ownership. Foreign or unresolved launcher identity blocks setup before any
backup or write, including an OpenCode legacy entry that migration would remove.
Review the existing launcher and manually rename or remove that registration to
approve replacing it. A different existing installation path is foreign to the
current entrypoint; a path that cannot be resolved remains unknown. Cordis uses
its stable REA entry ID instead of launcher identity.
Setup also retains valid environment overrides in REA's existing server entry,
such as `REA_MCP_INPUT_SCHEMA_PROFILE`, `REA_MCP_OUTPUT_SCHEMA_PROFILE`, and
`REA_MCP_MAX_RESPONSE_BYTES`, and `REA_MCP_TOOL_FAMILIES`.
docs/javascript-artifact-reconstruction.md
+9 -6
View File
@@ -72,14 +72,17 @@ code-point order. Dependency manifests remain package observations but do not
declare Electron main or renderer entries. If only dependency manifests are
present, the inventory artifact remains the graph root.
Package observations keep the legacy `renderer` property as a derived entrypoint
candidate: the first nonempty string from `renderer`, `browser`, then `module`.
It does not assert that the source declares a literal `renderer` field.
`renderer_declarations` records the actual present fields: strings, including
empty strings, remain exact; non-string values use `{ "status": "non-string" }`
Package observations use `renderer_declarations` to record the actual present
fields: strings, including empty strings, remain exact; non-string values use `{ "status": "non-string" }`
without copying nested browser mappings. An empty record means no such fields
were present; `null` means package metadata was invalid or unavailable. Renderer
role mechanisms and package load edges name the field that supplied the candidate.
role mechanisms and package load edges name the field that supplied the candidate,
selected as the first nonempty string from `renderer`, `browser`, then `module`.
The package load edge retains `declared_path`, `source_field`, and resolution
facts for that selection. New analyses no longer emit the derived package
`renderer` alias; consumers must read declarations for source metadata and load
edges for the selected candidate. Previously retained Evidence keeps its original
properties and remains readable without migration.
## Select sources before analysis
docs/managed-code-analysis.md
+14 -7
View File
@@ -660,13 +660,15 @@ name and raw signature digest available while marking that signature malformed
and coverage partial.
A fat method header's nonzero LocalVarSigTok must name an existing StandAloneSig
row whose blob is a structurally valid LocalVarSig. REA validates its local
types and exact blob coverage but does not project their identities. An invalid
types and exact blob coverage, retaining `local_var_signature` with the raw blob
digest and whether named types, modifiers, or generic variables require metadata
or declaring context. Historical Evidence defaults this field to null. An invalid
token or malformed signature leaves decoded CIL anchors intact but marks the
body malformed and coverage partial. Unsupported local type encodings retain
the readable body with partial coverage. Zero continues to mean no locals
signature. Comparison detects whether a locals declaration is present, but raw
table tokens are not cross-build identities and local type contents are not
compared.
signature. Raw table tokens are not cross-build identities. Comparison uses
validated context-free signature digests; signatures requiring metadata or
declaring context remain unknown across artifacts.
## Decoded-CIL fingerprint
@@ -868,9 +870,14 @@ MethodDef flags or implementation flags value differs, including accessibility
or synchronization changes. Observed method-body `init_locals` and `max_stack`
differences also contribute to `metadata`, even when CIL decoding is incomplete.
For complete bodies, presence or absence of a locals signature also contributes;
the locals table token is build-local and is not compared. Local type contents
are not currently compared, so this facet does not establish equivalent locals
declarations.
the locals table token is build-local and is not compared. Different validated
context-free LocalVarSig digests report a metadata encoding difference without
asserting a semantic type change. Equal digests establish unchanged bytes only
when neither signature requires metadata or declaring context. Historical
unrecorded signatures and context-dependent signatures keep local identity
unknown, preserving any independently observed changes. The same method token
in an artifact with the same authenticated SHA-256 identifies the same local
signature bytes, including for retained historical Evidence.
An unavailable header value keeps body coverage unknown rather than establishing
equality or a difference. These comparisons are separate from the CIL/signature
matching tiers: identical instructions do not establish unchanged method metadata.
docs/mcp-contracts.md
+2
View File
@@ -1491,6 +1491,8 @@ validates every result before delivery. On catalogs large enough that full
input schemas alone approach the frame limit, keep the input profile at its
`compact` default as well; with full input schemas the advertisement is
dominated by the input side.
The full input profile has no one-megabyte catalog-frame guarantee, including
when output schemas are omitted.
Only client-side schema discovery and validation are reduced or omitted, so
hosts using these profiles should rely on REA's validation and retain their
docs/native-investigation.md
+4 -1
View File
@@ -332,7 +332,10 @@ back. The effective heap stays unestablished when either flag is present,
including when an absolute `--max-old-space-size` is also on the command
line or in the recovered `NODE_OPTIONS`. Node accepts the config path as
`=` or the next argument.
Literal mode retains Unicode casefold matching.
Literal mode retains Python Unicode full case folding, which can expand a
character into a sequence: for example, `ss` matches `ß`. This differs from
REA's Unicode simple case folding used by other native search providers and
Hopper regex mode.
## Bounded static traces
docs/pcap-analysis.md
+7 -3
View File
@@ -81,9 +81,13 @@ Its separate inferred truncation lengths do not replace that limit; an unset
file-header limit remains `null`.
Addresses count once per packet, including loopback and nested IP occurrences.
DNS addresses are associated only with one question in the same response packet;
additional records may be included, so this does not prove final resolution.
Ambiguous addresses remain in a separate inventory. Protocol distribution counts
DNS address attribution requires one question in one response and complete,
consistent RR owner/type and section counts. Only answer-section A/AAAA records
matching that question (or its unambiguous answer-section CNAME chain) are
associated, with the requested address type respected. Authority, additional,
unrelated, malformed, and ambiguous addresses remain in the unattributed
inventory and make coverage partial. These packet-local relationships do not
prove final resolution or destination authenticity. Protocol distribution counts
exact nonempty `frame.protocols` stack strings. Frames for which TShark reports
an empty stack, including zero-length packets, still contribute to the packet
count without adding a protocol row. An all-empty stack inventory can have
docs/process-capture.md
+13
View File
@@ -500,6 +500,11 @@ Every capture requires `truncation_details`, with separate accounting for:
An unreadable directory is recorded in `enumeration_omissions` with its
root-aliased path and operating-system error code. Entries observed before
that directory remain available, while descendants beneath it stay unknown.
A basename that is not valid UTF-8 is never looked up through replacement
characters. It consumes entry capacity and records an `unrepresentable_name`
omission at its containing directory, with exact basename bytes in
`raw_name_hex` and a null `system_code`. That root's enumeration remains
incomplete, so missing paths beneath it remain unknown.
Each retained regular file without a digest has an aliased path, size,
remaining budget and reason: `file_bytes_budget`,
`file_changed_or_short_read`, or `file_unavailable`. Its `system_code` is
@@ -663,3 +668,11 @@ REA then preserves ownership uncertainty and can report `cleanup_incomplete`;
it does not disable verification to continue. A sandbox that explicitly allows
this read-only process inspector is a separate prerequisite from the compiler
and cache settings.
Symlink observations preserve their stored target bytes in `symlink_target_hex`
(lowercase hexadecimal). `symlink_target` retains valid UTF-8 text and is `null`
when the target bytes cannot be decoded as UTF-8. Filesystem effects compare
these exact bytes without resolving the link. Older captures without the hex
field still parse; a legacy target containing U+FFFD has ambiguous byte identity,
so matching target text alone produces an `unknown` effect. Symlink descendants
remain outside filesystem observation regardless of target encoding.
docs/specs/agent-environments/scenario-saved-values/plan.md
+22
View File
@@ -0,0 +1,22 @@
# Plan
1. Schema: `save` on request steps, `save_response` on click and press, the
`saved` value source in headers and credential parts, `basic_credentials`
with two parts, and the `{saved:name}` placeholder.
2. Validation: every saved value a step sends was saved by an earlier step.
3. Runtime: read the response body when a step saves, start waiting for a UI
step's response before it acts, keep saved values with their JSON type,
fill placeholders, build Basic credentials, and redact secret saves.
4. Evidence: leave an empty `save` out of the recorded step.
5. Environment build: warn about credential-named saved values not marked
secret.
6. Skill and docs: write through the UI first and save ids instead of guessing
them; bump the skill version.
7. Tests: schema, runtime with a fake page, Evidence parameters, the build
warning, the catalog budget, and a real-Chrome recording.
8. Write model: compose strings from generated ids and other fields, learn the
`&` spelling of slugs, tell apart addresses whose parts are the same text,
and check templates on later creates; census every recorded capture.
9. End to end: agents build environments from prompts with main and with this
branch on the same applications, and the branch's recordings are rebuilt
before and after the write-model changes.
docs/specs/agent-environments/scenario-saved-values/spec.md
+74
View File
@@ -0,0 +1,74 @@
# Saved values in browser scenarios
## Outcome
A browser scenario could not carry a value from one step to a later one. An
agent recording an application from a prompt therefore guessed the id of the
object it had just created, created probe objects to learn the next id, or
read the object back by list only. An API that takes Basic auth, or a header a
frontend builds from a login token, could not be sent at all, so the agent
stopped and asked for an API key.
Steps now save strings and numbers from JSON responses, and later steps send
them. A click or key press saves from a newly observed matching response,
so a scenario can create an object the way the application's UI does and then
open, read, edit or delete it by the id the application assigned. A request
header can send Basic credentials built from literal, secret, saved or page
values.
## Requirements
- A `request` step saves `[{name, json_pointer, secret}]` from its own JSON
response. A `click` or `press` step's `save_response: {method, path, save}`
saves from the first response that matches its method and a path on the start
origin (`{*}` matches one segment; the query is ignored), which must be 2xx
JSON. The wait starts before the step acts, and the step fails, saving
nothing, unless such a response arrives and is read before its timeout or
cancellation.
- A pointer that names no string or number, a number that is not finite (such as
`1e400`), a response that is not JSON or is larger than 4,000,000 characters
(a UI step reads none received as more than 12,000,000 bytes, three per
character), and a non-2xx `save_response` fail the step.
- `{saved:name}` sends a saved value, percent-encoded, in a request path and its
query and in a goto URL, and in a request's JSON body: a string that is only
the placeholder becomes the saved value with its JSON type, and a placeholder
inside a longer string is filled as text. Header values and Basic credential
parts send `{source: "saved", name}`. Other braces are sent as written. A
saved value that makes a path segment `.` or `..`, alone or with the text
beside it, fails the step.
- `basic_credentials` sends the base64 of `username:password` from `parts:
[username, password]`, each a literal, declared secret, saved value, cookie or
local storage entry. The encoded value and parts read from the page are
redacted as page values are. A username with a colon fails the step.
- Validation refuses a saved value no earlier step saves.
- A scenario keeps at most 4,000,000 characters of saved values at once, counting
the redaction spellings of values saved as secret; a save past that fails its
step before the value is kept.
- A value saved with `secret: true` is redacted from the selected source response
before retention and from subsequent records. The environment build warns about saved values named like credentials
that are not marked secret (`credential-named-saved-values`).
- A scenario that saves nothing records the same Evidence parameters and
`scenario_sha256` as before: an empty `save` is left out of the recorded
step.
- Earlier REA versions read captures that use saved values.
- The compact MCP catalog stays under its measured one-megabyte frame.
- The skill records writes the way the application's UI does and falls back to
request steps when the UI cannot reach an operation or coverage withholds the
UI-shaped write.
- A UI-made create must be served as faithfully as one made through the API:
- a string composed from a generated id and another field, such as Grafana's
folder address, is learned as composed, not as a template around the id;
- a slug learns how the application spells `&` from the call that shows it,
and held entities that spell it both ways leave no slug rule;
- an id that equals the title's slug is told apart by the entities the type
holds, or by a later create;
- a template a later create answers otherwise is withheld, as a composed
string is.
## Non-goals
- No general value transforms, string templates or script steps: the value
kinds are fixed.
- No saved values in `fill` or `select_option` values.
- No retroactive redaction of earlier unrelated records.
- No multi-factor sign-in.
docs/specs/agent-environments/scenario-saved-values/tasks.md
+15
View File
@@ -0,0 +1,15 @@
# Tasks
- [x] Record the outcome, requirements and non-goals.
- [x] Add saved values, `save_response` and Basic credentials to the scenario schema.
- [x] Validate that saved values are saved before use.
- [x] Save and send values in the Playwright runtime.
- [x] Keep scenarios that save nothing byte-identical in Evidence.
- [x] Warn about credential-named saved values not marked secret.
- [x] Update the skill and docs.
- [x] Add unit, boundary and real-browser checks.
- [x] Run the gates.
- [x] Build prompt-recorded environments with main and with this branch.
- [x] Make the skill's recording example write through the UI.
- [x] Learn the addresses a UI-made create spells, and check templates on later creates.
- [x] Rebuild the branch's recordings with the write-model changes and record every application again.
docs/specs/agent-environments/scenario-saved-values/verification.md
+138
View File
@@ -0,0 +1,138 @@
# Verification
This is Edgar Arakelyan's historical verification record from upstream PR #7162
(head `d39f386d202fe8e98592fb2d9d26e43d8d0fb94f`), retained with attribution.
These multi-application and model evaluations were not rerun during integration.
The independent integration measurements and their limits are recorded in
[saved-values integration verification](../../../verification/browser-saved-values-2026-10-26.md).
Executed on macOS with Node v24.22.1 at `feat/recorder-saved-values` e42362c9 (on main 2b84efe0), and on Ubuntu 24.04
VMs with snap Chromium for the agent recordings.
## Gates
- `npm run typecheck`, `lint:built`, `format:check`, `metadata:check`, `verify:architecture-guards`, `knip:production`,
`lint:typed`, `docs:check`, `mcp-catalog:check`, `verify:test-discovery`, `scan:todos` and `jscpd`: passed.
`knip` reports one unused export, `isToolFamilyId` in `src/contracts/toolFamilies.ts`, which this branch does not
touch and main 2b84efe0 reports too.
- `npx vitest run --project domain --project services --project adapters`: 18,261 passed and 1 failed, "stops Inspector
target authorization after the current stat settles", which fails the same way on main in this environment.
- Browser scenario CLI and MCP boundaries, scenario input diagnostics, cancellation and cleanup, the analysis-view CLI
and MCP boundaries, the environment build, export and serve CLI boundaries, the environment serve, served-writes and
bundle MCP boundaries, tool schema validity, MCP schema validation, and output schema constraints and profile: 17
files, 128 passed.
- The compact MCP catalog must stay under the one-megabyte frame `outputSchemaProfile.test.ts` measures. A first
design with `path_params` on request and goto steps, `json_values`, and separate `username` and `password` unions
added 9,391 bytes on main ff6d96bc and failed the test, which then had 5,731 bytes to spare. Saved values now travel
as `{saved:name}` placeholders, Basic credentials repeat their part union once, and repeated inner descriptions were
dropped: on main 2b84efe0 the catalog takes 1,038,975 bytes and this branch 1,042,639.
## Compatibility
- Scenarios that save nothing record the same Evidence parameters and `scenario_sha256` on main and on this branch: a
UI-only scenario with a goto to a URL holding literal braces, and a request step with a secret query value, a cookie
header, and literal braces in its path and JSON body.
- Main reads a capture that uses saved values: `environment-build --dry-run` of the real-Chrome capture below gives the
same manifest, state, coverage and file plan on main and on this branch. This branch adds the
`credential-named-saved-values` warning.
- Write plans for 109 captures (the 50 published captures, 29 earlier agent recordings and the 30 recorded in the
comparison below), built by main and by this branch: 524 plans and 247 served on each. Three plans differ, all
Grafana folder creates recorded below, whose `url` the branch composes from the uid and the title's slug where main
learned a template or nothing.
## Real Chrome
A local application with a cookie-session UI and an API that takes only Basic auth, recorded with Google Chrome:
- fill a name and click Save with `save_response` on `POST /ui/items`, saving the new id;
- `goto` `/items/{saved:item1}` and wait for the item's page;
- `GET /api/items/{saved:item1}` with Basic credentials whose username comes from local storage and password from a
declared secret;
- `POST /api/tokens`, saving the token both as a secret and, unmarked, as `session_token`;
- `POST /api/items` with the saved token in a header and `{"name": "Child of {saved:item1}", "ref": "{saved:item1}"}`;
- `PATCH /api/items/{saved:child}` and `GET /api/items?parent={saved:item1}`.
All 11 steps completed. The requests went to `/items/101`, `/api/items/101`, `/api/items/108` and `?parent=101`, and the
body sent was `{"name":"Child of 101","ref":101}`. The Evidence parameters keep the placeholders, the password appears
nowhere in the capture, and the token header is recorded as `[REDACTED:page:token.api_token]`. The token's own response
was redacted too in this run; the docs say it may not be, since the recorder reads bodies on their own schedule.
`environment-build` warned about `/parameters/actions/7/save/1`.
## Agent-built environments, main and this branch
Claude Code (`claude-sonnet-5-5`) built an environment of each application from the same prompt, with REA main
ff6d96bc (skill 49) and with this branch (skill 50), on freshly seeded instances. Each bundle was rebuilt by the REA it
was recorded with, and its write tasks run with `fireworks_ai/gpt-oss-120b` and `fireworks_ai/glm-5p3-flash`. Each task
checks its result from the environment's state, including that nothing else changed.
The branch was recorded three times as it changed:
1. With the skill as first written, EspoCRM's agent wrote through request steps with Basic credentials instead of the
UI, and EspoCRM's API answers in a shape the write model withholds (0/3). The skill's recording example created its
object with a request step, which agents copy; it now creates through the UI with `save_response`.
2. With that example, Grafana's agent created folders through the UI, where Grafana generates the uid. REA then learned
the folder's `url` as a template around the uid that kept the first folder's slug, and coverage did not check the
template against the second folder, so served creates answered with wrong addresses (0/2). The write-model fixes
in this PR compose the address from the uid and the title's slug, learn Grafana's `&` spelling, and check templates
on later creates.
3. With both, every application was recorded again.
**The same recordings, before and after the write-model fixes.** Run 2's captures, rebuilt by this branch before and
after the fixes (gpt-oss, GLM):
| Application | Before | After |
| ----------- | ------------- | ------------- |
| Miniflux | 3/3 3/3 | 3/3 3/3 |
| Paperless | 1/3 2/3 | 1/3 1/3 |
| Vikunja | 1/3 1/3 + 0/2 | 1/3 1/3 + 0/2 |
| Linkding | 0/2 0/2 | 0/2 0/2 |
| Homebox | 0/2 0/2 | 0/2 0/2 |
| Grafana | 0/2 0/2 | 2/2 2/2 |
| EspoCRM | 1/3 1/3 | 1/3 1/3 |
Only Grafana's plans change, and its tasks pass. Paperless's plans are identical in both builds; its GLM difference is
the eval model's.
**Fresh recordings, main and the final branch:**
| Application | Write tasks | Main: served, tasks | Branch: served, tasks |
| ----------- | ----------- | ---------------------- | --------------------- |
| Miniflux | 3 | no bundle | 4, 3/3 3/3 |
| Paperless | 3 | 3, 1/3 1/3; 6, 2/3 2/3 | 6, 1/3 2/3 |
| Vikunja | 3 + 2 | 1, 0/3 0/3 + 0/2 0/2 | 3, 1/3 1/3 + 2/2 2/2 |
| Linkding | 2 | 0, 0/2 0/2 | 3, 0/2 0/2 |
| Homebox | 2 | 1, 0/2 0/2 | 5, 0/2 0/2 |
| Grafana | 2 | 2, 2/2 2/2; 3, 2/2 2/2 | 1, 1/2 1/2 |
| EspoCRM | 3 | 2, 1/3 1/3 | 2, 0/3 0/3 |
| Total | 20 | 9, 4/20 4/20 | 24, 8/20 9/20 |
"Served" counts the write tools the final capture serves; tasks are gpt-oss then GLM, and Miniflux's count as failed
without a bundle. Paperless and Grafana were recorded twice on main; totals use the first run.
- **Miniflux:** main's agent stopped and asked for an API key, since the API takes Basic auth. With Basic credentials
the branch served category create, rename and delete in all three runs, and every category task passed with both
models.
- **Vikunja:** label edits and deletes are served and pass in two of the three branch runs, and task deletes in the
last two.
- **Saved ids:** every branch agent saved ids, with `save_response` on UI writes, and read objects back by them. On main
no step can carry a value to a later one, so agents chose their own ids or read lists back instead.
- **Paperless:** the two main runs differ as much as main and the branch do.
- **Linkding and Homebox** serve more writes, but their create tasks still fail. Linkding's served create requires every
member its recorded calls sent (`description`, `notes`, `unread`, `shared`, `tag_names`), so a task that sends a URL
and a title is refused.
- **Grafana:** the folder create is served with the right address and its task passes. The rename is withheld:
Grafana lists folders sorted by title, the recording renamed a folder to a title that sorts first, and REA keeps an
updated entity where it was in a list. Main's REA reads this recording the same way, and serves its create with the
template that keeps the first folder's slug, so the create task would fail too. Main's two recordings passed because
their titles kept their order.
- **EspoCRM:** the recording started with a contact an earlier attempt in the same session created and left, so the
environment holds it as seeded data and every task's "nothing else changed" check fails. The skill now asks agents to
delete what an earlier attempt created before recording again; that change was not measured.
- Agents spent $9.98 and 3,149 seconds in total on the fresh branch run, against $7.45 and 2,235 on main, a third of it
Homebox's.
## Not verified
- One fresh recording per application and version, two for Paperless and Grafana on main; agent runs vary.
- Lists sorted by a field that an update changes, and the skill's cleanup of earlier attempts.
- Multi-factor sign-in, and saved values in `fill` or `select_option` steps, which this change does not add.
docs/verification/browser-saved-values-2026-10-26.md
+271
View File
@@ -0,0 +1,271 @@
# Browser saved values: integration verification
## Pinned scope
Executed on Linux x64, Node 24.22.1 and Chromium through `/usr/bin/chromium`.
The primary comparison uses **Playwright 1.64.1 and the same integration tree
for both workflows**. Local verification commit `dd343d3b1092d697772f1447d94179d23bcc5cf2`
contains GitHub merge `1a3ef66c8e25f25fdd06053bb33255f10f65c5b6`
(main `030fa886f94e894e34ed2539e9d8dedff19c3327` plus PR head
`ed110935f520d56f6eaf5710ad2f9a58340492b4`), followed by response-scope fix
`2c5f9cc14b3395ef4a99071222e3feaf9501bb62`. The local verification commit is
not a published release. Recreate its tree from the public main/PR inputs,
then apply that fix. Final reconciliation adds `0d89c65409d28fec269cfdbb3ee62e0f153819fa`
(local current-dependency tree `221ba53af`) for query spelling and refused-slug
source constraints; the primary timing pair predates these two corrections.
Historical comparisons against pre-feature main
`6d5532a204b338a6c770fcbae61e0d55bda70793` used Playwright 1.58.2 and remain
below. They do not isolate the dependency upgrade. The implementation integrates
Edgar Arakelyan's #7162; upstream `d39f386d202fe8e98592fb2d9d26e43d8d0fb94f`
and subsequent fixes through `9f5191ab0` were compared. Its query redaction and refused-slug source
pinning were incorporated. Existing heap-derived URL/JSON/envelope admission
remains the owner of expansion bounds, rather than adding the upstream fixed
fill ceiling. Original multi-app/model
results are [attributed historical evidence](../specs/agent-environments/scenario-saved-values/verification.md),
not executions performed here.
## Real application and correctness oracles
PocketBase **0.40.6** runs on an owned loopback instance. Its official Linux amd64
archive is 12,824,710 bytes, SHA-256
`0e1f609158ed945f809f8a02ae012aa0427129c66ebb63c1aee76f2c0309373f`.
The [owned frontend](https://github.com/morluto/rea/blob/938052704/experiments/analysis-efficiency/fixtures/browser-saved-values/index.html)
uses actual superuser authentication and record APIs. It is a small test
frontend rather than PocketBase's entire administrative application.
Collection `rea_items` has required text `title` and numeric `quantity`, public
list/view, and superuser-only create/update/delete rules.
The completed task signs in, creates a record, reuses its generated ID and
numeric quantity, PATCHes it, GETs it with a query, and navigates to its API URL.
Independent PocketBase GETs verify identity, final title and numeric quantity;
retained request bytes verify a JSON number. Every declared action must finish.
The selected login response must actually occur once; no missing response can
pass the redaction oracle.
- Current integration public CLI: all **eight steps** completed, eight retained
JSON bodies, zero plaintext marked token or declared password. **4.351 s elapsed**,
**4.832 s CPU**, **1,381,789,696 B** sampled process-family peak.
- Current integration MCP: two equivalent tasks completed in both legacy and
saved-value modes. Legacy uses a retained event page and another capture;
saved bindings finish the same analyst task in one capture.
- Both current legacy runs actually observed the login body before the later
page-value read. That earlier unmarked token remained present, while later
observations used redaction. The mixed-scope unit cases additionally establish
that an unrelated page/Basic value remains prospective when response secrets
are also learned.
- A real negative control on the imported implementation retained the marked
authentication token despite completing its actions. The integrated source
response and retained-event checks reject that outcome.
## Privacy, provenance and resource boundaries
The public MCP source probe verifies:
1. Source and later JSON bodies with escaped strings and `1e3` become valid
redacted JSON while preserving unmarked `1e-400` and whitespace exactly.
2. Metadata-only capture redacts the selected source URL without retaining
network-content records. Unknown source values withhold the affected source
fields and expose a limitation.
3. A declared whole JSON string seeded into storage and POSTed to PocketBase is
withheld if a safe whole-body replacement cannot preserve JSON. HTTP **403**
and an unchanged integer record count independently confirm the outcome.
4. The frontend consumes the selected response, logs and visibly renders its
token, and immediately POSTs it in a URL/header/body. Retained console and
network data are protected. All three requested screenshots are `missing`,
with screenshot completeness ineligible for equality.
5. A separate nonsecret scenario captures a real PNG and validates its signature.
This expanded probe passed in **8.417 s elapsed**, **9.567 s CPU**,
**1,680,818,176 B** sampled family peak. A focused public CLI run of the shared
CI browser fixture also passed: three withheld sensitive screenshots, all other
selected artifacts captured, and explicit screenshot/network-cutoff gaps.
It took **5.956 s elapsed**, **6.868 s CPU**, **1,495,924,736 B** family peak;
that fixture server is inside the measured family.
The source parser owns strict UTF-8, exact selected numbers, pointer lookup and
source-token redaction. UI whole-body saving requires a positive known
identity-encoded size before reading; compressed/cache-unknown responses are
refused. The original author's fallback is not enabled. Two initial cached UI
probes were correctly refused; the owned fixture uses `cache: "no-store"` for
the known-size case.
Response-marked values can backfill retained source/follow-up event text.
Page-derived cookie/storage values and constructed Basic credentials retain
their existing prospective-only policy. A session-local response projection
borrows admitted compiled patterns/byte owners and is invalidated on admissions
or response classification. Promoting an already held value to an explicit
response secret preserves its first marker. Replacement work and retained output
are admitted before allocation; omissions and cleanup retry are accounted once.
UI matching is observational. Multiple observed matches fail without adopting
an ambiguous binding; a single observed match cannot prove UI causality or
exclude a later poll. Returned Playwright body bytes also do not establish
original-wire provenance. These limitations remain in the contract/Evidence.
Request JSON, saved URLs and the full request envelope are admitted before
expansion/dispatch using Node's string ceiling and a conservative V8-headroom
allowance. Repeated saved/header facts share request-local size/validation work.
Four cases reject 1,000 references to a 3,000,000-character value without dispatch;
those inputs were deliberately not run on unbounded code. The first repeated
header guard took 4.042 s in its source test; the final case no longer appears
in slow-test output (its final individual duration was not separately measured).
Raw pixels are not text-redactable. The secret owner establishes screenshot
withholding before execution for planned sensitive response/page/Basic values,
and whenever sensitive values are already known. The artifact producer then
returns `missing` without requesting pixels. No separate image-cleanup state or
publication hook remains.
## Completed-task measurements
Both columns use the current integration tree, provider and catalog. Each run
creates a fresh MCP server/browser context and completes a first and subsequent
task. The browser has already loaded the fixture; “first” is not a cold browser
or OS cache. One run per condition is an observation, not a general speed claim.
| Measurement | Legacy workflow | Saved values |
| -------------------------------------- | -----------------------: | -----------------------: |
| Task calls, each task | 3 | 1 |
| First task elapsed | 6.191 s | 1.879 s |
| Warm task elapsed | 1.120 s | 0.872 s |
| Whole verifier elapsed | 13.413 s | 8.092 s |
| Whole verifier CPU | 15.347 s | 9.387 s |
| Sampled family peak RSS | 2,018,435,072 B | 1,689,305,088 B |
| Input catalog + arguments/text replies | 34,885 estimated tokens | 25,881 estimated tokens |
| Full SDK catalog + arguments/results | 370,936 estimated tokens | 353,768 estimated tokens |
`o200k_base` BPE estimates serialize compact JSON and are not model billing.
The model projection counts all **53** tool names/descriptions/input schemas
(**18,579 tokens** in both columns), followed by the complete array of named
arguments and text-content results. The full SDK estimate counts the complete
catalog plus the array of full results, including output schemas and structured
results. Arrays are tokenized jointly rather than summing independently tokenized
cells. Both estimates include two extra correctness-proof exports: eight total
calls before and four after. No model ran; reasoning, retries, JSON-RPC framing
and initialization traffic are unmeasured.
Whole-verifier cost includes discovery, browser setup, proof/oracle work and
cleanup, distinct from timed task calls. PocketBase runs externally: its CPU
and memory are **unmeasured**. REA/Chromium families use two CPUs, a 3 GiB Node
heap and an 8 GiB family RSS guard. RSS is summed every 100 ms, may double-count
shared pages and miss short peaks. No guard stopped the completed runs.
Historical pre-feature comparisons (Playwright 1.58.2) remain observations:
| Candidate | First task before→after s | Warm before→after s | Whole elapsed before→after s | CPU before→after s | Peak GB before→after |
| ----------- | ------------------------: | ------------------: | ---------------------------: | -----------------: | -------------------: |
| `355e2dcb8` | 7.703→3.531 | 1.390→1.605 | 14.990→17.198 | 15.735→15.679 | 2.080→1.636 |
| `dd3085bf5` | 5.145→1.621 | 1.020→0.762 | 11.723→7.653 | 13.444→8.683 | 2.028→1.638 |
| `439a389fe` | 16.121→4.429 | 2.944→1.738 | 34.296→20.053 | 32.166→19.053 | 1.996→1.646 |
| `e7acc5d85` | 10.778→2.627 | 1.467→1.129 | 22.310→12.661 | 26.148→14.759 | 2.103→1.632 |
| `43aaaabbe` | 7.751→3.220 | 1.210→0.996 | 15.527→11.301 | 18.047→12.777 | 2.064→1.628 |
| `854b7a2c0` | 6.965→1.923 | 1.100→0.874 | 15.657→9.213 | 17.222→10.753 | 2.053→1.650 |
| `938052704` | 5.983→1.720 | 1.040→0.787 | 13.103→8.146 | 15.194→9.497 | 2.033→1.635 |
The first candidate was slower for warm/whole elapsed. Later differences also
reflect host/cache variance. Historical model estimates include 34,899→25,897
at `355e2dcb8`, 34,831→25,902 at `e7acc5d85`, 34,816→25,866 at `43aaaabbe`,
34,742→25,845 at `854b7a2c0`, and 34,820→25,897 at `938052704`. These are not
summed as independent model usage.
## Executed checks and negative controls
- Final feature branch: **560 tests in 12 source suites**, type/lint and build passed.
- Current integration/Playwright 1.64.1: **561 tests in the same 12 suites**,
build and `check:fast` passed; the additional test comes from main.
- Twenty explicit schema/profile tests passed earlier; all 41 distinct cases in
the broader five-suite run passed across its run and isolated rerun. One case
timed out at 30 s under two-CPU contention, then passed alone in 17.36 s.
An older-Node attempt failed before tests. Final real SDK calls also validate
the named tool definitions and advertised results.
- Negative controls fail before their fixes: decoded bounds/accounting/numeric/
slug cases; seven source-format/UI ambiguity cases; three source-publication
cases; two whole-JSON cases; two backslash substitutions plus immediate
follow-up; five final event/path cases; and ten screenshot cases. The
screenshot nonsecret control passes on the old revision.
- Final reconciliation re-ran the eight-step real CLI task and all four privacy
probes plus the positive PNG case on `221ba53af` successfully. A further
two-step public CLI case verifies the parsed query spelling of `ab'cd~ef`
is redacted. Together these checks used 16.550 s elapsed, 18.581 s CPU and
1,675,153,408 B sampled family peak; this is qualification, not a new timing pair.
- The final upstream query/slug regressions fail three cases on `34dc2c1c4`.
The query spelling lower-bound calculation also includes the new encoding
before allocating it.
- Three prospective-page/Basic regressions fail on `ed110935f`; final tests also
cover promotion, scope-cache invalidation and preservation of the first marker.
- The prior CI's own event-budget assertion was fixed with JSON admission order
and the whole-body guard. It was not bypassed. Two invalid newly added header
fixtures were corrected to canonical shapes before the final source run.
- A focused CI-fixture harness initially closed its server too early. Cleanup
order was corrected and the real CLI rerun passed. Failed receipts remain.
Initial CI/main controls reproduced a load-sensitive prewarm failure and two
SAZ partial-observation assertions on their exact parents. The prewarm case
passes alone on later main and is not claimed as a stable current failure.
Subsequent CI failures comprise those two SAZ assertions and Java diagnostic
`keeps empty Java bitness unresolved`; all three independently reproduce on
actual CI main parent `c6893d1ac6a9ccb8a11d64b4e9ba1b57e79cbfd8` with its isolated
1.64.1 locked dependencies. The first preparation refused stale shared 1.58.2
dependencies before any tests; the shared installation was preserved.
On head `34dc2c1c4`, all 18 distinct CI failures (artifact/inventory, snapshot,
cleanup, Apple-resource coverage and the two SAZ assertions) reproduce on exact
CI main parent `29ebd6740550bf3989129c39c65f17e5c61e3dd2`. The additional
control run also timed out two retained-ZIP cases and emitted two connection
closures; these were control-only incomplete results, not passing checks.
All fifteen actionable review threads were fixed and resolved. A further
final-head Codex review request was refused by the repository usage limit; no
fresh successful automated review is claimed.
Cancelled lanes remain incomplete, including cancelled shards that emitted
passing summaries. CI counts from overlapping revisions are not added together.
## Reproducing and limitations
Use an isolated worktree with the pinned Node/npm toolchain. For the primary
comparison, recreate the public main/PR integration inputs above, apply the
response-scope fix, install its lockfile, build once, and use **that same checkout**
as `--repository` for both modes below. Historical pre-feature measurements used
separate pinned checkouts.
On an owned loopback PocketBase instance, create the collection/rules described
above and a temporary superuser. Serve the committed `index.html`,
`whole-json.html` and `immediate-followup.html` from
`experiments/analysis-efficiency/fixtures/browser-saved-values/`. Store only
`{"email":"...","password":"..."}` in a private credentials file, mode 0600;
keep it and full retained Evidence out of Git.
Serve `escaped-values.json` with these exact minified bytes (unmarked spelling
is part of the oracle):
```text
{"token":"\u0073ource-token","code":1e3,"unknown":1e-400}
```
Run the committed driver with fresh private output directories:
```bash
node experiments/analysis-efficiency/browser-saved-values.mjs \
--repository /path/to/integration --mode before --output /private/before \
--origin http://127.0.0.1:18762 --credentials /private/credentials.json \
--browser /usr/bin/chromium
node experiments/analysis-efficiency/browser-saved-values.mjs \
--repository /path/to/integration --mode after --output /private/after \
--origin http://127.0.0.1:18762 --credentials /private/credentials.json \
--browser /usr/bin/chromium
node experiments/analysis-efficiency/browser-saved-values.mjs \
--repository /path/to/integration --mode after --source-only \
--output /private/privacy --origin http://127.0.0.1:18762 \
--credentials /private/credentials.json --browser /usr/bin/chromium
```
Run these under a process-family CPU/RSS supervisor for comparable resource
measurements. The driver owns its MCP client and browser/profile cleanup, checks
payload outcomes, records arguments/results/catalog, and exports retained parent
Evidence. Current legacy runs observed pre-read login bodies in both tasks; if
that ordering is not observed, the receipt says so rather than inventing timing.
Real-provider scope is Linux Chromium and the owned PocketBase/frontend plus
shared CI fixture. Compressed/cache-unknown UI saving is unsupported. Other
platforms, real Basic-auth servers, service-worker integrations and historical
multi-app/model evaluations were not rerun here. No model task success or actual
model-token billing is claimed.
docs/verification/browser-scenario-validation-2026-10-27.md
+57
View File
@@ -0,0 +1,57 @@
# Browser scenario offline validation
Executed on Linux x64 with Node 24.22.1/npm 11.16.0, based on main
`3fcbabe593611435ebc26e1eac86706fe60abb6e`. Integrates Ace's #7149
(`463c8bd73`), with one shared CLI input/admission helper and stronger boundary
coverage. The domain scenario contract remains the semantic owner.
The input is the preserved scenario from the successful real PocketBase 0.40.6
CLI workflow described in [saved-values verification](browser-saved-values-2026-10-26.md).
Its SHA-256 is `cfbb3428292250519a5a6f7f9c4b5fea494eb9e3ceccf7b777fd44ffe0f12223`.
The backend was stopped and its loopback endpoint confirmed unreachable before
these checks. Both declared secret environment variables were removed from each
child environment. The original scenario file was verified unchanged.
Each invocation used the public `validate-browser-scenario FILE --json` command.
All returned `{"valid":true,"actions":7,"secrets":2}` and exit 0. The seven
caller actions exclude the implicit initial capture step.
| Input / fresh CLI process | Elapsed s | CPU s | Sampled family peak bytes |
| ----------------------------------------------- | --------: | ----: | ------------------------: |
| Recorded scenario | 1.712 | 2.013 | 217,907,200 |
| Same input repeated | 1.722 | 1.989 | 217,239,552 |
| Browser replaced with a nonexistent executable | 1.667 | 1.933 | 219,025,408 |
| Browser replaced with disconnected loopback CDP | 1.557 | 1.880 | 218,779,648 |
Two CPUs, a 3 GiB Node heap and an 8 GiB family guard bounded execution. Family
RSS is the sum sampled every 100 ms; shared pages can be counted twice and short
peaks can be missed. No resource guard stopped a task. These are fresh processes
with warm filesystem caches, not cold-OS measurements or an analysis-cache claim.
There was no equivalent earlier public offline workflow, so no comparative
speedup is claimed. Profiling/tokenizer driver overhead is outside these tasks.
Each JSON reply is 38 bytes / **14 estimated o200k tokens**. The complete Node executable/script/CLI argument array plus parsed reply,
encoded as compact JSON `{argv, result}`, uses 75–77 estimated tokens, including
the local paths. Shell/profiler framing is unmeasured. The documents themselves are 1,926–1,942 bytes / 500–508 tokens if
read into context; this file-based task does not require returning them. There
is no model invocation/billing or MCP schema/discovery in these CLI tasks.
Boundary checks cover unavailable launch/connect resources, throwing provider
factory and enumerable secret getter spies, duplicate/reserved IDs, undeclared
secrets, duplicate secret IDs/environment names, ordered saved references,
unknown fields, unreadable/malformed files, caller-relative executable paths,
stdin and public CLI exit codes. Existing capture diagnostics and summary
publication remain covered. Initial test-helper type errors were corrected
before the final static gate; no passing result is claimed for that failed run.
Validation establishes schema and cross-reference validity. It does not establish
browser availability, secret availability, selector correctness or successful
execution, and it emits no analysis Evidence. Real-browser capture was verified
in the preceding feature; these offline checks construct no browser provider.
To reproduce with an existing scenario:
```bash
rea validate-browser-scenario ./browser-scenario.json --json
cat ./browser-scenario.json | rea validate-browser-scenario - --json
```
docs/verification/javascript-retained-facts-2026-10-15.md
+7
View File
@@ -36,6 +36,13 @@ not a claim of globally complete analysis. Full Evidence remains exportable.
The domain projector and shared application service own both CLI/MCP behavior;
no new provider, executor, persistent cache or live-state freshness claim exists.
The dated measurements below requested the then-emitted derived package
`renderer` property. New analyses use `renderer_declarations` for source fields
and package `loads` edges for the selected candidate; use
`"renderer_declarations"` in place of `"renderer"` when repeating the fact query
against a new analysis. Historical retained Evidence and measurements remain
unchanged.
## Real input and reproduction
Use the official [Joplin v3.7.21 Linux AppImage](https://github.com/laurent22/joplin/releases/download/v3.7.21/Joplin-3.7.21.AppImage).
docs/web-network-captures.md
+27 -5
View File
@@ -780,8 +780,16 @@ sorted stays sorted, unless a call stored its strings out of order as sent),
and a field neither rules holds one value. Members either call sent are accepted, only those every call sent are
required, and an echoed member a call left out takes the value that call stored
(`body.defaults`). A clock rule widens to `trimmed` only when no held entity or
earlier call showed that clock at a fixed width. A later create that disagrees
leaves the plan as it was. A create whose observed reference differs from the
earlier call showed that clock at a fixed width, and a slug to spelling `&` as
the word (`ampersand: "and"`) only when no call the plan accepted nor entity the
type held dropped one. A later
create that disagrees leaves the plan as it was. A field the first create left
unexplained, such as an address whose two parts were the same text, takes the
`composed` rule a later create learns for it when that rule spells the first
create's value too, and the first create's body contract is learned again with
it. A `composed` string or `template` a later create's answer shows otherwise
or leaves out is `unexplained-change`. A create whose observed reference
differs from the
entity the plan rebuilds, whether or not it sent the reference member, is
`unexplained-change`. So is a create that left out an echoed member yet shows a
value different from the learned default, or that stored a field no rule writes
@@ -789,9 +797,23 @@ otherwise than the plan keeps it.
A `composed` string is literal text and parts read from the fields other rules
write: a field's value, its slug, or a clock's calendar digits
(`YYYYMMDDhhmmss`, `YYYYMMDD`, `YYYY-MM-DD` or `YYYY/MM/DD`, in the clock's
zone). A create composes a string only when at least one other entity of the
type spells it with the same parts and none spells it otherwise, and never when
one text could come from two sources serving could tell apart. A clock format
zone). A slug drops `&` unless the application spells it as the word, as Go's
gosimple/slug and JavaScript's slugify do (`ampersand: "and"`, on a `slug` rule
or part). When a create slugs no `&`, the entities the type holds show which,
and no `slug` rule is learned when they spell it both ways; a create whose slug
spells `&` otherwise than they do, or alongside one whose
slug neither spelling gives, has no `slug` rule; such a held slug also keeps a
later create from widening the spelling. An update learns the spelling the same way: a later update whose slug spells
`&` as the word refines a rule an earlier update learned from a name without
`&`, unless an accepted call or a held entity dropped it, and every earlier
update must still replay. An update whose slug spells no `&` takes the spelling the
entity's earlier value and the type's other entities show, as a create does,
and learns no slug rule when they spell it both ways. A create composes a string only when at least one other entity of the
type spells it with the same parts and none spells it otherwise. Where one text
could come from two sources serving could tell apart, such as an id that equals
the title's slug, the entities the type holds must leave exactly one reading. A
`composed` string that reads a field besides the generated value a `template`
would rebuild it around replaces the template. A clock format
is `trimmed` when the field's values differ only in how many fraction digits
they show and none ends a fraction in a zero, as Go spells nanoseconds: its
`fraction` is the most it shows, and serving drops trailing zeros and the point
experiments/analysis-efficiency/README.md
+4
View File
@@ -1,5 +1,9 @@
# Analysis efficiency measurements
The [Android component checkpoint](android-components.md) records exact selected
declaration views, real JADX verification and completed-task token estimates,
including increased schema cost and no demonstrated speed or memory gain.
The [2026-10-21 coverage reconciliation](coverage-review-20261021.md) maps all
46 source directories to concrete workflow evidence, including original feature
PRs, historical no-benefit results and remaining verification boundaries.
experiments/analysis-efficiency/android-component-costs.py
+65
View File
@@ -0,0 +1,65 @@
"""Count completed Android MCP tasks; estimates do not establish model billing."""
import argparse
import hashlib
import json
from pathlib import Path
import tiktoken
def compact(value):
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False)
def inspect(path, encoding):
report = json.loads((path / "report.json").read_text())
catalog = json.loads((path / "catalog.json").read_text())
calls = json.loads((path / "calls.json").read_text())
assert len(report["runs"]) == 2 and len(calls) == 4
assert all(run["answer_verified"] and run["calls"] == 2 for run in report["runs"])
count = lambda text: len(encoding.encode(text, disallowed_special=()))
inputs = [{key: tool[key] for key in ["name", "description", "inputSchema"]}
for tool in catalog["tools"]]
question = (f"For {report['target']}, find the receiver declaring "
"android.provider.Telephony.SMS_RECEIVED and report its identity, permission, "
"enabled/process declarations, per-filter priority and source position. "
"Preserve source coverage and export complete parent Evidence. "
"Static declarations do not establish runtime behavior.")
rows = []
for index, run in enumerate(report["runs"]):
selected = calls[index * 2:index * 2 + 2]
assert [call["name"] for call in selected] == ["inspect_android_package", "export_evidence_bundle"]
assert not any(call["result"].get("isError") for call in selected)
arguments = sum(count(compact({"name": call["name"], "arguments": call["arguments"]}))
for call in selected)
text = sum(count(block["text"]) for call in selected
for block in call["result"]["content"] if block["type"] == "text")
wire_calls = [{key: call[key] for key in ["name", "arguments", "result"]} for call in selected]
rows.append({**run, "estimated_tokens": {
"question": count(question), "input_catalog_once": count(compact(inputs)),
"arguments": arguments, "text_replies": text,
"whole_task_text_projection": count(question) + count(compact(inputs)) + arguments + text,
"sdk_catalog_and_calls": count(compact({"catalog": catalog, "calls": wire_calls})),
}, "inspection_sdk_reply_bytes": len(compact(selected[0]["result"]).encode()),
"inspection_sdk_reply_tokens": count(compact(selected[0]["result"]))})
return {"case": str(path), "mode": report["mode"], "sha256": report["sha256"],
"runs": rows, "actual_model_usage": None,
"receipts_sha256": {name: hashlib.sha256((path / name).read_bytes()).hexdigest()
for name in ["report.json", "catalog.json", "calls.json"]}}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("cases", type=Path, nargs="+")
args = parser.parse_args()
encoding = tiktoken.get_encoding("o200k_base")
records = [inspect(path, encoding) for path in args.cases]
assert len({record["sha256"] for record in records}) == 1
args.output.write_text(json.dumps({"tokenizer": f"tiktoken {tiktoken.__version__}",
"encoding": encoding.name, "scope": "Text projection counts a fixed question, discovered name/description/input schemas once, all RPC arguments and text replies including proof export. Full SDK transport also counts output schemas and structuredContent. Host context injection and model billing are unmeasured.",
"records": records}, indent=2) + "\n")
if __name__ == "__main__":
main()
experiments/analysis-efficiency/android-components.md
+222
View File
@@ -0,0 +1,222 @@
# Android component declarations and selected retained facts
Addresses [#4448](https://github.com/morluto/rea/issues/4448), extending the inventory
already on main. Builds on Harsh's partial [#7264](https://github.com/morluto/rea/pull/7264)
revision `987b5aceb66c7526e47cf6d064c22029c2f48c04`, including its process budget.
The historical-field correction preserves omission and marks unknown coverage;
new fields are never invented as absent in old Evidence.
## Owner and compatibility
The Android manifest parser owns declaration interpretation. A shared domain
contract replaces duplicated provider/domain declarations; a single namespace
reader and one nested-declaration pass produce structured filters and existing
flattened action/category fields. Source positions address the exact retained
decoded text. Unsupported data constraints and filter constructs mark partial
coverage, including scheme-specific-part attributes outside the projected fields.
The producer emits all current fields; historical readers accept omitted fields.
Existing complete requests and the existing default summary remain available.
An explicit components view selects exact names/kinds/actions/exported declarations,
fields and pages. Component summaries count explicit/default exports separately.
CLI and MCP share the retained projection and retain complete parent Evidence.
Repeated views reuse validated sealed records under the existing session lifetime;
no new persistent provider cache or live-state freshness claim is introduced.
Rows always retain the parent array ordinal and declaration coverage. The APK
subject, engine identity, source status, limitations and parent linkage remain
available. Nothing here establishes runtime enabling, resource resolution, Android
URI matching, signature authenticity or execution of an app.
## Reproduction inputs
The old runtime is main `15a8f6847ccac581c7ff717b42062a54b1ba8084`.
The timing/token qualification below was captured on that pinned base before the
feature was rebased onto main `6207843814e5267e3a1d9796d5800346f69dc7da`.
The Android producer, bridge and view sources had no upstream changes in that
range; latest-main integration is checked separately rather than relabeling the
older runtime receipts.
Actual verification uses Linux x86-64, Node 24.22.1, Temurin 21.0.12.1,
JADX headless MCP 0.7.1 and the existing REA Java bridge protocol 7.
JAR SHA-256: `6e5eacf500b64292bfb73c49797c1958f6ee44646e43e868039ae7feb573ff75`.
The public [Appium ApiDemos 6.0.18 APK](https://github.com/appium/android-apidemos/releases/download/v6.0.18/ApiDemos-debug.apk)
is 6,521,217 bytes, SHA-256
`a9eecf37b26cd084855c530db81c2bb1b91f4c1b095a04f47aa7c20e2791f686`.
It has 355 declared components and 152,541 decoded manifest UTF-16 units.
The recorded sorted relative-path/SHA-256 tables cover compiled `dist/*.js` and
JSON recursively, the dispatcher/environment helper and Android bridge. Table
SHA-256: old runtime `9581afe8ba48c15d803dd7ed3ffc4d10e4e4b99857a0b90206c8012e266daffe`
(2,435 files), final candidate
`a78698c12ae1dc514ffd3f09ca1b881747c9dde7d1a8677ceed7de3233334c40`
(2,439 files). Raw tables remain with the private receipts.
`android-components.mjs` exercises actual public stdio MCP tools and validates
results against the selected named SDK tool definition. Supply `--repository`,
`--target`, a fresh `--output` directory, `--mode before|after`, `--java-home` and
`--jar`. It captures the selected catalog, every RPC, complete exported parents,
first-capture and warm-session results, and checks APK identity and an independent
stdlib ElementTree oracle. No production manifest parser is reused by that oracle.
Keep raw results, APKs and resource receipts outside Git. The selected tool families
are android/application/session, with compact input schemas, two allowed CPUs,
one analysis worker, 1 GiB JVM heap and a 3 GiB Node heap. A separate supervisor
samples process-family memory every 100 ms, with an 8 GiB early stop and a 4 GiB
host-available-memory floor. No guard stop or OOM occurred in these measurements.
## Equivalent completed question
Find the receiver declaring `android.provider.Telephony.SMS_RECEIVED` and report
its identity, permission, enabled/process declarations, per-filter priority and
source position. The old runtime needs complete manifest delivery for the new
facets. The selected candidate returns the matching row directly. Both treatments
make one inspection plus one complete Evidence export per capture; the oracle
verifies one positive match and the total count of 355 source declarations. An additional capture
uses the same server/engine session. First capture is a fresh process/session,
not a flushed OS page cache. Export costs and the extra call are included.
Initial candidate qualification passed the independent oracle. It measured
first/warm inspection RPCs of 11.395/0.329 seconds versus the old runtime's
10.534/0.314 seconds. The complete two-capture process family took 19.065 seconds,
27.687 CPU seconds and 1,193,615,360 peak sampled RSS bytes versus
16.534 seconds, 24.675 CPU seconds and 1,107,955,712 bytes. This run predates the
additional unsupported-data-attribute coverage correction. It establishes no
latency, CPU or memory improvement; retained data has more declaration facts.
The final coverage-corrected candidate and a repeated immutable-main control
also passed the identical oracle. Preserve all four receipts rather than choosing
a favorable timing:
| Runtime, execution order | First/warm inspection RPC seconds | Whole two-capture seconds | CPU seconds | Peak sampled family RSS bytes |
| ------------------------ | --------------------------------: | ------------------------: | ----------: | ----------------------------: |
| Main control 1 | 10.534 / 0.314 | 16.534 | 24.675 | 1,107,955,712 |
| Initial candidate | 11.395 / 0.329 | 19.065 | 27.687 | 1,193,615,360 |
| Final candidate | 14.558 / 0.571 | 23.369 | 36.031 | 1,145,073,664 |
| Main control 2 | 26.348 / 0.511 | 36.537 | 36.381 | 1,168,494,592 |
The repeated main control shows substantial host/run variation. These samples
establish neither a deterministic slowdown nor a speedup; warm candidate RPCs
are slightly slower in both observations. RSS is the sum of family RSS sampled
every 100 ms, so shared pages may count more than once and short peaks may be
missed; it is not a kernel/cgroup peak. Tokenizer processing is outside the timed
SDK runs. Local oracle work, hashing, discovery, serialization, proof export and
cleanup are inside. Cold/warm CPU and memory are not separately measured.
For the first main control and final candidate, `o200k_base` estimates give:
| Cost per first capture | Old complete | Selected components |
| ------------------------------------------------------------ | -----------: | ------------------: |
| Inspection reply bytes | 978,381 | 11,307 |
| Inspection reply estimated tokens | 225,926 | 2,715 |
| Selected family name/description/input-schema catalog tokens | 14,855 | 15,346 |
| Question + input catalog + both call arguments/text replies | 118,430 | 16,893 |
| Full SDK catalog, including output schemas, + both calls | 516,533 | 299,566 |
The primary text projection counts a fixed question, the input catalog once, all
call arguments and every text reply, including export receipts. The SDK transport
row additionally counts output schemas and structuredContent; it is not equated
with model context. Reproduce with `android-component-costs.py --output costs.json
BEFORE_DIRECTORY AFTER_DIRECTORY` (requires tiktoken). The manifest file exported locally is retained
proof, not inline model context. These are tokenizer estimates of scripted SDK
workflows, not billed model usage or a measured autonomous-agent task. They do
not establish fewer model rounds or a generic analysis speedup. Actual provider
work still reads the complete manifest, and schema discovery grows.
## Global catalog compatibility
The actual CI merge `10746a4e8f8c622357cf8dd529f98adc8bf06d2b` combines the feature
with main parent `86d4a21215cb0fbafaf555b5c7d9fcda3f255d82`. Real SDK discovery,
using the same complete contract registrations as the existing profile test, gives:
| Explicit wire profiles | Pure main bytes | CI merge bytes |
| ------------------------------ | --------------: | -------------: |
| Full input / full output | 4,398,561 | 4,425,163 |
| Compact input / compact output | 556,450 | 558,653 |
| Compact input / output off | 302,913 | 305,116 |
| Full input / output off | 1,047,039 | 1,052,239 |
Input presentation defaults to compact; output presentation defaults to full.
For frame-capped hosts, compact/off outputs are explicit compatibility choices.
The bounded compatibility combinations retain the existing one-megabyte test
limit and per-schema budgets. New selectors add 5,200 bytes with full inputs:
2,699 input bytes and 379 description bytes on `inspect_android_package`, plus
2,122 input bytes on the retained-view operation. Compact-input discovery grows
2,203 bytes. These global discovery costs are distinct from the selected-family
task token estimates above.
This change crosses the old test assertion for full input with output off.
That assertion was broader than its default-compact test name and the existing
profile documentation: full inputs deliberately preserve complete guidance and
can dominate the frame even without output schemas. The corrected test retains
both compact-input frame limits, compares every full input schema with its named
canonical advertisement under full/off outputs, preserves the entire tool set,
and verifies that off removes only output advertisements. No production limit is
raised and no full input validation/guidance is discarded. The owning guide now
states this full-profile boundary explicitly.
## Verification boundaries and failed attempts
The real Java regression exercises the existing raw/default-package class selector,
native absence, references and frozen metadata around neighbouring source generation.
An initial assertion compared the entire usage object: JADX legitimately populated
`def_pos` from 0 to 103 after generating source. The revised assertion excludes only
that presentation position and keeps class/method identity, raw DEX invoke IDs,
descriptor/overload facts and invoke offsets. Production bridge behavior and version
are unchanged.
One combined focused run passed 121 checks but hit the existing 30-second MCP test
timeout while advertising every unrelated tool family to the SDK. Its failed receipt
is retained. The Android-specific test now selects the same relevant families as
the real SDK workflow; its named schemas, retained manifest/source-removal behavior,
full/summary compatibility and complete Evidence export assertions remain enabled.
A subsequent run exposed a test assertion that incorrectly expected repeated
acquisitions to reuse the first Evidence ID. Each selected capture now proves its
own retained parent exists and has the same artifact and normalized facts; engine
reuse does not collapse distinct acquisition identities.
ApiDemos contains no URI data declarations. The checked-in manifest/resources and
Smali container permit a second source-built APK with real AAPT2 binary XML and
JADX decoding, including two filters, URI data, metadata, a resource boolean,
explicit process and provider URI grants. Compiled APK/DEX outputs are private.
Static declaration verification does not execute these component classes.
The source-built APK SHA-256 was
`aea6ac64630ad70e93f4d85de0237cc51a7012b077ca0d0e7b24cf22c5737532`.
Its real CLI selected view and retained bundle passed `verify-fixture.py`, including
AAPT2 dump comparison, all source positions, grouping, raw resource references,
metadata names and provider URI grants. The real SDK VIEW query passed the
independent ElementTree oracle on first and warm capture. An initial fixture build
used Smali's CLI and failed because the engine JAR does not bundle optional
JCommander; `BuildDex.java` now calls the already bundled Smali API directly.
The real metadata helper passed all Java markers and downstream CLI/MCP method
body and direct-invoke parity checks for both classes.dex and classes2.dex:
149.436 seconds, 204.026 CPU seconds and 1,070,514,176 sampled peak family RSS bytes.
No production bridge change was needed for default-package selectors.
A genuine complete parent from the old runtime also passed a new CLI retained
component query with an explicitly nonexistent engine JAR. Its newly introduced
fields remained omitted and row coverage remained unknown; no APK/provider
reopening was required. The current source checks cover 111 parser cases,
six projection cases, four CLI early-validation cases and four MCP cases,
including named SDK-converted schemas and retained source-removal behavior.
After the rebase onto `6207843814e5267e3a1d9796d5800346f69dc7da`, type/lint,
all 125 focused cases and the nine documentation conformance cases passed again.
A fresh real ApiDemos SDK qualification also passed the independent oracle:
first/warm RPCs 11.291/0.290 seconds, whole two-capture elapsed 17.008 seconds,
CPU 26.425 seconds and peak sampled family RSS 1,166,245,888 bytes. This is latest-main
integration evidence; it is not paired with a new baseline performance cohort.
Review follow-up: Greptile correctly identified that the original oracle filtered
before comparison, so the timing cohorts above verified selected-receiver facts
and the inventory count, not every unselected component's facts. The oracle now
returns an unfiltered inventory and derives the selected list from it. The driver
checks every retained parent's identity, enabled/raw-enabled/process/permission,
filter grouping/data/priority, metadata names, declaration coverage and component
source location before checking the selected page. The receipt records both the
verified count and scope; historical mode checks the available identity/permission
facets without inventing the new observations. A fresh real ApiDemos first/warm
run passed all 355 rows in both captures (16.538 seconds, 24.977 CPU seconds,
1,176,817,664 sampled peak family RSS bytes). The older receipts remain unchanged. The unfiltered historical-mode run also
passed all 355 available identity/permission rows in both captures; the compiled
manifest fixture passed all three current rows plus its selected VIEW page.
Platform coverage is Linux/JDK21/JADX0.7.1 only. Android devices, emulators, actual
runtime enabling/registration, other JDKs and Windows/macOS are unverified here.
experiments/analysis-efficiency/android-components.mjs
+294
View File
@@ -0,0 +1,294 @@
/** Real APK task through public MCP, with complete retained Evidence and an independent XML oracle. */
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { createReadStream } from "node:fs";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { createRequire } from "node:module";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { parseArgs } from "node:util";
import { performance } from "node:perf_hooks";
import { createVerifierCommands } from "../../scripts/lib/verify-commands.mjs";
const { values } = parseArgs({
options: {
repository: { type: "string" },
target: { type: "string" },
output: { type: "string" },
mode: { type: "string" },
"java-home": { type: "string" },
jar: { type: "string" },
action: {
type: "string",
default: "android.provider.Telephony.SMS_RECEIVED",
},
},
});
for (const name of [
"repository",
"target",
"output",
"mode",
"java-home",
"jar",
])
assert.ok(values[name], `Missing --${name}`);
assert.ok(["before", "after"].includes(values.mode));
const repo = resolve(values.repository),
target = resolve(values.target),
output = resolve(values.output);
await mkdir(output, { mode: 0o700 });
const digest = createHash("sha256");
for await (const chunk of createReadStream(target)) digest.update(chunk);
const sha256 = digest.digest("hex");
const require = createRequire(join(repo, "package.json"));
const { Client } = await import(
pathToFileURL(require.resolve("@modelcontextprotocol/client"))
);
const { StdioClientTransport } = await import(
pathToFileURL(require.resolve("@modelcontextprotocol/client/stdio"))
);
const commands = await createVerifierCommands();
const env = {
PATH: process.env.PATH,
JAVA_HOME: resolve(values["java-home"]),
REA_JADX_MCP_JAR: resolve(values.jar),
REA_JADX_HEAP_MIB: "1024",
REA_JADX_ACTIVE_PROCESSOR_COUNT: "2",
NODE_OPTIONS: "--max-old-space-size=3072",
UV_THREADPOOL_SIZE: "2",
REA_LOG_LEVEL: "silent",
REA_MCP_TOOL_FAMILIES: "android,application,session",
REA_MCP_INPUT_SCHEMA_PROFILE: "compact",
};
const transport = new StdioClientTransport({
command: process.execPath,
args: [join(repo, "scripts/rea.mjs"), "mcp"],
cwd: repo,
env,
stderr: "pipe",
maxBufferSize: 16 * 1024 * 1024,
});
const diagnostics = [];
let diagnosticBytes = 0;
let omittedDiagnosticBytes = 0;
transport.stderr?.on("data", (chunk) => {
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
const retained = bytes.subarray(0, Math.max(0, 65536 - diagnosticBytes));
if (retained.length > 0) diagnostics.push(retained);
diagnosticBytes += retained.length;
omittedDiagnosticBytes += bytes.length - retained.length;
});
const client = new Client({
name: "rea-real-android-components",
version: "1",
});
let catalog;
const calls = [],
runs = [];
const invoke = async (name, arguments_) => {
const definitions = catalog.tools.filter((tool) => tool.name === name);
assert.equal(definitions.length, 1);
const start = performance.now();
const result = await client.callTool(
{ name, arguments: arguments_ },
{ toolDefinition: definitions[0], timeout: 120000 },
);
calls.push({
name,
arguments: arguments_,
result,
elapsed_ms: performance.now() - start,
});
assert.notEqual(
result.isError,
true,
`${name}: ${JSON.stringify(result.content)}`,
);
return result.structuredContent;
};
const selectedFields = [
"kind",
"declared_name",
"name",
"exported",
"exported_default",
"permission",
"permission_source",
"enabled",
"enabled_declaration",
"process",
"intent_filters",
"meta_data",
"location",
];
const stripLocations = (row) => {
const { location: _location, ...fields } = row;
return {
...fields,
intent_filters: row.intent_filters?.map(
({
location: _filterLocation,
source_index: _filterIndex,
coverage: _coverage,
...filter
}) => ({
...filter,
data: filter.data.map(({ location: _dataLocation, ...data }) => data),
}),
),
meta_data: row.meta_data?.map(
({ location: _metadataLocation, ...metadata }) => metadata,
),
};
};
const assertOracleFacts = (row, expected, source) => {
assert.equal(row.declaration_coverage, "complete");
const cleaned = stripLocations(row);
for (const [key, value] of Object.entries(expected))
assert.deepEqual(
cleaned[key],
value,
`Independent oracle: ${expected.source_index}.${key}`,
);
assert.equal(
source.slice(
row.location.offset,
row.location.offset + row.kind.length + 1,
),
"<" + row.kind,
);
};
try {
await client.connect(transport);
catalog = await client.listTools();
await writeFile(join(output, "catalog.json"), JSON.stringify(catalog));
for (const temperature of ["first-capture", "warm-session"]) {
const firstCall = calls.length;
const start = performance.now();
const result = await invoke(
"inspect_android_package",
values.mode === "before"
? { path: target }
: {
path: target,
detail: "summary",
view: {
kind: "android",
facet: "components",
actions: [values.action],
fields: selectedFields,
limit: 64,
},
},
);
const rpcMs = performance.now() - start;
const normalized = result.normalized_result;
const parentId =
values.mode === "before"
? result.evidence_id
: normalized.parent_evidence_id;
const path = join(output, temperature + "-evidence.json");
await invoke("export_evidence_bundle", { path });
const bundle = JSON.parse(await readFile(path, "utf8"));
const parent = bundle.records.find(
(record) => record.evidence_id === parentId,
);
assert.ok(parent);
assert.equal(parent.subject.digest.sha256, sha256);
const oraclePath = join(
dirname(fileURLToPath(import.meta.url)),
"fixtures/android-components/manifest-oracle.py",
);
const oracle = JSON.parse(
(
await commands.exec(
"python3",
[oraclePath, path, parentId, values.action],
{ timeout: 15000, maxBuffer: 4 * 1024 * 1024 },
)
).stdout,
);
assert.ok(
oracle.components.length > 0,
"The completed task must exercise a real positive action match",
);
const complete = parent.normalized_result;
assert.equal(complete.components.items.length, oracle.inventory.length);
assert.equal(oracle.inventory.length, oracle.source_components);
for (const [index, expected] of oracle.inventory.entries()) {
const row = complete.components.items[index];
if (values.mode === "after")
assertOracleFacts(
{ ...row, source_index: index },
expected,
complete.manifest.text,
);
else {
for (const key of ["kind", "declared_name", "name", "permission"])
assert.deepEqual(
row[key],
expected[key],
`Legacy inventory: ${index}.${key}`,
);
}
}
if (values.mode === "after") {
assert.equal(normalized.facet, "components");
assert.equal(normalized.source_status, "complete");
assert.equal(normalized.source_components, oracle.source_components);
assert.equal(normalized.coverage.exhausted, true);
assert.equal(normalized.coverage.total, oracle.components.length);
assert.equal(normalized.items.length, oracle.components.length);
for (let i = 0; i < oracle.components.length; i++) {
const row = normalized.items[i],
expected = oracle.components[i];
assertOracleFacts(row, expected, complete.manifest.text);
}
} else {
for (const expected of oracle.components) {
const row = normalized.components.items[expected.source_index];
assert.equal(row.name, expected.name);
assert.ok(row.actions.includes(values.action));
}
}
runs.push({
temperature,
rpc_ms: rpcMs,
calls: calls.length - firstCall,
matched_components: oracle.components.length,
source_components: oracle.source_components,
parent_evidence_id: parentId,
answer_verified: true,
inventory_verified: oracle.inventory.length,
inventory_scope:
values.mode === "after"
? "declared-structured-facets"
: "legacy-identity-permission",
});
}
} finally {
// exec drains already adopted owners; adopt the live MCP client only at its final cleanup boundary.
commands.adopt(client, "real Android component MCP client");
await commands.close();
await writeFile(join(output, "stderr.log"), Buffer.concat(diagnostics));
await writeFile(join(output, "calls.json"), JSON.stringify(calls), {
mode: 0o600,
});
await writeFile(
join(output, "report.json"),
JSON.stringify(
{
mode: values.mode,
target,
sha256,
runs,
tool_count: catalog?.tools.length,
omitted_diagnostic_bytes: omittedDiagnosticBytes,
},
null,
2,
),
);
}
console.log(JSON.stringify({ mode: values.mode, runs, verified: true }));
experiments/analysis-efficiency/browser-saved-values.mjs
+713
View File
@@ -0,0 +1,713 @@
/** Compare complete real PocketBase create/update/read tasks through public MCP. */
import assert from "node:assert/strict";
import { readFile, writeFile, mkdir, mkdtemp, rm } from "node:fs/promises";
import { join, resolve } from "node:path";
import { tmpdir } from "node:os";
import { createRequire } from "node:module";
import { pathToFileURL } from "node:url";
import { parseArgs } from "node:util";
import { performance } from "node:perf_hooks";
import { createVerifierCommands } from "../../scripts/lib/verify-commands.mjs";
import { withVerifierCleanup } from "../../scripts/lib/verifier-cleanup.mjs";
const { values } = parseArgs({
options: {
repository: { type: "string" },
mode: { type: "string" },
output: { type: "string" },
origin: { type: "string" },
credentials: { type: "string" },
browser: { type: "string" },
"source-only": { type: "boolean", default: false },
},
});
assert.ok(
values.repository &&
values.output &&
values.origin &&
values.credentials &&
values.browser,
);
assert.ok(["before", "after"].includes(values.mode));
const repo = resolve(values.repository),
output = resolve(values.output),
origin = new URL(values.origin).origin;
assert.equal(
new URL(origin).hostname,
"127.0.0.1",
"Use an owned loopback fixture",
);
await mkdir(output, { mode: 0o700 });
const credentials = JSON.parse(await readFile(values.credentials, "utf8"));
const require = createRequire(join(repo, "package.json"));
const { chromium } = require("playwright-core");
const { Client } = await import(
pathToFileURL(require.resolve("@modelcontextprotocol/client"))
);
const { StdioClientTransport } = await import(
pathToFileURL(require.resolve("@modelcontextprotocol/client/stdio"))
);
const commands = await createVerifierCommands();
const profile = await mkdtemp(join(tmpdir(), "rea-saved-values-"));
const env = {
PATH: process.env.PATH,
NODE_OPTIONS: "--max-old-space-size=3072",
UV_THREADPOOL_SIZE: "2",
REA_LOG_LEVEL: "silent",
REA_MCP_TOOL_FAMILIES: "browser,application,session",
REA_MCP_INPUT_SCHEMA_PROFILE: "compact",
REA_TEST_EMAIL: credentials.email,
REA_TEST_PASSWORD: credentials.password,
REA_TEST_WHOLE_JSON: JSON.stringify({ token: "private-token-729d" }),
};
const calls = [];
const runs = [];
let catalog;
const transport = new StdioClientTransport({
command: process.execPath,
args: [join(repo, "scripts/rea.mjs"), "mcp"],
cwd: repo,
env,
stderr: "pipe",
});
transport.stderr?.resume();
const client = new Client({ name: "rea-saved-values-real-app", version: "1" });
commands.adopt(client, "saved-values MCP client");
const tokenHeader = (saved) => ({
name: "Authorization",
value: saved
? { source: "saved", name: "token" }
: { source: "local_storage", key: "rea_auth", json_pointer: "/token" },
});
const decodeBodies = (value) => {
const bodies = [];
const visit = (x) => {
if (x && typeof x === "object") {
if (x.encoding === "base64" && typeof x.content === "string") {
try {
bodies.push(
JSON.parse(Buffer.from(x.content, "base64").toString("utf8")),
);
} catch {}
}
for (const v of Object.values(x)) visit(v);
}
};
visit(value);
return bodies;
};
async function invoke(name, args) {
const defs = catalog.tools.filter((t) => t.name === name);
assert.equal(defs.length, 1);
const start = performance.now();
const result = await client.callTool(
{ name, arguments: args },
{ toolDefinition: defs[0], timeout: 90000 },
);
calls.push({
name,
arguments: args,
result,
elapsed_ms: performance.now() - start,
});
assert.notEqual(
result.isError,
true,
`${name} failed; retain the local receipt`,
);
return result.structuredContent;
}
const loginActions = (saved) => [
{
step_id: "email",
action: "fill",
locator: { kind: "css", selector: "#email" },
value: { source: "secret", secret_id: "email" },
},
{
step_id: "password",
action: "fill",
locator: { kind: "css", selector: "#password" },
value: { source: "secret", secret_id: "password" },
},
{
step_id: "login",
action: "click",
locator: { kind: "css", selector: "#login" },
...(saved
? {
save_response: {
method: "POST",
path: "/api/collections/_superusers/auth-with-password",
save: [{ name: "token", json_pointer: "/token", secret: true }],
},
}
: {}),
timeout_ms: 10000,
},
{
step_id: "ready",
action: "wait_for",
locator: { kind: "css", selector: '#result[data-ready="yes"]' },
state: "visible",
timeout_ms: 10000,
},
];
const edits = (id, saved) => [
{
step_id: "update",
action: "request",
method: "PATCH",
path: `/api/collections/rea_items/records/${id}`,
headers: [tokenHeader(saved)],
json: { title: `record-${id}`, quantity: saved ? "{saved:quantity}" : 11 },
expected_status: [200],
},
{
step_id: "read",
action: "request",
method: "GET",
path: `/api/collections/rea_items/records/${id}?at=${id}`,
headers: [tokenHeader(saved)],
expected_status: [200],
},
{
step_id: "open",
action: "goto",
destination: { url: `${origin}/api/collections/rea_items/records/${id}` },
wait_until: "load",
},
];
const captureOptions = {
after_each_step: ["url"],
at_end: ["url", "accessibility"],
events: ["network"],
network: { request_body: true, response_body: true, header_values: true },
};
try {
const context = await chromium.launchPersistentContext(profile, {
executablePath: resolve(values.browser),
headless: true,
args: ["--remote-debugging-port=0", "--no-sandbox"],
});
commands.adopt(context, "owned saved-values Chromium context");
const page = context.pages()[0];
await page.goto(origin, { waitUntil: "networkidle" });
const port = (
await readFile(join(profile, "DevToolsActivePort"), "utf8")
).split("\n")[0];
const endpoint = `http://127.0.0.1:${port}`;
const targets = await (await fetch(endpoint + "/json/list")).json();
const target = targets.find(
(t) => t.type === "page" && t.url === origin + "/",
);
assert.ok(target);
const browser = {
mode: "connect",
cdp_endpoint: endpoint,
target_id: target.id,
};
const declared = [
{ secret_id: "email", environment_variable: "REA_TEST_EMAIL" },
{ secret_id: "password", environment_variable: "REA_TEST_PASSWORD" },
];
const scenario = (actions, secrets = declared) => ({
browser,
start_url: { url: origin + "/" },
secrets,
actions,
capture: captureOptions,
detail: "summary",
});
await client.connect(transport);
catalog = await client.listTools();
await writeFile(join(output, "catalog.json"), JSON.stringify(catalog));
for (const temperature of values["source-only"] ? [] : ["cold", "warm"]) {
const saved = values.mode === "after";
const start = performance.now(),
firstCall = calls.length;
const create = {
step_id: "create",
action: "request",
method: "POST",
path: "/api/collections/rea_items/records",
headers: [tokenHeader(saved)],
json: { title: `created-${temperature}`, quantity: 11 },
expected_status: [200, 201],
...(saved
? {
save: [
{ name: "record_id", json_pointer: "/id" },
{ name: "quantity", json_pointer: "/quantity" },
],
}
: {}),
};
const firstActions = [...loginActions(saved), create];
if (saved) firstActions.push(...edits("{saved:record_id}", true));
const first = await invoke(
"capture_browser_scenario",
scenario(firstActions),
);
const firstSummary = first.normalized_result;
assert.ok(
firstSummary.summary.steps.every((step) => step.status === "completed"),
"Initial capture must complete",
);
let final = first;
let id;
let legacyPreReadObserved;
if (!saved) {
const selected = await invoke("inspect_analysis_view", {
source: {
kind: "retained-evidence",
evidence_id: firstSummary.parent_evidence_id,
},
view: {
kind: "browser-scenario",
collection: "events",
step_indices: [firstActions.length],
event_kinds: ["network-content"],
limit: 16,
},
});
assert.equal(selected.normalized_result.coverage.exhausted, true);
const records = decodeBodies(selected.normalized_result).filter(
(b) => b?.collectionName === "rea_items" && typeof b.id === "string",
);
assert.equal(records.length, 1);
id = records[0].id;
final = await invoke(
"capture_browser_scenario",
scenario(edits(id, false), []),
);
}
assert.ok(
final.normalized_result.summary.steps.every(
(step) => step.status === "completed",
),
"Completed task must not hide failed actions",
);
const completedMs = performance.now() - start,
taskCalls = calls.length - firstCall;
const path = join(output, `${temperature}-evidence.json`);
await invoke("export_evidence_bundle", { path });
const bundle = JSON.parse(await readFile(path, "utf8"));
const parent = bundle.records.find(
(r) => r.evidence_id === final.normalized_result.parent_evidence_id,
);
assert.ok(parent);
if (!saved) {
const legacy = bundle.records.find(
(record) => record.evidence_id === firstSummary.parent_evidence_id,
);
assert.ok(legacy);
const login = decodeBodies(legacy.normalized_result).filter(
(body) => typeof body?.token === "string",
);
assert.equal(login.length, 1);
const preReadLogin = decodeBodies({
events: legacy.normalized_result.events.items.filter(
(event) => event.step_index < firstActions.length,
),
}).filter((body) => typeof body?.token === "string");
legacyPreReadObserved = preReadLogin.length === 1;
if (legacyPreReadObserved) {
assert.ok(
!preReadLogin[0].token.startsWith("[REDACTED:"),
"An observed pre-read login response must preserve its unmarked token",
);
const later = legacy.normalized_result.events.items.filter(
(event) => event.step_index >= firstActions.length,
);
assert.ok(
!JSON.stringify({
events: later,
bodies: decodeBodies({ events: later }),
}).includes(preReadLogin[0].token),
"Post-read facts must use the current page-value policy",
);
}
}
const recordBodies = decodeBodies(parent.normalized_result).filter(
(b) => b?.collectionName === "rea_items" && typeof b.id === "string",
);
assert.ok(recordBodies.length >= 2);
if (saved) id = recordBodies[0].id;
assert.ok(recordBodies.every((b) => b.id === id));
assert.ok(
recordBodies.some((b) => b.title === `record-${id}` && b.quantity === 11),
);
const read = await fetch(
`${origin}/api/collections/rea_items/records/${id}`,
);
assert.equal(read.status, 200);
const actual = await read.json();
assert.equal(actual.title, `record-${id}`);
assert.equal(actual.quantity, 11);
const requestBodies = decodeBodies(parent.normalized_result).filter(
(b) => b?.title === `record-${id}` && !Object.hasOwn(b, "collectionName"),
);
assert.ok(requestBodies.length > 0);
assert.ok(requestBodies.every((b) => typeof b.quantity === "number"));
if (saved) {
const loginBodies = decodeBodies(parent.normalized_result).filter(
(b) => typeof b?.token === "string",
);
assert.equal(loginBodies.length, 1);
assert.ok(loginBodies[0].token.startsWith("[REDACTED:"));
}
runs.push({
temperature,
legacy_pre_read_observed: legacyPreReadObserved,
elapsed_ms: completedMs,
task_calls: taskCalls,
evidence_id: parent.evidence_id,
record_id: id,
quantity_type: typeof actual.quantity,
});
}
if (values["source-only"]) {
assert.equal(values.mode, "after");
const sourceScenario = scenario(
[
{
step_id: "source",
action: "request",
method: "GET",
path: "/escaped-values.json?credential=source-token",
save: [
{ name: "token", json_pointer: "/token", secret: true },
{ name: "code", json_pointer: "/code", secret: true },
],
},
{
step_id: "later",
action: "request",
method: "GET",
path: "/escaped-values.json",
},
],
[],
);
const captured = await invoke("capture_browser_scenario", sourceScenario);
assert.ok(
captured.normalized_result.summary.steps.every(
(step) => step.status === "completed",
),
);
const path = join(output, "source-evidence.json");
await invoke("export_evidence_bundle", { path });
const bundle = JSON.parse(await readFile(path, "utf8"));
const parent = bundle.records.find(
(record) =>
record.evidence_id === captured.normalized_result.parent_evidence_id,
);
assert.ok(parent);
const sourceRequests = parent.normalized_result.events.items.filter(
(event) =>
event.kind === "request" &&
event.step_index === 1 &&
event.url.url.includes("/escaped-values.json"),
);
assert.equal(sourceRequests.length, 1);
const transaction = sourceRequests[0].transaction_id;
const sourceEvents = parent.normalized_result.events.items.filter(
(event) => event.transaction_id === transaction,
);
assert.ok(sourceEvents.some((event) => event.kind === "response"));
assert.ok(
!JSON.stringify(sourceEvents).includes("source-token"),
"Source transaction metadata must not retain the value learned from its response",
);
const bodies = decodeBodies(parent.normalized_result).filter(
(body) =>
Object.hasOwn(body ?? {}, "token") && Object.hasOwn(body, "code"),
);
assert.equal(
bodies.length,
2,
"Both actual source and later response must be observed",
);
for (const body of bodies) {
assert.equal(body.token, "[REDACTED:page:source.token]");
assert.equal(body.code, "[REDACTED:page:source.code]");
}
const expected = Buffer.from(
'{"token":"[REDACTED:page:source.token]","code":"[REDACTED:page:source.code]","unknown":1e-400}',
).toString("base64");
assert.equal(
JSON.stringify(parent.normalized_result).split(JSON.stringify(expected))
.length - 1,
2,
"Both source bodies must preserve the unmarked numeric spelling exactly",
);
const metadata = await invoke("capture_browser_scenario", {
...sourceScenario,
capture: {
...sourceScenario.capture,
network: {
request_body: false,
response_body: false,
header_values: false,
},
},
});
assert.ok(
metadata.normalized_result.summary.steps.every(
(step) => step.status === "completed",
),
);
const metadataPath = join(output, "metadata-evidence.json");
await invoke("export_evidence_bundle", { path: metadataPath });
const metadataBundle = JSON.parse(await readFile(metadataPath, "utf8"));
const metadataParent = metadataBundle.records.find(
(record) =>
record.evidence_id === metadata.normalized_result.parent_evidence_id,
);
assert.ok(metadataParent);
const metadataRequest = metadataParent.normalized_result.events.items.find(
(event) =>
event.kind === "request" &&
event.step_index === 1 &&
event.url.url.includes("/escaped-values.json"),
);
assert.ok(metadataRequest);
assert.equal(metadataRequest.url.origin, origin);
assert.ok(metadataRequest.url.url.includes("/escaped-values.json"));
assert.ok(!metadataRequest.url.url.includes("source-token"));
assert.ok(
!metadataParent.normalized_result.events.items.some(
(event) => event.kind === "network-content",
),
);
const countUrl = `${origin}/api/collections/rea_items/records?perPage=1`;
const beforeCount = (await (await fetch(countUrl)).json()).totalItems;
assert.ok(Number.isSafeInteger(beforeCount));
const whole = await invoke("capture_browser_scenario", {
...sourceScenario,
start_url: { url: origin + "/whole-json.html" },
secrets: [
{
secret_id: "whole_json",
environment_variable: "REA_TEST_WHOLE_JSON",
},
],
storage: {
local_storage: [
{
origin,
entries: [
{
name: "whole_json",
value: { source: "secret", secret_id: "whole_json" },
},
],
},
],
},
actions: [
{
step_id: "settled",
action: "wait_for",
locator: { kind: "css", selector: '#result[data-ready="yes"]' },
state: "visible",
timeout_ms: 10000,
},
],
});
assert.ok(
whole.normalized_result.summary.steps.every(
(step) => step.status === "completed",
),
);
const wholePath = join(output, "whole-json-evidence.json");
await invoke("export_evidence_bundle", { path: wholePath });
const wholeBundle = JSON.parse(await readFile(wholePath, "utf8"));
const wholeParent = wholeBundle.records.find(
(record) =>
record.evidence_id === whole.normalized_result.parent_evidence_id,
);
assert.ok(wholeParent);
const post = wholeParent.normalized_result.events.items.find(
(event) => event.kind === "request" && event.method === "POST",
);
assert.ok(post);
const postEvents = wholeParent.normalized_result.events.items.filter(
(event) => event.transaction_id === post.transaction_id,
);
assert.ok(
postEvents.some(
(event) =>
event.kind === "network-content" &&
event.phase === "request" &&
event.body.state === "unavailable",
),
);
const postResponse = postEvents.find((event) => event.kind === "response");
assert.ok(postResponse && [400, 401, 403].includes(postResponse.status));
assert.ok(!JSON.stringify(wholeParent).includes("private-token-729d"));
assert.equal(
(await (await fetch(countUrl)).json()).totalItems,
beforeCount,
);
const followup = await invoke("capture_browser_scenario", {
...sourceScenario,
start_url: { url: origin + "/immediate-followup.html" },
capture: {
...sourceScenario.capture,
events: [...sourceScenario.capture.events, "console"],
after_each_step: [
...sourceScenario.capture.after_each_step,
"screenshot",
],
at_end: [...sourceScenario.capture.at_end, "screenshot"],
},
actions: [
{
step_id: "exchange",
action: "click",
locator: { kind: "css", selector: "#run" },
save_response: {
method: "GET",
path: "/escaped-values.json",
save: [{ name: "token", json_pointer: "/token", secret: true }],
},
timeout_ms: 10000,
},
{
step_id: "settled",
action: "wait_for",
locator: { kind: "css", selector: '#result[data-ready="yes"]' },
state: "visible",
timeout_ms: 10000,
},
],
});
const followupPath = join(output, "followup-evidence.json");
await invoke("export_evidence_bundle", { path: followupPath });
const followupBundle = JSON.parse(await readFile(followupPath, "utf8"));
const followupParent = followupBundle.records.find(
(record) =>
record.evidence_id === followup.normalized_result.parent_evidence_id,
);
assert.ok(followupParent);
assert.equal(await page.locator("#sensitive").innerText(), "source-token");
assert.equal(followupParent.normalized_result.steps.length, 3);
assert.ok(
followupParent.normalized_result.steps.every(
(step) =>
step.artifacts.screenshot.state === "missing" &&
step.completeness.missing_sections.includes("screenshot"),
),
);
assert.ok(
followupParent.normalized_result.completeness.missing_sections.includes(
"screenshot",
),
);
const consoleEvents = followupParent.normalized_result.events.items.filter(
(event) => event.kind === "console" && event.text.includes("REDACTED"),
);
assert.equal(consoleEvents.length, 1);
assert.ok(
!JSON.stringify(followupParent.normalized_result.events.items).includes(
"source-token",
),
);
assert.ok(
followup.normalized_result.summary.steps.every(
(step) => step.status === "completed",
),
);
const followupPost = followupParent.normalized_result.events.items.find(
(event) => event.kind === "request" && event.method === "POST",
);
assert.ok(followupPost);
const followupEvents = followupParent.normalized_result.events.items.filter(
(event) => event.transaction_id === followupPost.transaction_id,
);
assert.ok(!JSON.stringify(followupEvents).includes("source-token"));
const followupBodies = decodeBodies({ events: followupEvents });
assert.ok(
followupBodies.some(
(body) => body?.token === "[REDACTED:page:exchange.token]",
),
);
assert.ok(
followupEvents.some(
(event) => event.kind === "response" && event.status === 403,
),
);
assert.equal(
(await (await fetch(countUrl)).json()).totalItems,
beforeCount,
);
const publicScreenshot = await invoke("capture_browser_scenario", {
...sourceScenario,
start_url: { url: origin + "/immediate-followup.html" },
actions: [
{ step_id: "settle", action: "wait_for_timeout", duration_ms: 1 },
],
capture: {
...sourceScenario.capture,
events: [],
network: {
request_body: false,
response_body: false,
header_values: false,
},
after_each_step: [],
at_end: ["screenshot"],
},
});
const publicPath = join(output, "public-screenshot-evidence.json");
await invoke("export_evidence_bundle", { path: publicPath });
const publicBundle = JSON.parse(await readFile(publicPath, "utf8"));
const publicParent = publicBundle.records.find(
(record) =>
record.evidence_id ===
publicScreenshot.normalized_result.parent_evidence_id,
);
assert.ok(publicParent);
assert.equal(await page.locator("#sensitive").innerText(), "");
const png =
publicParent.normalized_result.steps.at(-1).artifacts.screenshot;
assert.equal(png.state, "captured");
assert.equal(
Buffer.from(png.value.data_base64, "base64")
.subarray(0, 8)
.toString("hex"),
"89504e470d0a1a0a",
);
runs.push({
verification: "source-and-later-JSON-spellings",
verified_bodies: bodies.length,
metadata_only_source_url: true,
whole_json_secret_withheld: true,
immediate_followup_protected: true,
immediate_console_protected: true,
rendered_secret_screenshots_withheld: true,
nonsecret_screenshot_captured: true,
whole_json_status: postResponse.status,
});
}
await writeFile(join(output, "calls.json"), JSON.stringify(calls), {
mode: 0o600,
});
await writeFile(
join(output, "metrics.json"),
JSON.stringify({ mode: values.mode, runs }, null, 2),
);
console.log(JSON.stringify({ mode: values.mode, runs }));
} finally {
await withVerifierCleanup(
() => commands.close(),
[
async () => {
if (commands.cleanupResources.length === 0)
await rm(profile, { recursive: true, force: true });
},
],
);
}
experiments/analysis-efficiency/fixtures/android-components/AndroidManifest.xml
+22
View File
@@ -0,0 +1,22 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android" package="com.rea.components" android:versionCode="1" android:versionName="1">
<uses-sdk android:minSdkVersion="21" android:targetSdkVersion="30"/>
<application android:permission="com.rea.components.PRIVATE" android:enabled="true" android:process=":application">
<activity android:name=".Main" android:exported="true" android:enabled="false" android:process=":activity">
<intent-filter android:priority="7">
<action android:name="android.intent.action.VIEW"/>
<category android:name="android.intent.category.DEFAULT"/>
<data android:scheme="https" android:host="example.test" android:pathPrefix="/open"/>
</intent-filter>
<intent-filter android:priority="@integer/filter_priority">
<action android:name="android.intent.action.EDIT"/>
<data android:scheme="content" android:mimeType="text/plain"/>
</intent-filter>
<meta-data android:name="public.configuration" android:value="retained-in-manifest"/>
</activity>
<receiver android:name=".SmsReceiver" android:exported="false" android:enabled="@bool/receiver_enabled" android:process=":sms" android:permission="android.permission.BROADCAST_SMS">
<intent-filter android:priority="-10"><action android:name="android.provider.Telephony.SMS_RECEIVED"/></intent-filter>
</receiver>
<provider android:name=".DemoProvider" android:enabled="true" android:exported="false" android:authorities="com.rea.components.provider" android:grantUriPermissions="true"/>
</application>
</manifest>
experiments/analysis-efficiency/fixtures/android-components/BuildDex.java
+15
View File
@@ -0,0 +1,15 @@
import com.android.tools.smali.smali.Smali;
import com.android.tools.smali.smali.SmaliOptions;
import java.util.List;
/** Assemble the local fixture without Smali CLI's optional JCommander dependency. */
class BuildDex {
public static void main(String[] args) throws Exception {
SmaliOptions options = new SmaliOptions();
options.apiLevel = 30;
options.jobs = 1;
options.outputDexFile = args[1];
if (!Smali.assemble(options, List.of(args[0])))
throw new IllegalStateException("Manifest fixture assembly failed");
}
}
experiments/analysis-efficiency/fixtures/android-components/ManifestFixture.smali
+3
View File
@@ -0,0 +1,3 @@
# A DEX container for a static manifest fixture; no Android app is executed.
.class public Lcom/rea/components/ManifestFixture;
.super Ljava/lang/Object;
experiments/analysis-efficiency/fixtures/android-components/README.md
+43
View File
@@ -0,0 +1,43 @@
# Static compiled-manifest fixture
These source-owned declarations cover URI data/filter association, exact resource
references, metadata names, component process/enablement and provider URI grants.
`ManifestFixture.smali` supplies a DEX container; it does not implement or execute
the declared Android components. The verification uses existing AAPT2 36.1.0,
Android 36 `android.jar`, JDK 21 and the digest-pinned JADX headless MCP 0.7.1 JAR.
No Android device, emulator or app execution is involved.
From the repository root, choose existing `AAPT2`, `ANDROID_JAR`, `JAVA_HOME` and
`REA_JADX_MCP_JAR` paths, then build into a fresh private directory:
```sh
fixture=experiments/analysis-efficiency/fixtures/android-components
work=$(mktemp -d)
"$AAPT2" compile --dir "$fixture/res" -o "$work/resources.zip"
"$AAPT2" link --manifest "$fixture/AndroidManifest.xml" -I "$ANDROID_JAR" \
-o "$work/manifest.apk" "$work/resources.zip"
"$JAVA_HOME/bin/java" -Xmx512m -XX:ActiveProcessorCount=2 \
-cp "$REA_JADX_MCP_JAR" "$fixture/BuildDex.java" \
"$fixture/ManifestFixture.smali" "$work/classes.dex"
python3 - "$work" <<'PY'
from pathlib import Path
import sys, zipfile
root = Path(sys.argv[1])
with zipfile.ZipFile(root / "manifest.apk", "a") as apk:
apk.write(root / "classes.dex", "classes.dex")
PY
"$AAPT2" dump xmltree --file AndroidManifest.xml "$work/manifest.apk" > "$work/aapt2-xmltree.txt"
"$AAPT2" dump resources "$work/manifest.apk" > "$work/aapt2-resources.txt"
REA_JADX_HEAP_MIB=1024 REA_JADX_ACTIVE_PROCESSOR_COUNT=2 \
node scripts/rea.mjs inspect-android-package "$work/manifest.apk" \
--detail summary --view '{"kind":"android","facet":"components"}' \
--evidence_output "$work/evidence.json" --json > "$work/cli.json"
python3 "$fixture/verify-fixture.py" "$work"
```
`verify-fixture.py` cross-checks concrete compiled XML facts with the independent
AAPT2 dump, structured CLI facts, parent APK identity and every retained manifest
source location. It deliberately keeps resource declarations unresolved even
though the resource table contains values. `manifest-oracle.py` is a separate
stdlib ElementTree oracle used by the public MCP task driver. Generated binaries,
Evidence and dumps stay outside Git.
experiments/analysis-efficiency/fixtures/android-components/manifest-oracle.py
+70
View File
@@ -0,0 +1,70 @@
"""Independent stdlib XML oracle over retained manifests in this verification corpus."""
import json
import sys
import xml.etree.ElementTree as ET
ANDROID = "{http://schemas.android.com/apk/res/android}"
KINDS = {"activity", "activity-alias", "service", "receiver", "provider"}
DATA_ATTRIBUTES = {
"scheme": "scheme", "host": "host", "port": "port", "path": "path",
"path_prefix": "pathPrefix", "path_pattern": "pathPattern",
"path_advanced_pattern": "pathAdvancedPattern", "path_suffix": "pathSuffix",
"mime_type": "mimeType",
}
def filter_facts(element):
def names(tag):
return [value.get(ANDROID + "name") for value in element.findall(tag)
if value.get(ANDROID + "name") is not None]
return {
"priority": element.get(ANDROID + "priority"),
"actions": names("action"), "categories": names("category"),
"data": [{key: value.get(ANDROID + attribute)
for key, attribute in DATA_ATTRIBUTES.items()}
for value in element.findall("data")],
}
def main():
with open(sys.argv[1], encoding="utf-8") as source:
bundle = json.load(source)
parent = next(record for record in bundle["records"] if record["evidence_id"] == sys.argv[2])
root = ET.fromstring(parent["normalized_result"]["manifest"]["text"])
package = root.get("package")
application = root.find("application")
assert application is not None and package, "Verification corpus requires a known package/application"
inventory = []
ordinal = 0
for component in application:
if component.tag not in KINDS:
continue
declared = component.get(ANDROID + "name")
if declared is None:
continue
filters = [filter_facts(value) for value in component.findall("intent-filter")]
enabled = component.get(ANDROID + "enabled")
permission = component.get(ANDROID + "permission")
if permission is None and component.tag != "activity-alias":
permission = application.get(ANDROID + "permission")
name = (package + declared if declared.startswith(".") else
package + "." + declared if "." not in declared else declared)
inventory.append({
"source_index": ordinal, "kind": component.tag,
"declared_name": declared, "name": name,
"enabled": True if enabled == "true" else False if enabled == "false" else None,
"enabled_declaration": enabled, "process": component.get(ANDROID + "process"),
"permission": permission or None, "intent_filters": filters,
"meta_data": [{"name": value.get(ANDROID + "name")}
for value in component.findall("meta-data")],
})
ordinal += 1
selected = [row for row in inventory
if any(sys.argv[3] in value["actions"] for value in row["intent_filters"])]
print(json.dumps({"components": selected, "inventory": inventory, "source_components": ordinal},
ensure_ascii=False, separators=(",", ":")))
if __name__ == "__main__":
main()
experiments/analysis-efficiency/fixtures/android-components/res/values/values.xml
+4
View File
@@ -0,0 +1,4 @@
<resources>
<bool name="receiver_enabled">false</bool>
<integer name="filter_priority">80</integer>
</resources>
experiments/analysis-efficiency/fixtures/android-components/verify-fixture.py
+58
View File
@@ -0,0 +1,58 @@
"""Check real AAPT2/JADX/CLI facts against the source-built manifest corpus."""
import hashlib
import json
from pathlib import Path
import sys
root = Path(sys.argv[1])
reply = json.loads((root / "cli.json").read_text())["summary"]
view = reply["normalized_result"]
bundle = json.loads((root / "evidence.json").read_text())
parent = next(record for record in bundle["records"]
if record["evidence_id"] == view["parent_evidence_id"])
sha256 = hashlib.sha256((root / "manifest.apk").read_bytes()).hexdigest()
assert parent["subject"]["digest"]["sha256"] == view["artifact"]["sha256"] == sha256
assert reply["evidence_links"] == [parent["evidence_id"]]
assert view["source_status"] == "complete" and view["source_components"] == 3
assert view["coverage"]["exhausted"] and view["coverage"]["total"] == 3
activity, receiver, provider = view["items"]
assert [row["source_index"] for row in view["items"]] == [0, 1, 2]
assert all(row["declaration_coverage"] == "complete" for row in view["items"])
assert activity["enabled"] is False and activity["process"] == ":activity"
first, second = activity["intent_filters"]
assert first["actions"] == ["android.intent.action.VIEW"] and first["priority"] == "7"
assert first["data"][0]["scheme"] == "https"
assert first["data"][0]["host"] == "example.test" and first["data"][0]["path_prefix"] == "/open"
assert second["actions"] == ["android.intent.action.EDIT"]
assert second["priority"] == "@integer/filter_priority"
assert second["data"][0]["scheme"] == "content" and second["data"][0]["mime_type"] == "text/plain"
assert activity["meta_data"][0]["name"] == "public.configuration"
assert "value" not in activity["meta_data"][0]
assert receiver["enabled"] is None and receiver["enabled_declaration"] == "@bool/receiver_enabled"
assert receiver["process"] == ":sms" and receiver["permission"] == "android.permission.BROADCAST_SMS"
assert receiver["intent_filters"][0]["priority"] == "-10"
assert provider["enabled"] is True and provider["grant_uri_permissions"] is True
assert provider["grant_uri_permissions_declaration"] == "true" and provider["process"] is None
manifest = parent["normalized_result"]["manifest"]["text"]
assert manifest.isascii() and "retained-in-manifest" in manifest
for row in view["items"]:
nodes = [(row, row["kind"])]
for element in row["intent_filters"]:
nodes.append((element, "intent-filter"))
nodes.extend((data, "data") for data in element["data"])
nodes.extend((metadata, "meta-data") for metadata in row["meta_data"])
for node, tag in nodes:
location = node["location"]
offset = location["offset"]
assert manifest[offset:].startswith("<" + tag)
assert manifest[:offset].count("\n") + 1 == location["line"]
assert offset - manifest.rfind("\n", 0, offset) == location["column"]
tree = (root / "aapt2-xmltree.txt").read_text()
for fragment in [':scheme(0x01010027)="https"', ':host(0x01010028)="example.test"',
':pathPrefix(0x0101002b)="/open"', ':priority(0x0101001c)=7',
':priority(0x0101001c)=-10', ':priority(0x0101001c)=@0x7f020000',
':enabled(0x0101000e)=@0x7f010000', ':grantUriPermissions(0x0101001b)=true']:
assert fragment in tree, fragment
print(json.dumps({"compiled_manifest_cli": "passed", "sha256": sha256,
"components": 3, "filter_grouping": True, "locations": True,
"resource_values_remain_declarations": True, "aapt2_oracle": True}))
experiments/analysis-efficiency/fixtures/browser-saved-values/immediate-followup.html
+30
View File
@@ -0,0 +1,30 @@
<!doctype html>
<meta charset="utf-8" />
<title>Immediate saved-response follow-up fixture</title>
<button id="run">Exchange and follow up</button>
<output id="result">pending</output>
<output id="sensitive"></output>
<script>
document.querySelector("#run").onclick = async () => {
const source = await (
await fetch("/escaped-values.json", { cache: "no-store" })
).json();
document.querySelector("#sensitive").textContent = source.token;
console.log(source.token);
const response = await fetch(
"/api/collections/rea_items/records?token=" +
encodeURIComponent(source.token),
{
method: "POST",
headers: {
"Content-Type": "application/json",
"X-Followup": source.token,
},
body: JSON.stringify({ token: source.token }),
},
);
const result = document.querySelector("#result");
result.textContent = String(response.status);
result.dataset.ready = "yes";
};
</script>
experiments/analysis-efficiency/fixtures/browser-saved-values/index.html
+39
View File
@@ -0,0 +1,39 @@
<!doctype html><meta charset="utf-8" /><title>
PocketBase saved value verifier
</title>
<label>Email<input id="email" /></label
><label>Password<input type="password" id="password" /></label
><button id="login">Sign in</button><button id="create">Create record</button>
<pre id="result"></pre>
<script>
document.querySelector("#login").onclick = async () => {
const r = await fetch("/api/collections/_superusers/auth-with-password", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
identity: document.querySelector("#email").value,
password: document.querySelector("#password").value,
}),
});
const data = await r.json();
localStorage.setItem("rea_auth", JSON.stringify(data));
document.querySelector("#result").textContent = r.ok
? "Signed in"
: "Sign in failed";
if (r.ok) document.querySelector("#result").dataset.ready = "yes";
};
document.querySelector("#create").onclick = async () => {
const token = JSON.parse(localStorage.getItem("rea_auth") || "{}").token;
const r = await fetch("/api/collections/rea_items/records", {
method: "POST",
headers: {
"Content-Type": "application/json",
...(token ? { Authorization: token } : {}),
},
body: JSON.stringify({ title: "rea-ui-record" }),
});
document.querySelector("#result").textContent = JSON.stringify(
await r.json(),
);
};
</script>
experiments/analysis-efficiency/fixtures/browser-saved-values/whole-json.html
+15
View File
@@ -0,0 +1,15 @@
<!doctype html>
<meta charset="utf-8" />
<title>Declared JSON payload privacy fixture</title>
<output id="result">pending</output>
<script>
fetch("/api/collections/rea_items/records", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: localStorage.getItem("whole_json"),
}).then((response) => {
const result = document.querySelector("#result");
result.textContent = String(response.status);
result.dataset.ready = "yes";
});
</script>
package.json
+4 -5
View File
@@ -112,21 +112,21 @@
"prebuild": "npm run deps:check",
"build": "node scripts/run-exclusive.mjs artifacts node scripts/run-turbo.mjs run artifacts:skills --force",
"precheck": "npm run deps:check",
"check": "node scripts/run-exclusive.mjs artifacts node scripts/run-turbo.mjs run typecheck lint:built format:check knip metadata:check && npm run verify:architecture-guards && npm run knip:production && npm run lint:typed",
"check": "node scripts/run-exclusive.mjs artifacts node scripts/run-turbo.mjs run typecheck lint format:check knip metadata:check && npm run verify:architecture-guards && npm run knip:production && npm run lint:typed",
"precheck:fast": "npm run deps:check",
"check:fast": "node scripts/run-turbo.mjs run typecheck lint --output-logs=errors-only",
"precheck:changed": "npm run deps:check",
"check:changed": "npm run check:fast && npm run test:changed",
"precheck:test": "npm run deps:check",
"check:test": "node scripts/run-exclusive.mjs artifacts node scripts/run-turbo.mjs run typecheck lint:built format:check knip metadata:check test:run",
"check:test": "node scripts/run-exclusive.mjs artifacts node scripts/run-turbo.mjs run typecheck lint format:check knip metadata:check test:run",
"precheck:pr": "npm run deps:check",
"check:pr": "node scripts/run-exclusive.mjs artifacts node scripts/run-turbo.mjs run typecheck lint:built format:check knip metadata:check test:run && npm run test:python && npm run docs:check && npm run jscpd && npm run scan:todos && npm run knip:production && npm run lint:typed",
"check:pr": "node scripts/run-exclusive.mjs artifacts node scripts/run-turbo.mjs run typecheck lint format:check knip metadata:check test:run && npm run test:python && npm run docs:check && npm run jscpd && npm run scan:todos && npm run knip:production && npm run lint:typed",
"precheck:ci": "npm run deps:check",
"precheck:ci:fast": "npm run deps:check",
"precheck:ci:deep": "npm run deps:check",
"check:ci": "npm run check:ci:fast && npm run check:ci:deep",
"check:ci:fast": "node scripts/run-exclusive.mjs artifacts npm run check:ci:fast:tasks",
"check:ci:fast:tasks": "node scripts/run-turbo.mjs run typecheck lint:built format:check",
"check:ci:fast:tasks": "node scripts/run-turbo.mjs run typecheck lint format:check",
"check:ci:deep": "node scripts/run-exclusive.mjs artifacts npm run check:ci:deep:tasks && npm run verify:architecture-guards && npm run verify:test-discovery && npm run test:python && npm run knip:production && npm run lint:typed",
"check:ci:deep:tasks": "node scripts/run-turbo.mjs run knip metadata:check jscpd scan:todos",
"ci:static-context": "node scripts/ci/static-task-context.mjs",
@@ -137,7 +137,6 @@
"format:check": "oxfmt --check .",
"lint": "node scripts/ci/static-check-report.mjs lint && npm run verify:module-boundaries",
"lint:details": "oxlint && npm run verify:module-boundaries",
"lint:built": "npm run lint",
"lint:fix": "oxlint --fix && npm run verify:module-boundaries",
"knip": "knip --cache",
"jscpd": "node scripts/ci/static-check-report.mjs duplicates",
scripts/ci/artifacts.mjs
+1 -1
View File
@@ -378,7 +378,7 @@ function blobIdentityStatus(diagnostics, blobs) {
}
function sameTaskHashes(diagnostics, left, right) {
return diagnostics.every((diagnostic) => {
const task = diagnostic.kind === "jscpd" ? "jscpd" : "lint:built";
const task = diagnostic.kind === "jscpd" ? "jscpd" : "lint";
return (
TASK_HASH.test(left.taskHashes[task] ?? "") &&
left.taskHashes[task] === right.taskHashes[task]
scripts/lib/browser-scenario-verifier.mjs
+14 -3
View File
@@ -233,11 +233,19 @@ export function assertScenarioCapture(capture) {
const final = capture.steps.at(-1);
if (
final === undefined ||
Object.values(final.artifacts).some(({ state }) => state !== "captured")
Object.entries(final.artifacts).some(
([kind, { state }]) => kind !== "screenshot" && state !== "captured",
)
)
throw new Error(
`Scenario launch did not capture every requested artifact: ${JSON.stringify(final?.artifacts ?? null)}`,
);
for (const step of capture.steps)
assert.ok(
step.artifacts.screenshot.state === "missing" &&
step.artifacts.screenshot.reason.includes("sensitive"),
"Sensitive scenario screenshots must be withheld from its first step",
);
const storageNames =
final.artifacts.storage.state === "captured"
? final.artifacts.storage.value.local_storage.map(({ name }) => name)
@@ -281,10 +289,13 @@ export function assertScenarioCapture(capture) {
);
// This fixture polls continuously. A request crossing the capture cutoff
// makes later activity unknown even when no response body was selected.
const expectedMissing = unfinished.length === 0 ? [] : ["events"];
const expectedMissing = [
...(unfinished.length === 0 ? [] : ["events"]),
"screenshot",
].sort();
if (
capture.events.items.length === 0 ||
capture.completeness.equality_eligible !== (unfinished.length === 0) ||
capture.completeness.equality_eligible !== false ||
JSON.stringify(capture.completeness.missing_sections) !==
JSON.stringify(expectedMissing) ||
capture.completeness.truncated_sections.length !== 0
scripts/rea.mjs
+3 -2
View File
@@ -80,14 +80,15 @@ if (!(await compiledRuntimeExists(runtimeFiles))) {
} else {
const {
cliConfigurationError,
normalizeCliArguments,
renderCliOutputArgumentError,
renderEmptyFilteredCliOutput,
renderVersionReport,
parseCliOutputArguments,
} = await import("../dist/cli-runtime/cliOutput.js");
guardClosedStdout();
const cliArgs = isShellCompletion ? args : normalizeCliArguments(args);
const cliArgs = isShellCompletion
? args
: builtinArguments.normalizeCliArguments(args);
// Completion words are transport data, including partial output flags.
const outputOptions = parseCliOutputArguments(
isShellCompletion ? [] : cliArgs,
scripts/verify-real-ghidra-inventory.mjs
+7 -1
View File
@@ -1,3 +1,4 @@
import { filterGhidraLiteralCandidates } from "../dist/ghidra/GhidraLiteralSearch.js";
import { parseGhidraInventoryResult } from "../dist/ghidra/GhidraInventoryValues.js";
import { findValue, matchesSymbol } from "./verify-real-ghidra-assertions.mjs";
@@ -16,7 +17,12 @@ export async function call(client, operation, parameters) {
);
const parsed = parseGhidraInventoryResult(operation, called.value);
if (!parsed.ok) throw parsed.error;
return parsed.value.value;
return (operation === "search_procedures" ||
operation === "search_strings") &&
parameters.mode === "literal" &&
parameters.case_sensitive === false
? filterGhidraLiteralCandidates(parsed.value.value, parameters.pattern)
: parsed.value.value;
}
export async function allItems(client, operation, parameters) {
scripts/verify/android/metadata.mjs
+4
View File
@@ -81,6 +81,10 @@ export const verifyAndroidMetadata = async ({
result.stdout,
/RAW_CLASS_SELECTOR_REGRESSION_PASS nested=true metadata=true references=true/u,
);
assert.match(
result.stdout,
/DEFAULT_PACKAGE_SELECTOR_REGRESSION_PASS raw=true display=true metadata=true method=true references=true frozen=true/u,
);
assert.match(
result.stdout,
/INITIALIZER_REGRESSION_PASS folded=2 bounded=true/u,
scripts/verify/architecture-guards.mjs
+7
View File
@@ -59,6 +59,13 @@ const check = (label, selected, pattern, allow = []) => {
);
};
// Prerequisite inspection lifetime is an application contract shared by adapters.
check(
"provider-owned inspection lifetime (use src/application/ProviderInspectionScope)",
files,
/providers\/ProviderInspectionScope\.js/,
);
check(
"src/domain uses localeCompare (use compareUnicodeCodePoints)",
inDir("src/domain/"),
src/adb/AdbCommand.capture.test.ts
+86
View File
@@ -0,0 +1,86 @@
import { expect, it } from "vitest";
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
import { runAdbCapture } from "./AdbCommand.js";
// Exercise the real subprocess capture and public error projection, substituting
// only the bring-your-own executable. These are not Android-device checks.
it.each([
{
name: "overflow",
source: 'process.stdout.write("X".repeat(4096))',
maxOutputBytes: 64,
code: "resource_constraint",
stdout: "X".repeat(64),
stderr: "",
truncated: true,
},
{
name: "timeout",
source:
'process.stdout.write("partial-雪"); process.stderr.write("waiting\\n"); setInterval(() => {}, 1000)',
code: "provider_timeout",
stdout: "partial-雪",
stderr: "waiting\n",
},
{
name: "offline device",
source:
'process.stdout.write("partial-雪"); process.stderr.write("adb: error: device offline\\n"); process.exitCode = 1',
code: "unsupported_target",
stdout: "partial-雪",
stderr: "adb: error: device offline\n",
},
{
name: "local path refusal",
source:
'process.stdout.write("partial-雪"); process.stderr.write("adb: error: cannot stat \'/host/token file\': Permission denied\\n"); process.exitCode = 1',
code: "unsupported_target",
stdout: "partial-雪",
stderr: "adb: error: cannot stat '/host/token file': Permission denied\n",
},
])("preserves bounded capture on $name", async (testCase) => {
const result = await runAdbCapture(
"pull_adb_file",
{
device: "caller-device",
path: "/remote/source",
localPath: "/host/token file",
},
{
binary: process.execPath,
pathSource: "environment",
environment: { REA_TEST_PRIVATE: "ambient-value-must-not-be-persisted" },
},
["-e", testCase.source],
{
timeoutMs: testCase.name === "timeout" ? 500 : 5_000,
maxOutputBytes: testCase.maxOutputBytes ?? 1024,
},
undefined,
);
expect(result.ok).toBe(false);
if (result.ok) return;
const projected = projectAnalysisError(result.error);
expect(projected).toMatchObject({
code: testCase.code,
details: {
captured_output: {
stdout: testCase.stdout,
stderr: testCase.stderr,
stdout_bytes: Buffer.byteLength(testCase.stdout),
stderr_bytes: Buffer.byteLength(testCase.stderr),
truncated: testCase.truncated ?? false,
},
},
});
if (testCase.code === "unsupported_target")
expect(projected.details?.path).toBe(
testCase.name === "local path refusal"
? "/host/token file"
: "caller-device",
);
expect(JSON.stringify(projected)).not.toContain(
"ambient-value-must-not-be-persisted",
);
});
src/adb/AdbCommand.ts
+13 -6
View File
@@ -130,6 +130,10 @@ export const runAdbCapture = async (
return err(
new AnalysisProtocolError(`adb failed for ${operation}`, { cause }),
);
const detail = {
cause,
capturedOutput: execFileFailureCapturedOutput(failure),
};
if (failure.outputTruncated) {
const name = `The adb output capture allowance of ${String(limits.maxOutputBytes)} bytes (this release has no runtime override)`;
const allowance =
@@ -145,7 +149,7 @@ export const runAdbCapture = async (
"memory",
`adb output exceeded the ${String(limits.maxOutputBytes)}-byte capture budget`,
{ max_output_bytes: limits.maxOutputBytes },
{ cause, ...allowance },
{ ...detail, ...allowance },
),
);
}
@@ -181,9 +185,12 @@ export const runAdbCapture = async (
)
)
return err(
new AnalysisUnsupportedTargetError(operation, target.device, excerpt, {
cause,
}),
new AnalysisUnsupportedTargetError(
operation,
target.device,
excerpt,
detail,
),
);
if (
/No such file or directory|Permission denied|read: Permission denied/iu.test(
@@ -201,13 +208,13 @@ export const runAdbCapture = async (
? localPath
: (target.path ?? target.device);
return err(
new AnalysisUnsupportedTargetError(operation, path, excerpt, { cause }),
new AnalysisUnsupportedTargetError(operation, path, excerpt, detail),
);
}
if (failure.killed)
return err(
new AnalysisTimeoutError(operation, limits.timeoutMs, {
cause,
...detail,
userMessage: `adb ${operation} reached its ${String(limits.timeoutMs)} ms deadline (deadlineExpired). This release has no runtime override.`,
remediationAction: `This ${String(limits.timeoutMs)} ms adb deadline has no runtime override. Allow execution, cleanup and delivery headroom in the outer MCP client deadline. Confirm outstanding work and cleanup before retrying.`,
}),
src/android/AndroidManifestComponentDetails.ts
+249
View File
@@ -0,0 +1,249 @@
import type { Element } from "@xmldom/xmldom";
import type { z } from "zod";
import type {
AndroidManifestComponent,
androidIntentDataSchema,
androidIntentFilterSchema,
androidComponentMetadataSchema,
} from "../domain/android/androidManifestComponents.js";
import {
androidAttribute,
createManifestLocationReader,
isAndroidAttribute,
isManifestElement,
} from "./AndroidManifestXml.js";
type Filter = z.output<typeof androidIntentFilterSchema>;
type Data = z.output<typeof androidIntentDataSchema>;
type Metadata = z.output<typeof androidComponentMetadataSchema>;
const projectedDataAttributes = new Set([
"scheme",
"host",
"port",
"path",
"pathPrefix",
"pathPattern",
"pathAdvancedPattern",
"pathSuffix",
"mimeType",
]);
const literalBoolean = (value: string | null): boolean | null =>
value === "true" ? true : value === "false" ? false : null;
/** One decoded-manifest lifetime owns nested declarations and cumulative copy budgets. */
export const createAndroidManifestDetailsReader = (
source: string,
limits: {
readonly intentFilters: number;
readonly intentValues: number;
readonly processBytes: number;
readonly detailBytes: number;
},
) => {
const locationOf = createManifestLocationReader(source);
let processRemaining = limits.processBytes;
let detailsRemaining = limits.detailBytes;
return (component: Element, declarationIndex: number) => {
let partial = false;
let detailLimited = false;
let processLimited = false;
let unsupported = 0;
let remainingRecords = limits.intentValues;
const take = (value: string | null): string | null => {
if (value === null) return null;
const bytes = Buffer.byteLength(value);
if (bytes > detailsRemaining) {
partial = true;
detailLimited = true;
return null;
}
detailsRemaining -= bytes;
return value;
};
const admit = (): boolean => {
if (remainingRecords === 0) {
partial = true;
detailLimited = true;
return false;
}
remainingRecords -= 1;
return true;
};
const enabledDeclaration = androidAttribute(component, "enabled");
const enabled = literalBoolean(enabledDeclaration);
const grantDeclaration =
component.nodeName === "provider"
? androidAttribute(component, "grantUriPermissions")
: null;
const grant = literalBoolean(grantDeclaration);
const declaredProcess = androidAttribute(component, "process");
let process: string | null = null;
if (declaredProcess !== null) {
const bytes = Buffer.byteLength(declaredProcess);
if (bytes > processRemaining) {
partial = true;
processLimited = true;
} else {
process = declaredProcess;
processRemaining -= bytes;
}
}
const filters: Filter[] = [];
const metadata: Metadata[] = [];
const actions: string[] = [];
const categories: string[] = [];
const actionNames = new Set<string>();
const categoryNames = new Set<string>();
let legacyValuesRemaining = limits.intentValues;
let legacyTruncated = false;
let filterCount = 0;
for (let index = 0; index < component.childNodes.length; index += 1) {
const child = component.childNodes.item(index);
if (!isManifestElement(child)) continue;
if (child.nodeName === "meta-data") {
if (admit())
metadata.push({
name: take(androidAttribute(child, "name")),
location: locationOf(child),
});
continue;
}
if (child.nodeName !== "intent-filter") continue;
const filterIndex = filterCount++;
if (filterIndex >= limits.intentFilters) {
partial = true;
legacyTruncated = true;
continue;
}
const filterActions: string[] = [];
const filterCategories: string[] = [];
const data: Data[] = [];
let filterPartial = false;
const priority = take(androidAttribute(child, "priority"));
for (let at = 0; at < child.childNodes.length; at += 1) {
const member = child.childNodes.item(at);
if (!isManifestElement(member)) continue;
if (member.nodeName === "action" || member.nodeName === "category") {
const value = androidAttribute(member, "name");
if (value === null) continue;
const names =
member.nodeName === "action" ? actionNames : categoryNames;
if (!names.has(value)) {
if (legacyValuesRemaining === 0) legacyTruncated = true;
else {
legacyValuesRemaining -= 1;
names.add(value);
(member.nodeName === "action" ? actions : categories).push(value);
}
}
if (!admit()) {
filterPartial = true;
continue;
}
const copied = take(value);
if (copied === null) {
filterPartial = true;
continue;
}
(member.nodeName === "action"
? filterActions
: filterCategories
).push(copied);
} else if (member.nodeName === "data") {
if (!admit()) {
filterPartial = true;
continue;
}
for (
let attributeIndex = 0;
attributeIndex < member.attributes.length;
attributeIndex += 1
) {
const attribute = member.attributes.item(attributeIndex);
if (
attribute !== null &&
isAndroidAttribute(attribute) &&
(attribute.localName === null ||
!projectedDataAttributes.has(attribute.localName))
) {
// A retained data declaration can have unprojected matching constraints.
unsupported += 1;
filterPartial = true;
partial = true;
break;
}
}
data.push({
location: locationOf(member),
scheme: take(androidAttribute(member, "scheme")),
host: take(androidAttribute(member, "host")),
port: take(androidAttribute(member, "port")),
path: take(androidAttribute(member, "path")),
path_prefix: take(androidAttribute(member, "pathPrefix")),
path_pattern: take(androidAttribute(member, "pathPattern")),
path_advanced_pattern: take(
androidAttribute(member, "pathAdvancedPattern"),
),
path_suffix: take(androidAttribute(member, "pathSuffix")),
mime_type: take(androidAttribute(member, "mimeType")),
});
} else {
// Newer platform filter constructs remain source-accessible, never an empty match claim.
unsupported += 1;
filterPartial = true;
partial = true;
}
}
filters.push({
source_index: filterIndex,
location: locationOf(child),
priority,
actions: filterActions,
categories: filterCategories,
data,
coverage: filterPartial || detailLimited ? "partial" : "complete",
});
}
const enabledRaw = take(enabledDeclaration);
const grantRaw = take(grantDeclaration);
const facts = {
declaration_index: declarationIndex,
location: locationOf(component),
declaration_coverage: partial ? "partial" : "complete",
enabled,
enabled_declaration: enabledRaw,
process,
grant_uri_permissions: grant,
grant_uri_permissions_declaration: grantRaw,
intent_filters: filters,
meta_data: metadata,
} satisfies Pick<
AndroidManifestComponent,
| "declaration_index"
| "location"
| "declaration_coverage"
| "enabled"
| "enabled_declaration"
| "process"
| "grant_uri_permissions"
| "grant_uri_permissions_declaration"
| "intent_filters"
| "meta_data"
>;
return {
facts,
actions,
categories,
filterCount,
legacyTruncated,
partial,
detailLimited,
processLimited,
unsupported,
nonliteralEnabled: enabledDeclaration !== null && enabled === null,
nonliteralGrant: grantDeclaration !== null && grant === null,
};
};
};
src/android/AndroidManifestComponents.ts
+71 -141
View File
@@ -1,16 +1,17 @@
import type { Element, Node } from "@xmldom/xmldom";
import { parseXmlDocument } from "../domain/xmlDocument.js";
/** Component kinds a decoded AndroidManifest can declare under <application>. */
export const ANDROID_COMPONENT_KINDS = [
"activity",
"activity-alias",
"service",
"receiver",
"provider",
] as const;
export type AndroidComponentKind = (typeof ANDROID_COMPONENT_KINDS)[number];
import {
androidAttribute,
plainAttribute,
childNodes,
isManifestElement,
} from "./AndroidManifestXml.js";
import { createAndroidManifestDetailsReader } from "./AndroidManifestComponentDetails.js";
import {
ANDROID_COMPONENT_KINDS,
type AndroidComponentKind,
type ObservedAndroidManifestComponent,
type ObservedAndroidManifestComponentInventory,
} from "../domain/android/androidManifestComponents.js";
/** Bounds keep a hostile manifest from exhausting the inspection. */
export const ANDROID_MANIFEST_LIMITS = {
@@ -23,103 +24,23 @@ export const ANDROID_MANIFEST_LIMITS = {
authorityBytes: 1024 * 1024,
/** Total copied permission-value bytes, including inherited defaults. */
permissionBytes: 1024 * 1024,
/** Total copied android:process bytes across projected declarations. */
processBytes: 1024 * 1024,
/** Copied nested declaration values; complete text remains retained. */
detailBytes: 1024 * 1024,
} as const;
/** The attribute namespace every component facet uses. */
const ANDROID_NAMESPACE = "http://schemas.android.com/apk/res/android";
export interface AndroidManifestComponent {
readonly kind: AndroidComponentKind;
/** Class name as written in the manifest. */
readonly declared_name: string;
/**
* Fully qualified name with the manifest package applied, or the declared
* name unchanged when it cannot be resolved or its syntax is not
* established.
*/
readonly name: string;
/**
* Declared android:exported, or null when the attribute is absent or set
* to a value other than the literals true or false, whose effective state
* the parser cannot resolve.
*/
readonly exported: boolean | null;
/**
* Android's default export state when android:exported is absent: the
* intent-filter rule for activities, services, receivers and aliases, and
* the target-SDK rule for providers. Null when the provider rule cannot be
* evaluated because the target SDK is unknown, or when the attribute is
* present but not a literal boolean, since the default does not govern a
* declared value.
*/
readonly exported_default: boolean | null;
readonly permission: string | null;
/** Source of the effective permission projection; unknown when its copy is capped. */
readonly permission_source: "component" | "application" | "none" | "unknown";
readonly intent_filter_count: number;
readonly actions: readonly string[];
readonly categories: readonly string[];
/** Declared android:foregroundServiceType on services; null otherwise. */
readonly foreground_service_type: string | null;
/** Exact declared provider authority segments, split on ";"; malformed segments remain raw. */
readonly authorities: readonly string[] | null;
/** Exact declared android:targetActivity on activity-alias; malformed references remain raw and make the inventory partial. */
readonly target_activity: string | null;
}
export interface AndroidManifestComponentInventory {
readonly status: "complete" | "partial" | "unavailable";
readonly items: readonly AndroidManifestComponent[];
readonly limitations: readonly string[];
}
const PARTIAL_LIMITATION =
"The decoded manifest text is truncated, so its component inventory is not established; the retained manifest is partial.";
const unavailable = (
limitation: string,
): AndroidManifestComponentInventory => ({
): ObservedAndroidManifestComponentInventory => ({
status: "unavailable",
items: [],
limitations: [limitation],
});
const childNodes = (element: Element): readonly Node[] =>
Array.from(element.childNodes);
/** Manifest tags are unnamespaced; Android's namespace qualifies attributes. */
const isManifestElement = (node: Node | null): node is Element =>
node !== null &&
node.nodeType === 1 &&
(node as Element).namespaceURI === null;
/** Read an unnamespaced attribute, the shape manifest `package` uses. */
const plainAttribute = (element: Element, name: string): string | null => {
const value = element.getAttribute(name);
return value === "" ? null : value;
};
/**
* Read an android-namespaced attribute. XML prefixes are aliases, so a
* same-local-name attribute from another namespace is never a match.
*/
const androidAttribute = (
element: Element,
localName: string,
): string | null => {
const attributes = element.attributes;
for (let index = 0; index < attributes.length; index += 1) {
const node = attributes.item(index);
if (node === null || node.localName !== localName) continue;
if (node.namespaceURI === ANDROID_NAMESPACE) return node.nodeValue;
// Without namespace information (an undeclared prefix), the conventional
// android: prefix is the only remaining signal.
if (node.namespaceURI === null && node.prefix === "android")
return node.nodeValue;
}
return null;
};
/** Android application-ID grammar, with the platform package exception. */
const hasSupportedAndroidPackageNameSyntax = (name: string): boolean =>
name === "android" ||
@@ -184,37 +105,6 @@ const androidTargetSdkLevel = (targetSdk: string | null): number | null => {
: null;
};
/**
* Collect unique intent-filter actions and categories. The value budget is
* shared across both arrays, matching the stated per-component bound.
*/
const intentValues = (
filters: readonly Element[],
): { actions: string[]; categories: string[]; truncated: boolean } => {
const actions: string[] = [];
const categories: string[] = [];
let truncated = filters.length > ANDROID_MANIFEST_LIMITS.intentFilters;
let budget = ANDROID_MANIFEST_LIMITS.intentValues;
for (const filter of filters.slice(0, ANDROID_MANIFEST_LIMITS.intentFilters))
for (const member of childNodes(filter)) {
if (!isManifestElement(member)) continue;
const element = member;
if (element.nodeName !== "action" && element.nodeName !== "category")
continue;
const value = androidAttribute(element, "name");
if (value === null) continue;
const target = element.nodeName === "action" ? actions : categories;
if (target.includes(value)) continue;
if (budget <= 0) {
truncated = true;
continue;
}
budget -= 1;
target.push(value);
}
return { actions, categories, truncated };
};
/** Split a provider authorities declaration within its own bound. */
const providerAuthorities = (
declared: string,
@@ -262,7 +152,7 @@ export const parseAndroidManifestComponents = (
coverage: "complete" | "partial",
declaredTargetSdk: string | null = null,
minimumSdk: string | null = null,
): AndroidManifestComponentInventory => {
): ObservedAndroidManifestComponentInventory => {
if (coverage === "partial")
return {
status: "partial",
@@ -295,7 +185,7 @@ export const parseAndroidManifestComponents = (
const packageNameBytes =
packageName === null ? 0 : Buffer.byteLength(packageName);
const limitations: string[] = [];
const items: AndroidManifestComponent[] = [];
const items: ObservedAndroidManifestComponent[] = [];
let truncated = false;
let applicationSeen = false;
let unnamed = 0;
@@ -313,9 +203,18 @@ export const parseAndroidManifestComponents = (
let capacityTruncated = false;
let permissionBudgetLimited = false;
let unresolvedPermissionTokens = 0;
let nonliteralEnabled = 0;
let nonliteralGrantUriPermissions = 0;
let processBudgetLimited = false;
let nameBudget = ANDROID_MANIFEST_LIMITS.resolvedNameBytes;
let authorityBudget = ANDROID_MANIFEST_LIMITS.authorityBytes;
let permissionBudget = ANDROID_MANIFEST_LIMITS.permissionBytes;
let detailBudgetLimited = false;
let unsupportedFilterMembers = 0;
const readDetails = createAndroidManifestDetailsReader(
manifestText,
ANDROID_MANIFEST_LIMITS,
);
const children = childNodes(root);
let targetSdkDeclared = false;
let minimumSdkDeclared = false;
@@ -375,11 +274,21 @@ export const parseAndroidManifestComponents = (
continue;
}
const unresolvedName = containsUnresolvedToken(declared);
const filters = childNodes(component).filter(
(entry): entry is Element =>
isManifestElement(entry) && entry.nodeName === "intent-filter",
);
const collected = intentValues(filters);
const details = readDetails(component, items.length + unnamed);
const collected = {
actions: details.actions,
categories: details.categories,
truncated: details.legacyTruncated,
};
if (details.partial) truncated = true;
if (details.detailLimited) detailBudgetLimited = true;
unsupportedFilterMembers += details.unsupported;
if (details.processLimited) {
processBudgetLimited = true;
capacityTruncated = true;
}
if (details.nonliteralEnabled) nonliteralEnabled += 1;
if (details.nonliteralGrant) nonliteralGrantUriPermissions += 1;
if (collected.truncated) {
truncated = true;
capacityTruncated = true;
@@ -435,7 +344,7 @@ export const parseAndroidManifestComponents = (
precedingActivityTargetNames.add(bareName);
}
}
let authorities: readonly string[] | null = null;
let authorities: string[] | null = null;
if (component.nodeName === "provider") {
if (
androidAttribute(component, "readPermission") !== null ||
@@ -549,7 +458,7 @@ export const parseAndroidManifestComponents = (
declaredExported === null &&
targetSdkLevel !== null &&
targetSdkLevel >= 31 &&
filters.length > 0 &&
details.filterCount > 0 &&
(component.nodeName === "activity" ||
component.nodeName === "activity-alias" ||
component.nodeName === "service" ||
@@ -557,7 +466,7 @@ export const parseAndroidManifestComponents = (
const filteredExportUnknown =
declaredExported === null &&
targetSdkLevel === null &&
filters.length > 0 &&
details.filterCount > 0 &&
(component.nodeName === "activity" ||
component.nodeName === "activity-alias" ||
component.nodeName === "service" ||
@@ -582,10 +491,10 @@ export const parseAndroidManifestComponents = (
? null
: component.nodeName === "provider"
? providerDefault
: filters.length > 0,
: details.filterCount > 0,
permission,
permission_source: effectivePermissionSource,
intent_filter_count: filters.length,
intent_filter_count: details.filterCount,
actions: collected.actions,
categories: collected.categories,
foreground_service_type:
@@ -594,6 +503,7 @@ export const parseAndroidManifestComponents = (
: null,
authorities,
target_activity: targetActivity,
...details.facts,
});
}
}
@@ -646,6 +556,14 @@ export const parseAndroidManifestComponents = (
limitations.push(
`${String(nonliteralExported)} component declaration(s) set android:exported to a value other than the literals true or false (for example a resource reference), so their effective export state is unknown and the Android default does not apply.`,
);
if (nonliteralEnabled > 0)
limitations.push(
`${String(nonliteralEnabled)} component declaration(s) set android:enabled to a nonliteral boolean; the declared enabled state is unknown.`,
);
if (nonliteralGrantUriPermissions > 0)
limitations.push(
`${String(nonliteralGrantUriPermissions)} provider(s) set android:grantUriPermissions to a nonliteral boolean; the declared URI-grant state is unknown.`,
);
if (unresolvedPackages > 0) {
limitations.push(
`${String(unresolvedPackages)} package-relative component name(s) remain unresolved because the manifest package is absent or contains a resource/placeholder token, or its syntax is unsupported; the declared name is retained.`,
@@ -668,6 +586,18 @@ export const parseAndroidManifestComponents = (
limitations.push(
`Projected component permission values are capped at ${String(ANDROID_MANIFEST_LIMITS.permissionBytes)} total bytes; omitted copies have permission_source "unknown".`,
);
if (detailBudgetLimited)
limitations.push(
`Nested component declarations reached the ${String(ANDROID_MANIFEST_LIMITS.intentValues)}-record per-component or ${String(ANDROID_MANIFEST_LIMITS.detailBytes)}-byte shared copy bound; affected declaration coverage is partial. Read the retained manifest at each source location for omitted facts.`,
);
if (unsupportedFilterMembers > 0)
limitations.push(
`${String(unsupportedFilterMembers)} intent-filter child declaration(s) use unsupported constructs; affected filter coverage is partial and the declarations remain in the retained manifest.`,
);
if (processBudgetLimited)
limitations.push(
`Projected component process declarations are capped at ${String(ANDROID_MANIFEST_LIMITS.processBytes)} total bytes; omitted values are null.`,
);
if (unresolvedPermissionTokens > 0) {
limitations.push(
`${String(unresolvedPermissionTokens)} permission value(s) contain unresolved resource or manifest-placeholder tokens; their effective values are unknown.`,
@@ -691,7 +621,7 @@ export const parseAndroidManifestComponents = (
}
if (capacityTruncated)
limitations.push(
`The component inventory reached a capacity limit: ${String(ANDROID_MANIFEST_LIMITS.components)} declarations, ${String(ANDROID_MANIFEST_LIMITS.intentFilters)} filters or ${String(ANDROID_MANIFEST_LIMITS.intentValues)} values per component, ${String(ANDROID_MANIFEST_LIMITS.authorities)} authorities per provider, ${String(ANDROID_MANIFEST_LIMITS.authorityBytes)} total authority bytes, or ${String(ANDROID_MANIFEST_LIMITS.permissionBytes)} total permission bytes.`,
`The component inventory reached a capacity limit: ${String(ANDROID_MANIFEST_LIMITS.components)} declarations, ${String(ANDROID_MANIFEST_LIMITS.intentFilters)} filters or ${String(ANDROID_MANIFEST_LIMITS.intentValues)} values per component, ${String(ANDROID_MANIFEST_LIMITS.authorities)} authorities per provider, ${String(ANDROID_MANIFEST_LIMITS.authorityBytes)} total authority bytes, or ${String(ANDROID_MANIFEST_LIMITS.permissionBytes)} total permission bytes, or ${String(ANDROID_MANIFEST_LIMITS.processBytes)} total process bytes.`,
);
return {
status: truncated ? "partial" : "complete",
src/android/AndroidManifestXml.ts
+50
View File
@@ -0,0 +1,50 @@
import type { Attr, Element, Node } from "@xmldom/xmldom";
import type { AndroidManifestLocation } from "../domain/android/androidManifestComponents.js";
import { XML_LINE_ENDING } from "../domain/xmlDocument.js";
const ANDROID_NAMESPACE = "http://schemas.android.com/apk/res/android";
export const childNodes = (element: Element): readonly Node[] =>
Array.from(element.childNodes);
export const isManifestElement = (node: Node | null): node is Element =>
node !== null &&
node.nodeType === 1 &&
(node as Element).namespaceURI === null;
export const plainAttribute = (
element: Element,
name: string,
): string | null => {
const value = element.getAttribute(name);
return value === "" ? null : value;
};
/** Namespace identity, never the spelling of a bound prefix, selects a fact. */
export const isAndroidAttribute = (attribute: Attr): boolean =>
attribute.namespaceURI === ANDROID_NAMESPACE ||
(attribute.namespaceURI === null && attribute.prefix === "android");
export const androidAttribute = (
element: Element,
localName: string,
): string | null => {
for (let index = 0; index < element.attributes.length; index += 1) {
const attribute = element.attributes.item(index);
if (attribute === null || attribute.localName !== localName) continue;
if (isAndroidAttribute(attribute)) return attribute.nodeValue;
}
return null;
};
/** Build the source line index once; all locations address the same retained text. */
export const createManifestLocationReader = (source: string) => {
const starts = [0];
for (const match of source.matchAll(XML_LINE_ENDING))
starts.push(match.index + match[0].length);
return (element: Element): AndroidManifestLocation | null => {
const line = element.lineNumber;
const column = element.columnNumber;
if (line === undefined || column === undefined || line < 1 || column < 1)
return null;
const start = starts[line - 1];
if (start === undefined) return null;
const offset = start + column - 1;
return source[offset] === "<" ? { line, column, offset } : null;
};
};
src/android/JadxAnalysis.ts
+7 -1
View File
@@ -1,6 +1,7 @@
import type { z } from "zod";
import type { AndroidRequest } from "../domain/android/androidAnalysis.js";
import { androidResultSchemas } from "../domain/android/androidAnalysis.js";
import { observedAndroidManifestComponentsSchema } from "../domain/android/androidManifestComponents.js";
import { parseAndroidManifestComponents } from "./AndroidManifestComponents.js";
import {
AnalysisInputError,
@@ -39,6 +40,11 @@ type Engine = z.infer<
>["engine"];
type ClassSummary = z.infer<typeof jadxClassSchema>;
const observedPackageSchema =
androidResultSchemas.inspect_android_package.extend({
components: observedAndroidManifestComponentsSchema,
});
const methodsOf = (summary: ClassSummary) => {
const seen = new Map<string, number>();
return summary.methods.map((method) => {
@@ -127,7 +133,7 @@ const inspectPackage = async (
const manifest = normalizeJadxText(
await tools.text("get_android_manifest", {}),
);
return androidResultSchemas.inspect_android_package.parse({
return observedPackageSchema.parse({
engine,
package_name: app.package || null,
version_name: app.version_name || null,
src/application/AnalysisProvider.ts
+1 -1
View File
@@ -1,5 +1,5 @@
import type { AnalysisObservation } from "../domain/analysisObservation.js";
import type { ProviderInspectionScope } from "../providers/ProviderInspectionScope.js";
import type { ProviderInspectionScope } from "./ProviderInspectionScope.js";
import type { BinaryTarget } from "../domain/binaryTargetTypes.js";
import type { OfficialToolName } from "../contracts/officialToolContracts.js";
import type { EnhancedToolName } from "../contracts/enhancedInputs.js";
src/application/AppBundleExecutable.ts
+3
View File
@@ -119,6 +119,9 @@ export const findAppBundleResourceRoot = async (
options.cwd === undefined ? {} : { cwd: options.cwd },
);
try {
// Admit the selected spelling before Darwin realpath can collapse file/..
// into a different, otherwise valid bundle resource root.
await stat(callerPath);
const canonicalInput = await realpath(callerPath);
const directLayout = await presentLayout(canonicalInput);
const wrapped =
src/application/AppBundleResourceRoot.test.ts
+27
View File
@@ -2,6 +2,7 @@ import {
chmod,
mkdir,
mkdtemp,
realpath,
rm,
symlink,
writeFile,
@@ -57,6 +58,32 @@ it("resolves resource roots without requiring or hashing CFBundleExecutable", as
}
});
it.skipIf(process.platform === "win32")(
"preserves valid symlink-parent traversal when selecting resource roots",
async () => {
const temp = await mkdtemp(join(tmpdir(), "rea-resource-parent-"));
try {
await mkdir(join(temp, "physical", "child"), { recursive: true });
await mkdir(join(temp, "physical", "Fixture.app"));
await writeFile(
join(temp, "physical", "Fixture.app", "Info.plist"),
"<plist><dict/></plist>",
);
await symlink(join(temp, "physical", "child"), join(temp, "alias"));
const selected = `${temp}/alias/../Fixture.app`;
const resolved = await resolveAppBundleResourceRoot(selected);
expect(resolved.ok).toBe(true);
if (!resolved.ok) throw resolved.error;
expect(resolved.value.callerPath).toBe(selected);
expect(resolved.value.canonicalPath).toBe(
await realpath(join(temp, "physical", "Fixture.app")),
);
} finally {
await rm(temp, { recursive: true, force: true });
}
},
);
it("resolves a flat bundle resource root without inferring executable identity", async () => {
const temp = await mkdtemp(join(tmpdir(), "rea-ios-resource-root-"));
try {
src/application/BrowserEvidence.ts
+17 -3
View File
@@ -31,6 +31,7 @@ import type {
BrowserCaptureComparisonInput,
} from "../domain/browserCaptureComparison.js";
import { commitBrowserScenarioNormalization } from "../domain/browserScenarioNormalization.js";
import { digestCanonicalValue } from "../domain/canonicalDigest.js";
import type {
CaptureWebScreenshotInput,
CompareWebScreenshotsInput,
@@ -81,13 +82,26 @@ export const createBrowserEvidence = (
): Evidence => {
const acquisition = isBundleAcquisition(result);
const observed = acquisition ? result.inspection : result;
const comparison =
operation === "compare_web_captures" ||
operation === "compare_web_screenshots";
return createEvidence(undefined, provider, {
predicateType: browserPredicate(operation),
operation,
parameters: browserParameters(input, result),
parameters: {
...browserParameters(input, result),
...(comparison
? {
comparison_input_sha256: digestCanonicalValue(
input,
"Browser comparison input",
),
}
: {}),
},
result,
confidence: "observed",
authority: "external-service",
confidence: comparison ? "derived" : "observed",
authority: comparison ? "analyst-inference" : "external-service",
environment:
"browser" in observed
? {
src/application/BrowserObservationService.test.ts
+59
View File
@@ -11,6 +11,7 @@ import {
AnalysisCancelledError,
AnalysisOutputError,
} from "../domain/analysisErrorCore.js";
import { resolvedAnalysisError } from "../domain/analysisErrorBase.js";
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
import {
expectDroppedPartial,
@@ -55,6 +56,21 @@ const provider = (
listTargets,
}) as unknown as BrowserObservationPort;
it("keeps live browser inventory Evidence observed with external-service authority", async () => {
const result = await listBrowserTargets(
provider(async () => ok(report)),
input,
);
expect(result).toMatchObject({
ok: true,
value: { confidence: "observed", authority: "external-service" },
});
if (result.ok)
expect(result.value.parameters).not.toHaveProperty(
"comparison_input_sha256",
);
});
it("retains completed browser inventory Evidence when the provider completes after abort", async () => {
const controller = new AbortController();
const result = await listBrowserTargets(
@@ -135,6 +151,49 @@ it("converts a raw page inspection retained after browser cleanup failure", asyn
}
});
it.each([true, false])(
"promotes cleanup primary facts once when wrapper retains them: %s",
async (wrapperRetainsFacts) => {
const inspection = passiveNetworkCapture();
const primary = new AnalysisCancelledError("inspect_web_page", {
partialObservation: inspection,
});
const failure = new BrowserObservationError(
"inspect_web_page",
"cleanup_failed",
{
cleanupPrimary: primary,
cleanup: {
reason: "target detach failed",
resources: ["browser target"],
},
...(wrapperRetainsFacts ? { partialObservation: inspection } : {}),
},
);
const browser = {
...provider(async () => ok(report)),
inspectPage: async () => err(failure),
} as unknown as BrowserObservationPort;
const result = await inspectWebPage(browser, pageInput());
expect(result.ok).toBe(false);
expect(parseEvidence(primary.partialObservation)).toMatchObject({
operation: "inspect_web_page",
normalized_result: inspection,
});
if (wrapperRetainsFacts)
expect(failure.partialObservation).toBe(primary.partialObservation);
failure.resolveCleanup();
expect(resolvedAnalysisError(failure)).toBe(primary);
expect(
parseEvidence(
projectAnalysisError(resolvedAnalysisError(failure)).details?.[
"partial_observation"
],
),
).toEqual(primary.partialObservation);
},
);
it("retains captured page Evidence when bundle analysis fails after capture", async () => {
const inspection = passiveNetworkCapture();
const failure = new BrowserObservationError(
src/application/BrowserObservationService.ts
+37 -17
View File
@@ -288,44 +288,71 @@ const retainBrowserObservation = (
provider: BrowserObservationPort,
errorOperation: "inspect_web_page" | "analyze_web_bundle",
): Result<Evidence, AnalysisError> => {
const partial = error.partialObservation;
const seen = new Set<AnalysisError>();
const promoted = new Map<object, Evidence>();
let current: AnalysisError | undefined = error;
while (current !== undefined && !seen.has(current)) {
seen.add(current);
const partial = current.partialObservation;
if (partial !== undefined) {
const evidence =
promoted.get(partial) ??
promoteBrowserObservation(partial, input, provider, errorOperation);
if (evidence !== undefined) {
promoted.set(partial, evidence);
current.partialObservation = evidence;
}
}
current = current.cleanupPrimary;
}
return err(
keepIncompleteObservation(
error,
errorOperation,
NON_JSON_BROWSER_OBSERVATION,
),
);
};
/** Each cleanup owner can become the selected error after cleanup succeeds. */
const promoteBrowserObservation = (
partial: NonNullable<AnalysisError["partialObservation"]>,
input: InspectWebPageInput | AnalyzeWebBundleInput,
provider: BrowserObservationPort,
errorOperation: "inspect_web_page" | "analyze_web_bundle",
): Evidence | undefined => {
if (
errorOperation === "analyze_web_bundle" &&
partial !== undefined &&
"kind" in partial &&
partial.kind === "web-bundle-acquisition"
) {
const acquisition =
webBundleAcquisitionPartialObservationSchema.safeParse(partial);
if (acquisition.success) {
error.partialObservation = createBrowserEvidence(
return createBrowserEvidence(
"analyze_web_bundle",
input,
acquisition.data,
provider.identity(),
);
return err(error);
}
}
if (
errorOperation === "analyze_web_bundle" &&
partial !== undefined &&
!("evidence_id" in partial) &&
"observations" in partial &&
"capture" in partial
) {
const bundle = webBundleAnalysisSchema.safeParse(partial);
if (bundle.success) {
error.partialObservation = createBrowserEvidence(
return createBrowserEvidence(
"analyze_web_bundle",
input,
bundle.data,
provider.identity(),
);
return err(error);
}
} else if (
partial !== undefined &&
!("evidence_id" in partial) &&
"target" in partial &&
"network" in partial &&
@@ -337,20 +364,13 @@ const retainBrowserObservation = (
"fetch_source_maps" in input
? (({ fetch_source_maps: _fetchSourceMaps, ...page }) => page)(input)
: input;
error.partialObservation = createBrowserEvidence(
return createBrowserEvidence(
"inspect_web_page",
pageInput,
inspection.data,
provider.identity(),
);
return err(error);
}
}
return err(
keepIncompleteObservation(
error,
errorOperation,
NON_JSON_BROWSER_OBSERVATION,
),
);
return undefined;
};
src/application/BrowserScenarioEvidence.test.ts
+63
View File
@@ -0,0 +1,63 @@
import { describe, expect, it } from "vitest";
import { scenarioCaptureFixture } from "../../tests/fixtures/browserScenarioCapture.js";
import { browserScenarioSchema } from "../domain/browserScenario.js";
import { digestCanonicalValue } from "../domain/canonicalDigest.js";
import { createBrowserScenarioEvidence } from "./BrowserScenarioEvidence.js";
const ORIGIN = "http://127.0.0.1:3101";
const parameters = (actions: readonly unknown[]) =>
createBrowserScenarioEvidence(
browserScenarioSchema.parse({
browser: { mode: "launch", executable_path: "/opt/chromium/chrome" },
start_url: { url: `${ORIGIN}/` },
actions,
}),
scenarioCaptureFixture(),
{ id: "fixture", name: "fixture", version: "1" },
).parameters;
const open = (fields: Record<string, unknown> = {}) => ({
step_id: "open",
action: "goto",
destination: { url: `${ORIGIN}/items` },
wait_until: "load",
...fields,
});
const create = (fields: Record<string, unknown> = {}) => ({
step_id: "create",
action: "request",
method: "POST",
path: "/api/items",
json: { title: "Item 1" },
...fields,
});
describe("browser scenario evidence parameters", () => {
it("records a scenario that saves nothing as it was recorded before saved values existed", () => {
const recorded = parameters([open(), create()]);
const actions = recorded["actions"] as Record<string, unknown>[];
expect(Object.keys(actions[1]!)).not.toContain("save");
expect(parameters([open(), create({ save: [] })])).toEqual(recorded);
const { scenario_sha256, ...scenario } = recorded;
expect(scenario_sha256).toBe(
digestCanonicalValue(scenario, "Browser scenario"),
);
});
it("records the values a step saves and the placeholders that send them", () => {
const recorded = parameters([
create({ save: [{ name: "item", json_pointer: "/id" }] }),
open({ destination: { url: `${ORIGIN}/items/{saved:item}` } }),
]);
expect(recorded["actions"]).toMatchObject([
{ save: [{ name: "item", json_pointer: "/id", secret: false }] },
{ destination: { url: `${ORIGIN}/items/%7Bsaved:item%7D` } },
]);
expect(recorded["scenario_sha256"]).not.toBe(
parameters([create(), open()])["scenario_sha256"],
);
});
});
src/application/BrowserScenarioEvidence.ts
+19 -4
View File
@@ -9,12 +9,27 @@ import {
} from "../domain/evidence.js";
import { jsonObjectSchema } from "../domain/jsonValue.js";
/** A step as Evidence records it: a request step leaves out an empty `save`, so a scenario that saves nothing records and hashes as it did before saved values existed. */
const recordedAction = (
action: BrowserScenario["actions"][number],
): Record<string, unknown> => {
if (action.action !== "request" || action.save.length > 0) return action;
const { save, ...rest } = action;
return rest;
};
const browserScenarioParameters = (
scenario: BrowserScenario,
): EvidenceObservation["parameters"] => ({
...jsonObjectSchema.parse(scenario),
scenario_sha256: digestCanonicalValue(scenario, "Browser scenario"),
});
): EvidenceObservation["parameters"] => {
const recorded = {
...scenario,
actions: scenario.actions.map(recordedAction),
};
return {
...jsonObjectSchema.parse(recorded),
scenario_sha256: digestCanonicalValue(recorded, "Browser scenario"),
};
};
/** Create Evidence without retaining resolved scenario secret values. */
export const createBrowserScenarioEvidence = (
src/application/ClientRegistration.ts
+28 -1
View File
@@ -1,4 +1,5 @@
import { z } from "zod";
import { PRODUCT_IDENTITY } from "../identity.js";
import { jsonRecordSchema } from "../domain/jsonRecord.js";
import { ConfigurationError } from "../domain/configurationErrors.js";
@@ -15,7 +16,12 @@ import {
type ClientRegistrationDialect,
} from "./ClientConfigurationDocument.js";
import type { SetupClient } from "./SupportedClients.js";
import { equivalentClientRegistrationCommands } from "./ClientRegistrationIdentity.js";
import {
classifyClientRegistrationCommand,
equivalentClientRegistrationCommands,
type ClientRegistrationCommandIdentity,
} from "./ClientRegistrationIdentity.js";
import { CURRENT_ENTRYPOINT_PATH } from "../process/CurrentEntrypointPath.js";
const tomlTimeoutSchema = z.union([
z.number().positive(),
@@ -107,6 +113,27 @@ export const parseClientRegistrationLauncher = (
: err(registrationError(parsed.error));
};
/** Launcher ownership is independent of readiness and caller-owned policy. */
export const clientRegistrationOwnership = (
value: unknown,
client: SetupClient,
): ClientRegistrationCommandIdentity => {
const parsed = parseClientRegistrationLauncher(value, client);
return parsed.ok
? classifyClientRegistrationCommand(
[parsed.value.command, ...parsed.value.args],
CURRENT_ENTRYPOINT_PATH,
)
: "unknown";
};
/** Explain the same ownership admission decision in setup and doctor. */
export const clientRegistrationOwnershipRemediation = (
ownership: Exclude<ClientRegistrationCommandIdentity, "owned">,
paths: readonly string[],
): string =>
`The existing "${PRODUCT_IDENTITY.mcpServerKey}" registration read from ${paths.join(", ")} has ${ownership} ownership and was not changed. Review its launcher and manually rename or remove that registration before rerunning setup. A different installation path does not prove REA ownership.`;
/** Policy belongs to the client dialect, and is validated before refresh writes. */
const validateClientPolicy = (
value: unknown,
src/application/ClientRegistrationStatus.ts
+7 -1
View File
@@ -41,6 +41,7 @@ import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.j
import {
parseClientRegistration,
clientRegistrationOwnershipRemediation,
setupPreservesDisabledActivation,
type ClientRegistration,
} from "./ClientRegistration.js";
@@ -332,7 +333,12 @@ export const readClientRegistrationInspections = async (
policyBlock === undefined
? "aligned"
: "stale",
policyBlock ??
(parsed.dialect !== "dsh" && commandFacts.identity !== "owned"
? clientRegistrationOwnershipRemediation(commandFacts.identity, [
configPath,
])
: undefined) ??
policyBlock ??
(packageVersionDrift
? packageVersionRemediation(packageReference ?? "", client)
: alignment === "unknown"
src/application/DirectAnalysis.ts
+46 -59
View File
@@ -8,6 +8,10 @@ import type { JsonValue } from "../domain/jsonValue.js";
import type { DirectAnalysisDependencies } from "./DirectAnalysisDependencies.js";
import type { BinarySession } from "./binary/BinarySession.js";
import type { SessionProviderRoute } from "./binary/SessionProviderRouter.js";
import {
routeAllowsSnapshotReplay,
routeProviderIdentity,
} from "./binary/SessionProviderPolicy.js";
import type { ResolvedSessionOpen } from "./binary/BinarySessionOpen.js";
import { silentLogger } from "../logger.js";
import type { Logger } from "pino";
@@ -128,7 +132,7 @@ export const runDirectAnalysis = async (
readonly architecture?: BinaryArchitecture;
readonly entryPoint?: string;
} = {},
): Promise<JsonValue> =>
): Promise<Result<Evidence, JsonValue>> =>
withProcessCancellation(options.signal, (signal) =>
runAnalysis(dependencies, path, tool, arguments_, {
logger: options.logger ?? silentLogger,
@@ -171,31 +175,35 @@ export const runProviderAnalysis = async (
readonly evidenceBundlePath?: string;
},
]
): Promise<JsonValue> => {
): Promise<Result<Evidence, JsonValue>> => {
if (delivery !== undefined && tool !== "inspect_managed_members")
return cliError(
new AnalysisInputError(tool, undefined, [
{
path: ["delivery"],
reason: "invalid_value",
message:
"Evidence delivery options are only supported by inspect_managed_members",
},
]),
return err(
cliError(
new AnalysisInputError(tool, undefined, [
{
path: ["delivery"],
reason: "invalid_value",
message:
"Evidence delivery options are only supported by inspect_managed_members",
},
]),
),
);
if (
delivery?.detail === "summary" &&
delivery.evidenceBundlePath === undefined
)
return cliError(
new AnalysisInputError(tool, undefined, [
{
path: ["evidence_bundle"],
reason: "missing_argument",
message:
"summary detail requires --evidence-bundle so complete Evidence remains available to later CLI commands",
},
]),
return err(
cliError(
new AnalysisInputError(tool, undefined, [
{
path: ["evidence_bundle"],
reason: "missing_argument",
message:
"summary detail requires --evidence-bundle so complete Evidence remains available to later CLI commands",
},
]),
),
);
return withProcessCancellation(signal, async (operationSignal) => {
// The selected CLI path is a request fact. Keep it out of provider inputs
@@ -211,14 +219,16 @@ export const runProviderAnalysis = async (
operationSignal,
);
await dispatchQueuedInterrupts();
if (!original.ok) return cliError(original.error);
if (!original.ok) return err(cliError(original.error));
const evidence = withEvidenceCallerInputPath(original.value, path);
if (operationSignal.aborted)
return cliError(
new AnalysisCancelledError(tool, {
cause: operationSignal.reason,
partialObservation: evidence,
}),
return err(
cliError(
new AnalysisCancelledError(tool, {
cause: operationSignal.reason,
partialObservation: evidence,
}),
),
);
const result =
delivery?.evidenceBundlePath === undefined
@@ -229,7 +239,7 @@ export const runProviderAnalysis = async (
delivery.detail,
operationSignal,
);
return result.ok ? result.value : cliError(result.error);
return result.ok ? ok(result.value) : err(cliError(result.error));
}
return runAnalysis(dependencies, path, tool, arguments_, {
logger,
@@ -317,7 +327,7 @@ const runAnalysis = async (
readonly architecture?: BinaryArchitecture;
readonly entryPoint?: string;
},
): Promise<JsonValue> => {
): Promise<Result<Evidence, JsonValue>> => {
const { logger, signal, snapshotPath } = options;
const evidenceParameters = options.evidenceParameters ?? arguments_;
let session: BinarySession;
@@ -325,7 +335,7 @@ const runAnalysis = async (
session = await options.createSession();
} else {
const config = dependencies.readConfiguration();
if (!config.ok) return cliError(config.error);
if (!config.ok) return err(cliError(config.error));
session = await (
tool === "inspect_native_dispatch_metadata" && snapshotPath === undefined
? dependencies.createNativeDispatchBinarySession
@@ -406,7 +416,9 @@ const runAnalysis = async (
!requiresNativeSourceAdmissionBeforeReplay(
route.binding?.identity.id,
) &&
allowsSnapshotReplay(route, tool)
tool !== "trace_native_ui_action" &&
tool !== "decompile_all" &&
routeAllowsSnapshotReplay(route, tool)
) {
const cached = snapshotEvidenceForQuery(snapshot, {
target: preview.value.target,
@@ -415,7 +427,7 @@ const runAnalysis = async (
parameters: arguments_,
provider: isWorkflowEvidenceTool(tool)
? REA_WORKFLOW_PROVIDER
: providerIdentityForRoute(route, tool),
: routeProviderIdentity(route, tool),
evidenceProfile,
});
if (cached !== undefined)
@@ -543,9 +555,9 @@ const runAnalysis = async (
outcome.cleanupError.retainDiagnostics({
analysis_snapshot: { ...publishedSnapshot },
});
return directAnalysisCleanupFailure(result, outcome.cleanupError);
return err(directAnalysisCleanupFailure(result, outcome.cleanupError));
}
return result.ok ? result.value : cliError(result.error);
return result.ok ? ok(result.value) : err(cliError(result.error));
};
const executeAnalysisTool = async (input: {
@@ -749,24 +761,6 @@ const inspectBundleResourceInput = async (
}
};
const allowsSnapshotReplay = (
route: SessionProviderRoute,
tool:
| NativeToolName
| ArtifactAnalysisOperation
| ManagedToolName
| DirectAnalysisTool,
): boolean => {
if (tool === "trace_native_ui_action" || tool === "decompile_all")
return false;
const descriptor = route.capabilities?.get(tool);
return descriptor === undefined
? ![...(route.capabilities?.values() ?? [])].some(
({ cachePolicy }) => cachePolicy === "live",
)
: descriptor.cachePolicy !== "live";
};
const analysisProfileForRoute = (
route: SessionProviderRoute,
tool:
@@ -788,7 +782,7 @@ const analysisProfileForRoute = (
}
if (isWorkflowEvidenceTool(tool))
return workflowAnalysisProfile(profile, tool);
const provider = providerIdentityForRoute(route, tool);
const provider = routeProviderIdentity(route, tool);
return provider.id === profile.provider.id ? profile : undefined;
};
@@ -800,13 +794,6 @@ const nativeDispatchBindingProfile = (
return profilelessNativeDispatchProfile(route);
};
/** Mirror BinarySession.providerIdentity for a route not opened yet. */
const providerIdentityForRoute = (
route: SessionProviderRoute,
operation: string,
): ProviderIdentity =>
route.capabilities?.get(operation)?.provider ?? route.identity;
const analysisProfileForEvidence = (
session: BinarySession,
tool:
src/application/DirectAnalysisCleanup.ts
+4 -1
View File
@@ -38,6 +38,7 @@ export const directAnalysisCleanupFailure = (
const cleanup = projectAnalysisError(error);
if (!result.ok) {
const primary = projectAnalysisError(result.error);
const combined = analysisErrorWithCleanupFailure(result.error, error);
return {
error: "Analysis failed",
...primary,
@@ -45,8 +46,10 @@ export const directAnalysisCleanupFailure = (
retryable: false,
remediation: cleanup.remediation,
details: {
resources: error.cleanupResources.slice(),
...primary.details,
// The domain composition owns the union of still-retained resources;
// neither projection may overwrite the other owner's cleanup state.
resources: [...combined.cleanupResources],
cleanup: "incomplete",
execution_failure: primary.details?.execution_failure ?? primary.code,
cleanup_error: cleanup,
src/application/Doctor.test.ts
+22 -16
View File
@@ -495,22 +495,28 @@ describe("doctor runtime check", () => {
);
});
it.each([undefined, "", "not-a-version", "14garbage"])(
"retains unknown host support for macOS probe value %s",
async (version) => {
const result = await runDoctor(
undefined,
host({ macosVersion: () => Promise.resolve(version) }),
);
expect(result.checks).toContainEqual(
expect.objectContaining({
name: "host",
ok: false,
classification: "unknown",
}),
);
},
);
it.each([
undefined,
"",
"not-a-version",
"14garbage",
"14.garbage",
"14.",
"14.0\n",
"99999999999999999",
])("retains unknown host support for macOS probe value %s", async (version) => {
const result = await runDoctor(
undefined,
host({ macosVersion: () => Promise.resolve(version) }),
);
expect(result.checks).toContainEqual(
expect.objectContaining({
name: "host",
ok: false,
classification: "unknown",
}),
);
});
it.skipIf(process.platform !== "darwin")(
"reads the macOS version without PATH utilities",
src/application/DoctorDiagnostics.ts
+2 -4
View File
@@ -3,6 +3,7 @@ import {
supportsJavaScriptRuntime,
} from "../domain/runtimeVersion.js";
import { hopperLauncherDiscovery } from "./HopperLauncherDiscovery.js";
import { macosMajorVersion } from "../domain/macosVersion.js";
import { LINUX_HOPPER_SETUP_HOSTS, supportsReaHost } from "./LinuxHopper.js";
import type {
DoctorCheck,
@@ -129,10 +130,7 @@ const hostCheck = async (host: DoctorHost): Promise<DoctorCheck> => {
host.platform === "darwin" ? await host.macosVersion() : undefined;
const linuxDistribution =
host.platform === "linux" ? await host.linuxDistribution() : undefined;
const macosMajor =
macosVersion !== undefined && /^\d+(?:\.\d+)*$/u.test(macosVersion)
? Number(macosVersion.split(".")[0])
: undefined;
const macosMajor = macosMajorVersion(macosVersion);
if (host.platform === "darwin" && macosMajor === undefined)
return {
name: "host",
src/application/EvidenceBundleCommands.ts
+119 -42
View File
@@ -31,26 +31,36 @@ import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
import {
readEvidenceBundle,
readImmutableEvidenceBundle,
readComparisonEvidenceBundles,
writeEvidenceBundle,
} from "./EvidenceBundleFiles.js";
import { inspectEvidenceBundle } from "./investigation/InspectEvidenceBundle.js";
import type { EvidenceRecordFilters } from "../domain/evidenceRecordSummary.js";
import { inspectAnalysisViewValidated } from "./analysisView/AnalysisViewService.js";
/** Validate a standalone bundle and report its complete record/revision counts. */
export const importEvidenceBundleCommand = async (
path: string,
signal?: AbortSignal,
): Promise<Result<JsonValue, AnalysisError>> => {
const loaded = await readEvidenceBundle(
path,
undefined,
undefined,
return withBundleReadResources(
async (resources) => {
const loaded = await readEvidenceBundle(
path,
signal,
resources,
"import_evidence_bundle",
);
if (!loaded.ok) return loaded;
return ok({
imported: loaded.value.records.length,
unknowns_added: loaded.value.unknowns.length,
total: loaded.value.records.length,
});
},
"import_evidence_bundle",
signal,
);
if (!loaded.ok) return loaded;
return ok({
imported: loaded.value.records.length,
unknowns_added: loaded.value.unknowns.length,
total: loaded.value.records.length,
});
};
/** Validate a source bundle and atomically export canonical bytes. */
@@ -58,19 +68,37 @@ export const exportEvidenceBundleCommand = async (
sourcePath: string,
outputPath: string,
overwrite: boolean,
signal?: AbortSignal,
): Promise<Result<JsonValue, AnalysisError>> => {
// The sealed snapshot lets the writer reuse this authentication.
const loaded = await readImmutableEvidenceBundle(
sourcePath,
undefined,
undefined,
const loaded = await withBundleReadResources(
(resources) =>
readImmutableEvidenceBundle(
sourcePath,
signal,
resources,
"export_evidence_bundle",
),
"export_evidence_bundle",
signal,
);
if (!loaded.ok) return loaded;
return projectWrite(
const written = await persistEvidenceBundleCommand(
loaded.value,
await persistEvidenceBundleCommand(loaded.value, outputPath, overwrite),
outputPath,
overwrite,
signal,
"export_evidence_bundle",
);
await dispatchQueuedInterrupts();
if (written.ok && signal?.aborted)
return err(
new AnalysisCancelledError("export_evidence_bundle", {
cause: signal.reason,
diagnostics: { evidence_bundle: { ...written.value } },
}),
);
return projectWrite(loaded.value, written);
};
/** Publish complete parent Evidence before delivering it or its linked summary. */
@@ -223,35 +251,32 @@ export const compareEvidenceBundlesCommand = async (input: {
readonly leftPath: string;
readonly rightPath: string;
readonly recordPairs?: readonly BundleRecordPair[];
readonly signal?: AbortSignal;
}): Promise<Result<JsonValue, AnalysisError>> => {
// Sealed snapshots let comparison and its bundle digests reuse this
// authentication instead of re-validating every record twice more.
const [left, right] = await Promise.all([
readImmutableEvidenceBundle(
input.leftPath,
undefined,
undefined,
"compare_bundles",
),
readImmutableEvidenceBundle(
input.rightPath,
undefined,
undefined,
"compare_bundles",
),
]);
if (!left.ok) return left;
if (!right.ok) return right;
try {
return ok(
jsonValueSchema.parse(
compareBundles(left.value, right.value, input.recordPairs),
),
);
} catch (cause: unknown) {
if (cause instanceof AnalysisInputError) return err(cause);
throw cause;
}
return withBundleReadResources(
async (resources) => {
const loaded = await readComparisonEvidenceBundles(
input.leftPath,
input.rightPath,
input.signal,
resources,
);
if (!loaded.ok) return loaded;
const { left, right } = loaded.value;
try {
return ok(
jsonValueSchema.parse(compareBundles(left, right, input.recordPairs)),
);
} catch (cause: unknown) {
if (cause instanceof AnalysisInputError) return err(cause);
throw cause;
}
},
"compare_bundles",
input.signal,
);
};
const projectWrite = (
@@ -265,3 +290,55 @@ const projectWrite = (
records: bundle.records.length,
})
: err(written.error);
/** Standalone bundle discovery shares the same read owner as import and comparison. */
export const inspectEvidenceBundleCommand = async (
input: {
readonly path: string;
readonly detail: "complete" | "summary";
readonly filters?: EvidenceRecordFilters | undefined;
},
signal?: AbortSignal,
) => {
return withBundleReadResources(
async (resources) => {
const loaded = await readEvidenceBundle(
input.path,
signal,
resources,
"get_evidence_bundle",
);
return loaded.ok ? inspectEvidenceBundle(loaded.value, input) : loaded;
},
"get_evidence_bundle",
signal,
);
};
/** Final read cleanup retries the exact leases already bound to a failed read. */
const withBundleReadResources = async <Value extends JsonValue>(
read: (
resources: ArtifactResourceScope,
) => Promise<Result<Value, AnalysisError>>,
operation: string,
signal?: AbortSignal,
): Promise<Result<Value, AnalysisError>> => {
const resources = new ArtifactResourceScope();
let result: Result<Value, AnalysisError>;
try {
result = await read(resources);
} finally {
// Read errors own their retained leases and observe resolution or remaining
// failure in place. A retry must not replace the original read failure.
await resources.close().catch(() => undefined);
await dispatchQueuedInterrupts();
}
return result.ok && signal?.aborted
? err(
new AnalysisCancelledError(operation, {
cause: signal.reason,
diagnostics: { completed_result: result.value },
}),
)
: result;
};
src/application/EvidenceBundleFiles.ts
+34
View File
@@ -1,3 +1,5 @@
import type { AnalysisError } from "../domain/analysisErrorBase.js";
import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js";
import { requireAuthenticationHeadroom } from "./JsonAuthenticationHeadroom.js";
import { absoluteCallerPath } from "../filesystem/callerPath.js";
@@ -52,6 +54,38 @@ export const readImmutableEvidenceBundle = (
): Promise<Result<EvidenceBundle, EvidenceReadFailure>> =>
readBundle(path, true, signal, resources, operation);
/** Join both comparison reads and preserve cleanup owned by either failed input. */
export const readComparisonEvidenceBundles = async (
leftPath: string,
rightPath: string,
signal?: AbortSignal,
resources?: ArtifactResourceScope,
operation = "compare_bundles",
): Promise<
Result<
{ readonly left: EvidenceBundle; readonly right: EvidenceBundle },
AnalysisError
>
> => {
const [left, right] = await Promise.all([
readImmutableEvidenceBundle(leftPath, signal, resources, operation),
readImmutableEvidenceBundle(rightPath, signal, resources, operation),
]);
if (!left.ok) {
if (!right.ok && right.error.cleanupIncomplete)
return err(
analysisErrorWithCleanupFailure(
left.error,
right.error,
operation,
).bindCleanupDependencies([left.error, right.error]),
);
return left;
}
if (!right.ok) return right;
return ok({ left: left.value, right: right.value });
};
const readBundle = async (
path: string,
immutable: boolean,
src/application/FunctionAnalysisEvidence.test.ts
+45 -1
View File
@@ -7,6 +7,7 @@ import { createAnalysisProfile } from "../domain/analysisProfile.js";
import { ghidraFunctionDossier } from "../domain/ghidraValues.fixture.js";
import type { BinaryTarget } from "../domain/binaryTargetTypes.js";
import { err, ok } from "../domain/result.js";
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
const provider = {
id: "dossier-fixture",
@@ -157,6 +158,49 @@ it.each(["provider-failure", "invalid-dossier"] as const)(
_tag: "AnalysisOutputError",
operation: "analyze_function",
});
expect(result.error.partialObservation).toBeUndefined();
if (kind === "provider-failure")
expect(result.error.partialObservation).toBeUndefined();
else {
const retained = parseEvidence(result.error.partialObservation);
expect(retained.normalized_result).toEqual({ procedure: "incomplete" });
expect(retained.subject?.local_path).toBe(target.path);
}
},
);
it("retains malformed dossier facts and source links through public error projection", async () => {
const source = createEvidence(undefined, provider, {
operation: "health",
parameters: {},
result: "retained source",
});
const malformed = { procedure: "incomplete", partial: ["caller fact"] };
const result = await executeFunctionAnalysisEvidence(
{
execute: async () =>
ok(
createAnalysisExecution(malformed, provider, {
subject: null,
rawResult: { text: "original provider output" },
limitations: ["Incomplete provider response"],
locations: [{ kind: "address", address: "0x401000" }],
sourceEvidence: [source],
}),
),
},
parameters,
target,
);
if (result.ok) throw new Error("Expected invalid dossier failure");
expect(result.sourceEvidence).toEqual([source]);
const projected = projectAnalysisError(result.error);
const retained = parseEvidence(projected.details?.partial_observation);
expect(retained.normalized_result).toEqual(malformed);
expect(retained.raw_result).toEqual({ text: "original provider output" });
expect(retained.subject).toBeNull();
expect(retained.evidence_links).toEqual([source.evidence_id]);
expect(retained.locations).toEqual([
{ kind: "address", address: "0x401000" },
]);
expect(retained.limitations).toContain("Incomplete provider response");
});
src/application/FunctionAnalysisEvidence.ts
+18 -6
View File
@@ -49,9 +49,25 @@ export const executeFunctionAnalysisEvidence = async (
);
const sources = sourceLedger.recordsForAnalysis();
if (!recorded.ok) return withEvidenceSources(recorded, sources);
const observation = {
...execution.value,
sourceEvidence: sources,
...(execution.value.subject === undefined && target !== undefined
? { subject: target }
: {}),
};
const dossier = parseFunctionDossier(execution.value.result);
if (!dossier.ok) return withEvidenceSources(dossier, sources);
const observation = execution.value;
if (!dossier.ok) {
dossier.error.retainPartialObservation(
createAnalysisExecutionEvidence(
"analyze_function",
arguments_,
observation,
"observed",
),
);
return withEvidenceSources(dossier, sources);
}
return withEvidenceSources(
completeProviderEvidence(
createAnalysisExecutionEvidence(
@@ -59,11 +75,7 @@ export const executeFunctionAnalysisEvidence = async (
arguments_,
{
...observation,
sourceEvidence: sources,
result: dossier.value,
...(observation.subject === undefined && target !== undefined
? { subject: target }
: {}),
},
"observed",
),
src/application/HermesClientHome.ts
+49 -22
View File
@@ -1,14 +1,10 @@
import { lstatSync, readFileSync, realpathSync, statSync } from "node:fs";
import { userInfo, type UserInfo } from "node:os";
import { basename, dirname, isAbsolute, join, relative, sep } from "node:path";
import {
basename,
dirname,
isAbsolute,
join,
relative,
resolve,
sep,
} from "node:path";
absoluteCallerPath,
appendCallerPath,
} from "../filesystem/callerPath.js";
/** An unresolved Hermes profile must not be configured as the default home. */
export class HermesClientPathError extends Error {
@@ -32,7 +28,7 @@ const defaultDirectory = (
env.LOCALAPPDATA?.trim() || join(home, "AppData", "Local"),
`hermes${suffix}`,
)
: join(home, `.hermes${suffix}`);
: appendCallerPath(home, `.hermes${suffix}`, platform);
};
const expandedDirectory = (
home: string,
@@ -60,7 +56,9 @@ const expandedDirectory = (
expanded.startsWith("~/") ||
(platform === "win32" && expanded.startsWith("~\\"))
)
return join(home, expanded.slice(2));
return absoluteCallerPath(
appendCallerPath(home, expanded.slice(2), platform),
);
if (expanded.startsWith("~")) {
// Named POSIX homes come from the account database, not HOME/USERPROFILE.
// Only the current account has a portable Node API; unresolved names must
@@ -78,23 +76,52 @@ const expandedDirectory = (
);
}
if (account.username === username && isAbsolute(account.homedir))
return join(
account.homedir,
separator < 0 ? "" : expanded.slice(separator + 1),
);
return separator < 0
? account.homedir
: absoluteCallerPath(
appendCallerPath(account.homedir, expanded.slice(separator + 1)),
);
}
throw new HermesClientPathError(
override,
"Cannot resolve this named Hermes home. Set an absolute HERMES_HOME before configuring Hermes.",
);
}
return resolve(expanded);
// Hermes expands environment/user syntax and then opens the resulting Path;
// preserve POSIX symlink/.. traversal when making a relative override absolute.
return absoluteCallerPath(expanded);
};
// Resolve existing directory aliases even when setup will create the final home.
// An unreadable path or dangling alias cannot prove root identity.
const physicalPath = (path: string): string => {
let ancestor = absoluteCallerPath(path);
const missing: string[] = [];
try {
return realpathSync(path);
while (true) {
try {
return join(realpathSync(ancestor), ...missing);
} catch (cause: unknown) {
if (
!(cause instanceof Error) ||
!("code" in cause) ||
cause.code !== "ENOENT" ||
dirname(ancestor) === ancestor ||
basename(ancestor) === ".."
)
throw cause;
// ENOENT may mean a dangling symlink rather than an absent entry. Do
// not reclassify such an alias as a literal child of its lexical parent.
if (lstatSync(ancestor, { throwIfNoEntry: false })?.isSymbolicLink())
throw cause;
missing.unshift(basename(ancestor));
ancestor = dirname(ancestor);
}
}
} catch {
return resolve(path);
throw new HermesClientPathError(
path,
"Cannot resolve the Hermes home root. Restore access to this path before configuring Hermes.",
);
}
};
const within = (root: string, path: string): boolean => {
@@ -119,14 +146,14 @@ const liveProfile = (path: string): boolean => {
try {
if (!statSync(path).isDirectory()) return false;
try {
statSync(join(dirname(path), ".deleted", basename(path)));
statSync(appendCallerPath(dirname(path), `.deleted/${basename(path)}`));
return false;
} catch {
/* No tombstone. */
}
return PROFILE_MARKERS.some((marker) => {
try {
const status = lstatSync(join(path, marker));
const status = lstatSync(appendCallerPath(path, marker));
return status.isFile() || status.isSymbolicLink();
} catch {
return false;
@@ -152,7 +179,7 @@ export const resolveHermesClientHome = (
try {
// Python uses utf-8-sig and ignores unreadable / invalid-UTF8 active_profile files.
name = new TextDecoder("utf-8", { fatal: true })
.decode(readFileSync(join(stickyRoot, "active_profile")))
.decode(readFileSync(appendCallerPath(stickyRoot, "active_profile")))
.trim();
} catch {
return root;
@@ -165,12 +192,12 @@ export const resolveHermesClientHome = (
RESERVED_PROFILES.has(canonical)
)
throw new HermesClientPathError(
join(stickyRoot, "active_profile"),
appendCallerPath(stickyRoot, "active_profile"),
`Invalid saved Hermes profile ${JSON.stringify(name)}. Run 'hermes profile use default' or repair active_profile.`,
);
// Hermes resolves both sticky selection and named profiles from its default
// root (including overrides beneath the native home), never mixed roots.
const selected = join(stickyRoot, "profiles", canonical);
const selected = appendCallerPath(stickyRoot, `profiles/${canonical}`);
if (!liveProfile(selected))
throw new HermesClientPathError(
selected,
src/application/HopperReleaseFeed.test.ts
+32
View File
@@ -121,3 +121,35 @@ it("distinguishes malformed UTF-8 from a genuine replacement character and toler
parseHopperReleaseJson(new TextEncoder().encode(bom + '{"x":"雪�"}')),
).toMatchObject({ ok: true, value: { x: "雪�" } });
});
describe("macOS Hopper public download channel", () => {
const macRelease = {
filename: "https://www.hopperapp.com:443/downloader/public/Hopper-demo.dmg",
file_length: "123",
file_hash: "a".repeat(40),
};
it.each([
"https://www.hopperapp.com/downloader/public/Hopper-demo.dmg",
"https://WWW.HOPPERAPP.COM:443/downloader/public/Hopper-demo.dmg",
"https://www.hopperapp.com:443/downloader/public/Hopper%20demo.dmg",
])("preserves the admitted source spelling %s", (filename) => {
expect(
parseMacHopperReleases({ "OS X": { ...macRelease, filename } }),
).toEqual({ ok: true, value: { ...macRelease, filename } });
});
it.each([
"https://www.hopperapp.com:443/downloader/public/../private/Hopper-demo.dmg",
"https://www.hopperapp.com:443/downloader/public/%2e%2e/private/Hopper-demo.dmg",
"https://www.hopperapp.com:443/downloader/public/Hopper-demo.dmg?",
"https://www.hopperapp.com:443/downloader/public/Hopper-demo.dmg#",
"https://user:password@www.hopperapp.com:443/downloader/public/Hopper-demo.dmg",
"https://other.example/downloader/public/Hopper-demo.dmg",
"http://www.hopperapp.com/downloader/public/Hopper-demo.dmg",
])("rejects a URL outside the public channel: %s", (filename) => {
expect(
parseMacHopperReleases({ "OS X": { ...macRelease, filename } }),
).toMatchObject({ ok: false, error: { reason: "release_metadata" } });
});
});
src/application/HopperReleaseFeed.ts
+10 -5
View File
@@ -52,6 +52,14 @@ const selectedPlatformsSchema = z.object({
Arch: z.unknown().optional(),
});
/** Interpret channel identity once using the same URL semantics as the downloader. */
const isPublicHopperDownload = (url: URL): boolean =>
url.origin === "https://www.hopperapp.com" &&
url.username === "" &&
url.password === "" &&
!urlDeclaresQueryOrFragment(url) &&
url.pathname.startsWith("/downloader/public/");
/** Admit the selected macOS row without projecting away vendor release metadata. */
export const parseMacHopperReleases = (
raw: unknown,
@@ -68,7 +76,7 @@ export const parseMacHopperReleases = (
),
);
const retained = release.data as MacHopperRelease;
if (!retained.filename.startsWith(HOPPER_DOWNLOAD_PREFIX))
if (!isPublicHopperDownload(new URL(retained.filename)))
return err(
new HopperReleaseError(
"release_metadata",
@@ -154,10 +162,7 @@ export const parseLinuxHopperRelease = (
);
const url = new URL(parsed.data.filename);
if (
url.origin !== "https://www.hopperapp.com" ||
url.username !== "" ||
url.password !== "" ||
urlDeclaresQueryOrFragment(url) ||
!isPublicHopperDownload(url) ||
!url.pathname.startsWith("/downloader/public/Hopper-") ||
!url.pathname.endsWith(`-Linux-demo${suffixes[family]}`)
)
src/application/LinuxHopper.ts
+2 -1
View File
@@ -1,3 +1,4 @@
import { appendCallerPath } from "../filesystem/callerPath.js";
import type { AnalysisCleanupObservation } from "../domain/analysisErrorBase.js";
import type { HopperInstallFailureReason } from "./SetupInstallFailure.js";
import { access, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
@@ -525,7 +526,7 @@ export const installLinuxHopper = async (
/** Canonical launcher path for a legacy user-local Linux Hopper installation. */
export const linuxHopperLauncherPath = (home: string): string =>
posix.join(home, ".local/share/rea/hopper/bin/Hopper");
appendCallerPath(home, ".local/share/rea/hopper/bin/Hopper", "linux");
/** REA's managed Linux install path, preferred when no explicit launcher is set. */
export const REA_MANAGED_LINUX_HOPPER_LAUNCHER = "/opt/hopper/bin/Hopper";
src/application/LinuxHopperCacheDirectory.ts
+5 -5
View File
@@ -1,4 +1,5 @@
import { isAbsolute, join } from "node:path";
import { isAbsolute } from "node:path";
import { appendCallerPath } from "../filesystem/callerPath.js";
import { homeDirectoryFromEnvironment } from "../config/homeDirectory.js";
import { err, ok, type Result } from "../domain/result.js";
import { HopperReleaseError } from "./HopperReleaseFeed.js";
@@ -18,14 +19,13 @@ export const linuxHopperReleaseCacheDirectory = (
),
);
return ok(
join(
appendCallerPath(
supplied ??
join(
appendCallerPath(
selectedHome ?? homeDirectoryFromEnvironment(environment, "linux"),
".cache",
),
"rea",
"hopper-releases",
"rea/hopper-releases",
),
);
} catch (cause: unknown) {
src/application/LinuxHopperPublisher.ts
+13 -6
View File
@@ -1,6 +1,7 @@
import { createHash, randomUUID } from "node:crypto";
import { copyFile, lstat, mkdir, rename, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { appendCallerPath } from "../filesystem/callerPath.js";
import { z } from "zod";
import {
@@ -299,9 +300,13 @@ export class LinuxHopperPublisher {
"verification_unavailable",
"The Hopper release cache is not a caller-owned directory.",
);
const target = join(this.cacheRoot, verified.release.main_exe_hash);
const target = appendCallerPath(
this.cacheRoot,
verified.release.main_exe_hash,
);
const lease = await this.#acquireCacheLease(
verified.release.main_exe_hash,
resources,
signal,
);
await this.#withOwnedResource(
@@ -408,6 +413,7 @@ export class LinuxHopperPublisher {
async #acquireCacheLease(
digest: string,
resources: ArtifactResourceScope,
signal?: AbortSignal,
): Promise<OwnedLease> {
const deadline = performance.now() + 120_000;
@@ -419,6 +425,7 @@ export class LinuxHopperPublisher {
cacheDirectory: this.cacheRoot,
launcherSha256: digest,
runId,
resources,
});
if (acquired.acquired) return acquired.lease;
const remaining = deadline - performance.now();
@@ -699,7 +706,7 @@ export class LinuxHopperPublisher {
resources: ArtifactResourceScope,
): Promise<Result<VerifiedLinuxHopperPackage, HopperReleaseError>> {
try {
const root = join(this.cacheRoot, digest);
const root = appendCallerPath(this.cacheRoot, digest);
try {
const directory = await lstat(root);
if (!directory.isDirectory() || directory.isSymbolicLink())
@@ -722,9 +729,9 @@ export class LinuxHopperPublisher {
// Distinguish proven malformed/missing evidence from unavailable reads or
// inspection tools. Only the former authorize approved cache replacement.
for (const name of ["release.json", "public-key.asc", "archive"])
await this.#assertCacheFile(join(root, name));
await this.#assertCacheFile(appendCallerPath(root, name));
const metadata = await this.#read(
join(root, "release.json"),
appendCallerPath(root, "release.json"),
RELEASE_METADATA_BYTES,
signal,
resources,
@@ -753,7 +760,7 @@ export class LinuxHopperPublisher {
"The retained Hopper metadata does not identify this launcher.",
);
const keyBytes = await this.#read(
join(root, "public-key.asc"),
appendCallerPath(root, "public-key.asc"),
RELEASE_METADATA_BYTES,
signal,
resources,
@@ -771,7 +778,7 @@ export class LinuxHopperPublisher {
return await this.#verifyArchive(
parsed.data.family,
release.value,
join(root, "archive"),
appendCallerPath(root, "archive"),
signal,
key,
resources,
src/application/MacHopper.test.ts
+16
View File
@@ -307,3 +307,19 @@ it.each([false, true])(
expect(host.installs).toBe(1);
},
);
it("refuses release URLs that normalize outside the public channel before package download", async () => {
const host = new FakeMacHopperHost();
const feed = await host.download("files-api");
const row = JSON.parse(new TextDecoder().decode(feed.bytes));
row["OS X"].filename =
"https://www.hopperapp.com:443/downloader/public/%2e%2e/private/Hopper-demo.dmg";
host.feedBytes = new TextEncoder().encode(JSON.stringify(row));
expect(await installMacHopper({}, host)).toMatchObject({
status: "failed",
reason: "release_metadata",
});
expect(host.packageDownloads).toBe(0);
expect(host.mounts).toBe(0);
});
src/application/MacHopper.ts
+16 -5
View File
@@ -12,6 +12,7 @@ import { tmpdir } from "node:os";
import { join, posix } from "node:path";
import { promisify } from "node:util";
import { appendCallerPath } from "../filesystem/callerPath.js";
import { homeDirectoryFromEnvironment } from "../config/homeDirectory.js";
import type { AnalysisCleanupObservation } from "../domain/analysisErrorBase.js";
import { HOPPER_PUBLISHER } from "../domain/hopperPublisher.js";
@@ -194,7 +195,11 @@ export const installMacHopper = async (
: { cleanupFailure: installation.cleanupFailure }),
};
bundleCleanupFailure = installation.cleanupFailure;
const launcherPath = posix.join(destination, "Contents/MacOS/hopper");
const launcherPath = appendCallerPath(
destination,
"Contents/MacOS/hopper",
"darwin",
);
let openFailure: string | undefined;
try {
await host.openApplication(destination);
@@ -260,8 +265,12 @@ export const installMacHopperBundle = async (
const stagingRoot = await operations.createStagingRoot(
posix.dirname(destination),
);
const stagedBundle = posix.join(stagingRoot, "new.app");
const previousBundle = posix.join(stagingRoot, "previous.app");
const stagedBundle = appendCallerPath(stagingRoot, "new.app", "darwin");
const previousBundle = appendCallerPath(
stagingRoot,
"previous.app",
"darwin",
);
let originalBackedUp = false;
let newCommitted = false;
let failureReason: "copy" | "launcher_missing" = "copy";
@@ -270,7 +279,7 @@ export const installMacHopperBundle = async (
await operations.copyBundle(source, stagedBundle);
failureReason = "launcher_missing";
await operations.requireLauncher(
posix.join(stagedBundle, "Contents/MacOS/hopper"),
appendCallerPath(stagedBundle, "Contents/MacOS/hopper", "darwin"),
);
failureReason = "copy";
if (replaceExisting) {
@@ -428,7 +437,9 @@ export const systemMacHopperInstallHost = (
{
async createStagingRoot(parent) {
await mkdir(parent, { recursive: true });
return mkdtemp(posix.join(parent, ".rea-hopper-stage-"));
return mkdtemp(
appendCallerPath(parent, ".rea-hopper-stage-", "darwin"),
);
},
copyBundle: async (bundle, staged) => {
await execFileAsync("ditto", [bundle, staged], {
src/application/MacHopperPaths.test.ts
+101 -1
View File
@@ -1,11 +1,24 @@
import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
import {
access,
mkdtemp,
mkdir,
readFile,
rename,
rm,
symlink,
writeFile,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, posix, relative } from "node:path";
import { installMacHopperBundle } from "./MacHopper.js";
import { linuxHopperLauncherPath } from "./LinuxHopper.js";
import { homeDirectoryFromEnvironment } from "../config/homeDirectory.js";
import { describe, expect, it } from "vitest";
import {
localMacHopperRenamedLauncherCandidates,
macHopperCanonicalLauncherCandidates,
macHopperUserApplicationPath,
macHopperRenamedLauncherCandidates,
readMacHopperRenamedLauncherCandidates,
} from "./MacHopperPaths.js";
@@ -36,6 +49,93 @@ describe("macOS Hopper launcher paths", () => {
]);
});
it.skipIf(process.platform === "win32")(
"preserves a selected home through symlink parent traversal",
async () => {
const temporary = await mkdtemp(join(tmpdir(), "rea-hopper-home-"));
try {
await mkdir(join(temporary, "actual", "nested"), { recursive: true });
await symlink(
join(temporary, "actual", "nested"),
join(temporary, "alias"),
);
const home = homeDirectoryFromEnvironment(
{ HOME: `${temporary}/alias/..` },
"darwin",
);
await mkdir(
join(temporary, "actual", "Applications", "Hopper Preview.app"),
{ recursive: true },
);
const expected = `${home}/Applications/Hopper Preview.app/Contents/MacOS/hopper`;
expect(
[...localMacHopperRenamedLauncherCandidates(home)].filter((path) =>
path.startsWith(home),
),
).toEqual([expected]);
expect(
(await readMacHopperRenamedLauncherCandidates(home)).filter((path) =>
path.startsWith(home),
),
).toEqual([expected]);
expect(macHopperUserApplicationPath(home)).toBe(
`${home}/Applications/Hopper Disassembler.app`,
);
expect(macHopperCanonicalLauncherCandidates(home)[1]).toBe(
`${home}/Applications/Hopper Disassembler.app/Contents/MacOS/hopper`,
);
expect(linuxHopperLauncherPath(home)).toBe(
`${home}/.local/share/rea/hopper/bin/Hopper`,
);
const source = join(temporary, "source.app");
await mkdir(join(source, "Contents/MacOS"), { recursive: true });
await writeFile(
join(source, "Contents/MacOS/hopper"),
"inert launcher",
);
const result = await installMacHopperBundle(
source,
macHopperUserApplicationPath(home),
false,
{
createStagingRoot: (parent) =>
mkdtemp(`${parent}/.rea-hopper-stage-`),
copyBundle: async (from, to) => {
await mkdir(`${to}/Contents/MacOS`, { recursive: true });
await writeFile(
`${to}/Contents/MacOS/hopper`,
await readFile(`${from}/Contents/MacOS/hopper`),
);
},
requireLauncher: (path) => access(path),
pathExists: async (path) =>
access(path).then(
() => true,
() => false,
),
rename,
removeStagingRoot: (path) => rm(path, { recursive: true }),
},
() => {
throw new Error("Unexpected retained cleanup");
},
);
expect(result).toMatchObject({ status: "installed" });
expect(
await readFile(
join(
temporary,
"actual/Applications/Hopper Disassembler.app/Contents/MacOS/hopper",
),
"utf8",
),
).toBe("inert launcher");
} finally {
await rm(temporary, { recursive: true, force: true });
}
},
);
it("finds the same renamed user bundles through runtime and doctor scans", async () => {
// Keep native ownership for all I/O, including UNC temporary directories.
// On Windows, a checkout-relative home models POSIX separators without
src/application/MacHopperPaths.ts
+14 -6
View File
@@ -1,6 +1,7 @@
import { readdirSync, type Dirent } from "node:fs";
import { readdir } from "node:fs/promises";
import { posix } from "node:path";
import { appendCallerPath } from "../filesystem/callerPath.js";
export const MAC_HOPPER_APPLICATION_DIRECTORY = "/Applications";
export const MAC_HOPPER_APPLICATION_NAME = "Hopper Disassembler.app";
@@ -13,26 +14,31 @@ export const MAC_HOPPER_SYSTEM_LAUNCHER = posix.join(
);
export const macHopperUserApplicationPath = (homeDirectory: string): string =>
posix.join(
appendCallerPath(
homeDirectory,
MAC_HOPPER_USER_APPLICATIONS_DIRECTORY,
MAC_HOPPER_APPLICATION_NAME,
`${MAC_HOPPER_USER_APPLICATIONS_DIRECTORY}/${MAC_HOPPER_APPLICATION_NAME}`,
"darwin",
);
export const macHopperApplicationRoots = (
homeDirectory: string,
): readonly string[] => [
MAC_HOPPER_APPLICATION_DIRECTORY,
posix.join(homeDirectory, MAC_HOPPER_USER_APPLICATIONS_DIRECTORY),
appendCallerPath(
homeDirectory,
MAC_HOPPER_USER_APPLICATIONS_DIRECTORY,
"darwin",
),
];
export const macHopperCanonicalLauncherCandidates = (
homeDirectory: string,
): readonly string[] => [
MAC_HOPPER_SYSTEM_LAUNCHER,
posix.join(
appendCallerPath(
macHopperUserApplicationPath(homeDirectory),
"Contents/MacOS/hopper",
"darwin",
),
];
@@ -50,7 +56,9 @@ export const macHopperRenamedLauncherCandidates = (
MAC_HOPPER_APPLICATION_NAME.toLocaleLowerCase("en-US"),
)
.sort()
.map((name) => posix.join(root, name, "Contents/MacOS/hopper")),
.map((name) =>
appendCallerPath(root, `${name}/Contents/MacOS/hopper`, "darwin"),
),
);
// A linked application bundle retains its selected spelling. Runtime and
src/providers/ProviderInspectionScope.ts → src/application/ProviderInspectionScope.ts
View File
File renamed without changes.
src/application/ReferenceSourceGitDirectory.ts
+19 -3
View File
@@ -1,4 +1,4 @@
import { lstat, realpath } from "node:fs/promises";
import { lstat, realpath, stat } from "node:fs/promises";
import { dirname, join } from "node:path";
import { absoluteCallerPath } from "../filesystem/callerPath.js";
@@ -40,7 +40,7 @@ export const locateReferenceSourceGitDirectory = async (
}
// Derived joins and the Git library normalize paths lexically. Admit the
// physical directory first, while the source pointer stays in its file.
const gitDirectory = await realpath(selectedGitDirectory);
const gitDirectory = await resolveGitDirectory(selectedGitDirectory);
let common: string | undefined;
try {
common = (await readText(join(gitDirectory, "commondir"))).replace(
@@ -54,11 +54,27 @@ export const locateReferenceSourceGitDirectory = async (
const commonDirectory =
common === undefined
? gitDirectory
: await realpath(absoluteCallerPath(common, { cwd: gitDirectory }));
: await resolveGitDirectory(
absoluteCallerPath(common, { cwd: gitDirectory }),
);
return { repository, gitDirectory, commonDirectory };
}
};
const resolveGitDirectory = async (path: string): Promise<string> => {
// Darwin realpath can collapse file/.. even when native lookup rejects it.
// Admit the selected traversal before deriving paths for metadata consumers.
if (!(await stat(path)).isDirectory())
throw Object.assign(
new Error(`Git metadata path is not a directory: ${path}`),
{
code: "ENOTDIR",
path,
},
);
return realpath(path);
};
const hasCode = (cause: unknown, code: string): boolean =>
typeof cause === "object" &&
cause !== null &&
src/application/ReferenceSourceImportEntries.test.ts
+173
View File
@@ -364,3 +364,176 @@ it("resolves extensionless imports and requires to .mts and .cts files and direc
]),
);
});
it("does not infer missing candidates or manifests from partial directory enumeration", () => {
const file = (
path: string,
content: string,
): ReferenceSourceRead["entries"][number] => ({
status: "read",
kind: "file",
path,
bytes: Buffer.from(content),
size: Buffer.byteLength(content),
});
const read: ReferenceSourceRead = {
root: "/selected/source",
bytesRead: 0,
limitations: [],
entries: [
file("package.json", '{"type":"commonjs"}'),
file(
"main.cjs",
'require("./pkg"); require("./other"); require("./exact.js");',
),
file("pkg/index.js", "module.exports = 1;"),
file("other/index.js", "module.exports = 1;"),
file("exact.js", "module.exports = 1;"),
file("pkg/source.js", 'require("../exact.js");'),
{
status: "failed",
kind: "directory",
path: "pkg",
code: "io",
message: "Partial enumeration",
},
{ status: "read", kind: "directory", path: "other" },
],
};
const parsed = parseReferenceSourceEntries(read, []);
expect(
parsed.relationships.filter(({ from_path }) => from_path === "main.cjs"),
).toEqual([
expect.objectContaining({ to: "pkg", resolution: "unresolved" }),
expect.objectContaining({ to: "other/index.js", resolution: "internal" }),
expect.objectContaining({ to: "exact.js", resolution: "internal" }),
]);
expect(
parsed.relationships.filter(
({ from_path }) => from_path === "pkg/source.js",
),
).not.toContainEqual(expect.objectContaining({ resolution: "internal" }));
const partialRoot = parseReferenceSourceEntries(
{ ...read, rootFailure: { code: "io", message: "Partial root" } },
[],
);
expect(
partialRoot.relationships.find(
({ from_path, to }) =>
from_path === "main.cjs" && to === "other/index.js",
),
).toBeUndefined();
expect(partialRoot.relationships).toContainEqual(
expect.objectContaining({
from_path: "main.cjs",
to: "exact.js",
resolution: "internal",
}),
);
});
it.each(["failed", "excluded"] as const)(
"keeps higher priority %s candidates unknown",
(state) => {
const files = [
["main.cjs", 'require("./pkg"); require("./pkg/index.js");'],
["pkg/index.js", "module.exports = 1;"],
] as const;
const read: ReferenceSourceRead = {
root: "/selected/source",
bytesRead: 0,
limitations: [],
entries: [
...files.map(([path, content]) => ({
status: "read" as const,
kind: "file" as const,
path,
bytes: Buffer.from(content),
size: Buffer.byteLength(content),
})),
...(state === "failed"
? [
{
status: "failed" as const,
kind: "file" as const,
path: "pkg.js",
code: "io" as const,
message: "Unreadable",
},
]
: []),
],
};
const parsed = parseReferenceSourceEntries(
read,
state === "excluded"
? [
{
path: "pkg.js",
entry_kind: "file",
reason: "caller-excluded",
pattern: "pkg.js",
},
]
: [],
);
expect(parsed.relationships).toEqual([
expect.objectContaining({ to: "pkg", resolution: "unresolved" }),
expect.objectContaining({ to: "pkg/index.js", resolution: "internal" }),
]);
},
);
it("retains observed manifests and closer completed scopes inside a partial inventory", () => {
const files = [
["main.cjs", 'require("./pkg");'],
["pkg/package.json", '{"main":"entry.js","type":"commonjs"}'],
["pkg/entry.js", 'require("./known.js");'],
["pkg/known.js", "module.exports = 1;"],
["complete/source.js", 'require("./known.js");'],
["complete/known.js", "module.exports = 1;"],
["complete/package.json", '{"type":"commonjs"}'],
] as const;
const parsed = parseReferenceSourceEntries(
{
root: "/selected/source",
bytesRead: 0,
limitations: [],
entries: [
...files.map(([path, content]) => ({
status: "read" as const,
kind: "file" as const,
path,
bytes: Buffer.from(content),
size: Buffer.byteLength(content),
})),
{
status: "failed",
kind: "directory",
path: "pkg",
code: "io",
message: "Partial enumeration",
},
{ status: "read", kind: "directory", path: "complete" },
],
},
[],
);
expect(parsed.relationships).toEqual([
expect.objectContaining({
from_path: "main.cjs",
to: "pkg/entry.js",
resolution: "internal",
}),
expect.objectContaining({
from_path: "pkg/entry.js",
to: "pkg/known.js",
resolution: "internal",
}),
expect.objectContaining({
from_path: "complete/source.js",
to: "complete/known.js",
resolution: "internal",
}),
]);
});
src/application/ReferenceSourceImportEntries.ts
+113 -3
View File
@@ -124,6 +124,7 @@ const UNKNOWN_PACKAGE: PackageMetadata = {
const readPackageMetadata = (
read: ReferenceSourceRead,
exclusions: HistoricalSourceGraphInput["exclusions"],
unobservedPath: (path: string) => boolean,
): {
readonly packages: ReadonlyMap<string, PackageMetadata>;
readonly pathLookup: ReferenceSourcePathLookup;
@@ -176,6 +177,22 @@ const readPackageMetadata = (
packages.set(directory, UNKNOWN_PACKAGE);
}
}
// An incomplete directory cannot establish that its manifest is absent.
const uncertainDirectories = [
...(read.rootFailure === undefined ? [] : ["."]),
...read.entries
.filter(
(entry) => entry.kind === "directory" && entry.status === "failed",
)
.map(({ path }) => path),
];
for (const directory of uncertainDirectories) {
const path = posix.join(directory, "package.json");
if (!packages.has(directory) && unobservedPath(path)) {
manifestPaths.push(path);
packages.set(directory, UNKNOWN_PACKAGE);
}
}
return {
packages,
pathLookup: createReferenceSourcePathLookup(manifestPaths),
@@ -226,8 +243,74 @@ interface SpecifierContext {
readonly lookupPath: ReferenceSourcePathLookup;
readonly packages: ReadonlyMap<string, PackageMetadata>;
readonly packageManifestPathLookup: ReferenceSourcePathLookup;
readonly unobservedPath: (path: string) => boolean;
}
/** Earlier unobserved candidates must not be treated as absent. */
const selectObservedCandidate = (
candidates: readonly string[],
context: SpecifierContext,
): Result<string | undefined, undefined> => {
const selected = context.lookupPath.resolveCandidates(candidates);
for (const candidate of candidates) {
if (
context.lookupPath.resolve(candidate) === selected &&
selected !== undefined
)
break;
if (context.unobservedPath(candidate)) return err(undefined);
}
return ok(selected);
};
/** The closest directory observation owns whether an absent child was examined. */
const unobservedReferencePath = (
read: ReferenceSourceRead,
exclusions: HistoricalSourceGraphInput["exclusions"],
): ((path: string) => boolean) => {
const observed = createReferenceSourcePathLookup(
read.entries.map(({ path }) => path),
);
const unavailable = createReferenceSourcePathLookup([
...read.entries
.filter(
(entry) =>
entry.kind !== "directory" &&
(entry.status === "failed" || entry.kind === "symlink"),
)
.map(({ path }) => path),
...exclusions
.filter((entry) => entry.entry_kind !== "directory")
.map(({ path }) => path),
]);
const excludedDirectories = createReferenceSourcePathLookup(
exclusions
.filter((entry) => entry.entry_kind === "directory")
.map(({ path }) => path),
);
const directories = new Map(
read.entries
.filter((entry) => entry.kind === "directory")
.map((entry) => [entry.path, entry.status]),
);
const directoryLookup = createReferenceSourcePathLookup(directories.keys());
return (path) => {
if (unavailable.has(path)) return true;
if (observed.has(path)) return false;
for (
let parent = posix.dirname(path);
parent !== "." && parent !== "/";
parent = posix.dirname(parent)
) {
if (excludedDirectories.has(parent)) return true;
const directory = directoryLookup.resolve(parent);
if (directory !== undefined)
return directories.get(directory) === "failed";
}
return read.rootFailure !== undefined;
};
};
const packageMainForDirectory = (
directory: string,
context: SpecifierContext,
@@ -291,7 +374,9 @@ const selectPackageMain = (
.filter((suffix) => suffix !== "")
.map((suffix) => posix.join(target, `index${suffix}`)),
];
const selected = context.lookupPath.resolveCandidates(candidates);
const selection = selectObservedCandidate(candidates, context);
if (!selection.ok) return { to: packageDirectory, resolution: "unresolved" };
const selected = selection.value;
return selected === undefined
? indexFallback === undefined
? { to: packageDirectory, resolution: "unresolved" }
@@ -390,7 +475,26 @@ const resolveInternalSpecifier = (
({ path }) => path,
),
);
const match = context.lookupPath.resolveCandidates(candidates);
const selection = selectObservedCandidate(candidates, context);
if (!selection.ok) {
// Node consults a known package main before probing directory indexes.
// Incomplete index coverage cannot invalidate that observed declaration.
if (
mode === "runtime-require" &&
packageMainForDirectory(normalized, context)?.main.state === "declared"
) {
const files = directoryReference
? ok(undefined)
: selectObservedCandidate(
nodeRequireFileCandidates(normalized),
context,
);
if (files.ok && files.value === undefined)
return selectPackageMain(normalized, undefined, context, mode);
}
return { to: normalized, resolution: "unresolved" };
}
const match = selection.value;
if (match === undefined)
return packageMainForDirectory(normalized, context) !== undefined
? selectPackageMain(normalized, undefined, context, mode)
@@ -547,13 +651,19 @@ export const parseReferenceSourceEntries = (
output.limitations.push(
`Reference root enumeration is incomplete: ${read.rootFailure.message}`,
);
const packageInventory = readPackageMetadata(read, exclusions);
const unobservedPath = unobservedReferencePath(read, exclusions);
const packageInventory = readPackageMetadata(
read,
exclusions,
unobservedPath,
);
const context: SpecifierContext = {
lookupPath: createReferenceSourcePathLookup(
read.entries
.filter((entry) => entry.status === "read" && entry.kind === "file")
.map((entry) => entry.path),
),
unobservedPath,
packages: packageInventory.packages,
packageManifestPathLookup: packageInventory.pathLookup,
};
src/application/Setup.recovery.test.ts
+4
View File
@@ -683,6 +683,10 @@ describe("setup Hopper install eligibility", () => {
it.each([
{ version: "", installs: false },
{ version: "14beta", installs: false },
{ version: "14.garbage", installs: false },
{ version: "14.", installs: false },
{ version: "14.0\n", installs: false },
{ version: "99999999999999999", installs: false },
{ version: "12", installs: true },
{ version: "12.0", installs: true },
])(
src/application/SetupClientConfiguration.ts
+121 -8
View File
@@ -16,6 +16,7 @@ import {
clientServerListedDisabled,
clientServerPath,
legacyClientServerPath,
shadowedClientServers,
OMP_DISABLED_SERVERS_KEY,
parseClientConfiguration,
parseZcodeFallbackConfiguration,
@@ -39,7 +40,11 @@ import {
type ProviderRequestBudgetResolver,
} from "./ClientRegistrationEnvironment.js";
import { refreshClientRegistration } from "./ClientRegistration.js";
import {
clientRegistrationOwnership,
clientRegistrationOwnershipRemediation,
refreshClientRegistration,
} from "./ClientRegistration.js";
import { PRODUCT_IDENTITY } from "../identity.js";
import { snapshotEnvironment } from "../process/snapshotEnvironment.js";
@@ -169,6 +174,18 @@ const configureClientDocument = async (
format,
);
parsed = clientConfigurationForWrite(parsed, files.value);
const ownershipBlocker = registrationOwnershipBlocker(
client,
parsed,
original ?? "",
files.value,
);
if (ownershipBlocker !== undefined)
return {
status: "failed",
reason: "registration_ownership",
detail: ownershipBlocker,
};
const policy = await readClientPolicyBlock(
client,
parsed,
@@ -195,8 +212,15 @@ const configureClientDocument = async (
}
const fallback = await readZcodeFallbackServers(client, parsed);
if (!fallback.ok) {
throwCleanupFailure(fallback.error);
return { status: "failed", reason: "readback" };
if (fallback.error.kind !== "registration_ownership")
throwCleanupFailure(fallback.error);
return fallback.error.kind === "registration_ownership"
? {
status: "failed",
reason: "registration_ownership",
detail: fallback.error.detail,
}
: { status: "failed", reason: "readback" };
}
const desired = await clientConfigurationDesired(
client,
@@ -310,8 +334,7 @@ const persistClientConfiguration = async ({
...(legacyPath === undefined ? [] : [legacyPath]),
...(enableOmp ? [[OMP_DISABLED_SERVERS_KEY]] : []),
],
// Setup deletes only REA's OpenCode V1 name, which the native V2 entry
// supersedes whatever its launcher; every repeat goes, as one member does.
// Ownership admission above covers every legacy member removed here.
{ removeShadowedMembersAt: legacyPath === undefined ? [] : [legacyPath] },
);
} catch (cause: unknown) {
@@ -542,6 +565,16 @@ const inspectClientDocument = async (
} catch (cause: unknown) {
if (cause instanceof RegularFileCleanupFailure) throw cause;
if (isMissing(cause)) {
const inherited = effectiveClientConfiguration(files.value, {
client,
document: undefined,
});
const ownershipBlocker =
inherited === undefined
? undefined
: registrationOwnershipBlocker(client, inherited, "", files.value);
if (ownershipBlocker !== undefined)
return { status: "invalid", remediation: ownershipBlocker };
// This read establishes absence; do not reuse an earlier user-file policy.
const policy = await readClientPolicyBlock(
client,
@@ -566,7 +599,8 @@ const inspectClientDocument = async (
return { status: "invalid", remediation: policy.value };
const fallback = await readZcodeFallbackServers(client);
if (!fallback.ok) {
throwCleanupFailure(fallback.error);
if (fallback.error.kind !== "registration_ownership")
throwCleanupFailure(fallback.error);
return { status: "invalid", remediation: fallback.error.detail };
}
return { status: "create" };
@@ -581,6 +615,14 @@ const inspectClientDocument = async (
parseClientConfiguration(original, client.format),
files.value,
);
const ownershipBlocker = registrationOwnershipBlocker(
client,
parsed,
original,
files.value,
);
if (ownershipBlocker !== undefined)
return { status: "invalid", remediation: ownershipBlocker };
const desired = await clientConfigurationDesired(
client,
providerEnvironment,
@@ -603,7 +645,8 @@ const inspectClientDocument = async (
return { status: "already_current" };
const fallback = await readZcodeFallbackServers(client, parsed);
if (!fallback.ok) {
throwCleanupFailure(fallback.error);
if (fallback.error.kind !== "registration_ownership")
throwCleanupFailure(fallback.error);
return { status: "invalid", remediation: fallback.error.detail };
}
} catch (cause: unknown) {
@@ -642,12 +685,71 @@ const throwCleanupFailure = (error: ClientConfigurationFileError): void => {
if (error.cleanupFailure !== undefined) throw error.cleanupFailure;
};
/** Admit only absent or proven-owned rows that this setup would replace. */
const registrationOwnershipBlocker = (
client: SetupClient,
parsed: ClientConfigurationDocument,
original: string,
sources: readonly ClientConfigurationSource[] = [],
): string | undefined => {
// Cordis owns registrations by stable entry ID, including malformed launchers.
if (parsed.dialect === "dsh") return undefined;
const name = PRODUCT_IDENTITY.mcpServerKey;
const entries: { value: unknown; paths: readonly string[] }[] = [];
const effective = effectiveClientConfiguration(sources, {
client,
document: parsed,
});
if (effective !== undefined && Object.hasOwn(effective.servers, name))
entries.push({
value: effective.servers[name],
paths: client.configPaths ?? [client.configPath],
});
const fragment = parsed.servers[name];
// A layered policy fragment inherits its launcher. Classify the effective
// launcher above, but also admit any explicit command this write replaces.
const inheritsLauncher =
effective !== undefined &&
effective !== parsed &&
typeof fragment === "object" &&
fragment !== null &&
!Array.isArray(fragment) &&
!Object.hasOwn(fragment, "command");
if (Object.hasOwn(parsed.servers, name) && !inheritsLauncher)
entries.push({ value: parsed.servers[name], paths: [client.configPath] });
if (legacyClientServerPath(parsed, name) !== undefined) {
entries.push({
value: parsed.legacyServers[name],
paths: [client.configPath],
});
entries.push(
...shadowedClientServers(original, parsed, name).legacyServers.map(
(value) => ({ value, paths: [client.configPath] }),
),
);
}
for (const entry of entries) {
const ownership = clientRegistrationOwnership(entry.value, client);
if (ownership !== "owned")
return clientRegistrationOwnershipRemediation(ownership, entry.paths);
}
return undefined;
};
/** Admit the fallback only when the first native ZCode server would suppress it. */
const readZcodeFallbackServers = async (
client: SetupClient,
parsed?: ClientConfigurationDocument,
): Promise<
Result<Readonly<Record<string, unknown>>, ClientConfigurationFileError>
Result<
Readonly<Record<string, unknown>>,
| ClientConfigurationFileError
| {
readonly kind: "registration_ownership";
readonly path: string;
readonly detail: string;
}
>
> => {
if (
client.format !== "zcode" ||
@@ -667,6 +769,17 @@ const readZcodeFallbackServers = async (
if (source.value === undefined) return ok({});
try {
const parsed = parseZcodeFallbackConfiguration(source.value);
const ownershipBlocker = registrationOwnershipBlocker(
{ ...client, configPath: path, format: "zcode_fallback" },
parsed,
source.value,
);
if (ownershipBlocker !== undefined)
return err({
kind: "registration_ownership",
path,
detail: ownershipBlocker,
});
const environment = clientRegistrationEnvironment(parsed);
if (!environment.ok)
return err({
src/application/SetupClients.ts
+2
View File
@@ -10,6 +10,8 @@ export const failedConfigurationMessage = (
result: Extract<ClientConfigurationResult, { readonly status: "failed" }>,
): string => {
switch (result.reason) {
case "registration_ownership":
return result.detail;
case "conflict":
return "Agent configuration changed during setup. Review the current file and rerun setup.";
case "target_changed":
src/application/SetupPlan.ts
+3 -5
View File
@@ -24,6 +24,7 @@ import { redirectedWriteBlocker } from "./MsixPackageStorage.js";
import type { DoctorReport } from "./Doctor.js";
import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
import { CURRENT_ENTRYPOINT_PATH } from "../process/CurrentEntrypointPath.js";
import { macosMajorVersion } from "../domain/macosVersion.js";
/** Whether automated Hopper installation can run on this host, and why not. */
export type HopperInstallEligibility =
@@ -408,11 +409,8 @@ const hopperInstallEligibility = (
linuxDistribution: LinuxDistribution | undefined,
): HopperInstallEligibility => {
if (platform === "darwin") {
const major =
macosVersion === undefined
? undefined
: /^(\d+)(?:\.|$)/u.exec(macosVersion)?.[1];
return major !== undefined && Number(major) >= 12
const major = macosMajorVersion(macosVersion);
return major !== undefined && major >= 12
? { supported: true }
: { supported: false, remediation: "Upgrade to macOS 12 or newer." };
}
src/application/SetupTypes.ts
+5
View File
@@ -35,6 +35,11 @@ export interface SetupInitialState {
/** Result of one backup/write/readback transaction. */
export type ClientConfigurationResult =
| {
readonly status: "failed";
readonly reason: "registration_ownership";
readonly detail: string;
}
| {
readonly status: "failed";
readonly reason: "conflict";
src/application/SupportedClients.ts
+5 -4
View File
@@ -196,8 +196,9 @@ const kimiDirectory = ({ home, env }: ClientPathContext): string =>
const qoderDirectory = ({ home, env }: ClientPathContext): string =>
env.QODER_CONFIG_DIR || join(home, ".qoder");
const factoryDirectory = ({ home }: ClientPathContext): string =>
join(home, ".factory");
/** Droid's override selects the home containing .factory, including personal skills. */
const factoryDirectory = ({ home, env }: ClientPathContext): string =>
join(env.FACTORY_HOME_OVERRIDE || home, ".factory");
/** Match Qwen Code's home override, including tilde and cwd-relative paths. */
const qwenCodeDirectory = ({ home, env }: ClientPathContext): string => {
@@ -684,9 +685,9 @@ export const SUPPORTED_CLIENT_DEFINITIONS = [
name: "hermes",
displayName: "Hermes",
skillPath: (context: ClientPathContext) =>
join(hermesDirectory(context), "skills"),
appendCallerPath(hermesDirectory(context), "skills"),
configPath: (context: ClientPathContext) =>
join(hermesDirectory(context), "config.yaml"),
appendCallerPath(hermesDirectory(context), "config.yaml"),
markerPath: hermesDirectory,
format: "hermes",
},
src/application/Uninstall.ts
+5 -20
View File
@@ -24,7 +24,7 @@ import { defaultKunaCacheDirectory } from "../config/kunaCacheDirectory.js";
import { linuxHopperReleaseCacheDirectory } from "./LinuxHopperCacheDirectory.js";
import { ClientConfigurationPublication } from "./ClientConfigurationPublication.js";
import { parseClientRegistrationLauncher } from "./ClientRegistration.js";
import { clientRegistrationOwnership } from "./ClientRegistration.js";
import { PRODUCT_IDENTITY } from "../identity.js";
import {
@@ -33,8 +33,6 @@ import {
readSkillFile,
skillDestinationSelection,
} from "./SetupSkill.js";
import { classifyClientRegistrationCommand } from "./ClientRegistrationIdentity.js";
import { CURRENT_ENTRYPOINT_PATH } from "../process/CurrentEntrypointPath.js";
import { dshDocument } from "./DshCordisPatch.js";
import { resolveClientConfigTransactionPath } from "./ClientConfigPath.js";
import {
@@ -785,11 +783,11 @@ const clientRegistrationRemoval = (
const removeNative =
(parsed.dialect === "dsh" && dshDocument(parsed.document).carriesEntry) ||
(Object.hasOwn(parsed.servers, name) &&
isOwnedRegistration(parsed.servers[name], client));
clientRegistrationOwnership(parsed.servers[name], client) === "owned");
const legacyPath = legacyClientServerPath(parsed, name);
const removeLegacy =
legacyPath !== undefined &&
isOwnedRegistration(parsed.legacyServers[name], client);
clientRegistrationOwnership(parsed.legacyServers[name], client) === "owned";
const ownership: RegistrationRemoval = {
removeNative,
legacyPath: removeLegacy ? legacyPath : undefined,
@@ -799,11 +797,11 @@ const clientRegistrationRemoval = (
if (
(removeNative &&
shadowed.servers.some(
(member) => !isOwnedRegistration(member, client),
(member) => clientRegistrationOwnership(member, client) !== "owned",
)) ||
(removeLegacy &&
shadowed.legacyServers.some(
(member) => !isOwnedRegistration(member, client),
(member) => clientRegistrationOwnership(member, client) !== "owned",
))
)
return err({
@@ -1024,19 +1022,6 @@ const resolveUninstallConfigPath = async (
);
};
/** Ownership rule for persistent MCP registrations created by REA setup. */
const isOwnedRegistration = (value: unknown, client: SetupClient): boolean => {
const parsed = parseClientRegistrationLauncher(value, client);
if (!parsed.ok) return false;
const { command, args } = parsed.value;
return (
classifyClientRegistrationCommand(
[command, ...args],
CURRENT_ENTRYPOINT_PATH,
) === "owned"
);
};
/** Purge the same default cache namespaces selected by their producers. */
const purgeSelectedData = async (
home: string,
src/application/Update.test.ts
+63
View File
@@ -1,4 +1,14 @@
import { join, resolve } from "node:path";
import {
mkdtemp,
mkdir,
realpath,
rm,
stat,
symlink,
writeFile,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import { describe, expect, it } from "vitest";
@@ -397,6 +407,59 @@ describe("Bun installation ownership", () => {
Promise.resolve(manifests.includes(path.replaceAll("\\", "/"))),
});
it.skipIf(process.platform === "win32")(
"preserves native symlink traversal through detection and pins the canonical global project",
async () => {
const directory = await mkdtemp(join(tmpdir(), "rea-bun-global-"));
try {
await mkdir(join(directory, "outer", "inner"), { recursive: true });
await symlink(
join(directory, "outer", "inner"),
join(directory, "alias"),
);
const selected = `${directory}/alias/../global`;
expect(
bunGlobalDirectory({ BUN_INSTALL: selected }, process.platform),
).toBe(resolve(selected, "install", "global"));
expect(
bunGlobalDirectory({ XDG_CACHE_HOME: selected }, process.platform),
).toBe(resolve(selected, ".bun", "install", "global"));
const selectedPackage = join(
directory,
"outer",
"global",
"node_modules",
"rea-agents",
);
await mkdir(selectedPackage, { recursive: true });
await writeFile(`${selected}/package.json`, "{}");
// The lexical alternative is also a valid project; it must not win.
await mkdir(join(directory, "global", "node_modules", "rea-agents"), {
recursive: true,
});
await writeFile(join(directory, "global", "package.json"), "{}");
const installation = await detectBunInstallation(
selectedPackage,
bunGlobalDirectory(
{ BUN_INSTALL_GLOBAL_DIR: selected },
process.platform,
),
{
canonicalPath: realpath,
isFile: async (path) => (await stat(path)).isFile(),
},
);
expect(installation).toEqual({
manager: "bun",
globalDirectory: await realpath(selected),
packageRoot: await realpath(selectedPackage),
});
} finally {
await rm(directory, { recursive: true, force: true });
}
},
);
it("identifies the package inside Bun's global project", async () => {
await expect(
detectBunInstallation(packageRoot, globalDirectory, host()),
src/application/UpdateRuntime.ts
+12 -4
View File
@@ -1,5 +1,6 @@
import { realpath, stat } from "node:fs/promises";
import { homeDirectoryFromEnvironment } from "../config/homeDirectory.js";
import { absoluteCallerPath } from "../filesystem/callerPath.js";
import { basename, dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { lt, valid } from "semver";
@@ -89,7 +90,9 @@ export const bunGlobalDirectory = (
platform: NodeJS.Platform,
): string => {
const explicit = nonemptyEnvironment(environment.BUN_INSTALL_GLOBAL_DIR);
if (explicit !== undefined) return resolve(explicit);
// Bun opens the explicit project path with native filesystem traversal;
// its derived BUN_INSTALL/XDG paths below are normalized lexically instead.
if (explicit !== undefined) return absoluteCallerPath(explicit);
const installRoot = nonemptyEnvironment(environment.BUN_INSTALL);
if (installRoot !== undefined)
return resolve(installRoot, "install", "global");
@@ -112,17 +115,22 @@ export const detectBunInstallation = async (
): Promise<BunInstallation | undefined> => {
try {
const canonicalPackageRoot = await host.canonicalPath(packageRoot);
const canonicalGlobalDirectory = await host.canonicalPath(globalDirectory);
const globalPackageRoot = await host.canonicalPath(
join(globalDirectory, "node_modules", PRODUCT_IDENTITY.packageName),
join(
canonicalGlobalDirectory,
"node_modules",
PRODUCT_IDENTITY.packageName,
),
);
if (
canonicalPackageRoot !== globalPackageRoot ||
!(await host.isFile(join(globalDirectory, "package.json")))
!(await host.isFile(join(canonicalGlobalDirectory, "package.json")))
)
return undefined;
return {
manager: "bun",
globalDirectory,
globalDirectory: canonicalGlobalDirectory,
packageRoot: canonicalPackageRoot,
};
} catch (cause: unknown) {
src/application/analysisView/AnalysisViewService.test.ts
+447 -1
View File
@@ -11,7 +11,7 @@ import {
analysisViewJavaScriptEvidence,
analysisViewLayoutEvidence,
} from "../../../tests/fixtures/analysisView.js";
import { describe, expect, it } from "vitest";
import { describe, expect, it, vi } from "vitest";
import { JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE } from "../../contracts/javascript/javascriptRuntimeReconciliationExample.js";
import {
@@ -33,6 +33,7 @@ import { EvidenceIntegrityError } from "../../domain/evidenceErrors.js";
import { err, ok } from "../../domain/result.js";
import { EvidenceLedger } from "../investigation/EvidenceLedger.js";
import { WEB_NETWORK_CAPTURE_LIMITS } from "../../domain/webNetworkCapture.js";
import { isImmutableJsonSnapshot } from "../../domain/immutableJson.js";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import {
@@ -41,6 +42,149 @@ import {
scenarioCaptureFixture,
} from "../../../tests/fixtures/browserScenarioCapture.js";
import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
import * as queuedInterrupts from "../../process/queuedInterrupts.js";
import { createEvidenceBundle } from "../../domain/evidenceBundle.js";
import * as evidenceFileSource from "./evidenceFileSource.js";
it("projects retained GPU source facts while delivering the caller's rebound subject", () => {
const sha256 = "a".repeat(64);
const original = createEvidence(
{ path: "/original/gpu", sha256, format: "elf" },
{ id: "fixture", name: "Fixture", version: "1" },
{
operation: "inspect_gpu_code_objects",
parameters: {},
result: {
artifact: {
path: "/original/gpu",
file_size: 0,
sha256,
bytes_read: 0,
},
format: "elf",
analysis: "inventory",
objects: [],
kernels: [],
ptx_artifacts: [],
coverage: {
discovery_scope: "supported-section-names-and-standalone-formats",
discovery: "partial",
payloads: "not-requested",
examined_sections: 0,
unknown_section_names: 0,
},
limitations: [],
},
},
);
const ledger = new EvidenceLedger();
expect(ledger.record(original).ok).toBe(true);
const retained = ledger.forAnalysis(original.evidence_id)!;
const payload = retained.normalized_result;
const selected = rebindEvidenceToTargetPath(original, {
path: "/selected/gpu",
sha256,
format: "elf",
});
expect(selected.evidence_id).toBe(original.evidence_id);
const retention = {
recordEvidence: (evidence: Evidence) => ledger.record(evidence),
evidenceById: (id: string) => ledger.forAnalysis(id),
};
for (const view of [
{ kind: "summary" as const },
{
kind: "gpu-code-objects" as const,
collection: "objects" as const,
offset: 0,
limit: 64,
},
]) {
const result = projectRetainedAnalysis(selected, retention, view);
if (!result.ok) throw result.error;
expect(result.value.subject?.local_path).toBe("/selected/gpu");
expect(result.value.normalized_result).toMatchObject({
parent_evidence_id: original.evidence_id,
artifact: { path: "/original/gpu", sha256 },
});
}
expect(ledger.forAnalysis(original.evidence_id)).toBe(retained);
// Local path metadata may be rebound before first retention or bundle import.
const aliasLedger = new EvidenceLedger();
expect(aliasLedger.record(selected).ok).toBe(true);
const imported = new EvidenceLedger();
expect(imported.import(aliasLedger.export()).ok).toBe(true);
// Importing metadata into an existing ledger must preserve view validity too.
expect(ledger.import(aliasLedger.export()).ok).toBe(true);
for (const sourceLedger of [aliasLedger, imported, ledger]) {
const result = projectRetainedAnalysis(
selected,
{
recordEvidence: (evidence) => sourceLedger.record(evidence),
evidenceById: (id) => sourceLedger.forAnalysis(id),
},
{ kind: "summary" },
);
if (!result.ok) throw result.error;
expect(result.value.subject?.local_path).toBe("/selected/gpu");
expect(result.value.normalized_result).toMatchObject({
artifact: { path: "/original/gpu", sha256 },
});
}
expect(retained.normalized_result).toBe(payload);
expect(retained.subject?.local_path).toBe("/original/gpu");
const mismatched = createEvidence(
{ path: "/original/gpu", sha256: "b".repeat(64), format: "elf" },
original.provider,
{
operation: original.operation,
parameters: {},
result: original.normalized_result,
},
);
expect(summarizeRetainedAnalysis(mismatched, retention)).toMatchObject({
ok: false,
error: { _tag: "AnalysisInputError" },
});
});
it.each(["inline", "retained-evidence", "invalid"] as const)(
"refuses an already cancelled %s view before source admission",
async (kind) => {
const parent = analysisViewLayoutEvidence();
const ledger = new EvidenceLedger();
expect(ledger.record(parent).ok).toBe(true);
const retained = ledger.forAnalysis(parent.evidence_id);
let lookups = 0;
const controller = new AbortController();
const reason = Object.freeze({ requested: "cancel-before-view-admission" });
controller.abort(reason);
const result = await inspectAnalysisView(
kind === "invalid"
? null
: {
source:
kind === "inline"
? { kind, evidence: parent }
: { kind, evidence_id: parent.evidence_id },
view: { kind: "summary" },
},
(id) => {
lookups++;
return ledger.forAnalysis(id);
},
{ signal: controller.signal },
);
expect(result).toMatchObject({
ok: false,
error: { _tag: "AnalysisCancelledError" },
});
expect(result.ok ? undefined : result.error.cause).toBe(reason);
expect(lookups).toBe(0);
expect(ledger.forAnalysis(parent.evidence_id)).toBe(retained);
},
);
it.each([false, true])(
"summarizes the selected observation path rather than older retained display metadata (history=%s)",
@@ -1185,6 +1329,308 @@ describe("inspectAnalysisView", () => {
});
});
it("prioritizes queued cancellation over a file authentication failure", async () => {
const controller = new AbortController();
const reason = Object.freeze({ requested: "cancel-after-authentication" });
// Model an event callback which queues the caller's cancellation notification
// as authentication finishes. The service must dispatch that notification
// before accepting the authentication failure.
const read = vi
.spyOn(evidenceFileSource, "readEvidenceFileSource")
.mockImplementationOnce(async () => {
setImmediate(() => setImmediate(() => controller.abort(reason)));
return err(new EvidenceIntegrityError("Invalid selected Evidence"));
});
try {
const result = await inspectAnalysisView(
{
source: { kind: "evidence-file", path: "/selected-evidence.json" },
view: manifestView,
},
undefined,
{ signal: controller.signal },
);
expect(result).toMatchObject({
ok: false,
error: { _tag: "AnalysisCancelledError" },
});
expect(result.ok ? undefined : result.error.cause).toBe(reason);
if (result.ok) throw new Error("Expected cancellation");
expect(result.error.secondaryFailures).toHaveLength(1);
expect(projectAnalysisError(result.error)).toMatchObject({
code: "cancelled",
details: {
diagnostics: { source_error: { code: "evidence_integrity_mismatch" } },
},
});
} finally {
read.mockRestore();
}
});
it("retries the owned descriptor when queued cancellation supersedes file cleanup failure", async () => {
const controller = new AbortController();
const path = join(
await createTestTempDirectory("rea-view-cleanup-"),
"evidence.json",
);
await writeFile(path, JSON.stringify(analysisViewCaptureEvidence()));
const resources = new ArtifactResourceScope();
const originalClose = OwnedFileHandle.prototype.close;
const owners = new Set<OwnedFileHandle>();
let failClose = true;
const close = vi
.spyOn(OwnedFileHandle.prototype, "close")
.mockImplementation(async function (this: OwnedFileHandle) {
owners.add(this);
if (failClose) {
setImmediate(() => setImmediate(() => controller.abort("stop")));
throw new Error("Fixture descriptor close unavailable");
}
return originalClose.call(this);
});
try {
const result = await inspectAnalysisView(
{ source: { kind: "evidence-file", path }, view: manifestView },
undefined,
{ signal: controller.signal, resources },
);
if (result.ok)
throw new Error("Expected cancellation with pending cleanup");
expect(result.error.cleanupIncomplete).toBe(true);
expect(result.error.cleanupResources).toEqual([path]);
expect(result.error.cleanupPrimary?.secondaryFailures).toHaveLength(1);
expect(projectAnalysisError(result.error)).toMatchObject({
details: {
diagnostics: {
primary_error: {
details: {
diagnostics: { source_error: { code: "cleanup_incomplete" } },
},
},
},
},
});
expect(resources.hasPending).toBe(true);
expect(owners.size).toBe(1);
expect([...owners][0]!.handle.fd).toBeGreaterThanOrEqual(0);
expect(projectAnalysisError(result.error)).toMatchObject({
code: "cleanup_incomplete",
details: { diagnostics: { primary_error: { code: "cancelled" } } },
});
failClose = false;
await resources.retryCleanup();
expect([...owners][0]!.handle.fd).toBe(-1);
expect(resources.hasPending).toBe(false);
expect(result.error.cleanupIncomplete).toBe(false);
expect(result.error.cleanupResources).toEqual([]);
} finally {
failClose = false;
await resources.close();
close.mockRestore();
}
});
it.each(["file", "bundle"] as const)(
"preserves the %s source authentication cancellation precedence",
async (kind) => {
const path = join(
await createTestTempDirectory("rea-view-invalid-"),
"invalid.json",
);
await writeFile(path, "{}");
const controller = new AbortController();
const dispatch = queuedInterrupts.dispatchQueuedInterrupts;
const interrupt = vi
.spyOn(queuedInterrupts, "dispatchQueuedInterrupts")
.mockImplementation(async () => {
await dispatch();
controller.abort("after-authentication");
});
try {
const result = await inspectAnalysisView(
{
source:
kind === "file"
? { kind: "evidence-file", path }
: {
kind: "retained-evidence",
evidence_id: "ev_" + "a".repeat(64),
},
view: manifestView,
},
undefined,
{
signal: controller.signal,
...(kind === "bundle" ? { evidenceBundlePath: path } : {}),
},
);
expect(result).toMatchObject({
ok: false,
error:
kind === "file"
? { _tag: "AnalysisCancelledError", cause: "after-authentication" }
: { _tag: "EvidenceIntegrityError" },
});
} finally {
interrupt.mockRestore();
}
},
);
it.each(["file", "bundle"] as const)(
"preserves authenticated %s parent Evidence when cancellation arrives before projection",
async (kind) => {
const parent = analysisViewCaptureEvidence();
const path = join(
await createTestTempDirectory("rea-view-admission-"),
"source.json",
);
await writeFile(
path,
JSON.stringify(kind === "file" ? parent : createEvidenceBundle([parent])),
);
const controller = new AbortController();
const dispatch = queuedInterrupts.dispatchQueuedInterrupts;
const interrupt = vi
.spyOn(queuedInterrupts, "dispatchQueuedInterrupts")
.mockImplementation(async () => {
await dispatch();
controller.abort("after-source-admission");
});
try {
const result = await inspectAnalysisView(
{
source:
kind === "file"
? { kind: "evidence-file", path }
: { kind: "retained-evidence", evidence_id: parent.evidence_id },
view: manifestView,
},
undefined,
{
signal: controller.signal,
...(kind === "bundle" ? { evidenceBundlePath: path } : {}),
},
);
if (result.ok) throw new Error("Expected cancellation before projection");
expect(result.error).toMatchObject({
_tag: "AnalysisCancelledError",
cause: "after-source-admission",
partialObservation: parent,
});
} finally {
interrupt.mockRestore();
}
},
);
it.each(["file", "bundle"] as const)(
"preserves completed %s view Evidence when cancellation arrives after projection",
async (kind) => {
const parent = analysisViewCaptureEvidence();
const path = join(
await createTestTempDirectory("rea-view-completion-"),
"source.json",
);
await writeFile(
path,
JSON.stringify(kind === "file" ? parent : createEvidenceBundle([parent])),
);
const controller = new AbortController();
const dispatch = queuedInterrupts.dispatchQueuedInterrupts;
let stages = 0;
const interrupt = vi
.spyOn(queuedInterrupts, "dispatchQueuedInterrupts")
.mockImplementation(async () => {
await dispatch();
stages += 1;
if (stages === 2) controller.abort("after-projection");
});
try {
const result = await inspectAnalysisView(
{
source:
kind === "file"
? { kind: "evidence-file", path }
: { kind: "retained-evidence", evidence_id: parent.evidence_id },
view: manifestView,
},
undefined,
{
signal: controller.signal,
...(kind === "bundle" ? { evidenceBundlePath: path } : {}),
},
);
if (result.ok) throw new Error("Expected cancellation after projection");
expect(result.error).toMatchObject({
_tag: "AnalysisCancelledError",
partialObservation: {
operation: "inspect_analysis_view",
evidence_links: [parent.evidence_id],
},
});
} finally {
interrupt.mockRestore();
}
},
);
it.each(["file", "bundle"] as const)(
"preserves a produced %s projection failure when cancellation arrives",
async (kind) => {
const parent = createEvidence(
undefined,
{ id: "fixture", name: "Fixture", version: "1" },
{
operation: "inspect_recorded_crash",
parameters: {},
result: { path: "/artifacts/crash.core" },
},
);
const path = join(
await createTestTempDirectory("rea-view-failure-"),
"source.json",
);
await writeFile(
path,
JSON.stringify(kind === "file" ? parent : createEvidenceBundle([parent])),
);
const controller = new AbortController();
const dispatch = queuedInterrupts.dispatchQueuedInterrupts;
let stages = 0;
const interrupt = vi
.spyOn(queuedInterrupts, "dispatchQueuedInterrupts")
.mockImplementation(async () => {
await dispatch();
if (++stages === 2) controller.abort("after-failure");
});
try {
const result = await inspectAnalysisView(
{
source:
kind === "file"
? { kind: "evidence-file", path }
: { kind: "retained-evidence", evidence_id: parent.evidence_id },
view: { kind: "summary" },
},
undefined,
{
signal: controller.signal,
...(kind === "bundle" ? { evidenceBundlePath: path } : {}),
},
);
expect(controller.signal.aborted).toBe(true);
expect(result).toMatchObject({
ok: false,
error: { _tag: "AnalysisUnsupportedTargetError" },
});
} finally {
interrupt.mockRestore();
}
},
);
describe("inspectAnalysisView scenario parents", () => {
const view = manifestView;
src/application/analysisView/AnalysisViewService.ts
+64 -38
View File
@@ -1,6 +1,7 @@
import { projectAnalysisErrorWithoutCapturedOutput } from "../../domain/analysisErrorProjection.js";
import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorCleanup.js";
import { parseAnalysisInput } from "../AnalysisInput.js";
import { isNativeObservationOperation } from "../../domain/native/nativeObservation.js";
import { setImmediate } from "node:timers/promises";
import { dispatchQueuedInterrupts } from "../../process/queuedInterrupts.js";
import { readImmutableEvidenceBundle } from "../EvidenceBundleFiles.js";
import { INSPECT_ANALYSIS_VIEW_OPERATION } from "../../domain/analysisView/analysisViewErrors.js";
@@ -65,6 +66,7 @@ export const inspectAnalysisView = async (
INSPECT_ANALYSIS_VIEW_OPERATION,
rawInput,
inspectAnalysisViewInputSchema,
options.signal,
);
if (!parsed.ok) return parsed;
if (options.evidenceBundlePath !== undefined)
@@ -95,32 +97,24 @@ const inspectFileView = async (
options,
input.source.evidence_id,
);
// Authentication and projection are synchronous; yield so a pending abort is dispatched before each check.
const cancelled = async (): Promise<boolean> => {
await setImmediate();
return options.signal?.aborted === true;
};
// Authentication and projection are synchronous; dispatch queued callbacks and
// the signal notifications they enqueue before accepting either result.
await dispatchQueuedInterrupts();
// A request cancelled during authentication reports cancellation, even when authentication failed.
if (await cancelled())
return err(
new AnalysisCancelledError(INSPECT_ANALYSIS_VIEW_OPERATION, {
cause: options.signal?.reason,
}),
);
if (!loaded.ok) return loaded;
const id = loaded.value.evidence_id;
const admitted = cancelledFileViewResult(
loaded,
options.signal,
loaded.ok ? loaded.value : undefined,
);
if (!admitted.ok) return admitted;
const id = admitted.value.evidence_id;
const projected = inspectAnalysisViewValidated(
{ ...input, source: { kind: "retained-evidence", evidence_id: id } },
(evidenceId) => (evidenceId === id ? loaded.value : lookup?.(evidenceId)),
(evidenceId) => (evidenceId === id ? admitted.value : lookup?.(evidenceId)),
);
// A request cancelled during projection delivers and records nothing.
if (await cancelled())
return err(
new AnalysisCancelledError(INSPECT_ANALYSIS_VIEW_OPERATION, {
cause: options.signal?.reason,
}),
);
return projected;
await dispatchQueuedInterrupts();
if (!projected.ok) return projected;
return cancelledFileViewResult(projected, options.signal, projected.value);
};
/** Resolve the caller's exact retained parent from an authenticated portable bundle. */
@@ -148,26 +142,58 @@ const inspectBundleView = async (
);
// Authentication and projection are synchronous; dispatch queued interrupts before accepting either result.
await dispatchQueuedInterrupts();
// Preserve an already-produced bundle failure before considering a later abort.
if (!loaded.ok) return loaded;
if (signal?.aborted)
return err(
new AnalysisCancelledError("inspect_analysis_view", {
cause: signal.reason,
}),
);
const parentId = input.source.evidence_id;
const admitted = cancelledFileViewResult(
loaded,
signal,
signal?.aborted
? loaded.value.records.find((record) => record.evidence_id === parentId)
: undefined,
);
if (!admitted.ok) return admitted;
const projected = inspectAnalysisViewValidated(
{ source: input.source, view: input.view },
(id) => loaded.value.records.find((record) => record.evidence_id === id),
(id) => admitted.value.records.find((record) => record.evidence_id === id),
);
await dispatchQueuedInterrupts();
if (projected.ok && signal?.aborted)
return err(
new AnalysisCancelledError("inspect_analysis_view", {
cause: signal.reason,
partialObservation: projected.value,
}),
);
return projected;
if (!projected.ok) return projected;
return cancelledFileViewResult(projected, signal, projected.value);
};
/** Retain completed work and pending cleanup when this stage accepts cancellation.
* File-source admission explicitly gives queued cancellation precedence over
* authentication errors; callers preserve other already-produced failures.
*/
const cancelledFileViewResult = <T>(
result: Result<T, AnalysisError>,
signal?: AbortSignal,
completedEvidence?: Evidence,
): Result<T, AnalysisError> => {
if (!signal?.aborted) return result;
const cancellation = new AnalysisCancelledError(
INSPECT_ANALYSIS_VIEW_OPERATION,
{
cause: signal.reason,
...(completedEvidence !== undefined
? { partialObservation: completedEvidence }
: {}),
},
);
if (!result.ok)
cancellation.retainSecondaryFailure(result.error, {
source_error: projectAnalysisErrorWithoutCapturedOutput(result.error),
});
return err(
!result.ok && result.error.cleanupIncomplete
? analysisErrorWithCleanupFailure(
cancellation,
result.error,
INSPECT_ANALYSIS_VIEW_OPERATION,
).bindCleanupDependencies([result.error])
: cancellation,
);
};
/** Project a parsed request whose selected source is inline or retained. */
src/application/analysisView/NativeObservationViews.test.ts
+45 -1
View File
@@ -18,7 +18,11 @@ import {
type AnalysisViewRequest,
type AnalysisViewResult,
} from "../../domain/analysisView/analysisView.js";
import { inspectAnalysisView } from "./AnalysisViewService.js";
import {
inspectAnalysisView,
projectRetainedAnalysis,
} from "./AnalysisViewService.js";
import { EvidenceLedger } from "../investigation/EvidenceLedger.js";
const provider = { id: "fixture", name: "Fixture provider", version: "1" };
const parent = (
@@ -494,6 +498,46 @@ it("keeps context identity across authenticated target path aliases while preser
});
expect(second.evidence.subject?.local_path).toBe("/other/alias.elf");
expect(second.evidence.evidence_id).not.toBe(first.evidence.evidence_id);
const ledger = new EvidenceLedger();
expect(ledger.record(original).ok).toBe(true);
const retained = ledger.forAnalysis(original.evidence_id)!;
const delivered = projectRetainedAnalysis(
rebound,
{
recordEvidence: (evidence) => ledger.record(evidence),
evidenceById: (id) => ledger.forAnalysis(id),
},
view,
);
if (!delivered.ok) throw delivered.error;
expect(delivered.value.subject?.local_path).toBe("/other/alias.elf");
expect(delivered.value.normalized_result).toMatchObject({
artifact: { path: "/other/alias.elf" },
items: {
rows: [{ record: { subject: { local_path: "/other/alias.elf" } } }],
},
});
expect(ledger.forAnalysis(original.evidence_id)).toBe(retained);
expect(retained.subject?.local_path).toBe("/selected/fixture.elf");
const imported = new EvidenceLedger();
const exporting = new EvidenceLedger();
expect(exporting.record(rebound).ok).toBe(true);
expect(imported.import(exporting.export()).ok).toBe(true);
const replay = await inspectAnalysisView(
{
source: { kind: "retained-evidence", evidence_id: rebound.evidence_id },
view,
},
(id) => imported.forAnalysis(id),
);
if (!replay.ok) throw replay.error;
expect(replay.value.subject?.local_path).toBe("/other/alias.elf");
expect(replay.value.normalized_result).toMatchObject({
artifact: { path: "/other/alias.elf" },
items: {
rows: [{ record: { subject: { local_path: "/other/alias.elf" } } }],
},
});
});
it("pages recorded empty graph levels without inventing nodes or dropping their identity", async () => {
src/application/android/AndroidPackageInspection.ts
+6 -2
View File
@@ -2,7 +2,7 @@ import type { ExecutionOptions } from "../AnalysisProvider.js";
import type { AndroidAnalysisService } from "./AndroidAnalysisService.js";
import {
requireAnalysisViewRetention,
summarizeRetainedAnalysis,
projectRetainedAnalysis,
type AnalysisViewRetention,
} from "../analysisView/AnalysisViewService.js";
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
@@ -36,5 +36,9 @@ export const inspectAndroidPackage = async (
options,
);
if (!executed.ok || parsed.value.detail !== "summary") return executed;
return summarizeRetainedAnalysis(executed.value, retention);
return projectRetainedAnalysis(
executed.value,
retention,
parsed.value.view ?? { kind: "summary" },
);
};
src/application/apkSignature/ApkSignatureService.test.ts
+29
View File
@@ -76,3 +76,32 @@ it("refuses summary acquisition before reading when complete Evidence cannot be
error: { _tag: "AnalysisCapabilityUnavailableError" },
});
});
it("rejects a live provider subject path that differs from the selected APK", async () => {
const root = await mkdtemp(join(tmpdir(), "rea-apk-subject-"));
const provider = new ApkSignatureProvider();
try {
const path = join(root, "selected.apk");
const writer = new ZipWriter(new Uint8ArrayWriter());
await writer.add("classes.dex", new TextReader("x"));
await writeFile(path, await writer.close());
const inspected = await provider.inspect({ path });
if (!inspected.ok) throw inspected.error;
const execution = inspected.value;
const service = new ApkSignatureService({
close: () => Promise.resolve(),
inspect: async () =>
ok({
...execution,
subject: { ...execution.subject, path: join(root, "other.apk") },
}),
});
expect(await service.inspect({ path, detail: "complete" })).toMatchObject({
ok: false,
error: { _tag: "AnalysisOutputError" },
});
} finally {
await provider.close();
await rm(root, { recursive: true, force: true });
}
});
src/application/apkSignature/ApkSignatureService.ts
+1
View File
@@ -70,6 +70,7 @@ export class ApkSignatureService {
if (
!report.success ||
report.data.artifact.path !== input.value.path ||
execution.subject.path !== input.value.path ||
!apkSignatureMatchesSubject(report.data.artifact, execution.subject)
)
return err(
src/application/binary/AnalysisClientCleanup.test.ts
+53
View File
@@ -88,3 +88,56 @@ it("preserves the original explanation and collected output alongside cleanup un
errors: [primary, cleanup],
});
});
it.each(["return", "reject"] as const)(
"reports a nested owner's failure once when close also %ss it",
async (outcome) => {
const failure = new ProviderCleanupError("fixture", ["owned-process"], {});
const notifications: unknown[] = [];
const child: AnalysisClient = {
execute: clientClosingWith(failure).execute,
close: (options) => {
options?.onCleanupFailure?.(failure);
return outcome === "return"
? Promise.resolve(err(failure))
: Promise.reject(failure);
},
};
const parent: AnalysisClient = {
execute: child.execute,
close: (options) => closeAnalysisClient(child, "fixture", options),
};
const closed = await closeAnalysisClient(parent, "composite", {
onCleanupFailure: (error) => notifications.push(error),
});
expect(closed).toEqual(err(failure));
expect(notifications).toEqual([failure]);
},
);
it("reports distinct failures during final cleanup and reports again on retry", async () => {
const first = new ProviderCleanupError("first", ["first-resource"], {});
const final = new ProviderCleanupError("second", ["second-resource"], {});
const notifications: unknown[] = [];
const client: AnalysisClient = {
execute: clientClosingWith(first).execute,
close: (options) => {
options?.onCleanupFailure?.(first);
return Promise.resolve(err(final));
},
};
const options = {
onCleanupFailure: (error: unknown) => notifications.push(error),
};
expect(await closeAnalysisClient(client, "composite", options)).toEqual(
err(final),
);
expect(notifications).toEqual([first, final]);
expect(await closeAnalysisClient(client, "composite", options)).toEqual(
err(final),
);
expect(notifications).toEqual([first, final, first, final]);
});
src/application/binary/AnalysisProviderRegistry.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { ProviderInspectionScope } from "../../providers/ProviderInspectionScope.js";
import { ProviderInspectionScope } from "../ProviderInspectionScope.js";
import { AnalysisCleanupLease } from "../../artifacts/AnalysisCleanupLease.js";
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js";
src/application/binary/BinarySession.ts
+31 -46
View File
@@ -1,5 +1,7 @@
import { randomUUID } from "node:crypto";
import { retainSavedAnalysisSnapshot } from "../../domain/savedAnalysisSnapshot.js";
import type { BinaryTarget } from "../../domain/binaryTargetTypes.js";
import {
analysisProfilesEqual,
@@ -10,7 +12,11 @@ import {
AnalysisCapabilityUnavailableError,
AnalysisOutputError,
} from "../../domain/analysisErrorCore.js";
import { AnalysisError } from "../../domain/analysisErrorBase.js";
import {
AnalysisError,
primaryAnalysisError,
} from "../../domain/analysisErrorBase.js";
import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
import { err, ok, type Result } from "../../domain/result.js";
import type { JsonValue } from "../../domain/jsonValue.js";
import { createAnalysisExecutionEvidence } from "../../domain/analysisObservation.js";
@@ -22,8 +28,6 @@ import type {
ProviderIdentity,
AnalysisOperation,
} from "../AnalysisProvider.js";
import { OFFICIAL_TOOL_CONTRACTS } from "../../contracts/officialToolContracts.js";
import { ENHANCED_TOOL_CONTRACTS } from "../../contracts/enhancedToolContracts.js";
import type {
BinarySessionPort,
SavedAnalysisSnapshot,
@@ -32,6 +36,10 @@ import {
SessionProviderRouter,
type SessionProviderRoute,
} from "./SessionProviderRouter.js";
import {
routeAllowsSnapshotReplay,
routeProviderIdentity,
} from "./SessionProviderPolicy.js";
import { InvestigationRecords } from "../investigation/InvestigationRecords.js";
import { recordEvidenceSources } from "../investigation/recordEvidenceSources.js";
import { BinarySessionRecords } from "./BinarySessionRecords.js";
@@ -68,12 +76,6 @@ import {
type ActiveBundleResourceSession,
type BundleResourceClientFactory,
} from "./BundleResourceSession.js";
const OFFICIAL_OPERATIONS: ReadonlySet<string> = new Set(
OFFICIAL_TOOL_CONTRACTS.map(({ name }) => name),
);
const ENHANCED_OPERATIONS: ReadonlySet<string> = new Set(
ENHANCED_TOOL_CONTRACTS.map(({ name }) => name),
);
interface SessionRun {
readonly client: AnalysisClient;
@@ -384,38 +386,14 @@ export class BinarySession
/** Identify the provider producing evidence for this session. */
providerIdentity(operation?: AnalysisOperation): ProviderIdentity {
const route = this.#currentRoute();
let selected = route.binding?.identity ?? route.identity;
if (operation !== undefined) {
const exact = route.capabilities.get(operation)?.provider;
if (exact !== undefined) selected = exact;
else if (ENHANCED_OPERATIONS.has(operation)) {
const providers = new Map<string, ProviderIdentity>();
for (const descriptor of route.capabilities.values())
if (
descriptor.available &&
OFFICIAL_OPERATIONS.has(descriptor.operation)
)
providers.set(descriptor.provider.id, descriptor.provider);
if (providers.size === 1) {
const provider = providers.values().next().value;
if (provider !== undefined) selected = provider;
}
}
}
return structuredClone(selected);
return routeProviderIdentity(this.#currentRoute(), operation);
}
/** External live state cannot be replayed by a direct or composed CLI workflow. */
allowsSnapshotReplay(operation: AnalysisOperation): boolean {
if (operation === "decompile_all") return false;
if (this.#activeBundleResources !== undefined) return false;
const capabilities = this.#active?.route.capabilities;
const descriptor = capabilities?.get(operation);
if (descriptor !== undefined) return descriptor.cachePolicy !== "live";
return ![...(capabilities?.values() ?? [])].some(
({ cachePolicy }) => cachePolicy === "live",
);
return routeAllowsSnapshotReplay(this.#active?.route, operation);
}
invalidateSnapshotForExternalMutation(): void {
@@ -848,15 +826,16 @@ export class BinarySession
overwrite,
);
if (!written.ok) return written;
const receipt: SavedAnalysisSnapshot = {
...written.value,
primitive_entries: snapshot.value.entries.length,
workflow_entries: snapshot.value.workflow_entries.length,
evidence_records: snapshot.value.evidence_bundle.records.length,
};
const closed = await this.#closeActive(options);
return closed.ok
? ok({
...written.value,
primitive_entries: snapshot.value.entries.length,
workflow_entries: snapshot.value.workflow_entries.length,
evidence_records: snapshot.value.evidence_bundle.records.length,
})
: closed;
? ok(receipt)
: err(retainSavedAnalysisSnapshot(closed.error, receipt));
});
}
@@ -1031,9 +1010,15 @@ export class BinarySession
primary: AnalysisError,
): Promise<AnalysisError> {
const restored = await this.#restore(previous);
return restored.ok
? primary
: sessionCleanupFailure(primary, restored.error, "open_binary");
if (restored.ok) return primary;
primary.retainSecondaryFailure(restored.error, {
restoration_error: projectAnalysisError(
primaryAnalysisError(restored.error),
),
});
return restored.error.cleanupIncomplete
? sessionCleanupFailure(primary, restored.error, "open_binary")
: primary;
}
/** Describe the active binary session. */
@@ -1460,7 +1445,7 @@ export class BinarySession
}
const closed = await this.#retireClient(run);
return closed.ok
? ok(null)
? err(started.error)
: err(sessionCleanupFailure(started.error, closed.error, "open_binary"));
}
src/application/binary/BinarySessionBundleResources.test.ts
+31 -1
View File
@@ -1,4 +1,4 @@
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { mkdir, mkdtemp, rm, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, expect, it } from "vitest";
@@ -44,6 +44,35 @@ afterEach(async () => {
);
});
it.skipIf(process.platform === "win32")(
"refuses invalid selected traversal before creating a bundle resource client",
async () => {
const root = await mkdtemp(join(tmpdir(), "rea-resource-selected-"));
temporaryDirectories.push(root);
await mkdir(join(root, "Fixture.app"));
await writeFile(
join(root, "Fixture.app", "Info.plist"),
"<plist><dict/></plist>",
);
await writeFile(join(root, "regular"), "file");
let created = false;
const session = new BinarySession(
SessionProviderRouter.selectable(new AnalysisProviderRegistry([]), []),
new InvestigationRecords(),
{
createClient: () => {
created = true;
throw new Error("Invalid selection must not create a client");
},
},
);
const selected = `${root}/regular/../Fixture.app`;
await expect(stat(selected)).rejects.toMatchObject({ code: "ENOTDIR" });
expect((await session.openBundleResources(selected)).ok).toBe(false);
expect(created).toBe(false);
},
);
it("opens a resource-only bundle without executable identity and keeps snapshot operations inapplicable", async () => {
const rootPath = await mkdtemp(join(tmpdir(), "rea-resource-session-"));
temporaryDirectories.push(rootPath);
@@ -84,6 +113,7 @@ it("opens a resource-only bundle without executable identity and keeps snapshot
const opened = await session.openBundleResources(rootPath);
expect(opened.ok).toBe(true);
if (!opened.ok) throw opened.error;
expect(session.allowsSnapshotReplay("inspect_plist")).toBe(false);
expect(opened.value.executableSha256).toBeNull();
expect(openedRoot).toBe(opened.value.canonicalPath);
expect(
Loaded 100 of 621 files, more files were not shown because too many files have changed in this diff. Show more