diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index e990e3923d..b659cff7ef 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -2,7 +2,7 @@ name: reverse-engineer-anything description: Reverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. metadata: - version: "42" + version: "43" --- # REA @@ -256,6 +256,12 @@ Provider candidate facts describe possible providers, while top-level session capabilities describe the current route and `analysis_provider_binding` names the selected provider. Do not treat those scopes as interchangeable. +For a Ghidra startup timeout on a large native target, follow +[deadline recovery](references/native-and-artifacts.md#ghidra-startup-deadline-recovery) +before abandoning deep analysis or repeating the same cold import. The default +330 seconds is configurable; changing the server's environment requires a new +REA process. + ## Work summary-first Start with the default result and use its inline Evidence and graph context. diff --git a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md index 80c2cddd46..675de228cc 100644 --- a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md +++ b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md @@ -155,6 +155,36 @@ operation's full ordered list. Candidate capabilities describe possible providers. Top-level session capabilities describe the current route, and `analysis_provider_binding` identifies the selected provider. +## Ghidra startup deadline recovery + +`provider_timeout` during the first Ghidra query can cover import, auto-analysis, +bridge startup, and readiness. The default 330,000 ms is configurable through +`REA_GHIDRA_STARTUP_TIMEOUT_MS`; a smaller query still needs the same cold import. + +Preserve useful Evidence before closing the session or restarting REA. Check +`close_binary` and resolve reported cleanup failures. Set a larger startup budget +in the environment that launches REA, then restart REA and reconnect the MCP +client before reopening the same target. Closing and reopening a binary in the +existing server keeps its original configuration and starts import again. +Environment changes in an agent's shell do not update an already running MCP +server. If its launch configuration cannot be changed, report that constraint +and the exact retry setting instead of claiming a fixed technical limit. + +The client's tool-call deadline must allow startup plus the requested query; +server settings and progress notifications cannot override it. CLI analysis +avoids the MCP client deadline, but retains the provider deadline. For example: + +```sh +REA_GHIDRA_STARTUP_TIMEOUT_MS=1800000 rea analyze /absolute/path/to/app.exe --provider ghidra --json +``` + +Thirty minutes is an example budget, not a completion guarantee. Keep host CPU +and memory controls in place. Disable specific expensive analyzers only when +their missing coverage is acceptable and retain the resulting limitations. +Exported C and embedded resources remain useful independent evidence, but a +timeout does not establish complete analysis or absence of code. See the +[deadline and client guide](https://github.com/morluto/rea/blob/main/docs/mcp-contracts.md#ghidra-first-query-deadlines-and-recovery). + ## Static memory and original file bytes For Ghidra, `read_bytes` reads initialized analysis memory, including loader diff --git a/.agents/skills/reverse-engineer-anything/references/runtime-observation.md b/.agents/skills/reverse-engineer-anything/references/runtime-observation.md index df75bcaf3d..a67648c802 100644 --- a/.agents/skills/reverse-engineer-anything/references/runtime-observation.md +++ b/.agents/skills/reverse-engineer-anything/references/runtime-observation.md @@ -69,6 +69,12 @@ with `inspect_analysis_view` and `view: {kind: "capture", detail: "preview", off Select complete record pages only for needed headers, metadata or payloads; views read the historical parent without repeating capture decoding. +For selected protobuf/gRPC bodies, keep raw bytes as the source and use +`inspect_analysis_view` kind `protobuf`, collection `bodies`, `frames` or +`fields`. Batch exact record ordinals and field numbers/IDs, and select +`schema_value` or `text` only when needed. Supply a local descriptor set and +message type for schema interpretation; compressed payloads remain unknown. + Inspect native `.saz` snapshots with `inspect_web_network_capture` and `format: "saz"`; encrypted archives are unsupported. For live traffic analysis, connect Everywhere's separate MCP server and use its diff --git a/docs/installation.md b/docs/installation.md index 184778a6b3..9007209830 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -987,6 +987,24 @@ can time out earlier even when Ghidra would complete within its startup deadline See [Ghidra first-query deadlines and recovery](mcp-contracts.md#ghidra-first-query-deadlines-and-recovery) for client options and the close/reopen recovery flow. +The 330-second default is configurable. Configuration is read when REA starts: +changing an environment variable in another shell, or closing and reopening a +binary in the same MCP server, does not change that server's startup allowance. +For a CLI retry with an example 30-minute allowance: + +```sh +REA_GHIDRA_STARTUP_TIMEOUT_MS=1800000 rea analyze /absolute/path/to/app.exe --provider ghidra --json +``` + +For MCP, export useful Evidence before restarting, close the binary and resolve +any cleanup failure, update the environment in the client's REA server launch +configuration, then restart REA and reconnect before reopening the target. Allow +startup plus the query in the client's own tool-call deadline. Every retry uses +a fresh ephemeral import; smaller result requests do not reduce the prerequisite +auto-analysis. A larger time allowance does not raise the JVM heap limit or +guarantee completion. Keep [heap and host CPU controls](#ghidra-heap-and-cpu-controls) +appropriate for the machine. + One session contains exactly one imported Program; use `provider_id: "ghidra"`, `--provider ghidra`, or `REA_ANALYSIS_PROVIDER=ghidra` when both Hopper and Ghidra support the target. One persistent decompiler is owned by the Program, and a serial queue keeps diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 77daba44f3..a753543ecc 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -22,6 +22,14 @@ directly to `compare_functions`, and `inspect_artifact` Evidence to `compare_artifacts`. Use `get_evidence_bundle` when the task needs broader retained session history or an explicit bundle for transfer. +Ghidra inventories and function replies use `raw_result: null` when their facts +are already preserved in `normalized_result`, limitations, locations and +relationship coverage. Distinct overview details, measured load-image +observations, file-offset selection roles and database types supplemented with +derived metadata keep their upstream representation. Read function results from +`normalized_result`; older retained Evidence keeps its original representation +and identity. + Artifact inventories declare `coverage.scope` (`root` or `container-entries`) and `coverage.status` (`complete` or `unknown`) independently of diagnostic wording. Comparison requires matching scopes, complete graph members and complete @@ -775,6 +783,13 @@ the current client, so it is not a fresh-start recovery. On the same connection: `provider_id: "ghidra"`, then retry the first query with an appropriate client deadline. The connection and selected provider do not need to change. +If REA's provider startup allowance also needs raising, preserve Evidence and +close as above, then update `REA_GHIDRA_STARTUP_TIMEOUT_MS` in REA's launch +environment and restart the server. Reconnect the MCP client before reopening +the target. Configuration is read at process startup; close/reopen on the same +connection cannot apply a changed environment. See the +[CLI retry example](installation.md#ghidra). + This close/reopen flow was exercised on one real Linux stdio connection after a controlled startup timeout, followed by a successful overview and function analysis. A separate Windows x64 stdio run with Ghidra 12.1.4, JDK 21 and Node diff --git a/docs/specs/agent-environments/state-view/verification.md b/docs/specs/agent-environments/state-view/verification.md index 26101f651a..48db9935b2 100644 --- a/docs/specs/agent-environments/state-view/verification.md +++ b/docs/specs/agent-environments/state-view/verification.md @@ -91,7 +91,7 @@ workflow: 22.836 s / 19.880 CPU s / 1,072.43 MiB; actual browser-state workflow: exhausts an isolated decoder heap and verifies that MCP remains usable; no workspace OOM occurred. -Before/after used immutable pre-fix `29e745923` and the final source, the same +The first follow-up benchmark used pre-fix `29e745923` and `6ad63bcc2`, the same completed Evidence files, Node and CPU affinity. One fresh process per revision, with a first call and two warm repeats; filesystem caches were not cleared. Complete normalized output digests matched for both synthetic projection tasks. @@ -116,3 +116,41 @@ No new macOS/Windows runtime, state serving, AgentEnv data-plane or write-effect claim is made. A type first observed only after an unrelated-root write may still lack the earlier observation needed to detect its effect. Item-shaped and delete-only document paths remain recorded, as specified. + +## Post-merge write-cutoff follow-up + +Base `3fca2e0ca` includes #5319 and the validated immutable-capture reuse from +#5349. The remaining review finding was repeated write-root parsing for each +read type and again during output assembly. One admitted prefix/wildcard index +now owns parsed write roots, with per-projection positive/negative cutoff +caching. Backed type membership is collected once, replacing another plan scan. +The old linear matcher is removed. Prefix, parameter, redacted-root and timeline +semantics are preserved; no cross-request cache or new authority is introduced. + +- Source checks: 41 files, 508 passed, 3 existing skips, including independent + matching-oracle and 12,000-segment iterative-lookup checks. +- Serial static/build/artifact gate: 11 tasks passed, 197.356 s wall / + 197.195 CPU s / 4,744.92 MiB peak process-family RSS. +- Both selected compiled state CLI/MCP cases passed after rebuilding. +- Live Gitea HAR workflows and fresh actual Chromium capture passed again: + initial values, recorded fallbacks, CLI/default stdio parity and six exact + HTTP replays. HAR workflow: 22.834 s wall / 19.623 CPU s / 1,067.36 MiB; + browser workflow: 22.858 s / 20.912 CPU s / 1,546.59 MiB. Owned processes stopped. + +The same completed Evidence with 1,000 collection reads and 1,000 unrelated +writes was projected before and after on one CPU. Two larger full-HAR inputs +(3,000/4,000 total records) were refused by the existing 96 MiB admission budget; +the limit was retained. One fresh process per revision, first call plus one +warm repeat, filesystem caches not cleared: + +| Complete file-to-state projection | Main before | Follow-up | +| -------------------------------------------------------- | ------------------- | ------------------- | +| First call wall / CPU (ms) | 1,727.89 / 1,727.36 | 1,244.82 / 1,244.05 | +| Warm call wall / CPU (ms) | 1,300.84 / 1,300.42 | 624.01 / 623.89 | +| Process wall / CPU (s), including imports and both calls | 4.690 / 4.363 | 3.370 / 3.062 | +| Sampled process-family peak RSS (MiB) | 365.84 | 360.03 | + +Parent Evidence IDs and complete normalized output digests were identical. +RSS sampling is every 50 ms, includes descendants and can double-count shared +pages; these few runs do not establish a general memory reduction. Model token +usage remains unmeasured. No workspace OOM occurred. diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md index 8f12239c13..9345e8e0cd 100644 --- a/docs/web-network-captures.md +++ b/docs/web-network-captures.md @@ -122,29 +122,96 @@ XML node tree; optional `_w.txt` and `_g.txt` files retain opaque WebSocket and gRPC message bytes. Missing members remain null, distinct from empty bodies. See Telerik's [SAZ format reference](https://www.telerik.com/fiddler/fiddler-everywhere/documentation/knowledge-base/fiddler-archives). -The existing ZIP reader verifies read members; expanded archive contents are -limited to 32 MiB in addition to the input/output budgets below. Exceeding this -expanded-byte budget returns `invalid_request` / `invalid_input` through both -CLI and MCP, with an `out_of_range` issue at -`capture_path` and `maximum_expanded_capture_bytes`. Selecting records does not -bypass the complete-archive admission check or return partial Evidence. Encrypted -archives and unrecognized raw session members are unsupported. SAZ requires an -enforced decoder memory bound: Node's fixed heap, or the Windows Job Object -commit limit. Bun on hosts without that Job Object limit returns capability -unavailable before starting the SAZ decoder. Non-session -members are inventoried without decoding their content. Original files remain -unchanged. Linux verification includes independently checked public recorded -Mattermost captures from [FOREST](https://github.com/dfrc-korea/FOREST/tree/405c9154658a128ea203a6c10040cec55111ed03/api-results/dataset) -and the Fiddler author's [encoding corpus](https://github.com/ericlaw1979/Clearinet/blob/b6faf78cde1e72042980364f180ceb34b77d1434/Test/data/Encodings.saz), -plus synthetic malformed/limit fixtures. Their exporting product versions are -not authenticated. An interactive export from a current Fiddler Everywhere -installation and its live MCP server remain unverified. Support here is the -bounded documented raw HTTP/XML SAZ layout, not every product/version profile. +## Protobuf and gRPC bodies -For visual inspection, use Everywhere's **Import Snapshot**, then **Inspectors** -and **Overview**. HAR remains an alternative: select sessions, choose -**Export > HTTPArchive**, and confirm `log.version` is `"1.2"`. Preserve the -original SAZ when exporting. See the [import/export instructions](https://www.telerik.com/fiddler/fiddler-everywhere/documentation/collaboration/exporting-and-importing). +Selecting `record_ordinals` (CLI `--record`) also decodes retained bodies with +recognized protobuf or gRPC content types. Whole-capture inspection does not +perform this extra work. The original binary sidecar, SHA-256 and producer +ordinal remain the source; decoding adds flat body, frame and field facts. + +Native gRPC admits only data-frame flags 0/1; gRPC-web additionally admits its +terminal trailer frame. `grpc-web-text` supports independently padded base64 +chunks. Its offsets refer to **base64-decoded body bytes**; other offsets refer +to retained body bytes. Every field has a body-scoped `field_id` derived from +its byte offset, a `frame_index`, and a `parent_field_id` for schema-driven +nested messages or groups. Repeated field numbers remain separate occurrences. + +Without a schema, varints and fixed-width values retain exact unsigned decimal +representations. Length-delimited UTF-8, nested-message and packed readings +are candidates; REA does not choose one or invent field names or signedness. +Nested candidate ranges remain accessible without recursively expanding them. +Malformed messages retain the parsed prefix and an exact error offset. +Compressed frames and encoded HTTP bodies remain unknown; they are not +silently treated as empty messages. + +For schema interpretation, supply a local compiled `FileDescriptorSet` and its +fully qualified message type. All imports must be included. REA never fetches +schemas or executes the selected application. Descriptor identity and parser +version remain in the body facts. Scalars, repeated numeric packing and nested +messages are interpreted alongside the unchanged wire facts; disagreements +remain explicit. Map semantics, extensions and required-field validation are +unsupported. A supplied schema does not establish the sender's actual schema. +An invalid nested interpretation records `schema_status: "invalid-message"` +and `schema_error` locally; later outer fields remain available. Valid string +contents, including a leading U+FEFF, are preserved. + +```bash +# Compile your own schema with your existing protoc, including local imports: +protoc --include_imports --descriptor_set_out=/evidence/api.pb api.proto +rea inspect-web-network-capture /evidence/session.har har --record 3 \ + --protobuf-schema /evidence/api.pb --protobuf-message example.Response \ + --detail summary --evidence_output /evidence/session-evidence.json --json +``` + +MCP accepts the same choices as `protobuf: {schema: {descriptor_set_path: +"/evidence/api.pb", message_type: "example.Response"}}`, with explicit +`record_ordinals`. Optional `body_pointers` batch-select exact binary pointers; +`framing` can explicitly select `raw`, `grpc`, `grpc-web` or `grpc-web-text` +when MIME metadata is unavailable. CLI exposes `--protobuf-body-pointer` and +`--protobuf-framing`. Filters never invent bytes excluded by source redaction. Decoding an extra +base64 layer also checks explicit exclusions; newly recognized trailer +credentials exclude the entire affected binary sidecar and its reported +encoding, preserving a redaction coordinate instead of a credential digest. +Credential inspection covers every selected body before descriptor reads or +field expansion. Later capacity or schema failures can therefore retain the +sanitized source even with explicit sensitivity options. If that source pass +is interrupted, unchecked bodies are not published as partial Evidence. +Framing overrides apply only to format-owned body coordinates and preserve +exclusions for the transport declared in MIME metadata. +Unknown or explicitly excluded HTTP coding metadata is not treated as absent: +SAZ/native bodies remain opaque until identity coding is established. Raw +header values and retained body bytes remain available in source Evidence. +Skipping wire analysis still runs the independent credential check. If HTTP +coding metadata is unavailable and that check cannot complete, the operation +returns unavailable without publishing unchecked capture bodies as Evidence. + +Use `detail: "summary"`, then `inspect_analysis_view` with its retained parent: + +```json +{"kind":"protobuf","collection":"bodies","offset":0,"limit":16} +{"kind":"protobuf","collection":"frames","record_ordinals":[3],"offset":0,"limit":16} +{"kind":"protobuf","collection":"fields","record_ordinals":[3],"field_numbers":[1,4],"fields":["number","schema_name","schema_value"],"offset":0,"limit":32} +``` + +Filters intersect; an empty filter selects nothing. Field pages also accept +`field_ids`, `frame_indices` and `schema_names`. Default field pages omit `text`, +`schema_value` and enum alias inventories; select these columns when needed. Identity, source offset +basis and body status survive column selection. Complete Evidence retains +all decoded values, and ordinary capture previews expose counts. Page exhaustion +covers retained facts only; a partial source does not establish absence. + +Authenticated immutable parents reuse validation and the most recent field +filter per capture. These caches live only with retained Evidence, hold references +to existing facts and never reopen either file. Cached selector text is bounded +by retained field cardinality; larger selections use an uncached scan. Mutable +input is revalidated. +The existing 96 MiB report budget includes the original decoder reply plus +conservative charges for new facts, strings and descriptor expansion. The explicit +`protobuf_coverage` reports selected and inspected record counts, preserving +partial decoding through composed views. Capacity failure retains the source +observation as partial Evidence; cancellation yields +between bounded work batches. Real elapsed time and process-family memory +still depend on the selected capture and schema. ## Fiddler Everywhere MCP and public skills diff --git a/experiments/analysis-efficiency/README.md b/experiments/analysis-efficiency/README.md index 3d9b728aa5..27667bac93 100644 --- a/experiments/analysis-efficiency/README.md +++ b/experiments/analysis-efficiency/README.md @@ -5,6 +5,11 @@ reconciles the earlier audit queue and measures immutable historical-capture validation reuse on public compact and large qlogs, including first-view costs, small-input limits and unchanged Evidence delivery. +The [Ghidra Evidence projection checkpoint](ghidra-evidence-projection.md) +measures removal of redundant raw copies on real `true` and jq workflows, +with complete provenance, unchanged response budgets, native CLI/MCP checks +and explicit cold-analysis variance. + For Hopper literal string queries, see the [decoded-record checkpoint](hopper-string-records.md). It measures record reuse through real Linux CLI/MCP workflows, preserving native recognition gaps, guard-stopped attempts and negative whole-task costs. diff --git a/experiments/analysis-efficiency/ghidra-evidence-projection.json b/experiments/analysis-efficiency/ghidra-evidence-projection.json new file mode 100644 index 0000000000..edb99d43ab --- /dev/null +++ b/experiments/analysis-efficiency/ghidra-evidence-projection.json @@ -0,0 +1,719 @@ +{ + "issue": 5337, + "initial_base_revision": "4e716b06078d725a53234503cbefbb340731e809", + "integration_base_revision": "3fca2e0ca6e54ba700c286d9a85a9600dee6a19d", + "environment": { + "platform": "Linux x86_64", + "node": "24.18.0", + "npm": "11.16.0", + "mcp_sdk": "2.3.1", + "ghidra": "12.1.4", + "java": "Eclipse Adoptium 21.0.12.1+1-LTS", + "configured_java_heap": "768M", + "configured_active_processors": 1, + "query_cpu_quota_percent": 100, + "query_cpu_affinity": [15], + "query_hard_memory_bytes": 4294967296, + "query_early_stop_bytes": 3221225472, + "query_task_limit": 192, + "swap_bytes": 0, + "os_caches": "warm and unflushed; fresh servers, homes and Java analysis", + "initial_startup_and_rpc_deadline_ms": 300000, + "integrated_startup_and_rpc_deadline_ms": 600000, + "product_default_deadlines_changed": false + }, + "method": { + "task": "Full observed procedure inventory, overview/load image and first internal non-thunk function facts; repeat inventory/pseudocode, export Evidence and close.", + "calls": 15, + "catalog_discoveries": 1, + "fact_comparison_excludes_only": ["evidence_id", "raw_result"], + "bytes": "Node JSON.stringify UTF-8 of complete SDK catalog, tool name/argument objects and reply envelopes; initialization, wire framing and model rendering unmeasured.", + "actual_model_usage": null, + "tokenizer_estimates": null, + "memory": "Kernel cgroup family peak including file cache at payload exit; post-GC retained heap unmeasured.", + "counter_scope": "Verified payload exit while the observer remains alive; final whole-scope counters after removal unavailable.", + "latency_claim": "No whole-task latency, CPU or peak-memory improvement established.", + "candidate_control_overhead": "Baseline fact-file parsing/comparison is included on the candidate side; its isolated cost is unmeasured." + }, + "runtime_tables": { + "baseline": { + "files": 1841, + "sha256_sorted_path_tab_digest_lines": "549bb479ed41e9c55505c169e726552b632f0e5341e63194380b8d0ded344658", + "revision": "4e716b06078d725a53234503cbefbb340731e809" + }, + "candidate": { + "files": 1841, + "sha256_sorted_path_tab_digest_lines": "0ebceab15311cc5ccdcfb92a0891022fc37736f2e7d01e2c1d22e88920df1ca8", + "base_revision": "4e716b06078d725a53234503cbefbb340731e809", + "source_diff_sha256": "7c460367e77daf2fad507800f1346dcd9e74553999fb049232b33b27684c8f49" + }, + "integrated-baseline": { + "files": 1847, + "sha256_sorted_path_tab_digest_lines": "0d1672378dd3989663f7ef9ead3a91f56283dd06868f4574cae4d062dc83b041", + "base_revision": "3fca2e0ca6e54ba700c286d9a85a9600dee6a19d" + }, + "integrated-candidate": { + "files": 1847, + "sha256_sorted_path_tab_digest_lines": "d4348d25b4b18e371ac63375accd8f14e72197588934f66b8ab2bb7049ddb7d3", + "base_revision": "3fca2e0ca6e54ba700c286d9a85a9600dee6a19d", + "source_diff_sha256": "7c460367e77daf2fad507800f1346dcd9e74553999fb049232b33b27684c8f49", + "driver_sha256": "b0463f8da110dcbae45769045a690dc5accf74791fa53f5086a60e8383f074de" + } + }, + "driver_versions": { + "earlier": "b5bf4b378102de16f871c5f13e7302cbdcecbe0a14cc90b30ab80031920aaa06", + "integrated": "b0463f8da110dcbae45769045a690dc5accf74791fa53f5086a60e8383f074de" + }, + "completed_tasks": [ + { + "name": "baseline-true-4", + "expected_projection": "duplicate", + "artifact_sha256": "c79bf44242829108e323378531f4ac839513ca1fba45efd6583643526e1e9fd2", + "artifact_bytes": 35664, + "procedure_count": 231, + "selected_procedure": "0x102000", + "facts_sha256": "5b826454c00709fb26777e3b6663aa69ce879f51597c54d6ec3021fc1984eb74", + "call_count": 15, + "catalog": { + "elapsed_ms": 1132.755321, + "bytes": 3255374, + "sha256": "8a646b95cc5607ee57e5a3c0ef480b77e4cec7e5168498e72f511a8af40daf61" + }, + "rpc_argument_bytes": 908, + "rpc_reply_bytes": 473344, + "catalog_arguments_replies_bytes": 3729626, + "normalized_value_bytes_once_per_selected_reply": 94357, + "raw_value_bytes_once_per_selected_reply": 90839, + "bundle_bytes": 177922, + "phases": { + "open_binary": { + "elapsed_ms": 913.6441110000001, + "reply_bytes": 499 + }, + "list_documents": { + "elapsed_ms": 27770.679519, + "reply_bytes": 6029 + }, + "list_procedures": { + "elapsed_ms": 156.53601400000116, + "reply_bytes": 125181 + }, + "binary_overview": { + "elapsed_ms": 101.32018699999753, + "reply_bytes": 17387 + }, + "inspect_native_load_image": { + "elapsed_ms": 265.31070799999725, + "reply_bytes": 101123 + }, + "procedure_info": { + "elapsed_ms": 80.39036699999997, + "reply_bytes": 7643 + }, + "procedure_pseudo_code": { + "elapsed_ms": 81.57453700000042, + "reply_bytes": 6549 + }, + "procedure_assembly": { + "elapsed_ms": 53.14257799999905, + "reply_bytes": 6715 + }, + "procedure_callers": { + "elapsed_ms": 22.954539999998815, + "reply_bytes": 6709 + }, + "procedure_callees": { + "elapsed_ms": 8.837930000001506, + "reply_bytes": 6753 + }, + "analyze_function": { + "elapsed_ms": 294.8276279999991, + "reply_bytes": 56545 + }, + "warm-inventory": { + "elapsed_ms": 49.338327999998, + "reply_bytes": 125181 + }, + "warm-pseudocode": { + "elapsed_ms": 16.670878999997512, + "reply_bytes": 6549 + }, + "export_evidence_bundle": { + "elapsed_ms": 51.058947999998054, + "reply_bytes": 312 + }, + "close_binary": { + "elapsed_ms": 494.6449689999936, + "reply_bytes": 169 + } + }, + "scope_seconds": 34.369, + "payload_cpu_seconds": 33.150037, + "payload_family_peak_bytes": 993923072, + "payload_cleanup_verified": true, + "zero_limit_events_at_payload_exit": true, + "final_whole_scope_counters_verified": false + }, + { + "name": "candidate-true-1", + "expected_projection": "compact", + "artifact_sha256": "c79bf44242829108e323378531f4ac839513ca1fba45efd6583643526e1e9fd2", + "artifact_bytes": 35664, + "procedure_count": 231, + "selected_procedure": "0x102000", + "facts_sha256": "5b826454c00709fb26777e3b6663aa69ce879f51597c54d6ec3021fc1984eb74", + "call_count": 15, + "catalog": { + "elapsed_ms": 1208.093488, + "bytes": 3255374, + "sha256": "8a646b95cc5607ee57e5a3c0ef480b77e4cec7e5168498e72f511a8af40daf61" + }, + "rpc_argument_bytes": 909, + "rpc_reply_bytes": 326642, + "catalog_arguments_replies_bytes": 3582925, + "normalized_value_bytes_once_per_selected_reply": 94357, + "raw_value_bytes_once_per_selected_reply": 22114, + "bundle_bytes": 137135, + "phases": { + "open_binary": { + "elapsed_ms": 941.0834749999995, + "reply_bytes": 499 + }, + "list_documents": { + "elapsed_ms": 27897.447016, + "reply_bytes": 5981 + }, + "list_procedures": { + "elapsed_ms": 136.00869700000112, + "reply_bytes": 65589 + }, + "binary_overview": { + "elapsed_ms": 124.65224699999817, + "reply_bytes": 17387 + }, + "inspect_native_load_image": { + "elapsed_ms": 249.8859930000035, + "reply_bytes": 101123 + }, + "procedure_info": { + "elapsed_ms": 95.36772800000472, + "reply_bytes": 6671 + }, + "procedure_pseudo_code": { + "elapsed_ms": 84.8552570000029, + "reply_bytes": 6355 + }, + "procedure_assembly": { + "elapsed_ms": 97.34392800000205, + "reply_bytes": 6305 + }, + "procedure_callers": { + "elapsed_ms": 8.018889999999374, + "reply_bytes": 6177 + }, + "procedure_callees": { + "elapsed_ms": 6.999988999996276, + "reply_bytes": 6199 + }, + "analyze_function": { + "elapsed_ms": 271.9528430000064, + "reply_bytes": 31929 + }, + "warm-inventory": { + "elapsed_ms": 30.13376900000003, + "reply_bytes": 65589 + }, + "warm-pseudocode": { + "elapsed_ms": 7.71154899999965, + "reply_bytes": 6355 + }, + "export_evidence_bundle": { + "elapsed_ms": 71.93626799999765, + "reply_bytes": 314 + }, + "close_binary": { + "elapsed_ms": 620.6287440000015, + "reply_bytes": 169 + } + }, + "scope_seconds": 34.901, + "payload_cpu_seconds": 33.834228, + "payload_family_peak_bytes": 1025228800, + "payload_cleanup_verified": true, + "zero_limit_events_at_payload_exit": true, + "final_whole_scope_counters_verified": false + }, + { + "name": "baseline-jq-1", + "expected_projection": "duplicate", + "artifact_sha256": "b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f", + "artifact_bytes": 2267912, + "procedure_count": 2276, + "selected_procedure": "0x401000", + "facts_sha256": "f7b95167c93f2b9129b5a4332590ef6bbb9d7b646d554ba4c396db1055ef0d24", + "call_count": 15, + "catalog": { + "elapsed_ms": 2761.8931860000002, + "bytes": 3255374, + "sha256": "8a646b95cc5607ee57e5a3c0ef480b77e4cec7e5168498e72f511a8af40daf61" + }, + "rpc_argument_bytes": 965, + "rpc_reply_bytes": 2372592, + "catalog_arguments_replies_bytes": 5628931, + "normalized_value_bytes_once_per_selected_reply": 536023, + "raw_value_bytes_once_per_selected_reply": 533635, + "bundle_bytes": 609212, + "phases": { + "open_binary": { + "elapsed_ms": 1856.9621389999993, + "reply_bytes": 617 + }, + "list_documents": { + "elapsed_ms": 290349.418768, + "reply_bytes": 6167 + }, + "list_procedures": { + "elapsed_ms": 624.2473109999555, + "reply_bytes": 1092959 + }, + "binary_overview": { + "elapsed_ms": 214.38793299999088, + "reply_bytes": 15727 + }, + "inspect_native_load_image": { + "elapsed_ms": 432.1906869999948, + "reply_bytes": 75633 + }, + "procedure_info": { + "elapsed_ms": 106.62601700000232, + "reply_bytes": 7813 + }, + "procedure_pseudo_code": { + "elapsed_ms": 165.65909500000998, + "reply_bytes": 6869 + }, + "procedure_assembly": { + "elapsed_ms": 98.534897000005, + "reply_bytes": 6855 + }, + "procedure_callers": { + "elapsed_ms": 23.43981900002109, + "reply_bytes": 6891 + }, + "procedure_callees": { + "elapsed_ms": 9.403080000018235, + "reply_bytes": 7479 + }, + "analyze_function": { + "elapsed_ms": 327.3197410000139, + "reply_bytes": 45277 + }, + "warm-inventory": { + "elapsed_ms": 396.67419799999334, + "reply_bytes": 1092959 + }, + "warm-pseudocode": { + "elapsed_ms": 6.517379999975674, + "reply_bytes": 6869 + }, + "export_evidence_bundle": { + "elapsed_ms": 56.13109899999108, + "reply_bytes": 308 + }, + "close_binary": { + "elapsed_ms": 550.6349530000007, + "reply_bytes": 169 + } + }, + "scope_seconds": 304.543, + "payload_cpu_seconds": 294.743485, + "payload_family_peak_bytes": 1159921664, + "payload_cleanup_verified": true, + "zero_limit_events_at_payload_exit": true, + "final_whole_scope_counters_verified": false + }, + { + "name": "integrated-baseline-jq", + "expected_projection": "duplicate", + "artifact_sha256": "b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f", + "artifact_bytes": 2267912, + "procedure_count": 2279, + "selected_procedure": "0x401000", + "facts_sha256": "87722aab25011f1b047eb2bf8302cdbc5a4256c1b7f17d7bd68404a7d191e070", + "call_count": 15, + "catalog": { + "elapsed_ms": 1411.786849, + "bytes": 3263492, + "sha256": "c754f774156866d70be754e050335d0e4d3a7a6cbb156746cf1f7b5ffa42b26b" + }, + "rpc_argument_bytes": 974, + "rpc_reply_bytes": 2375466, + "catalog_arguments_replies_bytes": 5639932, + "normalized_value_bytes_once_per_selected_reply": 536689, + "raw_value_bytes_once_per_selected_reply": 534301, + "bundle_bytes": 609878, + "phases": { + "open_binary": { + "elapsed_ms": 983.5489280000002, + "reply_bytes": 617 + }, + "list_documents": { + "elapsed_ms": 270159.690556, + "reply_bytes": 6167 + }, + "list_procedures": { + "elapsed_ms": 717.9032449999941, + "reply_bytes": 1094387 + }, + "binary_overview": { + "elapsed_ms": 203.3595150000183, + "reply_bytes": 15727 + }, + "inspect_native_load_image": { + "elapsed_ms": 362.37314199999673, + "reply_bytes": 75633 + }, + "procedure_info": { + "elapsed_ms": 92.72515800001565, + "reply_bytes": 7813 + }, + "procedure_pseudo_code": { + "elapsed_ms": 85.43632900004741, + "reply_bytes": 6869 + }, + "procedure_assembly": { + "elapsed_ms": 78.87645799998427, + "reply_bytes": 6855 + }, + "procedure_callers": { + "elapsed_ms": 32.66119900002377, + "reply_bytes": 6891 + }, + "procedure_callees": { + "elapsed_ms": 8.895469999988563, + "reply_bytes": 7479 + }, + "analyze_function": { + "elapsed_ms": 343.2701520000119, + "reply_bytes": 45277 + }, + "warm-inventory": { + "elapsed_ms": 338.9290720000281, + "reply_bytes": 1094387 + }, + "warm-pseudocode": { + "elapsed_ms": 9.391289999999572, + "reply_bytes": 6869 + }, + "export_evidence_bundle": { + "elapsed_ms": 82.86341799999354, + "reply_bytes": 326 + }, + "close_binary": { + "elapsed_ms": 679.7842850000015, + "reply_bytes": 169 + } + }, + "scope_seconds": 279.699, + "payload_cpu_seconds": 277.696079, + "payload_family_peak_bytes": 1111494656, + "payload_cleanup_verified": true, + "zero_limit_events_at_payload_exit": true, + "final_whole_scope_counters_verified": false + }, + { + "name": "integrated-candidate-jq", + "expected_projection": "compact", + "artifact_sha256": "b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f", + "artifact_bytes": 2267912, + "procedure_count": 2279, + "selected_procedure": "0x401000", + "facts_sha256": "87722aab25011f1b047eb2bf8302cdbc5a4256c1b7f17d7bd68404a7d191e070", + "call_count": 15, + "catalog": { + "elapsed_ms": 1213.2912340000003, + "bytes": 3263492, + "sha256": "c754f774156866d70be754e050335d0e4d3a7a6cbb156746cf1f7b5ffa42b26b" + }, + "rpc_argument_bytes": 975, + "rpc_reply_bytes": 1264884, + "catalog_arguments_replies_bytes": 4529351, + "normalized_value_bytes_once_per_selected_reply": 536689, + "raw_value_bytes_once_per_selected_reply": 16143, + "bundle_bytes": 345626, + "phases": { + "open_binary": { + "elapsed_ms": 927.8219500000005, + "reply_bytes": 617 + }, + "list_documents": { + "elapsed_ms": 391135.781249, + "reply_bytes": 6119 + }, + "list_procedures": { + "elapsed_ms": 646.0353969999705, + "reply_bytes": 550261 + }, + "binary_overview": { + "elapsed_ms": 198.11136500001885, + "reply_bytes": 15727 + }, + "inspect_native_load_image": { + "elapsed_ms": 456.49483100004727, + "reply_bytes": 75633 + }, + "procedure_info": { + "elapsed_ms": 99.12928700004704, + "reply_bytes": 6825 + }, + "procedure_pseudo_code": { + "elapsed_ms": 114.82446700002765, + "reply_bytes": 6584 + }, + "procedure_assembly": { + "elapsed_ms": 80.73368800000753, + "reply_bytes": 6444 + }, + "procedure_callers": { + "elapsed_ms": 10.588890000013635, + "reply_bytes": 6337 + }, + "procedure_callees": { + "elapsed_ms": 19.5354899999802, + "reply_bytes": 6631 + }, + "analyze_function": { + "elapsed_ms": 352.812751999998, + "reply_bytes": 26364 + }, + "warm-inventory": { + "elapsed_ms": 197.59054599999217, + "reply_bytes": 550261 + }, + "warm-pseudocode": { + "elapsed_ms": 8.122649999975692, + "reply_bytes": 6584 + }, + "export_evidence_bundle": { + "elapsed_ms": 61.80356899998151, + "reply_bytes": 328 + }, + "close_binary": { + "elapsed_ms": 614.0456879999838, + "reply_bytes": 169 + } + }, + "scope_seconds": 399.097, + "payload_cpu_seconds": 352.659548, + "payload_family_peak_bytes": 1088741376, + "payload_cleanup_verified": true, + "zero_limit_events_at_payload_exit": true, + "final_whole_scope_counters_verified": false + } + ], + "incomplete_attempts": [ + { + "name": "baseline-true-1", + "reason": "Harness selected subject.sha256 instead of subject.digest.sha256.", + "reported_status": "failed", + "payload_exit_code": 1, + "scope_seconds": 41.401, + "payload_cpu_seconds": 39.027824, + "payload_family_peak_bytes": 1132675072, + "observed_procedures": null, + "payload_cleanup_verified": true, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "is_completed_performance_sample": false + }, + { + "name": "baseline-true-2", + "reason": "Harness assumed the composed overview provider was Ghidra rather than rea-workflow.", + "reported_status": "failed", + "payload_exit_code": 1, + "scope_seconds": 37.538, + "payload_cpu_seconds": 36.599406, + "payload_family_peak_bytes": 1001476096, + "observed_procedures": 231, + "payload_cleanup_verified": true, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "is_completed_performance_sample": false + }, + { + "name": "baseline-true-3", + "reason": "Facts and cleanup completed, but the harness incorrectly read a status property from the boolean finalizer and exited 1.", + "reported_status": "passed", + "payload_exit_code": 1, + "scope_seconds": 35.796, + "payload_cpu_seconds": 34.736678, + "payload_family_peak_bytes": 1001455616, + "observed_procedures": 231, + "payload_cleanup_verified": true, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "is_completed_performance_sample": false + }, + { + "name": "candidate-jq-1", + "reason": "SDK five-minute first-query deadline expired while awaiting list_documents; cleanup completed.", + "reported_status": "failed", + "payload_exit_code": 1, + "scope_seconds": 316.605, + "payload_cpu_seconds": 293.28205, + "payload_family_peak_bytes": 1065631744, + "observed_procedures": null, + "payload_cleanup_verified": true, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "is_completed_performance_sample": false + }, + { + "name": "integrated-baseline-jq-control", + "reason": "Unmodified-main repeat differs by three observed procedure rows; strict comparison rejected it before export. Cause unestablished.", + "reported_status": "failed", + "payload_exit_code": 1, + "scope_seconds": 276.557, + "payload_cpu_seconds": 274.502165, + "payload_family_peak_bytes": 1200726016, + "observed_procedures": 2276, + "payload_cleanup_verified": true, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "is_completed_performance_sample": false + } + ], + "capacity_control": { + "baseline_revision": "3fca2e0ca6e54ba700c286d9a85a9600dee6a19d", + "records": 30000, + "baseline": "regression fails with resource_constraint for list_procedures at the unchanged 10485760-byte stdio budget", + "candidate": "all 30000 records delivered exactly; raw_result null", + "real_provider": false, + "production_boundary": "GhidraProvider -> BinarySession -> public MCP" + }, + "executed_validation": [ + { + "check": "npm run check", + "base": "4e716b060", + "result": "6 tasks and architecture guards passed" + }, + { + "check": "npm run check", + "base": "3fca2e0ca", + "result": "6 tasks and architecture guards passed; before adding the capacity regression" + }, + { + "check": "Focused Ghidra inventory/function/provider and four MCP files", + "base": "4e716b060", + "result": "246 assertions passed across 7 files" + }, + { + "check": "Capacity regression alone", + "base": "3fca2e0ca", + "result": "candidate passes; unchanged production baseline fails with resource_constraint; six other tests filtered out" + }, + { + "check": "Existing real Ghidra large-result helper through a private stdio harness", + "result": "passed", + "successful_calls": 15, + "expected_rejections": 4, + "runtime_empty_after_provider_close": true, + "scope": "oversized lookup/read/annotation/dossier retention and recovery, complete CLI readback, unchanged source bytes" + }, + { + "check": "node scripts/verify-real-ghidra-address-context.mjs", + "result": "passed with real Ghidra/CLI/MCP and verified empty descendant set", + "scope": "PIE load identity, file-offset decompilation/roles, address round trips, failure guidance/recovery, unchanged source and complete cleanup", + "fixture_sha256": "093f8ba3aef80f3d7c0e8f22557e9a8af29505e286835578282b8e32ff804a02" + }, + { + "check": "Final focused Ghidra source and MCP regression selection", + "base": "3fca2e0ca", + "result": "247 assertions across 7 files passed" + }, + { + "check": "npm run docs:check", + "result": "generated metadata/documentation checks and 5 README conformance assertions passed" + } + ], + "unmodified_main_control": { + "same_target_digest_and_analysis_profile": true, + "expected_procedures": 2279, + "observed_procedures": 2276, + "missing_relative_to_first_run": ["0x4f9930", "0x5260b0", "0x529640"], + "cause": "unestablished; no engine/default-analysis repair included", + "interpretation": "The matched pair is an observation, not a guarantee of cross-session procedure discovery or a latency distribution." + }, + "limitations": [ + "No actual model-token or tokenizer measurement.", + "Single completed pairs, not latency distributions.", + "Unmodified-main procedure-inventory repeat failed the strict oracle; retained as an incomplete control.", + "First three successful query receipts used the earlier driver; archived warm results were rechecked when explicit repeat assertions were added.", + "Real Windows/macOS Ghidra, other engine versions and real NativeAOT overlays remain unqualified.", + "Initial cached baseline build had an unverified payload-exit cleanup observation; a later systemd inspection found its scope inactive. It is not a query measurement." + ], + "resource_stopped_validation": [ + { + "check": "Final focused regression selection, first attempt", + "abort": "host_available_memory_early_stop", + "peak_family_bytes_sampled": 837332992, + "outcome": "guard-stopped without final test results; not counted as passing", + "retry_budget": "4 GiB hard/3 GiB early, based on the earlier focused run peak of about 1.3 GiB" + } + ] +} diff --git a/experiments/analysis-efficiency/ghidra-evidence-projection.md b/experiments/analysis-efficiency/ghidra-evidence-projection.md new file mode 100644 index 0000000000..7f23fedd41 --- /dev/null +++ b/experiments/analysis-efficiency/ghidra-evidence-projection.md @@ -0,0 +1,165 @@ +# Ghidra Evidence projection + +Continues [#5337](https://github.com/morluto/rea/issues/5337). The bridge +adapter supplied a complete `rawResult` for every successful operation, even +when strict parsing preserved the same value in `normalized_result`. Function +observation envelopes repeated their value and already-projected qualifications. +The generic Evidence contract already permits `raw_result: null` in this case. + +## Owning change + +The Ghidra inventory and function parsers now declare whether they retain an +independent upstream representation. Execution consumes those parsed facts and +named metadata, replacing the positional helper argument that required every +caller to supply another complete report. CLI and MCP use the same projection. + +Ordinary inventories and function observations omit the redundant copy. +Limitations, relationship coverage and locations remain canonical Evidence +fields. The following representations remain available: + +- The overview's upstream segment permissions, sections and provenance. +- Load-image observations before local attestation and enrichment. +- File-offset selection roles, which generic locations alone do not express. +- The database type reply before adding a locally derived NativeAOT overlay. + +No tool, schema, query default, provider protocol, cache, target execution or +annotation authority is added. Existing source Evidence is not rewritten. +New records have the identities appropriate to their new representation; +artifact and procedure identities remain unchanged. Consumers should read +function results from `normalized_result`, rather than `raw_result.value`. + +## Real inputs and completed question + +Both ELF x86-64 targets are read without executing them: + +| Target | Bytes | SHA-256 | Provenance | +| ---------------- | --------: | ------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| `true` | 35,664 | `c79bf44242829108e323378531f4ac839513ca1fba45efd6583643526e1e9fd2` | Installed Debian coreutils 9.1-1; package metadata is a local observation. | +| `jq-linux-amd64` | 2,267,912 | `b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f` | [Official jq 1.8.2 asset](https://github.com/jqlang/jq/releases/tag/jq-1.8.2); matches both the corpus and the release API's asset digest/size. | + +Each fresh production stdio MCP session discovers the complete catalog, opens +the target, reads documents and the complete observed procedure inventory, +selects the first internal non-thunk procedure, reads overview/load image and +six function operations, repeats inventory/pseudocode, exports Evidence and +closes. This is one catalog discovery and 15 RPCs, including cleanup. +It qualifies this inventory/function-read question, rather than application +feature reconstruction or complete native-code discovery. + +The live named SDK contracts validate replies. Each selected record binds to +the original target digest and Ghidra profile. The candidate's complete selected +fact vector equals the baseline after excluding only `evidence_id` and +`raw_result`: subject, parameters, normalized facts, provider/profile, authority, +limitations, relationships, locations and Evidence links remain equal. Exported +records match returned Evidence, repeated facts agree, and target hashes remain +unchanged. This is representation equivalence, not an independent qualification +of every Ghidra interpretation. + +## Measurements and limits + +Linux x86-64, Node 24.18.0/npm 11.16.0, MCP SDK 2.3.1, Ghidra 12.1.4 and +Eclipse Adoptium 21.0.12.1+1-LTS. Sessions and Java analysis are fresh; OS caches +are warm and unflushed. Configured Java heap is 768 MiB with one active processor. +The whole process family uses one logical CPU, 4 GiB hard/3 GiB early-stop memory, +no swap and 192 tasks. No completed query task has an OOM/limit event or cleanup +failure at verified payload exit. Final whole-scope counters after scope removal +remain unavailable. Post-GC retained heap is unmeasured. + +The small pair uses main `4e716b06078d725a53234503cbefbb340731e809` and its +candidate. Main advanced during the experiment; the large pair uses +`3fca2e0ca6e54ba700c286d9a85a9600dee6a19d` on both sides. Existing timing +samples keep their original runtime fingerprints. The [curated receipt](ghidra-evidence-projection.json) +retains individual observations and incomplete attempts; raw data stays outside Git. + +| Recording / metric | Before | After | +| ----------------------------------------------- | ---------: | ---------: | +| `true`: observed procedures | 231 | 231 | +| `true`: all RPC result bytes | 473,344 | 326,642 | +| `true`: catalog + request arguments + results | 3,729,626 | 3,582,925 | +| `true`: complete scope wall | 34.369 s | 34.901 s | +| `true`: payload-family CPU | 33.150 s | 33.834 s | +| `true`: kernel family peak including file cache | 947.9 MiB | 977.7 MiB | +| `jq`: observed procedures | 2,279 | 2,279 | +| `jq`: all RPC result bytes | 2,375,466 | 1,264,884 | +| `jq`: catalog + request arguments + results | 5,639,932 | 4,529,351 | +| `jq`: complete scope wall | 279.699 s | 399.097 s | +| `jq`: payload-family CPU | 277.696 s | 352.660 s | +| `jq`: kernel family peak including file cache | 1060.0 MiB | 1038.3 MiB | + +RPC result bytes include the SDK's text and structured representations. Request +objects and the complete catalog are included in the combined byte count; +initialization, wire framing and model rendering are unmeasured. Actual model +tokens and tokenizer estimates are unmeasured. The observed byte reduction +does not establish a token saving or better agent investigation outcome. + +The large result envelopes shrink by 46.8%; combined serialization shrinks by +19.7%, with unchanged catalog and call count. Small combined serialization +shrinks by 3.9%. These are single pairs, not distributions. No whole-task latency, +CPU or peak-memory improvement is established. The large candidate's first +`list_documents` costs 391.136 s versus 270.160 s before; this phase includes +startup, analysis wait and client work, not isolated engine time. Changed +projection code executes after the bridge returns its observation, but the +cause of the preparation-time variation is unestablished. One warm inventory +read costs 338.93 → 197.59 ms; this is not a general latency qualification. + +A subsequent unchanged-main repeat with the same target/profile returned 2,276 +procedures instead of 2,279, omitting `0x4f9930`, `0x5260b0` and `0x529640` +relative to its earlier inventory. The strict oracle rejected that control +before export. Its cause is unestablished; it is retained as an incomplete +attempt, not added to performance averages or treated as a projection defect. + +## Verification scope + +The production-boundary capacity regression supplies 30,000 valid simulated +Ghidra procedure rows through `GhidraProvider`, `BinarySession` and MCP. On +unchanged production main it fails with `resource_constraint` at the existing +10 MiB response budget. The candidate delivers every row exactly, without a +budget increase or truncation. This qualifies response admission, not a real +30,000-function Ghidra analysis or a universal maximum procedure count. + +Source/parser tests cover strict rejection, sidecars, distinct file-offset +mapping roles, overview details, NativeAOT enrichment and unchanged observations. +Executed checks and real-provider verification are recorded in the receipt. +NativeAOT overlay tests use producer fixtures; real Windows/macOS and other +engine versions remain unqualified here. + +The existing real large-result helper passed through a private stdio harness: +oversized lookup/read/annotation/dossier replies retained complete Evidence, +subsequent queries recovered, complete CLI readback matched the original bytes +with `raw_result: null`, and generated target bytes remained unchanged. +The public `verify-real-ghidra-address-context.mjs` also passed with real +Ghidra/CLI/MCP, including file-offset selection, address round trips, recovery, +unchanged source and verified cleanup. These capacity/address fixtures use a +separate 6 GiB family cap; they are correctness checks, not latency pairs. + +Three early `true` harness attempts used an incorrect subject-digest selector, +assumed the composed overview's provider was Ghidra, or misread the existing +finalizer's boolean result. The last completed its facts but exited nonzero; +none is a completed performance sample. A first `jq` candidate hit the SDK's +five-minute deadline while awaiting readiness, then cleaned up successfully. +The final driver's startup/RPC deadlines are ten minutes on both sides; product +defaults are unchanged. Earlier successful warm facts were rechecked from their +archived production replies when explicit warm-result assertions were added. + +## Reproduce + +Build and retain separate baseline/candidate runtimes. Use the same absolute +target path and caller-supplied Ghidra/JDK for both: + +```sh +python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \ + node experiments/analysis-efficiency/ghidra-evidence-projection.mjs \ + --runtime /absolute/baseline --target /absolute/application \ + --ghidra /absolute/ghidra --java /absolute/jdk \ + --expect duplicate --output /absolute/new-before + +python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \ + node experiments/analysis-efficiency/ghidra-evidence-projection.mjs \ + --runtime /absolute/candidate --target /absolute/application \ + --ghidra /absolute/ghidra --java /absolute/jdk \ + --expect compact --control /absolute/new-before/facts.json \ + --output /absolute/new-after +``` + +The candidate's independent baseline comparison is included in harness costs; +its isolated verification overhead is unmeasured. Static checks use a separate +8 GiB memory cap. All heavy operations run serially. diff --git a/experiments/analysis-efficiency/ghidra-evidence-projection.mjs b/experiments/analysis-efficiency/ghidra-evidence-projection.mjs new file mode 100644 index 0000000000..3686f11d7b --- /dev/null +++ b/experiments/analysis-efficiency/ghidra-evidence-projection.mjs @@ -0,0 +1,216 @@ +/** Real Ghidra inventory/function Evidence; raw inputs and receipts stay outside Git. */ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { createWriteStream } from "node:fs"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { finished } from "node:stream/promises"; +import { parseArgs } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { createRetainedQueryCaller } from "./retained-query-calls.mjs"; +import { finalizeRetainedQueryReceipt } from "./retained-query-finalization.mjs"; + +const { values } = parseArgs({ + options: Object.fromEntries( + ["runtime", "target", "output", "ghidra", "java", "expect", "control"].map( + (name) => [name, { type: "string" }], + ), + ), +}); +for (const name of ["runtime", "target", "output", "ghidra", "java"]) + assert.ok(values[name] && values[name] === resolve(values[name]), name); +assert.ok(["duplicate", "compact"].includes(values.expect)); +const hash = (value) => createHash("sha256").update(value).digest("hex"); +const output = values.output; +await mkdir(output, { mode: 0o700 }); +const bytes = await readFile(values.target); +const home = await mkdtemp(join(tmpdir(), "rea-ghidra-evidence-")); +const client = new Client({ name: "ghidra-evidence-projection", version: "1" }); +const transport = new StdioClientTransport({ + command: process.execPath, + args: [join(values.runtime, "scripts/rea.mjs"), "mcp"], + env: { + HOME: home, + PATH: process.env.PATH, + NODE_OPTIONS: "--max-old-space-size=2048", + GHIDRA_INSTALL_DIR: values.ghidra, + JAVA_HOME: values.java, + GHIDRA_HEADLESS_MAXMEM: "768M", + JAVA_TOOL_OPTIONS: "-XX:ActiveProcessorCount=1", + REA_GHIDRA_STARTUP_TIMEOUT_MS: "600000", + REA_LOG_LEVEL: "silent", + }, + stderr: "pipe", +}); +const serverLog = createWriteStream(join(output, "server.log")); +const serverLogFinished = finished(serverLog); +serverLogFinished.catch(() => {}); +transport.stderr?.pipe(serverLog); +const report = { + artifact: { path: values.target, bytes: bytes.length, sha256: hash(bytes) }, + expected_projection: values.expect, + calls: [], +}; +const transcript = []; +const evidence = []; +const call = createRetainedQueryCaller(client, (receipt, event) => { + report.calls.push(receipt); + transcript.push(event); +}); +const result = async (name, input, phase = name) => { + const reply = await call(name, input, phase); + assert.notEqual(reply.isError, true, JSON.stringify(reply)); + assert.ok(reply.structuredContent); + return reply.structuredContent; +}; +const inspect = async (name, input, phase = name) => { + const value = await result(name, input, phase); + assert.equal(value.subject.digest.sha256, report.artifact.sha256); + assert.equal( + value.provider.id, + name === "binary_overview" ? "rea-workflow" : "ghidra", + ); + assert.equal( + value.provider.version, + name === "binary_overview" ? "1" : "12.1.4", + ); + assert.ok(value.analysis_profile); + evidence.push({ phase, value }); + return value; +}; +const started = performance.now(); +let opened = false; +try { + await client.connect(transport); + const discovery = performance.now(); + const catalog = await client.listTools(); + assert.equal(catalog.nextCursor, undefined); + report.catalog = { + elapsed_ms: performance.now() - discovery, + bytes: Buffer.byteLength(JSON.stringify(catalog)), + sha256: hash(JSON.stringify(catalog)), + }; + await writeFile(join(output, "catalog.json"), JSON.stringify(catalog)); + const definitions = new Map(catalog.tools.map((tool) => [tool.name, tool])); + const original = client.callTool.bind(client); + client.callTool = (input, options) => + original(input, { + ...options, + timeout: 600000, + toolDefinition: definitions.get(input.name), + }); + await result("open_binary", { path: values.target, provider_id: "ghidra" }); + opened = true; + const documents = await inspect("list_documents", {}); + assert.equal(documents.normalized_result.length, 1); + const procedures = await inspect("list_procedures", {}); + assert.ok(procedures.normalized_result.length > 0); + const selected = procedures.normalized_result.find( + (row) => row.procedure.external === false && row.procedure.thunk === false, + ); + assert.ok(selected); + report.procedure = selected.address; + report.procedure_count = procedures.normalized_result.length; + await inspect("binary_overview", {}); + await inspect("inspect_native_load_image", {}); + for (const name of [ + "procedure_info", + "procedure_pseudo_code", + "procedure_assembly", + "procedure_callers", + "procedure_callees", + "analyze_function", + ]) + await inspect(name, { procedure: selected.address }); + const warmInventory = await inspect("list_procedures", {}, "warm-inventory"); + assert.deepEqual( + warmInventory.normalized_result, + procedures.normalized_result, + ); + const warmPseudocode = await inspect( + "procedure_pseudo_code", + { procedure: selected.address }, + "warm-pseudocode", + ); + assert.deepEqual( + warmPseudocode.normalized_result, + evidence.find(({ phase }) => phase === "procedure_pseudo_code").value + .normalized_result, + ); + for (const { phase, value } of evidence) { + if (phase === "binary_overview") { + assert.equal(value.raw_result, null); + continue; + } + const distinct = phase === "inspect_native_load_image"; + if (values.expect === "compact" && !distinct) + assert.equal(value.raw_result, null, phase); + else assert.notEqual(value.raw_result, null, phase); + if (values.expect === "duplicate" && !distinct) { + const raw = value.raw_result; + const wrapped = + raw !== null && + !Array.isArray(raw) && + Object.hasOwn(raw, "value") && + Object.hasOwn(raw, "limitations"); + assert.deepEqual( + wrapped ? raw.value : raw, + value.normalized_result, + phase, + ); + } + } + const facts = evidence.map(({ phase, value }) => { + const { evidence_id: _id, raw_result: _raw, ...facts } = value; + return { phase, facts }; + }); + report.facts_sha256 = hash(JSON.stringify(facts)); + await writeFile(join(output, "facts.json"), JSON.stringify(facts)); + if (values.control) + assert.deepEqual(facts, JSON.parse(await readFile(values.control, "utf8"))); + const path = join(output, "evidence.json"); + await result("export_evidence_bundle", { path }); + const bundle = JSON.parse(await readFile(path, "utf8")); + for (const { value } of evidence) + assert.deepEqual( + bundle.records.find((record) => record.evidence_id === value.evidence_id), + value, + ); + assert.equal(hash(await readFile(values.target)), report.artifact.sha256); + report.status = "passed"; +} catch (cause) { + report.status = "failed"; + report.failure = String(cause?.stack ?? cause); + process.exitCode = 1; +} finally { + if (opened) { + try { + await result("close_binary", {}); + } catch (cause) { + report.status = "failed"; + report.close_failure = String(cause?.stack ?? cause); + } + } + report.elapsed_before_cleanup_ms = performance.now() - started; + try { + await writeFile( + join(output, "transcript.json"), + JSON.stringify(transcript), + ); + } catch (cause) { + report.status = "failed"; + report.transcript_failure = String(cause); + } + const finalized = await finalizeRetainedQueryReceipt({ + report, + client, + transport, + account: home, + serverLog, + serverLogFinished, + output, + }); + if (report.status !== "passed" || !finalized) process.exitCode = 1; +} diff --git a/licenses/README.md b/licenses/README.md index c811a4f767..fa698f449e 100644 --- a/licenses/README.md +++ b/licenses/README.md @@ -8,6 +8,7 @@ upstream texts; they do not change [REA's MIT license](../LICENSE). | ------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `npm-dependencies.txt` | Every package path in the root lockfile, with its exact version, declared license, scope, download URL, and integrity. Includes development and optional platform entries. | | `npm-runtime-notices.txt` | Original license, copyright, and NOTICE texts collected from integrity-verified locked runtime npm archives across all platforms; also the license of Rolldown's generated runtime helpers. The archive manifest's name and version are checked. | +| `protobuf-es-v2.16.0-LICENSE.txt` | Original Apache license omitted from the protobuf-es npm archive, preserved from its v2.16.0 source commit. Its embedded Google BSD notice is retained in `npm-runtime-notices.txt`. | | `nativeaot-parser-MIT.txt` | Original Washi notice for derived NativeAOT parser code; source revision is recorded in the parser and the root notices. | | `dotnet-runtime-MIT.txt` | Original .NET Foundation MIT text from the NativeAOT reference revision `a3b2d40328be0be3f8dd43f0e5cc925b8827b044`. | | `ghidra-12.1.4-{LICENSE,NOTICE,DISCLAIMER}.txt` | Unchanged top-level documents from `Ghidra_12.1.4_build`. Ghidra is not bundled; these reference copies do not replace its component and GPL support-program notices. The US-government-specific disclaimer applies to Ghidra, not to REA. | diff --git a/licenses/npm-dependencies.txt b/licenses/npm-dependencies.txt index 19fb7e1ee6..c2c2280df1 100644 --- a/licenses/npm-dependencies.txt +++ b/licenses/npm-dependencies.txt @@ -2,7 +2,7 @@ REA npm dependency inventory Prepared from the root package-lock.json; refresh this inventory when locked dependencies change. License identifiers are upstream lockfile declarations, not a legal assessment. Runtime includes transitive and optional platform packages; development includes build, test and website tooling. -Locked package entries: 750; runtime: 178. +Locked package entries: 751; runtime: 179. Full runtime copyright/license/NOTICE texts: npm-runtime-notices.txt. @algolia/abtesting@1.25.0 | development | MIT @@ -335,6 +335,11 @@ obug@3.0.0 | development | MIT Package: https://www.npmjs.com/package/@bcoe/v8-coverage/v/1.0.2 Archive: https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz Integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA== +@bufbuild/protobuf@2.16.0 | runtime | (Apache-2.0 AND BSD-3-Clause) + Lockfile path: node_modules/@bufbuild/protobuf + Package: https://www.npmjs.com/package/@bufbuild/protobuf/v/2.16.0 + Archive: https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.16.0.tgz + Integrity: sha512-FWa0sPlqYGJgpTs6OxBRcL/AW4JT0OIO+W1ajerluoTVh8CV0B7XM1qsNCRgnkSCkRCgOICKHRQlY17jrsm0+Q== @cfworker/json-schema@4.1.1 | runtime | MIT Lockfile path: node_modules/@cfworker/json-schema Package: https://www.npmjs.com/package/@cfworker/json-schema/v/4.1.1 diff --git a/licenses/npm-runtime-notices.txt b/licenses/npm-runtime-notices.txt index d9efda90fa..0fc82a1312 100644 --- a/licenses/npm-runtime-notices.txt +++ b/licenses/npm-runtime-notices.txt @@ -1,7 +1,7 @@ REA third-party npm runtime copyright, license and NOTICE texts Collected from integrity-verified locked npm archives. Release-specific supplements and upstream notice gaps are identified in their entries. External engines are separately obtained; see THIRD_PARTY_NOTICES.md. -Runtime dependency fingerprint (SHA-256): 131100b2e1e6ad5e55b10a0baf4be2d7a0838fd724fb307b9bafd73446b0d3ec +Runtime dependency fingerprint (SHA-256): 4125777159e02cb749e17c959e77f59528cfffe3f7a5220b8a21ce3c49ad3061 ===== @babel/helper-string-parser@8.0.6 ===== Declared license: MIT @@ -126,6 +126,254 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +===== @bufbuild/protobuf@2.16.0 ===== +Declared license: (Apache-2.0 AND BSD-3-Clause) +Archive: https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.16.0.tgz +Integrity: sha512-FWa0sPlqYGJgpTs6OxBRcL/AW4JT0OIO+W1ajerluoTVh8CV0B7XM1qsNCRgnkSCkRCgOICKHRQlY17jrsm0+Q== +Scope: npm runtime dependency, including optional platform packages. + +The locked archive omits standalone license files. The Apache license is preserved from its v2.16.0 upstream commit; the BSD notice is embedded in the integrity-verified archive. + +--- licenses/protobuf-es-v2.16.0-LICENSE.txt (see upstream-sources.json) --- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2021-2026 Buf Technologies, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--- package/dist/esm/wire/varint.js (unchanged leading BSD notice) --- + +// Copyright 2008 Google Inc. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Code generated by the Protocol Buffer compiler is owned by the owner +// of the input file used when generating it. This code is not +// standalone and requires a support library to be linked with it. This +// support library is itself covered by the above license. + + ===== @cfworker/json-schema@4.1.1 ===== Declared license: MIT Archive: https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz diff --git a/licenses/protobuf-es-v2.16.0-LICENSE.txt b/licenses/protobuf-es-v2.16.0-LICENSE.txt new file mode 100644 index 0000000000..0852eae3ad --- /dev/null +++ b/licenses/protobuf-es-v2.16.0-LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2021-2026 Buf Technologies, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/licenses/upstream-sources.json b/licenses/upstream-sources.json index fb1dad9759..2fbf6a8855 100644 --- a/licenses/upstream-sources.json +++ b/licenses/upstream-sources.json @@ -68,5 +68,10 @@ "file": "licenses/LGPL-2.0.txt", "source": "https://www.gnu.org/licenses/old-licenses/lgpl-2.0.txt", "sha256": "cc535c21133c895b56b374c8a1dc1eb948d99003ed2b47372069456b62f42b24" + }, + { + "file": "licenses/protobuf-es-v2.16.0-LICENSE.txt", + "source": "https://raw.githubusercontent.com/bufbuild/protobuf-es/b3b232f7f466eda27c66e8f63c82f930578b6cc6/LICENSE", + "sha256": "c04b4216f1cd4c5a4f7fb2f2a1b0ae70d847e9e0cac7c9dee9bf8cc03177c449" } ] diff --git a/package-lock.json b/package-lock.json index 2841a5d1a7..67a74129f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@babel/parser": "8.0.4", "@babel/types": "8.0.4", + "@bufbuild/protobuf": "2.16.0", "@clack/prompts": "1.7.0", "@electron/asar": "4.2.0", "@jridgewell/trace-mapping": "0.3.31", @@ -1090,6 +1091,12 @@ "node": ">=18" } }, + "node_modules/@bufbuild/protobuf": { + "version": "2.16.0", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.16.0.tgz", + "integrity": "sha512-FWa0sPlqYGJgpTs6OxBRcL/AW4JT0OIO+W1ajerluoTVh8CV0B7XM1qsNCRgnkSCkRCgOICKHRQlY17jrsm0+Q==", + "license": "(Apache-2.0 AND BSD-3-Clause)" + }, "node_modules/@cfworker/json-schema": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", diff --git a/package.json b/package.json index 2032234383..5f77580431 100644 --- a/package.json +++ b/package.json @@ -323,6 +323,7 @@ "dependencies": { "@babel/parser": "8.0.4", "@babel/types": "8.0.4", + "@bufbuild/protobuf": "2.16.0", "@clack/prompts": "1.7.0", "@electron/asar": "4.2.0", "@jridgewell/trace-mapping": "0.3.31", diff --git a/scripts/lib/real-ghidra-large-results.mjs b/scripts/lib/real-ghidra-large-results.mjs index fa1c5115ff..6890bd4a28 100644 --- a/scripts/lib/real-ghidra-large-results.mjs +++ b/scripts/lib/real-ghidra-large-results.mjs @@ -167,26 +167,18 @@ export async function verifyGhidraLargeResults({ assert.equal(read.returned_bytes, length); assert.equal(read.complete, true); assert.equal(read.bytes_hex, expected.toString("hex")); - assert.equal(readRecord.raw_result.bytes_hex, read.bytes_hex); + assert.equal(readRecord.raw_result, null); const annotationRecord = retained(annotationId); const annotation = annotationRecord.normalized_result; assert.equal(annotation.annotations.name, "rea_frame_probe"); assert.equal(annotation.annotations.comment, comment); assert.equal(annotation.annotations.inline_comment, comment); - assert.equal(annotationRecord.raw_result.annotations.comment, comment); - assert.equal( - annotationRecord.raw_result.annotations.inline_comment, - comment, - ); + assert.equal(annotationRecord.raw_result, null); const dossierRecord = retained(dossierId); const dossier = dossierRecord.normalized_result; assert.equal(dossier.procedure.name, "rea_frame_probe"); assert.ok(dossier.comments.some(({ text }) => text === comment)); - assert.ok( - dossierRecord.raw_result.value.comments.some( - ({ text }) => text === comment, - ), - ); + assert.equal(dossierRecord.raw_result, null); const { stdout } = await exec( process.execPath, [ @@ -202,7 +194,9 @@ export async function verifyGhidraLargeResults({ ], { env, timeout: 240000, maxBuffer: 72 * 1024 * 1024 }, ); - assert.deepEqual(JSON.parse(stdout).normalized_result, read); + const cliRead = JSON.parse(stdout); + assert.deepEqual(cliRead.normalized_result, read); + assert.equal(cliRead.raw_result, null); assert.deepEqual(await readFile(path), bytes); await call("close_binary"); await call("open_binary", { path: target.path, provider_id: "ghidra" }); diff --git a/scripts/lib/real-hopper-owner-exit.mjs b/scripts/lib/real-hopper-owner-exit.mjs index ff59b95860..9cdf0d3cf8 100644 --- a/scripts/lib/real-hopper-owner-exit.mjs +++ b/scripts/lib/real-hopper-owner-exit.mjs @@ -7,6 +7,7 @@ import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { verifyNoTokenOwnedProcesses } from "../../dist/process/ProcessOwnership.js"; +import { ProcessOwnershipEvidenceOwner } from "../../dist/process/ProcessOwnershipEvidence.js"; import { observeProcessStartIdentity, signalProcessWithStartIdentity, @@ -64,6 +65,7 @@ export async function verifyLinuxHopperOwnerExit({ const relation = { leaderPid: ownerPid, processGroupId: ownerProcess.processGroupId, + evidenceOwner: new ProcessOwnershipEvidenceOwner(), }; requireMcpLifecycleResult( await client.callTool( @@ -93,16 +95,16 @@ export async function verifyLinuxHopperOwnerExit({ const runId = status.analysis_run?.run_id; assert.equal(typeof runId, "string"); assert.equal(status.analysis_provider_binding?.provider.id, "hopper"); - assert.equal( - ( - await verifyNoTokenOwnedProcesses( - runId, - systemProcessOwnershipHost, - baseline, - relation, - ) - ).cleaned, - false, + const active = await verifyNoTokenOwnedProcesses( + runId, + systemProcessOwnershipHost, + baseline, + relation, + ); + assert.equal(active.cleaned, false, JSON.stringify(active)); + assert.ok( + relation.evidenceOwner.coverage.retainedOwned > 0, + `The active Hopper session has no token-verified process ownership evidence: ${JSON.stringify(active)}`, ); if (mode === "graceful") await client.close(); else { @@ -127,7 +129,7 @@ export async function verifyLinuxHopperOwnerExit({ if (cleanup.cleaned) break; await delay(100); } while (Date.now() < deadline); - assert.equal(cleanup.cleaned, true, cleanup.reason); + assert.equal(cleanup.cleaned, true, JSON.stringify(cleanup)); // The SIGKILL case must recover the stale lease for both target identities. for (const target of [primary, secondary]) { const result = await execFileAsync( diff --git a/scripts/verify-real-ghidra-function.mjs b/scripts/verify-real-ghidra-function.mjs index 8acfeb1021..dbf4e77b7f 100644 --- a/scripts/verify-real-ghidra-function.mjs +++ b/scripts/verify-real-ghidra-function.mjs @@ -49,7 +49,7 @@ async function inventoryCall(client, operation, parameters) { ); const parsed = parseGhidraInventoryResult(operation, called.value); if (!parsed.ok) throw parsed.error; - return parsed.value; + return parsed.value.value; } export async function verifyDebugFunctionOperations( diff --git a/scripts/verify-real-ghidra-inventory.mjs b/scripts/verify-real-ghidra-inventory.mjs index e8c240789a..d5e033a99a 100644 --- a/scripts/verify-real-ghidra-inventory.mjs +++ b/scripts/verify-real-ghidra-inventory.mjs @@ -16,7 +16,7 @@ export async function call(client, operation, parameters) { ); const parsed = parseGhidraInventoryResult(operation, called.value); if (!parsed.ok) throw parsed.error; - return parsed.value; + return parsed.value.value; } export async function allItems(client, operation, parameters) { diff --git a/scripts/verify-real-ghidra-windows.mjs b/scripts/verify-real-ghidra-windows.mjs index fa297fd71a..ca64f38040 100644 --- a/scripts/verify-real-ghidra-windows.mjs +++ b/scripts/verify-real-ghidra-windows.mjs @@ -378,7 +378,7 @@ async function inventory(operation, parameters) { const result = parseGhidraInventoryResult(operation, called.value); if (!result.ok) throw result.error; observed.add(operation); - return result.value; + return result.value.value; } async function functionOperation(operation, parameters) { diff --git a/src/application/WebNetworkCaptureService.test.ts b/src/application/WebNetworkCaptureService.test.ts index 7dfc1d8ab7..17510384e2 100644 --- a/src/application/WebNetworkCaptureService.test.ts +++ b/src/application/WebNetworkCaptureService.test.ts @@ -543,3 +543,29 @@ it("reports malformed retained facts while keeping their cleanup owner retryable expect(result.error.cleanupIncomplete).toBe(false); expect(projectAnalysisError(result.error).code).toBe("unreadable_output"); }); + +it("rejects relative descriptor paths before starting a capture decoder", async () => { + const service = new WebNetworkCaptureService({ + inspect: () => { + throw new Error("Invalid input must fail before provider admission"); + }, + }); + const result = await service.inspect({ + capture_path: "/capture.har", + format: "har", + record_ordinals: [0], + protobuf: { + schema: { + descriptor_set_path: "relative.pb", + message_type: "capture.Message", + }, + }, + }); + if (result.ok) throw new Error("Expected typed invalid input"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "invalid_request", + details: { + issues: [{ path: ["protobuf", "schema", "descriptor_set_path"] }], + }, + }); +}); diff --git a/src/application/WebNetworkCaptureService.ts b/src/application/WebNetworkCaptureService.ts index 0c70daccd9..e1c397b51b 100644 --- a/src/application/WebNetworkCaptureService.ts +++ b/src/application/WebNetworkCaptureService.ts @@ -80,6 +80,30 @@ export class WebNetworkCaptureService { }, ]), ); + if (input.protobuf !== undefined && input.record_ordinals === undefined) + return failure( + new AnalysisInputError(OPERATION, undefined, [ + { + path: ["record_ordinals"], + reason: "invalid_format", + message: + "Protobuf options require explicit producer record ordinals.", + }, + ]), + ); + if ( + input.protobuf?.schema !== undefined && + !isAbsolute(input.protobuf.schema.descriptor_set_path) + ) + return failure( + new AnalysisInputError(OPERATION, undefined, [ + { + path: ["protobuf", "schema", "descriptor_set_path"], + reason: "invalid_format", + message: "Expected an absolute descriptor-set path on this host.", + }, + ]), + ); const loaded = await this.port.inspect(input, options); if (!loaded.ok) { const partial = webNetworkCaptureSchema.safeParse( @@ -201,6 +225,30 @@ export class WebNetworkCaptureService { ? {} : { format: input.format }), record_ordinals: selected, + ...(input.protobuf === undefined + ? {} + : { + protobuf: { + framing: input.protobuf.framing, + ...(input.protobuf.body_pointers === undefined + ? {} + : { + body_pointers: input.protobuf.body_pointers.map(redactText), + }), + ...(input.protobuf.schema === undefined + ? {} + : { + schema: { + descriptor_set_path: redactText( + input.protobuf.schema.descriptor_set_path, + ), + message_type: redactText( + input.protobuf.schema.message_type, + ), + }, + }), + }, + }), sensitive_value_count: input.sensitive_values.length, }); return ok( diff --git a/src/browser/history/CaptureFailures.ts b/src/browser/history/CaptureFailures.ts index 6c42ac8e3b..7bb07fb165 100644 --- a/src/browser/history/CaptureFailures.ts +++ b/src/browser/history/CaptureFailures.ts @@ -1,5 +1,7 @@ import type { ExecutionOptions } from "../../application/AnalysisProvider.js"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; +import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorCleanup.js"; +import { bindArtifactFileCleanup } from "../../artifacts/ArtifactFileOperation.js"; import { artifactFileCancellationCause } from "../../artifacts/ArtifactFileOperation.js"; import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { @@ -33,10 +35,18 @@ export const CAPTURE_PROCESS_MEMORY_LIMIT_REASON = export const historicalCaptureFailure = ( input: InspectWebNetworkCaptureInput, cause: unknown, - phase: "capture-read" | "decoder", + phase: "capture-read" | "descriptor-read" | "decoder", options?: ExecutionOptions, ): AnalysisError => { if (cause instanceof AnalysisError) return cause; + const descriptorRead = phase === "descriptor-read"; + const sourcePath = descriptorRead + ? (input.protobuf?.schema?.descriptor_set_path ?? input.capture_path) + : input.capture_path; + const inputPath = descriptorRead + ? ["protobuf", "schema", "descriptor_set_path"] + : ["capture_path"]; + const sourceLabel = descriptorRead ? "descriptor set" : "capture"; if (cause instanceof OwnedCommandFailure) { if (cause.cleanupFailure !== null) return new ProviderCleanupError( @@ -99,18 +109,18 @@ export const historicalCaptureFailure = ( ); } if ( - phase === "capture-read" && + phase !== "decoder" && cause instanceof ArtifactReaderFailure && cause.reason === "integrity" ) return new AnalysisArtifactChangedError( OPERATION, - input.capture_path, + sourcePath, cause.message, { cause }, ); if ( - phase === "capture-read" && + phase !== "decoder" && cause instanceof ArtifactReaderFailure && cause.reason === "cancelled" ) @@ -118,7 +128,7 @@ export const historicalCaptureFailure = ( cause: artifactFileCancellationCause(cause), }); if ( - phase === "capture-read" && + phase !== "decoder" && cause instanceof ArtifactReaderFailure && (cause.reason === "limit" || cause.reason === "format" || @@ -126,44 +136,44 @@ export const historicalCaptureFailure = ( ) return new AnalysisInputError(OPERATION, { cause }, [ { - path: ["capture_path"], + path: inputPath, reason: cause.reason === "limit" ? "out_of_range" : "invalid_format", message: cause.reason === "limit" - ? `${cause.message}. Select a capture within the ${WEB_NETWORK_CAPTURE_LIMITS.inputBytes}-byte input budget.` + ? `${cause.message}. Select a ${sourceLabel} within the ${WEB_NETWORK_CAPTURE_LIMITS.inputBytes}-byte input budget.` : cause.message, ...(cause.reason === "limit" ? { expected: { - maximum_capture_bytes: WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + [descriptorRead + ? "maximum_descriptor_set_bytes" + : "maximum_capture_bytes"]: + WEB_NETWORK_CAPTURE_LIMITS.inputBytes, }, } : {}), }, ]); if ( - phase === "capture-read" && + phase !== "decoder" && cause instanceof Error && "code" in cause && (cause.code === "EACCES" || cause.code === "EPERM") ) - return new AnalysisAccessDeniedError( - OPERATION, - input.capture_path, - cause.code, - { cause }, - ); + return new AnalysisAccessDeniedError(OPERATION, sourcePath, cause.code, { + cause, + }); if ( - phase === "capture-read" && + phase !== "decoder" && cause instanceof Error && "code" in cause && ["ENOENT", "ENOTDIR"].includes(String(cause.code)) ) return new AnalysisInputError(OPERATION, { cause }, [ { - path: ["capture_path"], + path: inputPath, reason: "invalid_value", - message: `Selected capture could not be read (${String(cause.code)}): ${input.capture_path}.`, + message: `Selected ${sourceLabel} could not be read (${String(cause.code)}): ${sourcePath}.`, }, ]); if ( @@ -182,6 +192,9 @@ export const historicalCaptureFailure = ( diagnostics: { phase, capture_path: input.capture_path, + ...(descriptorRead + ? { source_path: sourcePath, input_path: inputPath } + : {}), reason: cause instanceof Error ? cause.message : String(cause), ...(cause instanceof ArtifactReaderFailure ? { failure_kind: cause.reason } @@ -248,3 +261,36 @@ export const captureMemoryFailure = ( : {}), }, ); + +/** Preserve the exact file owner when capture or descriptor acquisition needs cleanup. */ +export const withCaptureReaderCleanup = ( + primary: AnalysisError, + cause: unknown, +): AnalysisError => { + if (!(cause instanceof ArtifactReaderFailure) || cause.cleanup === undefined) + return primary; + if (cause.reason === "unavailable") { + const cleanup = new ProviderCleanupError( + "web-network-capture", + cause.cleanup.resources, + { reason: cause.cleanup.reason }, + { operation: OPERATION, cause }, + ); + bindArtifactFileCleanup(cause, cleanup); + return cleanup; + } + const cleanup = new ProviderCleanupError( + "web-network-capture", + cause.cleanup.resources, + { reason: cause.cleanup.reason }, + { operation: OPERATION, cause }, + ); + bindArtifactFileCleanup(cause, cleanup); + const composed = analysisErrorWithCleanupFailure( + primary, + cleanup, + OPERATION, + ).bindCleanupDependencies([primary, cleanup]); + bindArtifactFileCleanup(cause, composed); + return composed; +}; diff --git a/src/browser/history/CaptureRedaction.ts b/src/browser/history/CaptureRedaction.ts index 63feb4899e..62f4eac469 100644 --- a/src/browser/history/CaptureRedaction.ts +++ b/src/browser/history/CaptureRedaction.ts @@ -9,6 +9,31 @@ const CREDENTIAL_HEADERS = new Set([ "cookie", "set-cookie", ]); +/** HTTP credential names apply equally to recorded headers and gRPC-web trailers. */ +export const isCaptureCredentialHeader = (name: string): boolean => + CREDENTIAL_HEADERS.has(name.toLowerCase()); + +/** Match an ASCII header name without allocating a potentially enormous line. */ +export const isCaptureCredentialHeaderBytes = ( + bytes: Uint8Array, + start: number, + end: number, +): boolean => { + for (const name of CREDENTIAL_HEADERS) { + if (end - start !== name.length) continue; + let matches = true; + for (let index = 0; index < name.length; index++) { + const byte = bytes[start + index]!; + const lower = byte >= 65 && byte <= 90 ? byte + 32 : byte; + if (lower !== name.charCodeAt(index)) { + matches = false; + break; + } + } + if (matches) return true; + } + return false; +}; /** Literal redaction and structural authentication exclusions preserve unrelated local evidence. */ export class CaptureRedaction { @@ -75,7 +100,7 @@ export class CaptureRedaction { return ( (/^\/(request|response)\/headers\/\d+\/value$/.test(pointer) && typeof parent.name === "string" && - CREDENTIAL_HEADERS.has(parent.name.toLowerCase())) || + isCaptureCredentialHeader(parent.name)) || /^\/(request|response)\/cookies\/\d+\/value$/.test(pointer) ); } diff --git a/src/browser/history/HistoricalCaptureDecoder.ts b/src/browser/history/HistoricalCaptureDecoder.ts index 550c96ca67..79e13071b8 100644 --- a/src/browser/history/HistoricalCaptureDecoder.ts +++ b/src/browser/history/HistoricalCaptureDecoder.ts @@ -1,7 +1,13 @@ +import { + decodeCaptureProtobuf, + prepareCaptureProtobuf, +} from "./protobuf/ProtobufCapture.js"; +import { CaptureFormatError } from "./CaptureFormatError.js"; import { CAPTURE_PROCESS_MEMORY_LIMIT_REASON, historicalCaptureFailure, captureMemoryFailure, + withCaptureReaderCleanup, } from "./CaptureFailures.js"; import { NODE_CAPTURE_HEAP_ENFORCED, @@ -17,7 +23,6 @@ import { z } from "zod"; import type { ExecutionOptions } from "../../application/AnalysisProvider.js"; import { readStableArtifact } from "../../artifacts/readStableArtifact.js"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; -import { bindArtifactFileCleanup } from "../../artifacts/ArtifactFileOperation.js"; import { ArtifactResourceScope, type ArtifactCleanupAttempt, @@ -29,6 +34,7 @@ import { AnalysisCancelledError, AnalysisInputError, AnalysisOutputError, + AnalysisResourceConstraintError, } from "../../domain/analysisErrorCore.js"; import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; @@ -104,7 +110,7 @@ export class HistoricalCaptureDecoder { ); return err( redactExplicitFailure( - this.withReaderCleanup(error, cause), + withCaptureReaderCleanup(error, cause), input.sensitive_values, ), ); @@ -118,6 +124,7 @@ export class HistoricalCaptureDecoder { let runtime: Pick | undefined; let result: Result; let phase: "capture-read" | "decoder" = "capture-read"; + let encodedReplyBytes = 0; try { const adapter = this.adapters.find( (candidate) => candidate.format === input.format, @@ -186,6 +193,7 @@ export class HistoricalCaptureDecoder { undefined, this.#resources, ); + encodedReplyBytes = replyFile.bytes.length; reply = replySchema.parse(JSON.parse(replyFile.bytes.toString("utf8"))); } catch (cause: unknown) { if (cause instanceof ArtifactReaderFailure) throw cause; @@ -295,11 +303,58 @@ export class HistoricalCaptureDecoder { ); result = ok({ ...reply.value, ...common, format: input.format }); } + if (result.ok) { + let safePartialSource = false; + try { + const prepared = await prepareCaptureProtobuf( + result.value, + input, + options?.signal, + ); + safePartialSource = true; + await decodeCaptureProtobuf( + result.value, + input, + this.#resources, + prepared, + options?.signal, + encodedReplyBytes + Buffer.byteLength(JSON.stringify(common)) + 16, + ); + } catch (cause) { + const error = + cause instanceof CaptureFormatError && cause.reason === "limit" + ? new AnalysisResourceConstraintError( + OPERATION, + "capacity", + cause.message, + { + maximum_encoded_capture_bytes: + WEB_NETWORK_CAPTURE_LIMITS.outputBytes, + }, + { + retryable: false, + remediationAction: + "Select smaller records, or inspect the retained original source bytes.", + }, + ) + : cause instanceof CaptureFormatError + ? new AnalysisInputError(OPERATION, undefined, [ + { + path: ["protobuf"], + reason: "invalid_format", + message: cause.message, + }, + ]) + : historicalCaptureFailure(input, cause, phase, options); + if (safePartialSource) error.retainPartialObservation(result.value); + throw error; + } + } } catch (cause: unknown) { const error = historicalCaptureFailure(input, cause, phase, options); result = err( redactExplicitFailure( - this.withReaderCleanup(error, cause), + withCaptureReaderCleanup(error, cause), input.sensitive_values, ), ); @@ -358,41 +413,6 @@ export class HistoricalCaptureDecoder { result.error.retainPartialObservation(completedCapture); return result; } - - private withReaderCleanup( - primary: AnalysisError, - cause: unknown, - ): AnalysisError { - if ( - !(cause instanceof ArtifactReaderFailure) || - cause.cleanup === undefined - ) - return primary; - if (cause.reason === "unavailable") { - const cleanup = new ProviderCleanupError( - "web-network-capture", - cause.cleanup.resources, - { reason: cause.cleanup.reason }, - { operation: OPERATION, cause }, - ); - bindArtifactFileCleanup(cause, cleanup); - return cleanup; - } - const cleanup = new ProviderCleanupError( - "web-network-capture", - cause.cleanup.resources, - { reason: cause.cleanup.reason }, - { operation: OPERATION, cause }, - ); - bindArtifactFileCleanup(cause, cleanup); - const composed = analysisErrorWithCleanupFailure( - primary, - cleanup, - OPERATION, - ).bindCleanupDependencies([primary, cleanup]); - bindArtifactFileCleanup(cause, composed); - return composed; - } } /** The Node capture decoder runs on this executable; only Node.js enforces its V8 heap flags. */ diff --git a/src/browser/history/protobuf/ProtobufBody.ts b/src/browser/history/protobuf/ProtobufBody.ts new file mode 100644 index 0000000000..7d040fd9c1 --- /dev/null +++ b/src/browser/history/protobuf/ProtobufBody.ts @@ -0,0 +1,194 @@ +import { + readGrpcFrame, + prepareProtobufBody, + type PreparedProtobufBody, +} from "./ProtobufFraming.js"; +import { CaptureFormatError } from "../CaptureFormatError.js"; +import type { + ProtobufDecoding, + ProtobufFrame, +} from "../../../domain/protobufWire.js"; +import { + decodeProtobufWire, + ProtobufBudget, + type WireSchema, +} from "./ProtobufWire.js"; + +/** Framing is protocol-specific: native gRPC has no in-body trailer frames. */ +export const PROTOBUF_FRAMING = { + "application/grpc": "grpc", + "application/grpc+proto": "grpc", + "application/grpc-web": "grpc-web", + "application/grpc-web+proto": "grpc-web", + "application/grpc-web-text": "grpc-web-text", + "application/grpc-web-text+proto": "grpc-web-text", + "application/x-protobuf": "raw", + "application/protobuf": "raw", + "application/vnd.google.protobuf": "raw", +} as const; + +export const PROTOBUF_LIMITATIONS = [ + "Wire numbers and ranges are observations. Length-delimited string, nested-message and packed readings are candidates without a supplied schema. Raw decoding does not infer field names, signedness or business meaning; nested candidate bytes remain accessible through their ranges.", + "Schema values are interpretations of the caller-supplied descriptor, not proof that the sender used that schema. Map semantics, extensions, required-field validation and unknown schema fields are not reconstructed.", +] as const; + +export const decodeProtobufBody = async ( + body: Uint8Array, + framing: ProtobufDecoding["framing"], + source: Pick, + redact: (value: string) => string, + budget: ProtobufBudget, + schema?: WireSchema, + prepared?: PreparedProtobufBody, +): Promise => { + const result: ProtobufDecoding = { + ...source, + decoder: "rea-protobuf-wire-v1", + framing, + offset_basis: + framing === "grpc-web-text" ? "base64-decoded-body" : "retained-body", + decoded_bytes: body.length, + status: "complete", + examined_bytes: 0, + frames: [], + fields: [], + limitations: [], + }; + const stop = (offset: number, reason: string): ProtobufDecoding => { + result.status = "partial"; + result.error = { offset, reason: redact(reason) }; + return result; + }; + // The production source pass already decoded text. Charge representation expansion + // here only after that pass made the retained source safe on failure. + if (framing === "grpc-web-text") budget.charge(body.length * 6); + const data = + prepared === undefined + ? ( + await prepareProtobufBody( + body, + framing, + { decoded: () => false, header: () => false }, + budget.signal, + ) + ).data + : prepared.data; + if (data === undefined) + return stop( + 0, + "gRPC-web text is not canonical base64 or independently padded base64 chunks", + ); + result.decoded_bytes = data.length; + let at = 0; + const addMessage = async (frame: ProtobufFrame) => { + if (frame.compressed) { + frame.status = "unknown"; + frame.error = { + offset: frame.payload_offset, + reason: + "Compressed gRPC payload requires its negotiated codec; source bytes remain retained", + }; + return; + } + const decoded = await decodeProtobufWire( + data, + frame.payload_offset, + frame.payload_offset + frame.bytes, + frame.index, + budget, + redact, + schema, + result.fields.length, + ); + for (const field of decoded.fields) result.fields.push(field); + frame.examined_bytes = decoded.examined_bytes; + if (decoded.error !== undefined) { + frame.status = "partial"; + frame.error = decoded.error; + } + }; + if (framing === "raw") { + const frame: ProtobufFrame = { + index: 0, + kind: "data", + offset: 0, + payload_offset: 0, + bytes: data.length, + compressed: false, + status: "complete", + examined_bytes: 0, + }; + result.frames.push(frame); + await addMessage(frame); + result.status = frame.status; + result.examined_bytes = frame.examined_bytes; + return result; + } + let trailerSeen = false; + while (at < data.length) { + await budget.checkpoint(); + if (trailerSeen) + return stop(at, "A gRPC-web trailer frame must terminate the body"); + const parsed = readGrpcFrame(data, at, framing); + if (!parsed.ok) return stop(parsed.error.offset, parsed.error.reason); + const { flags, bytes: length } = parsed.value; + budget.charge(1024); + const frame: ProtobufFrame = { + index: result.frames.length, + kind: flags & 128 ? "trailer" : "data", + offset: at, + payload_offset: at + 5, + bytes: length, + compressed: (flags & 1) !== 0, + status: "complete", + examined_bytes: 0, + }; + result.frames.push(frame); + if (frame.kind === "data") await addMessage(frame); + else { + trailerSeen = true; + if (frame.compressed) { + frame.status = "unknown"; + frame.error = { + offset: at + 5, + reason: "Compressed gRPC-web trailers remain opaque", + }; + } else { + budget.charge(length * 8); + try { + const text = new TextDecoder("utf-8", { + fatal: true, + ignoreBOM: true, + }).decode(data.subarray(at + 5, at + 5 + length)); + frame.trailers = []; + for (const line of text.split("\r\n")) { + if (line === "") continue; + const colon = line.indexOf(":"); + if ( + colon <= 0 || + !/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/.test(line.slice(0, colon)) + ) + throw new Error("Malformed trailer header"); + budget.charge(512); + frame.trailers.push({ + name: redact(line.slice(0, colon)), + value: redact(line.slice(colon + 1).replace(/^[ \t]+/, "")), + }); + } + frame.examined_bytes = length; + } catch (cause) { + if (cause instanceof CaptureFormatError) throw cause; + frame.status = "partial"; + frame.error = { + offset: at + 5, + reason: "Invalid gRPC-web trailer header block", + }; + } + } + } + if (frame.status !== "complete") result.status = "partial"; + result.examined_bytes += 5 + frame.examined_bytes; + at += 5 + length; + } + return result; +}; diff --git a/src/browser/history/protobuf/ProtobufCapture.test.ts b/src/browser/history/protobuf/ProtobufCapture.test.ts new file mode 100644 index 0000000000..7e711471be --- /dev/null +++ b/src/browser/history/protobuf/ProtobufCapture.test.ts @@ -0,0 +1,77 @@ +import { createHash } from "node:crypto"; +import { expect, it } from "vitest"; +import { ArtifactResourceScope } from "../../../artifacts/ArtifactResourceScope.js"; +import { + inspectWebNetworkCaptureInputSchema, + type WebNetworkCapture, +} from "../../../domain/webNetworkCapture.js"; +import { + prepareCaptureProtobuf, + decodeCaptureProtobuf, +} from "./ProtobufCapture.js"; + +it("restricts framing overrides to native body facts while preserving header bytes", async () => { + const binary = (pointer: string, bytes: Buffer) => ({ + pointer, + representation: "producer-bytes" as const, + state: "retained" as const, + bytes: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + content_base64: bytes.toString("base64"), + }); + const headers = [ + binary("/response/headers/0/0", Buffer.from("content-type")), + binary("/response/headers/0/1", Buffer.from("application/grpc")), + ]; + const report: WebNetworkCapture = { + artifact: { path: "/fixture.mitm", bytes: 100, sha256: "a".repeat(64) }, + decoder: { id: "fixture", name: "Native producer facts", version: "1" }, + container: { + reported: null, + numeric_literals: [], + redactions: [], + records_pointer: "/flows", + }, + format: "mitmproxy", + total_records: 1, + runtime_attribution: "unknown", + limitations: [], + records: [ + { + ordinal: 0, + location: { kind: "byte-range", offset: 0, bytes: 100 }, + reported: { response: { headers: [[null, null]], content: null } }, + binary_fields: [ + ...headers, + binary("/response/content", Buffer.from([0, 0, 0, 0, 2, 8, 1])), + ], + numeric_literals: [], + redactions: [], + limitations: [], + }, + ], + }; + const input = inspectWebNetworkCaptureInputSchema.parse({ + capture_path: report.artifact.path, + format: report.format, + record_ordinals: [0], + protobuf: { framing: "grpc" }, + }); + const resources = new ArtifactResourceScope(); + try { + const prepared = await prepareCaptureProtobuf(report, input); + await decodeCaptureProtobuf(report, input, resources, prepared); + } finally { + await resources.close(); + } + expect(report.records[0]?.protobuf).toMatchObject([ + { + pointer: "/response/content", + status: "complete", + fields: [{ number: 1, value: "1" }], + }, + ]); + expect(report.records[0]?.protobuf).toHaveLength(1); + expect(report.records[0]?.binary_fields.slice(0, 2)).toEqual(headers); + expect(report.protobuf_coverage?.status).toBe("complete"); +}); diff --git a/src/browser/history/protobuf/ProtobufCapture.ts b/src/browser/history/protobuf/ProtobufCapture.ts new file mode 100644 index 0000000000..0eaaf78806 --- /dev/null +++ b/src/browser/history/protobuf/ProtobufCapture.ts @@ -0,0 +1,501 @@ +import { isAbsolute } from "node:path"; +import { CaptureFormatError } from "../CaptureFormatError.js"; +import { + CaptureRedaction, + isCaptureCredentialHeaderBytes, +} from "../CaptureRedaction.js"; +import { createHash } from "node:crypto"; +import { readStableArtifact } from "../../../artifacts/readStableArtifact.js"; +import { + historicalCaptureFailure, + withCaptureReaderCleanup, +} from "../CaptureFailures.js"; +import type { ArtifactResourceScope } from "../../../artifacts/ArtifactResourceScope.js"; +import { + AnalysisInputError, + AnalysisCapabilityUnavailableError, +} from "../../../domain/analysisErrorCore.js"; +import { redactExplicitText } from "../../../domain/explicitSensitiveValues.js"; +import { + WEB_NETWORK_CAPTURE_LIMITS, + type InspectWebNetworkCaptureInput, + type WebNetworkCapture, + type WebNetworkCaptureRecord, +} from "../../../domain/webNetworkCapture.js"; +import type { JsonValue } from "../../../domain/jsonValue.js"; +import { ProtobufBudget } from "./ProtobufWire.js"; +import type { ProtobufSchema } from "./ProtobufSchema.js"; +import { + PROTOBUF_FRAMING, + PROTOBUF_LIMITATIONS, + decodeProtobufBody, +} from "./ProtobufBody.js"; +import { + prepareProtobufBody, + type PreparedProtobufBody, +} from "./ProtobufFraming.js"; +import type { ProtobufDecoding } from "../../../domain/protobufWire.js"; + +const object = ( + value: JsonValue | undefined, +): Readonly> | undefined => + value !== null && typeof value === "object" && !Array.isArray(value) + ? value + : undefined; + +/** Exclude the original encoded representation as well as its decoding sidecar. */ +const excludeBody = ( + record: WebNetworkCaptureRecord, + pointer: string, + reason: "explicit-sensitive-value" | "transport-credential", +) => { + record.binary_fields = record.binary_fields.map((field) => + field.pointer !== pointer + ? field + : { + pointer: field.pointer, + representation: field.representation, + state: "redacted", + content_base64: null, + bytes: null, + sha256: null, + }, + ); + const parts = pointer + .split("/") + .slice(1) + .map((part) => part.replaceAll("~1", "/").replaceAll("~0", "~")); + let parent: JsonValue | undefined = record.reported; + for (const part of parts.slice(0, -1)) { + if ( + parent === null || + typeof parent !== "object" || + !Object.hasOwn(parent, part) + ) { + parent = undefined; + break; + } + parent = Array.isArray(parent) ? parent[Number(part)] : parent[part]; + } + const key = parts.at(-1); + if ( + parent !== null && + typeof parent === "object" && + key !== undefined && + Object.hasOwn(parent, key) + ) + Object.defineProperty(parent, key, { + value: null, + writable: true, + configurable: true, + enumerable: true, + }); + record.redactions.push({ pointer, reason }); +}; + +/** Format-aware owner of body metadata; callers never reinterpret producer JSON. */ +const bodyMetadata = ( + record: WebNetworkCaptureRecord, + pointer: string, + format: string, + binaryByPointer: ReadonlyMap< + string, + WebNetworkCaptureRecord["binary_fields"][number] + >, +) => { + const side = /^\/(request|response)\//.exec(pointer)?.[1]; + if ( + side === undefined || + ![ + "/request/postData/text", + "/response/content/text", + "/request/body", + "/response/body", + "/request/content", + "/response/content", + ].includes(pointer) + ) + return undefined; + const message = object(object(record.reported)?.[side]); + if (message === undefined) return undefined; + const headers = new Map(); + let headerNamesKnown = Array.isArray(message.headers); + if (Array.isArray(message.headers)) { + for (const [index, item] of message.headers.entries()) { + const pair = object(item); + const text = ( + value: JsonValue | undefined, + coordinate: string, + ): string | undefined => { + const binary = binaryByPointer.get(coordinate); + if (binary !== undefined) + return binary.state === "retained" + ? Buffer.from(binary.content_base64, "base64").toString("latin1") + : undefined; + return typeof value === "string" ? value : undefined; + }; + const name = Array.isArray(item) + ? text(item[0], `/${side}/headers/${index}/0`) + : pair?.name; + const value = Array.isArray(item) + ? text(item[1], `/${side}/headers/${index}/1`) + : pair?.value; + if (typeof name === "string") { + const key = name.toLowerCase(); + const knownValue = typeof value === "string" ? value : undefined; + headers.set( + key, + headers.has(key) && headers.get(key) !== knownValue + ? undefined + : knownValue, + ); + } else headerNamesKnown = false; + } + } + if (format === "har") { + if ( + pointer !== `/${side}/${side === "request" ? "postData" : "content"}/text` + ) + return undefined; + const content = object( + message[side === "request" ? "postData" : "content"], + ); + return { + contentType: + typeof content?.mimeType === "string" + ? content.mimeType + : headers.get("content-type"), + httpIdentityKnown: true, + httpCodingKnown: true, + }; + } + if (pointer !== `/${side}/body` && pointer !== `/${side}/content`) + return undefined; + return { + contentType: headers.get("content-type"), + httpCodingKnown: + headerNamesKnown && + ["content-encoding", "transfer-encoding"].every( + (name) => !headers.has(name) || headers.get(name) !== undefined, + ), + httpIdentityKnown: + headerNamesKnown && + ["content-encoding", "transfer-encoding"].every( + (name) => + // HTTP OWS is SP/HTAB; require the entire identity token, including EOF. + !headers.has(name) || + /^[ \t]*identity[ \t]*(?![\s\S])/i.test(headers.get(name) ?? ""), + ), + }; +}; + +type RetainedBody = Extract< + WebNetworkCaptureRecord["binary_fields"][number], + { state: "retained" } +>; +interface PreparedBody { + readonly field: RetainedBody; + readonly body: Buffer; + readonly contentType: string | undefined; + readonly framing: ProtobufDecoding["framing"]; + readonly prepared: PreparedProtobufBody; +} +export interface PreparedProtobufCapture { + readonly records: readonly { + record: WebNetworkCaptureRecord; + bodies: readonly PreparedBody[]; + }[]; + readonly incomplete: boolean; +} + +/** Sanitize every selected body before any schema acquisition or field-expansion failure. + * Until this stage completes, the caller must not publish the original source as partial Evidence. + */ +export const prepareCaptureProtobuf = async ( + capture: WebNetworkCapture, + input: InspectWebNetworkCaptureInput, + signal?: AbortSignal, +): Promise => { + if (input.record_ordinals === undefined) return undefined; + capture.protobuf_coverage = { + status: "partial", + selected_records: input.record_ordinals.length, + inspected_records: 0, + }; + capture.limitations.push(...PROTOBUF_LIMITATIONS); + let incomplete = false; + const records: { record: WebNetworkCaptureRecord; bodies: PreparedBody[] }[] = + []; + const options = input.protobuf; + const pointers = + options?.body_pointers === undefined + ? undefined + : new Set(options.body_pointers); + const redactions = new CaptureRedaction(input.sensitive_values); + const work = new ProtobufBudget(signal); + for (const ordinal of input.record_ordinals) { + await work.checkpoint(); + const record = capture.records[ordinal]; + if (record === undefined) continue; + const bodies: PreparedBody[] = []; + records.push({ record, bodies }); + const binaryByPointer = new Map( + record.binary_fields.map((field) => [field.pointer, field]), + ); + for (const field of record.binary_fields) { + await work.checkpoint(); + if (pointers !== undefined && !pointers.has(field.pointer)) continue; + const metadata = bodyMetadata( + record, + field.pointer, + input.format, + binaryByPointer, + ); + // Framing overrides interpret bodies; header and extension bytes remain their own facts. + if (metadata === undefined) continue; + if (field.state !== "retained") { + incomplete = true; + continue; + } + const contentType = metadata.contentType; + const essence = contentType?.split(";", 1)[0]?.trim().toLowerCase(); + const automatic = + essence === undefined + ? undefined + : PROTOBUF_FRAMING[essence as keyof typeof PROTOBUF_FRAMING]; + const framing = + options?.framing === undefined || options.framing === "auto" + ? automatic + : options.framing; + if (framing === undefined) continue; + if ( + field.bytes > WEB_NETWORK_CAPTURE_LIMITS.inputBytes || + field.content_base64.length > + Math.ceil(WEB_NETWORK_CAPTURE_LIMITS.inputBytes / 3) * 4 + ) + throw new CaptureFormatError( + "limit", + "Protobuf body exceeds the admitted 32 MiB source bound", + ); + const body = Buffer.from(field.content_base64, "base64"); + if ( + body.length !== field.bytes || + createHash("sha256").update(body).digest("hex") !== field.sha256 + ) + throw new AnalysisInputError("inspect_web_network_capture", undefined, [ + { + path: ["capture_path"], + reason: "invalid_format", + message: + "Retained protobuf bytes contradict their source descriptor", + }, + ]); + const exclusions = { + decoded: (data: Uint8Array) => + redactions.sensitiveBytes( + Buffer.from(data.buffer, data.byteOffset, data.byteLength), + ), + header: isCaptureCredentialHeaderBytes, + }; + const prepared = await prepareProtobufBody( + body, + framing, + exclusions, + signal, + ); + // Caller overrides do not turn off exclusions for the transport declared by the source. + const declared = + prepared.excluded === undefined && + automatic !== undefined && + automatic !== framing + ? await prepareProtobufBody(body, automatic, exclusions, signal) + : prepared; + const excluded = prepared.excluded ?? declared.excluded; + if (excluded !== undefined) { + excludeBody(record, field.pointer, excluded); + binaryByPointer.delete(field.pointer); + incomplete = true; + continue; + } + // Analysis eligibility never bypasses the independent source exclusions. + if ( + !metadata.httpCodingKnown && + (!prepared.credentialsChecked || !declared.credentialsChecked) + ) + throw new AnalysisCapabilityUnavailableError( + input.format, + "inspect_web_network_capture", + "Credential inspection could not complete while HTTP coding metadata was unavailable; unchecked capture bodies were withheld.", + { + diagnostics: { body_pointer: field.pointer }, + userMessage: + "Use a capture with available HTTP coding metadata, or select records whose transport can be inspected. Unchecked source bytes were not published.", + }, + ); + if (!metadata.httpIdentityKnown) { + incomplete = true; + record.limitations.push( + "Selected protobuf body's HTTP content/transfer coding is encoded or unknown, so identity is not established; its exact stored bytes remain opaque.", + ); + continue; + } + bodies.push({ field, body, contentType, framing, prepared }); + } + } + return { records, incomplete }; +}; + +/** Interpret the sanitized source once, inside the capture adapter's owned operation. */ +export const decodeCaptureProtobuf = async ( + capture: WebNetworkCapture, + input: InspectWebNetworkCaptureInput, + resources: ArtifactResourceScope, + prepared: PreparedProtobufCapture | undefined, + signal?: AbortSignal, + encodedSourceBytes = 0, +): Promise => { + if (prepared === undefined || capture.protobuf_coverage === undefined) return; + let incomplete = prepared.incomplete; + const budget = new ProtobufBudget(signal); + budget.charge( + encodedSourceBytes + + 256 + + Buffer.byteLength(JSON.stringify(PROTOBUF_LIMITATIONS)), + ); + for (const entry of prepared.records) + for (const item of entry.bodies) + budget.charge( + item.body.length + + (item.prepared.data === item.body + ? 0 + : (item.prepared.data?.length ?? 0)), + ); + const redact = (text: string) => { + if (input.sensitive_values.length === 0) return text; + if (input.sensitive_values.some((literal) => text.includes(literal))) + budget.charge(text.length * 60); + return redactExplicitText(text, input.sensitive_values); + }; + const options = input.protobuf; + let schema: ProtobufSchema | undefined; + let schemaSource: ProtobufDecoding["schema"]; + if (options?.schema !== undefined) { + const path = options.schema.descriptor_set_path; + if (!isAbsolute(path)) + throw new AnalysisInputError("inspect_web_network_capture", undefined, [ + { + path: ["protobuf", "schema", "descriptor_set_path"], + reason: "invalid_format", + message: "Expected an absolute descriptor-set path", + }, + ]); + const snapshot = await readStableArtifact( + path, + WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + signal, + undefined, + resources, + ).catch((cause: unknown) => { + throw withCaptureReaderCleanup( + historicalCaptureFailure( + input, + cause, + "descriptor-read", + signal === undefined ? undefined : { signal }, + ), + cause, + ); + }); + const { ProtobufSchema } = await import("./ProtobufSchema.js"); + try { + schema = await ProtobufSchema.read( + snapshot.bytes, + options.schema.message_type, + budget, + redact, + ); + } catch (cause) { + if (signal?.aborted === true && cause === signal.reason) throw cause; + if (cause instanceof CaptureFormatError) throw cause; + throw new AnalysisInputError("inspect_web_network_capture", undefined, [ + { + path: ["protobuf", "schema"], + reason: "invalid_format", + message: + "The selected local FileDescriptorSet or message type is invalid or exceeds decoding budgets", + }, + ]); + } + schemaSource = { + decoder: "protobuf-es-2.16.0", + descriptor_set_path: redact(path), + descriptor_set_sha256: snapshot.sha256, + descriptor_set_bytes: snapshot.bytes.length, + message_type: redact(options.schema.message_type), + }; + } + for (const { record, bodies } of prepared.records) { + const decoded: ProtobufDecoding[] = []; + for (const { + field, + body, + contentType, + framing, + prepared: bodySource, + } of bodies) { + await budget.checkpoint(); + if ( + input.sensitive_values.some( + (value) => + field.pointer.includes(value) || + field.sha256.includes(value) || + schemaSource?.descriptor_set_sha256.includes(value), + ) + ) { + incomplete = true; + continue; + } + budget.charge( + 2048 + + 8 * + (field.pointer.length + + (contentType?.length ?? 0) + + (schemaSource?.descriptor_set_path.length ?? 0) + + (schemaSource?.message_type.length ?? 0)), + ); + schema?.reset(); + const result = await decodeProtobufBody( + body, + framing, + { + pointer: field.pointer, + content_type: contentType ?? null, + body_sha256: field.sha256, + }, + redact, + budget, + schema, + bodySource, + ); + if (schemaSource !== undefined) result.schema = schemaSource; + if ( + result.status !== "complete" || + result.fields.some( + (field) => + field.schema_status !== undefined && + field.schema_status !== "decoded", + ) + ) + incomplete = true; + decoded.push(result); + } + if (decoded.length > 0) record.protobuf = decoded; + capture.protobuf_coverage.inspected_records++; + } + if ( + !incomplete && + capture.protobuf_coverage.inspected_records === + capture.protobuf_coverage.selected_records + ) + capture.protobuf_coverage.status = "complete"; +}; diff --git a/src/browser/history/protobuf/ProtobufFraming.ts b/src/browser/history/protobuf/ProtobufFraming.ts new file mode 100644 index 0000000000..6c93feee8f --- /dev/null +++ b/src/browser/history/protobuf/ProtobufFraming.ts @@ -0,0 +1,144 @@ +import { err, ok } from "../../../domain/result.js"; +import type { ProtobufDecoding } from "../../../domain/protobufWire.js"; +import { WEB_NETWORK_CAPTURE_LIMITS } from "../../../domain/webNetworkCapture.js"; +import { CaptureFormatError } from "../CaptureFormatError.js"; +import { ProtobufBudget } from "./ProtobufWire.js"; + +/** One owner for frame boundaries used by exclusion and field interpretation. */ +export const readGrpcFrame = ( + data: Uint8Array, + at: number, + framing: Exclude, +) => { + if (data.length - at < 5) + return err({ offset: at, reason: "Truncated gRPC frame header" }); + const flags = data[at]!; + if ( + (framing === "grpc" && flags !== 0 && flags !== 1) || + (flags & ~0x81) !== 0 + ) + return err({ + offset: at, + reason: "Invalid flags for the selected gRPC framing", + }); + const length = + data[at + 1]! * 2 ** 24 + + data[at + 2]! * 2 ** 16 + + data[at + 3]! * 256 + + data[at + 4]!; + if (length > data.length - at - 5) + return err({ offset: at, reason: "gRPC frame exceeds the retained body" }); + return ok({ + flags, + bytes: length, + payload_offset: at + 5, + end: at + 5 + length, + }); +}; + +/** Decode padded stream chunks into one source-bounded buffer, never a chunk inventory. */ +const decodeText = async ( + body: Uint8Array, + work: ProtobufBudget, +): Promise => { + const text = Buffer.from(body.buffer, body.byteOffset, body.byteLength) + .toString("latin1") + .replace(/[ \r\n\t]/g, ""); + if (/[^A-Za-z0-9+/=]/.test(text)) return undefined; + const output = Buffer.alloc(Math.floor((text.length * 3) / 4)); + let at = 0; + let written = 0; + while (at < text.length) { + await work.checkpoint(); + let end = text.indexOf("=", at); + if (end < 0) end = text.length; + else { + end++; + if (text[end] === "=") end++; + } + const chunk = text.slice(at, end); + if ( + chunk.length % 4 === 1 || + (chunk.includes("=") && chunk.length % 4 !== 0) + ) + return undefined; + const bytes = Buffer.from(chunk, "base64"); + if ( + bytes.toString("base64").replace(/=+$/, "") !== chunk.replace(/=+$/, "") + ) + return undefined; + output.set(bytes, written); + written += bytes.length; + at = end; + } + return output.subarray(0, written); +}; + +export interface PreparedProtobufBody { + readonly data: Uint8Array | undefined; + readonly credentialsChecked: boolean; + readonly excluded?: "explicit-sensitive-value" | "transport-credential"; +} + +/** Complete credential inspection before schema I/O, fact budgets or recoverable errors. + * Its temporary storage is bounded by admitted source bytes, independently of field expansion. + */ +export const prepareProtobufBody = async ( + body: Uint8Array, + framing: ProtobufDecoding["framing"], + exclusions: { + decoded(bytes: Uint8Array): boolean; + header(bytes: Uint8Array, start: number, end: number): boolean; + }, + signal?: AbortSignal, +): Promise => { + if (body.length > WEB_NETWORK_CAPTURE_LIMITS.inputBytes) + throw new CaptureFormatError( + "limit", + "Protobuf body exceeds the admitted 32 MiB source bound", + ); + const work = new ProtobufBudget(signal); + await work.checkpoint(); + const data = + framing === "grpc-web-text" ? await decodeText(body, work) : body; + if (data === undefined) return { data, credentialsChecked: false }; + if (exclusions.decoded(data)) + return { + data, + credentialsChecked: true, + excluded: "explicit-sensitive-value", + }; + if (framing === "raw" || framing === "grpc") + return { data, credentialsChecked: true }; + let at = 0; + let credentialsChecked = true; + while (at < data.length) { + await work.checkpoint(); + const parsed = readGrpcFrame(data, at, framing); + if (!parsed.ok) { + credentialsChecked = false; + break; + } + const frame = parsed.value; + if (frame.flags === 129) credentialsChecked = false; + if (frame.flags === 128) { + let start = frame.payload_offset; + for (let index = start; index <= frame.end; index++) { + // Large/malformed lines remain source-bounded and cancellation stays cooperative. + if ((index - start) % 256 === 0) await work.checkpoint(); + if (index !== frame.end && data[index] !== 10) continue; + let colon = start; + while (colon < index && data[colon] !== 58) colon++; + if (colon < index && exclusions.header(data, start, colon)) + return { + data, + credentialsChecked: true, + excluded: "transport-credential", + }; + start = index + 1; + } + } + at = frame.end; + } + return { data, credentialsChecked }; +}; diff --git a/src/browser/history/protobuf/ProtobufSchema.ts b/src/browser/history/protobuf/ProtobufSchema.ts new file mode 100644 index 0000000000..8135b6c58d --- /dev/null +++ b/src/browser/history/protobuf/ProtobufSchema.ts @@ -0,0 +1,299 @@ +import { + createFileRegistry, + fromBinary, + ScalarType, + type DescField, + type DescEnum, + type DescMessage, +} from "@bufbuild/protobuf"; +import { + FileDescriptorSetSchema, + type FileDescriptorSet, +} from "@bufbuild/protobuf/wkt"; +import { BinaryReader } from "@bufbuild/protobuf/wire"; +import type { ProtobufField } from "../../../domain/protobufWire.js"; +import { WEB_NETWORK_CAPTURE_LIMITS } from "../../../domain/webNetworkCapture.js"; +import { CaptureFormatError } from "../CaptureFormatError.js"; +import { + decodeProtobufWire, + readProtobufVarint, + type ProtobufBudget, + type WireSchema, +} from "./ProtobufWire.js"; + +const scalarWire = (scalar: ScalarType): ProtobufField["wire_type"] => { + if (scalar === ScalarType.STRING || scalar === ScalarType.BYTES) return "len"; + if ( + [ScalarType.DOUBLE, ScalarType.FIXED64, ScalarType.SFIXED64].includes( + scalar, + ) + ) + return "i64"; + if ( + [ScalarType.FLOAT, ScalarType.FIXED32, ScalarType.SFIXED32].includes(scalar) + ) + return "i32"; + return "varint"; +}; + +const floatingValue = (value: number): string => + Object.is(value, -0) ? "-0" : String(value); + +/** Qualified names can expand a small descriptor into a much larger registry. */ +const chargeRegistryExpansion = async ( + descriptor: FileDescriptorSet, + budget: ProtobufBudget, +): Promise => { + for (const file of descriptor.file) { + budget.charge(file.name.length * 8 + 128); + const stack = file.messageType.map((message) => ({ + message, + prefix: file.package.length, + })); + const chargeQualifiedName = (name: string, prefix: number) => + budget.charge((prefix + 1 + name.length) * 8 + 128); + for (const enumeration of file.enumType) + chargeQualifiedName(enumeration.name, file.package.length); + for (const extension of file.extension) + chargeQualifiedName(extension.name, file.package.length); + for (const service of file.service) { + const length = file.package.length + 1 + service.name.length; + budget.charge(length * 8 + 128); + for (const method of service.method) + chargeQualifiedName(method.name, length); + } + while (stack.length > 0) { + await budget.checkpoint(); + const { message, prefix } = stack.pop()!; + const length = prefix + 1 + message.name.length; + budget.charge(length * 8 + 128); + for (const field of message.field) + chargeQualifiedName(field.name, length); + for (const extension of message.extension) + chargeQualifiedName(extension.name, length); + for (const enumeration of message.enumType) + chargeQualifiedName(enumeration.name, length); + for (const nested of message.nestedType) + stack.push({ message: nested, prefix: length }); + } + } +}; + +const scalarValue = (reader: BinaryReader, scalar: ScalarType): string => { + switch (scalar) { + case ScalarType.DOUBLE: + return floatingValue(reader.double()); + case ScalarType.FLOAT: + return floatingValue(reader.float()); + case ScalarType.INT64: + return String(reader.int64()); + case ScalarType.UINT64: + return String(reader.uint64()); + case ScalarType.INT32: + return String(reader.int32()); + case ScalarType.FIXED64: + return String(reader.fixed64()); + case ScalarType.FIXED32: + return String(reader.fixed32()); + case ScalarType.BOOL: + return String(reader.bool()); + case ScalarType.UINT32: + return String(reader.uint32()); + case ScalarType.SFIXED32: + return String(reader.sfixed32()); + case ScalarType.SFIXED64: + return String(reader.sfixed64()); + case ScalarType.SINT32: + return String(reader.sint32()); + case ScalarType.SINT64: + return String(reader.sint64()); + default: + throw new Error("Not a numeric protobuf scalar"); + } +}; + +/** The supplied schema interprets occurrences; it never overwrites raw wire facts. */ +export class ProtobufSchema implements WireSchema { + readonly #fields = new Map>(); + readonly #messages = new Map(); + readonly #enums = new WeakMap< + DescEnum, + ReadonlyMap + >(); + constructor( + readonly message: DescMessage, + readonly budget: ProtobufBudget, + readonly redact: (text: string) => string, + ) {} + + static async read( + bytes: Uint8Array, + typeName: string, + budget: ProtobufBudget, + redact: (text: string) => string, + ): Promise { + // Bound the object expansion of the descriptor before the SDK allocates its message tree. + const checked = await decodeProtobufWire( + bytes, + 0, + bytes.length, + 0, + budget, + redact, + new ProtobufSchema(FileDescriptorSetSchema, budget, redact), + ); + if ( + checked.error !== undefined || + checked.fields.some((field) => field.schema_status === "invalid-message") + ) + throw new Error("Malformed FileDescriptorSet wire format"); + const descriptor = fromBinary(FileDescriptorSetSchema, bytes, { + readUnknownFields: false, + recursionLimit: WEB_NETWORK_CAPTURE_LIMITS.depth, + }); + await chargeRegistryExpansion(descriptor, budget); + const message = createFileRegistry(descriptor).getMessage( + typeName.replace(/^\./, ""), + ); + if (message === undefined) + throw new Error( + "The supplied descriptor set does not define the selected message type", + ); + return new ProtobufSchema(message, budget, redact); + } + + reset(): void { + this.#messages.clear(); + } + + nested(field: ProtobufField): boolean { + return this.#messages.has(field.index); + } + + async annotate( + field: ProtobufField, + bytes: Uint8Array, + parent: ProtobufField | undefined, + ): Promise { + const message = + parent === undefined ? this.message : this.#messages.get(parent.index); + if (message === undefined) return; + let fields = this.#fields.get(message); + if (fields === undefined) { + fields = new Map(message.fields.map((item) => [item.number, item])); + this.#fields.set(message, fields); + } + const desc = fields.get(field.number); + if (desc === undefined) { + field.schema_status = "unknown-field"; + return; + } + this.budget.charge( + 8 * + (desc.name.length + + (desc.message?.typeName.length ?? 0) + + (desc.enum?.typeName.length ?? 0)), + ); + field.schema_name = this.redact(desc.name); + field.schema_status = "decoded"; + if (desc.fieldKind === "map") { + field.schema_type = "map"; + field.schema_status = "unsupported"; + return; + } + if (desc.message !== undefined) { + field.schema_type = this.redact(desc.message.typeName); + const expected = desc.delimitedEncoding ? "group" : "len"; + if (field.wire_type !== expected) + field.schema_status = "wire-type-mismatch"; + else this.#messages.set(field.index, desc.message); + return; + } + const scalar = desc.enum === undefined ? desc.scalar : ScalarType.INT32; + if (scalar === undefined) { + field.schema_status = "unsupported"; + return; + } + field.schema_type = + desc.enum === undefined + ? ScalarType[scalar].toLowerCase() + : this.redact(desc.enum.typeName); + const packed = + desc.fieldKind === "list" && + field.wire_type === "len" && + scalar !== ScalarType.STRING && + scalar !== ScalarType.BYTES; + if (!packed && field.wire_type !== scalarWire(scalar)) { + field.schema_status = "wire-type-mismatch"; + return; + } + const payload = bytes.subarray( + field.payload_offset, + field.payload_offset + field.payload_bytes, + ); + try { + if (scalar === ScalarType.BYTES) return; // Exact bytes already have an accessible source range. + if (scalar === ScalarType.STRING) { + // Wire decoding owns UTF-8 (including BOM) and literal exclusions. + // Reuse that fact; charge only its additional serialized representation. + this.budget.charge(payload.length * 6); + if (field.text === undefined) + field.schema_status = "wire-type-mismatch"; + else field.schema_value = field.text; + return; + } + const reader = new BinaryReader(payload); + const enumValues = new Map(); + const read = () => { + if ( + scalarWire(scalar) === "varint" && + readProtobufVarint(payload, reader.pos) === undefined + ) + throw new Error("Invalid packed varint"); + const value = scalarValue(reader, scalar); + if (desc.enum !== undefined) { + let names = this.#enums.get(desc.enum); + if (names === undefined) { + this.budget.charge(desc.enum.values.length * 256); + const indexed = new Map(); + for (const item of desc.enum.values) { + const number = String(item.number); + const aliases = indexed.get(number) ?? []; + aliases.push(item.name); + indexed.set(number, aliases); + } + names = indexed; + this.#enums.set(desc.enum, names); + } + const aliases = names.get(value); + if (aliases !== undefined && !enumValues.has(value)) { + this.budget.charge( + 256 + aliases.reduce((sum, name) => sum + name.length * 8, 0), + ); + enumValues.set(value, aliases.map(this.redact)); + } + } + return value; + }; + if (packed) { + const values: string[] = []; + while (reader.pos < reader.len) { + await this.budget.checkpoint(); + this.budget.charge(192); + values.push(this.redact(read())); + } + field.schema_value = JSON.stringify(values); + } else field.schema_value = this.redact(read()); + if (enumValues.size > 0) + field.schema_enum_values = [...enumValues].map(([number, names]) => ({ + number: this.redact(number), + names: [...names], + })); + } catch (cause) { + if (cause instanceof CaptureFormatError) throw cause; + if (this.budget.signal?.aborted && cause === this.budget.signal.reason) + throw cause; + field.schema_status = "wire-type-mismatch"; + } + } +} diff --git a/src/browser/history/protobuf/ProtobufWire.test.ts b/src/browser/history/protobuf/ProtobufWire.test.ts new file mode 100644 index 0000000000..1c7c6eb06c --- /dev/null +++ b/src/browser/history/protobuf/ProtobufWire.test.ts @@ -0,0 +1,522 @@ +import { expect, it } from "vitest"; +import { create, toBinary } from "@bufbuild/protobuf"; +import { FileDescriptorSetSchema } from "@bufbuild/protobuf/wkt"; +import { decodeProtobufWire, ProtobufBudget } from "./ProtobufWire.js"; +import { decodeProtobufBody } from "./ProtobufBody.js"; +import { ProtobufSchema } from "./ProtobufSchema.js"; +import { protobufCaptureDescriptor } from "../../../../tests/fixtures/protobufCapture.js"; + +const same = (value: string) => value; +const wire = (hex: string) => { + const bytes = Buffer.from(hex, "hex"); + return decodeProtobufWire( + bytes, + 0, + bytes.length, + 0, + new ProtobufBudget(), + same, + ); +}; +const source = { + pointer: "/response/content/text", + content_type: "application/grpc-web+proto", + body_sha256: "a".repeat(64), +}; +const frame = (bytes: Buffer, flag = 0) => { + const header = Buffer.alloc(5); + header[0] = flag; + header.writeUInt32BE(bytes.length, 1); + return Buffer.concat([header, bytes]); +}; + +it("rejects overflowing uint64 values without masking and preserves the maximum value", async () => { + expect(await wire("0880808080808080808002")).toMatchObject({ + fields: [], + error: { offset: 0 }, + }); + expect((await wire("08ffffffffffffffffff01")).fields[0]?.value).toBe( + "18446744073709551615", + ); + expect(await wire("0880808080808080808080")).toHaveProperty("error"); + expect(await wire("0880")).toHaveProperty("error"); +}); + +it("preserves repeated field occurrences, source ranges, ambiguous bytes and exact fixed values", async () => { + const result = await wire("0896010801120568656c6c6f1a02080125ffffffff"); + expect(result.error).toBeUndefined(); + expect(result.fields.map(({ number }) => number)).toEqual([1, 1, 2, 3, 4]); + expect(result.fields[2]).toMatchObject({ + field_id: "offset:5", + offset: 5, + payload_offset: 7, + payload_bytes: 5, + text: "hello", + interpretations: ["utf8-string", "packed-varint", "bytes"], + }); + expect(result.fields[3]?.interpretations).toContain("nested-message"); + expect(result.fields[3]?.text).toBe("\b\u0001"); + expect(result.fields[4]?.value).toBe("4294967295"); +}); + +it("handles proto2 groups and reports mismatched or missing end groups", async () => { + expect((await wire("0b10010c")).fields).toMatchObject([ + { wire_type: "group", bytes: 4 }, + { parent_field_id: "offset:0", number: 2, value: "1" }, + ]); + expect(await wire("0b100114")).toHaveProperty("error"); + expect(await wire("0b1001")).toHaveProperty("error"); + expect(await wire("0c")).toHaveProperty("error"); +}); + +it("keeps empty and control-containing UTF-8 among ambiguous length-delimited readings", async () => { + const empty = await wire("0a00"); + expect(empty.fields[0]).toMatchObject({ + text: "", + interpretations: [ + "utf8-string", + "nested-message", + "packed-varint", + "packed-fixed32", + "packed-fixed64", + "bytes", + ], + }); + expect((await wire("0a020801")).fields[0]).toMatchObject({ + text: "\b\u0001", + }); +}); + +it("keeps enum numbers separate from every declared alias", async () => { + const bytes = toBinary( + FileDescriptorSetSchema, + create(FileDescriptorSetSchema, { + file: [ + { + name: "aliases.proto", + package: "aliases", + syntax: "proto3", + enumType: [ + { + name: "Mode", + options: { allowAlias: true }, + value: [ + { name: "ZERO", number: 0 }, + { name: "ALSO_ZERO", number: 0 }, + ], + }, + ], + messageType: [ + { + name: "Message", + field: [ + { + name: "mode", + number: 1, + type: 14, + typeName: ".aliases.Mode", + label: 1, + }, + { name: "negative_zero", number: 2, type: 2, label: 1 }, + ], + }, + ], + }, + ], + }), + ); + const budget = new ProtobufBudget(); + const schema = await ProtobufSchema.read( + bytes, + "aliases.Message", + budget, + same, + ); + const decoded = await decodeProtobufBody( + Buffer.from([8, 0, 21, 0, 0, 0, 128]), + "raw", + source, + same, + budget, + schema, + ); + expect(decoded.fields[1]).toMatchObject({ + value: "2147483648", + schema_value: "-0", + }); + expect(decoded.fields[0]).toMatchObject({ + value: "0", + schema_type: "aliases.Mode", + schema_value: "0", + schema_enum_values: [{ number: "0", names: ["ZERO", "ALSO_ZERO"] }], + }); +}); + +it("bounds qualified descriptor-name expansion before creating the SDK registry", async () => { + const bytes = toBinary( + FileDescriptorSetSchema, + create(FileDescriptorSetSchema, { + file: [ + { + name: "expansion.proto", + syntax: "proto3", + messageType: [ + { + name: "A".repeat(10000), + nestedType: Array.from({ length: 2000 }, (_, index) => ({ + name: `Child${index}`, + })), + }, + ], + }, + ], + }), + ); + await expect( + ProtobufSchema.read(bytes, "Missing", new ProtobufBudget(), same), + ).rejects.toMatchObject({ reason: "limit" }); +}); + +it("distinguishes native gRPC flags from web trailers and retains partial messages", async () => { + const body = Buffer.concat([ + frame(Buffer.from("0801", "hex")), + frame(Buffer.from("grpc-status: 0\r\ngrpc-message: ok\r\n"), 128), + ]); + const web = await decodeProtobufBody( + body, + "grpc-web", + source, + same, + new ProtobufBudget(), + ); + expect(web.status).toBe("complete"); + expect(web.frames[1]?.trailers).toEqual([ + { name: "grpc-status", value: "0" }, + { name: "grpc-message", value: "ok" }, + ]); + expect( + (await decodeProtobufBody(body, "grpc", source, same, new ProtobufBudget())) + .error, + ).toMatchObject({ offset: 7 }); + expect( + ( + await decodeProtobufBody( + frame(Buffer.from("0880", "hex")), + "grpc", + source, + same, + new ProtobufBudget(), + ) + ).frames[0], + ).toMatchObject({ status: "partial", error: { offset: 5 } }); + expect( + ( + await decodeProtobufBody( + frame(Buffer.from("0801", "hex"), 2), + "grpc-web", + source, + same, + new ProtobufBudget(), + ) + ).status, + ).toBe("partial"); +}); + +it("decodes independently padded base64 chunks with explicit coordinates and rejects noncanonical text", async () => { + const first = frame(Buffer.from("0801", "hex")); + const second = frame(Buffer.from("0802", "hex")); + const body = Buffer.from( + first.toString("base64") + second.toString("base64"), + ); + const decoded = await decodeProtobufBody( + body, + "grpc-web-text", + source, + same, + new ProtobufBudget(), + ); + expect(decoded).toMatchObject({ + offset_basis: "base64-decoded-body", + decoded_bytes: 14, + status: "complete", + }); + expect( + decoded.fields.map(({ field_id, value }) => [field_id, value]), + ).toEqual([ + ["offset:5", "1"], + ["offset:12", "2"], + ]); + expect(decoded.fields.map(({ index }) => index)).toEqual([0, 1]); + expect( + ( + await decodeProtobufBody( + Buffer.from("AB=="), + "grpc-web-text", + source, + same, + new ProtobufBudget(), + ) + ).status, + ).toBe("partial"); +}); + +it("keeps compressed payloads explicitly unknown and rejects frames beyond the body", async () => { + const compressed = await decodeProtobufBody( + frame(Buffer.from("0801", "hex"), 1), + "grpc", + source, + same, + new ProtobufBudget(), + ); + expect(compressed.frames[0]).toMatchObject({ + status: "unknown", + compressed: true, + examined_bytes: 0, + }); + expect(compressed.fields).toEqual([]); + expect( + ( + await decodeProtobufBody( + Buffer.from("00000000090801", "hex"), + "grpc", + source, + same, + new ProtobufBudget(), + ) + ).error, + ).toBeDefined(); +}); + +it("uses supplied descriptors without replacing wire values and preserves schema disagreements", async () => { + const descriptor = toBinary( + FileDescriptorSetSchema, + create(FileDescriptorSetSchema, { + file: [ + { + name: "sample.proto", + package: "sample", + syntax: "proto3", + messageType: [ + { + name: "Child", + field: [{ name: "count", number: 1, type: 5, label: 1 }], + }, + { + name: "Message", + field: [ + { name: "signed", number: 1, type: 17, label: 1 }, + { name: "text", number: 2, type: 9, label: 1 }, + { + name: "child", + number: 3, + type: 11, + label: 1, + typeName: ".sample.Child", + }, + { name: "values", number: 4, type: 13, label: 3 }, + ], + }, + ], + }, + ], + }), + ); + const budget = new ProtobufBudget(); + const schema = await ProtobufSchema.read( + descriptor, + "sample.Message", + budget, + same, + ); + const body = Buffer.from("0801120568656c6c6f1a020801220201023801", "hex"); + const decoded = await decodeProtobufBody( + body, + "raw", + source, + same, + budget, + schema, + ); + expect(decoded.fields[0]).toMatchObject({ + value: "1", + schema_name: "signed", + schema_type: "sint32", + schema_value: "-1", + }); + expect( + decoded.fields.find(({ schema_name }) => schema_name === "text") + ?.schema_value, + ).toBe("hello"); + expect( + decoded.fields.find(({ schema_name }) => schema_name === "count"), + ).toMatchObject({ parent_field_id: "offset:9", value: "1" }); + expect( + decoded.fields.find(({ schema_name }) => schema_name === "values") + ?.schema_value, + ).toBe('["1","2"]'); + expect(decoded.fields.at(-1)?.schema_status).toBe("unknown-field"); + schema.reset(); + expect( + ( + await decodeProtobufBody( + Buffer.from("1001", "hex"), + "raw", + source, + same, + budget, + schema, + ) + ).fields[0]?.schema_status, + ).toBe("wire-type-mismatch"); +}); + +it("stops before exhausting fact capacity and yields for caller cancellation", async () => { + const budget = new ProtobufBudget(); + budget.charge(96 * 1024 * 1024 - 1023); + await expect( + decodeProtobufWire(Buffer.from([8, 1]), 0, 2, 0, budget, same), + ).rejects.toMatchObject({ reason: "limit" }); + const abort = new AbortController(); + const reason = new Error("caller stopped selected decoding"); + const data = Buffer.alloc(8192); + for (let at = 0; at < data.length; at += 2) { + data[at] = 8; + data[at + 1] = 1; + } + const task = decodeProtobufWire( + data, + 0, + data.length, + 0, + new ProtobufBudget(abort.signal), + same, + ); + setImmediate(() => abort.abort(reason)); + await expect(task).rejects.toBe(reason); +}); + +it("preserves a leading Unicode BOM in both the wire string and schema interpretation", async () => { + const budget = new ProtobufBudget(); + const schema = await ProtobufSchema.read( + protobufCaptureDescriptor(), + "capture.Message", + budget, + same, + ); + const bytes = Buffer.from("0a08efbbbf68656c6c6f", "hex"); + const result = await decodeProtobufWire( + bytes, + 0, + bytes.length, + 0, + budget, + same, + schema, + ); + expect(result.fields[0]).toMatchObject({ + text: "\uFEFFhello", + schema_value: "\uFEFFhello", + }); +}); + +it("keeps bad nested schema interpretations local and preserves subsequent outer fields", async () => { + const descriptor = toBinary( + FileDescriptorSetSchema, + create(FileDescriptorSetSchema, { + file: [ + { + name: "nested.proto", + package: "nested", + syntax: "proto3", + messageType: [ + { + name: "Child", + field: [{ name: "value", number: 1, type: 13, label: 1 }], + }, + { + name: "Message", + field: [ + { + name: "child", + number: 1, + type: 11, + label: 1, + typeName: ".nested.Child", + }, + { name: "value", number: 2, type: 13, label: 1 }, + ], + }, + ], + }, + ], + }), + ); + const budget = new ProtobufBudget(); + const schema = await ProtobufSchema.read( + descriptor, + "nested.Message", + budget, + same, + ); + const bytes = Buffer.from("0a01ff1001", "hex"); + const result = await decodeProtobufWire( + bytes, + 0, + bytes.length, + 0, + budget, + same, + schema, + ); + expect(result.error).toBeUndefined(); + expect(result.examined_bytes).toBe(bytes.length); + expect(result.fields).toMatchObject([ + { + number: 1, + schema_status: "invalid-message", + schema_error: { offset: 2 }, + }, + { number: 2, value: "1", schema_value: "1", parent_field_id: null }, + ]); +}); + +it("yields and preserves the cancellation reason inside one large packed interpretation", async () => { + const abort = new AbortController(); + const budget = new ProtobufBudget(abort.signal); + const schema = await ProtobufSchema.read( + protobufCaptureDescriptor(), + "capture.Message", + budget, + same, + ); + // Field 2, length 100000, followed by 100000 independently expected uint32 ones. + const bytes = Buffer.concat([ + Buffer.from([18, 160, 141, 6]), + Buffer.alloc(100000, 1), + ]); + const parsed = await decodeProtobufWire( + bytes, + 0, + bytes.length, + 0, + new ProtobufBudget(), + same, + ); + const field = parsed.fields[0]; + if (field === undefined) throw new Error("Expected the packed wire field"); + const task = schema.annotate(field, bytes, undefined); + const reason = new Error("stop during packed annotation"); + setImmediate(() => abort.abort(reason)); + await expect(task).rejects.toBe(reason); +}); + +it("rejects malformed descriptor wire even when the SDK would skip the unknown field", async () => { + // File t.proto defines empty M, then unknown field 99 carries an overflowing uint64. + const bytes = Buffer.from( + "0a1a0a07742e70726f746f22030a014d980680808080808080808002", + "hex", + ); + await expect( + ProtobufSchema.read(bytes, "M", new ProtobufBudget(), same), + ).rejects.toThrow("Malformed FileDescriptorSet wire format"); +}); diff --git a/src/browser/history/protobuf/ProtobufWire.ts b/src/browser/history/protobuf/ProtobufWire.ts new file mode 100644 index 0000000000..a1eb384303 --- /dev/null +++ b/src/browser/history/protobuf/ProtobufWire.ts @@ -0,0 +1,269 @@ +import { setImmediate } from "node:timers/promises"; +import type { ProtobufField } from "../../../domain/protobufWire.js"; +import { CaptureFormatError } from "../CaptureFormatError.js"; +import { WEB_NETWORK_CAPTURE_LIMITS } from "../../../domain/webNetworkCapture.js"; + +/** One charge covers all selected records, including temporary interpretation strings. */ +export class ProtobufBudget { + #remaining = WEB_NETWORK_CAPTURE_LIMITS.outputBytes; + #work = 0; + constructor(readonly signal?: AbortSignal) {} + charge(bytes: number): void { + if (!Number.isSafeInteger(bytes) || bytes < 0 || bytes > this.#remaining) + throw new CaptureFormatError( + "limit", + "Protobuf decoding exceeds the shared 96 MiB encoded-result and working-set budget.", + ); + this.#remaining -= bytes; + } + async checkpoint(): Promise { + this.signal?.throwIfAborted(); + if (++this.#work % 256 === 0) { + await setImmediate(); + this.signal?.throwIfAborted(); + } + } +} + +/** Reject overflowing uint64s before conversion; never wrap malformed bytes. */ +export const readProtobufVarint = ( + bytes: Uint8Array, + start: number, + end = bytes.length, +): { value: bigint; end: number } | undefined => { + let value = 0n; + for (let index = 0; index < 10 && start + index < end; index++) { + const byte = bytes[start + index]!; + if (index === 9 && byte > 1) return undefined; + value |= BigInt(byte & 127) << BigInt(index * 7); + if (byte < 128) return { value, end: start + index + 1 }; + } + return undefined; +}; + +const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const textOf = (bytes: Uint8Array): string | undefined => { + try { + return utf8.decode(bytes); + } catch { + return undefined; + } +}; + +/** An allocation-free syntactic candidate check, not a claim about the payload's type. */ +const messageCandidate = async ( + bytes: Uint8Array, + start: number, + end: number, + budget: ProtobufBudget, +): Promise => { + if (start === end) return true; + while (start < end) { + await budget.checkpoint(); + const tag = readProtobufVarint(bytes, start, end); + if ( + tag === undefined || + tag.value >> 3n === 0n || + tag.value >> 3n > 0x1fffffffn + ) + return false; + const type = Number(tag.value & 7n); + start = tag.end; + if (type === 0) { + const value = readProtobufVarint(bytes, start, end); + if (value === undefined) return false; + start = value.end; + } else if (type === 1 || type === 5) start += type === 1 ? 8 : 4; + else if (type === 2) { + const size = readProtobufVarint(bytes, start, end); + if (size === undefined || size.value > BigInt(end - size.end)) + return false; + start = size.end + Number(size.value); + } else return false; + } + return start === end; +}; + +const packedVarintCandidate = async ( + bytes: Uint8Array, + start: number, + end: number, + budget: ProtobufBudget, +): Promise => { + while (start < end) { + await budget.checkpoint(); + const value = readProtobufVarint(bytes, start, end); + if (value === undefined) return false; + start = value.end; + } + return true; +}; + +export interface WireSchema { + annotate( + field: ProtobufField, + bytes: Uint8Array, + parent: ProtobufField | undefined, + ): void | Promise; + nested(field: ProtobufField): boolean; +} + +/** Decode one message into flat occurrence facts with offsets in the decoded body. */ +export const decodeProtobufWire = async ( + bytes: Uint8Array, + start: number, + end: number, + frameIndex: number, + budget: ProtobufBudget, + redact: (text: string) => string, + schema?: WireSchema, + baseIndex = 0, +): Promise<{ + fields: ProtobufField[]; + examined_bytes: number; + error?: { offset: number; reason: string }; +}> => { + const fields: ProtobufField[] = []; + const walk = async ( + from: number, + until: number, + parent: ProtobufField | undefined, + depth: number, + group?: number, + ): Promise<{ + end: number; + payloadEnd?: number; + error?: { offset: number; reason: string }; + }> => { + let at = from; + const fail = (reason: string) => ({ + end: at, + error: { offset: at, reason }, + }); + while (at < until) { + await budget.checkpoint(); + const tag = readProtobufVarint(bytes, at, until); + if (tag === undefined) return fail("Truncated or overflowing field tag"); + const number = Number(tag.value >> 3n); + const type = Number(tag.value & 7n); + if (number === 0 || tag.value >> 3n > 0x1fffffffn) + return fail("Invalid field number"); + if (type === 4) + return group === number + ? { end: tag.end, payloadEnd: at } + : fail("Unexpected or mismatched end-group tag"); + let payloadOffset = tag.end; + let next: number; + let value: string | undefined; + if (type === 0) { + const parsed = readProtobufVarint(bytes, tag.end, until); + if (parsed === undefined) + return fail("Truncated or overflowing uint64 varint"); + next = parsed.end; + value = parsed.value.toString(); + } else if (type === 1 || type === 5) { + next = tag.end + (type === 1 ? 8 : 4); + if (next > until) return fail("Truncated fixed-width field"); + let integer = 0n; + for (let p = next - 1; p >= tag.end; p--) + integer = (integer << 8n) | BigInt(bytes[p]!); + value = integer.toString(); + } else if (type === 2) { + const length = readProtobufVarint(bytes, tag.end, until); + if (length === undefined) + return fail("Truncated or overflowing length prefix"); + if (length.value > BigInt(until - length.end)) + return fail("Length-delimited field exceeds its message"); + payloadOffset = length.end; + next = payloadOffset + Number(length.value); + } else if (type === 3) { + if (depth >= WEB_NETWORK_CAPTURE_LIMITS.depth) + return fail("Group nesting exceeds the 64-level decoder budget"); + next = tag.end; + } else return fail(`Invalid wire type ${type}`); + budget.charge(1024); + const field: ProtobufField = { + index: baseIndex + fields.length, + frame_index: frameIndex, + parent_index: parent?.index ?? null, + field_id: `offset:${at}`, + parent_field_id: + parent === undefined ? null : `offset:${parent.offset}`, + number, + wire_type: + type === 0 + ? "varint" + : type === 1 + ? "i64" + : type === 2 + ? "len" + : type === 3 + ? "group" + : "i32", + offset: at, + bytes: next - at, + payload_offset: payloadOffset, + payload_bytes: next - payloadOffset, + ...(value === undefined ? {} : { value: redact(value) }), + }; + fields.push(field); + if (type === 2) { + // Account for UTF-16 allocation and JSON escaping before decoding any string. + budget.charge(field.payload_bytes * 8); + const text = textOf(bytes.subarray(payloadOffset, next)); + const candidate = await messageCandidate( + bytes, + payloadOffset, + next, + budget, + ); + const packed = await packedVarintCandidate( + bytes, + payloadOffset, + next, + budget, + ); + field.interpretations = [ + ...(text === undefined ? [] : ["utf8-string" as const]), + ...(candidate ? ["nested-message" as const] : []), + ...(packed ? ["packed-varint" as const] : []), + ...(field.payload_bytes % 4 === 0 ? ["packed-fixed32" as const] : []), + ...(field.payload_bytes % 8 === 0 ? ["packed-fixed64" as const] : []), + "bytes", + ]; + if (text !== undefined) field.text = redact(text); + } + if (schema !== undefined) await schema.annotate(field, bytes, parent); + if (type === 3) { + const nested = await walk(tag.end, until, field, depth + 1, number); + field.bytes = nested.end - at; + field.payload_bytes = (nested.payloadEnd ?? nested.end) - payloadOffset; + if (nested.error !== undefined) return nested; + next = nested.end; + } else if (type === 2 && schema?.nested(field)) { + const error = + depth >= WEB_NETWORK_CAPTURE_LIMITS.depth + ? { + offset: payloadOffset, + reason: + "Schema message nesting exceeds the 64-level decoder budget", + } + : (await walk(payloadOffset, next, field, depth + 1)).error; + // The outer length is an observed boundary. A bad interpretation of its + // payload must never hide valid fields following that boundary. + if (error !== undefined) { + field.schema_status = "invalid-message"; + field.schema_error = error; + } + } + at = next; + } + return group === undefined ? { end: at } : fail("Missing end-group tag"); + }; + const result = await walk(start, end, undefined, 0); + return { + fields, + examined_bytes: result.end - start, + ...(result.error === undefined ? {} : { error: result.error }), + }; +}; diff --git a/src/cli/webNetworkCaptureCommands.ts b/src/cli/webNetworkCaptureCommands.ts index 81a7ec4c52..5f2306f35e 100644 --- a/src/cli/webNetworkCaptureCommands.ts +++ b/src/cli/webNetworkCaptureCommands.ts @@ -7,7 +7,11 @@ import type { Logger } from "pino"; import type { CliInstance } from "./types.js"; import { cliIntegerArgument } from "./options.js"; import { withCommandCancellation } from "./commandCancellation.js"; -import { finishAnalysisCommand } from "./finishAnalysisCommand.js"; +import { + analysisEvidenceDeliveryOptions, + deliverCliAnalysisEvidence, +} from "./analysisEvidenceDelivery.js"; +import { absoluteCallerPath } from "../filesystem/callerPath.js"; /** Expose offline historical capture inspection with the same selection contract as MCP. */ export const registerWebNetworkCaptureCommands = ( @@ -26,6 +30,23 @@ export const registerWebNetworkCaptureCommands = ( .describe("Producer capture format"), }), options: z.object({ + ...analysisEvidenceDeliveryOptions, + protobufFraming: z + .enum(["auto", "raw", "grpc", "grpc-web", "grpc-web-text"]) + .optional() + .describe("Override framing for selected retained binary bodies"), + protobufBodyPointer: z + .array(z.string()) + .optional() + .describe("Exact body pointers; repeat to select several"), + protobufSchema: z + .string() + .optional() + .describe("Absolute local compiled FileDescriptorSet path"), + protobufMessage: z + .string() + .optional() + .describe("Fully qualified message type in the descriptor set"), record: z .array(cliIntegerArgument(0)) .optional() @@ -43,13 +64,42 @@ export const registerWebNetworkCaptureCommands = ( logger, CLI_COMMANDS.inspectWebNetworkCapture, async () => { - return finishAnalysisCommand( - () => + return deliverCliAnalysisEvidence({ + run: () => service.inspect( { - capture_path: args.capturePath, + capture_path: absoluteCallerPath(args.capturePath), format: args.format, sensitive_values: options.sensitiveValue, + ...([ + options.protobufFraming, + options.protobufBodyPointer, + options.protobufSchema, + options.protobufMessage, + ].every((value) => value === undefined) + ? {} + : { + protobuf: { + framing: options.protobufFraming ?? "auto", + ...(options.protobufBodyPointer === undefined + ? {} + : { body_pointers: options.protobufBodyPointer }), + ...(options.protobufSchema === undefined && + options.protobufMessage === undefined + ? {} + : { + schema: { + descriptor_set_path: + options.protobufSchema === undefined + ? undefined + : absoluteCallerPath( + options.protobufSchema, + ), + message_type: options.protobufMessage, + }, + }), + }, + }), ...(options.record === undefined ? {} : { record_ordinals: options.record }), @@ -57,11 +107,12 @@ export const registerWebNetworkCaptureCommands = ( { signal }, ), service, - "web-network-capture", - "inspect_web_network_capture", + providerId: "web-network-capture", + operation: "inspect_web_network_capture", signal, - "Web network capture resources", - ); + detail: options.detail, + evidenceOutput: options.evidence_output, + }); }, ), ), diff --git a/src/contracts/analysisViewToolContracts.ts b/src/contracts/analysisViewToolContracts.ts index 01b3140f2d..f7db46eece 100644 --- a/src/contracts/analysisViewToolContracts.ts +++ b/src/contracts/analysisViewToolContracts.ts @@ -17,7 +17,7 @@ export const ANALYSIS_VIEW_TOOL_CONTRACTS = [ ...toolContractMetadata("inspect_analysis_view"), kind: "application", description: - 'Inspect selected views of completed binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_macho, decode_interface_builder, inspect_evm_interface, inspect_managed_artifact, inspect_managed_members, inspect_android_package, inspect_dart_aot, inspect_il2cpp_metadata or inspect_web_network_capture Evidence without repeating analysis. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. The environment-manifest view projects the same HAR requests on the primary origin (most JSON-like records) into a replayable tool surface: view {kind: "environment-manifest", template_rules: ["after-uid-literal", "opaque-token", "sibling-variance"], include_query_parameter_names: true}. Each tool has a deterministic name, a path template and parameters from the four format rules plus the selected opt-in rules, observed query parameter names, a value-free body shape for POST/PUT/PATCH, status and content-type sets and record ordinals; records are pointers into the parent, excluded groups carry a reason and three fixed unknowns are always stated. Templates, parameters and names are inferences; the view copies no header, cookie or body values and refuses parents inspected with sensitive values. A capture_browser_scenario parent that selected response bodies or header values projects the same way, one record per network transaction in request order; it relies on capture-time redaction of declared secrets instead of that refusal, and a transaction whose URL was redacted is left unrepresented. The environment-state view extracts, from the same parents, the initial state the recorded reads observed: view {kind: "environment-state", template_rules: [...]} with the rules of the manifest it accompanies. It reads response bodies recorded before the first related write and reports entity types with an identity field (proved by a path parameter or inferred from a field name), entities in collection order with per-view values where views differ, documents for non-parameterized reads that are not entities and have one initial request line (item-shaped paths stay recorded), and a render plan per read tool: item and list views reproduce their latest recordings from the state, and every other read tool is a recorded view with a reason. Types and identity are inferences; values are recorded data, so it refuses parents inspected with sensitive values, and a scenario parent must have selected response bodies; its values keep the capture redaction markers, and a write whose URL it redacted ends the state of every type. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. A completed Evidence JSON file on this host uses {kind: "evidence-file", path: ""}: REA reads and authenticates it, then projects it like retained Evidence, so a large capture is bounded by the file reader rather than by inline-input accounting. For layout, JavaScript, inventory, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Typed module references use view {kind: "javascript", collection: "modules", projection: "references", where: {path: "main.js", kind: "javascript-module"}, offset: 0, limit: 64}; where matches exact retained strings (null selects unavailable paths). Rows include canonical identity, retained source-range count and observation support states. Source coverage and selection remain explicit; exhausting matching rows does not prove runtime loading or complete analysis. For exact file/native facts and processing gaps, use view {kind: "javascript-facts", where: {paths: ["package.json"]}, property_keys: ["name", "operation", "analysis_status"]}. Rows page original observations with canonical identities and Evidence; missing properties remain omitted. Paths match recorded artifact-path/source-range locations; optional kinds filter nodes. To discover field names without values, opt into include_available_property_keys: true with property_keys: []. Canonical identity and Evidence are separate from properties; keep each node_id/observation_id pair together when composing results. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions. Occurrences accept exact logical_paths and occurrence_ids; nodes accept artifact_ids. Batch filters intersect (omitted means all available, [] means none); fields selects columns while retaining identity links. source_complete describes retained graph membership; source_coverage describes producer coverage, including unknown encrypted PCK directories. Page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. Interface Builder views use kind interface-builder and collection documents/nodes/edges/coverage; nodes and edges accept an exact document relative_path. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Managed member summaries keep identity, selection with unmatched names, coverage and per-collection counts; managed-members pages use collection types/exported_types/fields/methods/member_refs/call_edges/field_accesses/user_strings and carry the source selection and coverage. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. IL2CPP metadata views use kind il2cpp-metadata and collection sections/images/types/methods, with exact indices/names, fields and paging; types also select image_indices/namespaces/declaring_types and methods select declaring_types. Rows always keep their original index scoped to the parent artifact digest. Desktop directory views use kind desktop-application and collection files/omissions/candidates/components/next_probes, with fields and pagination; files and omissions accept exact paths or their stable IDs, and candidates accept families. Source coverage and stable row IDs remain available. Native function Evidence requires kind native: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', + 'Inspect selected views of completed binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_macho, decode_interface_builder, inspect_evm_interface, inspect_managed_artifact, inspect_managed_members, inspect_android_package, inspect_dart_aot, inspect_il2cpp_metadata or inspect_web_network_capture Evidence without repeating analysis. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. The environment-manifest view projects the same HAR requests on the primary origin (most JSON-like records) into a replayable tool surface: view {kind: "environment-manifest", template_rules: ["after-uid-literal", "opaque-token", "sibling-variance"], include_query_parameter_names: true}. Each tool has a deterministic name, a path template and parameters from the four format rules plus the selected opt-in rules, observed query parameter names, a value-free body shape for POST/PUT/PATCH, status and content-type sets and record ordinals; records are pointers into the parent, excluded groups carry a reason and three fixed unknowns are always stated. Templates, parameters and names are inferences; the view copies no header, cookie or body values and refuses parents inspected with sensitive values. A capture_browser_scenario parent that selected response bodies or header values projects the same way, one record per network transaction in request order; it relies on capture-time redaction of declared secrets instead of that refusal, and a transaction whose URL was redacted is left unrepresented. The environment-state view extracts, from the same parents, the initial state the recorded reads observed: view {kind: "environment-state", template_rules: [...]} with the rules of the manifest it accompanies. It reads response bodies recorded before the first related write and reports entity types with an identity field (proved by a path parameter or inferred from a field name), entities in collection order with per-view values where views differ, documents for non-parameterized reads that are not entities and have one initial request line (item-shaped paths stay recorded), and a render plan per read tool: item and list views reproduce their latest recordings from the state, and every other read tool is a recorded view with a reason. Types and identity are inferences; values are recorded data, so it refuses parents inspected with sensitive values, and a scenario parent must have selected response bodies; its values keep the capture redaction markers, and a write whose URL it redacted ends the state of every type. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. A completed Evidence JSON file on this host uses {kind: "evidence-file", path: ""}: REA reads and authenticates it, then projects it like retained Evidence, so a large capture is bounded by the file reader rather than by inline-input accounting. For layout, JavaScript, inventory, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Typed module references use view {kind: "javascript", collection: "modules", projection: "references", where: {path: "main.js", kind: "javascript-module"}, offset: 0, limit: 64}; where matches exact retained strings (null selects unavailable paths). Rows include canonical identity, retained source-range count and observation support states. Source coverage and selection remain explicit; exhausting matching rows does not prove runtime loading or complete analysis. For exact file/native facts and processing gaps, use view {kind: "javascript-facts", where: {paths: ["package.json"]}, property_keys: ["name", "operation", "analysis_status"]}. Rows page original observations with canonical identities and Evidence; missing properties remain omitted. Paths match recorded artifact-path/source-range locations; optional kinds filter nodes. To discover field names without values, opt into include_available_property_keys: true with property_keys: []. Canonical identity and Evidence are separate from properties; keep each node_id/observation_id pair together when composing results. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions. Occurrences accept exact logical_paths and occurrence_ids; nodes accept artifact_ids. Batch filters intersect (omitted means all available, [] means none); fields selects columns while retaining identity links. source_complete describes retained graph membership; source_coverage describes producer coverage, including unknown encrypted PCK directories. Page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. Interface Builder views use kind interface-builder and collection documents/nodes/edges/coverage; nodes and edges accept an exact document relative_path. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Managed member summaries keep identity, selection with unmatched names, coverage and per-collection counts; managed-members pages use collection types/exported_types/fields/methods/member_refs/call_edges/field_accesses/user_strings and carry the source selection and coverage. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. Protobuf views use kind protobuf, collection bodies/frames/fields, with record_ordinals, body_pointers and fields; field pages also select frame_indices, field_ids, field_numbers or schema_names. Byte identity, offset basis and body status remain present. Text and schema_value are opt-in page columns. IL2CPP metadata views use kind il2cpp-metadata and collection sections/images/types/methods, with exact indices/names, fields and paging; types also select image_indices/namespaces/declaring_types and methods select declaring_types. Rows always keep their original index scoped to the parent artifact digest. Desktop directory views use kind desktop-application and collection files/omissions/candidates/components/next_probes, with fields and pagination; files and omissions accept exact paths or their stable IDs, and candidates accept families. Source coverage and stable row IDs remain available. Native function Evidence requires kind native: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', inputSchema: inspectAnalysisViewInputSchema, outputSchema: evidenceResultOf(analysisViewResultSchema), examples: [ diff --git a/src/contracts/webNetworkCaptureToolContracts.ts b/src/contracts/webNetworkCaptureToolContracts.ts index 592dd9b54e..17370b27b4 100644 --- a/src/contracts/webNetworkCaptureToolContracts.ts +++ b/src/contracts/webNetworkCaptureToolContracts.ts @@ -16,7 +16,7 @@ export const WEB_NETWORK_CAPTURE_TOOL_CONTRACTS = [ name: "inspect_web_network_capture", ...toolContractMetadata("inspect_web_network_capture"), description: - "Request detail summary for capture identity and counts; then page retained records with inspect_analysis_view kind capture (preview or complete), avoiding repeated decoding. Default detail complete preserves existing inline delivery. Inspect retained local HAR 1.2, native mitmproxy, Fiddler SAZ or aioquic 1.5.0 JSON qlog 0.3 events without contacting recorded URLs. Returns original ordinals, pointers/byte ranges/archive member names, preserved producer fields, precise numeric representations and exact exposed binary bytes inline. Excludes known transport credentials and explicitly marked values. Decodes every record before optional ordinal selection; malformed or oversized captures return no partial success. Duplicate JSON members are invalid; prototype-named members such as __proto__ are preserved as ordinary own JSON members without prototype mutation. Qlog reports trace-scoped STREAM interval unions and endpoint loss declarations; it does not recover bodies, prove network loss or correlate PCAP. Other qlog layouts are unsupported. Uses pinned upstream parsers, owned temporary files and bounded offline processes. SAZ preserves HTTP fields, XML metadata and encoded body/message bytes without replay or HAR conversion; encrypted archives are unsupported. Native mitmproxy requires caller-supplied mitmdump 12.2.3 on Linux. Browser transaction IDs, runtime attribution and original bytes of HAR Unicode text remain unknown.", + "Explicit record_ordinals also decode retained protobuf/gRPC bodies whose declared content type is recognized. Optional protobuf selects exact body_pointers, framing, and a caller-supplied local FileDescriptorSet with message_type. Native gRPC and gRPC-web framing remain distinct. Wire fields are flat byte-range facts; ambiguous readings stay candidates, compressed payloads stay unknown. Page retained decoding with inspect_analysis_view kind protobuf, collection bodies/frames/fields, exact batch selectors and fields. Large text and schema values require explicit fields selection in those pages. Request detail summary for capture identity and counts; then page retained records with inspect_analysis_view kind capture (preview or complete), avoiding repeated decoding. Default detail complete preserves existing inline delivery. Inspect retained local HAR 1.2, native mitmproxy, Fiddler SAZ or aioquic 1.5.0 JSON qlog 0.3 events without contacting recorded URLs. Returns original ordinals, pointers/byte ranges/archive member names, preserved producer fields, precise numeric representations and exact exposed binary bytes inline. Excludes known transport credentials and explicitly marked values. Decodes every record before optional ordinal selection; malformed or oversized captures return no partial success. Duplicate JSON members are invalid; prototype-named members such as __proto__ are preserved as ordinary own JSON members without prototype mutation. Qlog reports trace-scoped STREAM interval unions and endpoint loss declarations; it does not recover bodies, prove network loss or correlate PCAP. Other qlog layouts are unsupported. Uses pinned upstream parsers, owned temporary files and bounded offline processes. SAZ preserves HTTP fields, XML metadata and encoded body/message bytes without replay or HAR conversion; encrypted archives are unsupported. Native mitmproxy requires caller-supplied mitmdump 12.2.3 on Linux. Browser transaction IDs, runtime attribution and original bytes of HAR Unicode text remain unknown.", kind: "browser-provider", inputSchema: inspectWebNetworkCaptureInputSchema.extend({ detail: analysisDetailSchema, diff --git a/src/domain/analysisView/EnvironmentStateExtraction.ts b/src/domain/analysisView/EnvironmentStateExtraction.ts index 076ef5afc3..8a25199c26 100644 --- a/src/domain/analysisView/EnvironmentStateExtraction.ts +++ b/src/domain/analysisView/EnvironmentStateExtraction.ts @@ -21,7 +21,6 @@ import { locateItem, locateList, rootOf, - rootsRelated, scalarText, segmentsOf, apiBaseLengths, @@ -52,6 +51,7 @@ import { type StateGateContext, type StatePlan, } from "./environmentStateGate.js"; +import { StateWriteCutoffs } from "./environmentStateWriteCutoffs.js"; class StateBudgetExhausted extends Error {} @@ -79,6 +79,8 @@ export class EnvironmentStateExtraction { readonly #writes: readonly StateRecordFacts[]; readonly #writeWithin: ReturnType; readonly #bases: ReadonlyMap; + #writeCutoffs: StateWriteCutoffs | undefined; + readonly #initLimits = new Map(); readonly #identity = new Map< string, { readonly field: string; readonly basis: StateType["id_basis"] } @@ -116,6 +118,15 @@ export class EnvironmentStateExtraction { /** The extracted state, or null when the representation budget is exhausted. */ run(): ExtractedState | null { try { + if (this.#writes.length > 0) + this.#writeCutoffs = new StateWriteCutoffs( + this.#writes, + this.#bases, + (left, right) => + this.#position.get(left.ordinal)! < + this.#position.get(right.ordinal)!, + (bytes) => this.#retain(bytes), + ); const candidates = this.#candidates(); for (const candidate of candidates.filter((entry) => entry.item)) this.#planItem(candidate); @@ -185,10 +196,12 @@ export class EnvironmentStateExtraction { /** The first write whose root relates to the template's bounds its init recordings. */ #initLimit(template: string): StateRecordFacts | undefined { - const root = rootOf(template, this.#bases); - return this.#writes.find((write) => - rootsRelated(root, rootOf(write.path, this.#bases)), - ); + if (this.#writeCutoffs === undefined) return undefined; + if (this.#initLimits.has(template)) return this.#initLimits.get(template); + const first = this.#writeCutoffs.first(rootOf(template, this.#bases)); + this.#retain(ENTRY_BYTES + textBytes(template.length)); + this.#initLimits.set(template, first); + return first; } /** Successful recordings that finished before the first related write began. */ @@ -546,17 +559,19 @@ export class EnvironmentStateExtraction { /** Types and entities that at least one state-backed view renders; view values only for those views. */ #result(documents: StateDocument[]): ExtractedState { - const backed = new Set( - [...this.#views.values()].flatMap((view) => - view.shape === "item" || view.shape === "list" ? [view.tool] : [], - ), - ); + const backed = new Set(); + const backedTypes = new Set(); + for (const view of this.#views.values()) + if (view.shape === "item" || view.shape === "list") { + this.#retain(ENTRY_BYTES); + backed.add(view.tool); + if (!backedTypes.has(view.type)) { + this.#retain(ENTRY_BYTES); + backedTypes.add(view.type); + } + } const types = [...this.#identity.entries()] - .filter(([type]) => - this.#plans.some( - (plan) => plan.type === type && backed.has(plan.tool.name), - ), - ) + .filter(([type]) => backedTypes.has(type)) .map(([type, { field, basis }]): StateType => ({ collection_template: type, id_field: field, diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index 9a30bde79c..04e5cc7826 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -1,3 +1,8 @@ +import { + protobufViewRequestSchema, + protobufViewItemsSchema, + protobufViewContextSchema, +} from "./protobufCaptureView.js"; import { isAbsolute } from "node:path"; import { localPathStringSchema } from "../localPath.js"; @@ -247,6 +252,7 @@ export const analysisViewRequestSchema = z.discriminatedUnion("kind", [ environmentManifestViewRequestSchema, environmentStateViewRequestSchema, captureViewRequestSchema, + protobufViewRequestSchema, qlogViewRequestSchema, pcapViewRequestSchema, dartAotViewRequestSchema, @@ -549,6 +555,14 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ il2cppMetadataSummarySchema, ]), }), + z.strictObject({ + kind: z.literal("protobuf"), + view: protobufViewRequestSchema, + items: protobufViewItemsSchema, + ...viewResultBase, + ...protobufViewContextSchema.shape, + parent_operation: z.literal("inspect_web_network_capture"), + }), z.strictObject({ kind: z.literal("capture"), view: captureViewRequestSchema, @@ -710,79 +724,10 @@ export interface AnalysisViewParent { } /** View payload before the content digest is attached. */ -export type UnsignedAnalysisView = - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit, "view_digest"> - | Omit, "view_digest"> - | Omit, "view_digest"> - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit, "view_digest"> - | Omit, "view_digest"> - | Omit< - Extract, - "view_digest" - > - | Omit, "view_digest"> - | Omit, "view_digest"> - | Omit, "view_digest"> - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - > - | Omit< - Extract, - "view_digest" - >; +type WithoutViewDigest = T extends { readonly view_digest: string } + ? Omit + : never; +export type UnsignedAnalysisView = WithoutViewDigest; /** Attach the content digest that must change when projected bytes change. */ export const sealAnalysisView = ( diff --git a/src/domain/analysisView/environmentStateRules.ts b/src/domain/analysisView/environmentStateRules.ts index d029ff9d25..4e90c733fd 100644 --- a/src/domain/analysisView/environmentStateRules.ts +++ b/src/domain/analysisView/environmentStateRules.ts @@ -266,18 +266,6 @@ export const rootOf = ( return segments.slice(0, end + 1); }; -/** Roots relate when they agree on their common length; template parameters match any segment. */ -export const rootsRelated = ( - left: readonly string[], - right: readonly string[], -): boolean => - left - .slice(0, right.length) - .every( - (segment, index) => - segment === right[index] || isParam(segment) || isParam(right[index]!), - ); - /** Value bytes as retained state, charged before a value is kept. */ export const jsonBytes = (value: JsonValue): number => { if (typeof value === "string") return textBytes(value.length); diff --git a/src/domain/analysisView/environmentStateView.test.ts b/src/domain/analysisView/environmentStateView.test.ts index cd5eb2b39f..1d1869997b 100644 --- a/src/domain/analysisView/environmentStateView.test.ts +++ b/src/domain/analysisView/environmentStateView.test.ts @@ -889,6 +889,64 @@ it("proves the identity field against every item response", () => { expect(state.entities.map((entity) => entity.id)).toEqual(["1", "2"]); }); +it.each(["/api", "/api/items/1/update"])( + "bounds a type by a write at a related root (%s)", + (path) => { + const state = stateOf( + captureOf([ + ["GET", "/api/items/1", { id: 1, title: "initial" }], + ["POST", path, {}], + ["GET", "/api/items/1", { id: 1, title: "after" }], + ]), + ); + expect(state.types).toMatchObject([{ init_before_ordinal: 1 }]); + expect(state.entities).toMatchObject([{ fields: { title: "initial" } }]); + }, +); + +it("chooses the earliest related write in timeline order, across ancestor and descendant roots", () => { + const parent = captureOf([ + ["GET", "/api/items/1", { id: 1, title: "initial" }], + ["POST", "/api/items/1/update", {}], + ["POST", "/api", {}], + ["GET", "/api/items/1", { id: 1, title: "after" }], + ]); + const data = structuredClone(parent.normalizedResult) as { + records: { reported: { startedDateTime: string } }[]; + }; + const first = data.records[1]!.reported.startedDateTime; + data.records[1]!.reported.startedDateTime = + data.records[2]!.reported.startedDateTime; + data.records[2]!.reported.startedDateTime = first; + const state = stateOf({ ...parent, normalizedResult: data }); + expect(state.types).toMatchObject([{ init_before_ordinal: 2 }]); + expect(state.entities).toMatchObject([{ fields: { title: "initial" } }]); +}); + +it("bounds a parameterized namespace by concrete writes before publishing new items", () => { + const result = projectAnalysisView( + captureOf([ + ["GET", "/api/v1/repos/alice/alpha/issues/1", { id: 1 }], + ["GET", "/api/v1/repos/bob/beta/issues/2", { id: 2 }], + ["GET", "/api/v1/repos/carol/gamma/issues/3", { id: 3 }], + ["POST", "/api/v1/repos/dana/delta/issues", { id: 4 }], + ["GET", "/api/v1/repos/dana/delta/issues/4", { id: 4 }], + ]), + { + kind: "environment-state", + template_rules: [...ALL_RULES, "sibling-pair-variance"], + }, + ); + if (!result.ok || result.value.kind !== "environment-state") + throw new Error("Expected state"); + expect(result.value.entities.map((entity) => entity.id)).toEqual([ + "1", + "2", + "3", + ]); + expect(result.value.types).toMatchObject([{ init_before_ordinal: 3 }]); +}); + it.each(["totalcount", "itemcount"])( "recognizes a compact compound item total (%s)", (name) => { diff --git a/src/domain/analysisView/environmentStateWriteCutoffs.test.ts b/src/domain/analysisView/environmentStateWriteCutoffs.test.ts new file mode 100644 index 0000000000..664a90034d --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteCutoffs.test.ts @@ -0,0 +1,86 @@ +import { expect, it } from "vitest"; +import { + isParam, + rootOf, + type StateRecordFacts, +} from "./environmentStateRules.js"; +import { StateWriteCutoffs } from "./environmentStateWriteCutoffs.js"; + +const write = (path: string, ordinal: number): StateRecordFacts => ({ + path, + ordinal, + method: "POST", + query: undefined, + status: 200, + start: ordinal, + end: ordinal, +}); + +it("matches prefix and parameter semantics against an independent linear oracle", () => { + const bases = new Map([ + ["api", 8], + ["other", 8], + ]); + const paths = [ + "/api/a", + "/api/b/c", + "/api/{owner}/d", + "/other/x", + "/api/a/{item}/e", + "", + ]; + const writes = paths.map(write); + const patterns: string[][] = [[]]; + let level: string[][] = [[]]; + for (let depth = 0; depth < 4; depth += 1) { + level = level.flatMap((prefix) => + ["api", "a", "b", "d", "{key}"].map((segment) => [...prefix, segment]), + ); + patterns.push(...level); + } + for (const selection of [ + writes, + writes.slice(0, -1), + [...writes].reverse(), + ]) { + const index = new StateWriteCutoffs( + selection, + bases, + (a, b) => a.start < b.start, + () => {}, + ); + for (const root of patterns) { + const expected = [...selection] + .sort((a, b) => a.start - b.start) + .find((record) => { + const candidate = rootOf(record.path, bases); + const shared = Math.min(root.length, candidate.length); + return Array.from({ length: shared }, (_, i) => i).every( + (i) => + root[i] === candidate[i] || + isParam(root[i]!) || + isParam(candidate[i]!), + ); + }); + expect(index.first(root), JSON.stringify(root)).toBe(expected); + } + } +}); + +it("handles deep roots iteratively within admitted storage", () => { + const root = ["api", ...Array(12_000).fill("a")]; + const record = write(`/${root.join("/")}`, 1); + let admitted = 0; + const index = new StateWriteCutoffs( + [record], + new Map([["api", root.length]]), + () => false, + (bytes) => { + admitted += bytes; + if (admitted > 16 * 1024 * 1024) + throw new Error("Test storage budget exceeded"); + }, + ); + expect(index.first(root)).toBe(record); + expect(index.first(["other"])).toBeUndefined(); +}); diff --git a/src/domain/analysisView/environmentStateWriteCutoffs.ts b/src/domain/analysisView/environmentStateWriteCutoffs.ts new file mode 100644 index 0000000000..91f148c361 --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteCutoffs.ts @@ -0,0 +1,101 @@ +import { ENTRY_BYTES, textBytes } from "./webNetworkRouteTemplates.js"; +import { + isParam, + rootOf, + type StateRecordFacts, +} from "./environmentStateRules.js"; + +interface WriteRootNode { + readonly children: Map; + wildcard: WriteRootNode | undefined; + exact: StateRecordFacts | undefined; + first: StateRecordFacts | undefined; +} + +const node = (): WriteRootNode => ({ + children: new Map(), + wildcard: undefined, + exact: undefined, + first: undefined, +}); + +/** Parsed write roots with the earliest write at each prefix and subtree. */ +export class StateWriteCutoffs { + readonly #root: WriteRootNode; + + constructor( + writes: readonly StateRecordFacts[], + bases: ReadonlyMap, + private readonly before: ( + left: StateRecordFacts, + right: StateRecordFacts, + ) => boolean, + retain: (bytes: number) => void, + ) { + // Admission covers node maps, references and the iterative lookup frontier. + retain(ENTRY_BYTES * 4); + this.#root = node(); + for (const write of writes) { + let current = this.#root; + current.first = this.#earlier(current.first, write); + for (const segment of rootOf(write.path, bases)) { + const wildcard = isParam(segment); + let child = wildcard ? current.wildcard : current.children.get(segment); + if (child === undefined) { + retain(ENTRY_BYTES * 4 + textBytes(segment.length)); + child = node(); + if (wildcard) current.wildcard = child; + else current.children.set(segment, child); + } + current = child; + current.first = this.#earlier(current.first, write); + } + current.exact = this.#earlier(current.exact, write); + } + } + + #earlier( + left: StateRecordFacts | undefined, + right: StateRecordFacts | undefined, + ): StateRecordFacts | undefined { + if (right === undefined) return left; + return left === undefined || this.before(right, left) ? right : left; + } + + /** Earliest ancestor/descendant match; parameters on either side match any one segment. */ + first(root: readonly string[]): StateRecordFacts | undefined { + const nodes = [this.#root]; + const depths = [0]; + let first: StateRecordFacts | undefined; + while (nodes.length > 0) { + const current = nodes.pop()!; + const depth = depths.pop()!; + first = this.#earlier(first, current.exact); + if ( + depth === root.length || + (depth === root.length - 1 && isParam(root[depth]!)) + ) { + first = this.#earlier(first, current.first); + continue; + } + const segment = root[depth]!; + if (isParam(segment)) { + for (const child of current.children.values()) { + nodes.push(child); + depths.push(depth + 1); + } + } else { + const child = current.children.get(segment); + if (child !== undefined) { + nodes.push(child); + depths.push(depth + 1); + } + } + if (current.wildcard !== undefined) { + nodes.push(current.wildcard); + depths.push(depth + 1); + } + } + return first; + } +} diff --git a/src/domain/analysisView/protobufCaptureView.test.ts b/src/domain/analysisView/protobufCaptureView.test.ts new file mode 100644 index 0000000000..04d613a5a5 --- /dev/null +++ b/src/domain/analysisView/protobufCaptureView.test.ts @@ -0,0 +1,223 @@ +import { expect, it } from "vitest"; +import { createEvidence } from "../evidence.js"; +import { projectAnalysisView } from "./analysisView.js"; +import { webNetworkCaptureSchema } from "../webNetworkCapture.js"; + +const capture = async () => { + const report = webNetworkCaptureSchema.parse({ + artifact: { path: "/recordings/a.har", bytes: 100, sha256: "b".repeat(64) }, + decoder: { + id: "projection-fixture", + name: "Original projection facts", + version: "1", + }, + container: { + reported: null, + numeric_literals: [], + redactions: [], + records_pointer: "/log/entries", + }, + format: "har", + total_records: 1, + runtime_attribution: "unknown", + limitations: [], + records: [ + { + ordinal: 0, + location: { kind: "json-pointer", pointer: "/log/entries/0" }, + reported: null, + binary_fields: [], + numeric_literals: [], + redactions: [], + limitations: [], + protobuf: [ + { + decoder: "rea-protobuf-wire-v1", + pointer: "/response/content/text", + content_type: "application/grpc-web+proto", + body_sha256: "c".repeat(64), + framing: "grpc-web", + offset_basis: "retained-body", + decoded_bytes: 37, + status: "complete", + examined_bytes: 37, + limitations: [], + frames: [ + { + index: 0, + kind: "data", + offset: 0, + payload_offset: 5, + bytes: 11, + compressed: false, + status: "complete", + examined_bytes: 11, + }, + { + index: 1, + kind: "trailer", + offset: 16, + payload_offset: 21, + bytes: 16, + compressed: false, + status: "complete", + examined_bytes: 16, + }, + ], + fields: [ + { + index: 0, + frame_index: 0, + parent_index: null, + field_id: "offset:5", + parent_field_id: null, + number: 1, + wire_type: "len", + offset: 5, + bytes: 7, + payload_offset: 7, + payload_bytes: 5, + text: "hello", + interpretations: ["utf8-string", "packed-varint", "bytes"], + }, + { + index: 1, + frame_index: 0, + parent_index: null, + field_id: "offset:12", + parent_field_id: null, + number: 2, + wire_type: "len", + offset: 12, + bytes: 4, + payload_offset: 14, + payload_bytes: 2, + }, + ], + }, + ], + }, + ], + }); + const evidence = createEvidence( + { ...report.artifact, sha256: "b".repeat(64), format: "file" }, + report.decoder, + { + operation: "inspect_web_network_capture", + parameters: { record_ordinals: [0] }, + result: report, + confidence: "observed", + authority: "historical-reference", + }, + ); + return { + report, + parent: { + evidenceId: evidence.evidence_id, + operation: evidence.operation, + normalizedResult: report, + subject: evidence.subject, + limitations: evidence.limitations, + }, + }; +}; + +it("keeps coordinate identity and coverage with empty fields, batch filters and offset paging", async () => { + const { parent } = await capture(); + const viewed = projectAnalysisView(parent, { + kind: "protobuf", + collection: "fields", + field_ids: ["offset:5", "offset:12"], + fields: [], + offset: 1, + limit: 1, + }); + if (!viewed.ok) throw viewed.error; + expect(viewed.value).toMatchObject({ + kind: "protobuf", + coverage: { total: 2, examined: 1, exhausted: true }, + items: [ + { + field_id: "offset:12", + record_ordinal: 0, + body_pointer: "/response/content/text", + body_status: "complete", + offset_basis: "retained-body", + }, + ], + }); + const empty = projectAnalysisView(parent, { + kind: "protobuf", + collection: "fields", + field_numbers: [], + offset: 0, + limit: 1, + }); + if (!empty.ok) throw empty.error; + expect(empty.value).toMatchObject({ + items: [], + coverage: { total: 0, exhausted: true }, + }); +}); + +it("omits large values by default, returns requested text inline and preserves parser uncertainty", async () => { + const { parent, report } = await capture(); + const brief = projectAnalysisView(parent, { + kind: "protobuf", + collection: "fields", + offset: 0, + limit: 1, + }); + if (!brief.ok) throw brief.error; + expect(brief.value).toMatchObject({ + items: [{ interpretations: ["utf8-string", "packed-varint", "bytes"] }], + }); + expect(JSON.stringify(brief.value)).not.toContain('"hello"'); + const preview = projectAnalysisView(parent, { + kind: "capture", + detail: "preview", + offset: 0, + limit: 1, + }); + if (!preview.ok) throw preview.error; + expect(preview.value).toMatchObject({ + items: [ + { + protobuf_summary: { + bodies: 1, + frames: 2, + fields: 2, + incomplete_bodies: 0, + }, + }, + ], + }); + expect(JSON.stringify(preview.value)).not.toContain('"field_id"'); + report.records[0]!.protobuf![0]!.status = "partial"; + const full = projectAnalysisView(parent, { + kind: "protobuf", + collection: "fields", + fields: ["text"], + offset: 0, + limit: 1, + }); + if (!full.ok) throw full.error; + expect(full.value).toMatchObject({ + source_status: "partial", + items: [{ text: "hello", body_status: "partial" }], + }); + const changedSubject = projectAnalysisView( + { ...parent, subject: { ...parent.subject!, local_path: "/other.har" } }, + { kind: "protobuf", collection: "bodies", offset: 0, limit: 1 }, + ); + expect(changedSubject.ok).toBe(false); + report.records[0]!.protobuf![0]!.fields[0]!.offset = 1000; + expect( + projectAnalysisView(parent, { + kind: "protobuf", + collection: "fields", + offset: 0, + limit: 1, + }).ok, + ).toBe(false); +}); diff --git a/src/domain/analysisView/protobufCaptureView.ts b/src/domain/analysisView/protobufCaptureView.ts new file mode 100644 index 0000000000..163dc48657 --- /dev/null +++ b/src/domain/analysisView/protobufCaptureView.ts @@ -0,0 +1,315 @@ +import { isImmutableJsonSnapshot } from "../immutableJson.js"; +import { z } from "zod"; +import { + protobufDecodingSchema, + protobufDecodingObjectSchema, + protobufFieldSchema, + protobufFrameSchema, + type ProtobufDecoding, + type ProtobufField, +} from "../protobufWire.js"; +import type { WebNetworkCapture } from "../webNetworkCapture.js"; +import { pageViewCoverage } from "./analysisViewCoverage.js"; + +const body = { + record_ordinal: z.number().int().nonnegative(), + body_pointer: z.string(), + body_sha256: z.string(), + body_status: protobufDecodingSchema.shape.status, + offset_basis: protobufDecodingSchema.shape.offset_basis, +}; +const bodyRow = protobufDecodingObjectSchema + .omit({ + pointer: true, + body_sha256: true, + offset_basis: true, + frames: true, + fields: true, + }) + .extend({ + ...body, + frame_count: z.number().int().nonnegative(), + field_count: z.number().int().nonnegative(), + }); +const frameRow = protobufFrameSchema.extend(body); +const fieldRow = protobufFieldSchema.extend(body); +const page = { + kind: z.literal("protobuf"), + record_ordinals: z.array(z.number().int().nonnegative()).optional(), + body_pointers: z.array(z.string()).optional(), + offset: z.number().int().nonnegative().default(0), + limit: z.number().int().positive().default(64), +}; + +/** Batch selectors intersect; omitted filters select all retained decoding facts. */ +export const protobufViewRequestSchema = z.discriminatedUnion("collection", [ + z.strictObject({ + ...page, + collection: z.literal("bodies"), + fields: z.array(bodyRow.keyof()).optional(), + }), + z.strictObject({ + ...page, + collection: z.literal("frames"), + frame_indices: z.array(z.number().int().nonnegative()).optional(), + fields: z.array(frameRow.keyof()).optional(), + }), + z.strictObject({ + ...page, + collection: z.literal("fields"), + frame_indices: z.array(z.number().int().nonnegative()).optional(), + field_ids: z.array(z.string()).optional(), + field_numbers: z.array(z.number().int().positive()).optional(), + schema_names: z.array(z.string()).optional(), + fields: z.array(fieldRow.keyof()).optional(), + }), +]); +export const protobufViewItemsSchema = z.union([ + z.array( + bodyRow.partial().required({ + record_ordinal: true, + body_pointer: true, + body_sha256: true, + body_status: true, + offset_basis: true, + }), + ), + z.array( + frameRow.partial().required({ + record_ordinal: true, + body_pointer: true, + body_sha256: true, + body_status: true, + offset_basis: true, + index: true, + offset: true, + }), + ), + z.array( + fieldRow.partial().required({ + record_ordinal: true, + body_pointer: true, + body_sha256: true, + body_status: true, + offset_basis: true, + index: true, + frame_index: true, + field_id: true, + parent_field_id: true, + offset: true, + payload_offset: true, + payload_bytes: true, + }), + ), +]); +export const protobufViewContextSchema = z.strictObject({ + source_status: z.enum(["complete", "partial", "unknown"]), + retained_decoded_bodies: z.number().int().nonnegative(), +}); +type Request = z.output; +type Body = { recordOrdinal: number; decoding: ProtobufDecoding }; +const checkedBodies = new WeakMap(); +interface FieldSelection { + readonly key: string; + readonly frames: ReadonlySet | undefined; + readonly ids: ReadonlySet | undefined; + readonly numbers: ReadonlySet | undefined; + readonly names: ReadonlySet | undefined; + readonly rows: WeakMap | undefined; +} +const fieldQueries = new WeakMap(); +const fieldSelection = ( + report: WebNetworkCapture, + bodies: readonly Body[], + view: Extract, + reusable: boolean, +): FieldSelection | undefined => { + if ( + view.frame_indices === undefined && + view.field_ids === undefined && + view.field_numbers === undefined && + view.schema_names === undefined + ) + return undefined; + const key = JSON.stringify([ + view.frame_indices, + view.field_ids, + view.field_numbers, + view.schema_names, + ]); + const cached = reusable ? fieldQueries.get(report) : undefined; + if (cached?.key === key) return cached; + // Selector strings must fit within one 8-byte reference per retained field. + // One filter per parent avoids multiplying large keys/sets by its body count. + const referenceBytes = bodies.reduce( + (sum, body) => sum + body.decoding.fields.length * 8, + 0, + ); + const cache = reusable && key.length * 2 <= referenceBytes; + const selection: FieldSelection = { + key, + frames: + view.frame_indices === undefined + ? undefined + : new Set(view.frame_indices), + ids: view.field_ids === undefined ? undefined : new Set(view.field_ids), + numbers: + view.field_numbers === undefined + ? undefined + : new Set(view.field_numbers), + names: + view.schema_names === undefined ? undefined : new Set(view.schema_names), + rows: cache ? new WeakMap() : undefined, + }; + if (cache) fieldQueries.set(report, selection); + return selection; +}; +const selectFields = ( + decoding: ProtobufDecoding, + selection: FieldSelection | undefined, +): readonly ProtobufField[] => { + if (selection === undefined) return decoding.fields; + const cached = selection.rows?.get(decoding); + if (cached !== undefined) return cached; + const { frames, ids, numbers, names } = selection; + const rows = decoding.fields.filter( + (field) => + (frames === undefined || frames.has(field.frame_index)) && + (ids === undefined || ids.has(field.field_id)) && + (numbers === undefined || numbers.has(field.number)) && + (names === undefined || + (field.schema_name !== undefined && names.has(field.schema_name))), + ); + selection.rows?.set(decoding, rows); + return rows; +}; + +/** Pages select owner-parsed facts; neither payloads nor descriptors are decoded again. */ +export const projectProtobufCapturePage = ( + report: WebNetworkCapture, + view: Request, +) => { + const reusable = isImmutableJsonSnapshot(report); + let bodies = reusable ? checkedBodies.get(report) : undefined; + if (bodies === undefined) { + bodies = report.records.flatMap((record) => + (record.protobuf ?? []).map((decoding) => ({ + recordOrdinal: record.ordinal, + decoding, + })), + ); + if (reusable) checkedBodies.set(report, bodies); + } + const selection = + view.collection === "fields" + ? fieldSelection(report, bodies, view, reusable) + : undefined; + const ordinals = + view.record_ordinals === undefined + ? undefined + : new Set(view.record_ordinals); + const pointers = + view.body_pointers === undefined ? undefined : new Set(view.body_pointers); + const frames = + "frame_indices" in view && view.frame_indices !== undefined + ? new Set(view.frame_indices) + : undefined; + let total = 0; + const items: Record[] = []; + let sourceStatus: "complete" | "partial" | "unknown" = + report.protobuf_coverage?.status === "partial" + ? "partial" + : bodies.length === 0 + ? "unknown" + : "complete"; + for (const entry of bodies) { + const { recordOrdinal, decoding } = entry; + if ( + (ordinals !== undefined && !ordinals.has(recordOrdinal)) || + (pointers !== undefined && !pointers.has(decoding.pointer)) + ) + continue; + if (decoding.status !== "complete") sourceStatus = "partial"; + const rows = + view.collection === "bodies" + ? [decoding] + : view.collection === "frames" + ? frames === undefined + ? decoding.frames + : decoding.frames.filter((frame) => frames.has(frame.index)) + : selectFields(decoding, selection); + const from = Math.max(0, view.offset - total); + const to = Math.min(rows.length, view.offset + view.limit - total); + for (let index = from; index < to; index++) { + const source = rows[index]!; + const row: Record = + view.collection === "bodies" + ? { + ...decoding, + frame_count: decoding.frames.length, + field_count: decoding.fields.length, + } + : source; + const mandatory = [ + "record_ordinal", + "body_pointer", + "body_sha256", + "body_status", + "offset_basis", + ...(view.collection === "fields" + ? [ + "index", + "frame_index", + "field_id", + "parent_field_id", + "offset", + "payload_offset", + "payload_bytes", + ] + : view.collection === "frames" + ? ["index", "offset"] + : []), + ]; + const keys = + view.fields ?? + Object.keys(row).filter( + (key) => + ![ + "frames", + "fields", + "pointer", + "text", + "schema_value", + "schema_enum_values", + ].includes(key), + ); + const chosen = new Set([...mandatory, ...keys]); + const projected: Record = { + record_ordinal: recordOrdinal, + body_pointer: decoding.pointer, + body_sha256: decoding.body_sha256, + body_status: decoding.status, + offset_basis: decoding.offset_basis, + }; + for (const key of keys) + if ( + chosen.has(key) && + Object.hasOwn(row, key) && + !["frames", "fields", "pointer"].includes(key) + ) + projected[key] = row[key]; + for (const key of mandatory) + if (Object.hasOwn(row, key)) projected[key] = row[key]; + items.push(projected); + } + total += rows.length; + } + return { + kind: "protobuf" as const, + view, + items: protobufViewItemsSchema.parse(items), + source_status: sourceStatus, + retained_decoded_bodies: bodies.length, + coverage: pageViewCoverage(view.offset, items.length, total), + }; +}; diff --git a/src/domain/analysisView/webNetworkCaptureView.ts b/src/domain/analysisView/webNetworkCaptureView.ts index f962110c31..83965f7367 100644 --- a/src/domain/analysisView/webNetworkCaptureView.ts +++ b/src/domain/analysisView/webNetworkCaptureView.ts @@ -1,3 +1,4 @@ +import { projectProtobufCapturePage } from "./protobufCaptureView.js"; import { analysisViewInputError } from "./analysisViewErrors.js"; import { createHash } from "node:crypto"; import { z } from "zod"; @@ -11,6 +12,7 @@ import { safeParseOwnedWebNetworkCapture, webNetworkCaptureArtifactSchema, webNetworkCaptureDecoderSchema, + protobufCaptureCoverageSchema, WEB_NETWORK_CAPTURE_FORMATS, type WebNetworkCaptureRecord, type WebNetworkCapture, @@ -38,21 +40,43 @@ export const captureViewRequestSchema = z.strictObject({ limit: z.number().int().positive().default(64), }); -const previewSchema = webNetworkCaptureRecordSchema.extend({ - binary_fields: z.array(webNetworkCaptureBinaryDescriptorSchema), +const protobufSummarySchema = z.strictObject({ + bodies: z.number().int().nonnegative(), + frames: z.number().int().nonnegative(), + fields: z.number().int().nonnegative(), + incomplete_bodies: z.number().int().nonnegative(), }); +const summarizeProtobuf = (records: readonly WebNetworkCaptureRecord[]) => { + const counts = { bodies: 0, frames: 0, fields: 0, incomplete_bodies: 0 }; + for (const record of records) + for (const body of record.protobuf ?? []) { + counts.bodies++; + counts.frames += body.frames.length; + counts.fields += body.fields.length; + counts.incomplete_bodies += Number(body.status !== "complete"); + } + return counts.bodies === 0 ? undefined : counts; +}; +const previewSchema = webNetworkCaptureRecordSchema + .omit({ protobuf: true }) + .extend({ + binary_fields: z.array(webNetworkCaptureBinaryDescriptorSchema), + protobuf_summary: protobufSummarySchema.exactOptional(), + }); export const captureViewItemsSchema = z.array( z.union([webNetworkCaptureRecordSchema, previewSchema]), ); /** Count/identity overview; bodies and the archive member inventory stay in the parent. */ export const captureSummarySchema = z.strictObject({ + protobuf_coverage: protobufCaptureCoverageSchema.exactOptional(), artifact: webNetworkCaptureArtifactSchema, decoder: webNetworkCaptureDecoderSchema, format: z.enum(WEB_NETWORK_CAPTURE_FORMATS), total_records: z.number().int().nonnegative(), retained_records: z.number().int().nonnegative(), runtime_attribution: z.literal("unknown"), + protobuf: protobufSummarySchema.exactOptional(), qlog: qlogSummarySchema.optional(), }); @@ -369,8 +393,13 @@ const preview = (record: WebNetworkCaptureRecord, format: string) => { ); } } + const { protobuf: _protobuf, ...sourceRecord } = record; + const protobufSummary = summarizeProtobuf([record]); return { - ...record, + ...sourceRecord, + ...(protobufSummary === undefined + ? {} + : { protobuf_summary: protobufSummary }), reported: source === null ? record.reported : reported, binary_fields: record.binary_fields.map( ({ content_base64: _content, ...field }) => field, @@ -389,6 +418,7 @@ const parseCaptureParent = ( parent: AnalysisViewParent, ): Result => { const parsed = safeParseOwnedWebNetworkCapture(parent.normalizedResult); + if (!parsed.success) return err( analysisInputErrorFromIssues( @@ -429,6 +459,7 @@ export const projectWebNetworkCaptureView = ( if ( view.kind !== "summary" && view.kind !== "capture" && + view.kind !== "protobuf" && view.kind !== "qlog" ) return err( @@ -463,6 +494,15 @@ export const projectWebNetworkCaptureView = ( : []), ], }; + if (view.kind === "protobuf") + return ok({ + ...shared, + ...projectProtobufCapturePage(report, view), + unknowns: [ + ...shared.unknowns, + "Decoding covers selected retained bodies only. Exhausting wire fields does not establish schema completeness or business meaning; compressed bodies and unsupported schema facets remain unknown.", + ], + }); if (view.kind === "summary") return ok({ kind: "summary", @@ -475,6 +515,12 @@ export const projectWebNetworkCaptureView = ( total_records: report.total_records, retained_records: report.records.length, runtime_attribution: report.runtime_attribution, + ...(report.protobuf_coverage === undefined + ? {} + : { protobuf_coverage: report.protobuf_coverage }), + ...(summarizeProtobuf(report.records) === undefined + ? {} + : { protobuf: summarizeProtobuf(report.records)! }), ...(report.format === "qlog" ? { qlog: summarizeQlogCapture(report.qlog) } : {}), diff --git a/src/domain/explicitSensitiveFailure.test.ts b/src/domain/explicitSensitiveFailure.test.ts index 603097f12b..aedb79d44e 100644 --- a/src/domain/explicitSensitiveFailure.test.ts +++ b/src/domain/explicitSensitiveFailure.test.ts @@ -14,6 +14,7 @@ import { ProviderCleanupError } from "./providerCleanupError.js"; import { ProviderAdapterError } from "./providerAdapterError.js"; import { analysisErrorWithCleanupFailure } from "./analysisErrorCleanup.js"; import { resolvedAnalysisError } from "./analysisErrorBase.js"; +import { createEvidence } from "./evidence.js"; import { redactExplicitFailure, redactExplicitPointer, @@ -55,7 +56,20 @@ it.each([ ), new ProviderAdapterError("provider", "inspect", context), ])("preserves sanitized common context for $._tag", (error) => { + const observation = createEvidence( + undefined, + { id: "fixture", name: "Owner facts", version: "1" }, + { + operation: "inspect", + parameters: {}, + result: { source: "already sanitized owner facts" }, + confidence: "observed", + authority: "historical-reference", + }, + ); + error.retainPartialObservation(observation); const sanitized = redactExplicitFailure(error, ["explicit-secret"]); + expect(sanitized.partialObservation).toBe(observation); expect(sanitized._tag).toBe(error._tag); const projected = projectAnalysisError(sanitized); expect(projected.details).toMatchObject({ diff --git a/src/domain/explicitSensitiveFailure.ts b/src/domain/explicitSensitiveFailure.ts index d14cb443f5..14d680fb2f 100644 --- a/src/domain/explicitSensitiveFailure.ts +++ b/src/domain/explicitSensitiveFailure.ts @@ -41,6 +41,10 @@ export const redactExplicitFailure = ( ): AnalysisError => { if (values.length === 0) return error; const redacted = redactFailure(error, values); + // Observation owners already apply format-aware exclusions. Keep their facts + // and identity across every diagnostic replacement, including input/capacity. + if (redacted !== error && error.partialObservation !== undefined) + redacted.retainPartialObservation(error.partialObservation); return redacted === error ? error : mirrorCleanup(error, redacted, (value) => @@ -118,12 +122,10 @@ const redactFailure = ( return new AnalysisCancelledError(error.operation, { ...context, cause: error.cause, - partialObservation: error.partialObservation, }); if (error instanceof AnalysisTimeoutError) return new AnalysisTimeoutError(error.operation, error.timeoutMs, { ...context, - partialObservation: error.partialObservation, }); if (error instanceof AnalysisAccessDeniedError) return new AnalysisAccessDeniedError( @@ -199,17 +201,11 @@ const redactFailure = ( { ...context, operation: error.operation, - ...(error.partialObservation === undefined - ? {} - : { partialObservation: error.partialObservation }), }, ); if (error instanceof ProviderAdapterError) return new ProviderAdapterError(error.providerId, error.operation, { ...context, - ...(error.partialObservation === undefined - ? {} - : { partialObservation: error.partialObservation }), ...(error.userMessage === undefined ? {} : { userMessage: text(error.userMessage) }), diff --git a/src/domain/protobufWire.ts b/src/domain/protobufWire.ts new file mode 100644 index 0000000000..67c8f86860 --- /dev/null +++ b/src/domain/protobufWire.ts @@ -0,0 +1,206 @@ +import { z } from "zod"; +import { localPathStringSchema } from "./localPath.js"; + +/** Optional decoding applies only to explicitly selected capture records. */ +export const protobufOptionsSchema = z.strictObject({ + framing: z + .enum(["auto", "raw", "grpc", "grpc-web", "grpc-web-text"]) + .default("auto"), + body_pointers: z.array(z.string()).optional(), + schema: z + .strictObject({ + descriptor_set_path: localPathStringSchema, + message_type: z.string().min(1), + }) + .optional(), +}); + +const index = z.number().int().nonnegative(); +const issueSchema = z.strictObject({ offset: index, reason: z.string() }); + +/** Indices are scoped to the capture artifact, producer ordinal and body pointer. */ +export const protobufFieldSchema = z.strictObject({ + index, + frame_index: index, + field_id: z.string(), + parent_field_id: z.string().nullable(), + parent_index: index.nullable(), + number: z.number().int().positive().max(0x1fffffff), + wire_type: z.enum(["varint", "i64", "len", "i32", "group"]), + offset: index, + bytes: index, + payload_offset: index, + payload_bytes: index, + value: z.string().exactOptional(), + text: z.string().exactOptional(), + interpretations: z + .array( + z.enum([ + "utf8-string", + "nested-message", + "packed-varint", + "packed-fixed32", + "packed-fixed64", + "bytes", + ]), + ) + .exactOptional(), + schema_name: z.string().exactOptional(), + schema_type: z.string().exactOptional(), + schema_value: z.string().exactOptional(), + schema_enum_values: z + .array(z.strictObject({ number: z.string(), names: z.array(z.string()) })) + .exactOptional(), + schema_status: z + .enum([ + "decoded", + "unknown-field", + "wire-type-mismatch", + "invalid-message", + "unsupported", + ]) + .exactOptional(), + schema_error: issueSchema.exactOptional(), +}); + +export const protobufFrameSchema = z.strictObject({ + index, + kind: z.enum(["data", "trailer"]), + offset: index, + payload_offset: index, + bytes: index, + compressed: z.boolean(), + status: z.enum(["complete", "partial", "unknown"]), + examined_bytes: index, + error: issueSchema.exactOptional(), + trailers: z + .array(z.strictObject({ name: z.string(), value: z.string() })) + .exactOptional(), +}); + +/** Flat facts avoid recursive advertised schemas and preserve same-number occurrences. */ +export const protobufDecodingObjectSchema = z.strictObject({ + pointer: z.string(), + content_type: z.string().nullable(), + decoder: z.literal("rea-protobuf-wire-v1"), + body_sha256: z.string().regex(/^[0-9a-f]{64}$/), + framing: z.enum(["raw", "grpc", "grpc-web", "grpc-web-text"]), + offset_basis: z.enum(["retained-body", "base64-decoded-body"]), + decoded_bytes: index, + status: z.enum(["complete", "partial", "unknown"]), + examined_bytes: index, + frames: z.array(protobufFrameSchema), + fields: z.array(protobufFieldSchema), + error: issueSchema.exactOptional(), + limitations: z.array(z.string()), + schema: z + .strictObject({ + decoder: z.literal("protobuf-es-2.16.0"), + descriptor_set_path: z.string(), + descriptor_set_sha256: z.string().regex(/^[0-9a-f]{64}$/), + descriptor_set_bytes: index, + message_type: z.string(), + }) + .exactOptional(), +}); +/** Validate coordinate relationships at the owner, before any consumer projects rows. */ +export const protobufDecodingSchema = protobufDecodingObjectSchema.superRefine( + (body, context) => { + const issue = (path: (string | number)[], message: string) => + context.addIssue({ code: "custom", path, message }); + if (body.examined_bytes > body.decoded_bytes) + issue(["examined_bytes"], "Examined bytes exceed the decoded body"); + if ( + body.status === "complete" && + (body.error !== undefined || + body.frames.some((frame) => frame.status !== "complete")) + ) + issue(["status"], "Complete body contradicts frame coverage"); + let expectedOffset = 0; + for (const [index, frame] of body.frames.entries()) { + if ( + frame.offset !== expectedOffset || + frame.payload_offset !== frame.offset + (body.framing === "raw" ? 0 : 5) + ) + issue( + ["frames", index, "offset"], + "Frames must preserve contiguous protocol ranges", + ); + expectedOffset = frame.payload_offset + frame.bytes; + if ( + frame.index !== index || + frame.payload_offset < frame.offset || + frame.payload_offset + frame.bytes > body.decoded_bytes || + frame.examined_bytes > frame.bytes + ) + issue( + ["frames", index], + "Frame identity or range contradicts the body", + ); + if ( + frame.status === "complete" && + (frame.error !== undefined || + frame.examined_bytes !== frame.bytes || + frame.compressed) + ) + issue( + ["frames", index, "status"], + "Complete frame contradicts decoding coverage", + ); + if (body.framing === "grpc" && frame.kind === "trailer") + issue( + ["frames", index, "kind"], + "Native gRPC has no in-body trailer frame", + ); + } + if ( + body.status === "complete" && + (body.examined_bytes !== body.decoded_bytes || + expectedOffset !== body.decoded_bytes) + ) + issue(["status"], "Complete decoding must cover the whole decoded body"); + const ids = new Set(); + for (const [index, field] of body.fields.entries()) { + const frame = body.frames[field.frame_index]; + if ( + field.index !== index || + field.field_id !== `offset:${field.offset}` || + ids.has(field.field_id) + ) + issue(["fields", index], "Field occurrence identity is inconsistent"); + ids.add(field.field_id); + if ( + frame === undefined || + frame.kind !== "data" || + field.offset < frame.payload_offset || + field.offset + field.bytes > frame.payload_offset + frame.bytes || + field.payload_offset < field.offset || + field.payload_offset + field.payload_bytes > field.offset + field.bytes + ) + issue(["fields", index], "Field range contradicts its frame"); + const parent = + field.parent_index === null + ? undefined + : body.fields[field.parent_index]; + if ( + field.parent_index === null + ? field.parent_field_id !== null + : parent === undefined || + parent.index >= index || + parent.field_id !== field.parent_field_id || + parent.frame_index !== field.frame_index || + field.offset < parent.payload_offset || + field.offset + field.bytes > + parent.payload_offset + parent.payload_bytes + ) + issue( + ["fields", index, "parent_field_id"], + "Field parent contradicts its source range", + ); + } + }, +); + +export type ProtobufDecoding = z.output; +export type ProtobufField = z.output; +export type ProtobufFrame = z.output; diff --git a/src/domain/webNetworkCapture.ts b/src/domain/webNetworkCapture.ts index 1f40d3d1b7..99d652bbdc 100644 --- a/src/domain/webNetworkCapture.ts +++ b/src/domain/webNetworkCapture.ts @@ -1,6 +1,10 @@ import { z } from "zod"; import { jsonValueSchema } from "./jsonValue.js"; import { localPathStringSchema } from "./localPath.js"; +import { + protobufDecodingSchema, + protobufOptionsSchema, +} from "./protobufWire.js"; import { qlogCaptureFactsSchema } from "./qlogCapture.js"; import { canBorrowParsedJsonSnapshot } from "./immutableJson.js"; @@ -32,6 +36,7 @@ export const inspectWebNetworkCaptureInputSchema = z.strictObject({ .describe( "Zero-based producer record ordinals, in caller-selected order. Omit to inspect every record.", ), + protobuf: protobufOptionsSchema.optional(), sensitive_values: z .array(z.string().min(1)) .default([]) @@ -128,6 +133,12 @@ export const webNetworkCaptureRecordSchema = z.strictObject({ numeric_literals: z.array(numberSchema), redactions: z.array(redactionSchema), limitations: z.array(z.string()), + protobuf: z + .array(protobufDecodingSchema) + .exactOptional() + .describe( + "Flat decoding facts for explicitly selected records. Field indices are local to a body; offset_basis states whether ranges refer to retained or base64-decoded bytes. Interpretations without a caller-supplied schema are candidates.", + ), }); /** The selected offline parser profile, independent of authenticated producer identity. */ @@ -138,7 +149,26 @@ export const webNetworkCaptureDecoderSchema = z.strictObject({ }); /** Source coverage is shared by every offline decoder. */ +export const protobufCaptureCoverageSchema = z + .strictObject({ + status: z.enum(["complete", "partial"]), + selected_records: z.number().int().nonnegative(), + inspected_records: z.number().int().nonnegative(), + }) + .superRefine((coverage, context) => { + if ( + coverage.inspected_records > coverage.selected_records || + (coverage.status === "complete" && + coverage.inspected_records !== coverage.selected_records) + ) + context.addIssue({ + code: "custom", + path: ["status"], + message: "Protobuf coverage contradicts its inspected selection.", + }); + }); const payloadShape = { + protobuf_coverage: protobufCaptureCoverageSchema.exactOptional(), decoder: webNetworkCaptureDecoderSchema, container: z.strictObject({ reported: jsonValueSchema, diff --git a/src/generatedPackageMetadata.ts b/src/generatedPackageMetadata.ts index 1939af764b..05e68adc43 100644 --- a/src/generatedPackageMetadata.ts +++ b/src/generatedPackageMetadata.ts @@ -5,5 +5,5 @@ export const PACKAGE_METADATA = { serverSdkVersion: "2.3.1", clientSdkVersion: "2.3.1", coreSdkVersion: "2.3.1", - skillVersion: "42", + skillVersion: "43", } as const; diff --git a/src/ghidra/GhidraFunctionValues.test.ts b/src/ghidra/GhidraFunctionValues.test.ts index 71c8ddb504..6ee0d39fcd 100644 --- a/src/ghidra/GhidraFunctionValues.test.ts +++ b/src/ghidra/GhidraFunctionValues.test.ts @@ -193,7 +193,7 @@ describe("Ghidra relationship metadata validation", () => { it("preserves file-offset resolution in Evidence locations and rejects inexact offsets", () => { const selection = { file_offset: 0x210, - mapped_address: "0x140001010", + mapped_address: "0x140001014", procedure_address: "0x140001010", }; const wire = { value: "return 42;", limitations: [], selection }; @@ -204,9 +204,10 @@ describe("Ghidra relationship metadata validation", () => { if (!parsed.ok) throw parsed.error; expect(parsed.value.locations).toEqual([ { kind: "file-offset", offset: 0x210 }, - { kind: "address", address: "0x140001010" }, + { kind: "address", address: "0x140001014" }, { kind: "address", address: "0x140001010" }, ]); + expect(parsed.value.rawResult).toEqual(wire); expect( parseGhidraFunctionResult("procedure_pseudo_code", { ...wire, diff --git a/src/ghidra/GhidraFunctionValues.ts b/src/ghidra/GhidraFunctionValues.ts index c8890aae8b..8de099c083 100644 --- a/src/ghidra/GhidraFunctionValues.ts +++ b/src/ghidra/GhidraFunctionValues.ts @@ -437,6 +437,7 @@ const functionObservationEnvelope = z.strictObject({ export interface ParsedGhidraFunctionResult { readonly value: JsonValue; + readonly rawResult?: JsonValue; readonly limitations: readonly string[]; readonly locations?: readonly EvidenceLocation[]; readonly relationshipCoverage?: RelationshipCoverage; @@ -488,6 +489,8 @@ export const parseGhidraFunctionResult = ( ...(observation.selection === undefined ? {} : { + // Generic locations retain coordinates but not their mapping roles. + rawResult: value, locations: [ { kind: "file-offset" as const, diff --git a/src/ghidra/GhidraInventoryValues.test.ts b/src/ghidra/GhidraInventoryValues.test.ts index b66fed3ca5..5f70c88706 100644 --- a/src/ghidra/GhidraInventoryValues.test.ts +++ b/src/ghidra/GhidraInventoryValues.test.ts @@ -53,7 +53,41 @@ describe("provider-owned overview boundary", () => { parseGhidraInventoryResult("read_binary_inventory_summary", overview), ).toEqual({ ok: true, - value: { ...overview, segment_count: 0 }, + value: { value: { ...overview, segment_count: 0 }, rawResult: overview }, + }); + }); + + it("keeps the independent segment permissions and provenance behind an overview", () => { + const reported = { + ...overview, + segments: [ + { + name: ".text", + start: "0x1000", + end: "0x100f", + readable: true, + writable: false, + executable: true, + permissions: { available: true, source: "ghidra-memory-block" }, + provenance: "ghidra-memory-block", + address_space: "ram", + image_base: "0x1000", + initialized: true, + overlay: false, + sections: [], + }, + ], + }; + const parsed = parseGhidraInventoryResult( + "read_binary_inventory_summary", + reported, + ); + if (!parsed.ok) throw parsed.error; + expect(parsed.value.rawResult).toEqual(reported); + expect(parsed.value.value).toMatchObject({ + procedure_count: 0, + string_count: 0, + segments: [{ name: ".text", start: "0x1000", end: "0x100f" }], }); }); diff --git a/src/ghidra/GhidraInventoryValues.ts b/src/ghidra/GhidraInventoryValues.ts index 1d8fff19ee..36ddeb5159 100644 --- a/src/ghidra/GhidraInventoryValues.ts +++ b/src/ghidra/GhidraInventoryValues.ts @@ -357,28 +357,47 @@ const resultSchemas = { set_addresses_names: z.record(z.string(), z.boolean()), } satisfies Readonly>; -/** Validate exact Java-bridge output before creating Evidence. */ +/** The format owner retains raw output only when projection omits reported facts. */ +export interface ParsedGhidraInventoryResult { + readonly value: Value; + readonly rawResult?: JsonValue; +} + +/** Validate exact Java-bridge output and own its Evidence representation. */ export function parseGhidraInventoryResult( operation: "inspect_native_load_image", value: JsonValue, -): Result; +): Result< + ParsedGhidraInventoryResult, + AnalysisOutputError +>; export function parseGhidraInventoryResult( operation: GhidraInventoryOperation, value: JsonValue, -): Result; +): Result; export function parseGhidraInventoryResult( operation: GhidraInventoryOperation, value: JsonValue, -): Result { +): Result< + ParsedGhidraInventoryResult, + AnalysisOutputError +> { if (operation === "inspect_native_load_image") { const parsed = resultSchemas.inspect_native_load_image.safeParse(value); return parsed.success - ? ok(parsed.data) + ? ok({ value: parsed.data, rawResult: value }) : err(inventoryOutputFailure(operation, parsed.error)); } const parsed = resultSchemas[operation].safeParse(value); return parsed.success - ? ok(jsonValueSchema.parse(parsed.data)) + ? ok({ + value: jsonValueSchema.parse(parsed.data), + // The overview omits detailed segment permissions, sections and provenance. + // Load-image observations above undergo separate local attestation. + ...(operation === "read_binary_inventory_summary" + ? { rawResult: value } + : {}), + }) : err(inventoryOutputFailure(operation, parsed.error)); } diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index 49aee30f3b..38b7644fc9 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -1142,7 +1142,7 @@ describe("Ghidra client projection", () => { }, analysisProfile: resolved.value.profile, result: [{ address: "0x1000", value: "fixture_main" }], - rawResult: [{ procedure: { external: false, thunk: false } }], + rawResult: null, }, }); expect(startCount).toBe(0); @@ -1181,10 +1181,7 @@ describe("Ghidra result projection", () => { if (!execution.ok) throw execution.error; expect(execution.value.result).toBe("void main() {}"); expect(execution.value.limitations).toContain(limitation); - expect(execution.value.rawResult).toEqual({ - value: "void main() {}", - limitations: [limitation], - }); + expect(execution.value.rawResult).toBeNull(); await client.close(); }); @@ -1672,6 +1669,7 @@ it("returns read-only NativeAOT metadata inline and drills down by derived Metho expect(detail).toMatchObject({ ok: true, value: { + rawResult: unavailableNativeAotDatabaseType("0x140002280"), result: { status: "unavailable", source: "analysis-database", @@ -2933,6 +2931,12 @@ describe("Ghidra deadline guidance", () => { expect(projected.remediation.action).toContain( "REA_GHIDRA_DISABLED_ANALYZERS", ); + expect(projected.remediation.action).toContain("restart REA"); + expect(projected.remediation.action).toContain("reconnect the MCP client"); + expect(projected.remediation.action).toContain("export useful Evidence"); + expect(projected.remediation.action).toContain( + "Reopening alone keeps the current server settings", + ); expect(projected.remediation.action).not.toContain("rea doctor"); }); diff --git a/src/ghidra/GhidraProviderClient.ts b/src/ghidra/GhidraProviderClient.ts index f6e6b8f16c..4bdf2912c4 100644 --- a/src/ghidra/GhidraProviderClient.ts +++ b/src/ghidra/GhidraProviderClient.ts @@ -120,6 +120,13 @@ export type GhidraProviderClientFactory = ( type GhidraProviderClientInstance = ReturnType; +interface GhidraExecutionMetadata { + readonly rawResult?: JsonValue; + readonly limitations?: readonly string[]; + readonly locations?: readonly EvidenceLocation[]; + readonly relationshipCoverage?: RelationshipCoverage; +} + /** Build one AnalysisClient for an admitted Ghidra target and profile. */ export const createGhidraProviderClient = (input: { readonly config: AppConfig; @@ -499,19 +506,14 @@ export const createGhidraProviderClient = (input: { const projectExecution = ( operation: AnalysisOperation, normalized: JsonValue, - rawResult: JsonValue, - observationLimitations: readonly string[] = [], - locations: readonly EvidenceLocation[] = [], - relationshipCoverage?: RelationshipCoverage, + observation: GhidraExecutionMetadata = {}, ) => { return createAnalysisExecution(normalized, committedProfile.provider, { - rawResult, - locations, + ...observation, analysisProfile: committedProfile, - ...(relationshipCoverage === undefined ? {} : { relationshipCoverage }), limitations: [ ...limitationsFor(operation), - ...observationLimitations, + ...(observation.limitations ?? []), ...sessionLimitations(), ], }); @@ -531,12 +533,14 @@ export const createGhidraProviderClient = (input: { ); const parsed = parseGhidraInventoryResult(operation, called.value); if (!parsed.ok) return parsed; - return ok(projectExecution(operation, parsed.value, called.value)); + const { value, ...observation } = parsed.value; + return ok(projectExecution(operation, value, observation)); }; const loadImageExecution = async (raw: JsonValue, signal?: AbortSignal) => { const operation = "inspect_native_load_image"; const measured = parseGhidraInventoryResult(operation, raw); if (!measured.ok) return measured; + const { value: observed, ...observation } = measured.value; let derived: NativeAotPeResult | undefined; if (target.format === "pe") { if (nativeAotOwner === undefined) @@ -549,7 +553,7 @@ export const createGhidraProviderClient = (input: { }), ); const result = await nativeAotOwner.derive( - measured.value.image_base, + observed.image_base, operation, signal, raw, @@ -560,7 +564,7 @@ export const createGhidraProviderClient = (input: { const attested = await attestGhidraNativeLoadImage({ target, operation, - measured: measured.value, + measured: observed, client, mapSessionError: (failure) => projectSessionError( @@ -577,7 +581,7 @@ export const createGhidraProviderClient = (input: { projectExecution( operation, jsonValueSchema.parse(attested.value), - raw, + observation, ), ) : attested; @@ -662,22 +666,11 @@ export const createGhidraProviderClient = (input: { ); if (operation === "inspect_native_load_image") return loadImageExecution(called.value, options?.signal); - let normalized: JsonValue; - let observationLimitations: readonly string[] = []; - let observationLocations: readonly EvidenceLocation[] = []; - let relationshipCoverage: RelationshipCoverage | undefined; - if (isGhidraFunctionOperation(operation)) { - const result = parseGhidraFunctionResult(operation, called.value); - if (!result.ok) return result; - normalized = result.value.value; - observationLimitations = result.value.limitations; - observationLocations = result.value.locations ?? []; - relationshipCoverage = result.value.relationshipCoverage; - } else { - const result = parseGhidraInventoryResult(operation, called.value); - if (!result.ok) return result; - normalized = result.value; - } + const result = isGhidraFunctionOperation(operation) + ? parseGhidraFunctionResult(operation, called.value) + : parseGhidraInventoryResult(operation, called.value); + if (!result.ok) return result; + let { value: normalized, ...observation } = result.value; if ( operation === "inspect_native_data_type" && normalized !== null && @@ -712,23 +705,17 @@ export const createGhidraProviderClient = (input: { if (!derived.ok) return derived; } const metadata = nativeAotOwner?.readTypeDetail(address); - if (metadata !== undefined) + if (metadata !== undefined) { + // Preserve the database reply independently of the derived overlay. + observation = { ...observation, rawResult: called.value }; normalized = jsonValueSchema.parse({ ...normalized, metadata_recovery: metadata, }); + } } } - return ok( - projectExecution( - operation, - normalized, - called.value, - observationLimitations, - observationLocations, - relationshipCoverage, - ), - ); + return ok(projectExecution(operation, normalized, observation)); }, executeIfReady: async (operation, parameters, options) => { if (operation !== "list_documents" && operation !== "list_procedures") @@ -1109,8 +1096,8 @@ const projectObservedJvmMemoryFailure = ( /** Name the settings that change Ghidra's import-and-analysis startup deadline. */ const ghidraStartupTimeoutGuidance = (timeoutMs: number) => ({ - userMessage: `Ghidra did not finish importing and analyzing the target within its ${String(timeoutMs)} ms startup deadline (REA_GHIDRA_STARTUP_TIMEOUT_MS).`, - remediationAction: `Large binaries can need longer: set REA_GHIDRA_STARTUP_TIMEOUT_MS above ${String(timeoutMs)} in the environment that starts REA, or skip expensive analyzers with REA_GHIDRA_DISABLED_ANALYZERS, then open the target again. An MCP client's own tool-call timeout is separate and may also need raising.`, + userMessage: `Ghidra did not finish importing and analyzing the target within its configured ${String(timeoutMs)} ms startup deadline (REA_GHIDRA_STARTUP_TIMEOUT_MS).`, + remediationAction: `Large binaries can need longer: set REA_GHIDRA_STARTUP_TIMEOUT_MS above ${String(timeoutMs)} in REA's launch environment; REA_GHIDRA_DISABLED_ANALYZERS can skip selected expensive analyzers with reduced coverage. Before restarting, export useful Evidence and close the binary; resolve any cleanup failure. Then restart REA, reconnect the MCP client if used, and reopen the target. Reopening alone keeps the current server settings and repeats import from scratch. The MCP client's tool-call deadline must also allow startup and the query.`, }); /** A provider already observed as running can be restarted by reopening. */ diff --git a/tests/acceptance/tools/protobufCapture.test.ts b/tests/acceptance/tools/protobufCapture.test.ts new file mode 100644 index 0000000000..3720c57104 --- /dev/null +++ b/tests/acceptance/tools/protobufCapture.test.ts @@ -0,0 +1,530 @@ +import { z } from "zod"; +import { readFile, unlink, writeFile, mkdir, truncate } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { + protobufCaptureDescriptor, + protobufHar, + protobufFrame, + protobufCaptureBody, +} from "../../fixtures/protobufCapture.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { + webNetworkCaptureSchema, + inspectWebNetworkCaptureInputSchema, +} from "../../../src/domain/webNetworkCapture.js"; +import { readEvidenceBundle } from "../../../src/application/EvidenceBundleFiles.js"; +import { analysisViewResultSchema } from "../../../src/domain/analysisView/analysisView.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; + +type CaptureCall = Awaited>["call"]; +type CaptureRequest = z.input; + +const retainedCaptureDetailsSchema = z.object({ + partial_observation: z.strictObject({ + kind: z.literal("retained-evidence"), + evidence_id: z.string(), + }), +}); + +async function verifyCaptureExclusions( + call: CaptureCall, + args: CaptureRequest, + root: string, +): Promise { + const redacted = await call("inspect_web_network_capture", { + ...args, + sensitive_values: ["hello"], + }); + expect(redacted.isError, JSON.stringify(redacted)).not.toBe(true); + const excluded = webNetworkCaptureSchema.parse( + parseEvidence(redacted.structuredContent).normalized_result, + ); + expect(excluded.records[0]?.protobuf).toBeUndefined(); + expect(excluded.protobuf_coverage).toMatchObject({ + status: "partial", + selected_records: 1, + inspected_records: 1, + }); + expect(JSON.stringify(redacted)).not.toContain("hello"); + const textPath = join(root, "text.har"); + await writeFile( + textPath, + protobufHar( + Buffer.from(protobufCaptureBody().toString("base64")), + "application/grpc-web-text", + ), + ); + const textExcluded = await call("inspect_web_network_capture", { + capture_path: textPath, + format: "har", + record_ordinals: [0], + sensitive_values: ["hello"], + }); + expect(textExcluded.isError, JSON.stringify(textExcluded)).not.toBe(true); + const excludedText = webNetworkCaptureSchema.parse( + parseEvidence(textExcluded.structuredContent).normalized_result, + ); + expect(excludedText.records[0]?.binary_fields[0]).toMatchObject({ + state: "redacted", + content_base64: null, + sha256: null, + }); + expect(excludedText.records[0]?.reported).toMatchObject({ + response: { content: { text: null } }, + }); + expect(excludedText.records[0]?.protobuf).toBeUndefined(); + const credentialPath = join(root, "trailer.har"); + await writeFile( + credentialPath, + protobufHar( + Buffer.concat([ + protobufFrame(Buffer.from("0a0568656c6c6f", "hex")), + protobufFrame( + Buffer.from( + "grpc-status: 0\r\nauthorization: Bearer fixture-only-credential\r\n", + ), + 128, + ), + ]), + ), + ); + const credential = await call("inspect_web_network_capture", { + capture_path: credentialPath, + format: "har", + record_ordinals: [0], + }); + expect(credential.isError, JSON.stringify(credential)).not.toBe(true); + const excludedCredential = webNetworkCaptureSchema.parse( + parseEvidence(credential.structuredContent).normalized_result, + ); + expect(excludedCredential.records[0]?.binary_fields[0]).toMatchObject({ + state: "redacted", + content_base64: null, + }); + expect(excludedCredential.records[0]?.redactions).toContainEqual({ + pointer: "/response/content/text", + reason: "transport-credential", + }); + expect(excludedCredential.records[0]?.protobuf).toBeUndefined(); + expect(JSON.stringify(credential)).not.toContain("fixture-only-credential"); +} + +async function verifyPartialCaptureSources( + call: CaptureCall, + args: CaptureRequest, + root: string, + descriptor: string, +): Promise { + const missing = await call("inspect_web_network_capture", { + ...args, + sensitive_values: ["unrelated fixture exclusion"], + protobuf: { + schema: { + descriptor_set_path: descriptor, + message_type: "capture.Missing", + }, + }, + }); + const missingError = parseMcpToolError(missing).error; + expect(missingError.code).toBe("invalid_request"); + const retainedSource = retainedCaptureDetailsSchema.parse( + missingError.details, + ).partial_observation; + const retainedSummary = await call("inspect_analysis_view", { + source: retainedSource, + view: { kind: "summary" }, + }); + expect(retainedSummary.isError, JSON.stringify(retainedSummary)).not.toBe( + true, + ); + expect( + analysisViewResultSchema.parse( + parseEvidence(retainedSummary.structuredContent).normalized_result, + ), + ).toMatchObject({ + summary: { + protobuf_coverage: { status: "partial", inspected_records: 0 }, + }, + }); + const missingView = await call("inspect_analysis_view", { + source: retainedSource, + view: { kind: "protobuf", collection: "fields", offset: 0, limit: 1 }, + }); + expect(missingView.isError, JSON.stringify(missingView)).not.toBe(true); + expect( + analysisViewResultSchema.parse( + parseEvidence(missingView.structuredContent).normalized_result, + ), + ).toMatchObject({ source_status: "partial", items: [] }); + const capacityPath = join(root, "capacity.har"); + const repeated = Buffer.alloc(200000); + for (let index = 0; index < repeated.length; index += 2) { + repeated[index] = 8; + repeated[index + 1] = 1; + } + await writeFile(capacityPath, protobufHar(protobufFrame(repeated))); + const capacity = await call("inspect_web_network_capture", { + capture_path: capacityPath, + format: "har", + record_ordinals: [0], + sensitive_values: ["unrelated fixture exclusion"], + }); + const capacityError = parseMcpToolError(capacity).error; + expect(capacityError.code).toBe("resource_constraint"); + expect(capacityError.retryable).toBe(false); + const capacitySource = retainedCaptureDetailsSchema.parse( + capacityError.details, + ).partial_observation; + const capacitySummary = await call("inspect_analysis_view", { + source: capacitySource, + view: { kind: "summary" }, + }); + expect(capacitySummary.isError, JSON.stringify(capacitySummary)).not.toBe( + true, + ); + expect( + analysisViewResultSchema.parse( + parseEvidence(capacitySummary.structuredContent).normalized_result, + ), + ).toMatchObject({ + summary: { + protobuf_coverage: { status: "partial", inspected_records: 0 }, + }, + }); + expect(JSON.stringify(capacity)).not.toContain("content_base64"); +} + +async function verifyCredentialPreflight( + call: CaptureCall, + root: string, + descriptor: string, +): Promise { + const repeated = Buffer.alloc(200000); + for (let index = 0; index < repeated.length; index += 2) { + repeated[index] = 8; + repeated[index + 1] = 1; + } + const large = protobufFrame(repeated); + const credential = protobufFrame( + Buffer.from( + "grpc-status: 0\r\nauthorization: fixture-only-trailer-credential\r\n", + ), + 128, + ); + const small = protobufFrame(Buffer.from([8, 1])); + for (const [name, body, mimeType, framing] of [ + [ + "large", + Buffer.concat([large, credential]), + "application/grpc-web+proto", + "auto", + ], + [ + "malformed", + Buffer.concat([ + small, + protobufFrame( + Buffer.from( + "malformed\r\nauthorization: fixture-only-trailer-credential\r\n", + ), + 128, + ), + ]), + "application/grpc-web+proto", + "auto", + ], + [ + "text-override", + Buffer.from(Buffer.concat([small, credential]).toString("base64")), + "application/grpc-web-text", + "raw", + ], + ] as const) { + const path = join(root, `credential-${name}.har`); + await writeFile(path, protobufHar(body, mimeType)); + const response = await call("inspect_web_network_capture", { + capture_path: path, + format: "har", + record_ordinals: [0], + protobuf: { framing }, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const capture = webNetworkCaptureSchema.parse( + parseEvidence(response.structuredContent).normalized_result, + ); + expect(capture.records[0]?.binary_fields[0]).toMatchObject({ + state: "redacted", + content_base64: null, + sha256: null, + }); + expect(capture.records[0]?.protobuf).toBeUndefined(); + expect(capture.protobuf_coverage?.status).toBe("partial"); + } + const path = join(root, "credential-later-record.har"); + await writeFile( + path, + protobufHar(large, "application/grpc-web+proto", [ + Buffer.concat([small, credential]), + ]), + ); + for (const schema of [ + undefined, + { descriptor_set_path: descriptor, message_type: "capture.Missing" }, + ]) { + const response = await call("inspect_web_network_capture", { + capture_path: path, + format: "har", + record_ordinals: [0, 1], + sensitive_values: ["unrelated fixture exclusion"], + ...(schema === undefined ? {} : { protobuf: { schema } }), + }); + const error = parseMcpToolError(response).error; + expect(error.code).toBe( + schema === undefined ? "resource_constraint" : "invalid_request", + ); + const source = retainedCaptureDetailsSchema.parse( + error.details, + ).partial_observation; + const retained = await call("inspect_analysis_view", { + source, + view: { kind: "capture", detail: "complete", offset: 0, limit: 2 }, + }); + expect(retained.isError, JSON.stringify(retained)).not.toBe(true); + const view = analysisViewResultSchema.parse( + parseEvidence(retained.structuredContent).normalized_result, + ); + expect(view).toMatchObject({ + items: [ + expect.anything(), + { + binary_fields: [ + { state: "redacted", content_base64: null, sha256: null }, + ], + }, + ], + }); + } +} + +async function verifyDescriptorReadFailures( + call: CaptureCall, + args: CaptureRequest, + root: string, +): Promise { + const directory = join(root, "descriptor-directory"); + const oversized = join(root, "oversized.pb"); + await mkdir(directory); + await writeFile(oversized, ""); + await truncate(oversized, 32 * 1024 * 1024 + 1); + for (const [path, reason] of [ + [join(root, "missing.pb"), "invalid_value"], + [directory, "invalid_format"], + [oversized, "out_of_range"], + ] as const) { + const response = await call("inspect_web_network_capture", { + ...args, + sensitive_values: ["unrelated fixture exclusion"], + protobuf: { + schema: { descriptor_set_path: path, message_type: "capture.Message" }, + }, + }); + const error = parseMcpToolError(response).error; + expect(error.code).toBe("invalid_request"); + expect(error.details).toMatchObject({ + issues: [{ path: ["protobuf", "schema", "descriptor_set_path"], reason }], + }); + expect( + retainedCaptureDetailsSchema.parse(error.details).partial_observation + .kind, + ).toBe("retained-evidence"); + } +} + +cliTest( + "decodes selected capture bytes with a supplied descriptor and pages exported facts after both sources disappear", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-protobuf-cli-"); + const capture = join(root, "capture.har"); + const descriptor = join(root, "capture.pb"); + const bundle = join(root, "evidence.json"); + await writeFile(capture, protobufHar()); + await writeFile(descriptor, protobufCaptureDescriptor()); + const original = await cli.run({ + arguments: ["inspect-web-network-capture", capture, "har", "--json"], + }); + expect(original.exitCode, original.stderr).toBe(0); + expect( + webNetworkCaptureSchema.parse( + parseEvidence(original.json).normalized_result, + ).records[0], + ).not.toHaveProperty("protobuf"); + const decoded = await cli.run({ + arguments: [ + "inspect-web-network-capture", + capture, + "har", + "--record", + "0", + "--protobuf-schema", + descriptor, + "--protobuf-message", + "capture.Message", + "--detail", + "summary", + "--evidence_output", + bundle, + "--json", + ], + }); + expect(decoded.exitCode, decoded.stdout + decoded.stderr).toBe(0); + const loaded = await readEvidenceBundle(bundle); + if (!loaded.ok) throw loaded.error; + const parent = loaded.value.records.find( + (record) => record.operation === "inspect_web_network_capture", + ); + if (parent === undefined) + throw new Error("Complete capture Evidence must be retained"); + const report = webNetworkCaptureSchema.parse(parent.normalized_result); + expect(report.records[0]?.protobuf?.[0]).toMatchObject({ + status: "complete", + schema: { message_type: "capture.Message" }, + }); + expect(report.records[0]?.protobuf?.[0]?.fields[0]).toMatchObject({ + schema_name: "sentence", + schema_value: "hello", + field_id: "offset:5", + }); + await unlink(capture); + await unlink(descriptor); + const request = join(root, "view.json"); + await writeFile( + request, + JSON.stringify({ + source: { kind: "inline", evidence: parent }, + view: { + kind: "protobuf", + collection: "fields", + field_ids: ["offset:5"], + fields: ["schema_name", "schema_value"], + offset: 0, + limit: 1, + }, + }), + ); + const viewed = await cli.run({ + arguments: ["inspect-analysis-view", request, "--json"], + }); + expect(viewed.exitCode, viewed.stdout + viewed.stderr).toBe(0); + const view = analysisViewResultSchema.parse( + parseEvidence(viewed.json).normalized_result, + ); + expect(view).toMatchObject({ + kind: "protobuf", + parent_evidence_id: parent.evidence_id, + source_status: "complete", + coverage: { total: 1, examined: 1, exhausted: true }, + items: [ + { + record_ordinal: 0, + body_pointer: "/response/content/text", + field_id: "offset:5", + schema_name: "sentence", + schema_value: "hello", + }, + ], + }); + expect(JSON.stringify(view)).not.toContain("content_base64"); + expect((await readFile(bundle)).length).toBeGreaterThan(0); + }, +); + +cliTest( + "uses production MCP schemas, retained pages and CLI-equivalent decoding without reopening sources", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-protobuf-mcp-"); + const capture = join(root, "capture.har"); + const descriptor = join(root, "capture.pb"); + await writeFile(capture, protobufHar()); + await writeFile(descriptor, protobufCaptureDescriptor()); + const { call } = await connectLocalToolsMcp(); + const args: CaptureRequest = { + capture_path: capture, + format: "har", + record_ordinals: [0], + protobuf: { + schema: { + descriptor_set_path: descriptor, + message_type: "capture.Message", + }, + }, + }; + const complete = await call("inspect_web_network_capture", args); + expect(complete.isError, JSON.stringify(complete)).not.toBe(true); + const parent = parseEvidence(complete.structuredContent); + const cliResult = await cli.run({ + arguments: [ + "inspect-web-network-capture", + capture, + "har", + "--record", + "0", + "--protobuf-schema", + descriptor, + "--protobuf-message", + "capture.Message", + "--json", + ], + }); + expect(cliResult.exitCode, cliResult.stdout + cliResult.stderr).toBe(0); + expect(parseEvidence(cliResult.json).evidence_id).toBe(parent.evidence_id); + await verifyCaptureExclusions(call, args, root); + await verifyPartialCaptureSources(call, args, root, descriptor); + await verifyCredentialPreflight(call, root, descriptor); + await verifyDescriptorReadFailures(call, args, root); + const summary = await call("inspect_web_network_capture", { + ...args, + detail: "summary", + }); + expect(summary.isError, JSON.stringify(summary)).not.toBe(true); + const result = analysisViewResultSchema.parse( + parseEvidence(summary.structuredContent).normalized_result, + ); + expect(result).toMatchObject({ + kind: "summary", + parent_evidence_id: parent.evidence_id, + }); + await unlink(capture); + await unlink(descriptor); + const viewed = await call("inspect_analysis_view", { + source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, + view: { + kind: "protobuf", + collection: "fields", + field_numbers: [2], + fields: ["schema_name", "schema_value"], + offset: 0, + limit: 1, + }, + }); + expect(viewed.isError, JSON.stringify(viewed)).not.toBe(true); + expect( + analysisViewResultSchema.parse( + parseEvidence(viewed.structuredContent).normalized_result, + ), + ).toMatchObject({ + kind: "protobuf", + items: [{ schema_name: "values", schema_value: '["1","2"]' }], + coverage: { total: 1, exhausted: true }, + }); + const invalid = await call("inspect_web_network_capture", { + ...args, + record_ordinals: undefined, + }); + expect(invalid.isError).toBe(true); + expect(parseMcpToolError(invalid).error.code).toBe("invalid_request"); + }, +); diff --git a/tests/boundary/browser/sazWebCapture.test.ts b/tests/boundary/browser/sazWebCapture.test.ts index 9a53ff65a8..a325304db7 100644 --- a/tests/boundary/browser/sazWebCapture.test.ts +++ b/tests/boundary/browser/sazWebCapture.test.ts @@ -16,6 +16,7 @@ import { } from "../../../src/domain/webNetworkCapture.js"; import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { protobufFrame } from "../../fixtures/protobufCapture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { @@ -113,6 +114,181 @@ const cli = async (path: string, extra: string[] = []) => { return parseEvidence(JSON.parse(result.stdout)); }; +it("uses HTTP optional whitespace for SAZ identity codings without changing raw headers", async () => { + const body = Buffer.from([0, 0, 0, 0, 2, 8, 1]); + const codings = [ + " \tidentity\t ", + "\tIdEnTiTy ", + " gzip ", + " \u00a0identity", + ]; + const entries: [string, Buffer][] = codings.map((coding, index) => [ + `raw/${index + 1}_s.txt`, + Buffer.concat([ + Buffer.from( + `HTTP/1.1 200 OK\r\nContent-Type: application/grpc\r\nContent-Encoding:${coding}\r\nContent-Length: ${body.length}\r\n\r\n`, + "latin1", + ), + body, + ]), + ]); + const path = await fixture(await archive(entries)); + const evidence = await cli(path, [ + "--record", + "0", + "--record", + "1", + "--record", + "2", + "--record", + "3", + "--protobuf-framing", + "grpc", + ]); + const capture = webNetworkCaptureSchema.parse(evidence.normalized_result); + for (const [index, coding] of codings.entries()) { + expect(capture.records[index]?.reported).toMatchObject({ + response: { + headers: expect.arrayContaining([ + { name: "Content-Encoding", value: coding }, + ]), + }, + }); + if (index < 2) + expect(capture.records[index]?.protobuf).toMatchObject([ + { + pointer: "/response/body", + status: "complete", + fields: [{ number: 1, value: "1" }], + }, + ]); + else expect(capture.records[index]?.protobuf).toBeUndefined(); + } + const { call } = await connectLocalToolsMcp(); + const response = await call("inspect_web_network_capture", { + capture_path: path, + format: "saz", + record_ordinals: [0, 1, 2, 3], + protobuf: { framing: "grpc" }, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(parseEvidence(response.structuredContent).evidence_id).toBe( + evidence.evidence_id, + ); + for (const sensitive of ["gzip", "Content-Encoding"]) { + const excluded = await call("inspect_web_network_capture", { + capture_path: path, + format: "saz", + record_ordinals: [0, 1, 2, 3], + protobuf: { framing: "grpc" }, + sensitive_values: [sensitive], + }); + expect(excluded.isError, JSON.stringify(excluded)).not.toBe(true); + const report = webNetworkCaptureSchema.parse( + parseEvidence(excluded.structuredContent).normalized_result, + ); + expect(report.protobuf_coverage).toMatchObject({ + status: "partial", + inspected_records: 4, + }); + if (sensitive === "gzip") + expect(report.records[2]?.protobuf).toBeUndefined(); + else + expect( + report.records.every((record) => record.protobuf === undefined), + ).toBe(true); + for (const record of report.records) + expect(record.binary_fields).toContainEqual( + expect.objectContaining({ + pointer: "/response/body", + state: "retained", + }), + ); + } +}); + +it("checks trailer credentials before hidden metadata blocks analysis and withholds unchecked source", async () => { + const credential = Buffer.concat([ + protobufFrame(Buffer.from([8, 1])), + protobufFrame(Buffer.from("authorization: Bearer fixture-only\r\n"), 128), + ]); + const responseBytes = (body: Buffer, contentType: string, extra = "") => + Buffer.concat([ + Buffer.from( + `HTTP/1.1 200 OK\r\nContent-Type: ${contentType}\r\nX-Debug: harmless\r\n${extra}\r\n`, + ), + body, + ]); + const bytes = await archive([ + ["raw/1_s.txt", responseBytes(credential, "application/grpc-web+proto")], + [ + "raw/2_s.txt", + responseBytes(Buffer.from([0, 0, 0]), "application/grpc-web+proto"), + ], + [ + "raw/3_s.txt", + responseBytes( + gzipSync(credential), + "application/grpc-web+proto", + "Content-Encoding: gzip\r\n", + ), + ], + [ + "raw/4_s.txt", + responseBytes(Buffer.from("invalid*base64"), "application/grpc-web-text"), + ], + ]); + const path = await fixture(bytes); + const { call } = await connectLocalToolsMcp(); + const base = { + capture_path: path, + format: "saz", + sensitive_values: ["X-Debug"], + }; + const protectedReply = await call("inspect_web_network_capture", { + ...base, + record_ordinals: [0], + }); + expect(protectedReply.isError, JSON.stringify(protectedReply)).not.toBe(true); + const protectedCapture = webNetworkCaptureSchema.parse( + parseEvidence(protectedReply.structuredContent).normalized_result, + ); + expect(protectedCapture.records[0]?.binary_fields).toContainEqual( + expect.objectContaining({ + pointer: "/response/body", + state: "redacted", + content_base64: null, + sha256: null, + }), + ); + expect(protectedCapture.records[0]?.protobuf).toBeUndefined(); + for (const ordinals of [[0, 1], [2], [3]]) { + const response = await call("inspect_web_network_capture", { + ...base, + record_ordinals: ordinals, + }); + const error = parseMcpToolError(response).error; + expect(error.code).toBe("capability_unavailable"); + expect(JSON.stringify(response)).not.toContain("content_base64"); + expect(error.details).not.toHaveProperty("partial_observation"); + } + const opaqueReply = await call("inspect_web_network_capture", { + capture_path: path, + format: "saz", + record_ordinals: [2], + }); + expect(opaqueReply.isError, JSON.stringify(opaqueReply)).not.toBe(true); + const opaque = webNetworkCaptureSchema.parse( + parseEvidence(opaqueReply.structuredContent).normalized_result, + ); + expect(opaque.records[0]?.binary_fields).toContainEqual( + expect.objectContaining({ pointer: "/response/body", state: "retained" }), + ); + expect(opaque.records[0]?.protobuf).toBeUndefined(); + expect(opaque.protobuf_coverage?.status).toBe("partial"); + expect(await readFile(path)).toEqual(bytes); +}); + it("inspects native SAZ through the public CLI and MCP with equal evidence, bytes and archive coordinates", async () => { const bytes = await archive(); const path = await fixture(bytes); diff --git a/tests/boundary/mcp/ghidraEvidenceMcp.test.ts b/tests/boundary/mcp/ghidraEvidenceMcp.test.ts index bb8ed8ce7d..c249a8b823 100644 --- a/tests/boundary/mcp/ghidraEvidenceMcp.test.ts +++ b/tests/boundary/mcp/ghidraEvidenceMcp.test.ts @@ -182,10 +182,7 @@ it("retains terminal-call qualifications across function views and snapshot repl reply.structuredContent, ); expect(evidence.normalized_result, name).toEqual(value); - expect(evidence.raw_result, name).toEqual({ - value, - limitations: [limitation], - }); + expect(evidence.raw_result, name).toBeNull(); expect(evidence.limitations, name).toContain(limitation); expect(evidence.analysis_profile?.parameters).toMatchObject({ function_boundary_observations: @@ -240,7 +237,7 @@ it("retains handler-region limitations on incoming and outgoing reference Eviden reply.structuredContent, ); expect(evidence.normalized_result).toMatchObject({ direction }); - expect(evidence.raw_result).toMatchObject({ limitations: [limitation] }); + expect(evidence.raw_result).toBeNull(); expect(evidence.limitations).toContain(limitation); const replay = await harness.session.execute( "procedure_references", diff --git a/tests/boundary/mcp/ghidraInventoryMcp.test.ts b/tests/boundary/mcp/ghidraInventoryMcp.test.ts index b269087ac0..130088cc64 100644 --- a/tests/boundary/mcp/ghidraInventoryMcp.test.ts +++ b/tests/boundary/mcp/ghidraInventoryMcp.test.ts @@ -25,6 +25,35 @@ import { createServer } from "../../../src/server/createServer.js"; import { waitForExit } from "../../support/process/processFixture.js"; import { GhidraSessionError } from "../../../src/ghidra/GhidraSessionError.js"; +it("delivers a complete Ghidra inventory within the existing MCP response budget", async () => { + const items = Array.from({ length: 30_000 }, (_, index) => ({ + address: `0x${(0x1000 + index).toString(16)}`, + value: `procedure_${String(index).padStart(6, "0")}`, + procedure: { external: false, thunk: false, thunk_target: null }, + })); + const harness = await connectGhidraMcp( + "inventory-response-budget", + (operation) => { + if (operation !== "list_procedures") + throw new Error(`Unexpected operation: ${operation}`); + return Promise.resolve(ok(items)); + }, + ); + try { + const reply = await harness.mcp.callTool({ + name: "list_procedures", + arguments: {}, + }); + if (reply.isError) throw new Error(JSON.stringify(reply)); + const evidence = sessionEvidence(harness.session, reply.structuredContent); + expect(evidence.raw_result).toBeNull(); + expect(evidence.normalized_result).toEqual(items); + expect(harness.calls).toEqual(["list_procedures"]); + } finally { + await harness.close(); + } +}); + it("retains byte-search coverage and rejects malformed pages before creating Evidence", async () => { const page = { matches: [ @@ -245,6 +274,7 @@ it("searches bytes through MCP and the socket bridge with pagination and skipped }); expect(first.isError, JSON.stringify(first)).not.toBe(true); const firstEvidence = sessionEvidence(session, first.structuredContent); + expect(firstEvidence.raw_result).toBeNull(); expect(firstEvidence.normalized_result).toMatchObject({ matches: [ { diff --git a/tests/fixtures/protobufCapture.ts b/tests/fixtures/protobufCapture.ts new file mode 100644 index 0000000000..183bacecd6 --- /dev/null +++ b/tests/fixtures/protobufCapture.ts @@ -0,0 +1,83 @@ +import { create, toBinary } from "@bufbuild/protobuf"; +import { FileDescriptorSetSchema } from "@bufbuild/protobuf/wkt"; + +export const protobufFrame = (body: Buffer, flag = 0): Buffer => { + const header = Buffer.alloc(5); + header[0] = flag; + header.writeUInt32BE(body.length, 1); + return Buffer.concat([header, body]); +}; + +export const protobufCaptureBody = () => + Buffer.concat([ + protobufFrame(Buffer.from("0a0568656c6c6f12020102", "hex")), + protobufFrame(Buffer.from("grpc-status: 0\r\n"), 128), + ]); + +/** Original fixture schema; independent expected field values come from the literal wire bytes. */ +export const protobufCaptureDescriptor = () => + toBinary( + FileDescriptorSetSchema, + create(FileDescriptorSetSchema, { + file: [ + { + name: "capture.proto", + package: "capture", + syntax: "proto3", + messageType: [ + { + name: "Message", + field: [ + { name: "sentence", number: 1, type: 9, label: 1 }, + { name: "values", number: 2, type: 13, label: 3 }, + ], + }, + ], + }, + ], + }), + ); + +export const protobufHar = ( + body: Buffer = protobufCaptureBody(), + mimeType = "application/grpc-web+proto", + additionalBodies: readonly Buffer[] = [], +) => + JSON.stringify({ + log: { + version: "1.2", + creator: { name: "REA original fixture", version: "1" }, + entries: [body, ...additionalBodies].map((body) => ({ + startedDateTime: "2026-10-20T00:00:00Z", + time: 1, + request: { + method: "POST", + url: "https://example.invalid/api", + httpVersion: "HTTP/1.1", + cookies: [], + headers: [], + queryString: [], + headersSize: -1, + bodySize: 0, + }, + response: { + status: 200, + statusText: "OK", + httpVersion: "HTTP/1.1", + cookies: [], + headers: [], + redirectURL: "", + headersSize: -1, + bodySize: body.length, + content: { + size: body.length, + mimeType, + text: body.toString("base64"), + encoding: "base64", + }, + }, + cache: {}, + timings: { send: 0, wait: 1, receive: 0 }, + })), + }, + });