From f93a0335a8c1a60bbf6debdca16675ba303e4422 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 19 Oct 2026 19:33:52 -0700 Subject: [PATCH] feat(analysis-view): project rich captures from Evidence files and template path pairs (#5293) Project large local Evidence captures through the authenticated file-source boundary and add guarded opt-in sibling-pair path inference. Preserve request cancellation, resource ownership and file-relative validation diagnostics. Validated relevant source and compiled CLI/MCP/environment boundaries, static/build/docs/catalog gates, and a real isolated Gitea 28.2.0 capture through CLI, default stdio MCP and HTTP replay. Current-head automated review passed; unrelated long-running platform jobs may still be pending. Co-authored-by: Edgar Arakelyan Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Co-authored-by: gensokyo-dev --- docs/cli.md | 7 + .../agent-environments/rich-captures/plan.md | 18 ++ .../agent-environments/rich-captures/spec.md | 56 +++++ .../agent-environments/rich-captures/tasks.md | 10 + .../rich-captures/verification.md | 104 ++++++++ .../scenario-capture/spec.md | 9 +- docs/web-network-captures.md | 30 ++- .../analysisView/AnalysisViewService.test.ts | 234 +++++++++++++++++- .../analysisView/AnalysisViewService.ts | 88 ++++++- .../analysisView/evidenceFileSource.test.ts | 64 +++++ .../analysisView/evidenceFileSource.ts | 125 ++++++++++ src/cli/analysisViewCommands.ts | 12 +- src/contracts/analysisViewToolContracts.ts | 2 +- src/domain/analysisErrorProjection.ts | 8 + .../EnvironmentManifestAccumulator.ts | 25 +- .../WebNetworkProjectionAccumulator.ts | 17 +- src/domain/analysisView/analysisView.ts | 16 +- .../analysisView/analysisViewDataAdmission.ts | 10 +- .../analysisView/environmentManifestRules.ts | 135 +++++++++- .../environmentManifestView.test.ts | 132 ++++++++++ .../analysisView/environmentManifestView.ts | 2 +- .../scenarioCaptureRecords.test.ts | 47 ++++ .../analysisView/scenarioCaptureRecords.ts | 43 +++- src/domain/evidenceErrors.ts | 5 + src/server/createServer.ts | 2 + src/server/registerAnalysisViewTool.ts | 13 +- tests/boundary/cli/analysisViewCli.test.ts | 43 ++++ tests/boundary/mcp/analysisViewMcp.test.ts | 36 ++- 28 files changed, 1227 insertions(+), 66 deletions(-) create mode 100644 docs/specs/agent-environments/rich-captures/plan.md create mode 100644 docs/specs/agent-environments/rich-captures/spec.md create mode 100644 docs/specs/agent-environments/rich-captures/tasks.md create mode 100644 docs/specs/agent-environments/rich-captures/verification.md create mode 100644 src/application/analysisView/evidenceFileSource.test.ts create mode 100644 src/application/analysisView/evidenceFileSource.ts diff --git a/docs/cli.md b/docs/cli.md index e937882ddb..c320900c67 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -30,6 +30,13 @@ jq -c '{source: {kind: "inline", evidence: .}, view: {kind: "summary"}}' app-evi rea inspect-analysis-view app-view.json ``` +The request can also name the saved file instead of embedding it: +`{"source": {"kind": "evidence-file", "path": "app-evidence.json"}, "view": {"kind": "summary"}}`. +The CLI resolves a relative path against the working directory; MCP clients +pass an absolute path. REA reads and authenticates the file, then projects it +like retained Evidence, so a large file is bounded by the file reader rather +than charged as inline input. + For example, a 334 MB Obsidian application Evidence file yields a summary view of about 10 KB. Replace the view with `{"kind": "page", "collection": "modules", "offset": 0, "limit": 32}` for a diff --git a/docs/specs/agent-environments/rich-captures/plan.md b/docs/specs/agent-environments/rich-captures/plan.md new file mode 100644 index 0000000000..f7601491cd --- /dev/null +++ b/docs/specs/agent-environments/rich-captures/plan.md @@ -0,0 +1,18 @@ +# Plan + +1. Add the `evidence-file` source to `analysisViewSourceSchema` + (`src/domain/analysisView/analysisView.ts`). +2. Add `inspectAnalysisViewWithFiles` / `inspectAnalysisViewValidatedWithFiles` + to `AnalysisViewService`: read with `readJsonFile`, authenticate with + `parseEvidence`, then project through the retained path with a lookup that + returns the file's Evidence. The synchronous path refuses an unread file + source. +3. Route the CLI command (with `source.path` resolved against the working + directory) and the MCP tool through the file-aware entry point. +4. Add `sibling-pair-variance` to `TEMPLATE_RULES`, with its merge pass before + `sibling-variance`, its parameter names and its segment-limit limitation. +5. Document both in the tool contract, `docs/cli.md` and + `docs/web-network-captures.md`. +6. Tests: service (file equals inline; a capture refused inline is accepted + from a file; refusals), CLI and MCP boundaries, and the pair rule's merges + and non-merges. diff --git a/docs/specs/agent-environments/rich-captures/spec.md b/docs/specs/agent-environments/rich-captures/spec.md new file mode 100644 index 0000000000..f669c5d714 --- /dev/null +++ b/docs/specs/agent-environments/rich-captures/spec.md @@ -0,0 +1,56 @@ +# Rich captures for the environment manifest + +Issue: https://github.com/morluto/rea/issues/4933 + +## Outcome + +A capture large enough to hold a rich application state can be projected into +an environment manifest whose tool list stays within what model providers +accept. Two changes: + +- `inspect_analysis_view` accepts a third source, + `{"kind": "evidence-file", "path": ""}`. REA reads one + completed Evidence JSON file, authenticates it, and projects it like retained + Evidence, so its strings are bounded by the file reader instead of being + charged as inline input. +- The environment-manifest view gains an opt-in template rule, + `sibling-pair-variance`, that templates two adjacent literal segments after a + plural literal, such as `/repos/{owner}/{repo}/…`. + +## Requirements + +- An `evidence-file` path must be absolute and free of NUL bytes; the CLI + resolves a relative path against the working directory before validation, + and MCP clients pass an absolute path. +- The file is read through the shared JSON file reader (regular file, heap and + string-length guards) and parsed as complete Evidence. A missing or unreadable + file is an `EvidenceFileError`; JSON that is not complete Evidence is invalid + input reported under `source.path`. +- Over MCP the read follows the request's cancellation and the server's + resource scope, so a failed descriptor close stays owned for retry; a request + cancelled before authentication, before projection or during projection + returns typed cancellation and records nothing. Before authentication, which + copies and hashes the parsed Evidence, REA checks that live heap headroom holds + those copies and otherwise returns `resource_constraint`. +- A file source projects exactly what the same Evidence projects inline or + retained, with the same parent Evidence ID, view parameters and result. +- The synchronous projection API refuses an unread `evidence-file` source with + a typed capability error; the CLI and MCP adapters read files first. +- `sibling-pair-variance` applies only when listed, before `sibling-variance`, so + a single-position merge cannot template one half of a pair first. It + compares groups equal in method, origin and segment count except at two + adjacent literal positions at index 2 or later that follow a plural literal + and precede another literal (a trailing pair, such as + `/users/alice/followers`, names a sub-resource rather than a namespace and + name), and merges a bucket only when it holds at least three distinct pairs and both + positions vary. Stop words and `v1`-style versions never qualify, and routes + of more than 32 segments are not compared, with a limitation saying so. +- Its parameters are named from the plural literal, singular plus `_namespace` + and then `_name`, and labelled `sibling-pair-variance`. + +## Non-goals + +- No change to the 96 MiB projection budget or to inline admission. +- No change to `sibling-variance`; pairs are a separate rule so every inferred + parameter keeps naming the rule that produced it. +- No inference of semantic names (`owner`, `repo`) beyond the plural literal. diff --git a/docs/specs/agent-environments/rich-captures/tasks.md b/docs/specs/agent-environments/rich-captures/tasks.md new file mode 100644 index 0000000000..4683b6971a --- /dev/null +++ b/docs/specs/agent-environments/rich-captures/tasks.md @@ -0,0 +1,10 @@ +# Tasks + +- [x] Record the outcome, requirements and non-goals. +- [x] Add the `evidence-file` source and the file-aware service entry point. +- [x] Route the CLI command and the MCP tool through it. +- [x] Add the `sibling-pair-variance` rule, its names and its limitation. +- [x] Document both; extend the tool contract text. +- [x] Add service, CLI, MCP and rule tests. +- [x] Run static, formatting, duplication, documentation and test gates. +- [x] Rebuild a rich capture with both changes and record the results. diff --git a/docs/specs/agent-environments/rich-captures/verification.md b/docs/specs/agent-environments/rich-captures/verification.md new file mode 100644 index 0000000000..3a6e5d4fe6 --- /dev/null +++ b/docs/specs/agent-environments/rich-captures/verification.md @@ -0,0 +1,104 @@ +# Verification + +Verified commit: a64367a9 on main 2dd5effe. + +## Gates + +Run on macOS 26.5.1 with Node 22.23.1 on that tree: `npm run typecheck`, `lint` +(oxlint plus `verify:module-boundaries`, 2172 files), `format:check`, `knip`, +`metadata:check`, `docs:check`, `jscpd` (3.43% duplicated lines), `scan:todos` +and `mcp-catalog:check` all exit 0. oxlint on the 12 changed TypeScript files +reports no finding that main does not already report for the same files. + +`npm run test:prepare`, then `npm run test:changed` (the tests related to the +changed files): 84 files, 1531 passed, 3 skipped, 0 failed. They include: + +- the service: an Evidence file projects exactly what the same Evidence projects + inline; a capture whose response text exceeds the inline budget is refused + inline with `resource_constraint` and projected from its file; a relative + path, a missing file and JSON that is not Evidence are refused; the + synchronous API refuses an unread file source; +- the compiled CLI: a request naming an Evidence file projects the same + manifest as the same Evidence inline; +- MCP: inline, retained and file sources give identical manifests, outputs + validate against the advertised schema, and a relative path is refused; +- the file source module: the size estimate, the authentication headroom + refusal, a cancelled request refused before reading, and a read within the + caller's resource scope; the service returns typed cancellation for a + cancelled file request; a request aborted while it runs, even after its read, returns typed + cancellation because the service yields before each check; +- the pair rule: three owner/repo pairs merge for collection and numeric detail + routes, while two pairs, or three pairs sharing one namespace, stay literal; + with `sibling-variance` also selected the pair still merges, because pairs + are merged first. Each pair parameter keeps its role when + `sibling-variance` later templates the segment before the pair. + +## A rich capture + +A Gitea 28.1.0 instance seeded through its API with an invented world (12 +people, 4 organisations, 16 public repositories, 320 issues with about 600 +comments, labels, milestones, commits, pull requests and releases) was read by +one `capture-browser-scenario` run of 1,217 API reads: 1,218 transactions in +24 s, 10.7 MB of Evidence, nothing dropped. Scenario parents are not on main, +so this run used this commit merged onto the scenario-capture change +(morluto/rea#5182) and packed for the host. + +| | Inline source, previous rules | Evidence-file source with `sibling-pair-variance` | +| --------------------- | ------------------------------------- | ------------------------------------------------- | +| Manifest | `resource_constraint` (96 MiB budget) | built in 4.1 s, 459 MB maximum RSS | +| Tools | 446 (with the budget raised by hand) | 63 | +| Recorded exchanges | 1,218 | 1,218 | +| Tools with one record | 77% | 49% | + +The pair rule templated 25 route families as `{repo_namespace}/{repo_name}`. +A model provider that accepts at most 128 tools refused every request for the +446-tool manifest with HTTP 400. + +The 63-tool environment was then exported, and in the AgentEnv runner its +smoke task passed and twelve generated retrieval tasks scored 12/12 with +`openai/gpt-oss-120b` and 12/12 with `zai-org/GLM-5.3-Flash`, with no provider +errors. The 446-tool manifest had scored 0/12 and 8/12 on its own generated +tasks, the 0 being the provider's tool limit rather than the model. + +Review then found that the rule as run here also merged one id's +sub-resources (`/users/alice/followers`, `/users/bob/following` and +`/users/carol/starred` became one tool), and a pair now qualifies only when a +literal follows it. The Gitea capture above predates that guard and has not +been projected with it: whether any of the 63 tools merged sub-resources this +way, and the tool count with bare `/repos/{owner}/{repo}` routes kept one per +pair, are not established here. + +## Linux follow-up (2026-10-20) + +Based on `dd2bf3565`, including main `5fa4defcb`, with Node 24.18.0 and npm +11.16.0. The file reader now owns authentication; invalid Evidence returns the +selected file path and file-relative `content_issues`. Service and advertised +MCP error-envelope regressions cover this boundary. + +- Domain analysis views, application analysis views and error projection: + 16 files, 298 tests passed. +- Compiled analysis-view CLI/MCP and environment serve/export boundaries: + 5 files, 48 tests passed. +- Serial Turbo run: typecheck, lint, format, knip, package metadata and required + runtime/contract artifacts: 11 tasks passed. Documentation/catalog validation: + 8 tasks passed. Architecture guards passed across 1,595 source files. +- Real Gitea 28.2.0 official Linux binary (published compressed SHA-256 checked), + isolated SQLite database, two users and three public repositories with large + issue bodies. Actual Chromium navigation captured nine API transactions, + 45 events, no drops, 12,584,312 bytes of Evidence. Inline manifest projection + was refused with `resource_constraint`; file CLI and default stdio MCP + projections agreed on five tools with all template rules selected. The + three trailing user sub-resources stayed separate. Malformed file diagnostics + named the file's JSON root. HTTP SDK replay verified six complete small + responses and twelve exact 256 KiB prefixes of large responses, each marked + truncated. Owned Gitea, browser and replay-server processes were stopped. + +One CPU and one test worker were used. Sampled sum of descendant RSS every +50 ms (shared pages can count twice): focused tests 15.834 s / 15.327 CPU s / +764.69 MiB peak; boundaries 194.916 s / 190.121 CPU s / 1,795.15 MiB; +static/build 266.311 s / 259.658 CPU s / 4,677.44 MiB; real Gitea workflow +31.631 s / 24.999 CPU s / 2,146.29 MiB. These are individual runs, not a +cold/warm comparison; model token usage was not measured. The first external +probe stopped at the expected CLI refusal because its driver rejected exit 1; +the driver was corrected and the complete workflow passed. No new macOS, +Windows, model-task scoring or AgentEnv runner verification is claimed here. diff --git a/docs/specs/agent-environments/scenario-capture/spec.md b/docs/specs/agent-environments/scenario-capture/spec.md index 5ffb098230..3654d71993 100644 --- a/docs/specs/agent-environments/scenario-capture/spec.md +++ b/docs/specs/agent-environments/scenario-capture/spec.md @@ -31,9 +31,12 @@ change. unrepresented records, as HAR records without a usable request line are. - The response media type is the captured body's `media_type`, or the `content-type` header value when only headers were captured. -- The existing projection representation budget accounts for copied and - transformed strings before schema body validation or request-body decoding, - including retained Evidence parents. Over-budget captures return +- The existing projection representation budget reserves, before schema body + validation or request-body decoding and for retained and file parents alike, + the scratch of validating the largest body (two bytes per base64 character) + and the decoded text of every request body; response bodies stay borrowed, + as HAR response text does, so a large capture read from its file is not + charged per character for them. Over-budget captures return `resource_constraint` before those allocations. Replay projection omits the request text used only for manifest body-shape inference; retained response bytes and their integrity checks are unchanged. diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md index c4fda1723e..124b63b9ca 100644 --- a/docs/web-network-captures.md +++ b/docs/web-network-captures.md @@ -300,7 +300,15 @@ exclusions and deduplicated limitations. It scans path spans without splitting literal paths into arrays. Before schema parsing, it reserves the capture clone and validation scratch by walking the parent without cloning it. Inline requests borrow the Evidence value through CLI/MCP envelope validation; the application -admits the entire portable envelope before authenticated parsing or hashing. +admits the entire portable envelope before authenticated parsing or hashing. An +`evidence-file` source is read and authenticated first and then admitted like +retained Evidence, so its strings are not charged as inline envelope text: a +capture too large to project inline can be projected from its file. Before +authentication REA checks that live heap headroom holds its copies of the parsed +Evidence, and over MCP the read follows the request's cancellation. +An invalid Evidence file returns `EvidenceFileError` with the selected `path` +and `content_issues` relative to the file's JSON root; those locations do not +refer to the request's `source.path` string. The envelope clone, hash scratch and capture clone stay charged during grouping. Temporary URL/path/MIME/query work and retained state must fit together within a 96 MiB projection representation budget. A whole group or @@ -374,11 +382,12 @@ template and `params` carry the four format rules with names `id`, `uuid`, only when listed, each at most once (a repeated rule is an `invalid_request`), in this order: -| Rule | Applies to | Parameter | -| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------- | -| `after-uid-literal` | The literal segment right after a literal `uid`, unless it is a stop word | `uid` | -| `opaque-token` | A lowercase alphanumeric segment of twelve or more characters mixing letters and digits that is not a `v1`-style version | `uid` | -| `sibling-variance` | Groups equal in method, origin and segment count that differ in one literal at index 2 or later; the siblings whose literal is identifier-like (a digit or `:`, or three or more hyphenated names under a plural literal) merge, never a stop word, and the other literals stay their own tools | singular previous literal plus `_id`, else `id` | +| Rule | Applies to | Parameter | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------- | +| `after-uid-literal` | The literal segment right after a literal `uid`, unless it is a stop word | `uid` | +| `opaque-token` | A lowercase alphanumeric segment of twelve or more characters mixing letters and digits that is not a `v1`-style version | `uid` | +| `sibling-pair-variance` | Groups equal in method, origin and segment count that differ in two adjacent literals right after a plural literal at index 2 or later and followed by a literal, with at least three distinct pairs and both positions varying (`/repos/{owner}/{repo}/issues`); stop words and `v1`-style versions never qualify | singular plural literal plus `_namespace`, then `_name` | +| `sibling-variance` | Groups equal in method, origin and segment count that differ in one literal at index 2 or later; the siblings whose literal is identifier-like (a digit or `:`, or three or more hyphenated names under a plural literal) merge, never a stop word, and the other literals stay their own tools | singular previous literal plus `_id`, else `id` | With `sibling-variance` selected, literal siblings within its comparison scope fold into existing opt-in parameters as observed values unless they are stop words. @@ -395,7 +404,7 @@ cut to their first label, `_by_` appended when parameters exist, sanitized to `[a-z0-9_]`, at most 60 characters and made unique with numeric suffixes: `get_dashboards_by_uid`, `get_plugins_settings_by_plugin_id`. -`sibling-variance` compares routes of at most 32 segments; longer routes keep +`sibling-variance` and `sibling-pair-variance` compare routes of at most 32 segments; longer routes keep their grouped templates and the manifest says so in a limitation. `body_shape` is derived for `POST`, `PUT` and `PATCH` from the tool's earliest record when it carries an `application/json` or `application/*+json` request body: top-level @@ -413,11 +422,16 @@ transaction whose URL was redacted is left unrepresented, and a response body keeps the capture's redaction markers, which a manifest limitation counts. The manifest accumulator shares the api-surface admission model: the capture -clone, the inline envelope, record pointers, groups, every route part, observed +clone, the inline envelope (for an inline source), record pointers, groups, every route part, observed values, query names and body-shape keys are admitted before allocation, and each recorded body is charged as temporary bytes before it is parsed. Retained state and temporary work must fit together within the 96 MiB projection representation budget; exceeding it returns `resource_constraint` without a partial manifest. +Name a large capture with `{"kind": "evidence-file", "path": "/abs/capture.json"}` +rather than inline: response bodies the manifest never reads are then charged +once per value in the capture clone, not per character. A +`capture_browser_scenario` parent is also charged, before validation, for +checking its largest body and for the decoded text of its request bodies. ## Resource and lifecycle boundaries diff --git a/src/application/analysisView/AnalysisViewService.test.ts b/src/application/analysisView/AnalysisViewService.test.ts index 3d9db3ddab..728aada4cb 100644 --- a/src/application/analysisView/AnalysisViewService.test.ts +++ b/src/application/analysisView/AnalysisViewService.test.ts @@ -1,10 +1,13 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + import { analysisViewCaptureFixture, analysisViewCaptureEvidence, analysisViewObjectDenseCapture, environmentManifestCaptureFixture, } from "../../../tests/fixtures/analysisView.js"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE } from "../../contracts/javascript/javascriptRuntimeReconciliationExample.js"; import { createEvidence, parseEvidence } from "../../domain/evidence.js"; @@ -19,6 +22,7 @@ import { import { inspectAnalysisView, inspectAnalysisViewValidated, + inspectAnalysisViewWithFiles, summarizeRetainedAnalysis, } from "./AnalysisViewService.js"; import type { Evidence } from "../../domain/evidence.js"; @@ -27,6 +31,16 @@ import { err, ok } from "../../domain/result.js"; import { EvidenceLedger } from "../investigation/EvidenceLedger.js"; import { WEB_NETWORK_CAPTURE_LIMITS } from "../../domain/webNetworkCapture.js"; import { isImmutableJsonSnapshot } from "../../domain/immutableJson.js"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import { + SCENARIO_EXCHANGES, + scenarioCaptureEvidence, + scenarioCaptureFixture, +} from "../../../tests/fixtures/browserScenarioCapture.js"; +import { readEvidenceFileSource } from "./evidenceFileSource.js"; +import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; + +vi.mock("./evidenceFileSource.js", { spy: true }); it.each(["layout", "javascript"])( "carries the retained %s source snapshot into repeatable shared views", @@ -860,3 +874,221 @@ it("admits raw_result and parameters before authenticating a portable capture", }); } }); + +const manifestView = { kind: "environment-manifest" } as const; +const writeEvidenceFile = async (value: unknown): Promise => { + const root = await createTestTempDirectory("rea-analysis-view-file-"); + const path = join(root, "capture.json"); + await writeFile(path, JSON.stringify(value)); + return path; +}; + +describe("inspectAnalysisViewWithFiles", () => { + const view = manifestView; + + it("projects an Evidence file exactly as the same Evidence inline", async () => { + const parent = analysisViewCaptureEvidence( + environmentManifestCaptureFixture(), + ); + const fromFile = await inspectAnalysisViewWithFiles({ + source: { kind: "evidence-file", path: await writeEvidenceFile(parent) }, + view, + }); + const inline = inspectAnalysisView({ + source: { kind: "inline", evidence: parent }, + view, + }); + if (!fromFile.ok) throw fromFile.error; + if (!inline.ok) throw inline.error; + expect(fromFile.value.normalized_result).toEqual( + inline.value.normalized_result, + ); + expect(fromFile.value.parameters).toEqual(inline.value.parameters); + }); + + it("accepts from a file a capture whose response text exceeds the inline budget", async () => { + const capture = environmentManifestCaptureFixture(); + const reported = capture.records[0]!.reported as { + response: { content: { text: string } }; + }; + // Inline admission charges about 14 bytes per character; this body alone overruns it. + reported.response.content.text = "a".repeat( + Math.ceil(WEB_NETWORK_CAPTURE_LIMITS.outputBytes / 14) + 1024, + ); + const parent = analysisViewCaptureEvidence(capture); + expect( + inspectAnalysisView({ + source: { kind: "inline", evidence: parent }, + view, + }), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError", resource: "memory" }, + }); + const fromFile = await inspectAnalysisViewWithFiles({ + source: { kind: "evidence-file", path: await writeEvidenceFile(parent) }, + view, + }); + if (!fromFile.ok) throw fromFile.error; + expect(fromFile.value.parameters).toMatchObject({ + parent_evidence_id: parent.evidence_id, + }); + }); + + it("refuses a relative path, a missing file and JSON that is not Evidence", async () => { + expect( + await inspectAnalysisViewWithFiles({ + source: { kind: "evidence-file", path: "capture.json" }, + view, + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisInputError" } }); + const missing = join( + await createTestTempDirectory("rea-analysis-view-missing-"), + "absent.json", + ); + expect( + await inspectAnalysisViewWithFiles({ + source: { kind: "evidence-file", path: missing }, + view, + }), + ).toMatchObject({ ok: false, error: { _tag: "EvidenceFileError" } }); + const notEvidence = await inspectAnalysisViewWithFiles({ + source: { + kind: "evidence-file", + path: await writeEvidenceFile({ hello: 1 }), + }, + view, + }); + expect(notEvidence).toMatchObject({ + ok: false, + error: { _tag: "EvidenceFileError", reason: "wrong-kind" }, + }); + if (notEvidence.ok) throw new Error("Expected invalid Evidence file"); + expect(projectAnalysisError(notEvidence.error)).toMatchObject({ + details: { + content_issues: expect.arrayContaining([ + { + path: ["evidence_id"], + reason: "missing_argument", + expected: "string", + }, + ]), + }, + }); + }); + + it("returns typed cancellation for a cancelled file request", async () => { + const controller = new AbortController(); + controller.abort(); + const parent = analysisViewCaptureEvidence( + environmentManifestCaptureFixture(), + ); + expect( + await inspectAnalysisViewWithFiles( + { + source: { + kind: "evidence-file", + path: await writeEvidenceFile(parent), + }, + view, + }, + undefined, + { signal: controller.signal }, + ), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisCancelledError" } }); + }); + + it("returns typed cancellation when a running file request is aborted", async () => { + const controller = new AbortController(); + const parent = analysisViewCaptureEvidence( + environmentManifestCaptureFixture(), + ); + const path = await writeEvidenceFile(parent); + const running = inspectAnalysisViewWithFiles( + { source: { kind: "evidence-file", path }, view }, + undefined, + { signal: controller.signal }, + ); + // Queued before the request yields after its synchronous work, so it is seen even if the read already finished. + setImmediate(() => controller.abort()); + expect(await running).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); + }); + + it("leaves file sources to the file-aware entry point", () => { + expect( + inspectAnalysisView({ + source: { kind: "evidence-file", path: "/tmp/capture.json" }, + view, + }), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCapabilityUnavailableError" }, + }); + }); +}); + +describe("inspectAnalysisViewWithFiles scenario parents and cancellation", () => { + const view = manifestView; + + it("accepts from a file a scenario capture whose response bodies exceed the inline budget", async () => { + const blob = "a".repeat( + Math.ceil(WEB_NETWORK_CAPTURE_LIMITS.outputBytes / 14) + 1024, + ); + const parent = scenarioCaptureEvidence( + scenarioCaptureFixture( + SCENARIO_EXCHANGES.map((exchange, index) => + index === 1 ? { ...exchange, body: `{"blob":"${blob}"}` } : exchange, + ), + ), + ); + expect( + inspectAnalysisView({ + source: { kind: "inline", evidence: parent }, + view, + }), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError", resource: "memory" }, + }); + const fromFile = await inspectAnalysisViewWithFiles({ + source: { kind: "evidence-file", path: await writeEvidenceFile(parent) }, + view, + }); + if (!fromFile.ok) throw fromFile.error; + expect(fromFile.value.parameters).toMatchObject({ + parent_evidence_id: parent.evidence_id, + }); + }); + + it("reports cancellation rather than the authentication failure of a file aborted while it is authenticated", async () => { + const controller = new AbortController(); + const parent = analysisViewCaptureEvidence( + environmentManifestCaptureFixture(), + ); + const path = await writeEvidenceFile({ + ...parent, + evidence_id: `ev_${"0".repeat(64)}`, + }); + const { readEvidenceFileSource: read } = await vi.importActual< + typeof import("./evidenceFileSource.js") + >("./evidenceFileSource.js"); + // The abort arrives once the read has finished, while the stale ID is being authenticated. + vi.mocked(readEvidenceFileSource).mockImplementationOnce( + async (...args) => { + const loaded = await read(...args); + controller.abort(); + return loaded; + }, + ); + expect( + await inspectAnalysisViewWithFiles( + { source: { kind: "evidence-file", path }, view }, + undefined, + { signal: controller.signal }, + ), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisCancelledError" } }); + }); +}); diff --git a/src/application/analysisView/AnalysisViewService.ts b/src/application/analysisView/AnalysisViewService.ts index 7ea9e0f645..fd2410f781 100644 --- a/src/application/analysisView/AnalysisViewService.ts +++ b/src/application/analysisView/AnalysisViewService.ts @@ -1,3 +1,4 @@ +import { setImmediate } from "node:timers/promises"; import { INSPECT_ANALYSIS_VIEW_OPERATION } from "../../domain/analysisView/analysisViewErrors.js"; import { z } from "zod"; @@ -6,7 +7,10 @@ import type { EvidenceLookup } from "../EvidenceInputResolver.js"; import { resolveEvidenceInput } from "../EvidenceInputResolver.js"; import type { EvidenceWriter } from "../investigation/InvestigationRecordPort.js"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; -import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCore.js"; +import { + AnalysisCancelledError, + AnalysisCapabilityUnavailableError, +} from "../../domain/analysisErrorCore.js"; import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; import { EvidenceIntegrityError } from "../../domain/evidenceErrors.js"; import { @@ -29,6 +33,10 @@ import { err, ok, type Result } from "../../domain/result.js"; import { WebNetworkApiSurfaceAccumulator } from "../../domain/analysisView/WebNetworkApiSurfaceAccumulator.js"; import { EnvironmentManifestAccumulator } from "../../domain/analysisView/EnvironmentManifestAccumulator.js"; import { admitAnalysisViewData } from "../../domain/analysisView/analysisViewDataAdmission.js"; +import { + readEvidenceFileSource, + type EvidenceFileReadOptions, +} from "./evidenceFileSource.js"; /** Authenticate a selected-view request and project already completed Evidence. */ export const inspectAnalysisView = ( @@ -48,6 +56,58 @@ export const inspectAnalysisView = ( return inspectAnalysisViewValidated(parsed.data, lookup); }; +/** Validate a request whose source may be an Evidence file, then project it. */ +export const inspectAnalysisViewWithFiles = async ( + rawInput: unknown, + lookup?: EvidenceLookup, + options: EvidenceFileReadOptions = {}, +): Promise> => { + const parsed = inspectAnalysisViewInputSchema.safeParse(rawInput); + if (!parsed.success) + return err( + analysisInputErrorFromIssues( + INSPECT_ANALYSIS_VIEW_OPERATION, + parsed.error.issues, + rawInput, + { cause: parsed.error }, + ), + ); + return inspectAnalysisViewValidatedWithFiles(parsed.data, lookup, options); +}; + +/** + * Read and authenticate an Evidence-file source, then project it as retained + * Evidence: its strings are bounded by the file reader rather than charged as + * inline input. Other sources project unchanged. + */ +export const inspectAnalysisViewValidatedWithFiles = async ( + input: InspectAnalysisViewInput, + lookup?: EvidenceLookup, + options: EvidenceFileReadOptions = {}, +): Promise> => { + if (input.source.kind !== "evidence-file") + return inspectAnalysisViewValidated(input, lookup); + const loaded = await readEvidenceFileSource(input.source.path, options); + // Authentication and projection are synchronous; yield so a pending abort is dispatched before each check. + const cancelled = async (): Promise => { + await setImmediate(); + return options.signal?.aborted === true; + }; + // A request cancelled during authentication reports cancellation, even when authentication failed. + if (await cancelled()) + return err(new AnalysisCancelledError(INSPECT_ANALYSIS_VIEW_OPERATION)); + if (!loaded.ok) return loaded; + const id = loaded.value.evidence_id; + const projected = inspectAnalysisViewValidated( + { ...input, source: { kind: "retained-evidence", evidence_id: id } }, + (evidenceId) => (evidenceId === id ? loaded.value : lookup?.(evidenceId)), + ); + // A request cancelled during projection delivers and records nothing. + return (await cancelled()) + ? err(new AnalysisCancelledError(INSPECT_ANALYSIS_VIEW_OPERATION)) + : projected; +}; + /** Authenticate borrowed inline Evidence after adapter envelope validation. */ export const inspectAnalysisViewValidated = ( input: InspectAnalysisViewInput, @@ -182,8 +242,28 @@ const resolveAnalysisViewSource = ( ): Result => { if (source.kind === "retained-evidence") return resolveEvidenceInput(source, lookup); + // Files are read by inspectAnalysisViewValidatedWithFiles, which the CLI and MCP adapters call. + if (source.kind === "evidence-file") + return err( + new AnalysisCapabilityUnavailableError( + ANALYSIS_VIEW_PROVIDER.id, + INSPECT_ANALYSIS_VIEW_OPERATION, + "evidence-file sources are read before projection", + ), + ); + return authenticateEvidence(source.evidence, ["source", "evidence"], { + source, + }); +}; + +/** Parse complete Evidence, reporting schema issues under the given request path. */ +const authenticateEvidence = ( + value: unknown, + path: readonly (string | number)[], + input: unknown, +): Result => { try { - return ok(parseEvidence(source.evidence)); + return ok(parseEvidence(value)); } catch (cause: unknown) { if (cause instanceof z.ZodError) return err( @@ -191,9 +271,9 @@ const resolveAnalysisViewSource = ( INSPECT_ANALYSIS_VIEW_OPERATION, cause.issues.map((issue) => ({ ...issue, - path: ["source", "evidence", ...issue.path], + path: [...path, ...issue.path], })), - { source }, + input, { cause }, ), ); diff --git a/src/application/analysisView/evidenceFileSource.test.ts b/src/application/analysisView/evidenceFileSource.test.ts new file mode 100644 index 0000000000..8bad7eed31 --- /dev/null +++ b/src/application/analysisView/evidenceFileSource.test.ts @@ -0,0 +1,64 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { analysisViewCaptureEvidence } from "../../../tests/fixtures/analysisView.js"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; +import { + estimateJsonValueBytes, + readEvidenceFileSource, + requireAuthenticationHeadroom, +} from "./evidenceFileSource.js"; + +describe("evidence-file source", () => { + it("estimates parsed JSON at no less than its UTF-16 text", () => { + const text = "x".repeat(1000); + expect(estimateJsonValueBytes(text)).toBeGreaterThanOrEqual(2000); + expect( + estimateJsonValueBytes({ outer: [text, { inner: text }] }), + ).toBeGreaterThan(2 * estimateJsonValueBytes(text)); + }); + + it("refuses authentication when live heap headroom cannot hold its copies", () => { + const value = { normalized_result: "y".repeat(10_000) }; + expect( + requireAuthenticationHeadroom(value, "/tmp/capture.json", 1024), + ).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisResourceConstraintError", + resource: "memory", + reportedLimits: { + input_path: "/tmp/capture.json", + available_heap_bytes: 1024, + }, + }, + }); + expect( + requireAuthenticationHeadroom(value, "/tmp/capture.json", 1 << 30), + ).toEqual({ ok: true, value: undefined }); + }); + + it("refuses a cancelled request before reading and reads within the caller's resource scope", async () => { + const controller = new AbortController(); + controller.abort(); + expect( + await readEvidenceFileSource("/nonexistent/capture.json", { + signal: controller.signal, + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisCancelledError" } }); + const path = join( + await createTestTempDirectory("rea-evidence-file-source-"), + "capture.json", + ); + const parent = analysisViewCaptureEvidence(); + await writeFile(path, JSON.stringify(parent)); + const resources = new ArtifactResourceScope(); + const read = await readEvidenceFileSource(path, { resources }); + await resources.close(); + if (!read.ok) throw read.error; + expect(read.value).toEqual(JSON.parse(JSON.stringify(parent))); + }); +}); diff --git a/src/application/analysisView/evidenceFileSource.ts b/src/application/analysisView/evidenceFileSource.ts new file mode 100644 index 0000000000..6e644b57ee --- /dev/null +++ b/src/application/analysisView/evidenceFileSource.ts @@ -0,0 +1,125 @@ +import { getHeapStatistics } from "node:v8"; +import { z } from "zod"; + +import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisResourceConstraintError, +} from "../../domain/analysisErrorCore.js"; +import { INSPECT_ANALYSIS_VIEW_OPERATION } from "../../domain/analysisView/analysisViewErrors.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { parseEvidence, type Evidence } from "../../domain/evidence.js"; +import { + EvidenceFileError, + EvidenceIntegrityError, +} from "../../domain/evidenceErrors.js"; +import { projectInputIssues } from "../../domain/inputIssueProjection.js"; +import { readJsonFile } from "../JsonFiles.js"; +import { RegularFileCleanupFailure } from "../RegularFileRead.js"; + +/** The request's cancellation and the scope that owns descriptors whose close failed. */ +export interface EvidenceFileReadOptions { + readonly signal?: AbortSignal; + readonly resources?: ArtifactResourceScope; +} + +/** Authentication clones the parsed Evidence and hashes its canonical JSON. */ +const AUTHENTICATION_COPIES = 2; + +/** Conservative bytes a parsed JSON value occupies: UTF-16 text plus container and slot overhead. */ +export const estimateJsonValueBytes = (value: unknown): number => { + let bytes = 0; + const stack: unknown[] = [value]; + while (stack.length > 0) { + const item = stack.pop(); + if (typeof item === "string") bytes += 64 + item.length * 2; + else if (Array.isArray(item)) { + bytes += 128 + item.length * 16; + for (const entry of item) stack.push(entry); + } else if (typeof item === "object" && item !== null) { + bytes += 128; + for (const [key, entry] of Object.entries(item)) { + bytes += 64 + key.length * 2; + stack.push(entry); + } + } else bytes += 16; + } + return bytes; +}; + +/** Refuse before authentication when live heap headroom cannot hold its copies of the parsed value. */ +export const requireAuthenticationHeadroom = ( + value: unknown, + path: string, + availableBytes: number = getHeapStatistics().total_available_size, +): Result => { + const valueBytes = estimateJsonValueBytes(value); + const requiredBytes = valueBytes * AUTHENTICATION_COPIES; + if (requiredBytes <= availableBytes) return ok(undefined); + return err( + new AnalysisResourceConstraintError( + INSPECT_ANALYSIS_VIEW_OPERATION, + "memory", + "Insufficient heap headroom to authenticate the Evidence file", + { + input_path: path, + projected_value_bytes: valueBytes, + authentication_bytes: requiredBytes, + available_heap_bytes: availableBytes, + }, + ), + ); +}; + +/** Read and authenticate one Evidence file under the request's cancellation and resource scope. */ +export const readEvidenceFileSource = async ( + path: string, + options: EvidenceFileReadOptions = {}, +): Promise> => { + const { signal, resources } = options; + if (signal?.aborted) + return err(new AnalysisCancelledError(INSPECT_ANALYSIS_VIEW_OPERATION)); + const loaded = await readJsonFile(path, signal, resources).catch( + (cause: unknown) => { + // A failed close transfers a live handle owner; preserve that failure. + if ( + signal?.aborted === true && + !(cause instanceof RegularFileCleanupFailure) + ) + return err( + new AnalysisCancelledError(INSPECT_ANALYSIS_VIEW_OPERATION, { + cause, + }), + ); + throw cause; + }, + ); + if (!loaded.ok) return loaded; + if (signal?.aborted) + return err(new AnalysisCancelledError(INSPECT_ANALYSIS_VIEW_OPERATION)); + const headroom = requireAuthenticationHeadroom(loaded.value, path); + if (!headroom.ok) return headroom; + try { + return ok(parseEvidence(loaded.value)); + } catch (cause: unknown) { + if (cause instanceof z.ZodError) + return err( + new EvidenceFileError("read", "wrong-kind", { + path, + cause, + contentIssues: projectInputIssues(cause.issues, loaded.value), + userMessage: + "The selected file does not contain valid Evidence. Repair or recreate the file and try again.", + }), + ); + if (cause instanceof TypeError) + return err( + new EvidenceIntegrityError(cause.message, { + cause, + userMessage: cause.message, + }), + ); + throw cause; + } +}; diff --git a/src/cli/analysisViewCommands.ts b/src/cli/analysisViewCommands.ts index 979dbc30ce..4d00e5bb13 100644 --- a/src/cli/analysisViewCommands.ts +++ b/src/cli/analysisViewCommands.ts @@ -1,6 +1,6 @@ import { z } from "incur"; -import { inspectAnalysisViewValidated } from "../application/analysisView/AnalysisViewService.js"; +import { inspectAnalysisViewValidatedWithFiles } from "../application/analysisView/AnalysisViewService.js"; import { readEvidenceBundle } from "../application/EvidenceBundleFiles.js"; import { inspectEvidenceBundle } from "../application/investigation/InspectEvidenceBundle.js"; import { @@ -8,7 +8,7 @@ import { importEvidenceBundleInputSchema, } from "../contracts/sessionToolSchemas.js"; import { CLI_COMMANDS } from "../cliCommandNames.js"; -import { parseCliJsonInput } from "../cliJsonInput.js"; +import { parseCliJsonInput, resolveCliJsonPaths } from "../cliJsonInput.js"; import { logCliCommand } from "../cliLogging.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; @@ -82,7 +82,9 @@ export const registerAnalysisViewCommands = ( CLI_COMMANDS.inspectAnalysisView, ); if (!input.ok) return input.error; - const parsed = inspectAnalysisViewInputSchema.safeParse(input.value); + // An evidence-file path is resolved against the operator's working directory. + const value = resolveCliJsonPaths(input.value, [["source", "path"]]); + const parsed = inspectAnalysisViewInputSchema.safeParse(value); if (!parsed.success) return { error: "Application workflow failed", @@ -90,11 +92,11 @@ export const registerAnalysisViewCommands = ( analysisInputErrorFromIssues( CLI_COMMANDS.inspectAnalysisView, parsed.error.issues, - input.value, + value, ), ), }; - const result = inspectAnalysisViewValidated(parsed.data); + const result = await inspectAnalysisViewValidatedWithFiles(parsed.data); return result.ok ? jsonValueSchema.parse(result.value) : { diff --git a/src/contracts/analysisViewToolContracts.ts b/src/contracts/analysisViewToolContracts.ts index 26a43b563f..e904022e8b 100644 --- a/src/contracts/analysisViewToolContracts.ts +++ b/src/contracts/analysisViewToolContracts.ts @@ -17,7 +17,7 @@ export const ANALYSIS_VIEW_TOOL_CONTRACTS = [ ...toolContractMetadata("inspect_analysis_view"), kind: "application", description: - 'Inspect selected views of completed binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_macho, decode_interface_builder, inspect_evm_interface, inspect_managed_artifact, inspect_managed_members, inspect_android_package, inspect_dart_aot, inspect_il2cpp_metadata or inspect_web_network_capture Evidence without repeating analysis. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. The environment-manifest view projects the same HAR requests on the primary origin (most JSON-like records) into a replayable tool surface: view {kind: "environment-manifest", template_rules: ["after-uid-literal", "opaque-token", "sibling-variance"], include_query_parameter_names: true}. Each tool has a deterministic name, a path template and parameters from the four format rules plus the selected opt-in rules, observed query parameter names, a value-free body shape for POST/PUT/PATCH, status and content-type sets and record ordinals; records are pointers into the parent, excluded groups carry a reason and three fixed unknowns are always stated. Templates, parameters and names are inferences; the view copies no header, cookie or body values and refuses parents inspected with sensitive values. A capture_browser_scenario parent that selected response bodies or header values projects the same way, one record per network transaction in request order; it relies on capture-time redaction of declared secrets instead of that refusal, and a transaction whose URL was redacted is left unrepresented. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. For layout, JavaScript, inventory, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Typed module references use view {kind: "javascript", collection: "modules", projection: "references", where: {path: "main.js", kind: "javascript-module"}, offset: 0, limit: 64}; where matches exact retained strings (null selects unavailable paths). Rows include canonical identity, retained source-range count and observation support states. Source coverage and selection remain explicit; exhausting matching rows does not prove runtime loading or complete analysis. For exact file/native facts and processing gaps, use view {kind: "javascript-facts", where: {paths: ["package.json"]}, property_keys: ["name", "operation", "analysis_status"]}. Rows page original observations with canonical identities and Evidence; missing properties remain omitted. Paths match recorded artifact-path/source-range locations; optional kinds filter nodes. To discover field names without values, opt into include_available_property_keys: true with property_keys: []. Canonical identity and Evidence are separate from properties; keep each node_id/observation_id pair together when composing results. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions. Occurrences accept exact logical_paths and occurrence_ids; nodes accept artifact_ids. Batch filters intersect (omitted means all available, [] means none); fields selects columns while retaining identity links. source_complete describes retained graph membership; source_coverage describes producer coverage, including unknown encrypted PCK directories. Page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. Interface Builder views use kind interface-builder and collection documents/nodes/edges/coverage; nodes and edges accept an exact document relative_path. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Managed member summaries keep identity, selection with unmatched names, coverage and per-collection counts; managed-members pages use collection types/exported_types/fields/methods/member_refs/call_edges/field_accesses/user_strings and carry the source selection and coverage. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. IL2CPP metadata views use kind il2cpp-metadata and collection sections/images/types/methods, with exact indices/names, fields and paging; types also select image_indices/namespaces/declaring_types and methods select declaring_types. Rows always keep their original index scoped to the parent artifact digest. Desktop directory views use kind desktop-application and collection files/omissions/candidates/components/next_probes, with fields and pagination; files and omissions accept exact paths or their stable IDs, and candidates accept families. Source coverage and stable row IDs remain available. Native function Evidence requires kind native: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', + 'Inspect selected views of completed binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_macho, decode_interface_builder, inspect_evm_interface, inspect_managed_artifact, inspect_managed_members, inspect_android_package, inspect_dart_aot, inspect_il2cpp_metadata or inspect_web_network_capture Evidence without repeating analysis. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. The environment-manifest view projects the same HAR requests on the primary origin (most JSON-like records) into a replayable tool surface: view {kind: "environment-manifest", template_rules: ["after-uid-literal", "opaque-token", "sibling-variance"], include_query_parameter_names: true}. Each tool has a deterministic name, a path template and parameters from the four format rules plus the selected opt-in rules, observed query parameter names, a value-free body shape for POST/PUT/PATCH, status and content-type sets and record ordinals; records are pointers into the parent, excluded groups carry a reason and three fixed unknowns are always stated. Templates, parameters and names are inferences; the view copies no header, cookie or body values and refuses parents inspected with sensitive values. A capture_browser_scenario parent that selected response bodies or header values projects the same way, one record per network transaction in request order; it relies on capture-time redaction of declared secrets instead of that refusal, and a transaction whose URL was redacted is left unrepresented. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. A completed Evidence JSON file on this host uses {kind: "evidence-file", path: ""}: REA reads and authenticates it, then projects it like retained Evidence, so a large capture is bounded by the file reader rather than by inline-input accounting. For layout, JavaScript, inventory, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Typed module references use view {kind: "javascript", collection: "modules", projection: "references", where: {path: "main.js", kind: "javascript-module"}, offset: 0, limit: 64}; where matches exact retained strings (null selects unavailable paths). Rows include canonical identity, retained source-range count and observation support states. Source coverage and selection remain explicit; exhausting matching rows does not prove runtime loading or complete analysis. For exact file/native facts and processing gaps, use view {kind: "javascript-facts", where: {paths: ["package.json"]}, property_keys: ["name", "operation", "analysis_status"]}. Rows page original observations with canonical identities and Evidence; missing properties remain omitted. Paths match recorded artifact-path/source-range locations; optional kinds filter nodes. To discover field names without values, opt into include_available_property_keys: true with property_keys: []. Canonical identity and Evidence are separate from properties; keep each node_id/observation_id pair together when composing results. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions. Occurrences accept exact logical_paths and occurrence_ids; nodes accept artifact_ids. Batch filters intersect (omitted means all available, [] means none); fields selects columns while retaining identity links. source_complete describes retained graph membership; source_coverage describes producer coverage, including unknown encrypted PCK directories. Page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. Interface Builder views use kind interface-builder and collection documents/nodes/edges/coverage; nodes and edges accept an exact document relative_path. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Managed member summaries keep identity, selection with unmatched names, coverage and per-collection counts; managed-members pages use collection types/exported_types/fields/methods/member_refs/call_edges/field_accesses/user_strings and carry the source selection and coverage. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. IL2CPP metadata views use kind il2cpp-metadata and collection sections/images/types/methods, with exact indices/names, fields and paging; types also select image_indices/namespaces/declaring_types and methods select declaring_types. Rows always keep their original index scoped to the parent artifact digest. Desktop directory views use kind desktop-application and collection files/omissions/candidates/components/next_probes, with fields and pagination; files and omissions accept exact paths or their stable IDs, and candidates accept families. Source coverage and stable row IDs remain available. Native function Evidence requires kind native: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', inputSchema: inspectAnalysisViewInputSchema, outputSchema: evidenceResultOf(analysisViewResultSchema), examples: [ diff --git a/src/domain/analysisErrorProjection.ts b/src/domain/analysisErrorProjection.ts index 58afc52df5..4060a560df 100644 --- a/src/domain/analysisErrorProjection.ts +++ b/src/domain/analysisErrorProjection.ts @@ -350,6 +350,14 @@ const artifactStateErrorDetails = ( operation: error.operation, reason: error.reason, ...(error.path === undefined ? {} : { path: error.path }), + ...(error.contentIssues === undefined + ? {} + : { + content_issues: error.contentIssues.map((issue) => ({ + ...issue, + path: [...issue.path], + })), + }), ...(error.published === undefined ? {} : { diff --git a/src/domain/analysisView/EnvironmentManifestAccumulator.ts b/src/domain/analysisView/EnvironmentManifestAccumulator.ts index aa4f325c6a..ff59a3410b 100644 --- a/src/domain/analysisView/EnvironmentManifestAccumulator.ts +++ b/src/domain/analysisView/EnvironmentManifestAccumulator.ts @@ -439,6 +439,22 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula }); } + /** One limitation per selected sibling rule when a route is too long for it to compare. */ + #segmentLimitLimitations(toolGroups: readonly ToolGroup[]): string[] { + if ( + !toolGroups.some( + (group) => group.parts.length > SIBLING_VARIANCE_SEGMENT_LIMIT, + ) + ) + return []; + return (["sibling-variance", "sibling-pair-variance"] as const) + .filter((rule) => this.#rules.has(rule)) + .map( + (rule) => + `Routes with more than ${String(SIBLING_VARIANCE_SEGMENT_LIMIT)} segments are not compared by ${rule}; they keep their grouped templates.`, + ); + } + /** Project the complete manifest; tools, records and exclusions are sorted deterministically. */ finish(): Result { const primary = this.#primaryOrigin(); @@ -494,14 +510,7 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula "A top-level __proto__ key in a recorded JSON body is omitted from body_shape; it cannot be an own property of the published object.", ] : []), - ...(this.#rules.has("sibling-variance") && - toolGroups.some( - (group) => group.parts.length > SIBLING_VARIANCE_SEGMENT_LIMIT, - ) - ? [ - `Routes with more than ${String(SIBLING_VARIANCE_SEGMENT_LIMIT)} segments are not compared by sibling-variance; they keep their grouped templates.`, - ] - : []), + ...this.#segmentLimitLimitations(toolGroups), ...(this.#unrepresented > 0 ? [ `${String(this.#unrepresented)} retained records without a usable request line are not represented in the manifest.`, diff --git a/src/domain/analysisView/WebNetworkProjectionAccumulator.ts b/src/domain/analysisView/WebNetworkProjectionAccumulator.ts index aa867eb267..d9c48b7070 100644 --- a/src/domain/analysisView/WebNetworkProjectionAccumulator.ts +++ b/src/domain/analysisView/WebNetworkProjectionAccumulator.ts @@ -63,18 +63,17 @@ export abstract class WebNetworkProjectionAccumulator { } /** Reserve schema output and validation scratch while the borrowed parent stays live. */ - admitCaptureClone( - value: unknown, - options?: { readonly copiedStrings: boolean }, - ): Result { - return admitAnalysisViewData( - value, - false, - (bytes) => (this.retain(bytes) ? ok(undefined) : this.overBudget()), - options?.copiedStrings, + admitCaptureClone(value: unknown): Result { + return admitAnalysisViewData(value, false, (bytes) => + this.retain(bytes) ? ok(undefined) : this.overBudget(), ); } + /** Reserve bytes a view derives from the borrowed parent before it allocates them. */ + admitDerivedBytes(bytes: number): Result { + return this.retain(bytes) ? ok(undefined) : this.overBudget(); + } + /** Admit the complete portable envelope before cloning or authenticating it. */ admitInlineEvidence(value: unknown): Result { return admitAnalysisViewData(value, true, (bytes) => diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index 2c539f284b..58cb333d48 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -1,3 +1,6 @@ +import { isAbsolute } from "node:path"; + +import { localPathStringSchema } from "../localPath.js"; import { z } from "zod"; import { desktopViewRequestSchema, @@ -250,7 +253,7 @@ const inlineEvidenceInputSchema = z.unknown().nonoptional(); inlineEvidenceInputSchema._zod.processJSONSchema = evidenceSchema._zod.processJSONSchema; -/** Unauthenticated inline input or an exact same-session retained reference. */ +/** Unauthenticated inline input, an exact same-session retained reference, or one local Evidence file. */ export const analysisViewSourceSchema = z.discriminatedUnion("kind", [ z.strictObject({ kind: z.literal("retained-evidence"), @@ -260,6 +263,17 @@ export const analysisViewSourceSchema = z.discriminatedUnion("kind", [ kind: z.literal("inline"), evidence: inlineEvidenceInputSchema, }), + z.strictObject({ + kind: z.literal("evidence-file"), + path: localPathStringSchema + .refine(isAbsolute, { + message: + "path must be an absolute local filesystem path (for example /tmp/capture.json or C:\\rea\\capture.json)", + }) + .describe( + "Absolute local path of one completed Evidence JSON file. REA reads and authenticates it, then projects it like retained Evidence, so large captures are not charged as inline input.", + ), + }), ]); /** Public inspect_analysis_view input. */ diff --git a/src/domain/analysisView/analysisViewDataAdmission.ts b/src/domain/analysisView/analysisViewDataAdmission.ts index a4847c0ae2..ad39d23418 100644 --- a/src/domain/analysisView/analysisViewDataAdmission.ts +++ b/src/domain/analysisView/analysisViewDataAdmission.ts @@ -10,14 +10,12 @@ const MEMBER_BYTES = 256; const CONTAINER_BYTES = 512; const textBytes = (length: number): number => 32 + length * 14; -/** Inspect borrowed JSON without accessors and reserve copied or transformed representations. */ +/** Inspect borrowed data without invoking accessors; only API-surface reserves bytes. */ export const admitAnalysisViewData = ( value: unknown, inline: boolean, reserve?: (bytes: number) => Result, - copiedStrings = false, ): Result => { - const countStrings = inline || copiedStrings; const ancestors = new Set(); // A single bounded stack preserves diagnostics without per-node path copies. const path: (string | number)[] = inline @@ -45,9 +43,7 @@ export const admitAnalysisViewData = ( return invalid(); const member = reserve?.( MEMBER_BYTES + - (countStrings && typeof item === "string" - ? textBytes(item.length) - : 0) + + (inline && typeof item === "string" ? textBytes(item.length) : 0) + (typeof item === "object" && item !== null ? CONTAINER_BYTES : 0), ); if (member !== undefined && !member.ok) return member; @@ -87,7 +83,7 @@ export const admitAnalysisViewData = ( // Zod reads named fields even when they are non-enumerable. for (const key of Object.getOwnPropertyNames(item)) { // Authentication hashes strings and keys alongside the schema clone. - const keyAdmission = countStrings + const keyAdmission = inline ? reserve?.(textBytes(key.length)) : undefined; if (keyAdmission !== undefined && !keyAdmission.ok) return keyAdmission; diff --git a/src/domain/analysisView/environmentManifestRules.ts b/src/domain/analysisView/environmentManifestRules.ts index 18b2d65518..d5bc200ae1 100644 --- a/src/domain/analysisView/environmentManifestRules.ts +++ b/src/domain/analysisView/environmentManifestRules.ts @@ -9,6 +9,7 @@ export const TEMPLATE_RULES = [ "after-uid-literal", "opaque-token", "sibling-variance", + "sibling-pair-variance", ] as const; export type TemplateRule = (typeof TEMPLATE_RULES)[number]; export type ManifestRule = RouteRule | TemplateRule; @@ -35,6 +36,8 @@ export interface ParamPart { readonly kind: "param"; readonly rule: ManifestRule; readonly values: ReadonlySet; + /** Which half of a sibling-pair-variance pair; fixed when the pair forms so later merges cannot change it. */ + readonly pairRole?: "namespace" | "name"; } export type RoutePart = LiteralPart | ParamPart; @@ -113,16 +116,18 @@ const mergeGroups = ( parts: shape.parts.map((part, index) => part.kind === "literal" ? part - : param( - part.rule, - groups.flatMap((group) => { - const sibling = group.parts[index]; - if (sibling === undefined) return []; - return sibling.kind === "param" - ? [...sibling.values] - : [sibling.value]; - }), - ), + : { + ...part, + values: new Set( + groups.flatMap((group) => { + const sibling = group.parts[index]; + if (sibling === undefined) return []; + return sibling.kind === "param" + ? [...sibling.values] + : [sibling.value]; + }), + ), + }, ), positions: groups .flatMap((group) => group.positions) @@ -287,6 +292,105 @@ const mergeSiblingVariance = (groups: readonly ToolGroup[]): ToolGroup[] => { return [...current, ...beyondLimit]; }; +/** Fewest distinct literal pairs that make two adjacent positions one templated pair. */ +export const SIBLING_PAIR_MINIMUM = 3; + +/** + * Both positions are literals that are neither stop words nor versions, right after a plural literal + * and before another literal: a trailing pair such as `/users/alice/followers` names a sub-resource. + */ +const pairCandidate = (group: ToolGroup, position: number): boolean => { + const previous = group.parts[position - 1]; + const first = group.parts[position]; + const second = group.parts[position + 1]; + return ( + previous?.kind === "literal" && + previous.value.endsWith("s") && + !STOP_WORDS.has(previous.value) && + first?.kind === "literal" && + second?.kind === "literal" && + group.parts[position + 2]?.kind === "literal" && + [first.value, second.value].every( + (value) => !STOP_WORDS.has(value) && !VERSION_TOKEN.test(value), + ) + ); +}; + +/** Merge one bucket when its pairs are distinct enough and both positions vary; otherwise keep its tools. */ +const mergePairBucket = ( + bucket: readonly ToolGroup[], + position: number, +): ToolGroup[] => { + const pairs = new Set( + bucket.map((group) => + compositeKey([ + literalAt(group, position), + literalAt(group, position + 1), + ]), + ), + ); + const firsts = new Set(bucket.map((group) => literalAt(group, position))); + const seconds = new Set( + bucket.map((group) => literalAt(group, position + 1)), + ); + if (pairs.size < SIBLING_PAIR_MINIMUM || firsts.size < 2 || seconds.size < 2) + return [...bucket]; + const first = earliest(bucket); + const shape = withParts( + first, + first.parts.map((part, index) => + index === position || index === position + 1 + ? { + ...param("sibling-pair-variance", [literalAt(first, index)]), + pairRole: index === position ? "namespace" : "name", + } + : part, + ), + ); + return [ + mergeGroups( + bucket.map((group) => (group === first ? shape : group)), + shape, + ), + ]; +}; + +/** Merge groups that differ in two adjacent literals after a plural literal, such as `/repos/{owner}/{repo}/…`. */ +const mergeSiblingPairs = (groups: readonly ToolGroup[]): ToolGroup[] => { + const beyondLimit = groups.filter( + (group) => group.parts.length > SIBLING_VARIANCE_SEGMENT_LIMIT, + ); + let current = groups.filter( + (group) => group.parts.length <= SIBLING_VARIANCE_SEGMENT_LIMIT, + ); + const longest = Math.max(0, ...current.map((group) => group.parts.length)); + for (let position = 2; position + 1 < longest; position += 1) { + const buckets = new Map(); + const output: ToolGroup[] = []; + for (const group of current) { + if (!pairCandidate(group, position)) { + output.push(group); + continue; + } + const key = compositeKey([ + group.method, + group.origin, + group.parts.length, + ...group.parts.map((part, index) => + index === position || index === position + 1 ? null : partKey(part), + ), + ]); + const bucket = buckets.get(key); + if (bucket === undefined) buckets.set(key, [group]); + else bucket.push(group); + } + for (const bucket of buckets.values()) + output.push(...mergePairBucket(bucket, position)); + current = output; + } + return [...current, ...beyondLimit]; +}; + /** Apply the selected opt-in rules in fixed order; every merge is an inference. */ export const applyTemplateRules = ( groups: readonly ToolGroup[], @@ -302,6 +406,9 @@ export const applyTemplateRules = ( withParts(group, templateOpaqueTokens(group.parts)), ); current = rebucket(current); + // Pairs merge first: a single-position merge could otherwise template one + // half of a pair and leave too few literal pairs to recognize. + if (rules.has("sibling-pair-variance")) current = mergeSiblingPairs(current); if (rules.has("sibling-variance")) current = mergeSiblingVariance(current); return current; }; @@ -343,6 +450,14 @@ const paramBaseName = (parts: readonly RoutePart[], index: number): string => { : ""; return stem.length > 0 ? `${stem}_id` : "id"; } + case "sibling-pair-variance": { + // The role was fixed when the pair formed; the plural literal precedes the pair's first half. + const role = part.pairRole ?? "namespace"; + const plural = parts[role === "name" ? index - 2 : index - 1]; + const stem = + plural?.kind === "literal" ? sanitizeToken(singular(plural.value)) : ""; + return stem.length > 0 ? `${stem}_${role}` : role; + } } }; diff --git a/src/domain/analysisView/environmentManifestView.test.ts b/src/domain/analysisView/environmentManifestView.test.ts index 2c2367f0ba..1fdd211ab3 100644 --- a/src/domain/analysisView/environmentManifestView.test.ts +++ b/src/domain/analysisView/environmentManifestView.test.ts @@ -540,6 +540,138 @@ it("merges identifier-like siblings at index two or later and never folds stop w ).toEqual(["blue-two", "green-three", "red-one"]); }); +it("templates two adjacent literals after a plural literal as a pair when three pairs vary in both positions", () => { + const projected = manifest( + ["sibling-pair-variance"], + syntheticCapture([ + [ + "GET", + "https://g.test/api/v1/repos/lighthouse-rota/lamp-room-log/issues", + ], + ["GET", "https://g.test/api/v1/repos/granite-ledger/ledger-core/issues"], + [ + "GET", + "https://g.test/api/v1/repos/blue-heron-kitchen/recipe-book/issues", + ], + [ + "GET", + "https://g.test/api/v1/repos/lighthouse-rota/tide-tables/issues/4", + ], + [ + "GET", + "https://g.test/api/v1/repos/granite-ledger/audit-trail/issues/9", + ], + [ + "GET", + "https://g.test/api/v1/repos/saltmarsh-depot/crate-tracker/issues/2", + ], + ["GET", "https://g.test/api/v1/teams/one/alpha/members"], + ["GET", "https://g.test/api/v1/teams/two/beta/members"], + ["GET", "https://g.test/api/v1/orgs/acme/x1/hooks"], + ["GET", "https://g.test/api/v1/orgs/acme/x2/hooks"], + ["GET", "https://g.test/api/v1/orgs/acme/x3/hooks"], + ]), + ); + expect(projected.tools.map((tool) => tool.path_template)).toEqual([ + "/api/v1/orgs/acme/x1/hooks", + "/api/v1/orgs/acme/x2/hooks", + "/api/v1/orgs/acme/x3/hooks", + "/api/v1/repos/{repo_namespace}/{repo_name}/issues", + "/api/v1/repos/{repo_namespace}/{repo_name}/issues/{numeric}", + "/api/v1/teams/one/alpha/members", + "/api/v1/teams/two/beta/members", + ]); + expect( + projected.tools.find( + (tool) => + tool.path_template === + "/api/v1/repos/{repo_namespace}/{repo_name}/issues", + ), + ).toMatchObject({ + name: "get_repos_issues_by_repo_namespace_repo_name", + params: [ + { + name: "repo_namespace", + rule: "sibling-pair-variance", + observed_values: [ + "blue-heron-kitchen", + "granite-ledger", + "lighthouse-rota", + ], + }, + { + name: "repo_name", + rule: "sibling-pair-variance", + observed_values: ["lamp-room-log", "ledger-core", "recipe-book"], + }, + ], + record_ordinals: [0, 1, 2], + }); + expect( + projected.tools + .find((tool) => tool.path_template.endsWith("/issues/{numeric}")) + ?.params.map((param) => [param.name, param.rule]), + ).toEqual([ + ["repo_namespace", "sibling-pair-variance"], + ["repo_name", "sibling-pair-variance"], + ["id", "numeric"], + ]); +}); + +it("keeps a trailing pair literal because its second half names a sub-resource", () => { + const projected = manifest( + ["sibling-pair-variance"], + syntheticCapture([ + ["GET", "https://g.test/api/v1/users/alice/followers"], + ["GET", "https://g.test/api/v1/users/bob/following"], + ["GET", "https://g.test/api/v1/users/carol/starred"], + ]), + ); + expect(projected.tools.map((tool) => tool.path_template)).toEqual([ + "/api/v1/users/alice/followers", + "/api/v1/users/bob/following", + "/api/v1/users/carol/starred", + ]); +}); + +it("merges pairs before sibling-variance so the two rules compose", () => { + const routes = syntheticCapture([ + ["GET", "https://g.test/api/repos/alice-1/common/issues"], + ["GET", "https://g.test/api/repos/bob-2/common/issues"], + ["GET", "https://g.test/api/repos/carol-3/other/issues"], + ["GET", "https://g.test/api/repos/dave-4/third/issues"], + ]); + for (const rules of [ + ["sibling-pair-variance"], + ["sibling-variance", "sibling-pair-variance"], + ] as const) + expect( + manifest([...rules], routes).tools.map((tool) => tool.path_template), + ).toEqual(["/api/repos/{repo_namespace}/{repo_name}/issues"]); +}); + +it("keeps each pair parameter's role when sibling-variance later templates the segment before the pair", () => { + const projected = manifest( + ["sibling-variance", "sibling-pair-variance"], + syntheticCapture([ + ["GET", "https://g.test/api/root/group1s/alice-1/common/issues"], + ["GET", "https://g.test/api/root/group1s/bob-2/ledger/issues"], + ["GET", "https://g.test/api/root/group1s/carol-3/books/issues"], + ["GET", "https://g.test/api/root/group2s/dave-4/x1/issues"], + ["GET", "https://g.test/api/root/group2s/erin-5/x2/issues"], + ["GET", "https://g.test/api/root/group2s/finn-6/x3/issues"], + ]), + ); + expect(projected.tools).toHaveLength(1); + expect( + projected.tools[0]?.params.map((param) => [param.name, param.rule]), + ).toEqual([ + ["root_id", "sibling-variance"], + ["namespace", "sibling-pair-variance"], + ["name", "sibling-pair-variance"], + ]); +}); + it("never folds a literal sibling into a format-rule parameter", () => { const projected = manifest( ALL_RULES, diff --git a/src/domain/analysisView/environmentManifestView.ts b/src/domain/analysisView/environmentManifestView.ts index 8d367b4ca3..5204274f17 100644 --- a/src/domain/analysisView/environmentManifestView.ts +++ b/src/domain/analysisView/environmentManifestView.ts @@ -25,7 +25,7 @@ export const environmentManifestViewRequestSchema = z.strictObject({ .meta({ uniqueItems: true }) .default([]) .describe( - "Opt-in template rules applied after the four format rules, each listed at most once: after-uid-literal templates the segment following a literal uid, opaque-token templates long mixed alphanumeric segments, sibling-variance merges groups that differ in one identifier-like segment. Literal sibling folding into existing opt-in parameters also requires sibling-variance. Merged parameters are inferences.", + "Opt-in template rules applied after the four format rules, each listed at most once: after-uid-literal templates the segment following a literal uid, opaque-token templates long mixed alphanumeric segments, sibling-variance merges groups that differ in one identifier-like segment, sibling-pair-variance merges groups that differ in two adjacent literals between a plural literal and another literal when at least three distinct pairs vary in both. Literal sibling folding into existing opt-in parameters also requires sibling-variance. Merged parameters are inferences.", ), include_query_parameter_names: z .boolean() diff --git a/src/domain/analysisView/scenarioCaptureRecords.test.ts b/src/domain/analysisView/scenarioCaptureRecords.test.ts index 53eed91940..9579dd118b 100644 --- a/src/domain/analysisView/scenarioCaptureRecords.test.ts +++ b/src/domain/analysisView/scenarioCaptureRecords.test.ts @@ -170,6 +170,53 @@ it("reserves retained request-body expansion before schema decoding or projectio } }); +it("reserves request text even when another event's body was not captured", () => { + const capture = scenarioCaptureFixture([ + { + method: "GET", + url: "/api/items", + status: 200, + mediaType: "application/json", + body: "[]", + }, + { + method: "POST", + url: "/api/items", + status: 201, + mediaType: "application/json", + body: "{}", + requestMediaType: "application/json", + requestBody: JSON.stringify({ value: "x".repeat(2 * 1024 * 1024) }), + }, + ]); + // The first response's body has no content, as a body that was not requested has. + const items = capture.events.items.map((event) => + event.kind === "network-content" && event.transaction_id === "request-1" + ? { ...event, body: { state: "not_requested" as const } } + : event, + ); + const decoder = vi.spyOn(TextDecoder.prototype, "decode"); + try { + // The capture's containers fit this budget; only its request text does not. + const projected = admitScenarioCaptureParent( + { + evidenceId: `ev_${"a".repeat(64)}`, + operation: "capture_browser_scenario", + normalizedResult: { ...capture, events: { ...capture.events, items } }, + limitations: [], + }, + new EnvironmentManifestAccumulator(8 * 1024 * 1024), + ); + expect(projected).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError", resource: "memory" }, + }); + expect(decoder).not.toHaveBeenCalled(); + } finally { + decoder.mockRestore(); + } +}); + it("does not decode request text when projecting replay records", () => { const capture = scenarioCaptureFixture(); const decoder = vi.spyOn(TextDecoder.prototype, "decode"); diff --git a/src/domain/analysisView/scenarioCaptureRecords.ts b/src/domain/analysisView/scenarioCaptureRecords.ts index df4378d1d7..19e108417d 100644 --- a/src/domain/analysisView/scenarioCaptureRecords.ts +++ b/src/domain/analysisView/scenarioCaptureRecords.ts @@ -15,6 +15,7 @@ import type { WebNetworkCaptureRecord } from "../webNetworkCapture.js"; import { analysisViewInputError } from "./analysisViewErrors.js"; import type { AnalysisViewParent } from "./analysisView.js"; import type { WebNetworkProjectionAccumulator } from "./WebNetworkProjectionAccumulator.js"; +import { textBytes } from "./webNetworkRouteTemplates.js"; type RequestEvent = Extract; type ResponseEvent = Extract; @@ -262,17 +263,49 @@ export const scenarioCaptureRecords = ( }; }; +/** An own data member of admitted parent data; the clone admission already refused accessors. */ +const member = (value: unknown, key: string): unknown => + typeof value === "object" && value !== null && Object.hasOwn(value, key) + ? (value as Readonly>)[key] + : undefined; + +/** + * Bytes the manifest derives from a borrowed scenario: validation decodes and re-encodes one body at + * a time (two bytes per base64 character), and projection keeps the decoded text of every request + * body. Response bodies stay borrowed, as HAR response text does. + */ +const derivedTextBytes = (value: unknown): number => { + const items = member(member(value, "events"), "items"); + let largestBody = 0; + let requestText = 0; + // Each event is read on its own, so a body without content leaves the others counted. + for (const event of Array.isArray(items) ? items : []) { + const content = member(member(event, "body"), "content"); + if ( + member(event, "kind") !== "network-content" || + typeof content !== "string" + ) + continue; + largestBody = Math.max(largestBody, content.length); + if (member(event, "phase") === "request") + requestText += textBytes(Math.ceil((content.length * 3) / 4)); + } + return 64 + largestBody * 2 + requestText; +}; + /** Admit a capture_browser_scenario parent whose network content identifies response media types. */ export const admitScenarioCaptureParent = ( parent: AnalysisViewParent, accumulator: WebNetworkProjectionAccumulator, ): Result => { - // Body validation decodes base64, and projection constructs UTF-8 request text. - // Reserve their expanded strings before either allocation, even for retained parents. - const admitted = accumulator.admitCaptureClone(parent.normalizedResult, { - copiedStrings: true, - }); + const admitted = accumulator.admitCaptureClone(parent.normalizedResult); if (!admitted.ok) return admitted; + // Reserve body validation and request text before either allocation, even for a parent read from + // a file, without charging the borrowed response bodies a large capture is made of. + const derived = accumulator.admitDerivedBytes( + derivedTextBytes(parent.normalizedResult), + ); + if (!derived.ok) return derived; const parsed = browserScenarioCaptureSchema.safeParse( parent.normalizedResult, ); diff --git a/src/domain/evidenceErrors.ts b/src/domain/evidenceErrors.ts index 1aae747fd9..6c173f5c00 100644 --- a/src/domain/evidenceErrors.ts +++ b/src/domain/evidenceErrors.ts @@ -2,6 +2,7 @@ import { AnalysisError, type AnalysisErrorOptions, } from "./analysisErrorBase.js"; +import type { AnalysisInputIssue } from "./analysisErrorCore.js"; /** Receipt retained when publication succeeded but staging cleanup did not. */ export interface PublishedEvidenceFile { @@ -51,6 +52,8 @@ export class EvidenceFileError extends AnalysisError { readonly path: string | undefined; /** Present only when a write committed before cleanup failed. */ readonly published: PublishedEvidenceFile | undefined; + /** Schema issues relative to the selected file's JSON root, never the request envelope. */ + readonly contentIssues: readonly AnalysisInputIssue[] | undefined; constructor( readonly operation: "read" | "write", @@ -65,11 +68,13 @@ export class EvidenceFileError extends AnalysisError { readonly path?: string; readonly published?: PublishedEvidenceFile; readonly userMessage?: string; + readonly contentIssues?: readonly AnalysisInputIssue[]; }, ) { super(`Evidence bundle ${operation} failed: ${reason}`, options); this.path = options?.path; this.published = options?.published; this.userMessage = options?.userMessage; + this.contentIssues = options?.contentIssues; } } diff --git a/src/server/createServer.ts b/src/server/createServer.ts index 301935670a..985e2e075a 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -676,6 +676,7 @@ const registerConfiguredAnalysisTools = ( il2cppMetadata, peResources, webNetworkCapture, + artifactResources, }: ServerToolContext, android: AndroidAnalysisPort, wasmArtifact: Pick, @@ -713,6 +714,7 @@ const registerConfiguredAnalysisTools = ( evidenceById, recordEvidence, withAdmittedAnalysis, + artifactResources, ); registerPeResourcesTool( server, diff --git a/src/server/registerAnalysisViewTool.ts b/src/server/registerAnalysisViewTool.ts index d427c0cb42..253eae45ce 100644 --- a/src/server/registerAnalysisViewTool.ts +++ b/src/server/registerAnalysisViewTool.ts @@ -1,6 +1,8 @@ import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; -import { inspectAnalysisView } from "../application/analysisView/AnalysisViewService.js"; +import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; + +import { inspectAnalysisViewWithFiles } from "../application/analysisView/AnalysisViewService.js"; import type { EvidenceLookup } from "../application/EvidenceInputResolver.js"; import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; import { toolContract } from "../contracts/toolContracts.js"; @@ -19,12 +21,19 @@ export const registerAnalysisViewTool = ( evidenceById?: EvidenceLookup, recordEvidence?: EvidenceWriter["recordEvidence"], withAdmittedAnalysis?: WithAdmittedAnalysis, + artifactResources?: ArtifactResourceScope, ): void => { const contract = toolContract("inspect_analysis_view"); const registration = toolRegistrationOptions(contract); const handler = createEvidenceToolHandler( server, - (input) => Promise.resolve(inspectAnalysisView(input, evidenceById)), + (input, options) => + inspectAnalysisViewWithFiles(input, evidenceById, { + ...(options.signal === undefined ? {} : { signal: options.signal }), + ...(artifactResources === undefined + ? {} + : { resources: artifactResources }), + }), logger, recordEvidence, withAdmittedAnalysis, diff --git a/tests/boundary/cli/analysisViewCli.test.ts b/tests/boundary/cli/analysisViewCli.test.ts index a8f8a83266..ca37b4f6af 100644 --- a/tests/boundary/cli/analysisViewCli.test.ts +++ b/tests/boundary/cli/analysisViewCli.test.ts @@ -547,6 +547,49 @@ cliTest( }, ); +cliTest( + "projects an environment manifest from an Evidence file through the compiled CLI", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-evidence-file-cli-"); + const capturePath = join(root, "capture.json"); + const parent = analysisViewCaptureEvidence( + environmentManifestCaptureFixture(), + ); + await writeFile(capturePath, JSON.stringify(parent)); + const requestPath = join(root, "view.json"); + const view = { + kind: "environment-manifest", + template_rules: ["opaque-token", "sibling-pair-variance"], + } as const; + await writeFile( + requestPath, + JSON.stringify({ + source: { kind: "evidence-file", path: capturePath }, + view, + }), + ); + const result = await cli.run({ + arguments: ["inspect-analysis-view", requestPath, "--json"], + }); + expect(result.exitCode, result.stdout).toBe(0); + const evidence = parseEvidence(result.json); + expect(evidence.parameters).toMatchObject({ + parent_evidence_id: parent.evidence_id, + }); + // The file source projects exactly what the same Evidence projects inline. + const inline = inspectAnalysisView({ + source: { kind: "inline", evidence: parent }, + view, + }); + if (!inline.ok) throw inline.error; + expect(evidence.normalized_result).toEqual(inline.value.normalized_result); + expect(evidence.normalized_result).toMatchObject({ + kind: "environment-manifest", + origin: "http://grafana.example", + }); + }, +); + cliTest( "projects an environment manifest through the compiled CLI without recorded values", async ({ cli }) => { diff --git a/tests/boundary/mcp/analysisViewMcp.test.ts b/tests/boundary/mcp/analysisViewMcp.test.ts index 4846f1bd21..4457a6c291 100644 --- a/tests/boundary/mcp/analysisViewMcp.test.ts +++ b/tests/boundary/mcp/analysisViewMcp.test.ts @@ -1167,7 +1167,7 @@ it.each(PUBLISHED_EXAMPLES)( }, ); -it("projects an environment manifest identically from retained and portable MCP Evidence", async () => { +it("projects an environment manifest identically from retained, portable and file MCP Evidence", async () => { const { client, session } = await connect(); const { ajv, inputSchema, outputSchema } = await advertisedMcpToolSchemas( client, @@ -1176,6 +1176,11 @@ it("projects an environment manifest identically from retained and portable MCP const capture = environmentManifestCaptureFixture(); const parent = analysisViewCaptureEvidence(capture); expect(session.recordEvidence(parent).ok).toBe(true); + const evidencePath = join( + await createTestTempDirectory("rea-evidence-file-mcp-"), + "capture.json", + ); + await writeFile(evidencePath, JSON.stringify(parent)); const view = { kind: "environment-manifest", template_rules: ["after-uid-literal", "opaque-token", "sibling-variance"], @@ -1184,6 +1189,7 @@ it("projects an environment manifest identically from retained and portable MCP for (const source of [ { kind: "inline", evidence: parent }, { kind: "retained-evidence", evidence_id: parent.evidence_id }, + { kind: "evidence-file", path: evidencePath }, ]) { expect(ajv.validate(inputSchema, { source, view })).toBe(true); expect( @@ -1224,6 +1230,34 @@ it("projects an environment manifest identically from retained and portable MCP outputs.push(evidence.normalized_result); } expect(outputs[0]).toEqual(outputs[1]); + expect(outputs[2]).toEqual(outputs[0]); + // JSON Schema cannot express "absolute"; the tool refuses a relative path when called. + const relative = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { kind: "evidence-file", path: "capture.json" }, + view, + }, + }); + expect(parseMcpToolError(relative).error.code).toBe("invalid_request"); + await writeFile(evidencePath, JSON.stringify({ hello: 1 })); + const invalidFile = await client.callTool({ + name: "inspect_analysis_view", + arguments: { source: { kind: "evidence-file", path: evidencePath }, view }, + }); + expect(parseMcpToolError(invalidFile).error).toMatchObject({ + details: { + path: evidencePath, + reason: "wrong-kind", + content_issues: expect.arrayContaining([ + { + path: ["evidence_id"], + reason: "missing_argument", + expected: "string", + }, + ]), + }, + }); const hidden = createEvidence( { path: capture.artifact.path,