From 6d5532a204b338a6c770fcbae61e0d55bda70793 Mon Sep 17 00:00:00 2001 From: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:37:39 +0800 Subject: [PATCH] feat(web): page retained bundle endpoint observations and inferences (#7259) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Page retained web-bundle endpoint observations and inferences through shared CLI/MCP projections, preserving source identities, scoped unknowns, coverage and complete Evidence access. Based on HarsH’s original implementation in #7213 (6b1d5d42). Real Chromium/CLI/SDK verification and whole-task cost limits are recorded in docs/verification/web-bundle-paging-2026-10-26.md. Explicit user authorization permits leaving human-readiness declarations unchecked and bypassing reproduced main failures (ADB listing fixture, package catalog expectation, untouched APK verifier formatting). No human-review attestation is made. Co-authored-by: HarsH --- .../references/javascript-applications.md | 9 + docs/javascript-route-inventory.md | 40 +- .../web-bundle-paging-2026-10-26.md | 138 ++++ .../analysis-efficiency/web-bundle-paging.mjs | 382 +++++++++++ src/cli/browserCommands.ts | 4 +- src/cli/javascriptRouteDelivery.ts | 7 + src/contracts/analysisViewToolContracts.ts | 18 +- src/contracts/browserToolContracts.ts | 15 +- src/domain/analysisView/analysisView.ts | 49 ++ .../analysisView/javascriptRoutesView.ts | 6 +- src/domain/analysisView/webBundleView.test.ts | 641 ++++++++++++++++++ src/domain/analysisView/webBundleView.ts | 395 +++++++++++ tests/boundary/mcp/webBundleViewsMcp.test.ts | 348 ++++++++++ 13 files changed, 2041 insertions(+), 11 deletions(-) create mode 100644 docs/verification/web-bundle-paging-2026-10-26.md create mode 100644 experiments/analysis-efficiency/web-bundle-paging.mjs create mode 100644 src/domain/analysisView/webBundleView.test.ts create mode 100644 src/domain/analysisView/webBundleView.ts create mode 100644 tests/boundary/mcp/webBundleViewsMcp.test.ts diff --git a/.agents/skills/reverse-engineer-anything/references/javascript-applications.md b/.agents/skills/reverse-engineer-anything/references/javascript-applications.md index a074dad75f..191060b90c 100644 --- a/.agents/skills/reverse-engineer-anything/references/javascript-applications.md +++ b/.agents/skills/reverse-engineer-anything/references/javascript-applications.md @@ -78,6 +78,15 @@ classification, and unavailable sources never establish route reachability or the complete application route denominator. See the [route inventory guide](https://github.com/morluto/rea/blob/main/docs/javascript-route-inventory.md). +For captured endpoint observations and inference facts, `analyze_web_bundle` +also accepts `view: {kind: "web-bundle", collection: "endpoints", limit: 32}` +with `detail: "summary"`. Read later `endpoints`, `inferences`, or `summary` +pages from the retained parent. Filter before requesting detail; retain +`source_index` together with parent ID and collection, and request `location` +or `basis` when tracing sources. Page exhaustion does not mean producer +completeness: check `completeness`, structured `source_unknowns`, and limitations. +Endpoint candidates remain inferences rather than confirmed network behavior. + ## Read a module without repeating analysis Use the actual returned parent ID, on the same connection, as `source.evidence_id`. diff --git a/docs/javascript-route-inventory.md b/docs/javascript-route-inventory.md index a7e9901842..bb20b18901 100644 --- a/docs/javascript-route-inventory.md +++ b/docs/javascript-route-inventory.md @@ -25,18 +25,50 @@ Keep the original analysis inputs in that call. For the CLI, pass the same selection as `--view '' --detail summary --evidence_output routes.json`. The complete Evidence bundle is written without overwriting existing files. Unselected delivery retains the existing complete result. A web-bundle summary -requires an explicit route view; application summaries also support the existing +requires an explicit route or web-bundle view; application summaries also support the existing general summary when `view` is omitted. Use the returned `normalized_result.parent_evidence_id` with `inspect_analysis_view` on the same connection. The CLI's exported bundle can be reopened with `inspect-evidence-bundle`; its `records` contain the complete -Evidence. To use `inspect_analysis_view`'s `evidence-file` source, extract the -selected parent record into a separate local Evidence JSON file (that source -accepts one Evidence record, not a bundle). Views do not reopen source files or +Evidence. `inspect_analysis_view`'s `evidence-file` source accepts either one +Evidence record or an exported bundle with the selected `evidence_id`. +Views do not reopen source files or reconnect to a browser. Historical Evidence without `route_inventory` is unmeasured, rather than an empty route set. +## Page captured endpoint observations and inferences + +For `analyze_web_bundle`, select `detail: "summary"` and +`view: {kind: "web-bundle", collection: "endpoints", limit: 32}`. The CLI +accepts the same JSON through `--view`, with `--detail summary` and +`--evidence_output bundle.json`. Query the returned parent with +`inspect_analysis_view` to read `endpoints`, `inferences`, or `summary` without +recapturing the page or repeating JavaScript analysis. + +Both row collections accept exact `indices`, `values`, `script_keys`, `fields`, +`offset`, and `limit`; endpoints also accept `mechanisms`, and inferences accept +`kinds`. Filters combine with AND, and an explicit empty selector matches no +rows. Every row retains its `source_index` in the original collection, including +when other fields are omitted. This identity is scoped to the parent Evidence +and collection. Original ordering and separate occurrences of duplicate values +are preserved; request endpoint `location` or inference `basis` for source details. + +Follow `coverage.next_offset` until `coverage.exhausted`; those fields describe +the selected rows, independently of producer `completeness`. Each response also +preserves structured `source_unknowns`, including `affected_script_keys`, so a +filtered or value-only page cannot hide a source gap. The summary's +`limitation_count` counts distinct parent and report limitations before display +bounding. An `endpoint_candidate` remains an inference, with its original +confidence and basis, and does not prove a backend endpoint or reachability. + +Pages obey the shared 1 MiB projection budget; MCP transport applies its own +wire-size guard. A byte-limited page returns a continuation; an indivisible +oversized record or metadata returns a typed resource error. Select fewer +fields or retrieve the complete retained parent through `get_evidence_bundle`, +`export_evidence_bundle`, or the CLI's exported bundle. Full Evidence is never +silently truncated. + Collections are `summary`, `routes`, `occurrences`, `sources`, and `unknowns`. All row collections support exact batch `where.ids`, field selection, `offset`, and `limit`. An explicit empty filter array matches nothing. Routes also support diff --git a/docs/verification/web-bundle-paging-2026-10-26.md b/docs/verification/web-bundle-paging-2026-10-26.md new file mode 100644 index 0000000000..a81f5e00cf --- /dev/null +++ b/docs/verification/web-bundle-paging-2026-10-26.md @@ -0,0 +1,138 @@ +# Retained web-bundle paging verification + +Issue [#6198](https://github.com/morluto/rea/issues/6198). Runtime source +`a92e07f484e444755c2ee33f355351a28d79f430`, based on main +`8423c5c6d08512568c2a75753b6b7c3a9b6bbd4e`. The original implementation is +HarsH's [#7213](https://github.com/morluto/rea/pull/7213), head +`6b1d5d42c6d6b4dba19295b8e9520cbeb55fc39e`. Subsequent commits update the +verifier, tests and this record; they do not change that runtime implementation. + +## Executed correctness scope + +The public CLI and actual stdio SDK read authentic completed browser Evidence. +Every returned row is compared with the unchanged producer's complete parent, +including original source index, ordering, values, locations, inference kind, +confidence and basis. This validates projection; it does not independently prove +that every producer inference is correct or that an endpoint is reachable. + +Inputs were unmodified Miniflux JavaScript assets at commit +[`d5c529b94ce9a172bd4c22a2b89da068c583783b`](https://github.com/miniflux/v2/tree/d5c529b94ce9a172bd4c22a2b89da068c583783b/internal/ui/static/js), +loaded by installed Chromium through an owned loopback static harness. Four +scripts total 60,395 bytes: `app.js`, `keyboard_handler.js`, `touch_handler.js`, +and `webauthn_handler.js`. The complete parent has five low-confidence endpoint +candidates, zero endpoint observations, and all 6,245 visited AST nodes examined. +The harness initially loaded `app.js` before `KeyboardHandler` and recorded a +page error; all four unmodified scripts were still captured and statically +parsed. This verifies shipped asset analysis, not a running Miniflux backend or +complete application behavior. Nonempty endpoint observations are covered by +the SDK boundary fixture, not claimed as real Miniflux findings. + +The CLI's initial selected analysis returned one inference and exported the +complete parent. Retained SDK paging reproduced all five full inference rows; +subsequent CLI Evidence-file queries reproduced the same shared projection. +The parent used for the equivalent query measurements is +`ev_2c4b96d1fbf9e0c4ec2baf23815adc2599448b459dffb7cc1cb9cf8a75526339`. + +Published `swagger-ui-dist@5.30.2` was also loaded locally from its npm tarball. +Its standalone preset (one captured script, 73,471 visited AST nodes) produced +one inference and an explicit `javascript_ast` widening unknown. SDK pages +preserved the exact unknown's reason and affected script keys, and retained +producer partial status even when the selected page was exhausted. The larger +1.5 MB main bundle exceeded its unchanged 60-second script analysis deadline; +its partial result had no findings. It is a limitation case, not a successful +large-inventory benchmark. No successful large real endpoint inventory or +whole-application backend verification is claimed. + +The boundary suites cover duplicate endpoint values at different locations, +field selection, intersecting/empty selectors, retained/inline/file sources, +wrong operations, malformed reports, original indices, continuation, producer +partial status and a record larger than the 1 MiB projection budget. Actual SDK +converted output schemas are validated. Source regressions additionally cover +historical omitted versus explicit-null origins, the full parent/report +limitation union before display bounding, and distinct script-scoped unknowns. +These three corrections fail against the imported upstream production code; +explicit-null and explicit-origin controls continue to pass. + +Automated review follow-up added an interleaved, intersecting-filter continuation +case: the second matching offset is `2`, while the joined original indices are +`2, 4, 5`. All 24 projection tests and both SDK boundary tests pass. Test temporary +directories now have the existing fixture as their sole cleanup owner. Both real +CLI launches in the verifier use `createVerifierCommands` and awaited shared +cleanup; real selected capture and both CLI collection readbacks pass after that +change. Command supervision is acquired lazily only for capture or explicit CLI +verification, outside the measured retained-query workflows above. + +## Equivalent completed query costs + +Both versions import the same already completed parent, discover the same 41 +`application,session` tools, and use compact input schemas. No browser or static +analyzer runs during these queries. The existing full-bundle query answers +both questions: (A) all observed endpoint/inference facts and counts, or (B) +the first endpoint candidate, its original provenance, candidate count and source +coverage. The new all-facts workflow uses a summary and both collections; the +selected B workflow needs just one filtered inference page. + +| Workflow | Cold query ms, including import | Warm query ms | Combined process elapsed s | Combined CPU s | Sampled family peak MiB | +| ------------------------- | ------------------------------: | ------------: | -------------------------: | -------------: | ----------------------: | +| Main, complete bundle | 227.1 | 40.5 | 4.125 | 3.940 | 451.3 | +| New, all-facts paging | 4,978.5 | 58.8 | 9.562 | 9.274 | 876.2 | +| New, first candidate only | 4,559.1 | 14.6 | 8.498 | 10.179 | 878.0 | + +Cold SDK output-schema compilation is materially more expensive for the existing +large `inspect_analysis_view` result union than for a full-bundle read. These +single runs do not establish a general latency or memory improvement. Warm +selected reads reuse the session's validated immutable parent; no new cache or +live-state freshness claim is introduced. Unfiltered projection builds only the +requested page; filtered projection counts all matches while allocating only +selected rows. CPU and memory above cover discovery, import, cold and warm runs +and cleanup together; per-temperature CPU/peak memory was not separately measured. + +| Workflow | Input-schema catalog estimate | Model projection estimate | Complete SDK payload estimate | +| ------------------------- | ----------------------------: | ------------------------: | ----------------------------: | +| Main, complete bundle | 13,828 | 20,083 | 268,729 | +| New, all-facts paging | 13,861 | 21,459 | 272,691 | +| New, first candidate only | 13,861 | 16,248 | 262,752 | + +All token figures are `cl100k_base` BPE estimates, **not actual model usage or +billing**. Model projection counts input tool definitions, every tool argument +and returned text-content block across cold and warm tasks. The complete SDK +payload estimate also counts output schemas and structured results, which may +not be placed in model context by an actual client; it excludes JSON-RPC framing. +Discovery is counted once. No real model investigation was executed. Selection +reduces estimated model input about 19% for question B; paging every row of this +small report increases it about 7%. Complete delivery remains useful for small +reports or questions requiring most fields. Failed harness attempts and separate +CLI/provider verification runs are excluded from this equivalent-task table. + +The host guard restricted the whole owned process family to two CPUs and stopped +at 8 GiB summed RSS, 4 GiB host available memory, or a command-specific deadline. +RSS was sampled every 100 ms, can double-count shared pages and can miss brief +peaks. The main Swagger timeout capture took 66.99 s elapsed, 78.39 CPU s and +1,982.8 MiB peak family RSS; the Miniflux capture took 5.87 s, 5.38 CPU s and +1,364.4 MiB. No guard stop or OOM occurred in those executed cases. + +## Reproduction + +Use the pinned Node/npm toolchain, `npm ci`, `npm run build:cached`, an existing +Chromium executable, and an owned directory containing original asset bytes plus +a local `index.html`. The verifier starts only an owned loopback server and +browser; it does not install dependencies or contact an application backend. +The original source files and their SHA-256 digests should be retained with the +capture. For example: + +```bash +node experiments/analysis-efficiency/web-bundle-paging.mjs \ + --repository "$REA_CHECKOUT" --mode capture --application "$ASSET_DIRECTORY" \ + --browser /usr/bin/chromium --output "$NEW_CAPTURE_DIRECTORY" +node experiments/analysis-efficiency/web-bundle-paging.mjs \ + --repository "$MAIN_CHECKOUT" --mode complete --parent "$PARENT_JSON" \ + --output "$NEW_BASELINE_DIRECTORY" +node experiments/analysis-efficiency/web-bundle-paging.mjs \ + --repository "$REA_CHECKOUT" --mode selected --task first-candidate \ + --parent "$PARENT_JSON" --output "$NEW_SELECTED_DIRECTORY" +``` + +Omit `--task first-candidate` to compare all rows. `--verify-cli` additionally +checks both collections through the real CLI; exclude that extra verification +from performance comparisons. Every output directory must be new. Keep resource +receipts and separate failed attempts; do not overwrite them with reruns. diff --git a/experiments/analysis-efficiency/web-bundle-paging.mjs b/experiments/analysis-efficiency/web-bundle-paging.mjs new file mode 100644 index 0000000000..6fbbef9c34 --- /dev/null +++ b/experiments/analysis-efficiency/web-bundle-paging.mjs @@ -0,0 +1,382 @@ +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import { readFile, writeFile, mkdir, mkdtemp, rm } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { tmpdir } from "node:os"; +import { createRequire } from "node:module"; +import { pathToFileURL } from "node:url"; +import { parseArgs } from "node:util"; +import { performance } from "node:perf_hooks"; +import { createVerifierCommands } from "../../scripts/lib/verify-commands.mjs"; +import { withVerifierCleanup } from "../../scripts/lib/verifier-cleanup.mjs"; +// Public CLI capture and same-session retained MCP queries. All assets and output +// paths are explicitly supplied; this verifier installs no browser or application. +const { values } = parseArgs({ + options: { + repository: { type: "string" }, + mode: { type: "string" }, + output: { type: "string" }, + parent: { type: "string" }, + application: { type: "string" }, + browser: { type: "string" }, + task: { type: "string", default: "all" }, + "verify-cli": { type: "boolean", default: false }, + }, +}); +assert.ok(values.repository && values.mode && values.output); +const repo = resolve(values.repository), + mode = values.mode, + out = resolve(values.output), + bundle = values.parent && resolve(values.parent); +assert.ok(["capture", "complete", "selected"].includes(mode)); +assert.ok(["all", "first-candidate"].includes(values.task)); +assert.ok(mode === "capture" ? values.application && values.browser : bundle); +await mkdir(out); +const require = createRequire(join(repo, "package.json")); +const { Client } = await import( + pathToFileURL(require.resolve("@modelcontextprotocol/client")) +); +const { StdioClientTransport } = await import( + pathToFileURL(require.resolve("@modelcontextprotocol/client/stdio")) +); +const env = { + PATH: process.env.PATH, + NODE_OPTIONS: "--max-old-space-size=3072", + UV_THREADPOOL_SIZE: "2", + REA_LOG_LEVEL: "silent", + REA_MCP_INPUT_SCHEMA_PROFILE: "compact", + REA_MCP_TOOL_FAMILIES: "application,session", +}; +const commands = + mode === "capture" || values["verify-cli"] + ? await createVerifierCommands() + : undefined; +if (mode === "capture") { + const { chromium } = require("playwright-core"); + const assets = resolve(values.application); + const allowed = new Set( + await (await import("node:fs/promises")).readdir(assets), + ); + const server = createServer(async (req, res) => { + try { + const name = + new URL(req.url, "http://127.0.0.1").pathname.slice(1) || "index.html"; + assert.ok(allowed.has(name)); + const data = await readFile(join(assets, name)); + res.setHeader( + "Content-Type", + name.endsWith(".js") + ? "text/javascript" + : name.endsWith(".css") + ? "text/css" + : name.endsWith(".json") + ? "application/json" + : "text/html", + ); + res.setHeader( + "Content-Type", + res.getHeader("Content-Type") + "; charset=utf-8", + ); + res.end(data); + } catch { + res.writeHead(404); + res.end(); + } + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const origin = `http://127.0.0.1:${server.address().port}`; + const userData = await mkdtemp(join(tmpdir(), "rea-6198-chrome-")); + let context; + await withVerifierCleanup(async () => { + context = await chromium.launchPersistentContext(userData, { + executablePath: resolve(values.browser), + headless: true, + args: ["--remote-debugging-port=0", "--no-sandbox"], + }); + context + .pages()[0] + .on("pageerror", (e) => console.error("PAGE ERROR", e.message)); + context + .pages()[0] + .on("response", (r) => console.error("RESPONSE", r.status(), r.url())); + await context.pages()[0].goto(origin, { waitUntil: "networkidle" }); + const port = ( + await readFile(join(userData, "DevToolsActivePort"), "utf8") + ).split("\n")[0]; + const endpoint = `http://127.0.0.1:${port}`; + const targets = await (await fetch(`${endpoint}/json/list`)).json(); + const target = targets.find( + (t) => t.type === "page" && t.url === origin + "/", + ); + assert.ok(target); + const started = performance.now(); + const { stdout, stderr } = await commands.exec( + process.execPath, + [ + join(repo, "scripts/rea.mjs"), + "analyze-web-bundle", + endpoint, + target.id, + "--observation-ms", + "1000", + "--max-heap-mb", + "1024", + "--analysis-timeout-ms", + "60000", + "--total-analysis-timeout-ms", + "90000", + "--detail", + "summary", + "--evidence_output", + join(out, "complete-evidence.json"), + "--view", + JSON.stringify({ + kind: "web-bundle", + collection: "inferences", + limit: 1, + }), + "--format", + "json", + ], + { cwd: repo, env, maxBuffer: 64 * 1024 ** 2, timeout: 120000 }, + ); + await writeFile(join(out, "stderr.txt"), stderr); + const delivery = JSON.parse(stdout); + const view = delivery.summary.normalized_result; + const complete = JSON.parse( + await readFile(join(out, "complete-evidence.json"), "utf8"), + ); + const evidence = complete.records.find( + (row) => row.evidence_id === view.parent_evidence_id, + ); + assert.ok(evidence); + assert.equal(evidence.operation, "analyze_web_bundle"); + assert.ok(evidence.normalized_result.observations); + assert.deepEqual( + view.items, + evidence.normalized_result.inferences + .slice(0, 1) + .map((row, source_index) => ({ source_index, ...row })), + ); + assert.deepEqual(view.source_unknowns, evidence.normalized_result.unknowns); + await writeFile(join(out, "delivery.json"), stdout); + await writeFile(join(out, "parent.json"), JSON.stringify(evidence)); + // Public export/import format comes from the CLI's complete Evidence publication. + console.log( + JSON.stringify({ + capture_ms: performance.now() - started, + evidence_id: evidence.evidence_id, + counts: { + endpoints: evidence.normalized_result.observations.endpoints.length, + inferences: evidence.normalized_result.inferences.length, + }, + completeness: evidence.normalized_result.completeness, + }), + ); + }, [ + () => commands.close(), + async () => { + await context?.close(); + await rm(userData, { recursive: true, force: true }); + }, + () => + new Promise((done, reject) => + server.close((error) => (error ? reject(error) : done())), + ), + ]); + process.exit(0); +} +const parent = JSON.parse(await readFile(bundle, "utf8")); +const report = parent.normalized_result; +const { createEvidenceBundle } = await import( + pathToFileURL(join(repo, "dist/domain/evidenceBundle.js")) +); +const bundleFile = join(out, "input-bundle.json"); +await writeFile(bundleFile, JSON.stringify(createEvidenceBundle([parent]))); +const transport = new StdioClientTransport({ + command: process.execPath, + args: [join(repo, "scripts/rea.mjs"), "mcp"], + cwd: repo, + env, + stderr: "pipe", +}); +transport.stderr?.resume(); +const client = new Client({ + name: "rea-web-bundle-paging-verifier", + version: "1", +}); +const calls = []; +let catalog; +async function invoke(name, args) { + const toolDefinition = catalog.tools.find((t) => t.name === name); + assert.ok(toolDefinition); + const start = performance.now(); + const result = await client.callTool( + { name, arguments: args }, + { toolDefinition, timeout: 90000 }, + ); + calls.push({ + name, + arguments: args, + result, + elapsed_ms: performance.now() - start, + }); + assert.notEqual(result.isError, true, JSON.stringify(result)); + return result.structuredContent; +} +await withVerifierCleanup(async () => { + await client.connect(transport); + catalog = await client.listTools(); + await writeFile(join(out, "catalog.json"), JSON.stringify(catalog)); + const runs = []; + for (const temperature of ["cold", "warm"]) { + const start = performance.now(); + const callStart = calls.length; + if (temperature === "cold") + await invoke("import_evidence_bundle", { path: bundleFile }); + if (mode === "complete") { + const full = await invoke("get_evidence_bundle", {}); + const actual = full.result.records.find( + (r) => r.evidence_id === parent.evidence_id, + ); + assert.deepEqual(actual, parent); + runs.push({ + temperature, + elapsed_ms: performance.now() - start, + calls: calls.length - callStart, + }); + continue; + } + const source = { + kind: "retained-evidence", + evidence_id: parent.evidence_id, + }; + if (values.task === "all") { + const summary = await invoke("inspect_analysis_view", { + source, + view: { kind: "web-bundle", collection: "summary" }, + }); + assert.equal( + summary.normalized_result.summary.counts.endpoints, + report.observations.endpoints.length, + ); + } + if (values.task === "first-candidate") { + const value = await invoke("inspect_analysis_view", { + source, + view: { + kind: "web-bundle", + collection: "inferences", + kinds: ["endpoint_candidate"], + limit: 1, + }, + }); + const index = report.inferences.findIndex( + (row) => row.kind === "endpoint_candidate", + ); + assert.equal( + value.normalized_result.coverage.total, + report.inferences.filter((row) => row.kind === "endpoint_candidate") + .length, + ); + assert.deepEqual( + value.normalized_result.items, + index < 0 ? [] : [{ source_index: index, ...report.inferences[index] }], + ); + assert.deepEqual( + value.normalized_result.source_unknowns, + report.unknowns, + ); + } else + for (const collection of ["endpoints", "inferences"]) { + const expected = + collection === "endpoints" + ? report.observations.endpoints + : report.inferences; + const rows = []; + let offset = 0; + do { + const value = await invoke("inspect_analysis_view", { + source, + view: { kind: "web-bundle", collection, offset, limit: 32 }, + }); + const page = value.normalized_result; + assert.deepEqual(page.source_unknowns, report.unknowns); + assert.deepEqual(page.completeness, report.completeness); + assert.equal(page.coverage.total, expected.length); + for (const row of page.items) { + assert.deepEqual(row, { + source_index: rows.length, + ...expected[rows.length], + }); + rows.push(row); + } + if (page.coverage.exhausted) break; + assert.ok(page.coverage.next_offset > offset); + offset = page.coverage.next_offset; + } while (true); + assert.equal(rows.length, expected.length); + } + runs.push({ + temperature, + elapsed_ms: performance.now() - start, + calls: calls.length - callStart, + }); + } + if (mode === "selected" && values["verify-cli"]) { + for (const collection of ["endpoints", "inferences"]) { + const input = JSON.stringify({ + source: { kind: "evidence-file", path: bundle }, + view: { + kind: "web-bundle", + collection, + limit: 32, + }, + }); + const { stdout } = await commands.exec( + process.execPath, + [ + join(repo, "scripts/rea.mjs"), + "inspect-analysis-view", + input, + "--format", + "json", + ], + { cwd: repo, env, maxBuffer: 64 * 1024 ** 2, timeout: 90000 }, + ); + const cli = JSON.parse(stdout); + assert.deepEqual( + cli.normalized_result.items, + (collection === "endpoints" + ? report.observations.endpoints + : report.inferences + ) + .slice(0, 32) + .map((r, source_index) => ({ + source_index, + ...r, + })), + ); + assert.deepEqual(cli.normalized_result.source_unknowns, report.unknowns); + await writeFile(join(out, `cli-${collection}.json`), stdout); + } + } + await writeFile(join(out, "calls.json"), JSON.stringify(calls)); + await writeFile( + join(out, "metrics.json"), + JSON.stringify( + { + mode, + task: values.task, + runs, + counts: { + endpoints: report.observations.endpoints.length, + inferences: report.inferences.length, + }, + }, + null, + 2, + ), + ); + console.log(JSON.stringify({ mode, task: values.task, runs })); +}, [() => commands?.close(), () => client.close()]); diff --git a/src/cli/browserCommands.ts b/src/cli/browserCommands.ts index 21868d3140..5a967c5134 100644 --- a/src/cli/browserCommands.ts +++ b/src/cli/browserCommands.ts @@ -1,6 +1,6 @@ import { parseJavaScriptRouteDelivery, - javascriptRouteDeliveryOptions, + webBundleDeliveryOptions, } from "./javascriptRouteDelivery.js"; import { deliverCliAnalysisEvidence } from "./analysisEvidenceDelivery.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; @@ -164,7 +164,7 @@ const registerBundleAnalysis = ( }), options: z.object({ ...browserScopeOptions, - ...javascriptRouteDeliveryOptions, + ...webBundleDeliveryOptions, observationMs: observationDuration(500), fetchSourceMaps: z .boolean() diff --git a/src/cli/javascriptRouteDelivery.ts b/src/cli/javascriptRouteDelivery.ts index 71e254c27c..05e07f5f14 100644 --- a/src/cli/javascriptRouteDelivery.ts +++ b/src/cli/javascriptRouteDelivery.ts @@ -18,6 +18,13 @@ export const javascriptRouteDeliveryOptions = { ), }; +export const webBundleDeliveryOptions = { + ...javascriptRouteDeliveryOptions, + view: javascriptRouteDeliveryOptions.view.describe( + "JSON web-bundle (endpoints, inferences, summary) or javascript-routes view; requires --detail summary and --evidence_output", + ), +}; + /** Validate delivery before opening artifacts or attaching a browser. */ export const parseJavaScriptRouteDelivery = ( operation: string, diff --git a/src/contracts/analysisViewToolContracts.ts b/src/contracts/analysisViewToolContracts.ts index dae34d172c..16547ed7b5 100644 --- a/src/contracts/analysisViewToolContracts.ts +++ b/src/contracts/analysisViewToolContracts.ts @@ -17,10 +17,26 @@ export const ANALYSIS_VIEW_TOOL_CONTRACTS = [ ...toolContractMetadata("inspect_analysis_view"), kind: "application", description: - 'Use javascript-routes collections to query route patterns, source-linked occurrences, source status, unknowns or summary counts from retained JavaScript application or web-bundle Evidence. Inspect selected views of supported completed Evidence without repeating analysis. Choose a view compatible with the parent operation described below. Command summary views preserve ordered GDB records or opaque objdump stdout/stderr once, source parameters, completion facts and parent identity. They do not establish current target state. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. Native export tables use kind native-exports, collection exports/issues, exact names or original indices, selected fields and matching-row pagination; parent Evidence plus source_index identifies each row. Source coverage stays separate from page coverage. BEAM views use kind beam, collections atoms/exports/imports/lambdas/chunks, exact names/indices/fields and matching-row pages; import modules intersects function names. source_index identifies original parent-and-collection rows, preserving every arity and raw atom index. beam.counts preserves null missing tables versus decoded empty tables; runtime absence remains unknown. PYC views use kind python-pyc, collection code_objects, names/indices/fields and paging; source_index scopes every same-name alternative to the parent, nested_code keeps code-constant references, and pyc.coverage preserves source coverage. Go metadata uses kind go, collection dependencies/settings, exact paths/keys or indices, fields and pagination. Row index refers to the original collection; empty selections return no rows, and dependency coverage remains unknown. Rust metadata uses kind rust, collection packages/edges, exact names/versions or original indices, selected fields, edge from_indices/to_indices and matching-row pagination. Embedded-record coverage is separate from unknown runtime dependencies. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. The environment-manifest view projects the same HAR requests on the primary origin (most JSON-like records) into a replayable tool surface: view {kind: "environment-manifest", template_rules: ["after-uid-literal", "opaque-token", "sibling-variance"], include_query_parameter_names: true}. Each tool has a deterministic name, a path template and parameters from the four format rules plus the selected opt-in rules, observed query parameter names, a value-free body shape for POST/PUT/PATCH, status and content-type sets and record ordinals; records are pointers into the parent, excluded groups carry a reason and three fixed unknowns are always stated. Templates, parameters and names are inferences; the view copies no header, cookie or body values and refuses parents inspected with sensitive values. A capture_browser_scenario parent also supports summary and browser-scenario steps/events/transactions/text pages, preserving step_indices, event_kinds, transaction_ids and selected artifacts. A capture_browser_scenario parent that selected response bodies or header values projects the same way, one record per network transaction in request order; it relies on capture-time redaction of declared secrets instead of that refusal, and a transaction whose URL was redacted is left unrepresented. The environment-state view extracts, from the same parents, the initial state the recorded reads observed: view {kind: "environment-state", template_rules: [...]} with the rules of the manifest it accompanies. It reads response bodies recorded before the first related write and reports entity types with an identity field (proved by a path parameter or inferred from a field name), entities in collection order with per-view values where views differ, documents for non-parameterized reads that are not entities and have one initial request line (item-shaped paths stay recorded), and a render plan per read tool: item and list views reproduce their latest recordings from the state (a list also learns the filters, search, sort and paging its recordings verify, and answers new values of them from the state), and every other read tool is a recorded view with a reason. It also reports a write plan per state-changing tool (create, update, delete or document, built from its first successful call) and write_clock; a plan is served only when replaying the capture shows a later read confirming its effect and none contradicting it. Types and identity are inferences; values are recorded data, so it refuses parents inspected with sensitive values, and a scenario parent must have selected response bodies; its values keep the capture redaction markers, and a write whose URL it redacted ends the state of every type. The environment-coverage view, view {kind: "environment-coverage", template_rules: [...]}, summarizes that state without its entities: reads.state and reads.recorded count read tools answered from the state and replayed (recorded_reasons per view reason), writes.served lists the served write tools, and writes.withheld lists each other write tool with its reason, plan record, contradicting read and a hint about what to record next; it refuses what the environment-state view refuses. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. A completed Evidence JSON file on this host uses {kind: "evidence-file", path: ""}: REA reads and authenticates it, then projects it like retained Evidence, so a large capture is bounded by the file reader rather than by inline-input accounting. The file may also be an Evidence bundle written by CLI --evidence_output or export_evidence_bundle; select evidence_id when it contains more than one record. For layout, JavaScript, inventory, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Typed module references use view {kind: "javascript", collection: "modules", projection: "references", where: {path: "main.js", kind: "javascript-module"}, offset: 0, limit: 64}; where matches exact retained strings (null selects unavailable paths). Rows include canonical identity, retained source-range count and observation support states. Source coverage and selection remain explicit; exhausting matching rows does not prove runtime loading or complete analysis. For exact file/native facts and processing gaps, use view {kind: "javascript-facts", where: {paths: ["package.json"]}, property_keys: ["name", "operation", "analysis_status"]}. Rows page original observations with canonical identities and Evidence; missing properties remain omitted. Paths match recorded artifact-path/source-range locations; optional kinds filter nodes. To discover field names without values, opt into include_available_property_keys: true with property_keys: []. Canonical identity and Evidence are separate from properties; keep each node_id/observation_id pair together when composing results. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions. Occurrences accept exact logical_paths and occurrence_ids; nodes accept artifact_ids. Batch filters intersect (omitted means all available, [] means none); fields selects columns while retaining identity links. source_complete describes retained graph membership; source_coverage describes producer coverage, including unknown encrypted PCK directories. Page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. Interface Builder views use kind interface-builder and collection documents/nodes/edges/coverage; nodes and edges accept an exact document relative_path. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Managed member summaries keep identity, selection with unmatched names, coverage and per-collection counts; managed-members pages use collection types/exported_types/fields/methods/member_refs/call_edges/field_accesses/user_strings and carry the source selection and coverage. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. Protobuf views use kind protobuf, collection bodies/frames/fields, with record_ordinals, body_pointers and fields; field pages also select frame_indices, field_ids, field_numbers or schema_names. Byte identity, offset basis and body status remain present. Text and schema_value are opt-in page columns. APK signing views use kind apk-signature and collection signers/digests/lineage, with ids, signer_ids, schemes, fields and paging. Identity links remain when fields is empty. Declared identities and digest matches do not establish signature verification or authenticity. Model views use kind model-artifact and collection metadata (keys), tensors (names/types) or types (derived totals), with indices, fields and paging. Header SHA-256 scopes declarations; full-file identity remains unknown unless full hashing was selected. Arrays retain typed eight-element previews with explicit omissions. IL2CPP metadata views use kind il2cpp-metadata and collection sections/images/types/methods/fields/properties/events/parameters/string_literals/code_gen_modules/method_addresses/return_types/field_types/parameter_types/generic_method_instances/generic_method_issues. Select indices, fields and pages; sections/images/types/methods/fields/properties/events/parameters/return_types/field_types/parameter_types/generic_method_instances accept exact names (native type names match retained rendered values, preserving same-name rows; null names never match), types also accept image_indices/namespaces/declaring_types, and methods/members accept declaring_types. Parameters and parameter_types accept method_indices; code_gen_modules accept image_indices; method_addresses and generic_method_instances accept addresses/states (generic_method_instances also method_indices), preserving every folded alternative; generic_method_issues accept rows. Row index is scoped to the metadata digest; native pages add binary_source. Uncollected native tables have null counts and remain unavailable. Desktop directory views use kind desktop-application and collection files/omissions/candidates/components/next_probes, with fields and pagination; files and omissions accept exact paths or their stable IDs, and candidates accept families. Source coverage and stable row IDs remain available. Native load-image Evidence uses kind native-load-image with collection source_files/mappings/relocations/entry_points/entry_context/checks/sections, original indices, collection-specific fields and offset/limit paging. Null collections remain unrecorded or inapplicable; pages preserve unsupported verification. Native inventory Evidence from list_names, list/search_procedures or list/search_strings uses view: {kind: "native-inventory", offset: 0, limit: 64, fields: ["value"]}. Rows keep source_index and address even with selected fields. Summary counts describe retained rows; provider discovery coverage remains unknown. Native function Evidence accepts kind summary for identity, availability and facet counts, or kind native for selected facts: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', + 'Use javascript-routes collections to query route patterns, source-linked occurrences, source status, unknowns or summary counts from retained JavaScript application or web-bundle Evidence. Use web-bundle collections endpoints or inferences to page endpoint observations and inferences from retained analyze_web_bundle Evidence; summary returns target URL, origin, and counts. Inspect selected views of supported completed Evidence without repeating analysis. Choose a view compatible with the parent operation described below. Command summary views preserve ordered GDB records or opaque objdump stdout/stderr once, source parameters, completion facts and parent identity. They do not establish current target state. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. Native export tables use kind native-exports, collection exports/issues, exact names or original indices, selected fields and matching-row pagination; parent Evidence plus source_index identifies each row. Source coverage stays separate from page coverage. BEAM views use kind beam, collections atoms/exports/imports/lambdas/chunks, exact names/indices/fields and matching-row pages; import modules intersects function names. source_index identifies original parent-and-collection rows, preserving every arity and raw atom index. beam.counts preserves null missing tables versus decoded empty tables; runtime absence remains unknown. PYC views use kind python-pyc, collection code_objects, names/indices/fields and paging; source_index scopes every same-name alternative to the parent, nested_code keeps code-constant references, and pyc.coverage preserves source coverage. Go metadata uses kind go, collection dependencies/settings, exact paths/keys or indices, fields and pagination. Row index refers to the original collection; empty selections return no rows, and dependency coverage remains unknown. Rust metadata uses kind rust, collection packages/edges, exact names/versions or original indices, selected fields, edge from_indices/to_indices and matching-row pagination. Embedded-record coverage is separate from unknown runtime dependencies. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. The environment-manifest view projects the same HAR requests on the primary origin (most JSON-like records) into a replayable tool surface: view {kind: "environment-manifest", template_rules: ["after-uid-literal", "opaque-token", "sibling-variance"], include_query_parameter_names: true}. Each tool has a deterministic name, a path template and parameters from the four format rules plus the selected opt-in rules, observed query parameter names, a value-free body shape for POST/PUT/PATCH, status and content-type sets and record ordinals; records are pointers into the parent, excluded groups carry a reason and three fixed unknowns are always stated. Templates, parameters and names are inferences; the view copies no header, cookie or body values and refuses parents inspected with sensitive values. A capture_browser_scenario parent also supports summary and browser-scenario steps/events/transactions/text pages, preserving step_indices, event_kinds, transaction_ids and selected artifacts. A capture_browser_scenario parent that selected response bodies or header values projects the same way, one record per network transaction in request order; it relies on capture-time redaction of declared secrets instead of that refusal, and a transaction whose URL was redacted is left unrepresented. The environment-state view extracts, from the same parents, the initial state the recorded reads observed: view {kind: "environment-state", template_rules: [...]} with the rules of the manifest it accompanies. It reads response bodies recorded before the first related write and reports entity types with an identity field (proved by a path parameter or inferred from a field name), entities in collection order with per-view values where views differ, documents for non-parameterized reads that are not entities and have one initial request line (item-shaped paths stay recorded), and a render plan per read tool: item and list views reproduce their latest recordings from the state (a list also learns the filters, search, sort and paging its recordings verify, and answers new values of them from the state), and every other read tool is a recorded view with a reason. It also reports a write plan per state-changing tool (create, update, delete or document, built from its first successful call) and write_clock; a plan is served only when replaying the capture shows a later read confirming its effect and none contradicting it. Types and identity are inferences; values are recorded data, so it refuses parents inspected with sensitive values, and a scenario parent must have selected response bodies; its values keep the capture redaction markers, and a write whose URL it redacted ends the state of every type. The environment-coverage view, view {kind: "environment-coverage", template_rules: [...]}, summarizes that state without its entities: reads.state and reads.recorded count read tools answered from the state and replayed (recorded_reasons per view reason), writes.served lists the served write tools, and writes.withheld lists each other write tool with its reason, plan record, contradicting read and a hint about what to record next; it refuses what the environment-state view refuses. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. A completed Evidence JSON file on this host uses {kind: "evidence-file", path: ""}: REA reads and authenticates it, then projects it like retained Evidence, so a large capture is bounded by the file reader rather than by inline-input accounting. The file may also be an Evidence bundle written by CLI --evidence_output or export_evidence_bundle; select evidence_id when it contains more than one record. For layout, JavaScript, inventory, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Typed module references use view {kind: "javascript", collection: "modules", projection: "references", where: {path: "main.js", kind: "javascript-module"}, offset: 0, limit: 64}; where matches exact retained strings (null selects unavailable paths). Rows include canonical identity, retained source-range count and observation support states. Source coverage and selection remain explicit; exhausting matching rows does not prove runtime loading or complete analysis. For exact file/native facts and processing gaps, use view {kind: "javascript-facts", where: {paths: ["package.json"]}, property_keys: ["name", "operation", "analysis_status"]}. Rows page original observations with canonical identities and Evidence; missing properties remain omitted. Paths match recorded artifact-path/source-range locations; optional kinds filter nodes. To discover field names without values, opt into include_available_property_keys: true with property_keys: []. Canonical identity and Evidence are separate from properties; keep each node_id/observation_id pair together when composing results. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions. Occurrences accept exact logical_paths and occurrence_ids; nodes accept artifact_ids. Batch filters intersect (omitted means all available, [] means none); fields selects columns while retaining identity links. source_complete describes retained graph membership; source_coverage describes producer coverage, including unknown encrypted PCK directories. Page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. Interface Builder views use kind interface-builder and collection documents/nodes/edges/coverage; nodes and edges accept an exact document relative_path. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Managed member summaries keep identity, selection with unmatched names, coverage and per-collection counts; managed-members pages use collection types/exported_types/fields/methods/member_refs/call_edges/field_accesses/user_strings and carry the source selection and coverage. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. Protobuf views use kind protobuf, collection bodies/frames/fields, with record_ordinals, body_pointers and fields; field pages also select frame_indices, field_ids, field_numbers or schema_names. Byte identity, offset basis and body status remain present. Text and schema_value are opt-in page columns. APK signing views use kind apk-signature and collection signers/digests/lineage, with ids, signer_ids, schemes, fields and paging. Identity links remain when fields is empty. Declared identities and digest matches do not establish signature verification or authenticity. Model views use kind model-artifact and collection metadata (keys), tensors (names/types) or types (derived totals), with indices, fields and paging. Header SHA-256 scopes declarations; full-file identity remains unknown unless full hashing was selected. Arrays retain typed eight-element previews with explicit omissions. IL2CPP metadata views use kind il2cpp-metadata and collection sections/images/types/methods/fields/properties/events/parameters/string_literals/code_gen_modules/method_addresses/return_types/field_types/parameter_types/generic_method_instances/generic_method_issues. Select indices, fields and pages; sections/images/types/methods/fields/properties/events/parameters/return_types/field_types/parameter_types/generic_method_instances accept exact names (native type names match retained rendered values, preserving same-name rows; null names never match), types also accept image_indices/namespaces/declaring_types, and methods/members accept declaring_types. Parameters and parameter_types accept method_indices; code_gen_modules accept image_indices; method_addresses and generic_method_instances accept addresses/states (generic_method_instances also method_indices), preserving every folded alternative; generic_method_issues accept rows. Row index is scoped to the metadata digest; native pages add binary_source. Uncollected native tables have null counts and remain unavailable. Desktop directory views use kind desktop-application and collection files/omissions/candidates/components/next_probes, with fields and pagination; files and omissions accept exact paths or their stable IDs, and candidates accept families. Source coverage and stable row IDs remain available. Native load-image Evidence uses kind native-load-image with collection source_files/mappings/relocations/entry_points/entry_context/checks/sections, original indices, collection-specific fields and offset/limit paging. Null collections remain unrecorded or inapplicable; pages preserve unsupported verification. Native inventory Evidence from list_names, list/search_procedures or list/search_strings uses view: {kind: "native-inventory", offset: 0, limit: 64, fields: ["value"]}. Rows keep source_index and address even with selected fields. Summary counts describe retained rows; provider discovery coverage remains unknown. Native function Evidence accepts kind summary for identity, availability and facet counts, or kind native for selected facts: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', inputSchema: inspectAnalysisViewInputSchema, outputSchema: evidenceResultOf(analysisViewResultSchema), examples: [ + { + title: + "Page endpoint observations from retained web bundle analysis Evidence", + input: { + source: { + kind: "retained-evidence", + evidence_id: RETAINED_EXAMPLE_ID, + }, + view: { + kind: "web-bundle", + collection: "endpoints", + offset: 0, + limit: 32, + }, + }, + }, { title: "Page selected native inventory fields without repeating the complete list", diff --git a/src/contracts/browserToolContracts.ts b/src/contracts/browserToolContracts.ts index a51bebc728..27b24d8ca6 100644 --- a/src/contracts/browserToolContracts.ts +++ b/src/contracts/browserToolContracts.ts @@ -3,7 +3,11 @@ import { javaScriptRoutesCompleteDeliveryShape, webRoutesSummaryDeliveryShape, } from "../domain/analysisView/javascriptRoutesView.js"; -import { javaScriptRoutesViewResultSchema } from "../domain/analysisView/analysisView.js"; +import { + javaScriptRoutesViewResultSchema, + webBundleSummaryViewResultSchema, + webBundlePageViewResultSchema, +} from "../domain/analysisView/analysisView.js"; import type { ToolContract } from "./toolContractTypes.js"; import { inspectWebPageInputSchema, @@ -48,7 +52,12 @@ export const analyzeWebBundleRequestSchema = z ]) .meta({ "x-rea-root-discriminator": "detail" }); const bundleOutputSchema = evidenceResultOf( - z.union([webBundleAnalysisSchema, javaScriptRoutesViewResultSchema]), + z.union([ + webBundleAnalysisSchema, + javaScriptRoutesViewResultSchema, + webBundleSummaryViewResultSchema, + webBundlePageViewResultSchema, + ]), ); const observationSessionOutputSchema = evidenceResultOf( webObservationSessionSchema, @@ -197,7 +206,7 @@ export const BROWSER_TOOL_CONTRACTS = [ name: "analyze_web_bundle", ...toolContractMetadata("analyze_web_bundle"), description: - "Capture JavaScript source from one selected CDP page and statically derive a chunk graph, route and endpoint candidates, vendor fingerprints, page-declared WebMCP metadata, and optionally fetch source maps from the selected target origin or explicitly requested origins. JavaScript is parsed but never executed. Select detail summary and an explicit javascript-routes view for filtered route patterns, occurrences, sources, unknowns or summary counts; complete Evidence remains retained for later views. Static candidates do not establish reachability or a complete application route denominator.", + "Capture JavaScript source from one selected CDP page and statically derive a chunk graph, route and endpoint candidates, vendor fingerprints, page-declared WebMCP metadata, and optionally fetch source maps from the selected target origin or explicitly requested origins. JavaScript is parsed but never executed. Select detail summary with a web-bundle view for filtered endpoint observations, inferences or counts, or a javascript-routes view for route patterns, occurrences, sources, unknowns or counts. Complete Evidence remains retained for later views. Static candidates do not establish reachability or a complete application route denominator.", kind: "browser-provider", inputSchema: analyzeWebBundleRequestSchema, outputSchema: bundleOutputSchema, diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index 84e9ef3c85..ddf8626eab 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -61,6 +61,14 @@ import { javaScriptRoutesSourceReferenceSchema, projectJavaScriptRoutesView, } from "./javascriptRoutesView.js"; +import { + webBundleViewRequestSchema, + webBundleViewItemsSchema, + webBundleSummarySchema, + projectWebBundleView, +} from "./webBundleView.js"; +import { webBundleAnalysisSchema } from "../webBundleAnalysis.js"; +import { recordedBrowserOriginSchema } from "../browserObservation.js"; import { javaScriptRouteInventorySchema } from "../javascript/javascriptRouteInventory.js"; import { sourceToBundleViewRequestSchema, @@ -395,6 +403,7 @@ export const analysisViewRequestSchema = z.discriminatedUnion("kind", [ pythonPycViewRequestSchema, nativeExportsViewRequestSchema, javaScriptRoutesViewRequestSchema, + webBundleViewRequestSchema, nativeFunctionMatchingViewRequestSchema, sourceToBundleViewRequestSchema, summaryViewSchema, @@ -881,6 +890,32 @@ export const javaScriptRoutesViewResultSchema = z.strictObject({ byte_limited: z.boolean(), }); +/** Compact web bundle findings retain endpoint observations and inference candidates. */ +export const webBundleSummaryViewResultSchema = z.strictObject({ + ...summaryViewResultBase, + artifact: artifactSchema.nullable(), + parent_operation: z.literal("analyze_web_bundle"), + target_url: z.string(), + target_origin: recordedBrowserOriginSchema.nullable().optional(), + source_unknowns: webBundleAnalysisSchema.shape.unknowns, + summary: webBundleSummarySchema, +}); + +export const webBundlePageViewResultSchema = z.strictObject({ + ...viewResultBase, + artifact: artifactSchema.nullable(), + kind: z.literal("web-bundle"), + parent_operation: z.literal("analyze_web_bundle"), + target_url: z.string(), + target_origin: recordedBrowserOriginSchema.nullable().optional(), + source_unknowns: webBundleAnalysisSchema.shape.unknowns, + view: webBundleViewRequestSchema, + completeness: webBundleAnalysisSchema.shape.completeness, + items: webBundleViewItemsSchema, + summary: webBundleSummarySchema.nullable(), + byte_limited: z.boolean(), +}); + export const gpuCodeObjectsSummaryViewResultSchema = z.strictObject({ ...summaryViewResultBase, parent_operation: z.literal("inspect_gpu_code_objects"), @@ -1072,6 +1107,7 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ nativeObservationPageViewResultSchema, nativeContextViewResultSchema, javaScriptRoutesViewResultSchema, + webBundlePageViewResultSchema, sourceToBundlePageViewResultSchema, z.strictObject({ ...viewResultBase, @@ -1159,6 +1195,7 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ nativeFunctionMatchingSummaryViewResultSchema, nativeFunctionSummaryViewResultSchema, sourceToBundleSummaryViewResultSchema, + webBundleSummaryViewResultSchema, ]), z.strictObject({ ...viewResultBase, @@ -1372,6 +1409,13 @@ export type InspectAnalysisViewInput = z.output< export type AnalysisViewResult = z.output; export type AnalysisViewCoverage = z.output; +export { + webBundleViewRequestSchema, + webBundleViewItemsSchema, + webBundleSummarySchema, + projectWebBundleView, +}; + /** Parent record already resolved from inline Evidence or a session reference. */ export interface AnalysisViewParent { readonly evidenceId: string; @@ -1490,6 +1534,7 @@ const retainedViewProjectors = new Map([ ["compare_source_to_bundle", projectSourceToBundleView], ["inspect_artifact", projectArtifactInventoryView], ["inventory_artifact", projectArtifactInventoryView], + ["analyze_web_bundle", projectWebBundleView], ]); /** Budgets retained across source admission and the selected grouping projection. */ @@ -1593,6 +1638,10 @@ export const projectAnalysisView = ( const projected = projectJavaScriptRoutesView(parent, view); return projected.ok ? ok(sealAnalysisView(projected.value)) : projected; } + if (view.kind === "web-bundle") { + const projected = projectWebBundleView(parent, view); + return projected.ok ? ok(sealAnalysisView(projected.value)) : projected; + } if (isGroupingView(view)) { const grouped = projectGroupingView(parent, view, accumulators); return grouped.ok ? ok(sealAnalysisView(grouped.value)) : grouped; diff --git a/src/domain/analysisView/javascriptRoutesView.ts b/src/domain/analysisView/javascriptRoutesView.ts index 99d5be1a5a..c2bab2d08b 100644 --- a/src/domain/analysisView/javascriptRoutesView.ts +++ b/src/domain/analysisView/javascriptRoutesView.ts @@ -1,5 +1,6 @@ import { javascriptSourceRangeSchema } from "../javascript/javascriptSourceCoordinates.js"; import { z } from "zod"; +import { webBundleViewRequestSchema } from "./webBundleView.js"; import { digestSchema } from "../digests.js"; import { isImmutableJsonSnapshot } from "../immutableJson.js"; import { err, ok, type Result } from "../result.js"; @@ -122,7 +123,10 @@ export const javaScriptRoutesSummaryDeliveryShape = { }; export const webRoutesSummaryDeliveryShape = { detail: z.literal("summary"), - view: javaScriptRoutesViewRequestSchema, + view: z.union([ + javaScriptRoutesViewRequestSchema, + webBundleViewRequestSchema, + ]), }; export const javaScriptRoutesDeliverySchema = z.discriminatedUnion("detail", [ z.strictObject(javaScriptRoutesCompleteDeliveryShape), diff --git a/src/domain/analysisView/webBundleView.test.ts b/src/domain/analysisView/webBundleView.test.ts new file mode 100644 index 0000000000..2adb08d8d2 --- /dev/null +++ b/src/domain/analysisView/webBundleView.test.ts @@ -0,0 +1,641 @@ +import { expect, it } from "vitest"; +import type { WebBundleAnalysis } from "../webBundleAnalysis.js"; +import { AnalysisInputError } from "../analysisErrorCore.js"; +import { + analysisViewRequestSchema, + projectAnalysisView, + type AnalysisViewParent, +} from "./analysisView.js"; + +const sampleAnalysis = (): WebBundleAnalysis => ({ + capture: { + target_url: "https://example.test/app", + target_origin: "https://example.test", + scripts_observed: 3, + scripts_analyzed: 2, + source_artifacts: [], + }, + observations: { + chunks: { + nodes: [ + { + script_key: "app.js", + url: "https://example.test/app.js", + artifact_sha256: "a".repeat(64), + bytes: 1200, + }, + { + script_key: "vendor.js", + url: "https://example.test/vendor.js", + artifact_sha256: "b".repeat(64), + bytes: 4500, + }, + ], + edges: [], + }, + routes: [], + endpoints: [ + { + value: "/api/users", + mechanism: "call:fetch", + location: { script_key: "app.js", line: 10, column: 2 }, + }, + { + value: "/api/users", + mechanism: "call:fetch", + location: { script_key: "vendor.js", line: 42, column: 8 }, + }, + { + value: "/api/orders", + mechanism: "call:sendBeacon", + location: { script_key: "app.js", line: 85, column: 4 }, + }, + ], + webmcp_declarations: [], + source_maps: { + status: "not_requested", + requested: 0, + processed: 0, + items: [], + }, + }, + inferences: [ + { + kind: "endpoint_candidate", + value: "/api/dynamic-user", + confidence: "low", + basis: [ + { + script_key: "app.js", + artifact_sha256: "a".repeat(64), + line: 25, + column: 4, + detector: "call:client.get:unverified-network-receiver", + }, + ], + }, + { + kind: "vendor_fingerprint", + value: "React", + confidence: "high", + basis: [ + { + script_key: "vendor.js", + artifact_sha256: "b".repeat(64), + line: 1, + column: 0, + detector: "vendor:react", + }, + ], + }, + ], + unknowns: [ + { + dimension: "network", + reason: "Unverified client receiver", + affected_script_keys: ["app.js"], + }, + ], + completeness: { + status: "partial", + parsed_scripts: 2, + parse_failures: 1, + visited_ast_nodes: 150, + }, + limitations: ["Script evaluation was omitted"], +}); + +const makeParent = ( + overrides?: Partial, +): AnalysisViewParent => ({ + evidenceId: `ev_${"c".repeat(64)}`, + operation: "analyze_web_bundle", + limitations: ["Parent run was bounded"], + normalizedResult: { + ...sampleAnalysis(), + ...overrides, + }, +}); + +const query = ( + source: AnalysisViewParent, + selection: Record, +) => { + const result = projectAnalysisView( + source, + analysisViewRequestSchema.parse(selection), + ); + if (!result.ok) throw result.error; + return result.value; +}; + +it.each([undefined, null, "https://example.test"])( + "preserves historical omitted and explicit target origins: %s", + (origin) => { + const report = sampleAnalysis(); + if (origin === undefined) delete report.capture.target_origin; + else report.capture.target_origin = origin; + const parent = makeParent(report); + for (const selection of [ + { kind: "summary" }, + { kind: "web-bundle", collection: "summary" }, + { kind: "web-bundle", collection: "endpoints", fields: ["value"] }, + ]) { + const view = query(parent, selection); + if (origin === undefined) + expect(view).not.toHaveProperty("target_origin"); + else expect(view).toHaveProperty("target_origin", origin); + if ( + view.kind === "summary" || + (view.kind === "web-bundle" && view.summary !== null) + ) { + if (origin === undefined) + expect(view.summary).not.toHaveProperty("target_origin"); + else expect(view.summary).toHaveProperty("target_origin", origin); + } + } + }, +); + +it("counts the distinct full parent/report limitation union before bounding", () => { + const base = makeParent({ + limitations: [ + "Parent run was bounded", + "Script evaluation was omitted", + "x".repeat(100_000), + ], + }); + const parent = { + ...base, + limitations: [...base.limitations, "Parent-only condition"], + }; + for (const selection of [ + { kind: "summary" }, + { kind: "web-bundle", collection: "summary" }, + ]) { + const view = query(parent, selection); + expect(view).toHaveProperty("summary.limitation_count", 4); + expect(view.limitations).not.toContain("x".repeat(100_000)); + expect(view.limitations).toContainEqual( + expect.stringContaining("additional limitations omitted"), + ); + } +}); + +it("retains distinct structured unknown scopes on filtered pages", () => { + const unknowns = [ + { + dimension: "parse", + reason: "Source unavailable", + affected_script_keys: ["app.js"], + }, + { + dimension: "parse", + reason: "Source unavailable", + affected_script_keys: ["vendor.js"], + }, + ]; + const parent = makeParent({ unknowns }); + for (const collection of ["endpoints", "inferences"]) { + const view = query(parent, { + kind: "web-bundle", + collection, + script_keys: ["app.js"], + fields: ["value"], + }); + expect(view).toHaveProperty("source_unknowns", unknowns); + expect(view).toHaveProperty("completeness.status", "partial"); + } +}); + +it("pages endpoints and concatenated pages match original records exactly, preserving duplicates", () => { + const parent = makeParent(); + const page1 = query(parent, { + kind: "web-bundle", + collection: "endpoints", + offset: 0, + limit: 2, + }); + const page2 = query(parent, { + kind: "web-bundle", + collection: "endpoints", + offset: 2, + limit: 2, + }); + + expect(page1.kind).toBe("web-bundle"); + expect(page2.kind).toBe("web-bundle"); + if (page1.kind !== "web-bundle" || page2.kind !== "web-bundle") return; + + expect(page1.items).toHaveLength(2); + expect(page2.items).toHaveLength(1); + expect(page1.coverage).toMatchObject({ + status: "page", + examined: 2, + total: 3, + next_offset: 2, + exhausted: false, + }); + expect(page2.coverage).toMatchObject({ + status: "page", + examined: 1, + total: 3, + next_offset: null, + exhausted: true, + }); + + const concatenated = [...page1.items, ...page2.items]; + const original = sampleAnalysis().observations.endpoints; + expect(concatenated).toEqual( + original.map((row, source_index) => ({ source_index, ...row })), + ); + // Preserves duplicate "/api/users" at different locations + expect(concatenated[0]!.value).toBe("/api/users"); + expect(concatenated[1]!.value).toBe("/api/users"); + const first = concatenated[0]!; + const second = concatenated[1]!; + if (!("location" in first) || !("location" in second)) { + throw new Error("Expected endpoint rows with location"); + } + expect(first.location).not.toEqual(second.location); +}); + +it("filters endpoints by values, script_keys, mechanisms, and field projections", () => { + const parent = makeParent(); + const filtered = query(parent, { + kind: "web-bundle", + collection: "endpoints", + values: ["/api/orders"], + fields: ["value"], + }); + + if (filtered.kind !== "web-bundle") + throw new Error("Expected web-bundle view"); + expect(filtered.items).toEqual([{ source_index: 2, value: "/api/orders" }]); + expect(filtered.coverage.total).toBe(1); + + const byScript = query(parent, { + kind: "web-bundle", + collection: "endpoints", + script_keys: ["vendor.js"], + }); + if (byScript.kind !== "web-bundle") + throw new Error("Expected web-bundle view"); + expect(byScript.items).toHaveLength(1); + const item = byScript.items[0]!; + if (!("location" in item)) { + throw new Error("Expected endpoint row with location"); + } + expect(item.location).toMatchObject({ script_key: "vendor.js" }); +}); + +it("pages inference records and filters by kind (endpoint_candidate)", () => { + const parent = makeParent(); + const candidatesOnly = query(parent, { + kind: "web-bundle", + collection: "inferences", + kinds: ["endpoint_candidate"], + }); + + if (candidatesOnly.kind !== "web-bundle") + throw new Error("Expected web-bundle view"); + expect(candidatesOnly.items).toHaveLength(1); + expect(candidatesOnly.items[0]).toMatchObject({ + kind: "endpoint_candidate", + value: "/api/dynamic-user", + confidence: "low", + }); + expect(candidatesOnly.coverage).toMatchObject({ + total: 1, + exhausted: true, + }); +}); + +it("preserves partial producer coverage even when view pagination is exhausted", () => { + const parent = makeParent({ + completeness: { + status: "partial", + parsed_scripts: 1, + parse_failures: 1, + visited_ast_nodes: 50, + }, + }); + + const page = query(parent, { + kind: "web-bundle", + collection: "endpoints", + offset: 0, + limit: 10, + }); + + if (page.kind !== "web-bundle") throw new Error("Expected web-bundle view"); + expect(page.coverage.exhausted).toBe(true); + // Must remain "partial" and never be upgraded to "complete" + expect(page.completeness.status).toBe("partial"); + expect(page.completeness.parse_failures).toBe(1); +}); + +it("serves web bundle summary via kind: summary and kind: web-bundle collection: summary", () => { + const parent = makeParent(); + + const summaryView = query(parent, { kind: "summary" }); + expect(summaryView.kind).toBe("summary"); + if (summaryView.kind === "summary") { + expect(summaryView.summary).toMatchObject({ + target_url: "https://example.test/app", + counts: { + endpoints: 3, + inferences: 2, + endpoint_candidates: 1, + }, + completeness: { status: "partial" }, + }); + } + + const collectionSummary = query(parent, { + kind: "web-bundle", + collection: "summary", + }); + expect(collectionSummary.kind).toBe("web-bundle"); + if (collectionSummary.kind === "web-bundle") { + expect(collectionSummary.summary).toMatchObject({ + target_url: "https://example.test/app", + counts: { + endpoints: 3, + inferences: 2, + endpoint_candidates: 1, + }, + }); + } +}); + +it("returns an empty page when offset is beyond total items", () => { + const parent = makeParent(); + const empty = query(parent, { + kind: "web-bundle", + collection: "endpoints", + offset: 100, + limit: 10, + }); + + if (empty.kind !== "web-bundle") throw new Error("Expected web-bundle view"); + expect(empty.items).toEqual([]); + expect(empty.coverage).toMatchObject({ + status: "empty", + examined: 0, + total: 3, + next_offset: null, + exhausted: true, + }); +}); + +it("returns typed AnalysisResourceConstraintError when a single record exceeds 1 MiB", () => { + const hugeEndpoint = { + value: "/api/" + "x".repeat(1024 * 1024), + mechanism: "call:fetch", + location: { script_key: "app.js", line: 1, column: 0 }, + }; + const parent = makeParent({ + observations: { + ...sampleAnalysis().observations, + endpoints: [hugeEndpoint], + }, + }); + + const result = projectAnalysisView( + parent, + analysisViewRequestSchema.parse({ + kind: "web-bundle", + collection: "endpoints", + }), + ); + + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected resource error"); + expect(result.error._tag).toBe("AnalysisResourceConstraintError"); + expect(result.error.message).toContain("byte budget"); +}); + +it.each([undefined, ["call:fetch"]])( + "counts all matching rows after a byte-limited page: %s", + (mechanisms) => { + const largeRecord = (index: number) => ({ + value: `/api/${index}/` + "x".repeat(300_000), + mechanism: "call:fetch", + location: { script_key: "app.js", line: index, column: 0 }, + }); + const parent = makeParent({ + observations: { + ...sampleAnalysis().observations, + endpoints: [ + largeRecord(1), + largeRecord(2), + largeRecord(3), + largeRecord(4), + ], + }, + }); + + const page = query(parent, { + kind: "web-bundle", + collection: "endpoints", + limit: 10, + mechanisms, + }); + + if (page.kind !== "web-bundle") throw new Error("Expected web-bundle view"); + expect(page.byte_limited).toBe(true); + expect(page.items.length).toBeGreaterThan(0); + expect(page.items.length).toBeLessThan(4); + expect(page.coverage.next_offset).toBe(page.items.length); + expect(page.coverage.exhausted).toBe(false); + expect(page.coverage.total).toBe(4); + }, +); + +it.each(["indices", "values", "script_keys", "mechanisms"])( + "treats an explicit empty %s selector as an empty selection", + (selector) => { + const page = query(makeParent(), { + kind: "web-bundle", + collection: "endpoints", + [selector]: [], + }); + expect(page).toHaveProperty("items", []); + expect(page.coverage).toMatchObject({ + total: 0, + exhausted: true, + next_offset: null, + }); + }, +); + +it("rejects non-web-bundle parent operation with invalid_value", () => { + const invalidParent: AnalysisViewParent = { + evidenceId: `ev_${"d".repeat(64)}`, + operation: "analyze_javascript_application", + limitations: [], + normalizedResult: sampleAnalysis(), + }; + + const result = projectAnalysisView( + invalidParent, + analysisViewRequestSchema.parse({ + kind: "web-bundle", + collection: "endpoints", + }), + ); + + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected failure"); + if (!(result.error instanceof AnalysisInputError)) throw result.error; + expect(result.error.issues[0]!.message).toContain( + "Web-bundle views require analyze_web_bundle", + ); + expect(result.error.issues[0]!.reason).toBe("invalid_value"); +}); + +it("continues filtered pages by matching ordinal while preserving original indices", () => { + const report = sampleAnalysis(); + const row = report.observations.endpoints[0]!; + const endpoints = [ + { ...row, mechanism: "call:sendBeacon" }, + { ...row, location: { ...row.location, script_key: "vendor.js" } }, + row, + { ...row, value: "/api/orders" }, + { ...row, location: { ...row.location, line: 99 } }, + { ...row, location: { ...row.location, line: 100 } }, + { ...row, location: { ...row.location, script_key: "vendor.js" } }, + ]; + const parent = makeParent({ + observations: { ...report.observations, endpoints }, + }); + const selection = { + kind: "web-bundle", + collection: "endpoints", + script_keys: ["app.js"], + values: ["/api/users"], + mechanisms: ["call:fetch"], + limit: 2, + }; + const first = query(parent, selection); + if (first.kind !== "web-bundle") throw new Error("Expected web-bundle page"); + expect(first.coverage).toMatchObject({ + total: 3, + examined: 2, + next_offset: 2, + exhausted: false, + }); + const second = query(parent, { + ...selection, + offset: first.coverage.next_offset, + }); + if (second.kind !== "web-bundle") throw new Error("Expected web-bundle page"); + expect(second.coverage).toMatchObject({ + total: 3, + examined: 1, + next_offset: null, + exhausted: true, + }); + expect([...first.items, ...second.items]).toEqual( + [2, 4, 5].map((source_index) => ({ + source_index, + ...endpoints[source_index], + })), + ); +}); + +it("preserves parent artifact identity across summary and page views", () => { + const artifact = { path: "/app.js", sha256: "e".repeat(64) }; + const parent: AnalysisViewParent = { + ...makeParent(), + artifact, + }; + + const summary = query(parent, { kind: "summary" }); + expect(summary.artifact).toEqual(artifact); + + const page = query(parent, { + kind: "web-bundle", + collection: "endpoints", + }); + expect(page.artifact).toEqual(artifact); +}); + +it("filters endpoints by indices and inferences by script_keys and fields", () => { + const parent = makeParent(); + + const byIndex = query(parent, { + kind: "web-bundle", + collection: "endpoints", + indices: [1], + }); + if (byIndex.kind !== "web-bundle") + throw new Error("Expected web-bundle view"); + expect(byIndex.items).toHaveLength(1); + expect(byIndex.items[0]).toMatchObject({ + value: "/api/users", + location: { script_key: "vendor.js" }, + }); + + const byScriptKey = query(parent, { + kind: "web-bundle", + collection: "inferences", + script_keys: ["vendor.js"], + fields: ["kind", "value"], + }); + if (byScriptKey.kind !== "web-bundle") + throw new Error("Expected web-bundle view"); + expect(byScriptKey.items).toEqual([ + { source_index: 1, kind: "vendor_fingerprint", value: "React" }, + ]); +}); + +it("correctly counts zero findings in summary for an empty web bundle", () => { + const emptyParent = makeParent({ + observations: { + chunks: { nodes: [], edges: [] }, + routes: [], + endpoints: [], + webmcp_declarations: [], + source_maps: { + status: "not_requested", + requested: 0, + processed: 0, + items: [], + }, + }, + inferences: [], + }); + + const summary = query(emptyParent, { kind: "summary" }); + if (summary.kind !== "summary") throw new Error("Expected summary"); + expect(summary.summary).toMatchObject({ + counts: { + endpoints: 0, + inferences: 0, + endpoint_candidates: 0, + chunks: 0, + routes: 0, + }, + }); +}); + +it("rejects malformed parent analysis report", () => { + const malformedParent: AnalysisViewParent = { + evidenceId: `ev_${"e".repeat(64)}`, + operation: "analyze_web_bundle", + limitations: [], + normalizedResult: { malformed: true }, + }; + + const result = projectAnalysisView( + malformedParent, + analysisViewRequestSchema.parse({ + kind: "web-bundle", + collection: "endpoints", + }), + ); + + expect(result.ok).toBe(false); +}); diff --git a/src/domain/analysisView/webBundleView.ts b/src/domain/analysisView/webBundleView.ts new file mode 100644 index 0000000000..56faccf53c --- /dev/null +++ b/src/domain/analysisView/webBundleView.ts @@ -0,0 +1,395 @@ +import { z } from "zod"; +import type { AnalysisError } from "../analysisErrorBase.js"; +import { AnalysisResourceConstraintError } from "../analysisErrorCore.js"; +import { jsonValueByteLength } from "../jsonValueByteLength.js"; +import type { JsonValue } from "../jsonValue.js"; +import { err, ok, type Result } from "../result.js"; +import { + webBundleAnalysisSchema, + type WebBundleAnalysis, +} from "../webBundleAnalysis.js"; +import { recordedBrowserOriginSchema } from "../browserObservation.js"; +import type { + AnalysisViewParent, + AnalysisViewRequest, + UnsignedAnalysisView, +} from "./analysisView.js"; +import { + boundedViewMessages, + completeWithinViewCoverage, + pageViewCoverage, +} from "./analysisViewCoverage.js"; +import { analysisViewInputError } from "./analysisViewErrors.js"; +import { createAnalysisViewResultReader } from "./analysisViewParent.js"; + +const PAGE_BYTES = 1024 * 1024; +const DIGEST_BYTES = jsonValueByteLength({ view_digest: "0".repeat(64) }) - 1; +const PROJECTION_BYTES = PAGE_BYTES - DIGEST_BYTES; + +const endpointSchema = + webBundleAnalysisSchema.shape.observations.shape.endpoints.element; +const inferenceSchema = webBundleAnalysisSchema.shape.inferences.element; +const sourceLocationSchema = endpointSchema.shape.location; +const basisSchema = inferenceSchema.shape.basis.element; + +const pageShape = { + kind: z.literal("web-bundle"), + offset: z.number().int().nonnegative().default(0), + limit: z.number().int().positive().default(64), +}; + +export const webBundleViewRequestSchema = z.discriminatedUnion("collection", [ + z.strictObject({ + ...pageShape, + collection: z.literal("endpoints"), + indices: z.array(z.number().int().nonnegative()).optional(), + values: z.array(z.string()).optional(), + script_keys: z.array(z.string()).optional(), + mechanisms: z.array(z.string()).optional(), + fields: z.array(endpointSchema.keyof()).optional(), + }), + z.strictObject({ + ...pageShape, + collection: z.literal("inferences"), + indices: z.array(z.number().int().nonnegative()).optional(), + kinds: z.array(inferenceSchema.shape.kind).optional(), + values: z.array(z.string()).optional(), + script_keys: z.array(z.string()).optional(), + fields: z.array(inferenceSchema.keyof()).optional(), + }), + z.strictObject({ + kind: z.literal("web-bundle"), + collection: z.literal("summary"), + }), +]); + +export type WebBundleViewRequest = z.output; + +export const webBundleSummarySchema = z.strictObject({ + target_url: z.string(), + target_origin: recordedBrowserOriginSchema.nullable().optional(), + scripts_observed: z.number().int().nonnegative(), + scripts_analyzed: z.number().int().nonnegative(), + counts: z.strictObject({ + endpoints: z.number().int().nonnegative(), + inferences: z.number().int().nonnegative(), + endpoint_candidates: z.number().int().nonnegative(), + chunks: z.number().int().nonnegative(), + routes: z.number().int().nonnegative(), + }), + completeness: webBundleAnalysisSchema.shape.completeness, + limitation_count: z.number().int().nonnegative(), +}); + +export const webBundleEndpointRowSchema = z.strictObject({ + source_index: z.number().int().nonnegative(), + value: endpointSchema.shape.value.optional(), + mechanism: endpointSchema.shape.mechanism.optional(), + location: sourceLocationSchema.optional(), +}); + +export const webBundleInferenceRowSchema = z.strictObject({ + source_index: z.number().int().nonnegative(), + kind: inferenceSchema.shape.kind.optional(), + value: inferenceSchema.shape.value.optional(), + confidence: inferenceSchema.shape.confidence.optional(), + basis: z.array(basisSchema).optional(), +}); + +export const webBundleViewItemsSchema = z.union([ + z.array(webBundleEndpointRowSchema), + z.array(webBundleInferenceRowSchema), +]); + +const readReport = createAnalysisViewResultReader(webBundleAnalysisSchema); + +const choose = ( + row: T, + fields: ReadonlySet | undefined, +): Partial => { + if (fields === undefined) return { ...row }; + const out: Partial = {}; + for (const key of Object.keys(row) as (keyof T)[]) { + if (fields.has(String(key))) out[key] = row[key]; + } + return out; +}; + +/** Historical projection of completed analyze_web_bundle Evidence. */ +export const projectWebBundleView = ( + parent: AnalysisViewParent, + selected: AnalysisViewRequest, +): Result => { + if (selected.kind !== "web-bundle" && selected.kind !== "summary") + return err( + analysisViewInputError([ + { + path: ["view", "kind"], + reason: "invalid_value", + message: "Select a web-bundle or summary view.", + }, + ]), + ); + + if (parent.operation !== "analyze_web_bundle") + return err( + analysisViewInputError([ + { + path: ["source", "operation"], + reason: "invalid_value", + message: "Web-bundle views require analyze_web_bundle Evidence.", + }, + ]), + ); + + const parsed = readReport(parent); + if (!parsed.ok) return parsed; + const report: WebBundleAnalysis = parsed.value; + + const targetOrigin = + report.capture.target_origin === undefined + ? {} + : { target_origin: report.capture.target_origin }; + const sourceLimitations = [ + ...new Set([...parent.limitations, ...report.limitations]), + ]; + + const shared = { + parent_evidence_id: parent.evidenceId, + parent_operation: "analyze_web_bundle" as const, + parent_digest: parent.evidenceId.slice(3), + artifact: parent.artifact ?? null, + target_url: report.capture.target_url, + ...targetOrigin, + source_unknowns: report.unknowns, + limitations: boundedViewMessages( + sourceLimitations, + parent.evidenceId, + "limitations", + ), + unknowns: boundedViewMessages( + [ + "Static bundle findings are observations or bounded inferences; REA does not execute captured JavaScript.", + "String-built routes and endpoints, encrypted configuration, and server-side behavior may remain unknown.", + ...report.unknowns.map((u) => `${u.dimension}: ${u.reason}`), + ], + parent.evidenceId, + "unknowns", + ), + }; + + const tooLarge = (collection: string): Result => + err( + new AnalysisResourceConstraintError( + "inspect_analysis_view", + "transport", + "The requested web-bundle page cannot fit the analysis-view byte budget; select fewer fields or a smaller selection. The complete parent is retained.", + { + parent_evidence_id: parent.evidenceId, + limit_bytes: PAGE_BYTES, + collection, + }, + ), + ); + + const buildSummary = (): z.output => ({ + target_url: report.capture.target_url, + ...targetOrigin, + scripts_observed: report.capture.scripts_observed, + scripts_analyzed: report.capture.scripts_analyzed, + counts: { + endpoints: report.observations.endpoints.length, + inferences: report.inferences.length, + endpoint_candidates: report.inferences.reduce( + (count, inference) => + count + Number(inference.kind === "endpoint_candidate"), + 0, + ), + chunks: report.observations.chunks.nodes.length, + routes: report.observations.routes.length, + }, + completeness: report.completeness, + limitation_count: sourceLimitations.length, + }); + + if (selected.kind === "summary") { + const summary = buildSummary(); + const result: UnsignedAnalysisView = { + kind: "summary", + view: selected, + ...shared, + summary, + coverage: completeWithinViewCoverage(1, 1), + }; + return jsonValueByteLength( + result as unknown as JsonValue, + PROJECTION_BYTES, + ) > PROJECTION_BYTES + ? tooLarge("summary") + : ok(result); + } + + const view = selected; + if (view.collection === "summary") { + const summary = buildSummary(); + const result: UnsignedAnalysisView = { + kind: "web-bundle", + view, + ...shared, + completeness: report.completeness, + items: [], + summary, + byte_limited: false, + coverage: completeWithinViewCoverage(1, 1), + }; + return jsonValueByteLength( + result as unknown as JsonValue, + PROJECTION_BYTES, + ) > PROJECTION_BYTES + ? tooLarge("summary") + : ok(result); + } + + type Item = + | z.output + | z.output; + const fields = + view.fields === undefined ? undefined : new Set(view.fields); + const deliverRows = ( + rows: readonly T[], + matches: ((row: T, source_index: number) => boolean) | undefined, + project: (row: T, source_index: number) => Item, + ): Result => { + const baseEnvelope = { + ...shared, + kind: "web-bundle" as const, + view, + completeness: report.completeness, + items: [], + summary: null, + byte_limited: false, + coverage: { + status: "complete-within-view" as const, + examined: Number.MAX_SAFE_INTEGER, + total: Number.MAX_SAFE_INTEGER, + next_offset: Number.MAX_SAFE_INTEGER, + exhausted: false, + }, + }; + + let bytes = + DIGEST_BYTES + + jsonValueByteLength( + baseEnvelope as unknown as JsonValue, + PROJECTION_BYTES, + ); + + if (bytes > PAGE_BYTES) return tooLarge(view.collection); + + const items: Item[] = []; + let byte_limited = false; + let total = matches === undefined ? rows.length : 0; + + // Unfiltered pages use the producer's array length and direct offsets. + // Filtered pages count every match but only allocate selected output rows. + for ( + let source_index = + matches === undefined ? Math.min(view.offset, rows.length) : 0; + source_index < rows.length; + source_index++ + ) { + const row = rows[source_index]!; + if (matches !== undefined && !matches(row, source_index)) continue; + const ordinal = matches === undefined ? source_index : total++; + if (ordinal < view.offset) continue; + if (items.length >= view.limit || byte_limited) { + if (matches === undefined) break; + continue; + } + const item = project(row, source_index); + const addedBytes = + jsonValueByteLength(item as JsonValue, PAGE_BYTES - bytes) + + (items.length > 0 ? 1 : 0); + if (bytes + addedBytes > PAGE_BYTES) { + if (items.length === 0) return tooLarge(view.collection); + byte_limited = true; + if (matches === undefined) break; + continue; + } + bytes += addedBytes; + items.push(item); + } + + const coverage = pageViewCoverage(view.offset, items.length, total); + const result: UnsignedAnalysisView = { + ...shared, + kind: "web-bundle" as const, + view, + completeness: report.completeness, + items: webBundleViewItemsSchema.parse(items), + summary: null, + byte_limited, + coverage, + }; + + return jsonValueByteLength( + result as unknown as JsonValue, + PROJECTION_BYTES, + ) > PROJECTION_BYTES + ? tooLarge(view.collection) + : ok(result); + }; + + if (view.collection === "endpoints") { + const indices = + view.indices !== undefined ? new Set(view.indices) : undefined; + const values = view.values !== undefined ? new Set(view.values) : undefined; + const scriptKeys = + view.script_keys !== undefined ? new Set(view.script_keys) : undefined; + const mechanisms = + view.mechanisms !== undefined ? new Set(view.mechanisms) : undefined; + + const filters = [indices, values, scriptKeys, mechanisms]; + return deliverRows( + filters.some((filter) => filter?.size === 0) + ? [] + : report.observations.endpoints, + filters.every((filter) => filter === undefined) + ? undefined + : (row, index) => + (indices === undefined || indices.has(index)) && + (values === undefined || values.has(row.value)) && + (scriptKeys === undefined || + scriptKeys.has(row.location.script_key)) && + (mechanisms === undefined || mechanisms.has(row.mechanism)), + (row, source_index) => ({ + source_index, + ...choose(row, fields), + }), + ); + } + + const indices = + view.indices !== undefined ? new Set(view.indices) : undefined; + const kinds = view.kinds !== undefined ? new Set(view.kinds) : undefined; + const values = view.values !== undefined ? new Set(view.values) : undefined; + const scriptKeys = + view.script_keys !== undefined ? new Set(view.script_keys) : undefined; + + const filters = [indices, kinds, values, scriptKeys]; + return deliverRows( + filters.some((filter) => filter?.size === 0) ? [] : report.inferences, + filters.every((filter) => filter === undefined) + ? undefined + : (row, index) => + (indices === undefined || indices.has(index)) && + (kinds === undefined || kinds.has(row.kind)) && + (values === undefined || values.has(row.value)) && + (scriptKeys === undefined || + row.basis.some((basis) => scriptKeys.has(basis.script_key))), + (row, source_index) => ({ + source_index, + ...choose(row, fields), + }), + ); +}; diff --git a/tests/boundary/mcp/webBundleViewsMcp.test.ts b/tests/boundary/mcp/webBundleViewsMcp.test.ts new file mode 100644 index 0000000000..b8d6b44be2 --- /dev/null +++ b/tests/boundary/mcp/webBundleViewsMcp.test.ts @@ -0,0 +1,348 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it, onTestFinished } from "vitest"; +import { createServer } from "../../../src/server/createServer.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { + webBundlePageViewResultSchema, + webBundleSummaryViewResultSchema, +} from "../../../src/domain/analysisView/analysisView.js"; +import type { WebBundleAnalysis } from "../../../src/domain/webBundleAnalysis.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; +import { advertisedMcpToolSchemas } from "../../fixtures/advertisedMcpTool.js"; + +const sampleWebBundleAnalysis = (): WebBundleAnalysis => ({ + capture: { + target_url: "https://example.test/app", + target_origin: "https://example.test", + scripts_observed: 3, + scripts_analyzed: 2, + source_artifacts: [], + }, + observations: { + chunks: { + nodes: [ + { + script_key: "app.js", + url: "https://example.test/app.js", + artifact_sha256: "a".repeat(64), + bytes: 1200, + }, + { + script_key: "vendor.js", + url: "https://example.test/vendor.js", + artifact_sha256: "b".repeat(64), + bytes: 4500, + }, + ], + edges: [], + }, + routes: [], + endpoints: [ + { + value: "/api/users", + mechanism: "call:fetch", + location: { script_key: "app.js", line: 10, column: 2 }, + }, + { + value: "/api/users", + mechanism: "call:fetch", + location: { script_key: "vendor.js", line: 42, column: 8 }, + }, + { + value: "/api/orders", + mechanism: "call:sendBeacon", + location: { script_key: "app.js", line: 85, column: 4 }, + }, + ], + webmcp_declarations: [], + source_maps: { + status: "not_requested", + requested: 0, + processed: 0, + items: [], + }, + }, + inferences: [ + { + kind: "endpoint_candidate", + value: "/api/dynamic-user", + confidence: "low", + basis: [ + { + script_key: "app.js", + artifact_sha256: "a".repeat(64), + line: 25, + column: 4, + detector: "call:client.get:unverified-network-receiver", + }, + ], + }, + { + kind: "vendor_fingerprint", + value: "React", + confidence: "high", + basis: [ + { + script_key: "vendor.js", + artifact_sha256: "b".repeat(64), + line: 1, + column: 0, + detector: "vendor:react", + }, + ], + }, + ], + unknowns: [ + { + dimension: "network", + reason: "Unverified client receiver", + affected_script_keys: ["app.js"], + }, + ], + completeness: { + status: "partial", + parsed_scripts: 2, + parse_failures: 1, + visited_ast_nodes: 150, + }, + limitations: ["Script evaluation was omitted"], +}); + +const makeParent = () => + createEvidence( + undefined, + { id: "rea-browser", name: "REA Browser", version: "6.8.0" }, + { + operation: "analyze_web_bundle", + parameters: { target_url: "https://example.test/app" }, + result: sampleWebBundleAnalysis(), + }, + ); + +const connect = async () => { + const schemaValidation = createMcpSchemaValidation(); + const session = createTestBinarySession(() => { + throw new Error("selected views must not start a deep provider"); + }); + const server = createServer( + { kind: "session", session }, + { + environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: "full" }, + }, + ); + const client = new Client( + { name: "web-bundle-view-mcp", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); + onTestFinished(async () => { + await client.close(); + await server.close(); + await session.close(); + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + const schemas = await advertisedMcpToolSchemas( + client, + "inspect_analysis_view", + schemaValidation.assertions, + ); + return { client, session, schemas }; +}; + +it("projects web bundle summary views over MCP using retained and inline sources", async () => { + const { client, session, schemas } = await connect(); + const parent = makeParent(); + expect(session.recordEvidence(parent).ok).toBe(true); + + const summaryCall = await client.callTool({ + ...schemas.toolDefinition, + arguments: { + source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, + view: { kind: "summary" }, + }, + }); + expect(summaryCall.isError, JSON.stringify(summaryCall)).not.toBe(true); + expect( + schemas.ajv.validate(schemas.outputSchema, summaryCall.structuredContent), + JSON.stringify(schemas.ajv.errors), + ).toBe(true); + const summaryResult = webBundleSummaryViewResultSchema.parse( + parseEvidence(summaryCall.structuredContent).normalized_result, + ); + expect(summaryResult.kind).toBe("summary"); + expect(summaryResult.parent_operation).toBe("analyze_web_bundle"); + expect(summaryResult.summary).toMatchObject({ + target_url: "https://example.test/app", + counts: { endpoints: 3, inferences: 2, endpoint_candidates: 1 }, + completeness: { status: "partial" }, + }); + + const collectionSummaryCall = await client.callTool({ + ...schemas.toolDefinition, + arguments: { + source: { kind: "inline", evidence: parent }, + view: { kind: "web-bundle", collection: "summary" }, + }, + }); + expect(collectionSummaryCall.isError).not.toBe(true); + expect( + schemas.ajv.validate( + schemas.outputSchema, + collectionSummaryCall.structuredContent, + ), + JSON.stringify(schemas.ajv.errors), + ).toBe(true); + const collectionSummaryResult = webBundlePageViewResultSchema.parse( + parseEvidence(collectionSummaryCall.structuredContent).normalized_result, + ); + expect(collectionSummaryResult.kind).toBe("web-bundle"); + expect(collectionSummaryResult.summary).toMatchObject({ + target_url: "https://example.test/app", + counts: { endpoints: 3, inferences: 2, endpoint_candidates: 1 }, + }); +}); + +it("pages web bundle endpoints and inferences over MCP using evidence-file source", async () => { + const { client, schemas } = await connect(); + const parent = makeParent(); + const tempDir = await createTestTempDirectory("rea-webbundle-mcp-"); + const evidenceFile = join(tempDir, "webbundle-evidence.json"); + await writeFile(evidenceFile, JSON.stringify(parent)); + + const page1Call = await client.callTool({ + ...schemas.toolDefinition, + arguments: { + source: { kind: "evidence-file", path: evidenceFile }, + view: { + kind: "web-bundle", + collection: "endpoints", + offset: 0, + limit: 2, + }, + }, + }); + expect(page1Call.isError).not.toBe(true); + expect( + schemas.ajv.validate(schemas.outputSchema, page1Call.structuredContent), + JSON.stringify(schemas.ajv.errors), + ).toBe(true); + const page1 = webBundlePageViewResultSchema.parse( + parseEvidence(page1Call.structuredContent).normalized_result, + ); + expect(page1.items).toHaveLength(2); + expect(page1.items[0]).toMatchObject({ + source_index: 0, + value: "/api/users", + }); + expect(page1.items[1]).toMatchObject({ + source_index: 1, + value: "/api/users", + }); + expect(page1.coverage).toMatchObject({ + status: "page", + examined: 2, + total: 3, + next_offset: 2, + exhausted: false, + }); + expect(page1.completeness.status).toBe("partial"); + + const page2Call = await client.callTool({ + ...schemas.toolDefinition, + arguments: { + source: { kind: "evidence-file", path: evidenceFile }, + view: { + kind: "web-bundle", + collection: "endpoints", + offset: 2, + limit: 2, + }, + }, + }); + expect(page2Call.isError).not.toBe(true); + expect( + schemas.ajv.validate(schemas.outputSchema, page2Call.structuredContent), + JSON.stringify(schemas.ajv.errors), + ).toBe(true); + const page2 = webBundlePageViewResultSchema.parse( + parseEvidence(page2Call.structuredContent).normalized_result, + ); + expect(page2.items).toHaveLength(1); + expect(page2.items[0]).toMatchObject({ + source_index: 2, + value: "/api/orders", + }); + expect(page2.coverage).toMatchObject({ + status: "page", + examined: 1, + total: 3, + next_offset: null, + exhausted: true, + }); + + const candidatesCall = await client.callTool({ + ...schemas.toolDefinition, + arguments: { + source: { kind: "evidence-file", path: evidenceFile }, + view: { + kind: "web-bundle", + collection: "inferences", + kinds: ["endpoint_candidate"], + }, + }, + }); + expect(candidatesCall.isError).not.toBe(true); + expect( + schemas.ajv.validate( + schemas.outputSchema, + candidatesCall.structuredContent, + ), + JSON.stringify(schemas.ajv.errors), + ).toBe(true); + const candidatesPage = webBundlePageViewResultSchema.parse( + parseEvidence(candidatesCall.structuredContent).normalized_result, + ); + expect(candidatesPage.items).toEqual([ + expect.objectContaining({ + source_index: 0, + kind: "endpoint_candidate", + value: "/api/dynamic-user", + confidence: "low", + }), + ]); + + const emptyCall = await client.callTool({ + ...schemas.toolDefinition, + arguments: { + source: { kind: "evidence-file", path: evidenceFile }, + view: { + kind: "web-bundle", + collection: "endpoints", + offset: 100, + limit: 10, + }, + }, + }); + expect(emptyCall.isError).not.toBe(true); + expect( + schemas.ajv.validate(schemas.outputSchema, emptyCall.structuredContent), + JSON.stringify(schemas.ajv.errors), + ).toBe(true); + const emptyPage = webBundlePageViewResultSchema.parse( + parseEvidence(emptyCall.structuredContent).normalized_result, + ); + expect(emptyPage.items).toEqual([]); + expect(emptyPage.coverage).toMatchObject({ + status: "empty", + examined: 0, + total: 3, + next_offset: null, + exhausted: true, + }); +});