diff --git a/.agents/skills/reverse-engineer-anything/references/javascript-applications.md b/.agents/skills/reverse-engineer-anything/references/javascript-applications.md index a074dad75f..191060b90c 100644 --- a/.agents/skills/reverse-engineer-anything/references/javascript-applications.md +++ b/.agents/skills/reverse-engineer-anything/references/javascript-applications.md @@ -78,6 +78,15 @@ classification, and unavailable sources never establish route reachability or the complete application route denominator. See the [route inventory guide](https://github.com/morluto/rea/blob/main/docs/javascript-route-inventory.md). +For captured endpoint observations and inference facts, `analyze_web_bundle` +also accepts `view: {kind: "web-bundle", collection: "endpoints", limit: 32}` +with `detail: "summary"`. Read later `endpoints`, `inferences`, or `summary` +pages from the retained parent. Filter before requesting detail; retain +`source_index` together with parent ID and collection, and request `location` +or `basis` when tracing sources. Page exhaustion does not mean producer +completeness: check `completeness`, structured `source_unknowns`, and limitations. +Endpoint candidates remain inferences rather than confirmed network behavior. + ## Read a module without repeating analysis Use the actual returned parent ID, on the same connection, as `source.evidence_id`. diff --git a/docs/javascript-route-inventory.md b/docs/javascript-route-inventory.md index a7e9901842..bb20b18901 100644 --- a/docs/javascript-route-inventory.md +++ b/docs/javascript-route-inventory.md @@ -25,18 +25,50 @@ Keep the original analysis inputs in that call. For the CLI, pass the same selection as `--view '' --detail summary --evidence_output routes.json`. The complete Evidence bundle is written without overwriting existing files. Unselected delivery retains the existing complete result. A web-bundle summary -requires an explicit route view; application summaries also support the existing +requires an explicit route or web-bundle view; application summaries also support the existing general summary when `view` is omitted. Use the returned `normalized_result.parent_evidence_id` with `inspect_analysis_view` on the same connection. The CLI's exported bundle can be reopened with `inspect-evidence-bundle`; its `records` contain the complete -Evidence. To use `inspect_analysis_view`'s `evidence-file` source, extract the -selected parent record into a separate local Evidence JSON file (that source -accepts one Evidence record, not a bundle). Views do not reopen source files or +Evidence. `inspect_analysis_view`'s `evidence-file` source accepts either one +Evidence record or an exported bundle with the selected `evidence_id`. +Views do not reopen source files or reconnect to a browser. Historical Evidence without `route_inventory` is unmeasured, rather than an empty route set. +## Page captured endpoint observations and inferences + +For `analyze_web_bundle`, select `detail: "summary"` and +`view: {kind: "web-bundle", collection: "endpoints", limit: 32}`. The CLI +accepts the same JSON through `--view`, with `--detail summary` and +`--evidence_output bundle.json`. Query the returned parent with +`inspect_analysis_view` to read `endpoints`, `inferences`, or `summary` without +recapturing the page or repeating JavaScript analysis. + +Both row collections accept exact `indices`, `values`, `script_keys`, `fields`, +`offset`, and `limit`; endpoints also accept `mechanisms`, and inferences accept +`kinds`. Filters combine with AND, and an explicit empty selector matches no +rows. Every row retains its `source_index` in the original collection, including +when other fields are omitted. This identity is scoped to the parent Evidence +and collection. Original ordering and separate occurrences of duplicate values +are preserved; request endpoint `location` or inference `basis` for source details. + +Follow `coverage.next_offset` until `coverage.exhausted`; those fields describe +the selected rows, independently of producer `completeness`. Each response also +preserves structured `source_unknowns`, including `affected_script_keys`, so a +filtered or value-only page cannot hide a source gap. The summary's +`limitation_count` counts distinct parent and report limitations before display +bounding. An `endpoint_candidate` remains an inference, with its original +confidence and basis, and does not prove a backend endpoint or reachability. + +Pages obey the shared 1 MiB projection budget; MCP transport applies its own +wire-size guard. A byte-limited page returns a continuation; an indivisible +oversized record or metadata returns a typed resource error. Select fewer +fields or retrieve the complete retained parent through `get_evidence_bundle`, +`export_evidence_bundle`, or the CLI's exported bundle. Full Evidence is never +silently truncated. + Collections are `summary`, `routes`, `occurrences`, `sources`, and `unknowns`. All row collections support exact batch `where.ids`, field selection, `offset`, and `limit`. An explicit empty filter array matches nothing. Routes also support diff --git a/experiments/analysis-efficiency/web-bundle-paging.mjs b/experiments/analysis-efficiency/web-bundle-paging.mjs new file mode 100644 index 0000000000..6c8c035fc5 --- /dev/null +++ b/experiments/analysis-efficiency/web-bundle-paging.mjs @@ -0,0 +1,341 @@ +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import { readFile, writeFile, mkdir, mkdtemp, rm } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { tmpdir } from "node:os"; +import { createRequire } from "node:module"; +import { pathToFileURL } from "node:url"; +import { execFile } from "node:child_process"; +import { promisify, parseArgs } from "node:util"; +import { performance } from "node:perf_hooks"; +// Public CLI capture and same-session retained MCP queries. All assets and output +// paths are explicitly supplied; this verifier installs no browser or application. +const { values } = parseArgs({ + options: { + repository: { type: "string" }, + mode: { type: "string" }, + output: { type: "string" }, + parent: { type: "string" }, + application: { type: "string" }, + browser: { type: "string" }, + task: { type: "string", default: "all" }, + "verify-cli": { type: "boolean", default: false }, + }, +}); +assert.ok(values.repository && values.mode && values.output); +const repo = resolve(values.repository), + mode = values.mode, + out = resolve(values.output), + bundle = values.parent && resolve(values.parent); +assert.ok(["capture", "complete", "selected"].includes(mode)); +assert.ok(["all", "first-candidate"].includes(values.task)); +assert.ok(mode === "capture" ? values.application && values.browser : bundle); +await mkdir(out); +const require = createRequire(join(repo, "package.json")); +const { Client } = await import( + pathToFileURL(require.resolve("@modelcontextprotocol/client")) +); +const { StdioClientTransport } = await import( + pathToFileURL(require.resolve("@modelcontextprotocol/client/stdio")) +); +const env = { + PATH: process.env.PATH, + NODE_OPTIONS: "--max-old-space-size=3072", + UV_THREADPOOL_SIZE: "2", + REA_LOG_LEVEL: "silent", + REA_MCP_INPUT_SCHEMA_PROFILE: "compact", + REA_MCP_TOOL_FAMILIES: "application,session", +}; +if (mode === "capture") { + const { chromium } = require("playwright-core"); + const assets = resolve(values.application); + const allowed = new Set( + await (await import("node:fs/promises")).readdir(assets), + ); + const server = createServer(async (req, res) => { + try { + const name = + new URL(req.url, "http://127.0.0.1").pathname.slice(1) || "index.html"; + assert.ok(allowed.has(name)); + const data = await readFile(join(assets, name)); + res.setHeader( + "Content-Type", + name.endsWith(".js") + ? "text/javascript" + : name.endsWith(".css") + ? "text/css" + : name.endsWith(".json") + ? "application/json" + : "text/html", + ); + res.setHeader( + "Content-Type", + res.getHeader("Content-Type") + "; charset=utf-8", + ); + res.end(data); + } catch { + res.writeHead(404); + res.end(); + } + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const origin = `http://127.0.0.1:${server.address().port}`; + const userData = await mkdtemp(join(tmpdir(), "rea-6198-chrome-")); + let context; + try { + context = await chromium.launchPersistentContext(userData, { + executablePath: resolve(values.browser), + headless: true, + args: ["--remote-debugging-port=0", "--no-sandbox"], + }); + context + .pages()[0] + .on("pageerror", (e) => console.error("PAGE ERROR", e.message)); + context + .pages()[0] + .on("response", (r) => console.error("RESPONSE", r.status(), r.url())); + await context.pages()[0].goto(origin, { waitUntil: "networkidle" }); + const port = ( + await readFile(join(userData, "DevToolsActivePort"), "utf8") + ).split("\n")[0]; + const endpoint = `http://127.0.0.1:${port}`; + const targets = await (await fetch(`${endpoint}/json/list`)).json(); + const target = targets.find( + (t) => t.type === "page" && t.url === origin + "/", + ); + assert.ok(target); + const started = performance.now(); + const { stdout, stderr } = await promisify(execFile)( + process.execPath, + [ + join(repo, "scripts/rea.mjs"), + "analyze-web-bundle", + endpoint, + target.id, + "--observation-ms", + "1000", + "--max-heap-mb", + "1024", + "--analysis-timeout-ms", + "60000", + "--total-analysis-timeout-ms", + "90000", + "--format", + "json", + ], + { cwd: repo, env, maxBuffer: 64 * 1024 ** 2, timeout: 120000 }, + ); + await writeFile(join(out, "stderr.txt"), stderr); + const evidence = JSON.parse(stdout); + assert.equal(evidence.operation, "analyze_web_bundle"); + assert.ok(evidence.normalized_result.observations); + await writeFile(join(out, "parent.json"), JSON.stringify(evidence)); + // Public export/import format comes from the CLI's complete Evidence publication. + console.log( + JSON.stringify({ + capture_ms: performance.now() - started, + evidence_id: evidence.evidence_id, + counts: { + endpoints: evidence.normalized_result.observations.endpoints.length, + inferences: evidence.normalized_result.inferences.length, + }, + completeness: evidence.normalized_result.completeness, + }), + ); + } finally { + await context?.close(); + await new Promise((r) => server.close(r)); + await rm(userData, { recursive: true, force: true }); + } + process.exit(0); +} +const parent = JSON.parse(await readFile(bundle, "utf8")); +const report = parent.normalized_result; +const { createEvidenceBundle } = await import( + pathToFileURL(join(repo, "dist/domain/evidenceBundle.js")) +); +const bundleFile = join(out, "input-bundle.json"); +await writeFile(bundleFile, JSON.stringify(createEvidenceBundle([parent]))); +const transport = new StdioClientTransport({ + command: process.execPath, + args: [join(repo, "scripts/rea.mjs"), "mcp"], + cwd: repo, + env, + stderr: "pipe", +}); +transport.stderr?.resume(); +const client = new Client({ + name: "rea-web-bundle-paging-verifier", + version: "1", +}); +const calls = []; +let catalog; +async function invoke(name, args) { + const toolDefinition = catalog.tools.find((t) => t.name === name); + assert.ok(toolDefinition); + const start = performance.now(); + const result = await client.callTool( + { name, arguments: args }, + { toolDefinition, timeout: 90000 }, + ); + calls.push({ + name, + arguments: args, + result, + elapsed_ms: performance.now() - start, + }); + assert.notEqual(result.isError, true, JSON.stringify(result)); + return result.structuredContent; +} +try { + await client.connect(transport); + catalog = await client.listTools(); + await writeFile(join(out, "catalog.json"), JSON.stringify(catalog)); + const runs = []; + for (const temperature of ["cold", "warm"]) { + const start = performance.now(); + const callStart = calls.length; + if (temperature === "cold") + await invoke("import_evidence_bundle", { path: bundleFile }); + if (mode === "complete") { + const full = await invoke("get_evidence_bundle", {}); + const actual = full.result.records.find( + (r) => r.evidence_id === parent.evidence_id, + ); + assert.deepEqual(actual, parent); + runs.push({ + temperature, + elapsed_ms: performance.now() - start, + calls: calls.length - callStart, + }); + continue; + } + const source = { + kind: "retained-evidence", + evidence_id: parent.evidence_id, + }; + if (values.task === "all") { + const summary = await invoke("inspect_analysis_view", { + source, + view: { kind: "web-bundle", collection: "summary" }, + }); + assert.equal( + summary.normalized_result.summary.counts.endpoints, + report.observations.endpoints.length, + ); + } + if (values.task === "first-candidate") { + const value = await invoke("inspect_analysis_view", { + source, + view: { + kind: "web-bundle", + collection: "inferences", + kinds: ["endpoint_candidate"], + limit: 1, + }, + }); + const index = report.inferences.findIndex( + (row) => row.kind === "endpoint_candidate", + ); + assert.equal( + value.normalized_result.coverage.total, + report.inferences.filter((row) => row.kind === "endpoint_candidate") + .length, + ); + assert.deepEqual( + value.normalized_result.items, + index < 0 ? [] : [{ source_index: index, ...report.inferences[index] }], + ); + assert.deepEqual( + value.normalized_result.source_unknowns, + report.unknowns, + ); + } else + for (const collection of ["endpoints", "inferences"]) { + const expected = + collection === "endpoints" + ? report.observations.endpoints + : report.inferences; + const rows = []; + let offset = 0; + do { + const value = await invoke("inspect_analysis_view", { + source, + view: { kind: "web-bundle", collection, offset, limit: 32 }, + }); + const page = value.normalized_result; + assert.deepEqual(page.source_unknowns, report.unknowns); + assert.deepEqual(page.completeness, report.completeness); + assert.equal(page.coverage.total, expected.length); + for (const row of page.items) { + assert.deepEqual(row, { + source_index: rows.length, + ...expected[rows.length], + }); + rows.push(row); + } + if (page.coverage.exhausted) break; + assert.ok(page.coverage.next_offset > offset); + offset = page.coverage.next_offset; + } while (true); + assert.equal(rows.length, expected.length); + } + runs.push({ + temperature, + elapsed_ms: performance.now() - start, + calls: calls.length - callStart, + }); + } + if (mode === "selected" && values["verify-cli"]) { + const input = JSON.stringify({ + source: { kind: "evidence-file", path: bundle }, + view: { + kind: "web-bundle", + collection: "endpoints", + fields: ["value", "location"], + limit: 32, + }, + }); + const { stdout } = await promisify(execFile)( + process.execPath, + [ + join(repo, "scripts/rea.mjs"), + "inspect-analysis-view", + input, + "--format", + "json", + ], + { cwd: repo, env, maxBuffer: 64 * 1024 ** 2, timeout: 90000 }, + ); + const cli = JSON.parse(stdout); + assert.deepEqual( + cli.normalized_result.items, + report.observations.endpoints.slice(0, 32).map((r, source_index) => ({ + source_index, + value: r.value, + location: r.location, + })), + ); + await writeFile(join(out, "cli.json"), stdout); + } + await writeFile(join(out, "calls.json"), JSON.stringify(calls)); + await writeFile( + join(out, "metrics.json"), + JSON.stringify( + { + mode, + task: values.task, + runs, + counts: { + endpoints: report.observations.endpoints.length, + inferences: report.inferences.length, + }, + }, + null, + 2, + ), + ); + console.log(JSON.stringify({ mode, task: values.task, runs })); +} finally { + await client.close(); +} diff --git a/src/cli/browserCommands.ts b/src/cli/browserCommands.ts index 21868d3140..5a967c5134 100644 --- a/src/cli/browserCommands.ts +++ b/src/cli/browserCommands.ts @@ -1,6 +1,6 @@ import { parseJavaScriptRouteDelivery, - javascriptRouteDeliveryOptions, + webBundleDeliveryOptions, } from "./javascriptRouteDelivery.js"; import { deliverCliAnalysisEvidence } from "./analysisEvidenceDelivery.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; @@ -164,7 +164,7 @@ const registerBundleAnalysis = ( }), options: z.object({ ...browserScopeOptions, - ...javascriptRouteDeliveryOptions, + ...webBundleDeliveryOptions, observationMs: observationDuration(500), fetchSourceMaps: z .boolean() diff --git a/src/cli/javascriptRouteDelivery.ts b/src/cli/javascriptRouteDelivery.ts index 71e254c27c..05e07f5f14 100644 --- a/src/cli/javascriptRouteDelivery.ts +++ b/src/cli/javascriptRouteDelivery.ts @@ -18,6 +18,13 @@ export const javascriptRouteDeliveryOptions = { ), }; +export const webBundleDeliveryOptions = { + ...javascriptRouteDeliveryOptions, + view: javascriptRouteDeliveryOptions.view.describe( + "JSON web-bundle (endpoints, inferences, summary) or javascript-routes view; requires --detail summary and --evidence_output", + ), +}; + /** Validate delivery before opening artifacts or attaching a browser. */ export const parseJavaScriptRouteDelivery = ( operation: string, diff --git a/src/contracts/browserToolContracts.ts b/src/contracts/browserToolContracts.ts index b591838b3f..27b24d8ca6 100644 --- a/src/contracts/browserToolContracts.ts +++ b/src/contracts/browserToolContracts.ts @@ -206,7 +206,7 @@ export const BROWSER_TOOL_CONTRACTS = [ name: "analyze_web_bundle", ...toolContractMetadata("analyze_web_bundle"), description: - "Capture JavaScript source from one selected CDP page and statically derive a chunk graph, route and endpoint candidates, vendor fingerprints, page-declared WebMCP metadata, and optionally fetch source maps from the selected target origin or explicitly requested origins. JavaScript is parsed but never executed. Select detail summary and an explicit javascript-routes view for filtered route patterns, occurrences, sources, unknowns or summary counts; complete Evidence remains retained for later views. Static candidates do not establish reachability or a complete application route denominator.", + "Capture JavaScript source from one selected CDP page and statically derive a chunk graph, route and endpoint candidates, vendor fingerprints, page-declared WebMCP metadata, and optionally fetch source maps from the selected target origin or explicitly requested origins. JavaScript is parsed but never executed. Select detail summary with a web-bundle view for filtered endpoint observations, inferences or counts, or a javascript-routes view for route patterns, occurrences, sources, unknowns or counts. Complete Evidence remains retained for later views. Static candidates do not establish reachability or a complete application route denominator.", kind: "browser-provider", inputSchema: analyzeWebBundleRequestSchema, outputSchema: bundleOutputSchema, diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index b6ebc54737..ddf8626eab 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -897,6 +897,7 @@ export const webBundleSummaryViewResultSchema = z.strictObject({ parent_operation: z.literal("analyze_web_bundle"), target_url: z.string(), target_origin: recordedBrowserOriginSchema.nullable().optional(), + source_unknowns: webBundleAnalysisSchema.shape.unknowns, summary: webBundleSummarySchema, }); @@ -907,6 +908,7 @@ export const webBundlePageViewResultSchema = z.strictObject({ parent_operation: z.literal("analyze_web_bundle"), target_url: z.string(), target_origin: recordedBrowserOriginSchema.nullable().optional(), + source_unknowns: webBundleAnalysisSchema.shape.unknowns, view: webBundleViewRequestSchema, completeness: webBundleAnalysisSchema.shape.completeness, items: webBundleViewItemsSchema, diff --git a/src/domain/analysisView/webBundleView.test.ts b/src/domain/analysisView/webBundleView.test.ts index f4522fb2f3..3475a08e3a 100644 --- a/src/domain/analysisView/webBundleView.test.ts +++ b/src/domain/analysisView/webBundleView.test.ts @@ -1,5 +1,6 @@ import { expect, it } from "vitest"; import type { WebBundleAnalysis } from "../webBundleAnalysis.js"; +import { AnalysisInputError } from "../analysisErrorCore.js"; import { analysisViewRequestSchema, projectAnalysisView, @@ -128,6 +129,85 @@ const query = ( return result.value; }; +it.each([undefined, null, "https://example.test"])( + "preserves historical omitted and explicit target origins: %s", + (origin) => { + const report = sampleAnalysis(); + if (origin === undefined) delete report.capture.target_origin; + else report.capture.target_origin = origin; + const parent = makeParent(report); + for (const selection of [ + { kind: "summary" }, + { kind: "web-bundle", collection: "summary" }, + { kind: "web-bundle", collection: "endpoints", fields: ["value"] }, + ]) { + const view = query(parent, selection); + if (origin === undefined) + expect(view).not.toHaveProperty("target_origin"); + else expect(view).toHaveProperty("target_origin", origin); + if ( + view.kind === "summary" || + (view.kind === "web-bundle" && view.summary !== null) + ) { + if (origin === undefined) + expect(view.summary).not.toHaveProperty("target_origin"); + else expect(view.summary).toHaveProperty("target_origin", origin); + } + } + }, +); + +it("counts the distinct full parent/report limitation union before bounding", () => { + const base = makeParent({ + limitations: [ + "Parent run was bounded", + "Script evaluation was omitted", + "x".repeat(100_000), + ], + }); + const parent = { + ...base, + limitations: [...base.limitations, "Parent-only condition"], + }; + for (const selection of [ + { kind: "summary" }, + { kind: "web-bundle", collection: "summary" }, + ]) { + const view = query(parent, selection); + expect(view).toHaveProperty("summary.limitation_count", 4); + expect(view.limitations).not.toContain("x".repeat(100_000)); + expect(view.limitations).toContainEqual( + expect.stringContaining("additional limitations omitted"), + ); + } +}); + +it("retains distinct structured unknown scopes on filtered pages", () => { + const unknowns = [ + { + dimension: "parse", + reason: "Source unavailable", + affected_script_keys: ["app.js"], + }, + { + dimension: "parse", + reason: "Source unavailable", + affected_script_keys: ["vendor.js"], + }, + ]; + const parent = makeParent({ unknowns }); + for (const collection of ["endpoints", "inferences"]) { + const view = query(parent, { + kind: "web-bundle", + collection, + script_keys: ["app.js"], + fields: ["value"], + }); + expect(view).toHaveProperty("source_unknowns", unknowns); + expect(view).toHaveProperty("completeness.status", "partial"); + } +}); + it("pages endpoints and concatenated pages match original records exactly, preserving duplicates", () => { const parent = makeParent(); const page1 = query(parent, { @@ -336,37 +416,59 @@ it("returns typed AnalysisResourceConstraintError when a single record exceeds 1 expect(result.error.message).toContain("byte budget"); }); -it("bounds multi-item page before budget overflow and sets byte_limited: true", () => { - const largeRecord = (index: number) => ({ - value: `/api/${index}/` + "x".repeat(300_000), - mechanism: "call:fetch", - location: { script_key: "app.js", line: index, column: 0 }, - }); - const parent = makeParent({ - observations: { - ...sampleAnalysis().observations, - endpoints: [ - largeRecord(1), - largeRecord(2), - largeRecord(3), - largeRecord(4), - ], - }, - }); +it.each([undefined, ["call:fetch"]])( + "counts all matching rows after a byte-limited page: %s", + (mechanisms) => { + const largeRecord = (index: number) => ({ + value: `/api/${index}/` + "x".repeat(300_000), + mechanism: "call:fetch", + location: { script_key: "app.js", line: index, column: 0 }, + }); + const parent = makeParent({ + observations: { + ...sampleAnalysis().observations, + endpoints: [ + largeRecord(1), + largeRecord(2), + largeRecord(3), + largeRecord(4), + ], + }, + }); - const page = query(parent, { - kind: "web-bundle", - collection: "endpoints", - limit: 10, - }); + const page = query(parent, { + kind: "web-bundle", + collection: "endpoints", + limit: 10, + mechanisms, + }); - if (page.kind !== "web-bundle") throw new Error("Expected web-bundle view"); - expect(page.byte_limited).toBe(true); - expect(page.items.length).toBeGreaterThan(0); - expect(page.items.length).toBeLessThan(4); - expect(page.coverage.next_offset).toBe(page.items.length); - expect(page.coverage.exhausted).toBe(false); -}); + if (page.kind !== "web-bundle") throw new Error("Expected web-bundle view"); + expect(page.byte_limited).toBe(true); + expect(page.items.length).toBeGreaterThan(0); + expect(page.items.length).toBeLessThan(4); + expect(page.coverage.next_offset).toBe(page.items.length); + expect(page.coverage.exhausted).toBe(false); + expect(page.coverage.total).toBe(4); + }, +); + +it.each(["indices", "values", "script_keys", "mechanisms"])( + "treats an explicit empty %s selector as an empty selection", + (selector) => { + const page = query(makeParent(), { + kind: "web-bundle", + collection: "endpoints", + [selector]: [], + }); + expect(page).toHaveProperty("items", []); + expect(page.coverage).toMatchObject({ + total: 0, + exhausted: true, + next_offset: null, + }); + }, +); it("rejects non-web-bundle parent operation with invalid_value", () => { const invalidParent: AnalysisViewParent = { @@ -386,10 +488,11 @@ it("rejects non-web-bundle parent operation with invalid_value", () => { expect(result.ok).toBe(false); if (result.ok) throw new Error("Expected failure"); - expect((result.error as any).issues[0].message).toContain( + if (!(result.error instanceof AnalysisInputError)) throw result.error; + expect(result.error.issues[0]!.message).toContain( "Web-bundle views require analyze_web_bundle", ); - expect((result.error as any).issues[0].reason).toBe("invalid_value"); + expect(result.error.issues[0]!.reason).toBe("invalid_value"); }); it("preserves parent artifact identity across summary and page views", () => { diff --git a/src/domain/analysisView/webBundleView.ts b/src/domain/analysisView/webBundleView.ts index 9799ee0964..56faccf53c 100644 --- a/src/domain/analysisView/webBundleView.ts +++ b/src/domain/analysisView/webBundleView.ts @@ -103,17 +103,14 @@ export const webBundleViewItemsSchema = z.union([ const readReport = createAnalysisViewResultReader(webBundleAnalysisSchema); -const choose = >( +const choose = ( row: T, - fields: readonly string[] | undefined, -): Record => { + fields: ReadonlySet | undefined, +): Partial => { if (fields === undefined) return { ...row }; - const allowed = new Set(fields); - const out: Record = {}; - for (const [k, v] of Object.entries(row)) { - if (allowed.has(k)) { - out[k] = v; - } + const out: Partial = {}; + for (const key of Object.keys(row) as (keyof T)[]) { + if (fields.has(String(key))) out[key] = row[key]; } return out; }; @@ -149,15 +146,24 @@ export const projectWebBundleView = ( if (!parsed.ok) return parsed; const report: WebBundleAnalysis = parsed.value; + const targetOrigin = + report.capture.target_origin === undefined + ? {} + : { target_origin: report.capture.target_origin }; + const sourceLimitations = [ + ...new Set([...parent.limitations, ...report.limitations]), + ]; + const shared = { parent_evidence_id: parent.evidenceId, parent_operation: "analyze_web_bundle" as const, parent_digest: parent.evidenceId.slice(3), artifact: parent.artifact ?? null, target_url: report.capture.target_url, - target_origin: report.capture.target_origin ?? null, + ...targetOrigin, + source_unknowns: report.unknowns, limitations: boundedViewMessages( - [...parent.limitations, ...report.limitations], + sourceLimitations, parent.evidenceId, "limitations", ), @@ -188,20 +194,22 @@ export const projectWebBundleView = ( const buildSummary = (): z.output => ({ target_url: report.capture.target_url, - target_origin: report.capture.target_origin ?? null, + ...targetOrigin, scripts_observed: report.capture.scripts_observed, scripts_analyzed: report.capture.scripts_analyzed, counts: { endpoints: report.observations.endpoints.length, inferences: report.inferences.length, - endpoint_candidates: report.inferences.filter( - (i) => i.kind === "endpoint_candidate", - ).length, + endpoint_candidates: report.inferences.reduce( + (count, inference) => + count + Number(inference.kind === "endpoint_candidate"), + 0, + ), chunks: report.observations.chunks.nodes.length, routes: report.observations.routes.length, }, completeness: report.completeness, - limitation_count: report.limitations.length, + limitation_count: sourceLimitations.length, }); if (selected.kind === "summary") { @@ -242,9 +250,15 @@ export const projectWebBundleView = ( : ok(result); } - const deliverRows = >( - rows: readonly { readonly row: T; readonly source_index: number }[], - project: (row: T, source_index: number) => Record, + type Item = + | z.output + | z.output; + const fields = + view.fields === undefined ? undefined : new Set(view.fields); + const deliverRows = ( + rows: readonly T[], + matches: ((row: T, source_index: number) => boolean) | undefined, + project: (row: T, source_index: number) => Item, ): Result => { const baseEnvelope = { ...shared, @@ -272,29 +286,41 @@ export const projectWebBundleView = ( if (bytes > PAGE_BYTES) return tooLarge(view.collection); - const items: Record[] = []; + const items: Item[] = []; let byte_limited = false; + let total = matches === undefined ? rows.length : 0; + // Unfiltered pages use the producer's array length and direct offsets. + // Filtered pages count every match but only allocate selected output rows. for ( - let index = view.offset; - index < rows.length && items.length < view.limit; - index++ + let source_index = + matches === undefined ? Math.min(view.offset, rows.length) : 0; + source_index < rows.length; + source_index++ ) { - const { row, source_index } = rows[index]!; + const row = rows[source_index]!; + if (matches !== undefined && !matches(row, source_index)) continue; + const ordinal = matches === undefined ? source_index : total++; + if (ordinal < view.offset) continue; + if (items.length >= view.limit || byte_limited) { + if (matches === undefined) break; + continue; + } const item = project(row, source_index); const addedBytes = - jsonValueByteLength(item, PAGE_BYTES - bytes) + + jsonValueByteLength(item as JsonValue, PAGE_BYTES - bytes) + (items.length > 0 ? 1 : 0); if (bytes + addedBytes > PAGE_BYTES) { if (items.length === 0) return tooLarge(view.collection); byte_limited = true; - break; + if (matches === undefined) break; + continue; } bytes += addedBytes; items.push(item); } - const coverage = pageViewCoverage(view.offset, items.length, rows.length); + const coverage = pageViewCoverage(view.offset, items.length, total); const result: UnsignedAnalysisView = { ...shared, kind: "web-bundle" as const, @@ -323,27 +349,22 @@ export const projectWebBundleView = ( const mechanisms = view.mechanisms !== undefined ? new Set(view.mechanisms) : undefined; - const matched: { - readonly row: (typeof report.observations.endpoints)[number]; - readonly source_index: number; - }[] = []; - for (const [index, row] of report.observations.endpoints.entries()) { - if (indices !== undefined && !indices.has(index)) continue; - if (values !== undefined && !values.has(row.value)) continue; - if (scriptKeys !== undefined && !scriptKeys.has(row.location.script_key)) - continue; - if (mechanisms !== undefined && !mechanisms.has(row.mechanism)) continue; - matched.push({ row, source_index: index }); - } - + const filters = [indices, values, scriptKeys, mechanisms]; return deliverRows( - matched as unknown as readonly { - readonly row: Record; - readonly source_index: number; - }[], + filters.some((filter) => filter?.size === 0) + ? [] + : report.observations.endpoints, + filters.every((filter) => filter === undefined) + ? undefined + : (row, index) => + (indices === undefined || indices.has(index)) && + (values === undefined || values.has(row.value)) && + (scriptKeys === undefined || + scriptKeys.has(row.location.script_key)) && + (mechanisms === undefined || mechanisms.has(row.mechanism)), (row, source_index) => ({ source_index, - ...choose(row, view.fields), + ...choose(row, fields), }), ); } @@ -355,30 +376,20 @@ export const projectWebBundleView = ( const scriptKeys = view.script_keys !== undefined ? new Set(view.script_keys) : undefined; - const matched: { - readonly row: (typeof report.inferences)[number]; - readonly source_index: number; - }[] = []; - for (const [index, row] of report.inferences.entries()) { - if (indices !== undefined && !indices.has(index)) continue; - if (kinds !== undefined && !kinds.has(row.kind)) continue; - if (values !== undefined && !values.has(row.value)) continue; - if ( - scriptKeys !== undefined && - !row.basis.some((b) => scriptKeys.has(b.script_key)) - ) - continue; - matched.push({ row, source_index: index }); - } - + const filters = [indices, kinds, values, scriptKeys]; return deliverRows( - matched as unknown as readonly { - readonly row: Record; - readonly source_index: number; - }[], + filters.some((filter) => filter?.size === 0) ? [] : report.inferences, + filters.every((filter) => filter === undefined) + ? undefined + : (row, index) => + (indices === undefined || indices.has(index)) && + (kinds === undefined || kinds.has(row.kind)) && + (values === undefined || values.has(row.value)) && + (scriptKeys === undefined || + row.basis.some((basis) => scriptKeys.has(basis.script_key))), (row, source_index) => ({ source_index, - ...choose(row, view.fields), + ...choose(row, fields), }), ); };