fix(web): preserve provenance and bound retained bundle paging

This commit is contained in:
gensokyo-dev committed 2026-10-26 16:56:23 +00:00
1 parent 48d554ed2a
commit a92e07f484
9 files changed
+609 -104

No files matched your search

@@ -78,6 +78,15 @@ classification, and unavailable sources never establish route reachability or
the complete application route denominator. See the
[route inventory guide](https://github.com/morluto/rea/blob/main/docs/javascript-route-inventory.md).
For captured endpoint observations and inference facts, `analyze_web_bundle`
also accepts `view: {kind: "web-bundle", collection: "endpoints", limit: 32}`
with `detail: "summary"`. Read later `endpoints`, `inferences`, or `summary`
pages from the retained parent. Filter before requesting detail; retain
`source_index` together with parent ID and collection, and request `location`
or `basis` when tracing sources. Page exhaustion does not mean producer
completeness: check `completeness`, structured `source_unknowns`, and limitations.
Endpoint candidates remain inferences rather than confirmed network behavior.
## Read a module without repeating analysis
Use the actual returned parent ID, on the same connection, as `source.evidence_id`.
+36 -4
View File
@@ -25,18 +25,50 @@ Keep the original analysis inputs in that call. For the CLI, pass the same
selection as `--view '<JSON>' --detail summary --evidence_output routes.json`.
The complete Evidence bundle is written without overwriting existing files.
Unselected delivery retains the existing complete result. A web-bundle summary
requires an explicit route view; application summaries also support the existing
requires an explicit route or web-bundle view; application summaries also support the existing
general summary when `view` is omitted.
Use the returned `normalized_result.parent_evidence_id` with
`inspect_analysis_view` on the same connection. The CLI's exported bundle can be
reopened with `inspect-evidence-bundle`; its `records` contain the complete
Evidence. To use `inspect_analysis_view`'s `evidence-file` source, extract the
selected parent record into a separate local Evidence JSON file (that source
accepts one Evidence record, not a bundle). Views do not reopen source files or
Evidence. `inspect_analysis_view`'s `evidence-file` source accepts either one
Evidence record or an exported bundle with the selected `evidence_id`.
Views do not reopen source files or
reconnect to a browser. Historical Evidence without `route_inventory` is
unmeasured, rather than an empty route set.
## Page captured endpoint observations and inferences
For `analyze_web_bundle`, select `detail: "summary"` and
`view: {kind: "web-bundle", collection: "endpoints", limit: 32}`. The CLI
accepts the same JSON through `--view`, with `--detail summary` and
`--evidence_output bundle.json`. Query the returned parent with
`inspect_analysis_view` to read `endpoints`, `inferences`, or `summary` without
recapturing the page or repeating JavaScript analysis.
Both row collections accept exact `indices`, `values`, `script_keys`, `fields`,
`offset`, and `limit`; endpoints also accept `mechanisms`, and inferences accept
`kinds`. Filters combine with AND, and an explicit empty selector matches no
rows. Every row retains its `source_index` in the original collection, including
when other fields are omitted. This identity is scoped to the parent Evidence
and collection. Original ordering and separate occurrences of duplicate values
are preserved; request endpoint `location` or inference `basis` for source details.
Follow `coverage.next_offset` until `coverage.exhausted`; those fields describe
the selected rows, independently of producer `completeness`. Each response also
preserves structured `source_unknowns`, including `affected_script_keys`, so a
filtered or value-only page cannot hide a source gap. The summary's
`limitation_count` counts distinct parent and report limitations before display
bounding. An `endpoint_candidate` remains an inference, with its original
confidence and basis, and does not prove a backend endpoint or reachability.
Pages obey the shared 1 MiB projection budget; MCP transport applies its own
wire-size guard. A byte-limited page returns a continuation; an indivisible
oversized record or metadata returns a typed resource error. Select fewer
fields or retrieve the complete retained parent through `get_evidence_bundle`,
`export_evidence_bundle`, or the CLI's exported bundle. Full Evidence is never
silently truncated.
Collections are `summary`, `routes`, `occurrences`, `sources`, and `unknowns`.
All row collections support exact batch `where.ids`, field selection, `offset`,
and `limit`. An explicit empty filter array matches nothing. Routes also support
@@ -0,0 +1,341 @@
import assert from "node:assert/strict";
import { createServer } from "node:http";
import { readFile, writeFile, mkdir, mkdtemp, rm } from "node:fs/promises";
import { join, resolve } from "node:path";
import { tmpdir } from "node:os";
import { createRequire } from "node:module";
import { pathToFileURL } from "node:url";
import { execFile } from "node:child_process";
import { promisify, parseArgs } from "node:util";
import { performance } from "node:perf_hooks";
// Public CLI capture and same-session retained MCP queries. All assets and output
// paths are explicitly supplied; this verifier installs no browser or application.
const { values } = parseArgs({
options: {
repository: { type: "string" },
mode: { type: "string" },
output: { type: "string" },
parent: { type: "string" },
application: { type: "string" },
browser: { type: "string" },
task: { type: "string", default: "all" },
"verify-cli": { type: "boolean", default: false },
},
});
assert.ok(values.repository && values.mode && values.output);
const repo = resolve(values.repository),
mode = values.mode,
out = resolve(values.output),
bundle = values.parent && resolve(values.parent);
assert.ok(["capture", "complete", "selected"].includes(mode));
assert.ok(["all", "first-candidate"].includes(values.task));
assert.ok(mode === "capture" ? values.application && values.browser : bundle);
await mkdir(out);
const require = createRequire(join(repo, "package.json"));
const { Client } = await import(
pathToFileURL(require.resolve("@modelcontextprotocol/client"))
);
const { StdioClientTransport } = await import(
pathToFileURL(require.resolve("@modelcontextprotocol/client/stdio"))
);
const env = {
PATH: process.env.PATH,
NODE_OPTIONS: "--max-old-space-size=3072",
UV_THREADPOOL_SIZE: "2",
REA_LOG_LEVEL: "silent",
REA_MCP_INPUT_SCHEMA_PROFILE: "compact",
REA_MCP_TOOL_FAMILIES: "application,session",
};
if (mode === "capture") {
const { chromium } = require("playwright-core");
const assets = resolve(values.application);
const allowed = new Set(
await (await import("node:fs/promises")).readdir(assets),
);
const server = createServer(async (req, res) => {
try {
const name =
new URL(req.url, "http://127.0.0.1").pathname.slice(1) || "index.html";
assert.ok(allowed.has(name));
const data = await readFile(join(assets, name));
res.setHeader(
"Content-Type",
name.endsWith(".js")
? "text/javascript"
: name.endsWith(".css")
? "text/css"
: name.endsWith(".json")
? "application/json"
: "text/html",
);
res.setHeader(
"Content-Type",
res.getHeader("Content-Type") + "; charset=utf-8",
);
res.end(data);
} catch {
res.writeHead(404);
res.end();
}
});
await new Promise((r) => server.listen(0, "127.0.0.1", r));
const origin = `http://127.0.0.1:${server.address().port}`;
const userData = await mkdtemp(join(tmpdir(), "rea-6198-chrome-"));
let context;
try {
context = await chromium.launchPersistentContext(userData, {
executablePath: resolve(values.browser),
headless: true,
args: ["--remote-debugging-port=0", "--no-sandbox"],
});
context
.pages()[0]
.on("pageerror", (e) => console.error("PAGE ERROR", e.message));
context
.pages()[0]
.on("response", (r) => console.error("RESPONSE", r.status(), r.url()));
await context.pages()[0].goto(origin, { waitUntil: "networkidle" });
const port = (
await readFile(join(userData, "DevToolsActivePort"), "utf8")
).split("\n")[0];
const endpoint = `http://127.0.0.1:${port}`;
const targets = await (await fetch(`${endpoint}/json/list`)).json();
const target = targets.find(
(t) => t.type === "page" && t.url === origin + "/",
);
assert.ok(target);
const started = performance.now();
const { stdout, stderr } = await promisify(execFile)(
process.execPath,
[
join(repo, "scripts/rea.mjs"),
"analyze-web-bundle",
endpoint,
target.id,
"--observation-ms",
"1000",
"--max-heap-mb",
"1024",
"--analysis-timeout-ms",
"60000",
"--total-analysis-timeout-ms",
"90000",
"--format",
"json",
],
{ cwd: repo, env, maxBuffer: 64 * 1024 ** 2, timeout: 120000 },
);
await writeFile(join(out, "stderr.txt"), stderr);
const evidence = JSON.parse(stdout);
assert.equal(evidence.operation, "analyze_web_bundle");
assert.ok(evidence.normalized_result.observations);
await writeFile(join(out, "parent.json"), JSON.stringify(evidence));
// Public export/import format comes from the CLI's complete Evidence publication.
console.log(
JSON.stringify({
capture_ms: performance.now() - started,
evidence_id: evidence.evidence_id,
counts: {
endpoints: evidence.normalized_result.observations.endpoints.length,
inferences: evidence.normalized_result.inferences.length,
},
completeness: evidence.normalized_result.completeness,
}),
);
} finally {
await context?.close();
await new Promise((r) => server.close(r));
await rm(userData, { recursive: true, force: true });
}
process.exit(0);
}
const parent = JSON.parse(await readFile(bundle, "utf8"));
const report = parent.normalized_result;
const { createEvidenceBundle } = await import(
pathToFileURL(join(repo, "dist/domain/evidenceBundle.js"))
);
const bundleFile = join(out, "input-bundle.json");
await writeFile(bundleFile, JSON.stringify(createEvidenceBundle([parent])));
const transport = new StdioClientTransport({
command: process.execPath,
args: [join(repo, "scripts/rea.mjs"), "mcp"],
cwd: repo,
env,
stderr: "pipe",
});
transport.stderr?.resume();
const client = new Client({
name: "rea-web-bundle-paging-verifier",
version: "1",
});
const calls = [];
let catalog;
async function invoke(name, args) {
const toolDefinition = catalog.tools.find((t) => t.name === name);
assert.ok(toolDefinition);
const start = performance.now();
const result = await client.callTool(
{ name, arguments: args },
{ toolDefinition, timeout: 90000 },
);
calls.push({
name,
arguments: args,
result,
elapsed_ms: performance.now() - start,
});
assert.notEqual(result.isError, true, JSON.stringify(result));
return result.structuredContent;
}
try {
await client.connect(transport);
catalog = await client.listTools();
await writeFile(join(out, "catalog.json"), JSON.stringify(catalog));
const runs = [];
for (const temperature of ["cold", "warm"]) {
const start = performance.now();
const callStart = calls.length;
if (temperature === "cold")
await invoke("import_evidence_bundle", { path: bundleFile });
if (mode === "complete") {
const full = await invoke("get_evidence_bundle", {});
const actual = full.result.records.find(
(r) => r.evidence_id === parent.evidence_id,
);
assert.deepEqual(actual, parent);
runs.push({
temperature,
elapsed_ms: performance.now() - start,
calls: calls.length - callStart,
});
continue;
}
const source = {
kind: "retained-evidence",
evidence_id: parent.evidence_id,
};
if (values.task === "all") {
const summary = await invoke("inspect_analysis_view", {
source,
view: { kind: "web-bundle", collection: "summary" },
});
assert.equal(
summary.normalized_result.summary.counts.endpoints,
report.observations.endpoints.length,
);
}
if (values.task === "first-candidate") {
const value = await invoke("inspect_analysis_view", {
source,
view: {
kind: "web-bundle",
collection: "inferences",
kinds: ["endpoint_candidate"],
limit: 1,
},
});
const index = report.inferences.findIndex(
(row) => row.kind === "endpoint_candidate",
);
assert.equal(
value.normalized_result.coverage.total,
report.inferences.filter((row) => row.kind === "endpoint_candidate")
.length,
);
assert.deepEqual(
value.normalized_result.items,
index < 0 ? [] : [{ source_index: index, ...report.inferences[index] }],
);
assert.deepEqual(
value.normalized_result.source_unknowns,
report.unknowns,
);
} else
for (const collection of ["endpoints", "inferences"]) {
const expected =
collection === "endpoints"
? report.observations.endpoints
: report.inferences;
const rows = [];
let offset = 0;
do {
const value = await invoke("inspect_analysis_view", {
source,
view: { kind: "web-bundle", collection, offset, limit: 32 },
});
const page = value.normalized_result;
assert.deepEqual(page.source_unknowns, report.unknowns);
assert.deepEqual(page.completeness, report.completeness);
assert.equal(page.coverage.total, expected.length);
for (const row of page.items) {
assert.deepEqual(row, {
source_index: rows.length,
...expected[rows.length],
});
rows.push(row);
}
if (page.coverage.exhausted) break;
assert.ok(page.coverage.next_offset > offset);
offset = page.coverage.next_offset;
} while (true);
assert.equal(rows.length, expected.length);
}
runs.push({
temperature,
elapsed_ms: performance.now() - start,
calls: calls.length - callStart,
});
}
if (mode === "selected" && values["verify-cli"]) {
const input = JSON.stringify({
source: { kind: "evidence-file", path: bundle },
view: {
kind: "web-bundle",
collection: "endpoints",
fields: ["value", "location"],
limit: 32,
},
});
const { stdout } = await promisify(execFile)(
process.execPath,
[
join(repo, "scripts/rea.mjs"),
"inspect-analysis-view",
input,
"--format",
"json",
],
{ cwd: repo, env, maxBuffer: 64 * 1024 ** 2, timeout: 90000 },
);
const cli = JSON.parse(stdout);
assert.deepEqual(
cli.normalized_result.items,
report.observations.endpoints.slice(0, 32).map((r, source_index) => ({
source_index,
value: r.value,
location: r.location,
})),
);
await writeFile(join(out, "cli.json"), stdout);
}
await writeFile(join(out, "calls.json"), JSON.stringify(calls));
await writeFile(
join(out, "metrics.json"),
JSON.stringify(
{
mode,
task: values.task,
runs,
counts: {
endpoints: report.observations.endpoints.length,
inferences: report.inferences.length,
},
},
null,
2,
),
);
console.log(JSON.stringify({ mode, task: values.task, runs }));
} finally {
await client.close();
}
+2 -2
View File
@@ -1,6 +1,6 @@
import {
parseJavaScriptRouteDelivery,
javascriptRouteDeliveryOptions,
webBundleDeliveryOptions,
} from "./javascriptRouteDelivery.js";
import { deliverCliAnalysisEvidence } from "./analysisEvidenceDelivery.js";
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
@@ -164,7 +164,7 @@ const registerBundleAnalysis = (
}),
options: z.object({
...browserScopeOptions,
...javascriptRouteDeliveryOptions,
...webBundleDeliveryOptions,
observationMs: observationDuration(500),
fetchSourceMaps: z
.boolean()
+7
View File
@@ -18,6 +18,13 @@ export const javascriptRouteDeliveryOptions = {
),
};
export const webBundleDeliveryOptions = {
...javascriptRouteDeliveryOptions,
view: javascriptRouteDeliveryOptions.view.describe(
"JSON web-bundle (endpoints, inferences, summary) or javascript-routes view; requires --detail summary and --evidence_output",
),
};
/** Validate delivery before opening artifacts or attaching a browser. */
export const parseJavaScriptRouteDelivery = (
operation: string,
+1 -1
View File
@@ -206,7 +206,7 @@ export const BROWSER_TOOL_CONTRACTS = [
name: "analyze_web_bundle",
...toolContractMetadata("analyze_web_bundle"),
description:
"Capture JavaScript source from one selected CDP page and statically derive a chunk graph, route and endpoint candidates, vendor fingerprints, page-declared WebMCP metadata, and optionally fetch source maps from the selected target origin or explicitly requested origins. JavaScript is parsed but never executed. Select detail summary and an explicit javascript-routes view for filtered route patterns, occurrences, sources, unknowns or summary counts; complete Evidence remains retained for later views. Static candidates do not establish reachability or a complete application route denominator.",
"Capture JavaScript source from one selected CDP page and statically derive a chunk graph, route and endpoint candidates, vendor fingerprints, page-declared WebMCP metadata, and optionally fetch source maps from the selected target origin or explicitly requested origins. JavaScript is parsed but never executed. Select detail summary with a web-bundle view for filtered endpoint observations, inferences or counts, or a javascript-routes view for route patterns, occurrences, sources, unknowns or counts. Complete Evidence remains retained for later views. Static candidates do not establish reachability or a complete application route denominator.",
kind: "browser-provider",
inputSchema: analyzeWebBundleRequestSchema,
outputSchema: bundleOutputSchema,
+2
View File
@@ -897,6 +897,7 @@ export const webBundleSummaryViewResultSchema = z.strictObject({
parent_operation: z.literal("analyze_web_bundle"),
target_url: z.string(),
target_origin: recordedBrowserOriginSchema.nullable().optional(),
source_unknowns: webBundleAnalysisSchema.shape.unknowns,
summary: webBundleSummarySchema,
});
@@ -907,6 +908,7 @@ export const webBundlePageViewResultSchema = z.strictObject({
parent_operation: z.literal("analyze_web_bundle"),
target_url: z.string(),
target_origin: recordedBrowserOriginSchema.nullable().optional(),
source_unknowns: webBundleAnalysisSchema.shape.unknowns,
view: webBundleViewRequestSchema,
completeness: webBundleAnalysisSchema.shape.completeness,
items: webBundleViewItemsSchema,
+134 -31
View File
@@ -1,5 +1,6 @@
import { expect, it } from "vitest";
import type { WebBundleAnalysis } from "../webBundleAnalysis.js";
import { AnalysisInputError } from "../analysisErrorCore.js";
import {
analysisViewRequestSchema,
projectAnalysisView,
@@ -128,6 +129,85 @@ const query = (
return result.value;
};
it.each([undefined, null, "https://example.test"])(
"preserves historical omitted and explicit target origins: %s",
(origin) => {
const report = sampleAnalysis();
if (origin === undefined) delete report.capture.target_origin;
else report.capture.target_origin = origin;
const parent = makeParent(report);
for (const selection of [
{ kind: "summary" },
{ kind: "web-bundle", collection: "summary" },
{ kind: "web-bundle", collection: "endpoints", fields: ["value"] },
]) {
const view = query(parent, selection);
if (origin === undefined)
expect(view).not.toHaveProperty("target_origin");
else expect(view).toHaveProperty("target_origin", origin);
if (
view.kind === "summary" ||
(view.kind === "web-bundle" && view.summary !== null)
) {
if (origin === undefined)
expect(view.summary).not.toHaveProperty("target_origin");
else expect(view.summary).toHaveProperty("target_origin", origin);
}
}
},
);
it("counts the distinct full parent/report limitation union before bounding", () => {
const base = makeParent({
limitations: [
"Parent run was bounded",
"Script evaluation was omitted",
"x".repeat(100_000),
],
});
const parent = {
...base,
limitations: [...base.limitations, "Parent-only condition"],
};
for (const selection of [
{ kind: "summary" },
{ kind: "web-bundle", collection: "summary" },
]) {
const view = query(parent, selection);
expect(view).toHaveProperty("summary.limitation_count", 4);
expect(view.limitations).not.toContain("x".repeat(100_000));
expect(view.limitations).toContainEqual(
expect.stringContaining("additional limitations omitted"),
);
}
});
it("retains distinct structured unknown scopes on filtered pages", () => {
const unknowns = [
{
dimension: "parse",
reason: "Source unavailable",
affected_script_keys: ["app.js"],
},
{
dimension: "parse",
reason: "Source unavailable",
affected_script_keys: ["vendor.js"],
},
];
const parent = makeParent({ unknowns });
for (const collection of ["endpoints", "inferences"]) {
const view = query(parent, {
kind: "web-bundle",
collection,
script_keys: ["app.js"],
fields: ["value"],
});
expect(view).toHaveProperty("source_unknowns", unknowns);
expect(view).toHaveProperty("completeness.status", "partial");
}
});
it("pages endpoints and concatenated pages match original records exactly, preserving duplicates", () => {
const parent = makeParent();
const page1 = query(parent, {
@@ -336,37 +416,59 @@ it("returns typed AnalysisResourceConstraintError when a single record exceeds 1
expect(result.error.message).toContain("byte budget");
});
it("bounds multi-item page before budget overflow and sets byte_limited: true", () => {
const largeRecord = (index: number) => ({
value: `/api/${index}/` + "x".repeat(300_000),
mechanism: "call:fetch",
location: { script_key: "app.js", line: index, column: 0 },
});
const parent = makeParent({
observations: {
...sampleAnalysis().observations,
endpoints: [
largeRecord(1),
largeRecord(2),
largeRecord(3),
largeRecord(4),
],
},
});
it.each([undefined, ["call:fetch"]])(
"counts all matching rows after a byte-limited page: %s",
(mechanisms) => {
const largeRecord = (index: number) => ({
value: `/api/${index}/` + "x".repeat(300_000),
mechanism: "call:fetch",
location: { script_key: "app.js", line: index, column: 0 },
});
const parent = makeParent({
observations: {
...sampleAnalysis().observations,
endpoints: [
largeRecord(1),
largeRecord(2),
largeRecord(3),
largeRecord(4),
],
},
});
const page = query(parent, {
kind: "web-bundle",
collection: "endpoints",
limit: 10,
});
const page = query(parent, {
kind: "web-bundle",
collection: "endpoints",
limit: 10,
mechanisms,
});
if (page.kind !== "web-bundle") throw new Error("Expected web-bundle view");
expect(page.byte_limited).toBe(true);
expect(page.items.length).toBeGreaterThan(0);
expect(page.items.length).toBeLessThan(4);
expect(page.coverage.next_offset).toBe(page.items.length);
expect(page.coverage.exhausted).toBe(false);
});
if (page.kind !== "web-bundle") throw new Error("Expected web-bundle view");
expect(page.byte_limited).toBe(true);
expect(page.items.length).toBeGreaterThan(0);
expect(page.items.length).toBeLessThan(4);
expect(page.coverage.next_offset).toBe(page.items.length);
expect(page.coverage.exhausted).toBe(false);
expect(page.coverage.total).toBe(4);
},
);
it.each(["indices", "values", "script_keys", "mechanisms"])(
"treats an explicit empty %s selector as an empty selection",
(selector) => {
const page = query(makeParent(), {
kind: "web-bundle",
collection: "endpoints",
[selector]: [],
});
expect(page).toHaveProperty("items", []);
expect(page.coverage).toMatchObject({
total: 0,
exhausted: true,
next_offset: null,
});
},
);
it("rejects non-web-bundle parent operation with invalid_value", () => {
const invalidParent: AnalysisViewParent = {
@@ -386,10 +488,11 @@ it("rejects non-web-bundle parent operation with invalid_value", () => {
expect(result.ok).toBe(false);
if (result.ok) throw new Error("Expected failure");
expect((result.error as any).issues[0].message).toContain(
if (!(result.error instanceof AnalysisInputError)) throw result.error;
expect(result.error.issues[0]!.message).toContain(
"Web-bundle views require analyze_web_bundle",
);
expect((result.error as any).issues[0].reason).toBe("invalid_value");
expect(result.error.issues[0]!.reason).toBe("invalid_value");
});
it("preserves parent artifact identity across summary and page views", () => {
+77 -66
View File
@@ -103,17 +103,14 @@ export const webBundleViewItemsSchema = z.union([
const readReport = createAnalysisViewResultReader(webBundleAnalysisSchema);
const choose = <T extends Record<string, JsonValue>>(
const choose = <T extends object>(
row: T,
fields: readonly string[] | undefined,
): Record<string, JsonValue> => {
fields: ReadonlySet<string> | undefined,
): Partial<T> => {
if (fields === undefined) return { ...row };
const allowed = new Set(fields);
const out: Record<string, JsonValue> = {};
for (const [k, v] of Object.entries(row)) {
if (allowed.has(k)) {
out[k] = v;
}
const out: Partial<T> = {};
for (const key of Object.keys(row) as (keyof T)[]) {
if (fields.has(String(key))) out[key] = row[key];
}
return out;
};
@@ -149,15 +146,24 @@ export const projectWebBundleView = (
if (!parsed.ok) return parsed;
const report: WebBundleAnalysis = parsed.value;
const targetOrigin =
report.capture.target_origin === undefined
? {}
: { target_origin: report.capture.target_origin };
const sourceLimitations = [
...new Set([...parent.limitations, ...report.limitations]),
];
const shared = {
parent_evidence_id: parent.evidenceId,
parent_operation: "analyze_web_bundle" as const,
parent_digest: parent.evidenceId.slice(3),
artifact: parent.artifact ?? null,
target_url: report.capture.target_url,
target_origin: report.capture.target_origin ?? null,
...targetOrigin,
source_unknowns: report.unknowns,
limitations: boundedViewMessages(
[...parent.limitations, ...report.limitations],
sourceLimitations,
parent.evidenceId,
"limitations",
),
@@ -188,20 +194,22 @@ export const projectWebBundleView = (
const buildSummary = (): z.output<typeof webBundleSummarySchema> => ({
target_url: report.capture.target_url,
target_origin: report.capture.target_origin ?? null,
...targetOrigin,
scripts_observed: report.capture.scripts_observed,
scripts_analyzed: report.capture.scripts_analyzed,
counts: {
endpoints: report.observations.endpoints.length,
inferences: report.inferences.length,
endpoint_candidates: report.inferences.filter(
(i) => i.kind === "endpoint_candidate",
).length,
endpoint_candidates: report.inferences.reduce(
(count, inference) =>
count + Number(inference.kind === "endpoint_candidate"),
0,
),
chunks: report.observations.chunks.nodes.length,
routes: report.observations.routes.length,
},
completeness: report.completeness,
limitation_count: report.limitations.length,
limitation_count: sourceLimitations.length,
});
if (selected.kind === "summary") {
@@ -242,9 +250,15 @@ export const projectWebBundleView = (
: ok(result);
}
const deliverRows = <T extends Record<string, JsonValue>>(
rows: readonly { readonly row: T; readonly source_index: number }[],
project: (row: T, source_index: number) => Record<string, JsonValue>,
type Item =
| z.output<typeof webBundleEndpointRowSchema>
| z.output<typeof webBundleInferenceRowSchema>;
const fields =
view.fields === undefined ? undefined : new Set<string>(view.fields);
const deliverRows = <T>(
rows: readonly T[],
matches: ((row: T, source_index: number) => boolean) | undefined,
project: (row: T, source_index: number) => Item,
): Result<UnsignedAnalysisView, AnalysisError> => {
const baseEnvelope = {
...shared,
@@ -272,29 +286,41 @@ export const projectWebBundleView = (
if (bytes > PAGE_BYTES) return tooLarge(view.collection);
const items: Record<string, JsonValue>[] = [];
const items: Item[] = [];
let byte_limited = false;
let total = matches === undefined ? rows.length : 0;
// Unfiltered pages use the producer's array length and direct offsets.
// Filtered pages count every match but only allocate selected output rows.
for (
let index = view.offset;
index < rows.length && items.length < view.limit;
index++
let source_index =
matches === undefined ? Math.min(view.offset, rows.length) : 0;
source_index < rows.length;
source_index++
) {
const { row, source_index } = rows[index]!;
const row = rows[source_index]!;
if (matches !== undefined && !matches(row, source_index)) continue;
const ordinal = matches === undefined ? source_index : total++;
if (ordinal < view.offset) continue;
if (items.length >= view.limit || byte_limited) {
if (matches === undefined) break;
continue;
}
const item = project(row, source_index);
const addedBytes =
jsonValueByteLength(item, PAGE_BYTES - bytes) +
jsonValueByteLength(item as JsonValue, PAGE_BYTES - bytes) +
(items.length > 0 ? 1 : 0);
if (bytes + addedBytes > PAGE_BYTES) {
if (items.length === 0) return tooLarge(view.collection);
byte_limited = true;
break;
if (matches === undefined) break;
continue;
}
bytes += addedBytes;
items.push(item);
}
const coverage = pageViewCoverage(view.offset, items.length, rows.length);
const coverage = pageViewCoverage(view.offset, items.length, total);
const result: UnsignedAnalysisView = {
...shared,
kind: "web-bundle" as const,
@@ -323,27 +349,22 @@ export const projectWebBundleView = (
const mechanisms =
view.mechanisms !== undefined ? new Set(view.mechanisms) : undefined;
const matched: {
readonly row: (typeof report.observations.endpoints)[number];
readonly source_index: number;
}[] = [];
for (const [index, row] of report.observations.endpoints.entries()) {
if (indices !== undefined && !indices.has(index)) continue;
if (values !== undefined && !values.has(row.value)) continue;
if (scriptKeys !== undefined && !scriptKeys.has(row.location.script_key))
continue;
if (mechanisms !== undefined && !mechanisms.has(row.mechanism)) continue;
matched.push({ row, source_index: index });
}
const filters = [indices, values, scriptKeys, mechanisms];
return deliverRows(
matched as unknown as readonly {
readonly row: Record<string, JsonValue>;
readonly source_index: number;
}[],
filters.some((filter) => filter?.size === 0)
? []
: report.observations.endpoints,
filters.every((filter) => filter === undefined)
? undefined
: (row, index) =>
(indices === undefined || indices.has(index)) &&
(values === undefined || values.has(row.value)) &&
(scriptKeys === undefined ||
scriptKeys.has(row.location.script_key)) &&
(mechanisms === undefined || mechanisms.has(row.mechanism)),
(row, source_index) => ({
source_index,
...choose(row, view.fields),
...choose(row, fields),
}),
);
}
@@ -355,30 +376,20 @@ export const projectWebBundleView = (
const scriptKeys =
view.script_keys !== undefined ? new Set(view.script_keys) : undefined;
const matched: {
readonly row: (typeof report.inferences)[number];
readonly source_index: number;
}[] = [];
for (const [index, row] of report.inferences.entries()) {
if (indices !== undefined && !indices.has(index)) continue;
if (kinds !== undefined && !kinds.has(row.kind)) continue;
if (values !== undefined && !values.has(row.value)) continue;
if (
scriptKeys !== undefined &&
!row.basis.some((b) => scriptKeys.has(b.script_key))
)
continue;
matched.push({ row, source_index: index });
}
const filters = [indices, kinds, values, scriptKeys];
return deliverRows(
matched as unknown as readonly {
readonly row: Record<string, JsonValue>;
readonly source_index: number;
}[],
filters.some((filter) => filter?.size === 0) ? [] : report.inferences,
filters.every((filter) => filter === undefined)
? undefined
: (row, index) =>
(indices === undefined || indices.has(index)) &&
(kinds === undefined || kinds.has(row.kind)) &&
(values === undefined || values.has(row.value)) &&
(scriptKeys === undefined ||
row.basis.some((basis) => scriptKeys.has(basis.script_key))),
(row, source_index) => ({
source_index,
...choose(row, view.fields),
...choose(row, fields),
}),
);
};