fix(browser): budget retained observations independently of wire data

docs/browser-scenario-contract.md

@@ -368,8 +368,16 @@ Proxy-Authorization, Cookie, and Set-Cookie values are null with
 and their standard URI/form or JSON-escaped representations are replaced without
 decoding unrelated binary bytes. Other encodings are not interpreted. Digests describe the
 retained representation after redaction.
+The selected body-byte limit also bounds that retained representation after
+secret expansion, before base64 serialization. Credential header values are
+discarded before retained-header allocation accounting; their names and null
+projections still count. The transport/API has already materialized incoming
+bodies and headers, so these retained limits do not bound producer-side input
+allocation.
 
-Only completed responses are read. At the end of the final step, observation
+Only completed responses are read. The producer-observed terminal state is
+recorded independently of whether its event metadata fit the retention budget. An omitted completion event cannot turn a
+finished transaction into an unfinished one. When the final step ends, observation
 stops; requests still in flight emit `request-unfinished`, and their observed
 responses have explicitly unavailable bodies. Already-admitted content reads
 drain before browser cleanup. At most 16 reads run at once; queued work is

docs/web-runtime.md

@@ -52,7 +52,7 @@ The result contains directly registered listener types/flags, callback coordinat
 
 ## Bounds and verification
 
-Each protocol message has a 64 MiB budget; script/context metadata and request metadata each have an 8 MiB retained budget. Complete retained source text has a 32 MiB budget. Oversized or malformed evidence returns no partial successful result. Each command and the complete inspection/source-join phase use `command_timeout_ms` (default 20000). Cleanup uses a separate 5000 ms grace; `command_timeout_ms` does not raise or shorten it, and this release has no runtime override for that grace. Transport closure follows the grace. The selected observation duration must fit the host timer's positive 32-bit millisecond range. REA does not control the externally owned browser's CPU or memory.
+Each protocol message has a 64 MiB budget; script/context metadata and request metadata each have an 8 MiB retained budget. Request retention charges the projected request and initiator facts, including final source binding; discarded request bodies and headers do not consume that retained allowance. Complete retained source text has a 32 MiB budget. Retention overruns return a typed resource constraint naming the allowance, observed bytes and fixed limit; malformed evidence remains an output failure. Neither returns partial success. Each command and the complete inspection/source-join phase use `command_timeout_ms` (default 20000). Cleanup uses a separate 5000 ms grace; `command_timeout_ms` does not raise or shorten it, and this release has no runtime override for that grace. Transport closure follows the grace. Both CLI and MCP use the same positive 32-bit millisecond observation-duration contract. REA does not control the externally owned browser's CPU or memory.
 
 `npm run verify:browser:runtime` uses caller-supplied `REA_BROWSER_EXECUTABLE` and the native `ps` command on Linux/macOS against an owned synthetic fixture, through real public CLI and stdio MCP. It checks exact Unicode/CRLF text/digests, same-URL identities, same-document navigation, inert declared names, executed and zero-count branches, function-only unknowns on repeated observation, listener selection, initiator association, actual arming and preservation of the external page. A separate fixture page closes during an armed MCP window to verify prompt termination and inline cleanup diagnostics. The optional positional entrypoint to `scripts/verify-browser-runtime.mjs` permits the same checks against an isolated installed tarball.
 

src/browser/execution/CdpRuntimeInitiator.ts

@@ -9,7 +9,7 @@ const isObject = (value: Initiator[string] | undefined): value is Initiator =>
 /** Sanitize known transport fields while retaining declarations and every unknown producer field. */
 export const sanitizeRuntimeInitiator = (
   initiator: Initiator,
-  sources: CdpRuntimeSources,
+  sources: Pick<CdpRuntimeSources, "sourceUrl">,
 ): Initiator => {
   if (typeof initiator.url === "string")
     initiator.url = sanitizeBrowserUrl(initiator.url).url;

src/browser/execution/CdpRuntimeRequests.test.ts

@@ -0,0 +1,68 @@
+import { expect, it } from "vitest";
+import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
+import { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js";
+import { WEB_RUNTIME_LIMITS } from "../../domain/webRuntime.js";
+import { CdpRuntimeRequests } from "./CdpRuntimeRequests.js";
+
+const origin = "https://example.test";
+const sources = {
+  session: {
+    operation: "observe_web_execution" as const,
+    allowedOrigins: new Set([origin]),
+    target: { frame_id: "main" },
+  },
+  sourceUrl: (_scriptId: string, url: string) => url,
+  location: (): never => {
+    throw new Error("This parser-only initiator has no callsites");
+  },
+};
+const event = (
+  extraRequest: Record<string, string>,
+  extraInitiator: Record<string, string> = {},
+) => ({
+  method: "Network.requestWillBeSent",
+  params: {
+    requestId: "request",
+    frameId: "main",
+    timestamp: 1,
+    request: { url: `${origin}/resource`, method: "POST", ...extraRequest },
+    initiator: { type: "parser", ...extraInitiator },
+  },
+});
+
+it("budgets retained request facts independently of discarded POST bodies", () => {
+  const requests = new CdpRuntimeRequests(sources);
+  requests.ingest(
+    event({ postData: "p".repeat(WEB_RUNTIME_LIMITS.retainedEventBytes + 1) }),
+  );
+  const retained = requests.result();
+  expect(retained).toHaveLength(1);
+  expect(retained[0]).toMatchObject({
+    method: "POST",
+    reported_initiator: { type: "parser" },
+  });
+  expect(JSON.stringify(retained)).not.toContain("postData");
+});
+
+it("preserves a typed resource refusal for oversized retained initiator facts", () => {
+  const requests = new CdpRuntimeRequests(sources);
+  requests.ingest(
+    event(
+      {},
+      { retained: "r".repeat(WEB_RUNTIME_LIMITS.retainedEventBytes + 1) },
+    ),
+  );
+  try {
+    requests.result();
+    expect.fail(
+      "Expected the retained initiator allowance to refuse the result",
+    );
+  } catch (cause) {
+    expect(cause).toBeInstanceOf(AnalysisResourceConstraintError);
+    if (!(cause instanceof AnalysisResourceConstraintError)) throw cause;
+    expect(projectAnalysisError(cause)).toMatchObject({
+      code: "resource_constraint",
+      details: { resource: "capacity" },
+    });
+  }
+});

src/browser/execution/CdpRuntimeRequests.ts

@@ -1,8 +1,8 @@
 import type { WebExecution } from "../../domain/webExecution.js";
+import { AnalysisError } from "../../domain/analysisErrorBase.js";
 import { AnalysisOutputError } from "../../domain/analysisErrorCore.js";
 import { sanitizeBrowserUrl } from "../../domain/browserObservation.js";
 import { jsonObjectSchema } from "../../domain/jsonValue.js";
-import { WEB_RUNTIME_LIMITS } from "../../domain/webRuntime.js";
 import type { CdpEvent } from "../CdpConnection.js";
 import {
   runtimeRequestSchema,
@@ -10,14 +10,27 @@ import {
 } from "./CdpRuntimeProtocol.js";
 import type { CdpRuntimeSources } from "./CdpRuntimeSources.js";
 import { sanitizeRuntimeInitiator } from "./CdpRuntimeInitiator.js";
+import { assertRuntimeRetention } from "./CdpRuntimeRetention.js";
+
+type RuntimeRequestSources = Pick<
+  CdpRuntimeSources,
+  "location" | "sourceUrl"
+> & {
+  readonly session: Pick<
+    CdpRuntimeSources["session"],
+    "operation" | "allowedOrigins"
+  > & {
+    readonly target: Pick<CdpRuntimeSources["session"]["target"], "frame_id">;
+  };
+};
 
 /** Join producer initiators by script ID while preserving unresolved async parent identities. */
 export class CdpRuntimeRequests {
   readonly items: WebExecution["requests"] = [];
   excluded = 0;
   #bytes = 0;
-  #failure: AnalysisOutputError | undefined;
-  constructor(readonly sources: CdpRuntimeSources) {}
+  #failure: AnalysisError | undefined;
+  constructor(readonly sources: RuntimeRequestSources) {}
 
   /** Retain only armed-window requests from the selected main frame and selected origins. */
   ingest(event: CdpEvent): void {
@@ -37,11 +50,6 @@ export class CdpRuntimeRequests {
         this.excluded += 1;
         return;
       }
-      this.#bytes += Buffer.byteLength(JSON.stringify(event.params));
-      if (this.#bytes > WEB_RUNTIME_LIMITS.retainedEventBytes)
-        throw new Error(
-          "Request initiator evidence exceeds its complete 8 MiB event budget.",
-        );
       const callsites: WebExecution["requests"][number]["callsites"] = [];
       const parents: WebExecution["requests"][number]["async_parent_ids"] = [];
       let current: unknown = request.initiator.stack;
@@ -76,7 +84,7 @@ export class CdpRuntimeRequests {
             request.initiator.columnNumber ?? null,
           ),
         );
-      this.items.push({
+      const retained: WebExecution["requests"][number] = {
         request_id: request.requestId,
         url: url.url,
         method: request.request.method,
@@ -89,19 +97,29 @@ export class CdpRuntimeRequests {
         callsites,
         async_parent_ids: parents,
         causal_attribution: "unknown",
-      });
-    } catch (cause: unknown) {
-      this.#failure = new AnalysisOutputError(
+      };
+      this.#bytes += Buffer.byteLength(JSON.stringify(retained));
+      assertRuntimeRetention(
         this.sources.session.operation,
-        cause instanceof Error ? cause.message : String(cause),
+        "requests",
+        this.#bytes,
       );
+      this.items.push(retained);
+    } catch (cause: unknown) {
+      this.#failure =
+        cause instanceof AnalysisError
+          ? cause
+          : new AnalysisOutputError(
+              this.sources.session.operation,
+              cause instanceof Error ? cause.message : String(cause),
+            );
     }
   }
 
   /** Preserve all repeated/redirect requests and bind callsites after metadata collection ends. */
   result(): WebExecution["requests"] {
     this.check();
-    return this.items.map((request) => ({
+    const result = this.items.map((request) => ({
       ...request,
       reported_initiator: sanitizeRuntimeInitiator(
         jsonObjectSchema.parse(request.reported_initiator),
@@ -117,6 +135,13 @@ export class CdpRuntimeRequests {
         ),
       ),
     }));
+    // Late source binding may expand the output after the ingress projection.
+    assertRuntimeRetention(
+      this.sources.session.operation,
+      "requests",
+      Buffer.byteLength(JSON.stringify(result)),
+    );
+    return result;
   }
   /** Report malformed or oversized producer evidence without partial success. */
   check(): void {

src/browser/execution/CdpRuntimeRetention.ts

@@ -0,0 +1,29 @@
+import { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js";
+import { fixedAllowanceOptions } from "../../domain/fixedAllowanceRemediation.js";
+import { WEB_RUNTIME_LIMITS } from "../../domain/webRuntime.js";
+
+/** Enforce the retained runtime representation, separately from wire payloads. */
+export const assertRuntimeRetention = (
+  operation: string,
+  kind: "metadata" | "requests" | "sources",
+  retainedBytes: number,
+): void => {
+  const limit =
+    kind === "sources"
+      ? WEB_RUNTIME_LIMITS.sourceBytes
+      : WEB_RUNTIME_LIMITS.retainedEventBytes;
+  if (retainedBytes <= limit) return;
+  const allowance = `Complete browser runtime ${kind} retention (${String(limit)} bytes)`;
+  throw new AnalysisResourceConstraintError(
+    operation,
+    "capacity",
+    `${allowance} exceeded; no partial runtime result is returned.`,
+    {
+      boundary: "browser-runtime-retention",
+      kind,
+      retained_bytes: retainedBytes,
+      limit_bytes: limit,
+    },
+    fixedAllowanceOptions(allowance, "live"),
+  );
+};

src/browser/execution/CdpRuntimeSources.ts

@@ -5,7 +5,6 @@ import { AnalysisOutputError } from "../../domain/analysisErrorCore.js";
 import { sanitizeBrowserUrl } from "../../domain/browserObservation.js";
 import { jsonObjectSchema } from "../../domain/jsonValue.js";
 import {
-  WEB_RUNTIME_LIMITS,
   type WebRuntimeSource,
   type webRuntimeLocationSchema,
 } from "../../domain/webRuntime.js";
@@ -15,6 +14,7 @@ import {
   runtimeContextSchema,
   runtimeScriptParsedSchema,
 } from "./CdpRuntimeProtocol.js";
+import { assertRuntimeRetention } from "./CdpRuntimeRetention.js";
 
 const scriptIdentitySchema = runtimeScriptParsedSchema.pick({ scriptId: true });
 const contextIdentitySchema = z.object({
@@ -49,10 +49,11 @@ export class CdpRuntimeSources {
     try {
       if (event.method === "Runtime.executionContextCreated") {
         this.#retainedBytes += Buffer.byteLength(JSON.stringify(event.params));
-        if (this.#retainedBytes > WEB_RUNTIME_LIMITS.retainedEventBytes)
-          throw new Error(
-            "Context/script metadata exceeds the complete 8 MiB event budget.",
-          );
+        assertRuntimeRetention(
+          this.session.operation,
+          "metadata",
+          this.#retainedBytes,
+        );
         const context = runtimeContextSchema.parse(event.params).context;
         const previous = this.#contexts.get(context.id);
         if (
@@ -66,10 +67,11 @@ export class CdpRuntimeSources {
         if (retainNewMetadata) this.#contexts.set(context.id, context);
       } else if (event.method === "Debugger.scriptParsed") {
         this.#retainedBytes += Buffer.byteLength(JSON.stringify(event.params));
-        if (this.#retainedBytes > WEB_RUNTIME_LIMITS.retainedEventBytes)
-          throw new Error(
-            "Script metadata exceeds the complete 8 MiB event budget.",
-          );
+        assertRuntimeRetention(
+          this.session.operation,
+          "metadata",
+          this.#retainedBytes,
+        );
         const script = runtimeScriptParsedSchema.parse(event.params);
         const previous = this.scripts.get(script.scriptId);
         if (
@@ -83,10 +85,13 @@ export class CdpRuntimeSources {
         if (retainNewMetadata) this.scripts.set(script.scriptId, script);
       }
     } catch (cause: unknown) {
-      this.#failure = new AnalysisOutputError(
-        this.session.operation,
-        cause instanceof Error ? cause.message : String(cause),
-      );
+      this.#failure =
+        cause instanceof AnalysisError
+          ? cause
+          : new AnalysisOutputError(
+              this.session.operation,
+              cause instanceof Error ? cause.message : String(cause),
+            );
     }
   }
 
@@ -264,11 +269,11 @@ export class CdpRuntimeSources {
     const text = source.data.scriptSource;
     const bytes = Buffer.byteLength(text);
     this.#sourceBytes += bytes;
-    if (this.#sourceBytes > WEB_RUNTIME_LIMITS.sourceBytes)
-      throw new AnalysisOutputError(
-        this.session.operation,
-        "Complete runtime source evidence exceeds its 32 MiB budget; no partial evidence is returned.",
-      );
+    assertRuntimeRetention(
+      this.session.operation,
+      "sources",
+      this.#sourceBytes,
+    );
     return {
       state: "captured",
       text,

src/browser/network/PlaywrightNetworkContent.test.ts

@@ -0,0 +1,79 @@
+import { expect, it } from "vitest";
+import { browserScenarioSchema } from "../../domain/browserScenario.js";
+import { BrowserScenarioSecrets } from "../BrowserScenarioSecrets.js";
+import {
+  captureNetworkBody,
+  captureNetworkHeaders,
+} from "./PlaywrightNetworkContent.js";
+
+const scenario = browserScenarioSchema.parse({
+  browser: { mode: "launch", executable_path: "/fixture/chrome" },
+  start_url: { url: "https://example.test" },
+  actions: [{ step_id: "done", action: "wait_for_timeout", duration_ms: 1 }],
+  secrets: [{ secret_id: "declared", environment_variable: "SECRET" }],
+});
+const secrets = BrowserScenarioSecrets.resolve(scenario, { SECRET: "x" });
+if (secrets === undefined) throw new Error("Fixture secret was not resolved");
+const options = {
+  timeoutMs: 100,
+  maxBodyBytes: 4,
+  maxBodyAllocationBytes: 1024 * 1024,
+  maxHeaderAllocationBytes: 16384,
+  maxMetadataAllocationBytes: 4096,
+};
+const source = {
+  headers: () => ({ "content-type": "text/plain" }),
+  headersArray: async () => [],
+};
+
+it("enforces the retained body byte limit after secret expansion", async () => {
+  expect(
+    await captureNetworkBody(
+      async () => Buffer.from("xxxx"),
+      "response",
+      source,
+      secrets,
+      options,
+    ),
+  ).toMatchObject({
+    state: "unavailable",
+    reason: "body-size-limit",
+    limit_bytes: 4,
+  });
+  expect(
+    await captureNetworkBody(
+      async () => Buffer.from("safe"),
+      "response",
+      source,
+      secrets,
+      options,
+    ),
+  ).toMatchObject({
+    state: "captured",
+    bytes: 4,
+    content: Buffer.from("safe").toString("base64"),
+  });
+});
+
+it("charges discarded credential header values only for their retained projection", async () => {
+  const read = async (name: string) =>
+    captureNetworkHeaders(
+      {
+        headers: () => ({}),
+        headersArray: async () => [
+          { name, value: "v".repeat(50_000) },
+          { name: "X-Observed", value: "ordinary" },
+        ],
+      },
+      secrets,
+      options,
+    );
+  expect(await read("Cookie")).toMatchObject({
+    state: "captured",
+    items: [
+      { name: "Cookie", value: null, redacted: true },
+      { name: "X-Observed", value: "ordinary", redacted: false },
+    ],
+  });
+  expect(await read("X-Retained")).toMatchObject({ state: "unavailable" });
+});

src/browser/network/PlaywrightNetworkContent.ts

@@ -79,29 +79,32 @@ export const captureNetworkHeaders = async (
       options.signal,
     );
     let cost = 0;
+    const items: Extract<
+      BrowserNetworkHeaders,
+      { state: "captured" }
+    >["items"] = [];
     for (const { name, value } of headers) {
-      cost += secrets.redactionAllocationCost(name.length + value.length) + 128;
+      const credential = credentialHeaders.has(name.toLowerCase());
+      cost +=
+        secrets.redactionAllocationCost(
+          name.length + (credential ? 0 : value.length),
+        ) + 128;
       if (cost > options.maxHeaderAllocationBytes)
         return {
           state: "unavailable",
           message: `Headers exceed the ${String(options.maxHeaderAllocationBytes)}-byte allocation budget.`,
         };
+      const safeName = secrets.redact(name);
+      const safeValue = credential
+        ? null
+        : secrets.redactBytes(Buffer.from(value)).toString("utf8");
+      items.push({
+        name: safeName,
+        value: safeValue,
+        redacted: credential || safeName !== name || safeValue !== value,
+      });
     }
-    return {
-      state: "captured",
-      items: headers.map(({ name, value }) => {
-        const safeName = secrets.redact(name);
-        const credential = credentialHeaders.has(name.toLowerCase());
-        const safeValue = credential
-          ? null
-          : secrets.redactBytes(Buffer.from(value)).toString("utf8");
-        return {
-          name: safeName,
-          value: safeValue,
-          redacted: credential || safeName !== name || safeValue !== value,
-        };
-      }),
-    };
+    return { state: "captured", items };
   } catch (cause: unknown) {
     return {
       state: "unavailable",
@@ -177,6 +180,15 @@ export const captureNetworkBody = async (
     if (admission?.status === "omitted")
       return { state: "omitted", reason: admission.reason };
     const retained = secrets.redactBytes(observed);
+    if (retained.length > maxBodyBytes)
+      return {
+        state: "unavailable",
+        reason: "body-size-limit",
+        message:
+          "Secret redaction expanded the retained body beyond the selected byte limit.",
+        limit_bytes: maxBodyBytes,
+        observed_bytes: retained.length,
+      };
     return {
       state: "captured",
       representation:

src/browser/network/PlaywrightScenarioNetwork.test.ts

@@ -171,6 +171,44 @@ const bodyText = (event: BrowserScenarioEvent) => {
 
 afterEach(() => vi.useRealTimers());
 
+it("keeps observed completion independent of rejected completion metadata", async () => {
+  const { secrets, limits } = harness();
+  let omit = false;
+  const omitted: string[] = [];
+  const collector = new PlaywrightScenarioNetwork({
+    selection: {
+      request_body: false,
+      response_body: false,
+      header_values: false,
+    },
+    limits,
+    secrets,
+    admitText: () =>
+      omit
+        ? {
+            status: "omitted",
+            sequence: 1,
+            reason: "max-total-event-allocation-bytes",
+          }
+        : { status: "retained", sequence: 1 },
+    emit: () => ({ status: "retained", sequence: 1 }),
+    admitContentBody: () => ({ status: "retained", sequence: 1 }),
+    omitNetwork: (_id, kind) => {
+      omitted.push(kind);
+    },
+    omitEvent: (kind) => {
+      omitted.push(kind);
+    },
+  });
+  const request = new RequestFixture("ordinary");
+  collector.request(request);
+  omit = true;
+  collector.finished(request);
+  await collector.finish();
+  expect(omitted).toContain("request-finished");
+  expect(omitted).not.toContain("request-unfinished");
+});
+
 it.each(["finished", "failed", "unfinished"])(
   "counts selected omitted response content when %s",
   async (terminal) => {

src/browser/network/PlaywrightScenarioNetwork.ts

@@ -252,8 +252,8 @@ export class PlaywrightScenarioNetwork {
       this.omitOmittedTransactionPhase(request, "request-finished");
       return;
     }
-    if (!this.admitMetadata(transaction, "request-finished", request)) return;
     transaction.terminal = "finished";
+    if (!this.admitMetadata(transaction, "request-finished", request)) return;
     const admission = this.options.emit({
       ...this.metadata(request, transaction),
       kind: "request-finished",

src/cli/webRuntimeCommands.ts

@@ -3,11 +3,11 @@ import { createWebRuntimeService } from "../composition/webRuntime.js";
 import { browserCliError } from "./browserCliError.js";
 import {
   browserScopeOptions,
-  observationDuration,
   selectedCommandTimeout,
 } from "../cliObservationOptions.js";
 import { logCliCommand } from "../cliLogging.js";
 import { CLI_COMMANDS } from "../cliCommandNames.js";
+import { observeWebExecutionInputSchema } from "../domain/webExecution.js";
 import type { Logger } from "pino";
 import type { ProgressReporter } from "../application/ProgressReporter.js";
 import { withCommandCancellation } from "./commandCancellation.js";
@@ -36,7 +36,10 @@ export const registerWebRuntimeCommands = (
     }),
     options: z.object({
       ...browserScopeOptions,
-      observationMs: observationDuration(10_000, 1),
+      observationMs:
+        observeWebExecutionInputSchema.shape.observation_ms.describe(
+          "Observation duration in milliseconds",
+        ),
     }),
     run: ({ args, options }) =>
       withCommandCancellation((signal) =>