fix(web): preserve JSON identity and coverage contracts

Decode base64 JSON body bytes at the shape owner without replacement characters. Keep incomplete request correspondence unknown and use explicitly complete authored inventories for positive comparison controls. Repair semantic truncation and storage fixtures to satisfy their owning contracts without weakening assertions.

docs/browser-observation.md

@@ -421,7 +421,7 @@ REA retains local observation data and removes only URL userinfo credentials, ex
 - Supported DOM URL attributes are projected separately as resolved destinations. Empty `action` on HTML or XHTML forms and empty `formaction` on HTML or XHTML submit controls resolve to the document URL; other relative URL attributes resolve against the captured document base.
 - Accessibility structure and reported roles are retained by default, while names and descriptions require `include_accessibility_text: true`. The browser protocol may omit an individual role; that role remains unknown for comparison. Other observed accessibility differences can still establish a change.
 - Network observations retain method, status, MIME type, size, type, initiator stack location, and complete URLs with userinfo credentials removed. Redirects reported by CDP are attached as ordered `redirects` on the one request ID, with the prior request URL, CDP response URL, method/type, response status/MIME/encoded length, request timestamp, and the next `requestWillBeSent` event timestamp. CDP delivers `redirectResponse` with that next request event, so the latter is an event boundary rather than an exact response timestamp. Redirect response headers and bodies are discarded; hops outside the selected origins are discarded with the request chain. Headers are discarded after an allowlisted projection of length/encoding, structured CSP/Link/policy fields, and untrusted agent hints. Cookies and authorization headers are never retained.
-- Request/response bodies are not requested or parsed by default. When selected, only allowed-origin JSON media types are read, converted immediately to complete value-free typed paths, types and observation counts, and discarded. Each path segment is either `{ "kind": "property", "name": "raw property name" }` or `{ "kind": "array-element" }`; array-element types include primitives and nested arrays. Literal `*` properties remain distinct from array elements. `Network.getResponseBody` is never sent unless JSON body shape capture is selected. A JSON-typed request body that CDP reports through `hasPostData` but omits from `postData`, or that cannot be parsed as JSON, makes `json_body_shapes` unavailable. That request's `body_shapes.status` is `partial`, not `included`, when only its response shape was captured. Comparison uses body shapes only when both captures selected them, and compares request/response shapes only where both observations provide those facts. Unavailable or truncated selected coverage makes equivalence unknown; observed differences still report changed.
+- Request/response bodies are not requested or parsed by default. When selected, only allowed-origin JSON media types are read, converted immediately to complete value-free typed paths, types and observation counts, and discarded. Each path segment is either `{ "kind": "property", "name": "raw property name" }` or `{ "kind": "array-element" }`; array-element types include primitives and nested arrays. Literal `*` properties remain distinct from array elements. `Network.getResponseBody` is never sent unless JSON body shape capture is selected. A JSON-typed request body that CDP reports through `hasPostData` but omits from `postData`, or that cannot be parsed as JSON, makes `json_body_shapes` unavailable. That request's `body_shapes.status` is `partial`, not `included`, when only its response shape was captured. Base64 response bytes must be valid UTF-8 before JSON interpretation; invalid bytes leave the response shape unavailable and preserve any observed request shape and network metadata. Producer-decoded strings and genuine U+FFFD characters retain their reported meaning. Comparison uses body shapes only when both captures selected them, and compares request/response shapes only where both observations provide those facts. Unavailable or truncated selected coverage makes equivalence unknown; observed differences still report changed.
   For normalized passive inputs with complete request inventories on both sides,
   request additions and removals remain observable even when selected body shapes,
   response metadata, or WebSocket coverage are unavailable. HTTP request and

src/browser/CdpCaptureEventBodyShapes.ts

@@ -21,9 +21,7 @@ export const ingestResponseBodyShape = (
     invalidResponseBodyShape(state, requestId);
     return;
   }
-  const decoded = base64Encoded
-    ? decodePaddedBase64(body)?.toString("utf8")
-    : body;
+  const decoded = base64Encoded ? decodePaddedBase64(body) : body;
   if (decoded === undefined) {
     invalidResponseBodyShape(state, requestId);
     return;

src/browser/PlaywrightScenarioSession.test.ts

@@ -14,6 +14,7 @@ const mocks = vi.hoisted(() => ({
         initialize: ReturnType<typeof vi.fn>;
         settle: ReturnType<typeof vi.fn>;
         close: ReturnType<typeof vi.fn>;
+        limitations: ReturnType<typeof vi.fn>;
       }
     | undefined,
   finishEvents: vi.fn(),
@@ -208,6 +209,7 @@ const fixture = () => {
     close: vi.fn(async (stopLoading = false) => ({
       loadingStopped: stopLoading,
     })),
+    limitations: vi.fn(() => []),
   };
   return { cleanup, page, storage: mocks.storage };
 };

src/domain/javascript/javascriptSemanticGraphQuery.test.ts

@@ -51,8 +51,15 @@ it.each(["unknown", "unavailable", "partial"] as const)(
           truncated: true,
           limits: [{ name: "nodes", value: 1, unit: "items" }],
         },
+        limitations: [
+          "Authored truncation tests status and omission coupling.",
+        ],
       }),
-    ).toThrow(/Truncated semantic graph coverage/u);
+    ).toThrow(
+      status === "partial"
+        ? "Truncated semantic graph coverage must acknowledge omitted facts"
+        : "Truncated semantic graph coverage must be partial",
+    );
   },
 );
 
@@ -69,6 +76,7 @@ it("accepts truncated partial coverage with unknown omission extents", () => {
         omitted_relations: null,
         limits: [{ name: "nodes", value: 1, unit: "items" }],
       },
+      limitations: ["Node ceiling left semantic omission extents unknown."],
     }).coverage,
   ).toMatchObject({ status: "partial", truncated: true });
 });
@@ -92,6 +100,7 @@ it("accepts omitted unknown occurrences as the reason for semantic truncation",
           : coverage,
       ),
     },
+    limitations: ["Unknown ceiling omitted one resource-limit occurrence."],
   });
   expect(graph.coverage).toMatchObject({
     truncated: true,

src/domain/jsonShape.ts

@@ -43,13 +43,21 @@ interface ShapeNode {
   children?: Map<string | typeof arrayElementKey, ShapeNode>;
 }
 
-/** Parse approved JSON and immediately discard values after iterative shape inference. */
-export const inferJsonShape = (text: string): JsonShape | null => {
+/** Parse approved JSON text or exact UTF-8 bytes, then discard values after shape inference. */
+export const inferJsonShape = (
+  source: string | Uint8Array,
+): JsonShape | null => {
   let root: unknown;
   try {
+    const text =
+      typeof source === "string"
+        ? source
+        : new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
+            source,
+          );
     root = JSON.parse(text);
   } catch (cause: unknown) {
-    // Invalid JSON is represented by the null return.
+    // Invalid JSON or non-UTF-8 bytes cannot establish a shape.
     void cause;
     return null;
   }

tests/boundary/browser/cdpBrowserProvider.responseUtf8.test.ts

@@ -0,0 +1,158 @@
+import { expect, it } from "vitest";
+
+import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js";
+import { inspectWebPageInputSchema } from "../../../src/domain/browserObservation.js";
+import { startFakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js";
+import { trackBrowser } from "./cdpBrowserProvider.support.js";
+
+it.each([
+  {
+    name: "invalid UTF-8 property name",
+    body: Buffer.from([123, 34, 255, 34, 58, 116, 114, 117, 101, 125]),
+    property: null,
+  },
+  {
+    name: "invalid UTF-8 string value",
+    body: Buffer.from([123, 34, 107, 34, 58, 34, 255, 34, 125]),
+    property: null,
+  },
+  {
+    name: "overlong UTF-8 property name",
+    body: Buffer.from([123, 34, 192, 175, 34, 58, 116, 114, 117, 101, 125]),
+    property: null,
+  },
+  {
+    name: "valid UTF-8 property name",
+    body: Buffer.from('{"é":true}'),
+    property: "é",
+  },
+  {
+    name: "literal replacement-character property name",
+    body: Buffer.from('{"�":true}'),
+    property: "�",
+  },
+])("preserves JSON-shape uncertainty for $name", async ({ body, property }) => {
+  const browser = await startFakeCdpBrowser({
+    eventsBeforeReply: ["Network.enable"],
+    commandEvents(command, origin) {
+      if (command.method !== "Network.enable") return [];
+      const session =
+        command.sessionId === undefined ? {} : { sessionId: command.sessionId };
+      return [false, true].flatMap((hasRequestShape) => {
+        const requestId = hasRequestShape
+          ? "request-and-response"
+          : "response-only";
+        const url = `${origin}/${requestId}`;
+        return [
+          {
+            ...session,
+            method: "Network.requestWillBeSent",
+            params: {
+              requestId,
+              request: {
+                url,
+                method: hasRequestShape ? "POST" : "GET",
+                ...(hasRequestShape
+                  ? {
+                      headers: { "Content-Type": "application/json" },
+                      postData: '{"request":true}',
+                    }
+                  : {}),
+              },
+              initiator: { type: "other" },
+            },
+          },
+          {
+            ...session,
+            method: "Network.responseReceived",
+            params: {
+              requestId,
+              response: { url, status: 200, mimeType: "application/json" },
+            },
+          },
+          {
+            ...session,
+            method: "Network.loadingFinished",
+            params: { requestId, encodedDataLength: body.length },
+          },
+        ];
+      });
+    },
+    commandResult(command) {
+      return command.method === "Network.getResponseBody"
+        ? { body: body.toString("base64"), base64Encoded: true }
+        : undefined;
+    },
+  });
+  trackBrowser(browser);
+  const result = await new CdpBrowserProvider().inspectPage(
+    inspectWebPageInputSchema.parse({
+      cdp_endpoint: browser.endpoint,
+      allowed_origins: [browser.allowedOrigin],
+      target_id: "allowed-page",
+      observation_ms: 0,
+      include_json_body_shapes: true,
+    }),
+  );
+  if (!result.ok) throw result.error;
+  const requests = result.value.network.requests;
+  expect(requests).toHaveLength(2);
+  for (const request of requests) {
+    const hasRequestShape = request.request_id === "request-and-response";
+    expect(request).toMatchObject({
+      url: `${browser.allowedOrigin}/${request.request_id}`,
+      status: 200,
+      mime_type: "application/json",
+      encoded_data_length: body.length,
+      body_shapes: {
+        status:
+          property === null
+            ? hasRequestShape
+              ? "partial"
+              : "unavailable"
+            : "included",
+        request: hasRequestShape ? expect.any(Object) : null,
+        response:
+          property === null
+            ? null
+            : {
+                root_type: "object",
+                node_count: 2,
+                max_depth_observed: 1,
+                properties: [
+                  {
+                    path: [{ kind: "property", name: property }],
+                    types: ["boolean"],
+                    observations: 1,
+                  },
+                ],
+              },
+      },
+    });
+  }
+  expect(
+    browser.commands.filter(
+      (command) => command.method === "Network.getResponseBody",
+    ),
+  ).toHaveLength(2);
+  if (property === null) {
+    expect(result.value.completeness.unavailable_sections).toContain(
+      "json_body_shapes",
+    );
+    expect(result.value.completeness.excluded).toContainEqual({
+      section: "json_body_shapes",
+      reason: "provider_unavailable",
+      count: null,
+    });
+  } else {
+    expect(result.value.completeness.unavailable_sections).not.toContain(
+      "json_body_shapes",
+    );
+  }
+  expect(result.value.completeness.excluded).not.toContainEqual(
+    expect.objectContaining({
+      section: "json_body_shapes",
+      reason: "invalid_protocol_value",
+    }),
+  );
+});

tests/boundary/browser/cdpJsonShapeComparison.test.ts

@@ -11,6 +11,7 @@ import {
 } from "../../../src/domain/browserObservationSchemas.js";
 import { parseEvidence } from "../../../src/domain/evidence.js";
 import { webCaptureDiffSchema } from "../../../src/domain/webCaptureDiffSchemas.js";
+import { passiveNetworkCapture } from "../../fixtures/passiveNetworkCapture.js";
 
 const ORIGIN = "https://example.test";
 const CAPTURE_TIME = "2026-01-01T00:00:00.000Z";
@@ -104,6 +105,16 @@ it("captures all repeated array rows and empty arrays without retaining payload
 describe.each<BodySource>(["request", "response", "base64 response"])(
   "CDP %s JSON shape comparison",
   (source) => {
+    it("keeps differing retained shapes unknown when both request inventories are attach-limited", async () => {
+      const [before, after] = comparableCaptures(source, "[123]", '["a"]');
+      expect(
+        (await compareCaptures(before, after)).dimensions.network,
+      ).toMatchObject({
+        status: "unknown",
+        total_changes: 0,
+      });
+    });
+
     it.each(equivalentBodies)(
       "does not report a network modification for reordered $name",
       async ({ before: beforeBody, after: afterBody }) => {
@@ -144,7 +155,10 @@ describe.each<BodySource>(["request", "response", "base64 response"])(
         before + " ".repeat(length - Buffer.byteLength(before)),
         after + " ".repeat(length - Buffer.byteLength(after)),
       );
-      const result = await compareCaptures(...captures);
+      const result = await compareCaptures(
+        completeNormalizedCapture(captures[0]),
+        completeNormalizedCapture(captures[1]),
+      );
       expect(result.dimensions.network).toMatchObject({
         status: "changed",
         total_changes: 1,
@@ -167,7 +181,10 @@ describe.each<BodySource>(["request", "response", "base64 response"])(
           Buffer.byteLength(beforeBody) - Buffer.byteLength(changedBody),
         );
       const [before, after] = comparableCaptures(source, beforeBody, afterBody);
-      const result = await compareCaptures(before, after);
+      const result = await compareCaptures(
+        completeNormalizedCapture(before),
+        completeNormalizedCapture(after),
+      );
 
       expect(result.overall_status).toBe("changed");
       expect(result.dimensions.network).toEqual({
@@ -371,6 +388,16 @@ const compareCaptures = async (
   return webCaptureDiffSchema.parse(evidence.normalized_result);
 };
 
+/** Authored complete comparison inventories reuse parsed request facts, not live CDP coverage. */
+const completeNormalizedCapture = (
+  source: WebPageInspection,
+): WebPageInspection => {
+  const normalized = passiveNetworkCapture([]);
+  normalized.target = source.target;
+  normalized.network = source.network;
+  return normalized;
+};
+
 const captureBody = (
   source: BodySource,
   body: string,