fix(browser)!: preserve storage value code units (#6341)

Keep ordinary UTF-8 digests and secret redaction, separate malformed UTF-16 preimages, and record storage fingerprint provenance. Compare legacy hashed snapshots as unknown without treating encoding metadata as a target change. Cover producer boundaries and real owned-browser CLI/MCP workflows.

BREAKING CHANGE: legacy hashed browser storage remains readable but comparisons return unknown until both sides are recaptured.

Co-authored-by: gongfpp <22683417+gongfpp@users.noreply.github.com>

docs/browser-scenario-contract.md

@@ -387,6 +387,24 @@ observed) and excludes the sequence reference from normalization.
 Playwright scenario capture does not supply initiator stacks; passive CDP
 observation retains its own initiator evidence.
 
+## Storage value fingerprints
+
+New storage snapshots include `fingerprint_version: 2`. Non-secret, well-formed
+JavaScript strings retain their SHA-256 over UTF-8. Strings with unpaired UTF-16
+surrogates use SHA-256 over a raw `0xff` byte followed by their exact UTF-16LE
+code units; this prefix cannot occur in UTF-8. Declared secrets remain
+`redacted-secret` with a null digest, without a second or legacy digest.
+
+A missing version identifies legacy UTF-8 fingerprints, which replaced unpaired
+surrogates and cannot establish value identity. Storage containing any legacy
+hashed value is conservatively compared as `unknown`, including old-to-old
+comparisons; recapture both sides to compare reliably. This also conservatively
+withholds some otherwise provable legacy differences. Empty or entirely
+secret-redacted storage keeps its existing comparison semantics. The version is
+checked before normalization and excluded from content comparison: changing
+encoding metadata alone does not establish a website change. Existing captures
+and their completeness records are not rewritten.
+
 ## Scenario comparison
 
 `compare_web_captures` and `compare-web-captures` also accept two complete

scripts/lib/browser-scenario-storage-fingerprints-e2e.mjs

@@ -0,0 +1,227 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { fileURLToPath } from "node:url";
+import { Client } from "@modelcontextprotocol/client";
+import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
+
+import { parseEvidence } from "../../dist/domain/evidence.js";
+import { artifactCliEvidence, artifactMcpCall } from "./artifact-e2e.mjs";
+import {
+  runScenarioCli,
+  scenarioProfiles,
+} from "./browser-scenario-verifier.mjs";
+import { closeVerifierResources } from "./verifier-cleanup.mjs";
+
+const secretVariable = "REA_BROWSER_FINGERPRINT_VERIFIER_SECRET";
+const secret = "rea-storage-fingerprint-private";
+const utf8Digest = (value) => createHash("sha256").update(value).digest("hex");
+const literal = (value) => ({ source: "literal", value });
+const secretReference = { source: "secret", secret_id: "storage_secret" };
+
+const scenarioFor = (executable, origin, value) => {
+  const entries = [
+    { name: "value", value: literal(value) },
+    { name: "ordinary", value: literal("ordinary-value") },
+    { name: "secret", value: secretReference },
+  ];
+  return {
+    browser: { mode: "launch", executable_path: executable },
+    start_url: { url: `${origin}/scenario-environment` },
+    actions: [
+      { step_id: "settle", action: "wait_for_timeout", duration_ms: 1 },
+    ],
+    storage: {
+      // The shared site's favicon response also sets this same cookie.
+      cookies: [
+        ["rea-root-cookie", literal("root-fixture")],
+        ["ordinary", literal("ordinary-value")],
+        ["secret", secretReference],
+      ].map(([name, value]) => ({
+        name,
+        value,
+        destination: { url: origin },
+        http_only: false,
+        secure: false,
+        same_site: "Lax",
+      })),
+      local_storage: [{ origin, entries }],
+      session_storage: [{ origin, entries }],
+    },
+    secrets: [
+      { secret_id: "storage_secret", environment_variable: secretVariable },
+    ],
+    capture: { after_each_step: ["storage"], at_end: ["storage"], events: [] },
+  };
+};
+
+const assertCapture = (evidence) => {
+  const capture = evidence.normalized_result;
+  assert.equal(capture.browser.cleanup, "terminated-owned-process");
+  assert.equal(capture.completeness.equality_eligible, true);
+  assert.ok(!JSON.stringify(evidence).includes(secret));
+  assert.ok(!JSON.stringify(evidence).includes(utf8Digest(secret)));
+  for (const step of capture.steps) {
+    assert.equal(step.artifacts.storage.state, "captured");
+    const storage = step.artifacts.storage.value;
+    assert.equal(storage.fingerprint_version, 2);
+    for (const entries of [
+      storage.cookies,
+      storage.local_storage,
+      storage.session_storage,
+    ]) {
+      const byName = new Map(entries.map((entry) => [entry.name, entry]));
+      assert.equal(
+        byName.get("ordinary")?.value_sha256,
+        utf8Digest("ordinary-value"),
+      );
+      assert.equal(byName.get("secret")?.value_state, "redacted-secret");
+      assert.equal(byName.get("secret")?.value_sha256, null);
+    }
+  }
+};
+
+const assertComparison = (evidence, expected) => {
+  const result = evidence.normalized_result;
+  assert.equal(result.overall_status, expected);
+  for (const step of result.steps) {
+    assert.equal(step.status, expected);
+    assert.deepEqual(
+      step.artifact_diffs.map(({ artifact, status }) => ({ artifact, status })),
+      expected === "unchanged"
+        ? []
+        : [{ artifact: "storage", status: expected }],
+    );
+  }
+};
+
+const verifyValues = async (capture, compare) => {
+  const captures = [];
+  // The fourth capture is independent, not a self-comparison equality control.
+  for (const value of ["\ud800", "\ud801", "\ufffd", "\ud800"]) {
+    const evidence = await capture(value);
+    assertCapture(evidence);
+    captures.push(evidence);
+  }
+  for (const [index, expected] of [
+    [1, "changed"],
+    [2, "changed"],
+    [3, "unchanged"],
+  ])
+    assertComparison(
+      await compare(
+        captures[0].normalized_result,
+        captures[index].normalized_result,
+      ),
+      expected,
+    );
+  // Version-less imported snapshots must never receive new-scheme trust.
+  const current = captures[0].normalized_result;
+  const legacy = structuredClone(current);
+  for (const step of legacy.steps)
+    delete step.artifacts.storage.value.fingerprint_version;
+  for (const [before, after] of [
+    [legacy, legacy],
+    [legacy, current],
+    [current, legacy],
+  ])
+    assertComparison(await compare(before, after), "unknown");
+  return captures;
+};
+
+/** Real owned Chromium, public CLI and schema-validated stdio MCP storage identity. */
+export async function verifyScenarioStorageFingerprints(executable, origin) {
+  const previousSecret = process.env[secretVariable];
+  const profilesBefore = await scenarioProfiles();
+  process.env[secretVariable] = secret;
+  const transport = new StdioClientTransport({
+    command: process.execPath,
+    args: [fileURLToPath(new URL("../rea.mjs", import.meta.url)), "mcp"],
+    env: process.env,
+    stderr: "pipe",
+  });
+  const client = new Client({
+    name: "browser-storage-fingerprints-e2e",
+    version: "1",
+  });
+  let failure;
+  try {
+    const cli = await verifyValues(
+      (value) => runScenarioCli(scenarioFor(executable, origin, value)),
+      (before, after) =>
+        artifactCliEvidence("compare-web-captures", JSON.stringify(before), [
+          JSON.stringify(after),
+        ]),
+    );
+    await client.connect(transport);
+    const call = async (name, arguments_) => {
+      const result = await artifactMcpCall(client, {
+        name,
+        arguments: arguments_,
+      });
+      assert.notEqual(result.isError, true, JSON.stringify(result));
+      return result.structuredContent;
+    };
+    const mcp = await verifyValues(
+      async (value) =>
+        parseEvidence(
+          await call(
+            "capture_browser_scenario",
+            scenarioFor(executable, origin, value),
+          ),
+        ),
+      async (before, after) =>
+        parseEvidence(
+          await call("compare_web_captures", {
+            before_scenario: before,
+            after_scenario: after,
+          }),
+        ),
+    );
+    for (const [index, evidence] of cli.entries())
+      assert.deepEqual(
+        evidence.normalized_result.steps.map((step) => step.artifacts.storage),
+        mcp[index].normalized_result.steps.map(
+          (step) => step.artifacts.storage,
+        ),
+      );
+    const bundle = await call("get_evidence_bundle", {});
+    for (const evidence of mcp) {
+      const record = bundle.result.records.find(
+        ({ evidence_id }) => evidence_id === evidence.evidence_id,
+      );
+      assert.deepEqual(parseEvidence(record), evidence);
+    }
+    await client.ping();
+    return {
+      mocked: false,
+      cli: true,
+      stdio_mcp: true,
+      independent_captures: 8,
+      changed_code_units: true,
+      unchanged_control: true,
+      ordinary_utf8_preserved: true,
+      declared_secrets_redacted: true,
+      retained_evidence: true,
+      versionless_captures_unknown: true,
+    };
+  } catch (cause) {
+    failure = cause;
+    throw cause;
+  } finally {
+    if (previousSecret === undefined) delete process.env[secretVariable];
+    else process.env[secretVariable] = previousSecret;
+    await closeVerifierResources(
+      [
+        () => client.close(),
+        () => transport.close(),
+        async () =>
+          assert.ok(
+            [...(await scenarioProfiles())].every((profile) =>
+              profilesBefore.has(profile),
+            ),
+          ),
+      ],
+      failure,
+    );
+  }
+}

scripts/verify-real-browser.mjs

@@ -52,6 +52,7 @@ import { verifyBrowserCaptureMetadataBudget } from "./lib/browser-capture-metada
 import { artifactCliEvidence, artifactMcpResult } from "./lib/artifact-e2e.mjs";
 import { verifyScenarioEnvironment } from "./lib/browser-scenario-environment-e2e.mjs";
 import { verifyScenarioStorage } from "./lib/browser-scenario-storage-e2e.mjs";
+import { verifyScenarioStorageFingerprints } from "./lib/browser-scenario-storage-fingerprints-e2e.mjs";
 import { verifyScenarioServiceWorkers } from "./lib/browser-scenario-service-workers-e2e.mjs";
 import { verifyBrowserUrlUserInfo } from "./lib/browser-url-userinfo-e2e.mjs";
 
@@ -307,6 +308,11 @@ try {
     endpoint,
     targetId: target,
   });
+  process.stderr.write("Browser verifier: scenario storage fingerprints\n");
+  const scenarioStorageFingerprints = await verifyScenarioStorageFingerprints(
+    executable,
+    site.origin,
+  );
   process.stderr.write("Browser verifier: scenario service workers\n");
   const scenarioServiceWorkers = await verifyScenarioServiceWorkers(
     endpoint,
@@ -415,6 +421,7 @@ try {
     scenarioFailure,
     scenarioEnvironment,
     scenarioStorage,
+    scenarioStorageFingerprints,
     scenarioServiceWorkers,
     verified: true,
   };

src/browser/BrowserScenario.test.ts

@@ -1,3 +1,5 @@
+import { createHash } from "node:crypto";
+
 import { describe, expect, it } from "vitest";
 
 import { BrowserScenarioSecrets } from "./BrowserScenarioSecrets.js";
@@ -124,6 +126,60 @@ describe("browser scenario secret redaction", () => {
   });
 });
 
+describe("storage value fingerprint encoding", () => {
+  const secrets = BrowserScenarioSecrets.resolve(
+    browserScenarioSchema.parse(baseScenario()),
+    {
+      REA_TEST_SESSION: "declared-session",
+      REA_TEST_PASSWORD: "declared-password",
+      REA_TEST_COOKIE: "declared-cookie",
+    },
+  );
+  if (secrets === undefined) throw new Error("Expected resolved secrets");
+
+  it.each(["", "ordinary", "\u0000", "\u0620", "\ufffd", "\ud83d\ude00"])(
+    "preserves the UTF-8 digest of well-formed %j",
+    (value) => {
+      expect(secrets.fingerprint(value)).toEqual({
+        value_state: "hashed",
+        value_sha256: createHash("sha256").update(value).digest("hex"),
+      });
+    },
+  );
+
+  it.each([
+    ["\ud800", "\ud801"],
+    ["\ud800", "\ufffd"],
+    ["\ud801", "\ufffd"],
+    ["\udc00", "\ufffd"],
+    ["prefix\ud800suffix", "prefix\ud801suffix"],
+    ["\ud800\ud801", "\ud801\ud800"],
+    ["\udc00\ud800", "\ud800\udc00"],
+    // Bare UTF-16LE fallback would alias the UTF-8 bytes 00 d8 a0 00.
+    ["\ud800\u00a0", "\u0000\u0620\u0000"],
+  ])("distinguishes code-unit values %j and %j", (left, right) => {
+    expect(secrets.fingerprint(left).value_sha256).not.toBe(
+      secrets.fingerprint(right).value_sha256,
+    );
+  });
+
+  it("fixes the v2 fallback byte contract to raw FF followed by UTF-16LE", () => {
+    expect(secrets.fingerprint(String.fromCharCode(0xd800, 0x00a0))).toEqual({
+      value_state: "hashed",
+      value_sha256: createHash("sha256")
+        .update(Buffer.from([0xff, 0x00, 0xd8, 0xa0, 0x00]))
+        .digest("hex"),
+    });
+  });
+
+  it("redacts a declared secret before encoding an unpaired suffix", () => {
+    expect(secrets.fingerprint("declared-session\ud800")).toEqual({
+      value_state: "redacted-secret",
+      value_sha256: null,
+    });
+  });
+});
+
 describe("browserScenarioSchema", () => {
   it("accepts and normalizes a declared scenario", () => {
     const parsed = browserScenarioSchema.parse(baseScenario());

src/browser/BrowserScenarioSecrets.ts

@@ -396,9 +396,11 @@ export class BrowserScenarioSecrets {
       containsSecretPattern(value, normalizePercentEscapes(value), patterns)
     )
       return { value_state: "redacted-secret", value_sha256: null };
-    return {
-      value_state: "hashed",
-      value_sha256: createHash("sha256").update(value).digest("hex"),
-    };
+    const hash = createHash("sha256");
+    // In Unicode mode this class matches unpaired units, not valid pairs.
+    if (!/[\uD800-\uDFFF]/u.test(value)) hash.update(value);
+    // 0xff cannot occur in UTF-8, keeping the two preimage domains disjoint.
+    else hash.update(Buffer.from([0xff])).update(value, "utf16le");
+    return { value_state: "hashed", value_sha256: hash.digest("hex") };
   }
 }

src/browser/PlaywrightScenarioArtifacts.ts

@@ -5,6 +5,7 @@ import type { BrowserContext, Page } from "playwright-core";
 
 import type { BrowserScenario } from "../domain/browserScenario.js";
 import {
+  BROWSER_STORAGE_FINGERPRINT_VERSION,
   browserStepArtifactsSchema,
   type BrowserStepArtifacts,
 } from "../domain/browserScenarioCaptureValues.js";
@@ -154,6 +155,7 @@ const captureStorage = async (input: {
       }))()`),
     );
     const value = {
+      fingerprint_version: BROWSER_STORAGE_FINGERPRINT_VERSION,
       cookies: cookies.map((cookie) => ({
         name: secrets.redact(cookie.name),
         domain: secrets.redact(cookie.domain),

src/domain/browserScenarioCaptureValues.ts

@@ -98,7 +98,14 @@ const cookieShape = {
   same_site: z.enum(["Strict", "Lax", "None"]),
 };
 
+/** UTF-8 for well-formed strings; raw 0xff + UTF-16LE otherwise. */
+export const BROWSER_STORAGE_FINGERPRINT_VERSION = 2;
+
 const storageSnapshotSchema = z.strictObject({
+  // Absence retains the legacy, lossy UTF-8 interpretation.
+  fingerprint_version: z
+    .literal(BROWSER_STORAGE_FINGERPRINT_VERSION)
+    .optional(),
   cookies: z.array(
     z.discriminatedUnion("value_state", [
       z.strictObject({ ...cookieShape, ...hashedStorageValueShape }),

src/domain/browserScenarioDiff.test.ts

@@ -899,6 +899,158 @@ const multiStepScenarioCapture = (
   });
 };
 
+describe("storage fingerprint provenance", () => {
+  const withStorage = (value: unknown) => {
+    const base = scenarioCapture({});
+    return browserScenarioCaptureSchema.parse({
+      ...base,
+      steps: base.steps.map((step) => ({
+        ...step,
+        artifacts: {
+          ...step.artifacts,
+          storage: { state: "captured", value },
+        },
+      })),
+    });
+  };
+  const compare = (before: unknown, after: unknown) =>
+    compareBrowserScenarios(
+      compareBrowserScenariosInputSchema.parse({
+        before_scenario: withStorage(before),
+        after_scenario: withStorage(after),
+      }),
+    );
+  const empty = { cookies: [], local_storage: [], session_storage: [] };
+
+  it.each(["cookies", "local_storage", "session_storage"])(
+    "withholds legacy %s value identity on either side",
+    (kind) => {
+      const entry = {
+        name: "value",
+        value_state: "hashed",
+        value_sha256: "a".repeat(64),
+      };
+      const value = {
+        ...empty,
+        [kind]: [
+          kind === "cookies"
+            ? {
+                ...entry,
+                domain: "app.example.test",
+                path: "/",
+                secure: false,
+                http_only: false,
+                same_site: "Lax",
+              }
+            : entry,
+        ],
+      };
+      const current = { ...value, fingerprint_version: 2 };
+      for (const [before, after] of [
+        [value, value],
+        [value, current],
+        [current, value],
+      ]) {
+        const result = compare(before, after);
+        expect(result.overall_status).toBe("unknown");
+        expect(result.steps.flatMap((step) => step.artifact_diffs)).toEqual([
+          expect.objectContaining({
+            artifact: "storage",
+            status: "unknown",
+            reason: expect.stringContaining("Legacy storage fingerprints"),
+          }),
+          expect.objectContaining({
+            artifact: "storage",
+            status: "unknown",
+            reason: expect.stringContaining("Legacy storage fingerprints"),
+          }),
+        ]);
+      }
+      expect(compare(current, current).overall_status).toBe("unchanged");
+    },
+  );
+
+  it("conservatively withholds legacy comparisons even when digests differ", () => {
+    const entry = {
+      name: "value",
+      value_state: "hashed",
+      value_sha256: "a".repeat(64),
+    };
+    const left = { ...empty, local_storage: [entry] };
+    const right = {
+      ...empty,
+      local_storage: [{ ...entry, value_sha256: "b".repeat(64) }],
+    };
+    for (const [before, after] of [
+      [left, right],
+      [left, { ...right, fingerprint_version: 2 }],
+      [{ ...left, fingerprint_version: 2 }, right],
+    ])
+      expect(compare(before, after).overall_status).toBe("unknown");
+  });
+
+  it("checks legacy provenance before applying storage normalization", () => {
+    const legacy = {
+      ...empty,
+      local_storage: [
+        {
+          name: "value",
+          value_state: "hashed",
+          value_sha256: "a".repeat(64),
+        },
+      ],
+    };
+    const result = compareBrowserScenarios(
+      compareBrowserScenariosInputSchema.parse({
+        before_scenario: withStorage(legacy),
+        after_scenario: withStorage({ ...legacy, fingerprint_version: 2 }),
+        normalization: {
+          rules: [
+            {
+              rule_id: "rewrite-value-state",
+              artifacts: ["storage"],
+              match: "hashed",
+              replacement: "redacted-secret",
+            },
+          ],
+        },
+      }),
+    );
+    expect(result.overall_status).toBe("unknown");
+    expect(result.steps.flatMap((step) => step.artifact_diffs)).toEqual([
+      expect.objectContaining({
+        artifact: "storage",
+        status: "unknown",
+        reason: expect.stringContaining("Legacy storage fingerprints"),
+      }),
+      expect.objectContaining({
+        artifact: "storage",
+        status: "unknown",
+        reason: expect.stringContaining("Legacy storage fingerprints"),
+      }),
+    ]);
+  });
+
+  it.each([false, true])(
+    "does not turn encoding metadata alone into a storage change (redacted=%s)",
+    (redacted) => {
+      const entries = redacted
+        ? [
+            {
+              name: "secret",
+              value_state: "redacted-secret",
+              value_sha256: null,
+            },
+          ]
+        : [];
+      const legacy = { ...empty, local_storage: entries };
+      const current = { ...legacy, fingerprint_version: 2 };
+      expect(compare(legacy, current).overall_status).toBe("unchanged");
+      expect(compare(current, legacy).overall_status).toBe("unchanged");
+    },
+  );
+});
+
 interface ScenarioCaptureOptions {
   readonly stepId?: string;
   readonly action?: string;

src/domain/browserScenarioDiff.ts

@@ -11,6 +11,8 @@ import {
   prepareScenarioEvents,
   networkContentCoverageReason,
   screenshotProjection,
+  storageProjection,
+  storageFingerprintCoverageReason,
   type CaptureState,
 } from "./browserScenarioDiffEvidence.js";
 import {
@@ -363,6 +365,11 @@ const compareStepArtifacts = (
       artifact: "storage",
       before: input.beforeStep.artifacts.storage,
       after: input.afterStep.artifacts.storage,
+      project: storageProjection,
+      incomparableReason: storageFingerprintCoverageReason(
+        input.beforeStep.artifacts.storage,
+        input.afterStep.artifacts.storage,
+      ),
     }),
     compareScenarioEvents({
       ...common,

src/domain/browserScenarioDiffEvidence.ts

@@ -100,6 +100,41 @@ export const actionState = (step: BrowserScenarioStep) => ({
   error: step.error,
 });
 
+type StorageSnapshot = Extract<
+  BrowserStepArtifacts["storage"],
+  { state: "captured" }
+>["value"];
+
+/** Encoding provenance is not itself a target-storage difference. */
+export const storageProjection = ({
+  cookies,
+  local_storage,
+  session_storage,
+}: StorageSnapshot) => ({
+  cookies,
+  local_storage,
+  session_storage,
+});
+
+/** Legacy UTF-8 fingerprints have irreversibly lost unpaired code units. */
+export const storageFingerprintCoverageReason = (
+  ...states: readonly BrowserStepArtifacts["storage"][]
+): string | undefined =>
+  states.some(
+    (state) =>
+      state.state === "captured" &&
+      state.value.fingerprint_version === undefined &&
+      [
+        state.value.cookies,
+        state.value.local_storage,
+        state.value.session_storage,
+      ].some((entries) =>
+        entries.some(({ value_state }) => value_state === "hashed"),
+      ),
+  )
+    ? "Legacy storage fingerprints used lossy UTF-8; recapture both sides with code-unit-preserving storage fingerprints before comparing values."
+    : undefined;
+
 /** One step's ordered events and a single indexed network graph. */
 export interface ScenarioStepEvents {
   readonly events: readonly unknown[];