feat(browser): retain compact scenario summaries and selected views (#7120)

Retain complete browser scenario Evidence while delivering compact summaries and selected step, event, transaction and text views through shared CLI/MCP workflows. Preserve initial CLI failures across cleanup retries and validate with real Chromium/TodoMVC.

Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

experiments/analysis-efficiency/browser-scenario-summary.md

@@ -0,0 +1,63 @@
+# Browser scenario summary verification
+
+The verifier runs the actual compiled CLI/MCP and caller-selected Chromium against
+TodoMVC's JavaScript ES5 distribution served on loopback. Ten actions add five
+todos. Every step captures URL, DOM, accessibility and a screenshot; the oracle
+checks all completed steps, the five authored labels, retained events and
+transaction identities. Summary runs also read step, text and transaction pages
+and authenticate an exported complete parent.
+
+Download the public distribution at the pinned revision into a disposable folder:
+
+```bash
+export REA_TODOMVC_APP="$(mktemp -d)"
+python3 - "$REA_TODOMVC_APP" <<'PY'
+import hashlib, json, sys, urllib.request
+from pathlib import Path
+root = Path(sys.argv[1])
+revision = "ff43b02e59dfa604386bb382034b2cd07c2bcd8a"
+files = ["index.html", "app.js", "base.css", "base.js", "controller.js",
+         "helpers.js", "index.css", "model.js", "store.js", "template.js", "view.js"]
+manifest = []
+for name in files:
+    url = f"https://raw.githubusercontent.com/tastejs/todomvc/{revision}/examples/javascript-es5/dist/{name}"
+    with urllib.request.urlopen(url, timeout=30) as reply:
+        data = reply.read()
+    (root / name).write_bytes(data)
+    manifest.append({"path": name, "url": url, "bytes": len(data),
+                     "sha256": hashlib.sha256(data).hexdigest()})
+(root / "provenance.json").write_text(json.dumps({"revision": revision, "files": manifest}, indent=2))
+PY
+```
+
+Build each selected checkout with the repository's pinned Node/npm toolchain.
+Use new output directories; exports never overwrite an earlier run:
+
+```bash
+node experiments/analysis-efficiency/browser-scenario-summary.mjs \
+  --repository /absolute/path/to/baseline --mode complete \
+  --application "$REA_TODOMVC_APP" --browser /absolute/path/to/chromium \
+  --output /absolute/path/to/new-baseline-report
+node experiments/analysis-efficiency/browser-scenario-summary.mjs \
+  --mode summary --application "$REA_TODOMVC_APP" \
+  --browser /absolute/path/to/chromium \
+  --output /absolute/path/to/new-summary-report
+```
+
+`cold` is the first capture in a fresh MCP process; `warm` repeats the capture in
+that process with a new owned browser/profile. Neither label establishes cold OS
+file caches. Summary mode intentionally performs additional retained queries.
+Compare equivalent answers and include those calls when reporting task cost;
+smaller replies alone do not establish a faster task.
+
+Add `--verify-boundaries` to a separate summary run for a real failed-step focus,
+CLI summary publication, and MCP text readback from the CLI's complete bundle.
+Those extra checks make its whole-run resource cost unsuitable for comparison
+against a complete-only run.
+
+Reports retain the actual catalog, arguments, MCP text and complete Evidence.
+When estimating tokens, name the tokenizer and count discovery, arguments and
+all replies needed for the chosen task. Raw serialized catalog estimates are
+not actual model usage or a measurement of a client's rendered tool definitions.
+Measure CPU and peak process-family memory externally, state the accounting
+method, and include cleanup; the verifier's elapsed phase excludes final teardown.

experiments/analysis-efficiency/browser-scenario-summary.mjs

@@ -0,0 +1,412 @@
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { readFile, writeFile, mkdir, readdir } from "node:fs/promises";
+import { resolve, join, extname } from "node:path";
+import { pathToFileURL, fileURLToPath } from "node:url";
+import { createRequire } from "node:module";
+import { performance } from "node:perf_hooks";
+import { execFile } from "node:child_process";
+import { promisify, parseArgs } from "node:util";
+// A developer verifier: the caller selects the pinned TodoMVC distribution,
+// browser executable, output directory and baseline/final repository checkout.
+const { values } = parseArgs({
+  options: {
+    repository: {
+      type: "string",
+      default: fileURLToPath(new URL("../../", import.meta.url)),
+    },
+    mode: { type: "string", default: "summary" },
+    application: { type: "string" },
+    browser: { type: "string" },
+    output: { type: "string" },
+    port: { type: "string", default: "0" },
+    "verify-boundaries": { type: "boolean", default: false },
+  },
+});
+assert.ok(
+  values.application && values.browser && values.output,
+  "Pass --application <TodoMVC javascript-es5 dist> --browser <executable> --output <new directory>",
+);
+const repo = resolve(values.repository);
+const mode = values.mode;
+const output = resolve(values.output);
+const appRoot = resolve(values.application);
+const executablePath = resolve(values.browser);
+assert.ok(["complete", "summary"].includes(mode));
+assert.ok(
+  !values["verify-boundaries"] || mode === "summary",
+  "--verify-boundaries requires summary mode",
+);
+process.chdir(repo);
+const require = createRequire(join(repo, "package.json"));
+const { Client } = await import(
+  pathToFileURL(require.resolve("@modelcontextprotocol/client")).href
+);
+const { StdioClientTransport } = await import(
+  pathToFileURL(require.resolve("@modelcontextprotocol/client/stdio")).href
+);
+await mkdir(output);
+const files = await readdir(appRoot);
+const server = createServer(async (request, response) => {
+  try {
+    const name =
+      decodeURIComponent(
+        new URL(request.url, "http://127.0.0.1").pathname,
+      ).slice(1) || "index.html";
+    assert.ok(files.includes(name));
+    const data = await readFile(join(appRoot, name));
+    response.setHeader(
+      "Content-Type",
+      { ".html": "text/html", ".js": "text/javascript", ".css": "text/css" }[
+        extname(name)
+      ] ?? "application/octet-stream",
+    );
+    response.end(data);
+  } catch {
+    response.writeHead(404);
+    response.end();
+  }
+});
+await new Promise((done, reject) => {
+  server.once("error", reject);
+  server.listen(Number(values.port), "127.0.0.1", done);
+});
+const address = server.address();
+assert.ok(address && typeof address === "object");
+const origin = `http://127.0.0.1:${address.port}`;
+const environment = {
+  PATH: process.env.PATH,
+  NODE_OPTIONS: "--max-old-space-size=3072",
+  UV_THREADPOOL_SIZE: "2",
+  REA_LOG_LEVEL: "silent",
+  REA_MCP_INPUT_SCHEMA_PROFILE: "compact",
+};
+const transport = new StdioClientTransport({
+  command: process.execPath,
+  args: [join(repo, "scripts/rea.mjs"), "mcp"],
+  cwd: repo,
+  env: environment,
+  stderr: "pipe",
+});
+let stderr = "";
+transport.stderr?.on("data", (chunk) => {
+  if (stderr.length < 65536) stderr += chunk;
+});
+const client = new Client({
+  name: "rea-browser-scenario-summary-verifier",
+  version: "1",
+});
+const calls = [];
+const invoke = async (name, args, tools) => {
+  const toolDefinition = tools.find((tool) => tool.name === name);
+  assert.ok(
+    toolDefinition?.outputSchema,
+    `Missing named output schema: ${name}`,
+  );
+  const started = performance.now();
+  const result = await client.callTool(
+    { name, arguments: args },
+    { toolDefinition, timeout: 90000 },
+  );
+  calls.push({
+    name,
+    arguments: args,
+    content: result.content,
+    elapsed_ms: performance.now() - started,
+  });
+  assert.notEqual(result.isError, true, JSON.stringify(result));
+  return result.structuredContent;
+};
+const allStarted = performance.now();
+try {
+  await client.connect(transport);
+  const discoveryStarted = performance.now();
+  const catalog = await client.listTools();
+  const discoveryMs = performance.now() - discoveryStarted;
+  await writeFile(join(output, "catalog.json"), JSON.stringify(catalog));
+  const actions = Array.from({ length: 5 }, (_, index) => [
+    {
+      step_id: `fill_${index}`,
+      action: "fill",
+      locator: { kind: "css", selector: ".new-todo" },
+      value: { source: "literal", value: `Review evidence ${index + 1}` },
+      timeout_ms: 10000,
+    },
+    {
+      step_id: `add_${index}`,
+      action: "press",
+      locator: { kind: "css", selector: ".new-todo" },
+      key: "Enter",
+      timeout_ms: 10000,
+    },
+  ]).flat();
+  const scenario = {
+    browser: {
+      mode: "launch",
+      executable_path: executablePath,
+      headless: true,
+    },
+    start_url: { url: origin, query: [] },
+    actions,
+    capture: {
+      after_each_step: ["url", "dom", "accessibility", "screenshot"],
+      at_end: ["url", "dom", "accessibility"],
+      events: ["network"],
+    },
+  };
+  const runs = [];
+  for (const temperature of ["cold", "warm"]) {
+    const callStart = calls.length;
+    const result = await invoke(
+      "capture_browser_scenario",
+      { ...scenario, ...(mode === "summary" ? { detail: "summary" } : {}) },
+      catalog.tools,
+    );
+    const parentId =
+      mode === "summary"
+        ? result.normalized_result.parent_evidence_id
+        : result.evidence_id;
+    const bundlePath = join(output, `${temperature}-evidence.json`);
+    await invoke("export_evidence_bundle", { path: bundlePath }, catalog.tools);
+    const bundle = JSON.parse(await readFile(bundlePath, "utf8"));
+    const parent = bundle.records.find((row) => row.evidence_id === parentId);
+    assert.ok(parent);
+    const capture = parent.normalized_result;
+    assert.equal(capture.steps.length, 11);
+    assert.ok(capture.steps.every((step) => step.status === "completed"));
+    assert.equal(capture.browser.cleanup, "terminated-owned-process");
+    const final = capture.steps.at(-1).artifacts.accessibility;
+    assert.equal(final.state, "captured");
+    for (let index = 1; index <= 5; index++)
+      assert.ok(final.value.text.includes(`Review evidence ${index}`));
+    assert.ok(final.value.text.includes('strong: "5"'));
+    assert.ok(final.value.text.includes("text: items left"));
+    const transactionIds = new Set(
+      capture.events.items
+        .filter((event) => "transaction_id" in event)
+        .map((event) => event.transaction_id),
+    );
+    const details = {
+      temperature,
+      parentId,
+      steps: capture.steps.length,
+      retained_events: capture.events.retained,
+      transactions: transactionIds.size,
+    };
+    if (mode === "summary") {
+      const summary = result.normalized_result.summary;
+      assert.equal(summary.steps.length, 11);
+      assert.equal(
+        summary.final_accessibility.accessibility.value.text,
+        final.value.text,
+      );
+      assert.equal(summary.events.retained, capture.events.retained);
+      assert.equal(summary.limitation_count, parent.limitations.length);
+      assert.equal(summary.network.transactions, transactionIds.size);
+      const source = { kind: "retained-evidence", evidence_id: parentId };
+      const pagingStarted = performance.now();
+      for (const stepIndex of [1, 5, 10]) {
+        const page = await invoke(
+          "inspect_analysis_view",
+          {
+            source,
+            view: {
+              kind: "browser-scenario",
+              collection: "steps",
+              step_indices: [stepIndex],
+              artifacts: [],
+              limit: 1,
+            },
+          },
+          catalog.tools,
+        );
+        assert.equal(
+          page.normalized_result.items.rows[0].step_id,
+          capture.steps[stepIndex].step_id,
+        );
+      }
+      details.warm_page_total_ms = performance.now() - pagingStarted;
+      const text = await invoke(
+        "inspect_analysis_view",
+        {
+          source,
+          view: {
+            kind: "browser-scenario",
+            collection: "text",
+            step_index: 10,
+            artifact: "accessibility",
+            offset: 0,
+            limit: 32,
+          },
+        },
+        catalog.tools,
+      );
+      assert.equal(
+        text.normalized_result.items.text,
+        final.value.text.slice(0, 32),
+      );
+      const tx = await invoke(
+        "inspect_analysis_view",
+        {
+          source,
+          view: {
+            kind: "browser-scenario",
+            collection: "transactions",
+            limit: 2,
+          },
+        },
+        catalog.tools,
+      );
+      assert.deepEqual(
+        tx.normalized_result.items.rows.map((row) => row.transaction_id),
+        [...transactionIds].slice(0, 2),
+      );
+      details.returned_summary = summary;
+    }
+    details.calls = calls.slice(callStart);
+    runs.push(details);
+  }
+  const boundaryChecks = [];
+  if (values["verify-boundaries"]) {
+    const failed = await invoke(
+      "capture_browser_scenario",
+      {
+        ...scenario,
+        actions: [
+          {
+            step_id: "missing",
+            action: "click",
+            locator: {
+              kind: "css",
+              selector: "#rea-control-that-does-not-exist",
+            },
+            timeout_ms: 100,
+          },
+        ],
+        capture: {
+          after_each_step: ["url", "accessibility"],
+          at_end: ["url", "accessibility"],
+          events: ["network"],
+        },
+        detail: "summary",
+      },
+      catalog.tools,
+    );
+    assert.equal(failed.normalized_result.summary.steps[1].status, "failed");
+    assert.equal(
+      failed.normalized_result.summary.final_accessibility.focus_step_index,
+      1,
+    );
+    assert.equal(
+      failed.normalized_result.summary.completeness.equality_eligible,
+      false,
+    );
+    boundaryChecks.push("real failed-step focus and incomplete coverage");
+    const execute = promisify(execFile);
+    const bundlePath = join(output, "cli-evidence.json");
+    const { stdout } = await execute(
+      process.execPath,
+      [
+        join(repo, "scripts/rea.mjs"),
+        "capture-browser-scenario",
+        JSON.stringify(scenario),
+        "--detail",
+        "summary",
+        "--evidence_output",
+        bundlePath,
+        "--json",
+      ],
+      {
+        cwd: repo,
+        env: environment,
+        maxBuffer: 64 * 1024 * 1024,
+        timeout: 90000,
+      },
+    );
+    const cli = JSON.parse(stdout);
+    const parentId = cli.summary.normalized_result.parent_evidence_id;
+    const bundle = JSON.parse(await readFile(bundlePath, "utf8"));
+    const parent = bundle.records.find((row) => row.evidence_id === parentId);
+    assert.ok(parent);
+    assert.equal(parent.normalized_result.steps.length, 11);
+    assert.ok(
+      parent.normalized_result.steps.every(
+        (step) => step.status === "completed",
+      ),
+    );
+    assert.ok(!("detail" in parent.parameters));
+    assert.equal(
+      cli.summary.normalized_result.summary.final_accessibility.accessibility
+        .value.text,
+      parent.normalized_result.steps.at(-1).artifacts.accessibility.value.text,
+    );
+    const text = await invoke(
+      "inspect_analysis_view",
+      {
+        source: {
+          kind: "evidence-file",
+          path: bundlePath,
+          evidence_id: parentId,
+        },
+        view: {
+          kind: "browser-scenario",
+          collection: "text",
+          step_index: 10,
+          artifact: "accessibility",
+          offset: 0,
+          limit: 32,
+        },
+      },
+      catalog.tools,
+    );
+    assert.equal(
+      text.normalized_result.items.text,
+      parent.normalized_result.steps
+        .at(-1)
+        .artifacts.accessibility.value.text.slice(0, 32),
+    );
+    boundaryChecks.push(
+      "real CLI summary, complete bundle and MCP evidence-file readback",
+    );
+  }
+  await writeFile(
+    join(output, "result.json"),
+    JSON.stringify(
+      {
+        repo,
+        mode,
+        application: appRoot,
+        browser: executablePath,
+        input_paths: {
+          repository: values.repository,
+          application: values.application,
+          browser: values.browser,
+          output: values.output,
+        },
+        origin,
+        discovery_ms: discoveryMs,
+        elapsed_ms: performance.now() - allStarted,
+        runs,
+        boundaryChecks,
+        stderr,
+      },
+      null,
+      2,
+    ),
+  );
+  console.log(
+    JSON.stringify({
+      mode,
+      discovery_ms: discoveryMs,
+      runs: runs.map(
+        ({ calls: _calls, returned_summary: _summary, ...rest }) => rest,
+      ),
+      boundaryChecks,
+      elapsed_ms: performance.now() - allStarted,
+    }),
+  );
+} finally {
+  await client.close();
+  server.closeAllConnections();
+  await new Promise((done) => server.close(done));
+}