diff --git a/src/browser/history/playwrightTrace/PlaywrightTraceDecoder.test.ts b/src/browser/history/playwrightTrace/PlaywrightTraceDecoder.test.ts index 4b0923fda1..c9c137991c 100644 --- a/src/browser/history/playwrightTrace/PlaywrightTraceDecoder.test.ts +++ b/src/browser/history/playwrightTrace/PlaywrightTraceDecoder.test.ts @@ -5,10 +5,7 @@ import { expect, it } from "vitest"; import { ArtifactResourceScope } from "../../../resources/ArtifactResourceScope.js"; import { PlaywrightTraceDecoder } from "./PlaywrightTraceDecoder.js"; import { playwrightTraceSchema } from "../../../domain/playwrightTrace/playwrightTrace.js"; -import { - playwrightTraceFixture, - playwrightTracePathFixture, -} from "../../../../tests/fixtures/playwrightTrace.js"; +import { playwrightTraceFixture } from "../../../../tests/fixtures/playwrightTrace.js"; import { createTestTempDirectory } from "../../../../tests/fixtures/temporaryDirectory.js"; import { zstandardWindowFrameFixture, @@ -71,86 +68,6 @@ it("imports real v10 recorder text with per-stream identity and resource presenc } }); -it("preserves exact ZIP names for resources and paired stream profiles", async () => { - const fixture = await playwrightTracePathFixture(); - const resources = new ArtifactResourceScope(); - try { - const result = await new PlaywrightTraceDecoder(resources).inspect( - fixture.path, - ); - if (!result.ok) throw result.error; - const report = result.value; - expect(report.members.map((member) => member.path)).toContain( - "./resources/x", - ); - expect( - report.members.find((member) => member.path === "./resources/x"), - ).toMatchObject({ - logical_path: "resources/x", - limitations: [expect.stringContaining("leading current-directory")], - }); - expect( - report.members.find((member) => member.path === "resources/x"), - ).toMatchObject({ logical_path: "resources/x", limitations: [] }); - expect(report.events).toHaveLength(4); - expect(report.streams.map((stream) => stream.state)).toEqual([ - "supported", - "supported", - "supported", - "supported", - ]); - const members = new Map( - report.members.map((member) => [member.member_id, member]), - ); - for (const event of report.events) { - const source = members.get(event.source.member_id)!; - const expected = source.path.startsWith("./") - ? "./resources/x" - : "resources/x"; - expect(event.resource_references).toEqual([ - expect.objectContaining({ - path: expected, - state: "present-unread", - member_id: report.members.find((row) => row.path === expected) - ?.member_id, - matching_members: 1, - }), - ]); - } - expect( - report.events - .filter((event) => event.kind === "network") - .map((event) => event.status), - ).toEqual([200, 201]); - } finally { - await resources.close(); - } -}); - -it("keeps missing resource references explicit without turning page exhaustion into capture completeness", async () => { - const fixture = await playwrightTraceFixture(); - const resources = new ArtifactResourceScope(); - try { - const result = await new PlaywrightTraceDecoder(resources).inspect( - fixture.path, - ); - if (!result.ok) throw result.error; - expect(result.value.summary.unavailable_references).toBe(8); - expect(result.value.source_coverage.interpretation).toBe("partial"); - const references = result.value.events.flatMap( - (row) => row.resource_references, - ); - expect(references).toHaveLength(8); - expect( - references.every( - (row) => row.state === "missing" && row.matching_members === 0, - ), - ).toBe(true); - } finally { - await resources.close(); - } -}); - const header = JSON.stringify({ type: "context-options", version: 10, origin: "library" }) + "\n"; @@ -159,16 +76,20 @@ it("keeps failed-request status sentinels unknown without losing recorded respon const fixture = await playwrightTraceFixture({ trace: header, network: - statuses - .map((status) => - JSON.stringify({ - type: "resource-snapshot", - snapshot: { - request: { method: "GET", url: "http://127.0.0.1/failed" }, - response: { status, _failureText: "net::ERR_EMPTY_RESPONSE" }, - }, - }), - ) + [ + ...statuses.map((status) => ({ + type: "resource-snapshot", + snapshot: { + request: { method: "GET", url: "http://127.0.0.1/failed" }, + response: { status, _failureText: "net::ERR_EMPTY_RESPONSE" }, + }, + })), + { + type: "resource-snapshot", + snapshot: { response: { status: 200 } }, + }, + ] + .map((row) => JSON.stringify(row)) .join("\n") + "\n", }); const resources = new ArtifactResourceScope(); @@ -188,7 +109,13 @@ it("keeps failed-request status sentinels unknown without losing recorded respon 200, 600, 999, + 200, ]); + expect(report.events.at(-1)).toMatchObject({ + status: 200, + method: null, + url: null, + }); expect(report.events[0]).toMatchObject({ source: { line: 1, record_pointer: "/snapshot" }, reported: { @@ -201,7 +128,7 @@ it("keeps failed-request status sentinels unknown without losing recorded respon report.diagnostics .filter((row) => row.code === "partial-event") .map((row) => row.line), - ).toEqual([1, 2, 3, 4, 5]); + ).toEqual([1, 2, 3, 4, 5, 10]); } finally { await resources.close(); } @@ -247,31 +174,61 @@ it("retains valid siblings of malformed records and exposes exact producer lines } }); -it.each([ - ["", 0], - [before, 0], - [header + before + header + before, 1], -])( - "never guesses a missing or repeated context version %#", - async (trace, events) => { - const fixture = await playwrightTraceFixture({ trace, network: "" }); - const resources = new ArtifactResourceScope(); - try { - const result = await new PlaywrightTraceDecoder(resources).inspect( - fixture.path, - ); - if (!result.ok) throw result.error; - expect(result.value.events).toHaveLength(events); - expect(result.value.source_coverage.interpretation).toBe("partial"); - expect(result.value.streams.length).toBeGreaterThan(0); - expect( - result.value.streams.every((stream) => stream.state === "unknown"), - ).toBe(true); - } finally { - await resources.close(); - } - }, -); +it("requires an unambiguous v10 profile for each trace and paired network stream", async () => { + const network = + '{"type":"resource-snapshot","snapshot":{"request":{"method":"GET","url":"https://recorded.invalid/"},"response":{"status":200}}}\n'; + const fixture = await playwrightTraceFixture({ + trace: header + before + header + before, + network, + extraMembers: [ + { path: "empty.trace", contents: "" }, + { path: "empty.network", contents: network }, + { path: "missing-header.trace", contents: before }, + { path: "missing-header.network", contents: network }, + { + path: "duplicate-key.trace", + contents: + '{"type":"context-options","version":10,"version":11}\n' + before, + }, + { path: "duplicate-key.network", contents: network }, + ], + }); + const resources = new ArtifactResourceScope(); + try { + const result = await new PlaywrightTraceDecoder(resources).inspect( + fixture.path, + ); + if (!result.ok) throw result.error; + const { events, streams, members, diagnostics, source_coverage } = + result.value; + const sourcePath = (id: string | null) => + members.find((row) => row.member_id === id)?.path; + expect(streams).toHaveLength(8); + expect(streams.map((stream) => stream.state)).toEqual( + Array(8).fill("unknown"), + ); + expect( + events.map((event) => [ + sourcePath(event.source.member_id), + event.source.line, + event.call_id, + ]), + ).toEqual([["trace.trace", 2, "call-a"]]); + expect( + diagnostics.map((row) => [sourcePath(row.member_id), row.line, row.code]), + ).toEqual( + expect.arrayContaining([ + ["trace.trace", 3, "duplicate-context-header"], + ["empty.trace", null, "unknown-stream-version"], + ["missing-header.trace", 1, "missing-context-header"], + ["duplicate-key.trace", 1, "malformed-record"], + ]), + ); + expect(source_coverage.interpretation).toBe("partial"); + } finally { + await resources.close(); + } +}); it("refuses other declared trace versions with verified archive identity", async () => { const fixture = await playwrightTraceFixture({ @@ -335,28 +292,6 @@ it("keeps unsafe numeric literals exact instead of rounding recorded times", asy } }); -it.each([ - ['{"type":"context-options","version":10,"version":11}\n', "unknown"], - [header + '{"type":"resource-snapshot","snapshot":{}}\n', "supported"], -])( - "reports duplicate keys and incomplete known events explicitly %#", - async (trace, state) => { - const fixture = await playwrightTraceFixture({ trace, network: "" }); - const resources = new ArtifactResourceScope(); - try { - const result = await new PlaywrightTraceDecoder(resources).inspect( - fixture.path, - ); - if (!result.ok) throw result.error; - expect(result.value.streams[0]?.state).toBe(state); - expect(result.value.source_coverage.interpretation).toBe("partial"); - expect(result.value.diagnostics.length).toBeGreaterThan(0); - } finally { - await resources.close(); - } - }, -); - it("refuses oversized physical lines before parsing their representation", async () => { const trace = header + @@ -380,33 +315,6 @@ it("refuses oversized physical lines before parsing their representation", async } }); -it("never promotes facts from a stream whose ZIP CRC fails", async () => { - const fixture = await playwrightTraceFixture({ level: 0 }); - const bytes = await readFile(fixture.path); - const needle = Buffer.from('"method":"GET"'); - const offset = bytes.indexOf(needle); - expect(offset).toBeGreaterThan(0); - bytes[offset + 10] = 80; - await writeFile(fixture.path, bytes); - const resources = new ArtifactResourceScope(); - try { - const result = await new PlaywrightTraceDecoder(resources).inspect( - fixture.path, - ); - if (result.ok) throw new Error("Expected CRC refusal"); - expect(result.error._tag).toBe("AnalysisInputError"); - const partial = playwrightTraceSchema.parse( - result.error.partialObservation, - ); - expect(partial.summary.network).toBe(0); - expect(partial.summary.events).toBe(20); - expect(partial.source_coverage.interpretation).toBe("partial"); - expect(resources.hasPending).toBe(false); - } finally { - await resources.close(); - } -}); - it("rejects a sparse over-budget archive before hashing or expansion", async () => { const fixture = await playwrightTraceFixture(); const file = await open(fixture.path, "r+"); @@ -456,28 +364,7 @@ it("bounds JSON node expansion even when many tokens fit inside a small physical } }); -it("names the nesting allowance, not the node allowance, for a deeply nested small record", async () => { - const trace = - header + - `{"type":"console","args":${"[".repeat(80)}${"]".repeat(80)},"time":1}\n`; - const fixture = await playwrightTraceFixture({ trace, network: "" }); - const resources = new ArtifactResourceScope(); - try { - const result = await new PlaywrightTraceDecoder(resources).inspect( - fixture.path, - ); - if (result.ok) throw new Error("Expected nesting refusal"); - expect(result.error._tag).toBe("AnalysisResourceConstraintError"); - expect(projectAnalysisError(result.error).details).toMatchObject({ - refusal: { limit: 64, unit: "nesting levels" }, - }); - expect(resources.hasPending).toBe(false); - } finally { - await resources.close(); - } -}); - -it("keeps duplicate resource names ambiguous instead of selecting one ZIP occurrence", async () => { +it("distinguishes ambiguous and missing archive resource references", async () => { const fixture = await playwrightTraceFixture({ resources: true, level: 0 }); const frames = fixture.trace .split("\n") @@ -527,17 +414,30 @@ it("keeps duplicate resource names ambiguous instead of selecting one ZIP occurr expect( new Set(result.value.members.map((member) => member.member_id)).size, ).toBe(result.value.members.length); - expect( - result.value.events.flatMap((row) => row.resource_references), - ).toContainEqual( - expect.objectContaining({ - path: first, - state: "ambiguous", - member_id: null, - matching_members: 2, - }), + const references = result.value.events.flatMap( + (row) => row.resource_references, ); - expect(result.value.source_coverage.interpretation).toBe("partial"); + expect(references).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + path: first, + state: "ambiguous", + member_id: null, + matching_members: 2, + }), + expect.objectContaining({ + path: second, + state: "missing", + member_id: null, + matching_members: 0, + }), + ]), + ); + expect(result.value.summary.unavailable_references).toBe(2); + expect(result.value.source_coverage).toMatchObject({ + interpretation: "partial", + recording_completeness: "unknown", + }); } finally { await resources.close(); } diff --git a/tests/acceptance/applications/playwrightTrace.test.ts b/tests/acceptance/applications/playwrightTrace.test.ts index ba03f91be2..f6ada79d31 100644 --- a/tests/acceptance/applications/playwrightTrace.test.ts +++ b/tests/acceptance/applications/playwrightTrace.test.ts @@ -10,12 +10,13 @@ import { import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { advertisedMcpToolSchemas } from "../../fixtures/advertisedMcpTool.js"; -import { parseEvidence, type Evidence } from "../../../src/domain/evidence.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; import { playwrightTraceSchema } from "../../../src/domain/playwrightTrace/playwrightTrace.js"; import { analysisViewResultSchema } from "../../../src/domain/analysisView/analysisViewResult.js"; import { evidenceBundleSchema } from "../../../src/domain/evidenceBundle.js"; -type McpCall = Awaited>["call"]; +import { lifecycleResultOf } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; +import { evidenceRecordSelectionSchema } from "../../../src/domain/evidenceRecordSummary.js"; cliTest( "reports the fixed trace nesting limit through CLI and MCP", @@ -62,20 +63,6 @@ cliTest( REA_MCP_TOOL_FAMILIES: "application,session", PLAYWRIGHT_BROWSERS_PATH: "/rea-no-browser-required", }); - const parent = parseEvidence( - ( - await call("inspect_playwright_trace", { - path: fixture.path, - detail: "complete", - }) - ).structuredContent, - ); - const report = playwrightTraceSchema.parse(parent.normalized_result); - expect(report.events).toHaveLength(4); - expect(report.summary.unavailable_references).toBe(0); - expect(report.streams.every((stream) => stream.state === "supported")).toBe( - true, - ); const view = { kind: "playwright-trace", collection: "members", @@ -83,18 +70,17 @@ cliTest( fields: [], }; const selected = parseEvidence( - ( - await call("inspect_analysis_view", { - source: { - kind: "retained-evidence", - evidence_id: parent.evidence_id, - }, - view, - }) - ).structuredContent, + (await call("inspect_playwright_trace", { path: fixture.path, view })) + .structuredContent, ); + const result = await cli.run({ + arguments: ["inspect-playwright-trace", fixture.path, "--json"], + }); + expect(result.exitCode, result.stderr).toBe(0); + const parent = parseEvidence(result.json); expect(selected.normalized_result).toMatchObject({ kind: "playwright-trace", + parent_evidence_id: parent.evidence_id, items: [ { path: "./resources/x", @@ -105,50 +91,66 @@ cliTest( coverage: { total: 1, exhausted: true }, source_coverage: { interpretation: "partial" }, }); + const { members, streams, events, summary } = playwrightTraceSchema.parse( + parent.normalized_result, + ); + expect(members).toContainEqual( + expect.objectContaining({ + path: "resources/x", + logical_path: "resources/x", + limitations: [], + }), + ); + expect(streams.map((row) => row.state)).toEqual(Array(4).fill("supported")); + expect(events).toHaveLength(4); + expect(summary.unavailable_references).toBe(0); + for (const event of events) { + const source = members.find( + (row) => row.member_id === event.source.member_id, + ); + if (!source) throw new Error("Expected an exact source member"); + const path = source.path.startsWith("./") + ? "./resources/x" + : "resources/x"; + expect(event.resource_references).toEqual([ + expect.objectContaining({ + path, + state: "present-unread", + member_id: members.find((row) => row.path === path)?.member_id, + matching_members: 1, + }), + ]); + } expect( - (await call("inspect_playwright_trace", { path: fixture.path, view })) - .structuredContent, + events.filter((row) => row.kind === "network").map((row) => row.status), + ).toEqual([200, 201]); + expect( + ( + await call("inspect_analysis_view", { + source: { + kind: "retained-evidence", + evidence_id: parent.evidence_id, + }, + view, + }) + ).structuredContent, ).toEqual(selected); - const run = cli.run.bind(cli); - const result = await run({ - arguments: ["inspect-playwright-trace", fixture.path, "--json"], - }); - expect(result.exitCode, result.stderr).toBe(0); - expect(parseEvidence(result.json)).toEqual({ - ...parent, - caller_input_path: fixture.path, - }); expect((await readFile(fixture.path)).equals(original)).toBe(true); }, ); -const checkCliPublication = async ({ - cli, - fixture, - parent, - selected, - view, - call, -}: { - cli: TestCli; - fixture: Awaited>; - parent: Evidence; - selected: Evidence; - view: Record; - call: McpCall; -}) => { - const source = { kind: "retained-evidence", evidence_id: parent.evidence_id }; - const run = cli.run.bind(cli); - const fullCli = await run({ +const publishCliTrace = async ( + cli: TestCli, + fixture: Awaited>, + view: Record, +) => { + const full = await cli.run({ arguments: ["inspect-playwright-trace", fixture.path, "--json"], }); - expect(fullCli.exitCode, fullCli.stderr).toBe(0); - expect(parseEvidence(fullCli.json)).toEqual({ - ...parent, - caller_input_path: fixture.path, - }); + expect(full.exitCode, full.stderr).toBe(0); + const parent = parseEvidence(full.json); const bundlePath = join(fixture.root, "retained.bundle.json"); - const receipt = await run({ + const receipt = await cli.run({ arguments: [ "inspect-playwright-trace", fixture.path, @@ -162,36 +164,11 @@ const checkCliPublication = async ({ ], }); expect(receipt.exitCode, receipt.stderr).toBe(0); - expect(receipt.json).toMatchObject({ - summary: { - ...selected, - caller_input_path: fixture.path, - }, - }); expect( evidenceBundleSchema.parse(JSON.parse(await readFile(bundlePath, "utf8"))) .records, - ).toEqual([{ ...parent, caller_input_path: fixture.path }]); - await rm(fixture.path); - const retained = await call("inspect_analysis_view", { source, view }); - expect(parseEvidence(retained.structuredContent)).toEqual(selected); - const request = join(fixture.root, "page.json"); - await writeFile( - request, - JSON.stringify({ - source: { - kind: "evidence-file", - path: bundlePath, - evidence_id: parent.evidence_id, - }, - view, - }), - ); - const fromBundle = await run({ - arguments: ["inspect-analysis-view", request, "--json"], - }); - expect(fromBundle.exitCode, fromBundle.stderr).toBe(0); - expect(parseEvidence(fromBundle.json)).toEqual(selected); + ).toEqual([parent]); + return { parent, receipt, bundlePath }; }; cliTest( @@ -235,27 +212,46 @@ cliTest( expect(JSON.stringify(first.structuredContent)).not.toContain( "fixture loaded", ); - const complete = await call("inspect_playwright_trace", { - path: fixture.path, - detail: "complete", - }); - expect(complete.isError).not.toBe(true); - const parent = parseEvidence(complete.structuredContent); - const report = playwrightTraceSchema.parse(parent.normalized_result); - expect(summary.parent_evidence_id).toBe(parent.evidence_id); + if (summary.kind !== "summary") throw new Error("Expected trace summary"); const source = { kind: "retained-evidence", - evidence_id: parent.evidence_id, + evidence_id: summary.parent_evidence_id, }; + const view = { + kind: "playwright-trace", + collection: "events", + kinds: ["network"], + fields: [], + limit: 1, + }; + const { + parent: cliParent, + receipt, + bundlePath, + } = await publishCliTrace(cli, fixture, view); + await rm(fixture.path); + const bundle = await call("get_evidence_bundle", { + detail: "records", + filters: { evidence_id: source.evidence_id }, + }); + const { records } = lifecycleResultOf(evidenceRecordSelectionSchema).parse( + bundle.structuredContent, + ).result; + expect(records.map(({ evidence_id }) => evidence_id)).toEqual([ + source.evidence_id, + ]); + const parent = parseEvidence(records[0]); + expect(cliParent).toEqual({ ...parent, caller_input_path: fixture.path }); + const report = playwrightTraceSchema.parse(parent.normalized_result); const rows = []; - for (let offset = 0; offset < 23; offset += 5) { + for (const offset of [0, 12]) { const page = await call("inspect_analysis_view", { source, view: { kind: "playwright-trace", collection: "events", offset, - limit: 5, + limit: 12, fields: Object.keys(report.events[0]!), }, }); @@ -269,20 +265,14 @@ cliTest( expect(view.source_coverage).toEqual(report.source_coverage); } expect(rows).toEqual(report.events); - const view = { - kind: "playwright-trace", - collection: "events", - kinds: ["network"], - fields: [], - limit: 1, - }; - const selected = await call("inspect_playwright_trace", { - path: fixture.path, - view, - }); + const selected = await call("inspect_analysis_view", { source, view }); expect(selected.isError).not.toBe(true); + const selectedEvidence = parseEvidence(selected.structuredContent); + expect(receipt.json).toMatchObject({ + summary: { ...selectedEvidence, caller_input_path: fixture.path }, + }); const selectedView = analysisViewResultSchema.parse( - parseEvidence(selected.structuredContent).normalized_result, + selectedEvidence.normalized_result, ); expect(selectedView).toMatchObject({ kind: "playwright-trace", @@ -313,14 +303,23 @@ cliTest( parseEvidence(response.structuredContent).normalized_result, ).toMatchObject({ items: [], coverage: { total: 0, exhausted: true } }); } - await checkCliPublication({ - cli, - fixture, - parent, - selected: parseEvidence(selected.structuredContent), - view, - call, + const request = join(fixture.root, "page.json"); + await writeFile( + request, + JSON.stringify({ + source: { + kind: "evidence-file", + path: bundlePath, + evidence_id: source.evidence_id, + }, + view, + }), + ); + const fromBundle = await cli.run({ + arguments: ["inspect-analysis-view", request, "--json"], }); + expect(fromBundle.exitCode, fromBundle.stderr).toBe(0); + expect(parseEvidence(fromBundle.json)).toEqual(selectedEvidence); }, ); @@ -457,8 +456,7 @@ cliTest( expect( parseEvidence(full.structuredContent).normalized_result, ).toMatchObject({ items: [{ text: message }] }); - const run = cli.run.bind(cli); - const result = await run({ + const result = await cli.run({ arguments: ["inspect-playwright-trace", fixture.path, "--json"], }); expect(result.exitCode, result.stderr).toBe(0); @@ -473,12 +471,27 @@ cliTest( cliTest( "retains partial MCP Evidence when a trace stream fails after earlier streams were read", async () => { - const fixture = await playwrightTraceFixture({ level: 0 }); + const event = JSON.stringify({ + type: "resource-snapshot", + snapshot: { + request: { method: "GET", url: "https://recorded.invalid/value" }, + response: { status: 200 }, + }, + }); + const padding = JSON.stringify({ + type: "frame-snapshot", + snapshot: "x".repeat(8192), + }); + const tail = "trace-crc-tail"; + // Send the network event before the stored member's late CRC failure. + const fixture = await playwrightTraceFixture({ + level: 0, + network: `${event}\n${`${padding}\n`.repeat(80)}{"type":"${tail}"}\n`, + }); const bytes = await readFile(fixture.path); - // A stored network byte breaks only that member's ZIP CRC. - const offset = bytes.indexOf(Buffer.from('"method":"GET"')); + const offset = bytes.indexOf(tail); expect(offset).toBeGreaterThan(0); - bytes[offset + 10] = 80; + bytes[offset] = bytes[offset]! ^ 1; await writeFile(fixture.path, bytes); const { call } = await connectLocalToolsMcp({ REA_MCP_TOOL_FAMILIES: "application,session", @@ -490,6 +503,7 @@ cliTest( detail: "complete", }), ); + expect(failed.error.code).toBe("invalid_request"); const reference = failed.error.details?.partial_observation; expect(reference).toEqual({ kind: "retained-evidence", @@ -501,12 +515,18 @@ cliTest( view: { kind: "playwright-trace", collection: "events", fields: [] }, }); expect(page.isError, JSON.stringify(page.content)).not.toBe(true); - expect( + const partial = analysisViewResultSchema.parse( parseEvidence(page.structuredContent).normalized_result, - ).toMatchObject({ + ); + expect(partial).toMatchObject({ coverage: { total: 20, exhausted: true }, source_coverage: { interpretation: "partial" }, }); + if (partial.kind !== "playwright-trace") + throw new Error("Expected a trace page"); + expect( + partial.items.map((row) => ("kind" in row ? row.kind : undefined)), + ).not.toContain("network"); }, );