test(capture): consolidate Playwright trace contracts (#13207)

This commit is contained in:
morluto authored and GitHub committed 2026-11-21 20:55:20 +00:00
1 parent ff3919f3eb
commit 339f0ace57
2 files changed
+246 -326

No files matched your search

@@ -5,10 +5,7 @@ import { expect, it } from "vitest";
import { ArtifactResourceScope } from "../../../resources/ArtifactResourceScope.js";
import { PlaywrightTraceDecoder } from "./PlaywrightTraceDecoder.js";
import { playwrightTraceSchema } from "../../../domain/playwrightTrace/playwrightTrace.js";
import {
playwrightTraceFixture,
playwrightTracePathFixture,
} from "../../../../tests/fixtures/playwrightTrace.js";
import { playwrightTraceFixture } from "../../../../tests/fixtures/playwrightTrace.js";
import { createTestTempDirectory } from "../../../../tests/fixtures/temporaryDirectory.js";
import {
zstandardWindowFrameFixture,
@@ -71,86 +68,6 @@ it("imports real v10 recorder text with per-stream identity and resource presenc
}
});
it("preserves exact ZIP names for resources and paired stream profiles", async () => {
const fixture = await playwrightTracePathFixture();
const resources = new ArtifactResourceScope();
try {
const result = await new PlaywrightTraceDecoder(resources).inspect(
fixture.path,
);
if (!result.ok) throw result.error;
const report = result.value;
expect(report.members.map((member) => member.path)).toContain(
"./resources/x",
);
expect(
report.members.find((member) => member.path === "./resources/x"),
).toMatchObject({
logical_path: "resources/x",
limitations: [expect.stringContaining("leading current-directory")],
});
expect(
report.members.find((member) => member.path === "resources/x"),
).toMatchObject({ logical_path: "resources/x", limitations: [] });
expect(report.events).toHaveLength(4);
expect(report.streams.map((stream) => stream.state)).toEqual([
"supported",
"supported",
"supported",
"supported",
]);
const members = new Map(
report.members.map((member) => [member.member_id, member]),
);
for (const event of report.events) {
const source = members.get(event.source.member_id)!;
const expected = source.path.startsWith("./")
? "./resources/x"
: "resources/x";
expect(event.resource_references).toEqual([
expect.objectContaining({
path: expected,
state: "present-unread",
member_id: report.members.find((row) => row.path === expected)
?.member_id,
matching_members: 1,
}),
]);
}
expect(
report.events
.filter((event) => event.kind === "network")
.map((event) => event.status),
).toEqual([200, 201]);
} finally {
await resources.close();
}
});
it("keeps missing resource references explicit without turning page exhaustion into capture completeness", async () => {
const fixture = await playwrightTraceFixture();
const resources = new ArtifactResourceScope();
try {
const result = await new PlaywrightTraceDecoder(resources).inspect(
fixture.path,
);
if (!result.ok) throw result.error;
expect(result.value.summary.unavailable_references).toBe(8);
expect(result.value.source_coverage.interpretation).toBe("partial");
const references = result.value.events.flatMap(
(row) => row.resource_references,
);
expect(references).toHaveLength(8);
expect(
references.every(
(row) => row.state === "missing" && row.matching_members === 0,
),
).toBe(true);
} finally {
await resources.close();
}
});
const header =
JSON.stringify({ type: "context-options", version: 10, origin: "library" }) +
"\n";
@@ -159,16 +76,20 @@ it("keeps failed-request status sentinels unknown without losing recorded respon
const fixture = await playwrightTraceFixture({
trace: header,
network:
statuses
.map((status) =>
JSON.stringify({
type: "resource-snapshot",
snapshot: {
request: { method: "GET", url: "http://127.0.0.1/failed" },
response: { status, _failureText: "net::ERR_EMPTY_RESPONSE" },
},
}),
)
[
...statuses.map((status) => ({
type: "resource-snapshot",
snapshot: {
request: { method: "GET", url: "http://127.0.0.1/failed" },
response: { status, _failureText: "net::ERR_EMPTY_RESPONSE" },
},
})),
{
type: "resource-snapshot",
snapshot: { response: { status: 200 } },
},
]
.map((row) => JSON.stringify(row))
.join("\n") + "\n",
});
const resources = new ArtifactResourceScope();
@@ -188,7 +109,13 @@ it("keeps failed-request status sentinels unknown without losing recorded respon
200,
600,
999,
200,
]);
expect(report.events.at(-1)).toMatchObject({
status: 200,
method: null,
url: null,
});
expect(report.events[0]).toMatchObject({
source: { line: 1, record_pointer: "/snapshot" },
reported: {
@@ -201,7 +128,7 @@ it("keeps failed-request status sentinels unknown without losing recorded respon
report.diagnostics
.filter((row) => row.code === "partial-event")
.map((row) => row.line),
).toEqual([1, 2, 3, 4, 5]);
).toEqual([1, 2, 3, 4, 5, 10]);
} finally {
await resources.close();
}
@@ -247,31 +174,61 @@ it("retains valid siblings of malformed records and exposes exact producer lines
}
});
it.each([
["", 0],
[before, 0],
[header + before + header + before, 1],
])(
"never guesses a missing or repeated context version %#",
async (trace, events) => {
const fixture = await playwrightTraceFixture({ trace, network: "" });
const resources = new ArtifactResourceScope();
try {
const result = await new PlaywrightTraceDecoder(resources).inspect(
fixture.path,
);
if (!result.ok) throw result.error;
expect(result.value.events).toHaveLength(events);
expect(result.value.source_coverage.interpretation).toBe("partial");
expect(result.value.streams.length).toBeGreaterThan(0);
expect(
result.value.streams.every((stream) => stream.state === "unknown"),
).toBe(true);
} finally {
await resources.close();
}
},
);
it("requires an unambiguous v10 profile for each trace and paired network stream", async () => {
const network =
'{"type":"resource-snapshot","snapshot":{"request":{"method":"GET","url":"https://recorded.invalid/"},"response":{"status":200}}}\n';
const fixture = await playwrightTraceFixture({
trace: header + before + header + before,
network,
extraMembers: [
{ path: "empty.trace", contents: "" },
{ path: "empty.network", contents: network },
{ path: "missing-header.trace", contents: before },
{ path: "missing-header.network", contents: network },
{
path: "duplicate-key.trace",
contents:
'{"type":"context-options","version":10,"version":11}\n' + before,
},
{ path: "duplicate-key.network", contents: network },
],
});
const resources = new ArtifactResourceScope();
try {
const result = await new PlaywrightTraceDecoder(resources).inspect(
fixture.path,
);
if (!result.ok) throw result.error;
const { events, streams, members, diagnostics, source_coverage } =
result.value;
const sourcePath = (id: string | null) =>
members.find((row) => row.member_id === id)?.path;
expect(streams).toHaveLength(8);
expect(streams.map((stream) => stream.state)).toEqual(
Array(8).fill("unknown"),
);
expect(
events.map((event) => [
sourcePath(event.source.member_id),
event.source.line,
event.call_id,
]),
).toEqual([["trace.trace", 2, "call-a"]]);
expect(
diagnostics.map((row) => [sourcePath(row.member_id), row.line, row.code]),
).toEqual(
expect.arrayContaining([
["trace.trace", 3, "duplicate-context-header"],
["empty.trace", null, "unknown-stream-version"],
["missing-header.trace", 1, "missing-context-header"],
["duplicate-key.trace", 1, "malformed-record"],
]),
);
expect(source_coverage.interpretation).toBe("partial");
} finally {
await resources.close();
}
});
it("refuses other declared trace versions with verified archive identity", async () => {
const fixture = await playwrightTraceFixture({
@@ -335,28 +292,6 @@ it("keeps unsafe numeric literals exact instead of rounding recorded times", asy
}
});
it.each([
['{"type":"context-options","version":10,"version":11}\n', "unknown"],
[header + '{"type":"resource-snapshot","snapshot":{}}\n', "supported"],
])(
"reports duplicate keys and incomplete known events explicitly %#",
async (trace, state) => {
const fixture = await playwrightTraceFixture({ trace, network: "" });
const resources = new ArtifactResourceScope();
try {
const result = await new PlaywrightTraceDecoder(resources).inspect(
fixture.path,
);
if (!result.ok) throw result.error;
expect(result.value.streams[0]?.state).toBe(state);
expect(result.value.source_coverage.interpretation).toBe("partial");
expect(result.value.diagnostics.length).toBeGreaterThan(0);
} finally {
await resources.close();
}
},
);
it("refuses oversized physical lines before parsing their representation", async () => {
const trace =
header +
@@ -380,33 +315,6 @@ it("refuses oversized physical lines before parsing their representation", async
}
});
it("never promotes facts from a stream whose ZIP CRC fails", async () => {
const fixture = await playwrightTraceFixture({ level: 0 });
const bytes = await readFile(fixture.path);
const needle = Buffer.from('"method":"GET"');
const offset = bytes.indexOf(needle);
expect(offset).toBeGreaterThan(0);
bytes[offset + 10] = 80;
await writeFile(fixture.path, bytes);
const resources = new ArtifactResourceScope();
try {
const result = await new PlaywrightTraceDecoder(resources).inspect(
fixture.path,
);
if (result.ok) throw new Error("Expected CRC refusal");
expect(result.error._tag).toBe("AnalysisInputError");
const partial = playwrightTraceSchema.parse(
result.error.partialObservation,
);
expect(partial.summary.network).toBe(0);
expect(partial.summary.events).toBe(20);
expect(partial.source_coverage.interpretation).toBe("partial");
expect(resources.hasPending).toBe(false);
} finally {
await resources.close();
}
});
it("rejects a sparse over-budget archive before hashing or expansion", async () => {
const fixture = await playwrightTraceFixture();
const file = await open(fixture.path, "r+");
@@ -456,28 +364,7 @@ it("bounds JSON node expansion even when many tokens fit inside a small physical
}
});
it("names the nesting allowance, not the node allowance, for a deeply nested small record", async () => {
const trace =
header +
`{"type":"console","args":${"[".repeat(80)}${"]".repeat(80)},"time":1}\n`;
const fixture = await playwrightTraceFixture({ trace, network: "" });
const resources = new ArtifactResourceScope();
try {
const result = await new PlaywrightTraceDecoder(resources).inspect(
fixture.path,
);
if (result.ok) throw new Error("Expected nesting refusal");
expect(result.error._tag).toBe("AnalysisResourceConstraintError");
expect(projectAnalysisError(result.error).details).toMatchObject({
refusal: { limit: 64, unit: "nesting levels" },
});
expect(resources.hasPending).toBe(false);
} finally {
await resources.close();
}
});
it("keeps duplicate resource names ambiguous instead of selecting one ZIP occurrence", async () => {
it("distinguishes ambiguous and missing archive resource references", async () => {
const fixture = await playwrightTraceFixture({ resources: true, level: 0 });
const frames = fixture.trace
.split("\n")
@@ -527,17 +414,30 @@ it("keeps duplicate resource names ambiguous instead of selecting one ZIP occurr
expect(
new Set(result.value.members.map((member) => member.member_id)).size,
).toBe(result.value.members.length);
expect(
result.value.events.flatMap((row) => row.resource_references),
).toContainEqual(
expect.objectContaining({
path: first,
state: "ambiguous",
member_id: null,
matching_members: 2,
}),
const references = result.value.events.flatMap(
(row) => row.resource_references,
);
expect(result.value.source_coverage.interpretation).toBe("partial");
expect(references).toEqual(
expect.arrayContaining([
expect.objectContaining({
path: first,
state: "ambiguous",
member_id: null,
matching_members: 2,
}),
expect.objectContaining({
path: second,
state: "missing",
member_id: null,
matching_members: 0,
}),
]),
);
expect(result.value.summary.unavailable_references).toBe(2);
expect(result.value.source_coverage).toMatchObject({
interpretation: "partial",
recording_completeness: "unknown",
});
} finally {
await resources.close();
}
@@ -10,12 +10,13 @@ import {
import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js";
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
import { advertisedMcpToolSchemas } from "../../fixtures/advertisedMcpTool.js";
import { parseEvidence, type Evidence } from "../../../src/domain/evidence.js";
import { parseEvidence } from "../../../src/domain/evidence.js";
import { playwrightTraceSchema } from "../../../src/domain/playwrightTrace/playwrightTrace.js";
import { analysisViewResultSchema } from "../../../src/domain/analysisView/analysisViewResult.js";
import { evidenceBundleSchema } from "../../../src/domain/evidenceBundle.js";
type McpCall = Awaited<ReturnType<typeof connectLocalToolsMcp>>["call"];
import { lifecycleResultOf } from "../../../src/contracts/toolOutputSchemaPrimitives.js";
import { evidenceRecordSelectionSchema } from "../../../src/domain/evidenceRecordSummary.js";
cliTest(
"reports the fixed trace nesting limit through CLI and MCP",
@@ -62,20 +63,6 @@ cliTest(
REA_MCP_TOOL_FAMILIES: "application,session",
PLAYWRIGHT_BROWSERS_PATH: "/rea-no-browser-required",
});
const parent = parseEvidence(
(
await call("inspect_playwright_trace", {
path: fixture.path,
detail: "complete",
})
).structuredContent,
);
const report = playwrightTraceSchema.parse(parent.normalized_result);
expect(report.events).toHaveLength(4);
expect(report.summary.unavailable_references).toBe(0);
expect(report.streams.every((stream) => stream.state === "supported")).toBe(
true,
);
const view = {
kind: "playwright-trace",
collection: "members",
@@ -83,18 +70,17 @@ cliTest(
fields: [],
};
const selected = parseEvidence(
(
await call("inspect_analysis_view", {
source: {
kind: "retained-evidence",
evidence_id: parent.evidence_id,
},
view,
})
).structuredContent,
(await call("inspect_playwright_trace", { path: fixture.path, view }))
.structuredContent,
);
const result = await cli.run({
arguments: ["inspect-playwright-trace", fixture.path, "--json"],
});
expect(result.exitCode, result.stderr).toBe(0);
const parent = parseEvidence(result.json);
expect(selected.normalized_result).toMatchObject({
kind: "playwright-trace",
parent_evidence_id: parent.evidence_id,
items: [
{
path: "./resources/x",
@@ -105,50 +91,66 @@ cliTest(
coverage: { total: 1, exhausted: true },
source_coverage: { interpretation: "partial" },
});
const { members, streams, events, summary } = playwrightTraceSchema.parse(
parent.normalized_result,
);
expect(members).toContainEqual(
expect.objectContaining({
path: "resources/x",
logical_path: "resources/x",
limitations: [],
}),
);
expect(streams.map((row) => row.state)).toEqual(Array(4).fill("supported"));
expect(events).toHaveLength(4);
expect(summary.unavailable_references).toBe(0);
for (const event of events) {
const source = members.find(
(row) => row.member_id === event.source.member_id,
);
if (!source) throw new Error("Expected an exact source member");
const path = source.path.startsWith("./")
? "./resources/x"
: "resources/x";
expect(event.resource_references).toEqual([
expect.objectContaining({
path,
state: "present-unread",
member_id: members.find((row) => row.path === path)?.member_id,
matching_members: 1,
}),
]);
}
expect(
(await call("inspect_playwright_trace", { path: fixture.path, view }))
.structuredContent,
events.filter((row) => row.kind === "network").map((row) => row.status),
).toEqual([200, 201]);
expect(
(
await call("inspect_analysis_view", {
source: {
kind: "retained-evidence",
evidence_id: parent.evidence_id,
},
view,
})
).structuredContent,
).toEqual(selected);
const run = cli.run.bind(cli);
const result = await run({
arguments: ["inspect-playwright-trace", fixture.path, "--json"],
});
expect(result.exitCode, result.stderr).toBe(0);
expect(parseEvidence(result.json)).toEqual({
...parent,
caller_input_path: fixture.path,
});
expect((await readFile(fixture.path)).equals(original)).toBe(true);
},
);
const checkCliPublication = async ({
cli,
fixture,
parent,
selected,
view,
call,
}: {
cli: TestCli;
fixture: Awaited<ReturnType<typeof playwrightTraceFixture>>;
parent: Evidence;
selected: Evidence;
view: Record<string, unknown>;
call: McpCall;
}) => {
const source = { kind: "retained-evidence", evidence_id: parent.evidence_id };
const run = cli.run.bind(cli);
const fullCli = await run({
const publishCliTrace = async (
cli: TestCli,
fixture: Awaited<ReturnType<typeof playwrightTraceFixture>>,
view: Record<string, unknown>,
) => {
const full = await cli.run({
arguments: ["inspect-playwright-trace", fixture.path, "--json"],
});
expect(fullCli.exitCode, fullCli.stderr).toBe(0);
expect(parseEvidence(fullCli.json)).toEqual({
...parent,
caller_input_path: fixture.path,
});
expect(full.exitCode, full.stderr).toBe(0);
const parent = parseEvidence(full.json);
const bundlePath = join(fixture.root, "retained.bundle.json");
const receipt = await run({
const receipt = await cli.run({
arguments: [
"inspect-playwright-trace",
fixture.path,
@@ -162,36 +164,11 @@ const checkCliPublication = async ({
],
});
expect(receipt.exitCode, receipt.stderr).toBe(0);
expect(receipt.json).toMatchObject({
summary: {
...selected,
caller_input_path: fixture.path,
},
});
expect(
evidenceBundleSchema.parse(JSON.parse(await readFile(bundlePath, "utf8")))
.records,
).toEqual([{ ...parent, caller_input_path: fixture.path }]);
await rm(fixture.path);
const retained = await call("inspect_analysis_view", { source, view });
expect(parseEvidence(retained.structuredContent)).toEqual(selected);
const request = join(fixture.root, "page.json");
await writeFile(
request,
JSON.stringify({
source: {
kind: "evidence-file",
path: bundlePath,
evidence_id: parent.evidence_id,
},
view,
}),
);
const fromBundle = await run({
arguments: ["inspect-analysis-view", request, "--json"],
});
expect(fromBundle.exitCode, fromBundle.stderr).toBe(0);
expect(parseEvidence(fromBundle.json)).toEqual(selected);
).toEqual([parent]);
return { parent, receipt, bundlePath };
};
cliTest(
@@ -235,27 +212,46 @@ cliTest(
expect(JSON.stringify(first.structuredContent)).not.toContain(
"fixture loaded",
);
const complete = await call("inspect_playwright_trace", {
path: fixture.path,
detail: "complete",
});
expect(complete.isError).not.toBe(true);
const parent = parseEvidence(complete.structuredContent);
const report = playwrightTraceSchema.parse(parent.normalized_result);
expect(summary.parent_evidence_id).toBe(parent.evidence_id);
if (summary.kind !== "summary") throw new Error("Expected trace summary");
const source = {
kind: "retained-evidence",
evidence_id: parent.evidence_id,
evidence_id: summary.parent_evidence_id,
};
const view = {
kind: "playwright-trace",
collection: "events",
kinds: ["network"],
fields: [],
limit: 1,
};
const {
parent: cliParent,
receipt,
bundlePath,
} = await publishCliTrace(cli, fixture, view);
await rm(fixture.path);
const bundle = await call("get_evidence_bundle", {
detail: "records",
filters: { evidence_id: source.evidence_id },
});
const { records } = lifecycleResultOf(evidenceRecordSelectionSchema).parse(
bundle.structuredContent,
).result;
expect(records.map(({ evidence_id }) => evidence_id)).toEqual([
source.evidence_id,
]);
const parent = parseEvidence(records[0]);
expect(cliParent).toEqual({ ...parent, caller_input_path: fixture.path });
const report = playwrightTraceSchema.parse(parent.normalized_result);
const rows = [];
for (let offset = 0; offset < 23; offset += 5) {
for (const offset of [0, 12]) {
const page = await call("inspect_analysis_view", {
source,
view: {
kind: "playwright-trace",
collection: "events",
offset,
limit: 5,
limit: 12,
fields: Object.keys(report.events[0]!),
},
});
@@ -269,20 +265,14 @@ cliTest(
expect(view.source_coverage).toEqual(report.source_coverage);
}
expect(rows).toEqual(report.events);
const view = {
kind: "playwright-trace",
collection: "events",
kinds: ["network"],
fields: [],
limit: 1,
};
const selected = await call("inspect_playwright_trace", {
path: fixture.path,
view,
});
const selected = await call("inspect_analysis_view", { source, view });
expect(selected.isError).not.toBe(true);
const selectedEvidence = parseEvidence(selected.structuredContent);
expect(receipt.json).toMatchObject({
summary: { ...selectedEvidence, caller_input_path: fixture.path },
});
const selectedView = analysisViewResultSchema.parse(
parseEvidence(selected.structuredContent).normalized_result,
selectedEvidence.normalized_result,
);
expect(selectedView).toMatchObject({
kind: "playwright-trace",
@@ -313,14 +303,23 @@ cliTest(
parseEvidence(response.structuredContent).normalized_result,
).toMatchObject({ items: [], coverage: { total: 0, exhausted: true } });
}
await checkCliPublication({
cli,
fixture,
parent,
selected: parseEvidence(selected.structuredContent),
view,
call,
const request = join(fixture.root, "page.json");
await writeFile(
request,
JSON.stringify({
source: {
kind: "evidence-file",
path: bundlePath,
evidence_id: source.evidence_id,
},
view,
}),
);
const fromBundle = await cli.run({
arguments: ["inspect-analysis-view", request, "--json"],
});
expect(fromBundle.exitCode, fromBundle.stderr).toBe(0);
expect(parseEvidence(fromBundle.json)).toEqual(selectedEvidence);
},
);
@@ -457,8 +456,7 @@ cliTest(
expect(
parseEvidence(full.structuredContent).normalized_result,
).toMatchObject({ items: [{ text: message }] });
const run = cli.run.bind(cli);
const result = await run({
const result = await cli.run({
arguments: ["inspect-playwright-trace", fixture.path, "--json"],
});
expect(result.exitCode, result.stderr).toBe(0);
@@ -473,12 +471,27 @@ cliTest(
cliTest(
"retains partial MCP Evidence when a trace stream fails after earlier streams were read",
async () => {
const fixture = await playwrightTraceFixture({ level: 0 });
const event = JSON.stringify({
type: "resource-snapshot",
snapshot: {
request: { method: "GET", url: "https://recorded.invalid/value" },
response: { status: 200 },
},
});
const padding = JSON.stringify({
type: "frame-snapshot",
snapshot: "x".repeat(8192),
});
const tail = "trace-crc-tail";
// Send the network event before the stored member's late CRC failure.
const fixture = await playwrightTraceFixture({
level: 0,
network: `${event}\n${`${padding}\n`.repeat(80)}{"type":"${tail}"}\n`,
});
const bytes = await readFile(fixture.path);
// A stored network byte breaks only that member's ZIP CRC.
const offset = bytes.indexOf(Buffer.from('"method":"GET"'));
const offset = bytes.indexOf(tail);
expect(offset).toBeGreaterThan(0);
bytes[offset + 10] = 80;
bytes[offset] = bytes[offset]! ^ 1;
await writeFile(fixture.path, bytes);
const { call } = await connectLocalToolsMcp({
REA_MCP_TOOL_FAMILIES: "application,session",
@@ -490,6 +503,7 @@ cliTest(
detail: "complete",
}),
);
expect(failed.error.code).toBe("invalid_request");
const reference = failed.error.details?.partial_observation;
expect(reference).toEqual({
kind: "retained-evidence",
@@ -501,12 +515,18 @@ cliTest(
view: { kind: "playwright-trace", collection: "events", fields: [] },
});
expect(page.isError, JSON.stringify(page.content)).not.toBe(true);
expect(
const partial = analysisViewResultSchema.parse(
parseEvidence(page.structuredContent).normalized_result,
).toMatchObject({
);
expect(partial).toMatchObject({
coverage: { total: 20, exhausted: true },
source_coverage: { interpretation: "partial" },
});
if (partial.kind !== "playwright-trace")
throw new Error("Expected a trace page");
expect(
partial.items.map((row) => ("kind" in row ? row.kind : undefined)),
).not.toContain("network");
},
);