diff --git a/tests/boundary/browser/cdpHttpComparison.test.ts b/tests/boundary/browser/cdpHttpComparison.test.ts new file mode 100644 index 0000000000..2f4ad689cb --- /dev/null +++ b/tests/boundary/browser/cdpHttpComparison.test.ts @@ -0,0 +1,255 @@ +import { describe, expect, it } from "vitest"; + +import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js"; +import { inspectWebPageInputSchema } from "../../../src/domain/browserObservation.js"; +import type { WebPageInspection } from "../../../src/domain/browserObservationSchemas.js"; +import { compareWebCaptures } from "../../../src/domain/webCaptureDiff.js"; +import { compareWebCapturesInputSchema } from "../../../src/domain/webCaptureDiffSchemas.js"; +import type { FakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js"; +import { + startHttpComparisonBrowser, + type HttpRequestObservation, +} from "../../fixtures/httpComparisonBrowser.js"; +import { trackBrowser } from "./cdpBrowserProvider.support.js"; + +const inspect = async (browser: FakeCdpBrowser): Promise => { + const result = await new CdpBrowserProvider().inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + observation_ms: 0, + }), + ); + if (!result.ok) throw result.error; + return result.value; +}; + +const capturePair = async ( + before: readonly HttpRequestObservation[], + after: readonly HttpRequestObservation[], +): Promise => { + const fixture = await startHttpComparisonBrowser(before); + trackBrowser(fixture.browser); + const left = await inspect(fixture.browser); + fixture.setRequests(after); + return [left, await inspect(fixture.browser)]; +}; + +const compareNetwork = (before: WebPageInspection, after: WebPageInspection) => + compareWebCaptures( + compareWebCapturesInputSchema.parse({ + before: { inspection: before }, + after: { inspection: after }, + }), + ).dimensions.network; + +const response = { + requestId: "request-before", + status: 200, + mimeType: "application/json", +} as const; + +describe("HTTP comparison response coverage", () => { + it("does not compare an unavailable byte count with a completed response", async () => { + const [before, after] = await capturePair( + [response], + [{ ...response, requestId: "request-after", encodedDataLength: 100 }], + ); + expect(before.network.requests[0]?.encoded_data_length).toBeNull(); + expect(after.network.requests[0]?.encoded_data_length).toBe(100); + for (const [left, right] of [ + [before, after], + [after, before], + ] as const) + expect(compareNetwork(left, right)).toMatchObject({ + status: "unknown", + total_changes: 0, + changes: [], + }); + }); + + it("does not treat an unobserved response as a changed response", async () => { + const [before, after] = await capturePair( + [{ requestId: "request-before" }], + [{ ...response, encodedDataLength: 100 }], + ); + expect(before.network.requests).toEqual([ + expect.objectContaining({ + status: null, + mime_type: null, + encoded_data_length: null, + }), + ]); + for (const [left, right] of [ + [before, after], + [after, before], + ] as const) + expect(compareNetwork(left, right)).toMatchObject({ + status: "unknown", + total_changes: 0, + }); + }); + + it("retains observed status changes when response bytes are unavailable", async () => { + const [before, after] = await capturePair( + [response], + [{ ...response, status: 500 }], + ); + expect(compareNetwork(before, after)).toMatchObject({ + status: "changed", + changes: [expect.objectContaining({ change: "modified" })], + }); + }); + + it.each([ + { mimeType: "text/plain", encodedDataLength: 100 }, + { mimeType: "application/json", encodedDataLength: 200 }, + ])("retains observed response changes: %j", async (change) => { + const [before, after] = await capturePair( + [{ ...response, encodedDataLength: 100 }], + [{ ...response, ...change }], + ); + expect(compareNetwork(before, after).status).toBe("changed"); + }); +}); + +describe("HTTP comparison repeated request coverage", () => { + it("does not assign missing evidence by request order or transient ID", async () => { + const [before, after] = await capturePair( + [ + { ...response, encodedDataLength: 100 }, + { ...response, requestId: "second-before" }, + ], + [ + { ...response, requestId: "second-after", encodedDataLength: 200 }, + { ...response, requestId: "first-after", encodedDataLength: 100 }, + ], + ); + expect(before.network.requests).toHaveLength(2); + expect(after.network.requests).toHaveLength(2); + expect(compareNetwork(before, after)).toMatchObject({ + status: "unknown", + total_changes: 0, + }); + }); + + it("compares complete byte counts even when another field is incomplete", async () => { + const [before, after] = await capturePair( + [ + { ...response, encodedDataLength: 100 }, + { requestId: "second-before", encodedDataLength: 200 }, + ], + [ + { requestId: "second-after", encodedDataLength: 300 }, + { ...response, requestId: "first-after", encodedDataLength: 100 }, + ], + ); + expect(compareNetwork(before, after).status).toBe("changed"); + }); + + it("preserves the association between complete response fields", async () => { + const [before, after] = await capturePair( + [ + { ...response, encodedDataLength: 100 }, + { + ...response, + requestId: "second-before", + status: 201, + encodedDataLength: 200, + }, + ], + [ + { ...response, requestId: "first-after", encodedDataLength: 200 }, + { + ...response, + requestId: "second-after", + status: 201, + encodedDataLength: 100, + }, + ], + ); + expect(compareNetwork(before, after).status).toBe("changed"); + }); +}); + +it.each([ + { + name: "a pending request in an empty inventory", + before: [], + after: [{ requestId: "pending" }], + }, + { + name: "a known request beside an existing pending request", + before: [{ requestId: "pending" }], + after: [ + { requestId: "pending" }, + { + ...response, + requestId: "new", + path: "/api/new", + encodedDataLength: 100, + }, + ], + }, +])("preserves additions and removals of $name", async (observations) => { + const [before, after] = await capturePair( + observations.before, + observations.after, + ); + // Live CDP observations have attach-limited inventories. Their absence + // remains unknown even when a request is seen in the other capture. + for (const [left, right] of [ + [before, after], + [after, before], + ] as const) + expect(compareNetwork(left, right)).toMatchObject({ + status: "unknown", + total_changes: 0, + changes: [], + }); + + // Project explicitly complete inventory coverage without changing the + // producer's request facts; response fields remain independently unknown. + markNetworkInventoriesComplete(before, after); + for (const [left, right, change] of [ + [before, after, "added"], + [after, before, "removed"], + ] as const) + expect(compareNetwork(left, right)).toMatchObject({ + status: "changed", + total_changes: 1, + changes: [expect.objectContaining({ change })], + }); +}); + +it("requires response field coverage even when capture sections are complete", async () => { + const [before, after] = await capturePair( + [response], + [{ ...response, encodedDataLength: 100 }], + ); + markNetworkInventoriesComplete(before, after); + expect(compareNetwork(before, before).status).toBe("unknown"); + expect(compareNetwork(before, after).status).toBe("unknown"); + expect(compareNetwork(after, after).status).toBe("unchanged"); +}); + +const markNetworkInventoriesComplete = ( + ...inspections: WebPageInspection[] +): void => { + const sections = new Set([ + "network_requests", + "websocket_connections", + "websocket_frames", + ]); + for (const inspection of inspections) + for (const key of [ + "policy_filtered_sections", + "attach_limited_sections", + "truncated_sections", + "unavailable_sections", + ] as const) + inspection.completeness[key] = inspection.completeness[key].filter( + (section) => !sections.has(section), + ); +}; diff --git a/tests/boundary/browser/webCaptureDiff.test.ts b/tests/boundary/browser/webCaptureDiff.test.ts index 5c2ce87854..3680f7dfaa 100644 --- a/tests/boundary/browser/webCaptureDiff.test.ts +++ b/tests/boundary/browser/webCaptureDiff.test.ts @@ -446,91 +446,50 @@ describe("web capture WebSocket payload byte coverage", () => { }); describe("web capture diff", () => { - it("treats unavailable HTTP response facts as unknown while retaining observed changes", async () => { - const browser = await startFakeCdpBrowser(); - browsers.push(browser); - const captured = await new CdpBrowserProvider().inspectPage( - inspectWebPageInputSchema.parse({ - cdp_endpoint: browser.endpoint, - allowed_origins: [browser.allowedOrigin], - target_id: "allowed-page", - observation_ms: 0, - }), - ); - if (!captured.ok) throw captured.error; - const before = structuredClone(captured.value); - const request = before.network.requests[0]; - if (request === undefined) throw new Error("Missing network request"); - request.status = null; - request.mime_type = null; - request.encoded_data_length = null; - markSectionsComplete(before, [ - "network_requests", - "websocket_connections", - "websocket_frames", - "json_body_shapes", - ]); - for (const item of before.network.requests) - item.body_shapes.status = "not_approved"; + it.each(["status", "mime_type"] as const)( + "keeps an independently unavailable HTTP %s unknown", + async (field) => { + const browser = await startFakeCdpBrowser(); + browsers.push(browser); + const captured = await new CdpBrowserProvider().inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + observation_ms: 0, + }), + ); + if (!captured.ok) throw captured.error; + const observed = structuredClone(captured.value); + const request = observed.network.requests[0]; + if (request === undefined) throw new Error("Missing network request"); + request.status = 200; + request.mime_type = "application/json"; + request.encoded_data_length = 100; + markSectionsComplete(observed, [ + "network_requests", + "websocket_connections", + "websocket_frames", + "json_body_shapes", + ]); + for (const item of observed.network.requests) + item.body_shapes.status = "not_approved"; + expect(compareNetwork(observed, observed).status).toBe("unchanged"); - const after = structuredClone(before); - const responseObserved = after.network.requests[0]; - if (responseObserved === undefined) - throw new Error("Missing network request"); - responseObserved.status = 200; - responseObserved.mime_type = "application/json"; - responseObserved.encoded_data_length = 100; - expect(compareNetwork(before, after)).toMatchObject({ - status: "unknown", - total_changes: 0, - changes: [], - }); - expect(compareNetwork(after, before)).toMatchObject({ - status: "unknown", - total_changes: 0, - }); - expect(compareNetwork(before, before).status).toBe("unknown"); - - for (const field of [ - "status", - "mime_type", - "encoded_data_length", - ] as const) { - const partial = structuredClone(after); + const partial = structuredClone(observed); partial.network.requests[0]![field] = null; - expect(compareNetwork(partial, after).status).toBe("unknown"); - expect(compareNetwork(after, partial).status).toBe("unknown"); - } - - const inventoryBefore = structuredClone(before); - inventoryBefore.network.requests = []; - expect(compareNetwork(inventoryBefore, before)).toMatchObject({ - status: "changed", - total_changes: 1, - changes: [expect.objectContaining({ change: "added" })], - }); - - const statusChanged = structuredClone(after); - statusChanged.network.requests[0]!.status = 500; - expect(compareNetwork(after, statusChanged)).toMatchObject({ - status: "changed", - changes: [expect.objectContaining({ change: "modified" })], - }); - after.network.requests[0]!.encoded_data_length = null; - expect(compareNetwork(after, statusChanged).status).toBe("changed"); - - const repeated = structuredClone(statusChanged); - const duplicate = structuredClone(repeated.network.requests[0]!); - duplicate.status = 200; - duplicate.mime_type = "text/plain"; - repeated.network.requests.push(duplicate); - const reassociated = structuredClone(repeated); - reassociated.network.requests[0]!.mime_type = "text/plain"; - reassociated.network.requests.at(-1)!.mime_type = "application/json"; - expect(compareNetwork(repeated, reassociated).status).toBe("changed"); - repeated.network.requests[0]!.status = null; - expect(compareNetwork(repeated, reassociated).status).toBe("unknown"); - }); + for (const [left, right] of [ + [partial, observed], + [observed, partial], + [partial, partial], + ] as const) + expect(compareNetwork(left, right)).toMatchObject({ + status: "unknown", + total_changes: 0, + changes: [], + }); + }, + ); it("ignores transient request IDs and capture-approval state", async () => { const browser = await startFakeCdpBrowser(); diff --git a/tests/boundary/cli/httpCaptureComparison.test.ts b/tests/boundary/cli/httpCaptureComparison.test.ts new file mode 100644 index 0000000000..10d3733518 --- /dev/null +++ b/tests/boundary/cli/httpCaptureComparison.test.ts @@ -0,0 +1,77 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; + +import { webPageInspectionSchema } from "../../../src/domain/browserObservationSchemas.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { runCliJson } from "../../fixtures/cliJsonProcess.js"; +import { startHttpComparisonBrowser } from "../../fixtures/httpComparisonBrowser.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it("CLI compares incomplete HTTP responses without inventing changes", async () => { + const root = await createTestTempDirectory("rea-http-comparison-"); + const response = { + requestId: "request-1", + status: 200, + mimeType: "application/json", + }; + const fixture = await startHttpComparisonBrowser([response]); + try { + const inspect = async () => + webPageInspectionSchema.parse( + parseEvidence( + await runCliJson([ + "inspect-web-page", + fixture.browser.endpoint, + "allowed-page", + "--observation-ms", + "0", + "--json", + ]), + ).normalized_result, + ); + const pending = await inspect(); + fixture.setRequests([{ ...response, encodedDataLength: 100 }]); + const finished = await inspect(); + fixture.setRequests([{ ...response, status: 500, encodedDataLength: 100 }]); + const changed = await inspect(); + expect(pending.network.requests).toMatchObject([ + { status: 200, mime_type: "application/json", encoded_data_length: null }, + ]); + expect(finished.network.requests).toMatchObject([ + { status: 200, mime_type: "application/json", encoded_data_length: 100 }, + ]); + const paths = [pending, finished, changed].map((_, index) => + join(root, `capture-${String(index)}.json`), + ); + await Promise.all( + [pending, finished, changed].map((inspection, index) => + writeFile(paths[index]!, JSON.stringify({ inspection })), + ), + ); + for (const [left, right, status, count] of [ + [0, 1, "unknown", 0], + [1, 0, "unknown", 0], + [0, 2, "changed", 1], + ] as const) { + const compared = await runCliJson([ + "compare-web-captures", + paths[left]!, + paths[right]!, + "--json", + ]); + expect(compared).toMatchObject({ + operation: "compare_web_captures", + normalized_result: { + overall_status: status, + dimensions: { + network: { status, total_changes: count }, + }, + }, + }); + } + } finally { + await fixture.browser.close(); + } +}, 60_000); diff --git a/tests/fixtures/httpComparisonBrowser.ts b/tests/fixtures/httpComparisonBrowser.ts new file mode 100644 index 0000000000..94c7ed8566 --- /dev/null +++ b/tests/fixtures/httpComparisonBrowser.ts @@ -0,0 +1,83 @@ +import { startFakeCdpBrowser, type FakeCdpBrowser } from "./fakeCdpBrowser.js"; + +export type HttpRequestObservation = { + readonly requestId: string; + readonly path?: string; + readonly status?: number; + readonly mimeType?: string; + readonly encodedDataLength?: number; +}; + +type CdpEvent = Parameters[0]; + +const requestEvents = ( + observation: HttpRequestObservation, + origin: string, + sessionId: string | undefined, +): CdpEvent[] => { + const session = sessionId === undefined ? {} : { sessionId }; + const url = `${origin}${observation.path ?? "/api/items"}`; + const { requestId, status, mimeType, encodedDataLength } = observation; + const events: CdpEvent[] = [ + { + ...session, + method: "Network.requestWillBeSent", + params: { + requestId, + timestamp: 1, + type: "Fetch", + request: { url, method: "GET", headers: {} }, + initiator: { type: "other" }, + }, + }, + ]; + if (status !== undefined || mimeType !== undefined) + events.push({ + ...session, + method: "Network.responseReceived", + params: { + requestId, + timestamp: 2, + type: "Fetch", + response: { + url, + headers: {}, + ...(status === undefined ? {} : { status }), + ...(mimeType === undefined ? {} : { mimeType }), + }, + }, + }); + if (encodedDataLength !== undefined) + events.push({ + ...session, + method: "Network.loadingFinished", + params: { requestId, timestamp: 3, encodedDataLength }, + }); + return events; +}; + +/** Capture changing response coverage through the real CDP wire boundary. */ +export const startHttpComparisonBrowser = async ( + initial: readonly HttpRequestObservation[], +): Promise<{ + readonly browser: FakeCdpBrowser; + setRequests(next: readonly HttpRequestObservation[]): void; +}> => { + let requests = initial; + const browser = await startFakeCdpBrowser({ + eventsBeforeReply: ["Network.enable"], + commandEvents(command, origin) { + return command.method === "Network.enable" + ? requests.flatMap((request) => + requestEvents(request, origin, command.sessionId), + ) + : undefined; + }, + }); + return { + browser, + setRequests(next) { + requests = next; + }, + }; +};