mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
test(web): cover nullable HTTP captures through CDP and CLI (#1834)
* fix(browser): preserve unknown HTTP response comparison fields * fix(browser): retain proven request inventory changes * fix(javascript): preserve Electron identity and avoid quadratic property scans Share visible Electron mutations with wrapper expansion, index static object literal properties once, and align MCP assertions with current contracts. * fix(javascript): track writes through symbolic Electron aliases Resolve mutation targets through the same factories, module tables, containers, and conditional origins as call analysis. Share mutation coverage between direct and wrapper scans while isolating origin caches and preserving exact property paths, local rebinding semantics, and analysis resource limits. * fix(javascript): preserve container slots and imported mutation origins * perf(javascript): defer container mutation replay until property reads * fix(javascript): preserve captured object identities and write order * fix(javascript): preserve ordered mutation origins * fix(javascript): honor property mutation constraints * test(web): narrow PR to HTTP and CLI comparison regressions --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com>
This commit is contained in:
Yann Robertoandmorluto authored and GitHub committed
1 parent
9958754f6c
commit
5d12a0b4b6
4 files changed
+457
-83
No files matched your search
@@ -0,0 +1,255 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js";
|
||||
import { inspectWebPageInputSchema } from "../../../src/domain/browserObservation.js";
|
||||
import type { WebPageInspection } from "../../../src/domain/browserObservationSchemas.js";
|
||||
import { compareWebCaptures } from "../../../src/domain/webCaptureDiff.js";
|
||||
import { compareWebCapturesInputSchema } from "../../../src/domain/webCaptureDiffSchemas.js";
|
||||
import type { FakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js";
|
||||
import {
|
||||
startHttpComparisonBrowser,
|
||||
type HttpRequestObservation,
|
||||
} from "../../fixtures/httpComparisonBrowser.js";
|
||||
import { trackBrowser } from "./cdpBrowserProvider.support.js";
|
||||
|
||||
const inspect = async (browser: FakeCdpBrowser): Promise<WebPageInspection> => {
|
||||
const result = await new CdpBrowserProvider().inspectPage(
|
||||
inspectWebPageInputSchema.parse({
|
||||
cdp_endpoint: browser.endpoint,
|
||||
allowed_origins: [browser.allowedOrigin],
|
||||
target_id: "allowed-page",
|
||||
observation_ms: 0,
|
||||
}),
|
||||
);
|
||||
if (!result.ok) throw result.error;
|
||||
return result.value;
|
||||
};
|
||||
|
||||
const capturePair = async (
|
||||
before: readonly HttpRequestObservation[],
|
||||
after: readonly HttpRequestObservation[],
|
||||
): Promise<readonly [WebPageInspection, WebPageInspection]> => {
|
||||
const fixture = await startHttpComparisonBrowser(before);
|
||||
trackBrowser(fixture.browser);
|
||||
const left = await inspect(fixture.browser);
|
||||
fixture.setRequests(after);
|
||||
return [left, await inspect(fixture.browser)];
|
||||
};
|
||||
|
||||
const compareNetwork = (before: WebPageInspection, after: WebPageInspection) =>
|
||||
compareWebCaptures(
|
||||
compareWebCapturesInputSchema.parse({
|
||||
before: { inspection: before },
|
||||
after: { inspection: after },
|
||||
}),
|
||||
).dimensions.network;
|
||||
|
||||
const response = {
|
||||
requestId: "request-before",
|
||||
status: 200,
|
||||
mimeType: "application/json",
|
||||
} as const;
|
||||
|
||||
describe("HTTP comparison response coverage", () => {
|
||||
it("does not compare an unavailable byte count with a completed response", async () => {
|
||||
const [before, after] = await capturePair(
|
||||
[response],
|
||||
[{ ...response, requestId: "request-after", encodedDataLength: 100 }],
|
||||
);
|
||||
expect(before.network.requests[0]?.encoded_data_length).toBeNull();
|
||||
expect(after.network.requests[0]?.encoded_data_length).toBe(100);
|
||||
for (const [left, right] of [
|
||||
[before, after],
|
||||
[after, before],
|
||||
] as const)
|
||||
expect(compareNetwork(left, right)).toMatchObject({
|
||||
status: "unknown",
|
||||
total_changes: 0,
|
||||
changes: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("does not treat an unobserved response as a changed response", async () => {
|
||||
const [before, after] = await capturePair(
|
||||
[{ requestId: "request-before" }],
|
||||
[{ ...response, encodedDataLength: 100 }],
|
||||
);
|
||||
expect(before.network.requests).toEqual([
|
||||
expect.objectContaining({
|
||||
status: null,
|
||||
mime_type: null,
|
||||
encoded_data_length: null,
|
||||
}),
|
||||
]);
|
||||
for (const [left, right] of [
|
||||
[before, after],
|
||||
[after, before],
|
||||
] as const)
|
||||
expect(compareNetwork(left, right)).toMatchObject({
|
||||
status: "unknown",
|
||||
total_changes: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("retains observed status changes when response bytes are unavailable", async () => {
|
||||
const [before, after] = await capturePair(
|
||||
[response],
|
||||
[{ ...response, status: 500 }],
|
||||
);
|
||||
expect(compareNetwork(before, after)).toMatchObject({
|
||||
status: "changed",
|
||||
changes: [expect.objectContaining({ change: "modified" })],
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ mimeType: "text/plain", encodedDataLength: 100 },
|
||||
{ mimeType: "application/json", encodedDataLength: 200 },
|
||||
])("retains observed response changes: %j", async (change) => {
|
||||
const [before, after] = await capturePair(
|
||||
[{ ...response, encodedDataLength: 100 }],
|
||||
[{ ...response, ...change }],
|
||||
);
|
||||
expect(compareNetwork(before, after).status).toBe("changed");
|
||||
});
|
||||
});
|
||||
|
||||
describe("HTTP comparison repeated request coverage", () => {
|
||||
it("does not assign missing evidence by request order or transient ID", async () => {
|
||||
const [before, after] = await capturePair(
|
||||
[
|
||||
{ ...response, encodedDataLength: 100 },
|
||||
{ ...response, requestId: "second-before" },
|
||||
],
|
||||
[
|
||||
{ ...response, requestId: "second-after", encodedDataLength: 200 },
|
||||
{ ...response, requestId: "first-after", encodedDataLength: 100 },
|
||||
],
|
||||
);
|
||||
expect(before.network.requests).toHaveLength(2);
|
||||
expect(after.network.requests).toHaveLength(2);
|
||||
expect(compareNetwork(before, after)).toMatchObject({
|
||||
status: "unknown",
|
||||
total_changes: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("compares complete byte counts even when another field is incomplete", async () => {
|
||||
const [before, after] = await capturePair(
|
||||
[
|
||||
{ ...response, encodedDataLength: 100 },
|
||||
{ requestId: "second-before", encodedDataLength: 200 },
|
||||
],
|
||||
[
|
||||
{ requestId: "second-after", encodedDataLength: 300 },
|
||||
{ ...response, requestId: "first-after", encodedDataLength: 100 },
|
||||
],
|
||||
);
|
||||
expect(compareNetwork(before, after).status).toBe("changed");
|
||||
});
|
||||
|
||||
it("preserves the association between complete response fields", async () => {
|
||||
const [before, after] = await capturePair(
|
||||
[
|
||||
{ ...response, encodedDataLength: 100 },
|
||||
{
|
||||
...response,
|
||||
requestId: "second-before",
|
||||
status: 201,
|
||||
encodedDataLength: 200,
|
||||
},
|
||||
],
|
||||
[
|
||||
{ ...response, requestId: "first-after", encodedDataLength: 200 },
|
||||
{
|
||||
...response,
|
||||
requestId: "second-after",
|
||||
status: 201,
|
||||
encodedDataLength: 100,
|
||||
},
|
||||
],
|
||||
);
|
||||
expect(compareNetwork(before, after).status).toBe("changed");
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: "a pending request in an empty inventory",
|
||||
before: [],
|
||||
after: [{ requestId: "pending" }],
|
||||
},
|
||||
{
|
||||
name: "a known request beside an existing pending request",
|
||||
before: [{ requestId: "pending" }],
|
||||
after: [
|
||||
{ requestId: "pending" },
|
||||
{
|
||||
...response,
|
||||
requestId: "new",
|
||||
path: "/api/new",
|
||||
encodedDataLength: 100,
|
||||
},
|
||||
],
|
||||
},
|
||||
])("preserves additions and removals of $name", async (observations) => {
|
||||
const [before, after] = await capturePair(
|
||||
observations.before,
|
||||
observations.after,
|
||||
);
|
||||
// Live CDP observations have attach-limited inventories. Their absence
|
||||
// remains unknown even when a request is seen in the other capture.
|
||||
for (const [left, right] of [
|
||||
[before, after],
|
||||
[after, before],
|
||||
] as const)
|
||||
expect(compareNetwork(left, right)).toMatchObject({
|
||||
status: "unknown",
|
||||
total_changes: 0,
|
||||
changes: [],
|
||||
});
|
||||
|
||||
// Project explicitly complete inventory coverage without changing the
|
||||
// producer's request facts; response fields remain independently unknown.
|
||||
markNetworkInventoriesComplete(before, after);
|
||||
for (const [left, right, change] of [
|
||||
[before, after, "added"],
|
||||
[after, before, "removed"],
|
||||
] as const)
|
||||
expect(compareNetwork(left, right)).toMatchObject({
|
||||
status: "changed",
|
||||
total_changes: 1,
|
||||
changes: [expect.objectContaining({ change })],
|
||||
});
|
||||
});
|
||||
|
||||
it("requires response field coverage even when capture sections are complete", async () => {
|
||||
const [before, after] = await capturePair(
|
||||
[response],
|
||||
[{ ...response, encodedDataLength: 100 }],
|
||||
);
|
||||
markNetworkInventoriesComplete(before, after);
|
||||
expect(compareNetwork(before, before).status).toBe("unknown");
|
||||
expect(compareNetwork(before, after).status).toBe("unknown");
|
||||
expect(compareNetwork(after, after).status).toBe("unchanged");
|
||||
});
|
||||
|
||||
const markNetworkInventoriesComplete = (
|
||||
...inspections: WebPageInspection[]
|
||||
): void => {
|
||||
const sections = new Set([
|
||||
"network_requests",
|
||||
"websocket_connections",
|
||||
"websocket_frames",
|
||||
]);
|
||||
for (const inspection of inspections)
|
||||
for (const key of [
|
||||
"policy_filtered_sections",
|
||||
"attach_limited_sections",
|
||||
"truncated_sections",
|
||||
"unavailable_sections",
|
||||
] as const)
|
||||
inspection.completeness[key] = inspection.completeness[key].filter(
|
||||
(section) => !sections.has(section),
|
||||
);
|
||||
};
|
||||
@@ -446,91 +446,50 @@ describe("web capture WebSocket payload byte coverage", () => {
|
||||
});
|
||||
|
||||
describe("web capture diff", () => {
|
||||
it("treats unavailable HTTP response facts as unknown while retaining observed changes", async () => {
|
||||
const browser = await startFakeCdpBrowser();
|
||||
browsers.push(browser);
|
||||
const captured = await new CdpBrowserProvider().inspectPage(
|
||||
inspectWebPageInputSchema.parse({
|
||||
cdp_endpoint: browser.endpoint,
|
||||
allowed_origins: [browser.allowedOrigin],
|
||||
target_id: "allowed-page",
|
||||
observation_ms: 0,
|
||||
}),
|
||||
);
|
||||
if (!captured.ok) throw captured.error;
|
||||
const before = structuredClone(captured.value);
|
||||
const request = before.network.requests[0];
|
||||
if (request === undefined) throw new Error("Missing network request");
|
||||
request.status = null;
|
||||
request.mime_type = null;
|
||||
request.encoded_data_length = null;
|
||||
markSectionsComplete(before, [
|
||||
"network_requests",
|
||||
"websocket_connections",
|
||||
"websocket_frames",
|
||||
"json_body_shapes",
|
||||
]);
|
||||
for (const item of before.network.requests)
|
||||
item.body_shapes.status = "not_approved";
|
||||
it.each(["status", "mime_type"] as const)(
|
||||
"keeps an independently unavailable HTTP %s unknown",
|
||||
async (field) => {
|
||||
const browser = await startFakeCdpBrowser();
|
||||
browsers.push(browser);
|
||||
const captured = await new CdpBrowserProvider().inspectPage(
|
||||
inspectWebPageInputSchema.parse({
|
||||
cdp_endpoint: browser.endpoint,
|
||||
allowed_origins: [browser.allowedOrigin],
|
||||
target_id: "allowed-page",
|
||||
observation_ms: 0,
|
||||
}),
|
||||
);
|
||||
if (!captured.ok) throw captured.error;
|
||||
const observed = structuredClone(captured.value);
|
||||
const request = observed.network.requests[0];
|
||||
if (request === undefined) throw new Error("Missing network request");
|
||||
request.status = 200;
|
||||
request.mime_type = "application/json";
|
||||
request.encoded_data_length = 100;
|
||||
markSectionsComplete(observed, [
|
||||
"network_requests",
|
||||
"websocket_connections",
|
||||
"websocket_frames",
|
||||
"json_body_shapes",
|
||||
]);
|
||||
for (const item of observed.network.requests)
|
||||
item.body_shapes.status = "not_approved";
|
||||
expect(compareNetwork(observed, observed).status).toBe("unchanged");
|
||||
|
||||
const after = structuredClone(before);
|
||||
const responseObserved = after.network.requests[0];
|
||||
if (responseObserved === undefined)
|
||||
throw new Error("Missing network request");
|
||||
responseObserved.status = 200;
|
||||
responseObserved.mime_type = "application/json";
|
||||
responseObserved.encoded_data_length = 100;
|
||||
expect(compareNetwork(before, after)).toMatchObject({
|
||||
status: "unknown",
|
||||
total_changes: 0,
|
||||
changes: [],
|
||||
});
|
||||
expect(compareNetwork(after, before)).toMatchObject({
|
||||
status: "unknown",
|
||||
total_changes: 0,
|
||||
});
|
||||
expect(compareNetwork(before, before).status).toBe("unknown");
|
||||
|
||||
for (const field of [
|
||||
"status",
|
||||
"mime_type",
|
||||
"encoded_data_length",
|
||||
] as const) {
|
||||
const partial = structuredClone(after);
|
||||
const partial = structuredClone(observed);
|
||||
partial.network.requests[0]![field] = null;
|
||||
expect(compareNetwork(partial, after).status).toBe("unknown");
|
||||
expect(compareNetwork(after, partial).status).toBe("unknown");
|
||||
}
|
||||
|
||||
const inventoryBefore = structuredClone(before);
|
||||
inventoryBefore.network.requests = [];
|
||||
expect(compareNetwork(inventoryBefore, before)).toMatchObject({
|
||||
status: "changed",
|
||||
total_changes: 1,
|
||||
changes: [expect.objectContaining({ change: "added" })],
|
||||
});
|
||||
|
||||
const statusChanged = structuredClone(after);
|
||||
statusChanged.network.requests[0]!.status = 500;
|
||||
expect(compareNetwork(after, statusChanged)).toMatchObject({
|
||||
status: "changed",
|
||||
changes: [expect.objectContaining({ change: "modified" })],
|
||||
});
|
||||
after.network.requests[0]!.encoded_data_length = null;
|
||||
expect(compareNetwork(after, statusChanged).status).toBe("changed");
|
||||
|
||||
const repeated = structuredClone(statusChanged);
|
||||
const duplicate = structuredClone(repeated.network.requests[0]!);
|
||||
duplicate.status = 200;
|
||||
duplicate.mime_type = "text/plain";
|
||||
repeated.network.requests.push(duplicate);
|
||||
const reassociated = structuredClone(repeated);
|
||||
reassociated.network.requests[0]!.mime_type = "text/plain";
|
||||
reassociated.network.requests.at(-1)!.mime_type = "application/json";
|
||||
expect(compareNetwork(repeated, reassociated).status).toBe("changed");
|
||||
repeated.network.requests[0]!.status = null;
|
||||
expect(compareNetwork(repeated, reassociated).status).toBe("unknown");
|
||||
});
|
||||
for (const [left, right] of [
|
||||
[partial, observed],
|
||||
[observed, partial],
|
||||
[partial, partial],
|
||||
] as const)
|
||||
expect(compareNetwork(left, right)).toMatchObject({
|
||||
status: "unknown",
|
||||
total_changes: 0,
|
||||
changes: [],
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("ignores transient request IDs and capture-approval state", async () => {
|
||||
const browser = await startFakeCdpBrowser();
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { webPageInspectionSchema } from "../../../src/domain/browserObservationSchemas.js";
|
||||
import { parseEvidence } from "../../../src/domain/evidence.js";
|
||||
import { runCliJson } from "../../fixtures/cliJsonProcess.js";
|
||||
import { startHttpComparisonBrowser } from "../../fixtures/httpComparisonBrowser.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
|
||||
it("CLI compares incomplete HTTP responses without inventing changes", async () => {
|
||||
const root = await createTestTempDirectory("rea-http-comparison-");
|
||||
const response = {
|
||||
requestId: "request-1",
|
||||
status: 200,
|
||||
mimeType: "application/json",
|
||||
};
|
||||
const fixture = await startHttpComparisonBrowser([response]);
|
||||
try {
|
||||
const inspect = async () =>
|
||||
webPageInspectionSchema.parse(
|
||||
parseEvidence(
|
||||
await runCliJson([
|
||||
"inspect-web-page",
|
||||
fixture.browser.endpoint,
|
||||
"allowed-page",
|
||||
"--observation-ms",
|
||||
"0",
|
||||
"--json",
|
||||
]),
|
||||
).normalized_result,
|
||||
);
|
||||
const pending = await inspect();
|
||||
fixture.setRequests([{ ...response, encodedDataLength: 100 }]);
|
||||
const finished = await inspect();
|
||||
fixture.setRequests([{ ...response, status: 500, encodedDataLength: 100 }]);
|
||||
const changed = await inspect();
|
||||
expect(pending.network.requests).toMatchObject([
|
||||
{ status: 200, mime_type: "application/json", encoded_data_length: null },
|
||||
]);
|
||||
expect(finished.network.requests).toMatchObject([
|
||||
{ status: 200, mime_type: "application/json", encoded_data_length: 100 },
|
||||
]);
|
||||
const paths = [pending, finished, changed].map((_, index) =>
|
||||
join(root, `capture-${String(index)}.json`),
|
||||
);
|
||||
await Promise.all(
|
||||
[pending, finished, changed].map((inspection, index) =>
|
||||
writeFile(paths[index]!, JSON.stringify({ inspection })),
|
||||
),
|
||||
);
|
||||
for (const [left, right, status, count] of [
|
||||
[0, 1, "unknown", 0],
|
||||
[1, 0, "unknown", 0],
|
||||
[0, 2, "changed", 1],
|
||||
] as const) {
|
||||
const compared = await runCliJson([
|
||||
"compare-web-captures",
|
||||
paths[left]!,
|
||||
paths[right]!,
|
||||
"--json",
|
||||
]);
|
||||
expect(compared).toMatchObject({
|
||||
operation: "compare_web_captures",
|
||||
normalized_result: {
|
||||
overall_status: status,
|
||||
dimensions: {
|
||||
network: { status, total_changes: count },
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
await fixture.browser.close();
|
||||
}
|
||||
}, 60_000);
|
||||
+83
@@ -0,0 +1,83 @@
|
||||
import { startFakeCdpBrowser, type FakeCdpBrowser } from "./fakeCdpBrowser.js";
|
||||
|
||||
export type HttpRequestObservation = {
|
||||
readonly requestId: string;
|
||||
readonly path?: string;
|
||||
readonly status?: number;
|
||||
readonly mimeType?: string;
|
||||
readonly encodedDataLength?: number;
|
||||
};
|
||||
|
||||
type CdpEvent = Parameters<FakeCdpBrowser["emitEvent"]>[0];
|
||||
|
||||
const requestEvents = (
|
||||
observation: HttpRequestObservation,
|
||||
origin: string,
|
||||
sessionId: string | undefined,
|
||||
): CdpEvent[] => {
|
||||
const session = sessionId === undefined ? {} : { sessionId };
|
||||
const url = `${origin}${observation.path ?? "/api/items"}`;
|
||||
const { requestId, status, mimeType, encodedDataLength } = observation;
|
||||
const events: CdpEvent[] = [
|
||||
{
|
||||
...session,
|
||||
method: "Network.requestWillBeSent",
|
||||
params: {
|
||||
requestId,
|
||||
timestamp: 1,
|
||||
type: "Fetch",
|
||||
request: { url, method: "GET", headers: {} },
|
||||
initiator: { type: "other" },
|
||||
},
|
||||
},
|
||||
];
|
||||
if (status !== undefined || mimeType !== undefined)
|
||||
events.push({
|
||||
...session,
|
||||
method: "Network.responseReceived",
|
||||
params: {
|
||||
requestId,
|
||||
timestamp: 2,
|
||||
type: "Fetch",
|
||||
response: {
|
||||
url,
|
||||
headers: {},
|
||||
...(status === undefined ? {} : { status }),
|
||||
...(mimeType === undefined ? {} : { mimeType }),
|
||||
},
|
||||
},
|
||||
});
|
||||
if (encodedDataLength !== undefined)
|
||||
events.push({
|
||||
...session,
|
||||
method: "Network.loadingFinished",
|
||||
params: { requestId, timestamp: 3, encodedDataLength },
|
||||
});
|
||||
return events;
|
||||
};
|
||||
|
||||
/** Capture changing response coverage through the real CDP wire boundary. */
|
||||
export const startHttpComparisonBrowser = async (
|
||||
initial: readonly HttpRequestObservation[],
|
||||
): Promise<{
|
||||
readonly browser: FakeCdpBrowser;
|
||||
setRequests(next: readonly HttpRequestObservation[]): void;
|
||||
}> => {
|
||||
let requests = initial;
|
||||
const browser = await startFakeCdpBrowser({
|
||||
eventsBeforeReply: ["Network.enable"],
|
||||
commandEvents(command, origin) {
|
||||
return command.method === "Network.enable"
|
||||
? requests.flatMap((request) =>
|
||||
requestEvents(request, origin, command.sessionId),
|
||||
)
|
||||
: undefined;
|
||||
},
|
||||
});
|
||||
return {
|
||||
browser,
|
||||
setRequests(next) {
|
||||
requests = next;
|
||||
},
|
||||
};
|
||||
};
|
||||
Reference in new issue
Block a user