fix(browser): type screenshot, WebMCP and domain toggle CDP replies (#10647)

This commit is contained in:
morluto authored and GitHub committed 2026-11-14 21:11:43 -05:00
1 parent 3a29b6b8ea
commit 47925f3bef
10 files changed
+138 -76

No files matched your search

+5 -4
View File
@@ -1,17 +1,18 @@
import type { CdpSendParams, CdpSendResult } from "./protocol/CdpCommands.js";
import { CdpCommandRejection } from "./CdpCommandRejection.js";
import { CdpConnection } from "./CdpConnection.js";
/** Execute an optional CDP method while preserving transport and cancellation failures. */
export const optionalCdpCommand = async (
export const optionalCdpCommand = async <M extends string>(
context: {
readonly connection: CdpConnection;
readonly sessionId: string | undefined;
readonly signal?: AbortSignal;
},
method: string,
parameters: Readonly<Record<string, unknown>>,
method: M,
parameters: CdpSendParams<M>,
limitations: string[],
): Promise<unknown | undefined> => {
): Promise<CdpSendResult<M> | undefined> => {
try {
return await context.connection.send(
method,
+28 -54
View File
@@ -16,11 +16,6 @@ import {
import type { CdpFrameTree } from "./protocol/CdpFrameTree.js";
import type { CdpEndpointDiscovery, CdpEndpointTarget } from "./CdpEndpoint.js";
import type { CdpConnection, CdpEvent } from "./CdpConnection.js";
import {
recordValue,
cdpStringValue,
numberValue,
} from "./CdpCaptureValues.js";
import {
observeCdpFrameUrl,
classifyCdpFrameUrl,
@@ -228,39 +223,22 @@ export const captureCdpScreenshot = async (
/** Parse CSS clip geometry and device-pixel coverage at their owning boundary. */
const readScreenshotViewport = async (context: ScreenshotContext) => {
const metrics = recordValue(
await context.connection.send(
"Page.getLayoutMetrics",
{},
context.sessionId,
context.signal,
),
const { visualViewport, cssVisualViewport } = await context.connection.send(
"Page.getLayoutMetrics",
{},
context.sessionId,
context.signal,
);
const visualViewport = recordValue(metrics?.visualViewport);
const cssViewport = recordValue(metrics?.cssVisualViewport);
const pageX = numberValue(cssViewport?.pageX);
const pageY = numberValue(cssViewport?.pageY);
const width = numberValue(cssViewport?.clientWidth);
const height = numberValue(cssViewport?.clientHeight);
const scale = numberValue(cssViewport?.scale);
const pixelWidth = numberValue(visualViewport?.clientWidth);
const pixelHeight = numberValue(visualViewport?.clientHeight);
const pixelScale = numberValue(visualViewport?.scale);
if (
pageX === undefined ||
pageY === undefined ||
width === undefined ||
height === undefined ||
scale === undefined ||
pixelWidth === undefined ||
pixelHeight === undefined ||
pixelScale !== scale ||
width <= 0 ||
height <= 0 ||
scale <= 0 ||
pixelWidth <= 0 ||
pixelHeight <= 0
)
const {
pageX,
pageY,
clientWidth: width,
clientHeight: height,
scale,
} = cssVisualViewport;
const { clientWidth: pixelWidth, clientHeight: pixelHeight } = visualViewport;
// Both viewports describe one page zoom; disagreement is incoherent geometry.
if (visualViewport.scale !== scale)
throw new BrowserObservationError("inspect_web_page", "protocol_error");
// Keep ordinary surface captures, including their scrollbars and pixel
// density. The visual viewport excludes scrollbars and supplies only a
@@ -321,36 +299,32 @@ const captureScreenshotImage = async (
context: ScreenshotContext,
clip?: ScreenshotClip,
) => {
const result = recordValue(
await context.connection.send(
"Page.captureScreenshot",
{
format: "png",
fromSurface: true,
captureBeyondViewport: clip !== undefined,
...(clip === undefined ? {} : { clip }),
},
context.sessionId,
context.signal,
),
const { data } = await context.connection.send(
"Page.captureScreenshot",
{
format: "png",
fromSurface: true,
captureBeyondViewport: clip !== undefined,
...(clip === undefined ? {} : { clip }),
},
context.sessionId,
context.signal,
);
const screenshot = decodeScreenshot(result);
const screenshot = decodeScreenshot(data);
if (screenshot === undefined)
throw new BrowserObservationError("inspect_web_page", "protocol_error");
return screenshot;
};
const decodeScreenshot = (
result: ReturnType<typeof recordValue>,
encoded: string,
):
| {
readonly bytes: Buffer;
readonly dimensions: { width: number; height: number };
}
| undefined => {
const encoded = cdpStringValue(result?.data);
const bytes =
encoded === undefined ? undefined : decodeCanonicalBase64(encoded);
const bytes = decodeCanonicalBase64(encoded);
if (bytes === undefined) return undefined;
const dimensions = readPngHeaderDimensions(bytes);
return dimensions === undefined ? undefined : { bytes, dimensions };
+4 -3
View File
@@ -10,6 +10,7 @@ import { AnalysisCancelledError } from "../domain/analysisErrorCore.js";
import type { BrowserObservationOperation } from "../domain/browserObservationErrors.js";
import { CdpCommandRejection } from "./CdpCommandRejection.js";
import { CdpConnection } from "./CdpConnection.js";
import type { CdpDomainWith } from "./protocol/CdpCommands.js";
import {
CdpTargetSessionOwner,
bindCdpTransportCleanup,
@@ -257,7 +258,7 @@ const combineOpenFailure = (
/** Disable enabled domains, detach browser sessions, and release or close REA's socket. */
export const closeCdpTargetSession = async (
targetSession: CdpTargetSession,
enabledDomains: readonly string[],
enabledDomains: readonly CdpDomainWith<"disable">[],
signal?: AbortSignal,
owner?: CdpTargetSessionOwner,
reuseBrowserSocket = false,
@@ -280,7 +281,7 @@ export const closeCdpTargetSession = async (
const cleanupCdpTargetCommands = async (
targetSession: CdpTargetSession,
enabledDomains: readonly string[],
enabledDomains: readonly CdpDomainWith<"disable">[],
signal?: AbortSignal,
): Promise<boolean> => {
const { connection, sessionId } = targetSession;
@@ -297,7 +298,7 @@ const cleanupCdpTargetCommands = async (
if (domainSignal.aborted) break;
try {
await connection.send(
`${domain}.disable`,
`${domain}.disable` as const,
{},
sessionId,
domainSignal,
+1 -1
View File
@@ -19,7 +19,7 @@ export const captureWebEventListeners = async (
sources.ingest(event),
);
try {
for (const domain of ["Page", "Runtime", "Debugger", "DOM"])
for (const domain of ["Page", "Runtime", "Debugger", "DOM"] as const)
await session.enable(domain);
await session.assertDocument();
const document = await session.command("DOM.getDocument", { depth: 0 });
+7 -1
View File
@@ -20,7 +20,13 @@ export const captureWebExecution = async (
const collection = new CdpExecutionCollection(sources);
const { window, requests } = collection;
try {
for (const domain of ["Page", "Runtime", "Debugger", "Network", "Profiler"])
for (const domain of [
"Page",
"Runtime",
"Debugger",
"Network",
"Profiler",
] as const)
await session.enable(domain);
await session.assertDocument();
sources.check();
+12 -4
View File
@@ -13,7 +13,11 @@ import {
import { CdpTargetSessionOwner } from "../CdpTargetSessionOwner.js";
import { classifyCdpFrameUrl } from "../CdpCaptureDocuments.js";
import type { CdpFrameTree } from "../protocol/CdpFrameTree.js";
import type { CdpSendParams, CdpSendResult } from "../protocol/CdpCommands.js";
import type {
CdpDomainWith,
CdpSendParams,
CdpSendResult,
} from "../protocol/CdpCommands.js";
import {
WEB_RUNTIME_LIMITS,
type webRuntimeTargetSchema,
@@ -31,6 +35,10 @@ import {
type CdpTargetSession,
} from "../CdpTargetSession.js";
/** A domain this session can both enable and, on cleanup, disable. */
export type CdpRuntimeDomain = CdpDomainWith<"enable"> &
CdpDomainWith<"disable">;
type Scope = {
readonly cdp_endpoint: string;
readonly target_id: string;
@@ -40,7 +48,7 @@ type Scope = {
/** Owned transport and instrumentation; the selected browser/page remain externally owned. */
export class CdpRuntimeSession {
readonly enabledDomains: string[] = [];
readonly enabledDomains: CdpRuntimeDomain[] = [];
preciseCoverageMayBeActive = false;
#completion: AbortController | undefined;
#completionTimer: NodeJS.Timeout | undefined;
@@ -194,9 +202,9 @@ export class CdpRuntimeSession {
}
/** Register cleanup before sending: cancellation can occur after producer acceptance. */
async enable(domain: string): Promise<void> {
async enable(domain: CdpRuntimeDomain): Promise<void> {
this.enabledDomains.push(domain);
await this.command(`${domain}.enable`);
await this.command(`${domain}.enable` as const);
}
/** Check document identity before joining any live node or final coverage evidence. */
@@ -12,7 +12,9 @@ const root = fileURLToPath(new URL("../../../", import.meta.url));
* Literal names of CDP commands that have no result schema yet, per source
* file. A command registered in CdpCommands.ts no longer counts. This list may
* only shrink: migrate a file's commands, then lower or delete its entry.
* Dynamic names such as `${domain}.enable` are not counted.
* Names built at run time such as `${domain}.enable` are not matched here;
* their domains are typed `CdpDomainWith<verb>`, so an unregistered name fails
* to compile instead.
*/
const UNTYPED_CDP_COMMAND_BASELINE = JSON.parse(
readFileSync(
+41
View File
@@ -23,6 +23,23 @@ const acknowledgement = z.object({});
const coverageOffset = z.int().min(0);
const nodeId = z.int().min(1);
const positive = z.number().positive();
/** CSS clip geometry and device-pixel viewport for one screenshot. */
const layoutMetricsReply = z.object({
visualViewport: z.object({
clientWidth: positive,
clientHeight: positive,
scale: positive,
}),
cssVisualViewport: z.object({
pageX: z.number(),
pageY: z.number(),
clientWidth: positive,
clientHeight: positive,
scale: positive,
}),
});
/** Native listener descriptors; live remote handles are not read. */
const eventListenersReply = z.object({
@@ -77,6 +94,8 @@ const CDP_COMMAND_REPLIES = {
isSVG: z.boolean().optional(),
}),
}),
"DOM.disable": acknowledgement,
"DOM.enable": acknowledgement,
"DOM.getDocument": z.object({ root: z.object({ nodeId }) }),
// Zero is the protocol's "no match".
"DOM.querySelector": z.object({ nodeId: z.int().min(0) }),
@@ -84,6 +103,7 @@ const CDP_COMMAND_REPLIES = {
object: z.object({ objectId: z.string().min(1) }),
}),
"DOMDebugger.getEventListeners": eventListenersReply,
"Debugger.disable": acknowledgement,
"Debugger.enable": acknowledgement,
"Debugger.evaluateOnCallFrame": z.object({
result: z.looseObject({}),
@@ -98,12 +118,17 @@ const CDP_COMMAND_REPLIES = {
"Emulation.setFocusEmulationEnabled": acknowledgement,
"Emulation.setLocaleOverride": acknowledgement,
"Emulation.setTimezoneOverride": acknowledgement,
"Network.disable": acknowledgement,
"Network.enable": acknowledgement,
"Page.addScriptToEvaluateOnNewDocument": z.object({
identifier: z.string(),
}),
"Page.disable": acknowledgement,
"Page.enable": acknowledgement,
// `data` is base64 PNG; its canonical decoding stays with the screenshot owner.
"Page.captureScreenshot": z.object({ data: z.string() }),
"Page.getFrameTree": cdpFrameTreeReplySchema,
"Page.getLayoutMetrics": layoutMetricsReply,
"Page.getResourceTree": cdpResourceTreeReplySchema,
"Page.removeScriptToEvaluateOnNewDocument": acknowledgement,
"Page.setLifecycleEventsEnabled": acknowledgement,
@@ -112,9 +137,12 @@ const CDP_COMMAND_REPLIES = {
"Performance.getMetrics": z.object({
metrics: z.array(z.object({ name: z.string(), value: z.number() })),
}),
"Profiler.disable": acknowledgement,
"Profiler.enable": acknowledgement,
"Profiler.startPreciseCoverage": z.object({ timestamp: z.number().min(0) }),
"Profiler.stopPreciseCoverage": acknowledgement,
"Profiler.takePreciseCoverage": preciseCoverageReply,
"Runtime.disable": acknowledgement,
"Runtime.enable": acknowledgement,
"Runtime.releaseObjectGroup": acknowledgement,
"Runtime.runIfWaitingForDebugger": acknowledgement,
@@ -125,10 +153,23 @@ const CDP_COMMAND_REPLIES = {
targetInfo: z.object({ targetId: z.string() }),
}),
"Target.setAutoAttach": acknowledgement,
"WebMCP.disable": acknowledgement,
"WebMCP.enable": acknowledgement,
} as const satisfies { readonly [M in CdpProtocolMethod]?: z.ZodType };
export type CdpCommand = keyof typeof CDP_COMMAND_REPLIES;
type DomainOf<C, Verb extends string> = C extends `${infer Domain}.${Verb}`
? Domain
: never;
/**
* A domain whose `${domain}.${verb}` command is registered. Code that builds a
* command name at run time takes its domain as this type, so an unregistered
* name fails to compile.
*/
export type CdpDomainWith<Verb extends string> = DomainOf<CdpCommand, Verb>;
export type CdpCommandParams<M extends CdpCommand> = NonNullable<
Parameters<typeof protocolSend<M>>[1]
>;
@@ -30,13 +30,6 @@
"src/browser/CdpPageCaptureWorkers.ts": {
"Target.getTargets": 1
},
"src/browser/CdpScreenshot.ts": {
"Page.getLayoutMetrics": 1,
"Page.captureScreenshot": 1
},
"src/browser/CdpWebMcpDiscovery.ts": {
"WebMCP.enable": 1
},
"src/browser/PlaywrightScenarioServiceWorkers.ts": {
"Runtime.evaluate": 1
},
@@ -334,7 +334,11 @@ describe("screenshot viewport freshness", () => {
});
expect(result).toMatchObject({
ok: false,
error: { reason: "protocol_error" },
error: {
reason: "protocol_error",
command: "Page.getLayoutMetrics",
pointer: "/visualViewport",
},
});
expect(
browser.commands.filter(
@@ -342,6 +346,38 @@ describe("screenshot viewport freshness", () => {
),
).toHaveLength(stage - 1);
});
it("reports a screenshot reply without image data at the protocol boundary", async () => {
const { browser, result } = await captureWithFake({
commandResult: ({ method }) =>
method === "Page.getLayoutMetrics"
? withCssViewport({
visualViewport: {
pageX: 0,
pageY: 0,
clientWidth: 2,
clientHeight: 2,
scale: 1,
},
})
: method === "Page.captureScreenshot"
? { data: 5 }
: undefined,
});
expect(result).toMatchObject({
ok: false,
error: {
reason: "protocol_error",
command: "Page.captureScreenshot",
pointer: "/data",
},
});
expect(
browser.commands.some(
({ method }) => method === "Target.detachFromTarget",
),
).toBe(true);
});
});
describe("screenshot visual viewport capture", () => {