mirror of
https://github.com/morluto/rea.git
synced 2026-11-15 11:13:21 +00:00
fix(browser): type screenshot, WebMCP and domain toggle CDP replies (#10647)
This commit is contained in:
1 parent
3a29b6b8ea
commit
47925f3bef
10 files changed
+138
-76
No files matched your search
@@ -1,17 +1,18 @@
|
||||
import type { CdpSendParams, CdpSendResult } from "./protocol/CdpCommands.js";
|
||||
import { CdpCommandRejection } from "./CdpCommandRejection.js";
|
||||
import { CdpConnection } from "./CdpConnection.js";
|
||||
|
||||
/** Execute an optional CDP method while preserving transport and cancellation failures. */
|
||||
export const optionalCdpCommand = async (
|
||||
export const optionalCdpCommand = async <M extends string>(
|
||||
context: {
|
||||
readonly connection: CdpConnection;
|
||||
readonly sessionId: string | undefined;
|
||||
readonly signal?: AbortSignal;
|
||||
},
|
||||
method: string,
|
||||
parameters: Readonly<Record<string, unknown>>,
|
||||
method: M,
|
||||
parameters: CdpSendParams<M>,
|
||||
limitations: string[],
|
||||
): Promise<unknown | undefined> => {
|
||||
): Promise<CdpSendResult<M> | undefined> => {
|
||||
try {
|
||||
return await context.connection.send(
|
||||
method,
|
||||
|
||||
@@ -16,11 +16,6 @@ import {
|
||||
import type { CdpFrameTree } from "./protocol/CdpFrameTree.js";
|
||||
import type { CdpEndpointDiscovery, CdpEndpointTarget } from "./CdpEndpoint.js";
|
||||
import type { CdpConnection, CdpEvent } from "./CdpConnection.js";
|
||||
import {
|
||||
recordValue,
|
||||
cdpStringValue,
|
||||
numberValue,
|
||||
} from "./CdpCaptureValues.js";
|
||||
import {
|
||||
observeCdpFrameUrl,
|
||||
classifyCdpFrameUrl,
|
||||
@@ -228,39 +223,22 @@ export const captureCdpScreenshot = async (
|
||||
|
||||
/** Parse CSS clip geometry and device-pixel coverage at their owning boundary. */
|
||||
const readScreenshotViewport = async (context: ScreenshotContext) => {
|
||||
const metrics = recordValue(
|
||||
await context.connection.send(
|
||||
"Page.getLayoutMetrics",
|
||||
{},
|
||||
context.sessionId,
|
||||
context.signal,
|
||||
),
|
||||
const { visualViewport, cssVisualViewport } = await context.connection.send(
|
||||
"Page.getLayoutMetrics",
|
||||
{},
|
||||
context.sessionId,
|
||||
context.signal,
|
||||
);
|
||||
const visualViewport = recordValue(metrics?.visualViewport);
|
||||
const cssViewport = recordValue(metrics?.cssVisualViewport);
|
||||
const pageX = numberValue(cssViewport?.pageX);
|
||||
const pageY = numberValue(cssViewport?.pageY);
|
||||
const width = numberValue(cssViewport?.clientWidth);
|
||||
const height = numberValue(cssViewport?.clientHeight);
|
||||
const scale = numberValue(cssViewport?.scale);
|
||||
const pixelWidth = numberValue(visualViewport?.clientWidth);
|
||||
const pixelHeight = numberValue(visualViewport?.clientHeight);
|
||||
const pixelScale = numberValue(visualViewport?.scale);
|
||||
if (
|
||||
pageX === undefined ||
|
||||
pageY === undefined ||
|
||||
width === undefined ||
|
||||
height === undefined ||
|
||||
scale === undefined ||
|
||||
pixelWidth === undefined ||
|
||||
pixelHeight === undefined ||
|
||||
pixelScale !== scale ||
|
||||
width <= 0 ||
|
||||
height <= 0 ||
|
||||
scale <= 0 ||
|
||||
pixelWidth <= 0 ||
|
||||
pixelHeight <= 0
|
||||
)
|
||||
const {
|
||||
pageX,
|
||||
pageY,
|
||||
clientWidth: width,
|
||||
clientHeight: height,
|
||||
scale,
|
||||
} = cssVisualViewport;
|
||||
const { clientWidth: pixelWidth, clientHeight: pixelHeight } = visualViewport;
|
||||
// Both viewports describe one page zoom; disagreement is incoherent geometry.
|
||||
if (visualViewport.scale !== scale)
|
||||
throw new BrowserObservationError("inspect_web_page", "protocol_error");
|
||||
// Keep ordinary surface captures, including their scrollbars and pixel
|
||||
// density. The visual viewport excludes scrollbars and supplies only a
|
||||
@@ -321,36 +299,32 @@ const captureScreenshotImage = async (
|
||||
context: ScreenshotContext,
|
||||
clip?: ScreenshotClip,
|
||||
) => {
|
||||
const result = recordValue(
|
||||
await context.connection.send(
|
||||
"Page.captureScreenshot",
|
||||
{
|
||||
format: "png",
|
||||
fromSurface: true,
|
||||
captureBeyondViewport: clip !== undefined,
|
||||
...(clip === undefined ? {} : { clip }),
|
||||
},
|
||||
context.sessionId,
|
||||
context.signal,
|
||||
),
|
||||
const { data } = await context.connection.send(
|
||||
"Page.captureScreenshot",
|
||||
{
|
||||
format: "png",
|
||||
fromSurface: true,
|
||||
captureBeyondViewport: clip !== undefined,
|
||||
...(clip === undefined ? {} : { clip }),
|
||||
},
|
||||
context.sessionId,
|
||||
context.signal,
|
||||
);
|
||||
const screenshot = decodeScreenshot(result);
|
||||
const screenshot = decodeScreenshot(data);
|
||||
if (screenshot === undefined)
|
||||
throw new BrowserObservationError("inspect_web_page", "protocol_error");
|
||||
return screenshot;
|
||||
};
|
||||
|
||||
const decodeScreenshot = (
|
||||
result: ReturnType<typeof recordValue>,
|
||||
encoded: string,
|
||||
):
|
||||
| {
|
||||
readonly bytes: Buffer;
|
||||
readonly dimensions: { width: number; height: number };
|
||||
}
|
||||
| undefined => {
|
||||
const encoded = cdpStringValue(result?.data);
|
||||
const bytes =
|
||||
encoded === undefined ? undefined : decodeCanonicalBase64(encoded);
|
||||
const bytes = decodeCanonicalBase64(encoded);
|
||||
if (bytes === undefined) return undefined;
|
||||
const dimensions = readPngHeaderDimensions(bytes);
|
||||
return dimensions === undefined ? undefined : { bytes, dimensions };
|
||||
|
||||
@@ -10,6 +10,7 @@ import { AnalysisCancelledError } from "../domain/analysisErrorCore.js";
|
||||
import type { BrowserObservationOperation } from "../domain/browserObservationErrors.js";
|
||||
import { CdpCommandRejection } from "./CdpCommandRejection.js";
|
||||
import { CdpConnection } from "./CdpConnection.js";
|
||||
import type { CdpDomainWith } from "./protocol/CdpCommands.js";
|
||||
import {
|
||||
CdpTargetSessionOwner,
|
||||
bindCdpTransportCleanup,
|
||||
@@ -257,7 +258,7 @@ const combineOpenFailure = (
|
||||
/** Disable enabled domains, detach browser sessions, and release or close REA's socket. */
|
||||
export const closeCdpTargetSession = async (
|
||||
targetSession: CdpTargetSession,
|
||||
enabledDomains: readonly string[],
|
||||
enabledDomains: readonly CdpDomainWith<"disable">[],
|
||||
signal?: AbortSignal,
|
||||
owner?: CdpTargetSessionOwner,
|
||||
reuseBrowserSocket = false,
|
||||
@@ -280,7 +281,7 @@ export const closeCdpTargetSession = async (
|
||||
|
||||
const cleanupCdpTargetCommands = async (
|
||||
targetSession: CdpTargetSession,
|
||||
enabledDomains: readonly string[],
|
||||
enabledDomains: readonly CdpDomainWith<"disable">[],
|
||||
signal?: AbortSignal,
|
||||
): Promise<boolean> => {
|
||||
const { connection, sessionId } = targetSession;
|
||||
@@ -297,7 +298,7 @@ const cleanupCdpTargetCommands = async (
|
||||
if (domainSignal.aborted) break;
|
||||
try {
|
||||
await connection.send(
|
||||
`${domain}.disable`,
|
||||
`${domain}.disable` as const,
|
||||
{},
|
||||
sessionId,
|
||||
domainSignal,
|
||||
|
||||
@@ -19,7 +19,7 @@ export const captureWebEventListeners = async (
|
||||
sources.ingest(event),
|
||||
);
|
||||
try {
|
||||
for (const domain of ["Page", "Runtime", "Debugger", "DOM"])
|
||||
for (const domain of ["Page", "Runtime", "Debugger", "DOM"] as const)
|
||||
await session.enable(domain);
|
||||
await session.assertDocument();
|
||||
const document = await session.command("DOM.getDocument", { depth: 0 });
|
||||
|
||||
@@ -20,7 +20,13 @@ export const captureWebExecution = async (
|
||||
const collection = new CdpExecutionCollection(sources);
|
||||
const { window, requests } = collection;
|
||||
try {
|
||||
for (const domain of ["Page", "Runtime", "Debugger", "Network", "Profiler"])
|
||||
for (const domain of [
|
||||
"Page",
|
||||
"Runtime",
|
||||
"Debugger",
|
||||
"Network",
|
||||
"Profiler",
|
||||
] as const)
|
||||
await session.enable(domain);
|
||||
await session.assertDocument();
|
||||
sources.check();
|
||||
|
||||
@@ -13,7 +13,11 @@ import {
|
||||
import { CdpTargetSessionOwner } from "../CdpTargetSessionOwner.js";
|
||||
import { classifyCdpFrameUrl } from "../CdpCaptureDocuments.js";
|
||||
import type { CdpFrameTree } from "../protocol/CdpFrameTree.js";
|
||||
import type { CdpSendParams, CdpSendResult } from "../protocol/CdpCommands.js";
|
||||
import type {
|
||||
CdpDomainWith,
|
||||
CdpSendParams,
|
||||
CdpSendResult,
|
||||
} from "../protocol/CdpCommands.js";
|
||||
import {
|
||||
WEB_RUNTIME_LIMITS,
|
||||
type webRuntimeTargetSchema,
|
||||
@@ -31,6 +35,10 @@ import {
|
||||
type CdpTargetSession,
|
||||
} from "../CdpTargetSession.js";
|
||||
|
||||
/** A domain this session can both enable and, on cleanup, disable. */
|
||||
export type CdpRuntimeDomain = CdpDomainWith<"enable"> &
|
||||
CdpDomainWith<"disable">;
|
||||
|
||||
type Scope = {
|
||||
readonly cdp_endpoint: string;
|
||||
readonly target_id: string;
|
||||
@@ -40,7 +48,7 @@ type Scope = {
|
||||
|
||||
/** Owned transport and instrumentation; the selected browser/page remain externally owned. */
|
||||
export class CdpRuntimeSession {
|
||||
readonly enabledDomains: string[] = [];
|
||||
readonly enabledDomains: CdpRuntimeDomain[] = [];
|
||||
preciseCoverageMayBeActive = false;
|
||||
#completion: AbortController | undefined;
|
||||
#completionTimer: NodeJS.Timeout | undefined;
|
||||
@@ -194,9 +202,9 @@ export class CdpRuntimeSession {
|
||||
}
|
||||
|
||||
/** Register cleanup before sending: cancellation can occur after producer acceptance. */
|
||||
async enable(domain: string): Promise<void> {
|
||||
async enable(domain: CdpRuntimeDomain): Promise<void> {
|
||||
this.enabledDomains.push(domain);
|
||||
await this.command(`${domain}.enable`);
|
||||
await this.command(`${domain}.enable` as const);
|
||||
}
|
||||
|
||||
/** Check document identity before joining any live node or final coverage evidence. */
|
||||
|
||||
@@ -12,7 +12,9 @@ const root = fileURLToPath(new URL("../../../", import.meta.url));
|
||||
* Literal names of CDP commands that have no result schema yet, per source
|
||||
* file. A command registered in CdpCommands.ts no longer counts. This list may
|
||||
* only shrink: migrate a file's commands, then lower or delete its entry.
|
||||
* Dynamic names such as `${domain}.enable` are not counted.
|
||||
* Names built at run time such as `${domain}.enable` are not matched here;
|
||||
* their domains are typed `CdpDomainWith<verb>`, so an unregistered name fails
|
||||
* to compile instead.
|
||||
*/
|
||||
const UNTYPED_CDP_COMMAND_BASELINE = JSON.parse(
|
||||
readFileSync(
|
||||
|
||||
@@ -23,6 +23,23 @@ const acknowledgement = z.object({});
|
||||
|
||||
const coverageOffset = z.int().min(0);
|
||||
const nodeId = z.int().min(1);
|
||||
const positive = z.number().positive();
|
||||
|
||||
/** CSS clip geometry and device-pixel viewport for one screenshot. */
|
||||
const layoutMetricsReply = z.object({
|
||||
visualViewport: z.object({
|
||||
clientWidth: positive,
|
||||
clientHeight: positive,
|
||||
scale: positive,
|
||||
}),
|
||||
cssVisualViewport: z.object({
|
||||
pageX: z.number(),
|
||||
pageY: z.number(),
|
||||
clientWidth: positive,
|
||||
clientHeight: positive,
|
||||
scale: positive,
|
||||
}),
|
||||
});
|
||||
|
||||
/** Native listener descriptors; live remote handles are not read. */
|
||||
const eventListenersReply = z.object({
|
||||
@@ -77,6 +94,8 @@ const CDP_COMMAND_REPLIES = {
|
||||
isSVG: z.boolean().optional(),
|
||||
}),
|
||||
}),
|
||||
"DOM.disable": acknowledgement,
|
||||
"DOM.enable": acknowledgement,
|
||||
"DOM.getDocument": z.object({ root: z.object({ nodeId }) }),
|
||||
// Zero is the protocol's "no match".
|
||||
"DOM.querySelector": z.object({ nodeId: z.int().min(0) }),
|
||||
@@ -84,6 +103,7 @@ const CDP_COMMAND_REPLIES = {
|
||||
object: z.object({ objectId: z.string().min(1) }),
|
||||
}),
|
||||
"DOMDebugger.getEventListeners": eventListenersReply,
|
||||
"Debugger.disable": acknowledgement,
|
||||
"Debugger.enable": acknowledgement,
|
||||
"Debugger.evaluateOnCallFrame": z.object({
|
||||
result: z.looseObject({}),
|
||||
@@ -98,12 +118,17 @@ const CDP_COMMAND_REPLIES = {
|
||||
"Emulation.setFocusEmulationEnabled": acknowledgement,
|
||||
"Emulation.setLocaleOverride": acknowledgement,
|
||||
"Emulation.setTimezoneOverride": acknowledgement,
|
||||
"Network.disable": acknowledgement,
|
||||
"Network.enable": acknowledgement,
|
||||
"Page.addScriptToEvaluateOnNewDocument": z.object({
|
||||
identifier: z.string(),
|
||||
}),
|
||||
"Page.disable": acknowledgement,
|
||||
"Page.enable": acknowledgement,
|
||||
// `data` is base64 PNG; its canonical decoding stays with the screenshot owner.
|
||||
"Page.captureScreenshot": z.object({ data: z.string() }),
|
||||
"Page.getFrameTree": cdpFrameTreeReplySchema,
|
||||
"Page.getLayoutMetrics": layoutMetricsReply,
|
||||
"Page.getResourceTree": cdpResourceTreeReplySchema,
|
||||
"Page.removeScriptToEvaluateOnNewDocument": acknowledgement,
|
||||
"Page.setLifecycleEventsEnabled": acknowledgement,
|
||||
@@ -112,9 +137,12 @@ const CDP_COMMAND_REPLIES = {
|
||||
"Performance.getMetrics": z.object({
|
||||
metrics: z.array(z.object({ name: z.string(), value: z.number() })),
|
||||
}),
|
||||
"Profiler.disable": acknowledgement,
|
||||
"Profiler.enable": acknowledgement,
|
||||
"Profiler.startPreciseCoverage": z.object({ timestamp: z.number().min(0) }),
|
||||
"Profiler.stopPreciseCoverage": acknowledgement,
|
||||
"Profiler.takePreciseCoverage": preciseCoverageReply,
|
||||
"Runtime.disable": acknowledgement,
|
||||
"Runtime.enable": acknowledgement,
|
||||
"Runtime.releaseObjectGroup": acknowledgement,
|
||||
"Runtime.runIfWaitingForDebugger": acknowledgement,
|
||||
@@ -125,10 +153,23 @@ const CDP_COMMAND_REPLIES = {
|
||||
targetInfo: z.object({ targetId: z.string() }),
|
||||
}),
|
||||
"Target.setAutoAttach": acknowledgement,
|
||||
"WebMCP.disable": acknowledgement,
|
||||
"WebMCP.enable": acknowledgement,
|
||||
} as const satisfies { readonly [M in CdpProtocolMethod]?: z.ZodType };
|
||||
|
||||
export type CdpCommand = keyof typeof CDP_COMMAND_REPLIES;
|
||||
|
||||
type DomainOf<C, Verb extends string> = C extends `${infer Domain}.${Verb}`
|
||||
? Domain
|
||||
: never;
|
||||
|
||||
/**
|
||||
* A domain whose `${domain}.${verb}` command is registered. Code that builds a
|
||||
* command name at run time takes its domain as this type, so an unregistered
|
||||
* name fails to compile.
|
||||
*/
|
||||
export type CdpDomainWith<Verb extends string> = DomainOf<CdpCommand, Verb>;
|
||||
|
||||
export type CdpCommandParams<M extends CdpCommand> = NonNullable<
|
||||
Parameters<typeof protocolSend<M>>[1]
|
||||
>;
|
||||
|
||||
@@ -30,13 +30,6 @@
|
||||
"src/browser/CdpPageCaptureWorkers.ts": {
|
||||
"Target.getTargets": 1
|
||||
},
|
||||
"src/browser/CdpScreenshot.ts": {
|
||||
"Page.getLayoutMetrics": 1,
|
||||
"Page.captureScreenshot": 1
|
||||
},
|
||||
"src/browser/CdpWebMcpDiscovery.ts": {
|
||||
"WebMCP.enable": 1
|
||||
},
|
||||
"src/browser/PlaywrightScenarioServiceWorkers.ts": {
|
||||
"Runtime.evaluate": 1
|
||||
},
|
||||
|
||||
@@ -334,7 +334,11 @@ describe("screenshot viewport freshness", () => {
|
||||
});
|
||||
expect(result).toMatchObject({
|
||||
ok: false,
|
||||
error: { reason: "protocol_error" },
|
||||
error: {
|
||||
reason: "protocol_error",
|
||||
command: "Page.getLayoutMetrics",
|
||||
pointer: "/visualViewport",
|
||||
},
|
||||
});
|
||||
expect(
|
||||
browser.commands.filter(
|
||||
@@ -342,6 +346,38 @@ describe("screenshot viewport freshness", () => {
|
||||
),
|
||||
).toHaveLength(stage - 1);
|
||||
});
|
||||
|
||||
it("reports a screenshot reply without image data at the protocol boundary", async () => {
|
||||
const { browser, result } = await captureWithFake({
|
||||
commandResult: ({ method }) =>
|
||||
method === "Page.getLayoutMetrics"
|
||||
? withCssViewport({
|
||||
visualViewport: {
|
||||
pageX: 0,
|
||||
pageY: 0,
|
||||
clientWidth: 2,
|
||||
clientHeight: 2,
|
||||
scale: 1,
|
||||
},
|
||||
})
|
||||
: method === "Page.captureScreenshot"
|
||||
? { data: 5 }
|
||||
: undefined,
|
||||
});
|
||||
expect(result).toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
reason: "protocol_error",
|
||||
command: "Page.captureScreenshot",
|
||||
pointer: "/data",
|
||||
},
|
||||
});
|
||||
expect(
|
||||
browser.commands.some(
|
||||
({ method }) => method === "Target.detachFromTarget",
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("screenshot visual viewport capture", () => {
|
||||
|
||||
Reference in new issue
Block a user