diff --git a/src/application/desktop/DesktopBuildCapture.ts b/src/application/desktop/DesktopBuildCapture.ts index 79c4ce5571..c8497a0426 100644 --- a/src/application/desktop/DesktopBuildCapture.ts +++ b/src/application/desktop/DesktopBuildCapture.ts @@ -27,7 +27,10 @@ import type { JsonValue } from "../../domain/jsonValue.js"; import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; import { err, ok, type Result } from "../../domain/result.js"; import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvProviderIdentity.js"; -import type { AgentEnvComputer } from "../../desktop/AgentEnvComputer.js"; +import { + readComputerScreenshotBytes, + type AgentEnvComputer, +} from "../../desktop/AgentEnvComputer.js"; import { controller, parseBuildComputerJson, @@ -82,12 +85,10 @@ const keepScreenshot = async ( CHECKPOINT_SHOT, signal, ); - const image = shot.content.find((block) => block.type === "image")?.data; - if (shot.isError || typeof image !== "string") return; + const image = readComputerScreenshotBytes(shot); + if (image === undefined) return; await computer.addHomeFiles( - new Map([ - [`${folder}/screens/${name}.jpg`, Buffer.from(image, "base64")], - ]), + new Map([[`${folder}/screens/${name}.jpg`, image]]), signal, ); } catch (cause: unknown) { diff --git a/src/application/desktop/DesktopBuildWitnessBundle.test.ts b/src/application/desktop/DesktopBuildWitnessBundle.test.ts index 3572cea573..f5e01de1fe 100644 --- a/src/application/desktop/DesktopBuildWitnessBundle.test.ts +++ b/src/application/desktop/DesktopBuildWitnessBundle.test.ts @@ -1,4 +1,8 @@ import { expect, it } from "vitest"; +import { + INVALID_BUILD_SCREENSHOTS, + VALID_BUILD_SCREENSHOT, +} from "../../../tests/fixtures/buildScreenshotData.js"; import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; import { judgeScreenTask } from "./DesktopBuildWitnessBundle.js"; @@ -14,13 +18,14 @@ it("preserves earlier judge outcomes when a later check fails cleanup", async () { callTool: async () => ({ isError: false, - content: [{ type: "image", data: "aW1hZ2U=" }], + content: [{ type: "image", data: VALID_BUILD_SCREENSHOT.data }], structured: null, }), }, { path: "/bundle", - judge: async (_verifier, check) => { + judge: async (_verifier, check, screen) => { + expect(VALID_BUILD_SCREENSHOT.bytes.equals(screen)).toBe(true); if (check.id === "second") throw failure; return { id: check.id, passed: true, description: "first completed" }; }, @@ -48,3 +53,37 @@ it("preserves earlier judge outcomes when a later check fails cleanup", async () }, }); }); + +it.each(INVALID_BUILD_SCREENSHOTS)( + "refuses an invalid screen judge image ($name) before invoking the judge", + async ({ data }) => { + let judged = false; + const result = await judgeScreenTask( + { + callTool: async () => ({ + isError: false, + content: [{ type: "image", data }], + structured: null, + }), + }, + { + path: "/bundle", + judge: async (_verifier, check) => { + judged = true; + return { id: check.id, passed: true, description: "accepted" }; + }, + }, + { + name: "test", + prompt: "test", + checks: [{ kind: "screen_judge", id: "screen", question: "Changed?" }], + }, + new AbortController().signal, + ); + expect(judged).toBe(false); + expect(result).toMatchObject({ + ok: false, + error: { userMessage: expect.stringContaining("valid screenshot") }, + }); + }, +); diff --git a/src/application/desktop/DesktopBuildWitnessBundle.ts b/src/application/desktop/DesktopBuildWitnessBundle.ts index a8a53d707e..3276273dbb 100644 --- a/src/application/desktop/DesktopBuildWitnessBundle.ts +++ b/src/application/desktop/DesktopBuildWitnessBundle.ts @@ -30,6 +30,7 @@ import { safeParseUtf8Json } from "../../domain/safeJson.js"; import { guestFileAnswerCeiling, guestFileAnswerWorkingBytes, + readComputerScreenshotBytes, type AgentEnvComputer, } from "../../desktop/AgentEnvComputer.js"; import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvProviderIdentity.js"; @@ -136,10 +137,9 @@ export const judgeScreenTask = async ( { format: "jpeg", quality: 70 }, signal, ); - const image = shot.content.find((block) => block.type === "image")?.data; - if (shot.isError || typeof image !== "string") - throw new Error("the computer took no screenshot"); - const screen = Buffer.from(image, "base64"); + const screen = readComputerScreenshotBytes(shot); + if (screen === undefined) + throw new Error("the computer did not return valid screenshot bytes"); const verifier = join( bundle.path, "artifacts", diff --git a/src/desktop/AgentEnvComputer.ts b/src/desktop/AgentEnvComputer.ts index c2c306957a..a7682bff4e 100644 --- a/src/desktop/AgentEnvComputer.ts +++ b/src/desktop/AgentEnvComputer.ts @@ -8,6 +8,7 @@ import { z } from "zod"; import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; import { Deadline } from "../domain/deadline.js"; import { jsonObjectSchema, type JsonValue } from "../domain/jsonValue.js"; +import { isCanonicalBase64 } from "../domain/stringPatterns.js"; import { StructuredContentMcpClient } from "../providers/StructuredContentMcpClient.js"; import { validateMcpHttpResponse } from "../providers/Utf8McpResponse.js"; import type { ComputerHelper } from "./AgentEnvComputerHelper.js"; @@ -214,6 +215,17 @@ export interface DriverResult { readonly structured: Readonly> | null; } +/** Decode the driver's first screenshot only when its encoded bytes are intact. */ +export const readComputerScreenshotBytes = ( + reply: DriverResult, +): Buffer | undefined => { + if (reply.isError) return undefined; + const data = reply.content.find((block) => block.type === "image")?.data; + if (typeof data !== "string" || data.length === 0 || !isCanonicalBase64(data)) + return undefined; + return Buffer.from(data, "base64"); +}; + /** A response body read only while it stays within `maxBytes`. */ const readBounded = async ( response: HttpReply, diff --git a/tests/boundary/mcp/buildCaptureMcp.test.ts b/tests/boundary/mcp/buildCaptureMcp.test.ts index 4cc853767b..d578047c1f 100644 --- a/tests/boundary/mcp/buildCaptureMcp.test.ts +++ b/tests/boundary/mcp/buildCaptureMcp.test.ts @@ -24,6 +24,10 @@ import { expectNoToken, startBuildComputer, } from "../../fixtures/buildComputerMcp.js"; +import { + INVALID_BUILD_SCREENSHOTS, + VALID_BUILD_SCREENSHOT, +} from "../../fixtures/buildScreenshotData.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -195,12 +199,28 @@ it("captures the app inside the computer, acknowledges checkpoints, drafts the t expectNoToken(computer, [started, seed, stopped, exported]); }); -it("keeps a screenshot of each checkpoint in the computer, which the export takes as a screen judge's examples", async () => { - const { rea, client } = await fakes(); +it("keeps valid checkpoint screenshots byte for byte for export and skips invalid screenshots without failing checkpoints", async () => { + const screenshots = [ + { name: "seed", data: VALID_BUILD_SCREENSHOT.data }, + ...INVALID_BUILD_SCREENSHOTS, + { name: "end", data: VALID_BUILD_SCREENSHOT.data }, + ]; + const { rea, client } = await fakes({ + jpegScreenshotData: screenshots.map(({ data }) => data), + }); const id = await startBuildComputer(client, REQUEST); await callOk(client, "start_build_capture", { computer_id: id }); - for (const name of ["seed", "end"]) - await callOk(client, "mark_build_checkpoint", { computer_id: id, name }); + for (const { name } of screenshots) { + const marked = await callOk(client, "mark_build_checkpoint", { + computer_id: id, + name, + }); + expect( + toolContract("mark_build_checkpoint").outputSchema.parse( + marked.structuredContent, + ).checkpoint.name, + ).toBe(name); + } await callOk(client, "stop_build_capture", { computer_id: id }); await callOk(client, "export_build_environment", { computer_id: id, @@ -215,6 +235,10 @@ it("keeps a screenshot of each checkpoint in the computer, which the export take "screens/end.jpg", "screens/seed.jpg", ]); + for (const path of shots) { + const bytes = await readFile(join(rea.home, ".rea-build", path)); + expect(bytes.equals(VALID_BUILD_SCREENSHOT.bytes), path).toBe(true); + } const exported = ( (await rea.notes("calls.jsonl")) as { argv: string[] }[] ).findLast(({ argv }) => argv[0] === "export-desktop-application"); diff --git a/tests/fixtures/buildCaptureFakes.ts b/tests/fixtures/buildCaptureFakes.ts index 80b1afb44d..e13cb684e2 100644 --- a/tests/fixtures/buildCaptureFakes.ts +++ b/tests/fixtures/buildCaptureFakes.ts @@ -342,7 +342,8 @@ const realCliOutputs = async ( * `env` adds to the stand-in's environment; `install` replaces the captured * package, the archive's `notes`, and names the packages the real export * installs beside it; `screenshotDelaysMs` delays the computer's - * screenshots in order. `recorded` replaces what the capture records, with + * screenshots in order and `jpegScreenshotData` supplies encoded JPEG + * screenshot replies. `recorded` replaces what the capture records, with * `exportRefused` when its export refuses BUILD_TASK, or `tasks` in its * place, and `extraContent` the * retained bytes it holds beyond the note's; `selected` is what it records @@ -356,6 +357,7 @@ export const buildCaptureFakes = async ( readonly env?: Readonly>; readonly install?: CapturedInstall; readonly screenshotDelaysMs?: readonly number[]; + readonly jpegScreenshotData?: readonly string[]; readonly recorded?: DesktopApplicationCapture; readonly exportRefused?: boolean; readonly extraContent?: readonly string[]; @@ -400,6 +402,9 @@ export const buildCaptureFakes = async ( ...(options.screenshotDelaysMs === undefined ? {} : { screenshotDelaysMs: options.screenshotDelaysMs }), + ...(options.jpegScreenshotData === undefined + ? {} + : { jpegScreenshotData: options.jpegScreenshotData }), exec: { home: rea.home, bin: rea.bin, diff --git a/tests/fixtures/buildScreenshotData.ts b/tests/fixtures/buildScreenshotData.ts new file mode 100644 index 0000000000..4a0435e889 --- /dev/null +++ b/tests/fixtures/buildScreenshotData.ts @@ -0,0 +1,10 @@ +export const VALID_BUILD_SCREENSHOT = { + data: "aW1hZ2U=", + bytes: Buffer.from("image"), +}; + +export const INVALID_BUILD_SCREENSHOTS = [ + { name: "invalid-alphabet", data: "aW1hZ2U=!" }, + { name: "noncanonical-padding-bits", data: "aW1hZ2V=" }, + { name: "empty-image", data: "" }, +] as const; diff --git a/tests/fixtures/fakeAgentEnvComputer.ts b/tests/fixtures/fakeAgentEnvComputer.ts index 19c9271315..437e30879a 100644 --- a/tests/fixtures/fakeAgentEnvComputer.ts +++ b/tests/fixtures/fakeAgentEnvComputer.ts @@ -65,6 +65,8 @@ export interface FakeComputerOptions { readonly blankFrames?: number; /** How long each screenshot, in order, waits before its reply, unless its request is cancelled. */ readonly screenshotDelaysMs?: readonly number[]; + /** Encoded image data in response to JPEG screenshot requests, in order. */ + readonly jpegScreenshotData?: readonly string[]; /** Bytes data/get appends to each segment beyond its manifest size. */ readonly segmentPadding?: number; /** Recording starts, in order, whose recording begins but whose reply is lost. */ @@ -153,6 +155,7 @@ const startFakeDriver = async ( : "A".repeat(options.frameCharacters); let blankFrames = options.blankFrames ?? 0; let screenshots = 0; + let jpegScreenshots = 0; const mcp = new McpServer({ name: "fake-reference-driver", version: "1" }); for (const name of COMPUTER_TOOLS) mcp.registerTool( @@ -177,12 +180,16 @@ const startFakeDriver = async ( const frame = { frame_id: `f${String(calls.length)}`, width: 1280 }; const blank = name === "computer_screenshot" && blankFrames > 0; if (blank) blankFrames -= 1; + const jpegData = + name === "computer_screenshot" && args.format === "jpeg" + ? options.jpegScreenshotData?.[jpegScreenshots++] + : undefined; const image = name === "computer_screenshot" || args.screenshot === true ? [ { type: "image" as const, - data: blank ? FAKE_BLANK_PNG : frameData, + data: blank ? FAKE_BLANK_PNG : (jpegData ?? frameData), mimeType: "image/png", }, ]