fix(desktop): reject invalid build screenshot encodings (#12179)

* fix(desktop): reject invalid build screenshot encodings

* fix(desktop): finish screenshot admission review

---------

Co-authored-by: morluto <76467478+morluto@users.noreply.github.com>
This commit is contained in:
ShuaixinHuangandmorluto authored and GitHub committed 2026-11-19 05:08:14 +00:00
1 parent b946cb83dd
commit 3fae997886
8 files changed
+116 -18

No files matched your search

@@ -27,7 +27,10 @@ import type { JsonValue } from "../../domain/jsonValue.js";
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
import { err, ok, type Result } from "../../domain/result.js";
import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvProviderIdentity.js";
import type { AgentEnvComputer } from "../../desktop/AgentEnvComputer.js";
import {
readComputerScreenshotBytes,
type AgentEnvComputer,
} from "../../desktop/AgentEnvComputer.js";
import {
controller,
parseBuildComputerJson,
@@ -82,12 +85,10 @@ const keepScreenshot = async (
CHECKPOINT_SHOT,
signal,
);
const image = shot.content.find((block) => block.type === "image")?.data;
if (shot.isError || typeof image !== "string") return;
const image = readComputerScreenshotBytes(shot);
if (image === undefined) return;
await computer.addHomeFiles(
new Map([
[`${folder}/screens/${name}.jpg`, Buffer.from(image, "base64")],
]),
new Map([[`${folder}/screens/${name}.jpg`, image]]),
signal,
);
} catch (cause: unknown) {
@@ -1,4 +1,8 @@
import { expect, it } from "vitest";
import {
INVALID_BUILD_SCREENSHOTS,
VALID_BUILD_SCREENSHOT,
} from "../../../tests/fixtures/buildScreenshotData.js";
import { ProviderCleanupError } from "../../domain/providerCleanupError.js";
import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
import { judgeScreenTask } from "./DesktopBuildWitnessBundle.js";
@@ -14,13 +18,14 @@ it("preserves earlier judge outcomes when a later check fails cleanup", async ()
{
callTool: async () => ({
isError: false,
content: [{ type: "image", data: "aW1hZ2U=" }],
content: [{ type: "image", data: VALID_BUILD_SCREENSHOT.data }],
structured: null,
}),
},
{
path: "/bundle",
judge: async (_verifier, check) => {
judge: async (_verifier, check, screen) => {
expect(VALID_BUILD_SCREENSHOT.bytes.equals(screen)).toBe(true);
if (check.id === "second") throw failure;
return { id: check.id, passed: true, description: "first completed" };
},
@@ -48,3 +53,37 @@ it("preserves earlier judge outcomes when a later check fails cleanup", async ()
},
});
});
it.each(INVALID_BUILD_SCREENSHOTS)(
"refuses an invalid screen judge image ($name) before invoking the judge",
async ({ data }) => {
let judged = false;
const result = await judgeScreenTask(
{
callTool: async () => ({
isError: false,
content: [{ type: "image", data }],
structured: null,
}),
},
{
path: "/bundle",
judge: async (_verifier, check) => {
judged = true;
return { id: check.id, passed: true, description: "accepted" };
},
},
{
name: "test",
prompt: "test",
checks: [{ kind: "screen_judge", id: "screen", question: "Changed?" }],
},
new AbortController().signal,
);
expect(judged).toBe(false);
expect(result).toMatchObject({
ok: false,
error: { userMessage: expect.stringContaining("valid screenshot") },
});
},
);
@@ -30,6 +30,7 @@ import { safeParseUtf8Json } from "../../domain/safeJson.js";
import {
guestFileAnswerCeiling,
guestFileAnswerWorkingBytes,
readComputerScreenshotBytes,
type AgentEnvComputer,
} from "../../desktop/AgentEnvComputer.js";
import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvProviderIdentity.js";
@@ -136,10 +137,9 @@ export const judgeScreenTask = async (
{ format: "jpeg", quality: 70 },
signal,
);
const image = shot.content.find((block) => block.type === "image")?.data;
if (shot.isError || typeof image !== "string")
throw new Error("the computer took no screenshot");
const screen = Buffer.from(image, "base64");
const screen = readComputerScreenshotBytes(shot);
if (screen === undefined)
throw new Error("the computer did not return valid screenshot bytes");
const verifier = join(
bundle.path,
"artifacts",
+12
View File
@@ -8,6 +8,7 @@ import { z } from "zod";
import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js";
import { Deadline } from "../domain/deadline.js";
import { jsonObjectSchema, type JsonValue } from "../domain/jsonValue.js";
import { isCanonicalBase64 } from "../domain/stringPatterns.js";
import { StructuredContentMcpClient } from "../providers/StructuredContentMcpClient.js";
import { validateMcpHttpResponse } from "../providers/Utf8McpResponse.js";
import type { ComputerHelper } from "./AgentEnvComputerHelper.js";
@@ -214,6 +215,17 @@ export interface DriverResult {
readonly structured: Readonly<Record<string, JsonValue>> | null;
}
/** Decode the driver's first screenshot only when its encoded bytes are intact. */
export const readComputerScreenshotBytes = (
reply: DriverResult,
): Buffer | undefined => {
if (reply.isError) return undefined;
const data = reply.content.find((block) => block.type === "image")?.data;
if (typeof data !== "string" || data.length === 0 || !isCanonicalBase64(data))
return undefined;
return Buffer.from(data, "base64");
};
/** A response body read only while it stays within `maxBytes`. */
const readBounded = async (
response: HttpReply,
+28 -4
View File
@@ -24,6 +24,10 @@ import {
expectNoToken,
startBuildComputer,
} from "../../fixtures/buildComputerMcp.js";
import {
INVALID_BUILD_SCREENSHOTS,
VALID_BUILD_SCREENSHOT,
} from "../../fixtures/buildScreenshotData.js";
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
@@ -195,12 +199,28 @@ it("captures the app inside the computer, acknowledges checkpoints, drafts the t
expectNoToken(computer, [started, seed, stopped, exported]);
});
it("keeps a screenshot of each checkpoint in the computer, which the export takes as a screen judge's examples", async () => {
const { rea, client } = await fakes();
it("keeps valid checkpoint screenshots byte for byte for export and skips invalid screenshots without failing checkpoints", async () => {
const screenshots = [
{ name: "seed", data: VALID_BUILD_SCREENSHOT.data },
...INVALID_BUILD_SCREENSHOTS,
{ name: "end", data: VALID_BUILD_SCREENSHOT.data },
];
const { rea, client } = await fakes({
jpegScreenshotData: screenshots.map(({ data }) => data),
});
const id = await startBuildComputer(client, REQUEST);
await callOk(client, "start_build_capture", { computer_id: id });
for (const name of ["seed", "end"])
await callOk(client, "mark_build_checkpoint", { computer_id: id, name });
for (const { name } of screenshots) {
const marked = await callOk(client, "mark_build_checkpoint", {
computer_id: id,
name,
});
expect(
toolContract("mark_build_checkpoint").outputSchema.parse(
marked.structuredContent,
).checkpoint.name,
).toBe(name);
}
await callOk(client, "stop_build_capture", { computer_id: id });
await callOk(client, "export_build_environment", {
computer_id: id,
@@ -215,6 +235,10 @@ it("keeps a screenshot of each checkpoint in the computer, which the export take
"screens/end.jpg",
"screens/seed.jpg",
]);
for (const path of shots) {
const bytes = await readFile(join(rea.home, ".rea-build", path));
expect(bytes.equals(VALID_BUILD_SCREENSHOT.bytes), path).toBe(true);
}
const exported = (
(await rea.notes("calls.jsonl")) as { argv: string[] }[]
).findLast(({ argv }) => argv[0] === "export-desktop-application");
+6 -1
View File
@@ -342,7 +342,8 @@ const realCliOutputs = async (
* `env` adds to the stand-in's environment; `install` replaces the captured
* package, the archive's `notes`, and names the packages the real export
* installs beside it; `screenshotDelaysMs` delays the computer's
* screenshots in order. `recorded` replaces what the capture records, with
* screenshots in order and `jpegScreenshotData` supplies encoded JPEG
* screenshot replies. `recorded` replaces what the capture records, with
* `exportRefused` when its export refuses BUILD_TASK, or `tasks` in its
* place, and `extraContent` the
* retained bytes it holds beyond the note's; `selected` is what it records
@@ -356,6 +357,7 @@ export const buildCaptureFakes = async (
readonly env?: Readonly<Record<string, string>>;
readonly install?: CapturedInstall;
readonly screenshotDelaysMs?: readonly number[];
readonly jpegScreenshotData?: readonly string[];
readonly recorded?: DesktopApplicationCapture;
readonly exportRefused?: boolean;
readonly extraContent?: readonly string[];
@@ -400,6 +402,9 @@ export const buildCaptureFakes = async (
...(options.screenshotDelaysMs === undefined
? {}
: { screenshotDelaysMs: options.screenshotDelaysMs }),
...(options.jpegScreenshotData === undefined
? {}
: { jpegScreenshotData: options.jpegScreenshotData }),
exec: {
home: rea.home,
bin: rea.bin,
+10
View File
@@ -0,0 +1,10 @@
export const VALID_BUILD_SCREENSHOT = {
data: "aW1hZ2U=",
bytes: Buffer.from("image"),
};
export const INVALID_BUILD_SCREENSHOTS = [
{ name: "invalid-alphabet", data: "aW1hZ2U=!" },
{ name: "noncanonical-padding-bits", data: "aW1hZ2V=" },
{ name: "empty-image", data: "" },
] as const;
+8 -1
View File
@@ -65,6 +65,8 @@ export interface FakeComputerOptions {
readonly blankFrames?: number;
/** How long each screenshot, in order, waits before its reply, unless its request is cancelled. */
readonly screenshotDelaysMs?: readonly number[];
/** Encoded image data in response to JPEG screenshot requests, in order. */
readonly jpegScreenshotData?: readonly string[];
/** Bytes data/get appends to each segment beyond its manifest size. */
readonly segmentPadding?: number;
/** Recording starts, in order, whose recording begins but whose reply is lost. */
@@ -153,6 +155,7 @@ const startFakeDriver = async (
: "A".repeat(options.frameCharacters);
let blankFrames = options.blankFrames ?? 0;
let screenshots = 0;
let jpegScreenshots = 0;
const mcp = new McpServer({ name: "fake-reference-driver", version: "1" });
for (const name of COMPUTER_TOOLS)
mcp.registerTool(
@@ -177,12 +180,16 @@ const startFakeDriver = async (
const frame = { frame_id: `f${String(calls.length)}`, width: 1280 };
const blank = name === "computer_screenshot" && blankFrames > 0;
if (blank) blankFrames -= 1;
const jpegData =
name === "computer_screenshot" && args.format === "jpeg"
? options.jpegScreenshotData?.[jpegScreenshots++]
: undefined;
const image =
name === "computer_screenshot" || args.screenshot === true
? [
{
type: "image" as const,
data: blank ? FAKE_BLANK_PNG : frameData,
data: blank ? FAKE_BLANK_PNG : (jpegData ?? frameData),
mimeType: "image/png",
},
]