mirror of
https://github.com/morluto/rea.git
synced 2026-11-19 13:12:26 +00:00
fix(desktop): reject invalid build screenshot encodings (#12179)
* fix(desktop): reject invalid build screenshot encodings * fix(desktop): finish screenshot admission review --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com>
This commit is contained in:
1 parent
b946cb83dd
commit
3fae997886
8 files changed
+116
-18
No files matched your search
@@ -27,7 +27,10 @@ import type { JsonValue } from "../../domain/jsonValue.js";
|
||||
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvProviderIdentity.js";
|
||||
import type { AgentEnvComputer } from "../../desktop/AgentEnvComputer.js";
|
||||
import {
|
||||
readComputerScreenshotBytes,
|
||||
type AgentEnvComputer,
|
||||
} from "../../desktop/AgentEnvComputer.js";
|
||||
import {
|
||||
controller,
|
||||
parseBuildComputerJson,
|
||||
@@ -82,12 +85,10 @@ const keepScreenshot = async (
|
||||
CHECKPOINT_SHOT,
|
||||
signal,
|
||||
);
|
||||
const image = shot.content.find((block) => block.type === "image")?.data;
|
||||
if (shot.isError || typeof image !== "string") return;
|
||||
const image = readComputerScreenshotBytes(shot);
|
||||
if (image === undefined) return;
|
||||
await computer.addHomeFiles(
|
||||
new Map([
|
||||
[`${folder}/screens/${name}.jpg`, Buffer.from(image, "base64")],
|
||||
]),
|
||||
new Map([[`${folder}/screens/${name}.jpg`, image]]),
|
||||
signal,
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import { expect, it } from "vitest";
|
||||
import {
|
||||
INVALID_BUILD_SCREENSHOTS,
|
||||
VALID_BUILD_SCREENSHOT,
|
||||
} from "../../../tests/fixtures/buildScreenshotData.js";
|
||||
import { ProviderCleanupError } from "../../domain/providerCleanupError.js";
|
||||
import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
|
||||
import { judgeScreenTask } from "./DesktopBuildWitnessBundle.js";
|
||||
@@ -14,13 +18,14 @@ it("preserves earlier judge outcomes when a later check fails cleanup", async ()
|
||||
{
|
||||
callTool: async () => ({
|
||||
isError: false,
|
||||
content: [{ type: "image", data: "aW1hZ2U=" }],
|
||||
content: [{ type: "image", data: VALID_BUILD_SCREENSHOT.data }],
|
||||
structured: null,
|
||||
}),
|
||||
},
|
||||
{
|
||||
path: "/bundle",
|
||||
judge: async (_verifier, check) => {
|
||||
judge: async (_verifier, check, screen) => {
|
||||
expect(VALID_BUILD_SCREENSHOT.bytes.equals(screen)).toBe(true);
|
||||
if (check.id === "second") throw failure;
|
||||
return { id: check.id, passed: true, description: "first completed" };
|
||||
},
|
||||
@@ -48,3 +53,37 @@ it("preserves earlier judge outcomes when a later check fails cleanup", async ()
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it.each(INVALID_BUILD_SCREENSHOTS)(
|
||||
"refuses an invalid screen judge image ($name) before invoking the judge",
|
||||
async ({ data }) => {
|
||||
let judged = false;
|
||||
const result = await judgeScreenTask(
|
||||
{
|
||||
callTool: async () => ({
|
||||
isError: false,
|
||||
content: [{ type: "image", data }],
|
||||
structured: null,
|
||||
}),
|
||||
},
|
||||
{
|
||||
path: "/bundle",
|
||||
judge: async (_verifier, check) => {
|
||||
judged = true;
|
||||
return { id: check.id, passed: true, description: "accepted" };
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "test",
|
||||
prompt: "test",
|
||||
checks: [{ kind: "screen_judge", id: "screen", question: "Changed?" }],
|
||||
},
|
||||
new AbortController().signal,
|
||||
);
|
||||
expect(judged).toBe(false);
|
||||
expect(result).toMatchObject({
|
||||
ok: false,
|
||||
error: { userMessage: expect.stringContaining("valid screenshot") },
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -30,6 +30,7 @@ import { safeParseUtf8Json } from "../../domain/safeJson.js";
|
||||
import {
|
||||
guestFileAnswerCeiling,
|
||||
guestFileAnswerWorkingBytes,
|
||||
readComputerScreenshotBytes,
|
||||
type AgentEnvComputer,
|
||||
} from "../../desktop/AgentEnvComputer.js";
|
||||
import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvProviderIdentity.js";
|
||||
@@ -136,10 +137,9 @@ export const judgeScreenTask = async (
|
||||
{ format: "jpeg", quality: 70 },
|
||||
signal,
|
||||
);
|
||||
const image = shot.content.find((block) => block.type === "image")?.data;
|
||||
if (shot.isError || typeof image !== "string")
|
||||
throw new Error("the computer took no screenshot");
|
||||
const screen = Buffer.from(image, "base64");
|
||||
const screen = readComputerScreenshotBytes(shot);
|
||||
if (screen === undefined)
|
||||
throw new Error("the computer did not return valid screenshot bytes");
|
||||
const verifier = join(
|
||||
bundle.path,
|
||||
"artifacts",
|
||||
|
||||
@@ -8,6 +8,7 @@ import { z } from "zod";
|
||||
import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js";
|
||||
import { Deadline } from "../domain/deadline.js";
|
||||
import { jsonObjectSchema, type JsonValue } from "../domain/jsonValue.js";
|
||||
import { isCanonicalBase64 } from "../domain/stringPatterns.js";
|
||||
import { StructuredContentMcpClient } from "../providers/StructuredContentMcpClient.js";
|
||||
import { validateMcpHttpResponse } from "../providers/Utf8McpResponse.js";
|
||||
import type { ComputerHelper } from "./AgentEnvComputerHelper.js";
|
||||
@@ -214,6 +215,17 @@ export interface DriverResult {
|
||||
readonly structured: Readonly<Record<string, JsonValue>> | null;
|
||||
}
|
||||
|
||||
/** Decode the driver's first screenshot only when its encoded bytes are intact. */
|
||||
export const readComputerScreenshotBytes = (
|
||||
reply: DriverResult,
|
||||
): Buffer | undefined => {
|
||||
if (reply.isError) return undefined;
|
||||
const data = reply.content.find((block) => block.type === "image")?.data;
|
||||
if (typeof data !== "string" || data.length === 0 || !isCanonicalBase64(data))
|
||||
return undefined;
|
||||
return Buffer.from(data, "base64");
|
||||
};
|
||||
|
||||
/** A response body read only while it stays within `maxBytes`. */
|
||||
const readBounded = async (
|
||||
response: HttpReply,
|
||||
|
||||
@@ -24,6 +24,10 @@ import {
|
||||
expectNoToken,
|
||||
startBuildComputer,
|
||||
} from "../../fixtures/buildComputerMcp.js";
|
||||
import {
|
||||
INVALID_BUILD_SCREENSHOTS,
|
||||
VALID_BUILD_SCREENSHOT,
|
||||
} from "../../fixtures/buildScreenshotData.js";
|
||||
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
|
||||
@@ -195,12 +199,28 @@ it("captures the app inside the computer, acknowledges checkpoints, drafts the t
|
||||
expectNoToken(computer, [started, seed, stopped, exported]);
|
||||
});
|
||||
|
||||
it("keeps a screenshot of each checkpoint in the computer, which the export takes as a screen judge's examples", async () => {
|
||||
const { rea, client } = await fakes();
|
||||
it("keeps valid checkpoint screenshots byte for byte for export and skips invalid screenshots without failing checkpoints", async () => {
|
||||
const screenshots = [
|
||||
{ name: "seed", data: VALID_BUILD_SCREENSHOT.data },
|
||||
...INVALID_BUILD_SCREENSHOTS,
|
||||
{ name: "end", data: VALID_BUILD_SCREENSHOT.data },
|
||||
];
|
||||
const { rea, client } = await fakes({
|
||||
jpegScreenshotData: screenshots.map(({ data }) => data),
|
||||
});
|
||||
const id = await startBuildComputer(client, REQUEST);
|
||||
await callOk(client, "start_build_capture", { computer_id: id });
|
||||
for (const name of ["seed", "end"])
|
||||
await callOk(client, "mark_build_checkpoint", { computer_id: id, name });
|
||||
for (const { name } of screenshots) {
|
||||
const marked = await callOk(client, "mark_build_checkpoint", {
|
||||
computer_id: id,
|
||||
name,
|
||||
});
|
||||
expect(
|
||||
toolContract("mark_build_checkpoint").outputSchema.parse(
|
||||
marked.structuredContent,
|
||||
).checkpoint.name,
|
||||
).toBe(name);
|
||||
}
|
||||
await callOk(client, "stop_build_capture", { computer_id: id });
|
||||
await callOk(client, "export_build_environment", {
|
||||
computer_id: id,
|
||||
@@ -215,6 +235,10 @@ it("keeps a screenshot of each checkpoint in the computer, which the export take
|
||||
"screens/end.jpg",
|
||||
"screens/seed.jpg",
|
||||
]);
|
||||
for (const path of shots) {
|
||||
const bytes = await readFile(join(rea.home, ".rea-build", path));
|
||||
expect(bytes.equals(VALID_BUILD_SCREENSHOT.bytes), path).toBe(true);
|
||||
}
|
||||
const exported = (
|
||||
(await rea.notes("calls.jsonl")) as { argv: string[] }[]
|
||||
).findLast(({ argv }) => argv[0] === "export-desktop-application");
|
||||
|
||||
Vendored
+6
-1
@@ -342,7 +342,8 @@ const realCliOutputs = async (
|
||||
* `env` adds to the stand-in's environment; `install` replaces the captured
|
||||
* package, the archive's `notes`, and names the packages the real export
|
||||
* installs beside it; `screenshotDelaysMs` delays the computer's
|
||||
* screenshots in order. `recorded` replaces what the capture records, with
|
||||
* screenshots in order and `jpegScreenshotData` supplies encoded JPEG
|
||||
* screenshot replies. `recorded` replaces what the capture records, with
|
||||
* `exportRefused` when its export refuses BUILD_TASK, or `tasks` in its
|
||||
* place, and `extraContent` the
|
||||
* retained bytes it holds beyond the note's; `selected` is what it records
|
||||
@@ -356,6 +357,7 @@ export const buildCaptureFakes = async (
|
||||
readonly env?: Readonly<Record<string, string>>;
|
||||
readonly install?: CapturedInstall;
|
||||
readonly screenshotDelaysMs?: readonly number[];
|
||||
readonly jpegScreenshotData?: readonly string[];
|
||||
readonly recorded?: DesktopApplicationCapture;
|
||||
readonly exportRefused?: boolean;
|
||||
readonly extraContent?: readonly string[];
|
||||
@@ -400,6 +402,9 @@ export const buildCaptureFakes = async (
|
||||
...(options.screenshotDelaysMs === undefined
|
||||
? {}
|
||||
: { screenshotDelaysMs: options.screenshotDelaysMs }),
|
||||
...(options.jpegScreenshotData === undefined
|
||||
? {}
|
||||
: { jpegScreenshotData: options.jpegScreenshotData }),
|
||||
exec: {
|
||||
home: rea.home,
|
||||
bin: rea.bin,
|
||||
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
export const VALID_BUILD_SCREENSHOT = {
|
||||
data: "aW1hZ2U=",
|
||||
bytes: Buffer.from("image"),
|
||||
};
|
||||
|
||||
export const INVALID_BUILD_SCREENSHOTS = [
|
||||
{ name: "invalid-alphabet", data: "aW1hZ2U=!" },
|
||||
{ name: "noncanonical-padding-bits", data: "aW1hZ2V=" },
|
||||
{ name: "empty-image", data: "" },
|
||||
] as const;
|
||||
+8
-1
@@ -65,6 +65,8 @@ export interface FakeComputerOptions {
|
||||
readonly blankFrames?: number;
|
||||
/** How long each screenshot, in order, waits before its reply, unless its request is cancelled. */
|
||||
readonly screenshotDelaysMs?: readonly number[];
|
||||
/** Encoded image data in response to JPEG screenshot requests, in order. */
|
||||
readonly jpegScreenshotData?: readonly string[];
|
||||
/** Bytes data/get appends to each segment beyond its manifest size. */
|
||||
readonly segmentPadding?: number;
|
||||
/** Recording starts, in order, whose recording begins but whose reply is lost. */
|
||||
@@ -153,6 +155,7 @@ const startFakeDriver = async (
|
||||
: "A".repeat(options.frameCharacters);
|
||||
let blankFrames = options.blankFrames ?? 0;
|
||||
let screenshots = 0;
|
||||
let jpegScreenshots = 0;
|
||||
const mcp = new McpServer({ name: "fake-reference-driver", version: "1" });
|
||||
for (const name of COMPUTER_TOOLS)
|
||||
mcp.registerTool(
|
||||
@@ -177,12 +180,16 @@ const startFakeDriver = async (
|
||||
const frame = { frame_id: `f${String(calls.length)}`, width: 1280 };
|
||||
const blank = name === "computer_screenshot" && blankFrames > 0;
|
||||
if (blank) blankFrames -= 1;
|
||||
const jpegData =
|
||||
name === "computer_screenshot" && args.format === "jpeg"
|
||||
? options.jpegScreenshotData?.[jpegScreenshots++]
|
||||
: undefined;
|
||||
const image =
|
||||
name === "computer_screenshot" || args.screenshot === true
|
||||
? [
|
||||
{
|
||||
type: "image" as const,
|
||||
data: blank ? FAKE_BLANK_PNG : frameData,
|
||||
data: blank ? FAKE_BLANK_PNG : (jpegData ?? frameData),
|
||||
mimeType: "image/png",
|
||||
},
|
||||
]
|
||||
|
||||
Reference in new issue
Block a user