mirror of
https://github.com/morluto/rea.git
synced 2026-11-20 21:53:51 +00:00
* fix(desktop): reject invalid build screenshot encodings * fix(desktop): finish screenshot admission review --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com>
169 lines
6.0 KiB
TypeScript
169 lines
6.0 KiB
TypeScript
import { join } from "node:path";
|
|
import { setTimeout as delay } from "node:timers/promises";
|
|
import { isDeepStrictEqual } from "node:util";
|
|
|
|
import { z } from "zod";
|
|
|
|
import { AnalysisError } from "../../domain/analysisErrorBase.js";
|
|
import {
|
|
AnalysisInputError,
|
|
cancelledBy,
|
|
carriedReason,
|
|
} from "../../domain/analysisErrorCore.js";
|
|
import type { DesktopTask } from "../../domain/desktop/appTask.js";
|
|
import type { TaskCheckOutcome } from "../../domain/desktop/appTaskChecks.js";
|
|
import {
|
|
renderTaskSteps,
|
|
renderTaskVerifier,
|
|
sameTaskVerifier,
|
|
taskVerifierArtifact,
|
|
} from "../../domain/desktop/appTaskPython.js";
|
|
import {
|
|
SCREEN_SETTLE_MS,
|
|
isScreenTask,
|
|
screenExamplesOf,
|
|
type ScreenJudge,
|
|
} from "../../domain/desktop/appTaskScreen.js";
|
|
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
|
|
import { err, ok, type Result } from "../../domain/result.js";
|
|
import { safeParseUtf8Json } from "../../domain/safeJson.js";
|
|
import {
|
|
guestFileAnswerCeiling,
|
|
guestFileAnswerWorkingBytes,
|
|
readComputerScreenshotBytes,
|
|
type AgentEnvComputer,
|
|
} from "../../desktop/AgentEnvComputer.js";
|
|
import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvProviderIdentity.js";
|
|
import type { HostFileReader } from "../../ports/HostFileReader.js";
|
|
import { errorMessage } from "../../kernel/errorMessage.js";
|
|
|
|
const CHECK = "check_build_task";
|
|
|
|
/** The bundle folder a witnessed computer was started from, and the judge that runs its screen judges. */
|
|
export interface WitnessBundle {
|
|
readonly path: string;
|
|
readonly judge?: ScreenJudge;
|
|
}
|
|
|
|
/** A task's results on the app's state, whether its files had settled, and whether the bound cut a read. */
|
|
export type Graded = {
|
|
readonly results: TaskCheckOutcome[];
|
|
readonly settled: boolean;
|
|
readonly read_cut: boolean;
|
|
};
|
|
|
|
/**
|
|
* The most bytes REA reads of each of the bundle's task files. A task the
|
|
* export accepts has no smaller file cap, and the steps file holds the whole
|
|
* prompt. The bound is the same working budget as a guest file answer: a
|
|
* quarter of the heap, divided by the copies a read and a parse hold.
|
|
*/
|
|
const bundleFileBytes = () =>
|
|
guestFileAnswerWorkingBytes(guestFileAnswerCeiling());
|
|
const stepsSchema = z.array(
|
|
z.looseObject({ type: z.string(), env_id: z.string().optional() }),
|
|
);
|
|
|
|
/** Why the bundle's steps and verifier for the task are not what the export writes for it; undefined when they are. */
|
|
export const bundleTaskDifference = async (
|
|
bundle: string,
|
|
task: DesktopTask,
|
|
signal: AbortSignal,
|
|
readHostFile: HostFileReader,
|
|
): Promise<string | undefined> => {
|
|
const stepsPath = `tasks/${task.name}.json`;
|
|
const verifierPath = `artifacts/${taskVerifierArtifact(task)}/verify.py`;
|
|
const [steps, verifier] = await Promise.all(
|
|
[stepsPath, verifierPath].map((path) =>
|
|
readHostFile(
|
|
join(bundle, path),
|
|
bundleFileBytes(),
|
|
signal,
|
|
false,
|
|
(reason) => cancelledBy(CHECK, carriedReason(reason)),
|
|
),
|
|
),
|
|
);
|
|
if (!Buffer.isBuffer(steps) || !Buffer.isBuffer(verifier))
|
|
return `the bundle at ${bundle} has no readable ${Buffer.isBuffer(steps) ? verifierPath : stepsPath}.`;
|
|
const parsed = safeParseUtf8Json(steps);
|
|
const listed = parsed.ok ? parsed.value : undefined;
|
|
const app = stepsSchema
|
|
.safeParse(listed)
|
|
.data?.find((step) => step.type === "deploy_env")?.env_id;
|
|
if (
|
|
app === undefined ||
|
|
!isDeepStrictEqual(listed, JSON.parse(renderTaskSteps(task, app)))
|
|
)
|
|
return `the bundle's ${stepsPath} differs from this task's steps.`;
|
|
const source = verifier.toString("utf8");
|
|
const differs = `the bundle's ${verifierPath} differs from this task's checks.`;
|
|
if (!isScreenTask(task))
|
|
return sameTaskVerifier(renderTaskVerifier(task, app), source)
|
|
? undefined
|
|
: differs;
|
|
// A screen judge's examples are the build's screenshots, which only the bundle holds.
|
|
const examples = screenExamplesOf(source);
|
|
if (examples === undefined)
|
|
return `the bundle's ${verifierPath} holds no screen judge examples.`;
|
|
return source === renderTaskVerifier(task, app, examples)
|
|
? undefined
|
|
: differs;
|
|
};
|
|
|
|
/** A screen-judged task's checks on a fresh screenshot, each answered by the bundle's own judge. */
|
|
export const judgeScreenTask = async (
|
|
computer: Pick<AgentEnvComputer, "callTool">,
|
|
bundle: WitnessBundle | undefined,
|
|
task: DesktopTask,
|
|
signal: AbortSignal,
|
|
): Promise<Result<Graded, AnalysisError>> => {
|
|
const judge = bundle?.judge;
|
|
if (bundle === undefined || judge === undefined)
|
|
return err(
|
|
new AnalysisInputError(CHECK, {
|
|
userMessage:
|
|
"A screen_judge is checked by the bundle's own judge, so it needs a computer started from the bundle.",
|
|
remediationAction:
|
|
"Start a computer from the exported bundle with start_build_computer and check it there.",
|
|
conformsToInputContract: true,
|
|
}),
|
|
);
|
|
const results: TaskCheckOutcome[] = [];
|
|
try {
|
|
await delay(SCREEN_SETTLE_MS, undefined, { signal });
|
|
const shot = await computer.callTool(
|
|
"computer_screenshot",
|
|
{ format: "jpeg", quality: 70 },
|
|
signal,
|
|
);
|
|
const screen = readComputerScreenshotBytes(shot);
|
|
if (screen === undefined)
|
|
throw new Error("the computer did not return valid screenshot bytes");
|
|
const verifier = join(
|
|
bundle.path,
|
|
"artifacts",
|
|
taskVerifierArtifact(task),
|
|
"verify.py",
|
|
);
|
|
for (const check of task.checks)
|
|
if (check.kind === "screen_judge")
|
|
results.push(await judge(verifier, check, screen, signal));
|
|
return ok({ results, settled: true, read_cut: false });
|
|
} catch (cause: unknown) {
|
|
return err(
|
|
cause instanceof AnalysisError
|
|
? results.length === 0
|
|
? cause
|
|
: cause.retainDiagnostics({ completed_judge_checks: results })
|
|
: new ProviderAdapterError(AGENTENV_PROVIDER, CHECK, {
|
|
userMessage: `The screen judge could not check the screen: ${errorMessage(cause)}`,
|
|
cause,
|
|
...(results.length === 0
|
|
? {}
|
|
: { diagnostics: { completed_judge_checks: results } }),
|
|
}),
|
|
);
|
|
}
|
|
};
|