import { randomUUID } from "node:crypto"; import { z } from "zod"; import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { planOutputTree } from "./DesktopOutputTree.js"; import { AnalysisInputError } from "../../domain/analysisErrorCore.js"; import { parseDesktopTask, type desktopTaskSchema, type DesktopTask, } from "../../domain/desktop/appTask.js"; import { SETTLE_TIMING } from "../../domain/desktop/appSettle.js"; import { evaluateTaskChecks, type TaskCheckOutcome, } from "../../domain/desktop/appTaskChecks.js"; import { isScreenTask } from "../../domain/desktop/appTaskScreen.js"; import { admitWitnessStage, appStateViews, stageExpectation, stageVerdict, unrestoredChecks, unreadResetChecks, WITNESS_STAGES, type WitnessStage, type WitnessStageRecord, } from "../../domain/desktop/buildWitness.js"; import type { JsonValue } from "../../domain/jsonValue.js"; import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; import { err, ok, type Result } from "../../domain/result.js"; import type { AgentEnvComputer, RecordingManifest, } from "../../desktop/AgentEnvComputer.js"; import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvProviderIdentity.js"; import { silentProgressReporter, type ProgressReporter, } from "../../ports/ProgressReporter.js"; import type { ReadonlyActionLog } from "./DesktopBuildActionLog.js"; import { desktopOperationFailure } from "./DesktopCaptureRefusals.js"; import { settledAppData, type BuildSettle } from "./DesktopBuildSettle.js"; import { DesktopRemoteTaskReaders } from "./DesktopRemoteTaskReaders.js"; import { bundleTaskDifference, judgeScreenTask, type Graded, type WitnessBundle, } from "./DesktopBuildWitnessBundle.js"; import { writeWitnessRecord, type EndedWitness, type WitnessRecord, } from "./DesktopBuildWitnessRecord.js"; import { errorMessage } from "../../kernel/errorMessage.js"; import { sha256Hex } from "../../kernel/sha256Hex.js"; const CHECK = "check_build_task"; const FRESH = "Close this computer and start a fresh one from the bundle with start_build_computer."; /** The computer routes the witness reads. */ export type WitnessedComputer = Pick< AgentEnvComputer, | "appData" | "startRecording" | "endRecording" | "recordingSegments" | "exec" | "callTool" >; type WitnessSummary = { readonly stage: WitnessStage; readonly expectation: "some_fail" | "all_pass"; readonly held: boolean; readonly status: "open" | "held" | "failed"; readonly next_stage: WitnessStage | null; readonly record: WitnessRecord | null; }; /** One check of the task against the app's live state, and the witness stage it filed, if any. */ export type CheckedTask = { readonly task: string; readonly passed: number; readonly failed: number; readonly results: TaskCheckOutcome[]; /** Whether the app's files had held for the settle window when they were graded, rather than at its bound. */ readonly settled: boolean; /** Whether a read still in flight at the settle bound was cut, so the last completed read was graded. */ readonly read_cut: boolean; readonly witness: WitnessSummary | null; }; const refused = (userMessage: string, remediationAction: string) => new AnalysisInputError(CHECK, { userMessage, remediationAction, conformsToInputContract: true, }); const taskDigest = (task: DesktopTask) => sha256Hex(JSON.stringify(task)); /** A retry cannot redirect an ended witness to another folder. */ const recordDirectoryRefusal = ( expected: string, requested: string | undefined, ): AnalysisError | undefined => requested !== undefined && requested !== expected ? refused( `This witness's record belongs to ${expected}, not ${requested}.`, "Ask again without directory or with the original folder; another witness needs a fresh computer.", ) : undefined; /** * A witness from its seed on: `open` while stages are checked, `saving` * once a stage ended it and until its record is written, which a repeat of * that stage's check retries, and `saved` after, with the record a repeat * returns again. */ interface OpenWitness extends EndedWitness { state: "open" | "saving" | "saved"; record?: WitnessRecord; /** A manifest whose segments still need to be fetched. */ pendingManifest?: RecordingManifest | undefined; /** * The action log as it stood when the record was first written. A retry * uses this copy, so later activity cannot drop the witnessed actions. */ logged?: Pick; /** * A published record whose handles are still open. The error already carries * the record and the cleanup lease; every later close failure returns it. */ published?: | { readonly error: AnalysisError; readonly release: () => Promise; } | undefined; } /** What the witness says of one filed stage. */ const summaryOf = ( filed: WitnessStageRecord, next: WitnessStage | null, record: WitnessRecord | null, ): WitnessSummary => ({ stage: filed.stage, expectation: filed.expectation, held: filed.held, status: !filed.held ? "failed" : record === null ? "open" : "held", next_stage: next, record, }); /** The pass and fail counts of one set of results. */ const counted = (task: DesktopTask, { results, settled, read_cut }: Graded) => { const passed = results.filter((result) => result.passed).length; return { task: task.name, passed, failed: results.length - passed, results, settled, read_cut, }; }; /** * Grades an exported task against a build computer's app, as the bundle's * verifier does, and witnesses it: the seed, the edit, a near miss and the * seed again after a reset, recorded, with the action log, as Evidence. */ /** * Retry closing a published witness. A further failure returns the original * error, which already carries the record and the cleanup lease. */ export const retryPublishedCleanup = async ( published: AnalysisError, release: () => Promise, ): Promise> => { try { await release(); return ok(undefined); } catch { return err(published); } }; export class DesktopBuildWitness { #witness: OpenWitness | undefined; readonly readers: DesktopRemoteTaskReaders; /** * The request id of this computer's one witness recording, fixed before any * start is sent: a seed retried after a lost reply asks for the same id, * and the computer answers the recording it already began. */ readonly #recordingRequest = `rea-witness:${randomUUID()}`; readonly #bundle: WitnessBundle | undefined; readonly #redact: (value: JsonValue) => JsonValue; /** `bundle` is the folder the computer was started from, if it was; the witness takes only the task it holds. `redact` puts known secrets' handles in place of their values in the record. */ constructor( private readonly computer: WitnessedComputer, private readonly actions: ReadonlyActionLog, private readonly settle: BuildSettle = SETTLE_TIMING, options: { readonly bundle?: WitnessBundle | undefined; readonly redact?: (value: JsonValue) => JsonValue; } = {}, ) { this.#bundle = options.bundle; this.#redact = options.redact ?? ((value) => value); this.readers = new DesktopRemoteTaskReaders( (argv, timeout, signal, requestId) => this.computer.exec(argv, timeout, signal, requestId), ); } async check( input: { readonly task: z.output; readonly stage?: WitnessStage | undefined; readonly directory?: string | undefined; }, signal: AbortSignal, progress: ProgressReporter = silentProgressReporter, ): Promise> { const parsed = parseDesktopTask(input.task); if (!parsed.ok) return err( refused( `The task is refused: ${parsed.error}`, "Pass the task as export_build_environment took it.", ), ); const task = parsed.value; const { stage } = input; if (stage !== undefined && this.#witness?.state === "saving") { const directoryRefusal = recordDirectoryRefusal( this.#witness.directory, input.directory, ); return directoryRefusal === undefined ? this.#resave(this.#witness, task, stage, signal) : err(directoryRefusal); } const saved = stage === undefined ? undefined : this.#savedAgain(task, stage, input.directory); if (saved !== undefined) return saved; const admitted = stage === undefined ? undefined : await this.#admit(task, stage, input.directory, signal); if (admitted?.ok === false) return admitted; const graded = await this.#grade(task, signal, progress); if (!graded.ok) return graded; const checked = counted(task, graded.value); if (stage === undefined || admitted === undefined) return ok({ ...checked, witness: null }); const filed = await this.#file(stage, admitted.value, graded.value, signal); return filed.ok ? ok({ ...checked, witness: filed.value }) : filed; } /** The task's checks against the app's views once its files settle, and its readers' views read then, one result per check. */ async #grade( task: DesktopTask, signal: AbortSignal, progress: ProgressReporter, ): Promise> { if (isScreenTask(task)) return judgeScreenTask(this.computer, this.#bundle, task, signal); try { const read = await settledAppData( (turn) => this.computer.appData("data/get", turn), this.settle, signal, (message) => progress .report({ phase: CHECK, completed: 0, total: null, message }) .catch(() => undefined), ); const views = appStateViews(read.parts); const readers = await this.readers.read(task.readers ?? [], signal); return views.ok ? ok({ results: [ ...evaluateTaskChecks( task.checks, views.value, readers, this.#redact, ), ], settled: read.settled, read_cut: read.readCut, }) : err( refused( views.error, "Start the computer from a bundle export_build_environment wrote.", ), ); } catch (cause: unknown) { return err( cause instanceof AnalysisError ? cause : new ProviderAdapterError(AGENTENV_PROVIDER, CHECK, { userMessage: `Reading the app's state failed: ${errorMessage(cause)}`, cause, }), ); } } /** * File a stage's results; one that does not hold, or the last, ends the * witness and writes its record. One its checks leave undecided files * nothing and can be checked again. */ async #file( stage: WitnessStage, actionsBefore: number, { results, settled, read_cut }: Graded, signal: AbortSignal, ): Promise> { const witness = this.#witness; if (witness === undefined) throw new TypeError("the stage was admitted"); const verdict = stageVerdict(stage, results, witness.stages); if (verdict === "undecided") { const unreadSeed = stage === "after_reset" ? unreadResetChecks(witness.stages, results).map( (result) => result.id, ) : []; return err( refused( `The checks cannot decide ${stage}: each failure that would settle it may come from a view that could not be read or is cut short (${[ ...results.flatMap((result) => result.undecided === true ? [result.description] : [], ), ...unreadSeed.map( (id) => `${id} could not be compared with the seed`, ), ].join("; ")}). Nothing was filed.`, `Check ${stage} again once the app's files read whole; a view that is always cut short or unreadable needs a task whose checks read other files.`, ), ); } const held = verdict === "held"; const unrestored = stage === "after_reset" && !held ? unrestoredChecks(witness.stages, results).map((result) => result.id) : []; const filed: WitnessStageRecord = { stage, actions_before: actionsBefore, checked_at: new Date().toISOString(), expectation: stageExpectation(stage), held, settled, read_cut, results, ...(unrestored.length === 0 ? {} : { remediation: `The reset did not restore the seed: ${unrestored.join(", ")} came out unlike on the seed. Find what the app's data/reset leaves changed, export again and witness a fresh computer of the new bundle.`, }), }; witness.stages.push(filed); if (held && stage !== WITNESS_STAGES.at(-1)) return ok( summaryOf(filed, WITNESS_STAGES[witness.stages.length] ?? null, null), ); witness.state = "saving"; return this.#save(witness, filed, signal); } /** Write the ended witness's record; until that succeeds the witness keeps every stage for a retry. */ async #save( witness: OpenWitness, filed: WitnessStageRecord, signal: AbortSignal, ): Promise> { const published = witness.published; if (witness.record !== undefined && published !== undefined) { const released = await retryPublishedCleanup( published.error, published.release, ); if (!released.ok) return released; witness.published = undefined; witness.state = "saved"; return ok(summaryOf(filed, null, witness.record)); } witness.logged ??= { entries: [...this.actions.entries], dropped: this.actions.dropped, }; const written = await writeWitnessRecord( this.computer, witness, witness.logged, { signal, redact: this.#redact, pendingManifest: witness.pendingManifest, }, ); if (written.kind === "retry") { if (written.manifest !== undefined) witness.pendingManifest = written.manifest; return err(written.error); } witness.pendingManifest = undefined; witness.record = written.record; if (written.kind === "published") { if (written.release === undefined) witness.state = "saved"; else witness.published = { error: written.error, release: written.release }; return err(written.error); } witness.state = "saved"; return ok(summaryOf(filed, null, written.record)); } /** * The saved witness again, for a repeat of the check that ended it with the * same task, such as after a cancelled call whose record was saved anyway; * undefined for any other check. */ #savedAgain( task: DesktopTask, stage: WitnessStage, directory: string | undefined, ): Result | undefined { const witness = this.#witness; const filed = witness?.stages.at(-1); if ( witness?.record === undefined || filed?.stage !== stage || taskDigest(task) !== witness.digest ) return undefined; // A folder the record was not saved to gets nothing written, so it is refused. const directoryRefusal = recordDirectoryRefusal( witness.directory, directory, ); if (directoryRefusal !== undefined) return err(directoryRefusal); return ok({ ...counted(task, filed), witness: summaryOf(filed, null, witness.record), }); } /** Repeat the check of the stage that ended the witness to retry writing its record, without checking again. */ async #resave( witness: OpenWitness, task: DesktopTask, stage: WitnessStage, signal: AbortSignal, ): Promise> { const filed = witness.stages.at(-1); if ( filed === undefined || filed.stage !== stage || taskDigest(task) !== witness.digest ) return err( refused( "This witness has ended, but its record is not saved yet.", `Check ${filed?.stage ?? "the last stage"} again with the same task to save it.`, ), ); const saved = await this.#save(witness, filed, signal); return saved.ok ? ok({ ...counted(task, filed), witness: saved.value, }) : saved; } /** Open the witness at its seed, or admit the next stage of the open one; its value is where its actions end. */ async #admit( task: DesktopTask, stage: WitnessStage, directory: string | undefined, signal: AbortSignal, ): Promise> { const digest = taskDigest(task); const opened = stage === "seed" ? await this.#open(task, digest, directory, signal) : this.#continuing(digest, directory); if (!opened.ok) return opened; const admitted = admitWitnessStage( opened.value.stages, stage, this.actions, ); return admitted.ok ? admitted : err( refused( admitted.error, "Do what the message says, then check this stage again.", ), ); } /** Open the witness with its folder and begin its recording. */ async #open( task: DesktopTask, digest: string, directory: string | undefined, signal: AbortSignal, ): Promise> { const unfinished = this.#witness; if (unfinished !== undefined) return this.#retrySeed(unfinished, digest, directory); if (directory === undefined) return err( refused( "The seed stage opens the witness and needs directory.", "Give directory, a new or empty folder on this host for the record and recording.", ), ); // Admit the folder by the same rule the record is saved under, so the // seed cannot accept a folder (a symlink, a file, a denied path) that the // save would refuse. try { await planOutputTree(directory, { operation: CHECK, field: "directory" }); } catch (cause: unknown) { if (cause instanceof AnalysisError) return err(cause); throw cause; } let differs: string | undefined; if (this.#bundle !== undefined) { try { differs = await bundleTaskDifference(this.#bundle.path, task, signal); } catch (cause: unknown) { // A cancelled read, an I/O failure, or a handle that would not close // stays a Result. A cleanup lease stays bound to that error. return err( desktopOperationFailure( cause, signal, CHECK, "Reading the bundle's task", ), ); } } if (differs !== undefined) return err( refused( `The task is not the bundle's ${task.name} task: ${differs}`, "Give the task exactly as export_build_environment took it for this bundle.", ), ); try { const { recording_id: recordingId } = await this.computer.startRecording( this.#recordingRequest, signal, ); const witness: OpenWitness = { task, digest, directory, recordingId, stages: [], state: "open", }; this.#witness = witness; return ok(witness); } catch (cause: unknown) { return err( new ProviderAdapterError(AGENTENV_PROVIDER, CHECK, { userMessage: `The witness could not start its recording: ${errorMessage(cause)}`, cause, }), ); } } /** * A seed whose check failed after the witness opened, such as on a failed * read of the app's state, is checked again with the same task and folder, * keeping the recording it began. */ #retrySeed( witness: OpenWitness, digest: string, directory: string | undefined, ): Result { if (witness.state !== "open" || witness.stages.length > 0) return err(refused("This computer was already witnessed.", FRESH)); return witness.digest === digest && witness.directory === directory ? ok(witness) : err( refused( "A seed check for another task or folder opened this witness and did not finish.", "Check the seed again with the task and directory it was first given.", ), ); } /** The open witness a later stage continues, for the same task and without a folder. */ #continuing( digest: string, directory: string | undefined, ): Result { const witness = this.#witness; if (witness === undefined) return err( refused( "No witness is open on this computer.", "Check the seed stage first, with directory.", ), ); if (witness.state !== "open") return err(refused("This witness has ended.", FRESH)); if (directory !== undefined) return err( refused( "Only the seed stage takes directory.", "Leave directory out after the seed stage.", ), ); return witness.digest === digest ? ok(witness) : err( refused( "This task differs from the one the seed stage checked.", "Give every stage the task the seed stage checked.", ), ); } }