Files
reverse-engineering-anything/src/application/desktop/DesktopBuildExport.ts
T
morluto 12bf103367 fix(desktop): preserve cancellation after output publication (#10108)
* fix(desktop): preserve cancellation after output publication

* fix(desktop): retain cancellation in combined output failures
2026-11-11 09:01:07 -05:00

494 lines
17 KiB
TypeScript

import { Readable } from "node:stream";
import { z } from "zod";
import type { DownloadInstall } from "../../domain/desktop/appDownloadInstall.js";
import {
READER_RUNNER_SLACK_S,
readerTimeout,
} from "../../domain/desktop/appTaskReaders.js";
import { BUILD_REA_TIMEOUT_S } from "../../desktop/BuildComputerController.js";
import { BuildComputerLongRuns } from "../../desktop/BuildComputerLongRuns.js";
import { AnalysisError } from "../../domain/analysisErrorBase.js";
import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js";
import type { desktopTaskSchema } from "../../domain/desktop/appTask.js";
import { counted } from "../../domain/desktop/appExportUnowned.js";
import { END_CHECKPOINT } from "../../domain/desktop/buildCapture.js";
import {
SIGN_IN_END,
SIGN_IN_START,
type BuildConsent,
} from "../../domain/desktop/appWithheld.js";
import {
gzipInside,
isUncheckedGzip,
MAX_GUNZIPPED_BYTES,
redactSecrets,
redactSecretsInJson,
secretsIn,
type KnownSecret,
} from "../../domain/desktop/buildSecrets.js";
import { jsonValueSchema, type JsonValue } from "../../domain/jsonValue.js";
import { err, ok, type Result } from "../../domain/result.js";
import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js";
import {
fetchHomeFolder,
type ControlledComputer,
} from "../../desktop/BuildComputerController.js";
import {
adapterError,
ENDS,
errorText,
readOutput,
refused,
reportedFailure,
runRea,
WORK,
} from "./DesktopBuildCommands.js";
import {
planOutputTree,
writeOutputTree,
desktopOutputFailure,
type DesktopOutputCompletion,
} from "./DesktopOutputTree.js";
const EXPORT = "export_build_environment";
const exportResultSchema = z.looseObject({
environment: z.string(),
seed_files: z.array(z.string()),
scrubbed_files: z.array(z.string()),
// An export from before gzip seeds were listed apart has none.
recompressed_files: z.array(z.string()).default([]),
unscrubbed_files: z.array(z.string()),
limitations: z.array(z.string()),
tasks: z.array(jsonValueSchema),
// An export from before withheld files were listed has none.
withheld_files: z
.array(
z.looseObject({
file: z.string(),
variable: z.string(),
handle: z.string(),
}),
)
.default([]),
consented_files: z.array(z.string()).default([]),
});
/** One task the export takes, with the checkpoint its end state is read at. */
interface BuildExportTask {
readonly task: z.output<typeof desktopTaskSchema>;
readonly end_checkpoint?: string | undefined;
}
/** What the agent asks the export for: its tasks, the folder here, optionally the environment's name, files to withhold, and whether to seed withheld state with the person's consent. */
export interface BuildExportRequest {
readonly tasks: readonly BuildExportTask[];
readonly directory: string;
readonly name?: string | undefined;
readonly withhold?: readonly string[] | undefined;
readonly seed_sign_in?: boolean | undefined;
}
/** A withheld file as the export reports it: the variable a deploy reads and the handle of the secret file REA keeps its bytes in. */
type WithheldReport = {
readonly file: string;
readonly variable: string;
readonly handle: string;
};
/** An exported bundle brought back to this host. */
export type ExportedBuild = {
readonly directory: string;
readonly environment: string;
readonly files: number;
readonly bytes: number;
readonly tasks: JsonValue[];
readonly seed_files: string[];
readonly scrubbed_files: string[];
readonly recompressed_files: string[];
readonly unscrubbed_files: string[];
readonly limitations: string[];
readonly withheld_files: WithheldReport[];
readonly consented_files: string[];
};
/** Explicitly supplied secrets the bundle may not hold. */
interface BundleGuard {
readonly secrets: readonly KnownSecret[];
}
/** Why a fetched file is held back: known secrets or an unchecked gzip. */
const heldBack = (
file: { readonly path: string; readonly bytes: Buffer },
guard: BundleGuard,
) => {
// A gzip file is decompressed once for the secret checks.
const inner = guard.secrets.length === 0 ? undefined : gzipInside(file.bytes);
return {
handles: [
...new Set([
...secretsIn(Buffer.from(file.path), guard.secrets),
...secretsIn(file.bytes, guard.secrets, inner),
]),
],
unchecked: guard.secrets.length > 0 && isUncheckedGzip(file.bytes, inner),
};
};
/** The refusal of gzip files too large for REA to check for known secrets. */
const uncheckedRefusal = (
holding: readonly { readonly path: string }[],
directory: string,
) =>
refused(
EXPORT,
`REA could not check inside ${counted(holding.map(({ path }) => path))}: each is a gzip file that expands past ${String(MAX_GUNZIPPED_BYTES / 1024 / 1024)} MiB, the most REA decompresses to look for known secrets, so REA wrote none of the bundle to ${directory}.`,
"Keep each such file out of the seed: delete it in the app before marking seed, or capture again on a new build computer without it, then export again.",
);
/** The refusal of a bundle whose files hold secrets: each secret's handle and the files, the values in their names redacted. */
const secretsRefusal = (
holding: readonly { readonly path: string; readonly handles: string[] }[],
secrets: readonly KnownSecret[],
directory: string,
) => {
const handles = [...new Set(holding.flatMap(({ handles: found }) => found))];
const found = secrets.filter(({ handle }) => handles.includes(handle));
return refused(
EXPORT,
`The bundle holds the secret ${handles.join(", ")} in ${counted(holding.map(({ path }) => redactSecrets(path, found)))}, so REA wrote none of it to ${directory}: no bundle may hold a secret.`,
"Keep the secret out of the seed: delete the file that holds it in the app before marking seed, or capture again on a new build computer without it, then export again.",
);
};
/**
* Bring the bundle back straight into the caller's new or empty folder, each
* checked answer written before the next is fetched; a failure leaves no
* partial folder.
*/
const writeBundle = async (
computer: ControlledComputer,
bundle: string,
{
directory,
guard,
}: { readonly directory: string; readonly guard: BundleGuard },
signal: AbortSignal,
): Promise<{
readonly files: number;
readonly bytes: number;
readonly completion: DesktopOutputCompletion;
}> => {
let fetched = { files: 0, bytes: 0 };
const holding: {
path: string;
handles: string[];
unchecked: boolean;
}[] = [];
const completion = await writeOutputTree(
await planOutputTree(directory, { operation: EXPORT, field: "directory" }),
async (tree) => {
fetched = await fetchHomeFolder(
computer,
bundle,
{
file: async (file) => {
// Keep back known secrets and files REA could not check.
const held = heldBack(file, guard);
if (held.handles.length > 0 || held.unchecked) {
holding.push({ path: file.path, ...held });
return;
}
await tree.write(
file.path,
Readable.from([file.bytes]),
{ bytes: file.bytes.length, sha256: file.sha256 },
signal,
);
},
},
signal,
);
const secret = holding.filter(({ handles }) => handles.length > 0);
if (secret.length > 0)
throw secretsRefusal(secret, guard.secrets, directory);
const unchecked = holding.filter((held) => held.unchecked);
if (unchecked.length > 0) throw uncheckedRefusal(unchecked, directory);
},
{ operation: EXPORT, field: "directory" },
signal,
);
return { ...fetched, completion };
};
/** The export's arguments past the capture and the bundle: the tasks and each one's end, the ends, the install, and withheld state with its consents. */
const exportArguments = (
capture: {
readonly folder: string;
readonly packageUrl?: string;
readonly downloadInstall?: DownloadInstall;
readonly extraPackages: readonly string[];
readonly signIn: boolean;
readonly consents: readonly BuildConsent[];
},
request: BuildExportRequest,
files: {
readonly tasks: readonly string[];
readonly consent: string;
readonly withheld: string;
readonly download: string;
},
): string[] => [
...files.tasks.flatMap((path) => ["--task", path]),
...request.tasks.flatMap(({ task, end_checkpoint: end }) => [
"--task-end",
`${task.name}=${end ?? END_CHECKPOINT}`,
]),
...ENDS,
// The building agent made the seed in this computer for the task.
"--synthetic-seed",
"--screens",
`${capture.folder}/screens`,
...(request.name === undefined ? [] : ["--name", request.name]),
...(capture.packageUrl === undefined
? []
: ["--package-url", capture.packageUrl]),
...(capture.downloadInstall === undefined
? []
: ["--download-install", files.download]),
...capture.extraPackages.flatMap((name) => ["--extra-package", name]),
...(capture.signIn
? ["--withhold-from", SIGN_IN_START, "--withhold-to", SIGN_IN_END]
: []),
...(request.withhold ?? []).flatMap((path) => ["--withhold-path", path]),
...(capture.consents.length === 0 ? [] : ["--consent", files.consent]),
...(request.seed_sign_in === true ? ["--seed-withheld"] : []),
"--withheld-copy",
files.withheld,
"--json",
];
/** The consent records as `consent.json` holds them. */
const consentFile = (consents: readonly BuildConsent[]): string =>
`${JSON.stringify({ consents }, null, 2)}\n`;
/** Keep each withheld file's bytes as a secret, fetched from beside the bundle, reported by the handle that keeps it; none leaves REA otherwise. */
const keepWithheld = async (
computer: ControlledComputer,
folder: string,
files: readonly WithheldReport[],
{
keep,
signal,
completed,
}: {
readonly keep: (handle: string, bytes: Buffer) => Promise<string>;
readonly signal: AbortSignal;
readonly completed: WithheldReport[];
},
): Promise<WithheldReport[]> => {
if (files.length === 0) return [];
const handles = new Map(files.map((entry) => [entry.file, entry]));
const kept = new Map<string, string>();
await fetchHomeFolder(
computer,
folder,
{
file: async ({ path, bytes }) => {
const report = handles.get(path);
if (report !== undefined) {
const actual = await keep(report.handle, bytes);
kept.set(path, actual);
completed.push({ ...report, handle: actual });
}
},
},
signal,
);
if (kept.size !== handles.size)
throw new Error("The exported bundle did not supply every withheld file.");
return files.map(({ file, variable, handle }) => ({
file,
variable,
handle: kept.get(file) ?? handle,
}));
};
/** The stopped capture an export runs on, and what it holds back and records. */
interface ExportCapture {
readonly folder: string;
readonly attempt: number;
/** The URL a computer downloads the app's `.deb` from, when the capture says apt does not install it. */
readonly packageUrl?: string;
readonly downloadInstall?: DownloadInstall;
/** The packages the request named beside the app, which the bundle installs beside it too. */
readonly extraPackages: readonly string[];
/** The secrets no bundle may hold, and where the withheld files' bytes are kept as secrets. */
readonly secrets: readonly KnownSecret[];
readonly keep: (handle: string, bytes: Buffer) => Promise<string>;
/** Whether both sign-in checkpoints were marked, and the consents the person gave. */
readonly signIn: boolean;
readonly consents: readonly BuildConsent[];
/** The checkpoints the capture took, which say whether a task has a near miss its readers also read. */
readonly checkpoints?: ReadonlySet<string> | undefined;
}
/**
* Export a stopped capture in the computer with the agent's tasks, seeded at
* `seed` and each checked at its own end, `end` by default, into a new
* folder of the work folder, then bring the bundle back and write it into
* the caller's folder. A task the export refuses is refused with the
* export's own reason.
*/
export const exportBuild = async (
computer: ControlledComputer,
capture: ExportCapture,
request: BuildExportRequest,
signal: AbortSignal,
longRuns: BuildComputerLongRuns = new BuildComputerLongRuns(computer),
): Promise<Result<ExportedBuild, AnalysisError>> => {
const { folder } = capture;
const attempt = String(capture.attempt);
const bundle = `${folder}/bundle-${attempt}`;
const taskFiles = Object.fromEntries(
request.tasks.map(({ task }, index) => [
`${folder}/task-${attempt}-${String(index + 1)}.json`,
`${JSON.stringify(task, null, 2)}\n`,
]),
);
const consent = `${folder}/consent-${attempt}.json`;
const withheld = `${bundle}-withheld`;
const stdout = `${folder}/export-${attempt}.json`;
const downloadPath = `${folder}/download-install-${attempt}.json`;
const download = capture.downloadInstall;
const completed: WithheldReport[] = [];
let published:
| {
readonly directory: string;
readonly environment: string;
readonly files: number;
readonly bytes: number;
}
| undefined;
try {
const argv = [
"export-desktop-application",
`${folder}/capture`,
bundle,
...exportArguments(capture, request, {
tasks: Object.keys(taskFiles),
consent,
withheld,
download: downloadPath,
}),
];
// Each task's readers run at its end and the seed, and at its near miss when the capture took one.
const timeoutS =
BUILD_REA_TIMEOUT_S +
request.tasks.reduce(
(total, { task }) =>
total +
(capture.checkpoints?.has(`near-${task.name}`) === true ? 3 : 2) *
(task.readers ?? []).reduce(
(sum, reader) =>
sum + readerTimeout(reader) + READER_RUNNER_SLACK_S,
0,
),
0,
);
const options = {
stdout,
files: {
...taskFiles,
...(download === undefined
? {}
: { [downloadPath]: `${JSON.stringify(download, null, 2)}\n` }),
...(capture.consents.length === 0
? {}
: { [consent]: consentFile(capture.consents) }),
},
timeoutS,
signal,
};
const { stdoutBytes: bytes } =
timeoutS + 60 <= 3600
? await runRea(computer, argv, options)
: await longRuns.run(argv, { ...options, work: WORK });
const output = await readOutput(computer, stdout, bytes, signal);
if (output.kind === "failure")
return err(
output.failure.code === "invalid_request"
? refused(
EXPORT,
output.failure.message,
output.failure.remediation?.action ??
"Rewrite the task and export again.",
)
: reportedFailure(EXPORT, output),
);
const result = exportResultSchema.parse(output.result);
const written = await writeBundle(
computer,
bundle,
{
directory: request.directory,
guard: {
secrets: capture.secrets,
},
},
signal,
);
published = {
directory: request.directory,
environment: result.environment,
files: written.files,
bytes: written.bytes,
};
const outputFailure = desktopOutputFailure(
{ provider: AGENTENV_PROVIDER, operation: EXPORT, what: "bundle" },
request.directory,
written.completion,
);
if (outputFailure !== undefined) throw outputFailure;
const kept = await keepWithheld(computer, withheld, result.withheld_files, {
keep: capture.keep,
signal,
completed,
});
return ok({
directory: request.directory,
environment: result.environment,
files: written.files,
bytes: written.bytes,
tasks: result.tasks,
seed_files: result.seed_files,
scrubbed_files: result.scrubbed_files,
recompressed_files: result.recompressed_files,
unscrubbed_files: result.unscrubbed_files,
limitations: result.limitations,
withheld_files: kept,
consented_files: result.consented_files,
});
} catch (cause: unknown) {
const failure =
cause instanceof AnalysisError
? cause
: signal.aborted && cause === signal.reason
? new AnalysisCancelledError(EXPORT, { cause })
: adapterError(
EXPORT,
redactSecrets(errorText(cause), capture.secrets),
cause,
);
if (published !== undefined)
failure.retainDiagnostics({
published_bundle: redactSecretsInJson(
{ ...published, withheld_files: completed },
capture.secrets,
),
});
return err(failure);
}
};