mirror of
https://github.com/morluto/rea.git
synced 2026-11-16 19:53:50 +00:00
* fix(desktop): preserve cancellation after output publication * fix(desktop): retain cancellation in combined output failures
494 lines
17 KiB
TypeScript
494 lines
17 KiB
TypeScript
import { Readable } from "node:stream";
|
|
|
|
import { z } from "zod";
|
|
|
|
import type { DownloadInstall } from "../../domain/desktop/appDownloadInstall.js";
|
|
import {
|
|
READER_RUNNER_SLACK_S,
|
|
readerTimeout,
|
|
} from "../../domain/desktop/appTaskReaders.js";
|
|
import { BUILD_REA_TIMEOUT_S } from "../../desktop/BuildComputerController.js";
|
|
import { BuildComputerLongRuns } from "../../desktop/BuildComputerLongRuns.js";
|
|
import { AnalysisError } from "../../domain/analysisErrorBase.js";
|
|
import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js";
|
|
import type { desktopTaskSchema } from "../../domain/desktop/appTask.js";
|
|
import { counted } from "../../domain/desktop/appExportUnowned.js";
|
|
import { END_CHECKPOINT } from "../../domain/desktop/buildCapture.js";
|
|
import {
|
|
SIGN_IN_END,
|
|
SIGN_IN_START,
|
|
type BuildConsent,
|
|
} from "../../domain/desktop/appWithheld.js";
|
|
import {
|
|
gzipInside,
|
|
isUncheckedGzip,
|
|
MAX_GUNZIPPED_BYTES,
|
|
redactSecrets,
|
|
redactSecretsInJson,
|
|
secretsIn,
|
|
type KnownSecret,
|
|
} from "../../domain/desktop/buildSecrets.js";
|
|
import { jsonValueSchema, type JsonValue } from "../../domain/jsonValue.js";
|
|
import { err, ok, type Result } from "../../domain/result.js";
|
|
import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js";
|
|
import {
|
|
fetchHomeFolder,
|
|
type ControlledComputer,
|
|
} from "../../desktop/BuildComputerController.js";
|
|
import {
|
|
adapterError,
|
|
ENDS,
|
|
errorText,
|
|
readOutput,
|
|
refused,
|
|
reportedFailure,
|
|
runRea,
|
|
WORK,
|
|
} from "./DesktopBuildCommands.js";
|
|
import {
|
|
planOutputTree,
|
|
writeOutputTree,
|
|
desktopOutputFailure,
|
|
type DesktopOutputCompletion,
|
|
} from "./DesktopOutputTree.js";
|
|
|
|
const EXPORT = "export_build_environment";
|
|
const exportResultSchema = z.looseObject({
|
|
environment: z.string(),
|
|
seed_files: z.array(z.string()),
|
|
scrubbed_files: z.array(z.string()),
|
|
// An export from before gzip seeds were listed apart has none.
|
|
recompressed_files: z.array(z.string()).default([]),
|
|
unscrubbed_files: z.array(z.string()),
|
|
limitations: z.array(z.string()),
|
|
tasks: z.array(jsonValueSchema),
|
|
// An export from before withheld files were listed has none.
|
|
withheld_files: z
|
|
.array(
|
|
z.looseObject({
|
|
file: z.string(),
|
|
variable: z.string(),
|
|
handle: z.string(),
|
|
}),
|
|
)
|
|
.default([]),
|
|
consented_files: z.array(z.string()).default([]),
|
|
});
|
|
|
|
/** One task the export takes, with the checkpoint its end state is read at. */
|
|
interface BuildExportTask {
|
|
readonly task: z.output<typeof desktopTaskSchema>;
|
|
readonly end_checkpoint?: string | undefined;
|
|
}
|
|
|
|
/** What the agent asks the export for: its tasks, the folder here, optionally the environment's name, files to withhold, and whether to seed withheld state with the person's consent. */
|
|
export interface BuildExportRequest {
|
|
readonly tasks: readonly BuildExportTask[];
|
|
readonly directory: string;
|
|
readonly name?: string | undefined;
|
|
readonly withhold?: readonly string[] | undefined;
|
|
readonly seed_sign_in?: boolean | undefined;
|
|
}
|
|
|
|
/** A withheld file as the export reports it: the variable a deploy reads and the handle of the secret file REA keeps its bytes in. */
|
|
type WithheldReport = {
|
|
readonly file: string;
|
|
readonly variable: string;
|
|
readonly handle: string;
|
|
};
|
|
|
|
/** An exported bundle brought back to this host. */
|
|
export type ExportedBuild = {
|
|
readonly directory: string;
|
|
readonly environment: string;
|
|
readonly files: number;
|
|
readonly bytes: number;
|
|
readonly tasks: JsonValue[];
|
|
readonly seed_files: string[];
|
|
readonly scrubbed_files: string[];
|
|
readonly recompressed_files: string[];
|
|
readonly unscrubbed_files: string[];
|
|
readonly limitations: string[];
|
|
readonly withheld_files: WithheldReport[];
|
|
readonly consented_files: string[];
|
|
};
|
|
|
|
/** Explicitly supplied secrets the bundle may not hold. */
|
|
interface BundleGuard {
|
|
readonly secrets: readonly KnownSecret[];
|
|
}
|
|
|
|
/** Why a fetched file is held back: known secrets or an unchecked gzip. */
|
|
const heldBack = (
|
|
file: { readonly path: string; readonly bytes: Buffer },
|
|
guard: BundleGuard,
|
|
) => {
|
|
// A gzip file is decompressed once for the secret checks.
|
|
const inner = guard.secrets.length === 0 ? undefined : gzipInside(file.bytes);
|
|
return {
|
|
handles: [
|
|
...new Set([
|
|
...secretsIn(Buffer.from(file.path), guard.secrets),
|
|
...secretsIn(file.bytes, guard.secrets, inner),
|
|
]),
|
|
],
|
|
unchecked: guard.secrets.length > 0 && isUncheckedGzip(file.bytes, inner),
|
|
};
|
|
};
|
|
|
|
/** The refusal of gzip files too large for REA to check for known secrets. */
|
|
const uncheckedRefusal = (
|
|
holding: readonly { readonly path: string }[],
|
|
directory: string,
|
|
) =>
|
|
refused(
|
|
EXPORT,
|
|
`REA could not check inside ${counted(holding.map(({ path }) => path))}: each is a gzip file that expands past ${String(MAX_GUNZIPPED_BYTES / 1024 / 1024)} MiB, the most REA decompresses to look for known secrets, so REA wrote none of the bundle to ${directory}.`,
|
|
"Keep each such file out of the seed: delete it in the app before marking seed, or capture again on a new build computer without it, then export again.",
|
|
);
|
|
|
|
/** The refusal of a bundle whose files hold secrets: each secret's handle and the files, the values in their names redacted. */
|
|
const secretsRefusal = (
|
|
holding: readonly { readonly path: string; readonly handles: string[] }[],
|
|
secrets: readonly KnownSecret[],
|
|
directory: string,
|
|
) => {
|
|
const handles = [...new Set(holding.flatMap(({ handles: found }) => found))];
|
|
const found = secrets.filter(({ handle }) => handles.includes(handle));
|
|
return refused(
|
|
EXPORT,
|
|
`The bundle holds the secret ${handles.join(", ")} in ${counted(holding.map(({ path }) => redactSecrets(path, found)))}, so REA wrote none of it to ${directory}: no bundle may hold a secret.`,
|
|
"Keep the secret out of the seed: delete the file that holds it in the app before marking seed, or capture again on a new build computer without it, then export again.",
|
|
);
|
|
};
|
|
|
|
/**
|
|
* Bring the bundle back straight into the caller's new or empty folder, each
|
|
* checked answer written before the next is fetched; a failure leaves no
|
|
* partial folder.
|
|
*/
|
|
const writeBundle = async (
|
|
computer: ControlledComputer,
|
|
bundle: string,
|
|
{
|
|
directory,
|
|
guard,
|
|
}: { readonly directory: string; readonly guard: BundleGuard },
|
|
signal: AbortSignal,
|
|
): Promise<{
|
|
readonly files: number;
|
|
readonly bytes: number;
|
|
readonly completion: DesktopOutputCompletion;
|
|
}> => {
|
|
let fetched = { files: 0, bytes: 0 };
|
|
const holding: {
|
|
path: string;
|
|
handles: string[];
|
|
unchecked: boolean;
|
|
}[] = [];
|
|
const completion = await writeOutputTree(
|
|
await planOutputTree(directory, { operation: EXPORT, field: "directory" }),
|
|
async (tree) => {
|
|
fetched = await fetchHomeFolder(
|
|
computer,
|
|
bundle,
|
|
{
|
|
file: async (file) => {
|
|
// Keep back known secrets and files REA could not check.
|
|
const held = heldBack(file, guard);
|
|
if (held.handles.length > 0 || held.unchecked) {
|
|
holding.push({ path: file.path, ...held });
|
|
return;
|
|
}
|
|
await tree.write(
|
|
file.path,
|
|
Readable.from([file.bytes]),
|
|
{ bytes: file.bytes.length, sha256: file.sha256 },
|
|
signal,
|
|
);
|
|
},
|
|
},
|
|
signal,
|
|
);
|
|
const secret = holding.filter(({ handles }) => handles.length > 0);
|
|
if (secret.length > 0)
|
|
throw secretsRefusal(secret, guard.secrets, directory);
|
|
const unchecked = holding.filter((held) => held.unchecked);
|
|
if (unchecked.length > 0) throw uncheckedRefusal(unchecked, directory);
|
|
},
|
|
{ operation: EXPORT, field: "directory" },
|
|
signal,
|
|
);
|
|
return { ...fetched, completion };
|
|
};
|
|
|
|
/** The export's arguments past the capture and the bundle: the tasks and each one's end, the ends, the install, and withheld state with its consents. */
|
|
const exportArguments = (
|
|
capture: {
|
|
readonly folder: string;
|
|
readonly packageUrl?: string;
|
|
readonly downloadInstall?: DownloadInstall;
|
|
readonly extraPackages: readonly string[];
|
|
readonly signIn: boolean;
|
|
readonly consents: readonly BuildConsent[];
|
|
},
|
|
request: BuildExportRequest,
|
|
files: {
|
|
readonly tasks: readonly string[];
|
|
readonly consent: string;
|
|
readonly withheld: string;
|
|
readonly download: string;
|
|
},
|
|
): string[] => [
|
|
...files.tasks.flatMap((path) => ["--task", path]),
|
|
...request.tasks.flatMap(({ task, end_checkpoint: end }) => [
|
|
"--task-end",
|
|
`${task.name}=${end ?? END_CHECKPOINT}`,
|
|
]),
|
|
...ENDS,
|
|
// The building agent made the seed in this computer for the task.
|
|
"--synthetic-seed",
|
|
"--screens",
|
|
`${capture.folder}/screens`,
|
|
...(request.name === undefined ? [] : ["--name", request.name]),
|
|
...(capture.packageUrl === undefined
|
|
? []
|
|
: ["--package-url", capture.packageUrl]),
|
|
...(capture.downloadInstall === undefined
|
|
? []
|
|
: ["--download-install", files.download]),
|
|
...capture.extraPackages.flatMap((name) => ["--extra-package", name]),
|
|
...(capture.signIn
|
|
? ["--withhold-from", SIGN_IN_START, "--withhold-to", SIGN_IN_END]
|
|
: []),
|
|
...(request.withhold ?? []).flatMap((path) => ["--withhold-path", path]),
|
|
...(capture.consents.length === 0 ? [] : ["--consent", files.consent]),
|
|
...(request.seed_sign_in === true ? ["--seed-withheld"] : []),
|
|
"--withheld-copy",
|
|
files.withheld,
|
|
"--json",
|
|
];
|
|
|
|
/** The consent records as `consent.json` holds them. */
|
|
const consentFile = (consents: readonly BuildConsent[]): string =>
|
|
`${JSON.stringify({ consents }, null, 2)}\n`;
|
|
|
|
/** Keep each withheld file's bytes as a secret, fetched from beside the bundle, reported by the handle that keeps it; none leaves REA otherwise. */
|
|
const keepWithheld = async (
|
|
computer: ControlledComputer,
|
|
folder: string,
|
|
files: readonly WithheldReport[],
|
|
{
|
|
keep,
|
|
signal,
|
|
completed,
|
|
}: {
|
|
readonly keep: (handle: string, bytes: Buffer) => Promise<string>;
|
|
readonly signal: AbortSignal;
|
|
readonly completed: WithheldReport[];
|
|
},
|
|
): Promise<WithheldReport[]> => {
|
|
if (files.length === 0) return [];
|
|
const handles = new Map(files.map((entry) => [entry.file, entry]));
|
|
const kept = new Map<string, string>();
|
|
await fetchHomeFolder(
|
|
computer,
|
|
folder,
|
|
{
|
|
file: async ({ path, bytes }) => {
|
|
const report = handles.get(path);
|
|
if (report !== undefined) {
|
|
const actual = await keep(report.handle, bytes);
|
|
kept.set(path, actual);
|
|
completed.push({ ...report, handle: actual });
|
|
}
|
|
},
|
|
},
|
|
signal,
|
|
);
|
|
if (kept.size !== handles.size)
|
|
throw new Error("The exported bundle did not supply every withheld file.");
|
|
return files.map(({ file, variable, handle }) => ({
|
|
file,
|
|
variable,
|
|
handle: kept.get(file) ?? handle,
|
|
}));
|
|
};
|
|
|
|
/** The stopped capture an export runs on, and what it holds back and records. */
|
|
interface ExportCapture {
|
|
readonly folder: string;
|
|
readonly attempt: number;
|
|
/** The URL a computer downloads the app's `.deb` from, when the capture says apt does not install it. */
|
|
readonly packageUrl?: string;
|
|
readonly downloadInstall?: DownloadInstall;
|
|
/** The packages the request named beside the app, which the bundle installs beside it too. */
|
|
readonly extraPackages: readonly string[];
|
|
/** The secrets no bundle may hold, and where the withheld files' bytes are kept as secrets. */
|
|
readonly secrets: readonly KnownSecret[];
|
|
readonly keep: (handle: string, bytes: Buffer) => Promise<string>;
|
|
/** Whether both sign-in checkpoints were marked, and the consents the person gave. */
|
|
readonly signIn: boolean;
|
|
readonly consents: readonly BuildConsent[];
|
|
/** The checkpoints the capture took, which say whether a task has a near miss its readers also read. */
|
|
readonly checkpoints?: ReadonlySet<string> | undefined;
|
|
}
|
|
|
|
/**
|
|
* Export a stopped capture in the computer with the agent's tasks, seeded at
|
|
* `seed` and each checked at its own end, `end` by default, into a new
|
|
* folder of the work folder, then bring the bundle back and write it into
|
|
* the caller's folder. A task the export refuses is refused with the
|
|
* export's own reason.
|
|
*/
|
|
export const exportBuild = async (
|
|
computer: ControlledComputer,
|
|
capture: ExportCapture,
|
|
request: BuildExportRequest,
|
|
signal: AbortSignal,
|
|
longRuns: BuildComputerLongRuns = new BuildComputerLongRuns(computer),
|
|
): Promise<Result<ExportedBuild, AnalysisError>> => {
|
|
const { folder } = capture;
|
|
const attempt = String(capture.attempt);
|
|
const bundle = `${folder}/bundle-${attempt}`;
|
|
const taskFiles = Object.fromEntries(
|
|
request.tasks.map(({ task }, index) => [
|
|
`${folder}/task-${attempt}-${String(index + 1)}.json`,
|
|
`${JSON.stringify(task, null, 2)}\n`,
|
|
]),
|
|
);
|
|
const consent = `${folder}/consent-${attempt}.json`;
|
|
const withheld = `${bundle}-withheld`;
|
|
const stdout = `${folder}/export-${attempt}.json`;
|
|
const downloadPath = `${folder}/download-install-${attempt}.json`;
|
|
const download = capture.downloadInstall;
|
|
const completed: WithheldReport[] = [];
|
|
let published:
|
|
| {
|
|
readonly directory: string;
|
|
readonly environment: string;
|
|
readonly files: number;
|
|
readonly bytes: number;
|
|
}
|
|
| undefined;
|
|
try {
|
|
const argv = [
|
|
"export-desktop-application",
|
|
`${folder}/capture`,
|
|
bundle,
|
|
...exportArguments(capture, request, {
|
|
tasks: Object.keys(taskFiles),
|
|
consent,
|
|
withheld,
|
|
download: downloadPath,
|
|
}),
|
|
];
|
|
// Each task's readers run at its end and the seed, and at its near miss when the capture took one.
|
|
const timeoutS =
|
|
BUILD_REA_TIMEOUT_S +
|
|
request.tasks.reduce(
|
|
(total, { task }) =>
|
|
total +
|
|
(capture.checkpoints?.has(`near-${task.name}`) === true ? 3 : 2) *
|
|
(task.readers ?? []).reduce(
|
|
(sum, reader) =>
|
|
sum + readerTimeout(reader) + READER_RUNNER_SLACK_S,
|
|
0,
|
|
),
|
|
0,
|
|
);
|
|
const options = {
|
|
stdout,
|
|
files: {
|
|
...taskFiles,
|
|
...(download === undefined
|
|
? {}
|
|
: { [downloadPath]: `${JSON.stringify(download, null, 2)}\n` }),
|
|
...(capture.consents.length === 0
|
|
? {}
|
|
: { [consent]: consentFile(capture.consents) }),
|
|
},
|
|
timeoutS,
|
|
signal,
|
|
};
|
|
const { stdoutBytes: bytes } =
|
|
timeoutS + 60 <= 3600
|
|
? await runRea(computer, argv, options)
|
|
: await longRuns.run(argv, { ...options, work: WORK });
|
|
const output = await readOutput(computer, stdout, bytes, signal);
|
|
if (output.kind === "failure")
|
|
return err(
|
|
output.failure.code === "invalid_request"
|
|
? refused(
|
|
EXPORT,
|
|
output.failure.message,
|
|
output.failure.remediation?.action ??
|
|
"Rewrite the task and export again.",
|
|
)
|
|
: reportedFailure(EXPORT, output),
|
|
);
|
|
const result = exportResultSchema.parse(output.result);
|
|
const written = await writeBundle(
|
|
computer,
|
|
bundle,
|
|
{
|
|
directory: request.directory,
|
|
guard: {
|
|
secrets: capture.secrets,
|
|
},
|
|
},
|
|
signal,
|
|
);
|
|
published = {
|
|
directory: request.directory,
|
|
environment: result.environment,
|
|
files: written.files,
|
|
bytes: written.bytes,
|
|
};
|
|
const outputFailure = desktopOutputFailure(
|
|
{ provider: AGENTENV_PROVIDER, operation: EXPORT, what: "bundle" },
|
|
request.directory,
|
|
written.completion,
|
|
);
|
|
if (outputFailure !== undefined) throw outputFailure;
|
|
const kept = await keepWithheld(computer, withheld, result.withheld_files, {
|
|
keep: capture.keep,
|
|
signal,
|
|
completed,
|
|
});
|
|
return ok({
|
|
directory: request.directory,
|
|
environment: result.environment,
|
|
files: written.files,
|
|
bytes: written.bytes,
|
|
tasks: result.tasks,
|
|
seed_files: result.seed_files,
|
|
scrubbed_files: result.scrubbed_files,
|
|
recompressed_files: result.recompressed_files,
|
|
unscrubbed_files: result.unscrubbed_files,
|
|
limitations: result.limitations,
|
|
withheld_files: kept,
|
|
consented_files: result.consented_files,
|
|
});
|
|
} catch (cause: unknown) {
|
|
const failure =
|
|
cause instanceof AnalysisError
|
|
? cause
|
|
: signal.aborted && cause === signal.reason
|
|
? new AnalysisCancelledError(EXPORT, { cause })
|
|
: adapterError(
|
|
EXPORT,
|
|
redactSecrets(errorText(cause), capture.secrets),
|
|
cause,
|
|
);
|
|
if (published !== undefined)
|
|
failure.retainDiagnostics({
|
|
published_bundle: redactSecretsInJson(
|
|
{ ...published, withheld_files: completed },
|
|
capture.secrets,
|
|
),
|
|
});
|
|
return err(failure);
|
|
}
|
|
};
|