Files
reverse-engineering-anything/src/application/desktop/DesktopBuildExport.ts
T
morluto 5f718ce30e refactor(kernel): give error-message, byte-hex and SHA-256 hex helpers one owner in a new src/kernel layer (#11542)
* refactor(kernel): give error-to-message text one owner in a new src/kernel layer

* fix(kernel): keep the regex worker's own error text inside its isolated source

* refactor(kernel): exclude copies that run in another realm from the error-message guard by name

The regex worker's source string and the page.evaluate fetch in PlaywrightScenarioRequest cannot import errorMessage. The guard now names them, so the counted baseline does not invite a later slice to rewrite them.

* refactor(kernel): migrate the long tail of inline error-message copies to errorMessage (#11552)

Every provider and adapter folder with six or fewer copies now calls errorMessage from src/kernel. Local helpers whose whole body was the copy (failureMessage, gdbErrorMessage, message, describe, reasonOf, recoveryFailureMessage and four local errorMessage definitions) are removed and their call sites call the owner.

Refs #11548

* refactor(kernel): give byte-to-hex formatting one owner in src/kernel (#11560)

byteToHex returns the two lowercase hexadecimal digits of one byte. All eight inline copies (native, contracts, domain firmware and desktop, pcap) call it, keeping their own 0x, % or \x notation, and a counted guard with an empty baseline keeps new copies out.

Refs #11548

* refactor(kernel): give one-shot SHA-256 hex digests one owner in src/kernel (#11562)

sha256Hex digests one in-memory value. A counted guard records the one-shot createHash("sha256").update(x).digest("hex") chains (115 in 101 files) and the first slice migrates domain, contracts and process (35 copies), removing local wrappers (sha256, sha256Text, hash) and the exported sha256Text. Incremental and streamed hashing is not counted.

Refs #11548

* refactor(kernel): migrate the application layer's inline error-message copies to errorMessage (#11563)

All 69 copies in src/application call errorMessage from src/kernel. Local helpers whose whole body was the copy (errorDetail, causeReason and five local errorMessage definitions) are removed and their call sites call the owner.

Refs #11548

* refactor(kernel): migrate the native provider's inline error-message copies to errorMessage (#11567)

All 49 copies in src/native call errorMessage from src/kernel. The local errorReason helpers in LldbCallTracer and LldbRetainedObservations, whose whole body was the copy, are removed and their call sites call the owner.

Refs #11548

* refactor(kernel): migrate the remaining inline error-message copies to errorMessage (#11569)
2026-11-16 07:05:30 +08:00

501 lines
17 KiB
TypeScript

import { Readable } from "node:stream";
import { z } from "zod";
import type { DownloadInstall } from "../../domain/desktop/appDownloadInstall.js";
import {
READER_RUNNER_SLACK_S,
readerTimeout,
} from "../../domain/desktop/appTaskReaders.js";
import { BuildComputerLongRuns } from "../../desktop/BuildComputerLongRuns.js";
import { AnalysisError } from "../../domain/analysisErrorBase.js";
import { cancelledBy, carriedReason } from "../../domain/analysisErrorCore.js";
import type { desktopTaskSchema } from "../../domain/desktop/appTask.js";
import { counted } from "../../domain/desktop/appExportUnowned.js";
import { END_CHECKPOINT } from "../../domain/desktop/buildCapture.js";
import {
SIGN_IN_END,
SIGN_IN_START,
type BuildConsent,
} from "../../domain/desktop/appWithheld.js";
import {
gzipInside,
isUncheckedGzip,
MAX_GUNZIPPED_BYTES,
redactSecrets,
redactSecretsInJson,
secretsIn,
type KnownSecret,
} from "../../domain/desktop/buildSecrets.js";
import { jsonValueSchema, type JsonValue } from "../../domain/jsonValue.js";
import { err, ok, type Result } from "../../domain/result.js";
import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js";
import {
BUILD_REA_TIMEOUT_S,
fetchHomeFolder,
type ControlledComputer,
} from "../../desktop/BuildComputerController.js";
import {
adapterError,
ENDS,
readOutput,
refused,
reportedFailure,
runRea,
WORK,
} from "./DesktopBuildCommands.js";
import {
planOutputTree,
writeOutputTree,
desktopOutputFailure,
type DesktopOutputCompletion,
} from "./DesktopOutputTree.js";
import { errorMessage } from "../../kernel/errorMessage.js";
const EXPORT = "export_build_environment";
const exportResultSchema = z.looseObject({
environment: z.string(),
seed_files: z.array(z.string()),
scrubbed_files: z.array(z.string()),
// An export from before gzip seeds were listed apart has none.
recompressed_files: z.array(z.string()).default([]),
unscrubbed_files: z.array(z.string()),
limitations: z.array(z.string()),
tasks: z.array(jsonValueSchema),
// An export from before withheld files were listed has none.
withheld_files: z
.array(
z.looseObject({
file: z.string(),
variable: z.string(),
handle: z.string(),
}),
)
.default([]),
consented_files: z.array(z.string()).default([]),
});
/** One task the export takes, with the checkpoint its end state is read at. */
interface BuildExportTask {
readonly task: z.output<typeof desktopTaskSchema>;
readonly end_checkpoint?: string | undefined;
}
/** What the agent asks the export for: its tasks, the folder here, optionally the environment's name, files to withhold, and whether to seed withheld state with the person's consent. */
export interface BuildExportRequest {
readonly tasks: readonly BuildExportTask[];
readonly directory: string;
readonly name?: string | undefined;
readonly withhold?: readonly string[] | undefined;
readonly seed_sign_in?: boolean | undefined;
}
/** A withheld file as the export reports it: the variable a deploy reads and the handle of the secret file REA keeps its bytes in. */
type WithheldReport = {
readonly file: string;
readonly variable: string;
readonly handle: string;
};
/** An exported bundle brought back to this host. */
export type ExportedBuild = {
readonly directory: string;
readonly environment: string;
readonly files: number;
readonly bytes: number;
readonly tasks: JsonValue[];
readonly seed_files: string[];
readonly scrubbed_files: string[];
readonly recompressed_files: string[];
readonly unscrubbed_files: string[];
readonly limitations: string[];
readonly withheld_files: WithheldReport[];
readonly consented_files: string[];
};
/** Explicitly supplied secrets the bundle may not hold. */
interface BundleGuard {
readonly secrets: readonly KnownSecret[];
}
/** Why a fetched file is held back: known secrets or an unchecked gzip. */
const heldBack = (
file: { readonly path: string; readonly bytes: Buffer },
guard: BundleGuard,
) => {
// A gzip file is decompressed once for the secret checks.
const inner = guard.secrets.length === 0 ? undefined : gzipInside(file.bytes);
return {
handles: [
...new Set([
...secretsIn(Buffer.from(file.path), guard.secrets),
...secretsIn(file.bytes, guard.secrets, inner),
]),
],
unchecked: guard.secrets.length > 0 && isUncheckedGzip(file.bytes, inner),
};
};
/** The refusal of gzip files too large for REA to check for known secrets. */
const uncheckedRefusal = (
holding: readonly { readonly path: string }[],
directory: string,
) =>
refused(
EXPORT,
`REA could not check inside ${counted(holding.map(({ path }) => path))}: each is a gzip file that expands past ${String(MAX_GUNZIPPED_BYTES / 1024 / 1024)} MiB, the most REA decompresses to look for known secrets, so REA wrote none of the bundle to ${directory}.`,
"Keep each such file out of the seed: delete it in the app before marking seed, or capture again on a new build computer without it, then export again.",
);
/** The refusal of a bundle whose files hold secrets: each secret's handle and the files, the values in their names redacted. */
const secretsRefusal = (
holding: readonly { readonly path: string; readonly handles: string[] }[],
secrets: readonly KnownSecret[],
directory: string,
) => {
const handles = [...new Set(holding.flatMap(({ handles: found }) => found))];
const found = secrets.filter(({ handle }) => handles.includes(handle));
return refused(
EXPORT,
`The bundle holds the secret ${handles.join(", ")} in ${counted(holding.map(({ path }) => redactSecrets(path, found)))}, so REA wrote none of it to ${directory}: no bundle may hold a secret.`,
"Keep the secret out of the seed: delete the file that holds it in the app before marking seed, or capture again on a new build computer without it, then export again.",
);
};
/**
* Bring the bundle back straight into the caller's new or empty folder, each
* checked answer written before the next is fetched; a failure leaves no
* partial folder.
*/
const writeBundle = async (
computer: ControlledComputer,
bundle: string,
{
directory,
guard,
}: { readonly directory: string; readonly guard: BundleGuard },
signal: AbortSignal,
): Promise<{
readonly files: number;
readonly bytes: number;
readonly completion: DesktopOutputCompletion;
}> => {
let fetched = { files: 0, bytes: 0 };
const holding: {
path: string;
handles: string[];
unchecked: boolean;
}[] = [];
const completion = await writeOutputTree(
await planOutputTree(directory, { operation: EXPORT, field: "directory" }),
async (tree) => {
fetched = await fetchHomeFolder(
computer,
bundle,
{
file: async (file) => {
// Keep back known secrets and files REA could not check.
const held = heldBack(file, guard);
if (held.handles.length > 0 || held.unchecked) {
holding.push({ path: file.path, ...held });
return;
}
await tree.write(
file.path,
Readable.from([file.bytes]),
{ bytes: file.bytes.length, sha256: file.sha256 },
signal,
);
},
},
signal,
);
const secret = holding.filter(({ handles }) => handles.length > 0);
if (secret.length > 0)
throw secretsRefusal(secret, guard.secrets, directory);
const unchecked = holding.filter((held) => held.unchecked);
if (unchecked.length > 0) throw uncheckedRefusal(unchecked, directory);
},
{ operation: EXPORT, field: "directory" },
signal,
);
return { ...fetched, completion };
};
/** The export's arguments past the capture and the bundle: the tasks and each one's end, the ends, the install, and withheld state with its consents. */
const exportArguments = (
capture: {
readonly folder: string;
readonly packageUrl?: string;
readonly downloadInstall?: DownloadInstall;
readonly extraPackages: readonly string[];
readonly signIn: boolean;
readonly consents: readonly BuildConsent[];
},
request: BuildExportRequest,
files: {
readonly tasks: readonly string[];
readonly consent: string;
readonly withheld: string;
readonly download: string;
},
): string[] => [
...files.tasks.flatMap((path) => ["--task", path]),
...request.tasks.flatMap(({ task, end_checkpoint: end }) => [
"--task-end",
`${task.name}=${end ?? END_CHECKPOINT}`,
]),
...ENDS,
// The building agent made the seed in this computer for the task.
"--synthetic-seed",
"--screens",
`${capture.folder}/screens`,
...(request.name === undefined ? [] : ["--name", request.name]),
...(capture.packageUrl === undefined
? []
: ["--package-url", capture.packageUrl]),
...(capture.downloadInstall === undefined
? []
: ["--download-install", files.download]),
...capture.extraPackages.flatMap((name) => ["--extra-package", name]),
...(capture.signIn
? ["--withhold-from", SIGN_IN_START, "--withhold-to", SIGN_IN_END]
: []),
...(request.withhold ?? []).flatMap((path) => ["--withhold-path", path]),
...(capture.consents.length === 0 ? [] : ["--consent", files.consent]),
...(request.seed_sign_in === true ? ["--seed-withheld"] : []),
"--withheld-copy",
files.withheld,
"--json",
];
/** The consent records as `consent.json` holds them. */
const consentFile = (consents: readonly BuildConsent[]): string =>
`${JSON.stringify({ consents }, null, 2)}\n`;
/** Keep each withheld file's bytes as a secret, fetched from beside the bundle, reported by the handle that keeps it; none leaves REA otherwise. */
const keepWithheld = async (
computer: ControlledComputer,
folder: string,
files: readonly WithheldReport[],
{
keep,
signal,
completed,
}: {
readonly keep: (handle: string, bytes: Buffer) => Promise<string>;
readonly signal: AbortSignal;
readonly completed: WithheldReport[];
},
): Promise<WithheldReport[]> => {
if (files.length === 0) return [];
const handles = new Map(files.map((entry) => [entry.file, entry]));
const kept = new Map<string, string>();
await fetchHomeFolder(
computer,
folder,
{
file: async ({ path, bytes }) => {
const report = handles.get(path);
if (report !== undefined) {
const actual = await keep(report.handle, bytes);
kept.set(path, actual);
completed.push({ ...report, handle: actual });
}
},
},
signal,
);
if (kept.size !== handles.size)
throw new Error("The exported bundle did not supply every withheld file.");
return files.map(({ file, variable, handle }) => ({
file,
variable,
handle: kept.get(file) ?? handle,
}));
};
/** The stopped capture an export runs on, and what it holds back and records. */
interface ExportCapture {
readonly folder: string;
readonly attempt: number;
/** The URL a computer downloads the app's `.deb` from, when the capture says apt does not install it. */
readonly packageUrl?: string;
readonly downloadInstall?: DownloadInstall;
/** The packages the request named beside the app, which the bundle installs beside it too. */
readonly extraPackages: readonly string[];
/** The secrets no bundle may hold, and where the withheld files' bytes are kept as secrets. */
readonly secrets: readonly KnownSecret[];
readonly keep: (handle: string, bytes: Buffer) => Promise<string>;
/** Whether both sign-in checkpoints were marked, and the consents the person gave. */
readonly signIn: boolean;
readonly consents: readonly BuildConsent[];
/** The checkpoints the capture took, which say whether a task has a near miss its readers also read. */
readonly checkpoints?: ReadonlySet<string> | undefined;
}
/**
* Export a stopped capture in the computer with the agent's tasks, seeded at
* `seed` and each checked at its own end, `end` by default, into a new
* folder of the work folder, then bring the bundle back and write it into
* the caller's folder. A task the export refuses is refused with the
* export's own reason.
*/
export const exportBuild = async (
computer: ControlledComputer,
capture: ExportCapture,
request: BuildExportRequest,
signal: AbortSignal,
longRuns: BuildComputerLongRuns = new BuildComputerLongRuns(computer),
): Promise<Result<ExportedBuild, AnalysisError>> => {
const { folder } = capture;
const attempt = String(capture.attempt);
const bundle = `${folder}/bundle-${attempt}`;
const taskFiles = Object.fromEntries(
request.tasks.map(({ task }, index) => [
`${folder}/task-${attempt}-${String(index + 1)}.json`,
`${JSON.stringify(task, null, 2)}\n`,
]),
);
const consent = `${folder}/consent-${attempt}.json`;
const withheld = `${bundle}-withheld`;
const stdout = `${folder}/export-${attempt}.json`;
const downloadPath = `${folder}/download-install-${attempt}.json`;
const download = capture.downloadInstall;
const completed: WithheldReport[] = [];
let published:
| {
readonly directory: string;
readonly environment: string;
readonly files: number;
readonly bytes: number;
}
| undefined;
try {
// Admit the host folder by the rule the bundle is written under before
// running anything in the computer, so an unusable folder is refused
// before the in-computer export runs rather than when its bundle returns.
await planOutputTree(request.directory, {
operation: EXPORT,
field: "directory",
});
const argv = [
"export-desktop-application",
`${folder}/capture`,
bundle,
...exportArguments(capture, request, {
tasks: Object.keys(taskFiles),
consent,
withheld,
download: downloadPath,
}),
];
// Each task's readers run at its end and the seed, and at its near miss when the capture took one.
const timeoutS =
BUILD_REA_TIMEOUT_S +
request.tasks.reduce(
(total, { task }) =>
total +
(capture.checkpoints?.has(`near-${task.name}`) === true ? 3 : 2) *
(task.readers ?? []).reduce(
(sum, reader) =>
sum + readerTimeout(reader) + READER_RUNNER_SLACK_S,
0,
),
0,
);
const options = {
stdout,
files: {
...taskFiles,
...(download === undefined
? {}
: { [downloadPath]: `${JSON.stringify(download, null, 2)}\n` }),
...(capture.consents.length === 0
? {}
: { [consent]: consentFile(capture.consents) }),
},
timeoutS,
signal,
};
const { stdoutBytes: bytes } =
timeoutS + 60 <= 3600
? await runRea(computer, argv, options)
: await longRuns.run(argv, { ...options, work: WORK });
const output = await readOutput(computer, stdout, bytes, signal);
if (output.kind === "failure")
return err(
output.failure.code === "invalid_request"
? refused(
EXPORT,
output.failure.message,
output.failure.remediation?.action ??
"Rewrite the task and export again.",
)
: reportedFailure(EXPORT, output),
);
const result = exportResultSchema.parse(output.result);
const written = await writeBundle(
computer,
bundle,
{
directory: request.directory,
guard: {
secrets: capture.secrets,
},
},
signal,
);
published = {
directory: request.directory,
environment: result.environment,
files: written.files,
bytes: written.bytes,
};
const outputFailure = desktopOutputFailure(
{ provider: AGENTENV_PROVIDER, operation: EXPORT, what: "bundle" },
request.directory,
written.completion,
);
if (outputFailure !== undefined) throw outputFailure;
const kept = await keepWithheld(computer, withheld, result.withheld_files, {
keep: capture.keep,
signal,
completed,
});
return ok({
directory: request.directory,
environment: result.environment,
files: written.files,
bytes: written.bytes,
tasks: result.tasks,
seed_files: result.seed_files,
scrubbed_files: result.scrubbed_files,
recompressed_files: result.recompressed_files,
unscrubbed_files: result.unscrubbed_files,
limitations: result.limitations,
withheld_files: kept,
consented_files: result.consented_files,
});
} catch (cause: unknown) {
const failure =
cause instanceof AnalysisError
? cause
: signal.aborted && cause === signal.reason
? cancelledBy(EXPORT, carriedReason(cause))
: adapterError(
EXPORT,
redactSecrets(errorMessage(cause), capture.secrets),
cause,
);
if (published !== undefined)
failure.retainDiagnostics({
published_bundle: redactSecretsInJson(
{ ...published, withheld_files: completed },
capture.secrets,
),
});
return err(failure);
}
};