Files
reverse-engineering-anything/src/desktop/AgentEnvSolveHelper.test.ts
T

277 lines
9.8 KiB
TypeScript

import {
access,
chmod,
mkdir,
readdir,
rmdir,
writeFile,
} from "node:fs/promises";
import { join } from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import { describe, expect, it } from "vitest";
import { fakeAgentEnvSolves } from "../../tests/fixtures/fakeAgentEnvSolves.js";
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js";
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
import { cleanupOwnedProcessGroup } from "../process/ProcessOwnership.js";
import { spawnOwnedProviderProcess } from "../process/ProviderProcess.js";
import {
runSolveHelper,
type SolveCleanupOwner,
} from "./AgentEnvSolveHelper.js";
const SOLVE = { outcome: "passed" as const, checks: {}, turns: 3, usd: 0.1 };
const until = async (path: string, signal: AbortSignal) => {
for (;;) {
if (
await access(path).then(
() => true,
() => false,
)
)
return;
await delay(50, undefined, { signal });
}
};
const options = async (environment: Readonly<Record<string, string>>) => ({
bundle: await createTestTempDirectory("solve-bundle-"),
task: "add-eggs",
model: "m",
agent: "claude-code",
environment: { ...process.env, ...environment },
operation: "measure_build_task",
});
describe("runSolveHelper", () => {
it("stops a cancelled solve whose helper ignores SIGTERM within its stop grace", async () => {
const runner = await fakeAgentEnvSolves([SOLVE]);
const environment = { ...runner.environment, FAKE_SOLVE_HANGS: "1" };
const stop = new AbortController();
const polling = new AbortController();
const retained: SolveCleanupOwner[] = [];
const started = until(
join(runner.environment.FAKE_STATE ?? "", "solves.jsonl"),
polling.signal,
);
void started.catch(() => undefined);
const solved = runSolveHelper({
...(await options(environment)),
signal: stop.signal,
retainCleanup: (owner) => retained.push(owner),
// The solve deadline outlasts the test: only the stop grace ends it.
timing: { solveMs: 3_600_000, stopMs: 300 },
});
// agentenv keeps a sandbox's folder until its teardown is confirmed, so the
// emptied private root shows nothing is left up: only the cancellation remains.
try {
await awaitAcknowledgement(started, solved, "solve helper start");
stop.abort(new Error("cancelled by the test"));
const failure: unknown = await solved.then(
() => undefined,
(error: unknown) => error,
);
expect(failure).toBeInstanceOf(Error);
expect((failure as Error).message).toBe("cancelled by the test");
expect(failure).not.toHaveProperty("cleanupIncomplete", true);
expect(retained).toHaveLength(0);
} finally {
polling.abort();
stop.abort();
await Promise.allSettled([started, solved]);
}
});
it("records a helper stopped at its deadline without a report as a failed solve of unknown cost, its cleanup known", async () => {
const runner = await fakeAgentEnvSolves([SOLVE]);
const environment = { ...runner.environment, FAKE_SOLVE_HANGS: "1" };
const retained: SolveCleanupOwner[] = [];
const report = await runSolveHelper({
...(await options(environment)),
signal: new AbortController().signal,
retainCleanup: (owner) => retained.push(owner),
timing: { solveMs: 1_000, stopMs: 300 },
});
expect(report).toMatchObject({
outcome: "failed",
usd: null,
turns: null,
checks: [],
cleanup: { unknown: false, sandboxes: [] },
});
expect(report.error).toMatch(/stopped \(SIG[A-Z]+\) without a report/u);
expect(retained).toHaveLength(0);
});
it("keeps a helper whose cleanup failed, so closing can retry it, and reports what is left", async () => {
const runner = await fakeAgentEnvSolves([SOLVE]);
const retained: SolveCleanupOwner[] = [];
let cleanups = 0;
const solved = runSolveHelper({
...(await options(runner.environment)),
signal: new AbortController().signal,
retainCleanup: (owner) => retained.push(owner),
spawn: async (spawnOptions) => {
const owned = await spawnOwnedProviderProcess(spawnOptions);
return {
...owned,
cleanup: async () => {
cleanups += 1;
const cleaned = await cleanupOwnedProcessGroup(owned.ownership);
return cleanups === 1
? { cleaned: false as const, reason: "a refusal by the test" }
: cleaned;
},
};
},
});
const failure = await solved.catch((cause: unknown) => cause);
expect(failure).toBeInstanceOf(ProviderCleanupError);
expect(failure).toMatchObject({
report: { outcome: "passed", turns: 3, usd: 0.1 },
diagnostics: { solve_report: { outcome: "passed", turns: 3, usd: 0.1 } },
});
expect(retained).toHaveLength(1);
expect(await retained[0]?.stop()).toMatchObject({
status: "verified-cleanup",
});
retained[0]?.dispose();
});
});
it("retains a reported sandbox on failed removal, then retries that exact sandbox", async () => {
const runner = await fakeAgentEnvSolves([SOLVE]);
const root = await createTestTempDirectory("solve-sandbox-cleanup-");
const docker = join(root, "docker");
const permit = join(root, "permit");
await writeFile(
docker,
`#!/bin/sh\nif [ "$1" = ps ]; then echo owned-container; exit 0; fi\n[ -f "$FAKE_CLEANUP_PERMIT" ]\n`,
);
await chmod(docker, 0o755);
const retained: SolveCleanupOwner[] = [];
const failure = await runSolveHelper({
...(await options({
...runner.environment,
REA_DOCKER_COMMAND: docker,
FAKE_CLEANUP_PERMIT: permit,
FAKE_SOLVE_STILL_UP: '["local-owned"]',
})),
signal: new AbortController().signal,
retainCleanup: (owner) => retained.push(owner),
}).catch((cause: unknown) => cause);
expect(failure).toBeInstanceOf(ProviderCleanupError);
if (!(failure instanceof ProviderCleanupError)) throw failure;
expect(failure).toMatchObject({
cleanupResources: ["agentenv sandbox local-owned"],
report: { outcome: "passed", turns: 3, usd: 0.1 },
});
expect(retained).toHaveLength(1);
await writeFile(permit, "allowed");
expect(await retained[0]?.stop()).toMatchObject({
status: "verified-cleanup",
});
expect(failure.cleanupIncomplete).toBe(false);
expect(failure.cleanupResources).toEqual([]);
retained[0]?.dispose();
});
it("retains a completed report if cancellation arrives during final cleanup", async () => {
const runner = await fakeAgentEnvSolves([SOLVE]);
const controller = new AbortController();
const reason = { stage: "final cleanup" };
const failure = await runSolveHelper({
...(await options(runner.environment)),
signal: controller.signal,
retainCleanup: () => undefined,
spawn: async (spawnOptions) => {
const owned = await spawnOwnedProviderProcess(spawnOptions);
return {
...owned,
cleanup: async () => {
const result = await cleanupOwnedProcessGroup(owned.ownership);
controller.abort(reason);
return result;
},
};
},
}).catch((cause: unknown) => cause);
expect(failure).toMatchObject({
_tag: "AnalysisCancelledError",
cause: reason,
report: { outcome: "passed", turns: 3, usd: 0.1 },
});
});
it("retains an unrecognized sandbox root entry and retries after it is repaired", async () => {
const runner = await fakeAgentEnvSolves([SOLVE]);
const parent = await createTestTempDirectory("solve-owned-roots-");
const retained: SolveCleanupOwner[] = [];
const failure = await runSolveHelper({
...(await options({
...runner.environment,
AGENT_ENV_LOCAL_SANDBOX_DIR: parent,
FAKE_SOLVE_ROOT_ENTRY: "unexpected",
})),
signal: new AbortController().signal,
retainCleanup: (owner) => retained.push(owner),
}).catch((cause: unknown) => cause);
expect(failure).toBeInstanceOf(ProviderCleanupError);
expect(retained).toHaveLength(1);
const roots = await readdir(parent);
expect(roots).toHaveLength(1);
const root = join(parent, roots[0] ?? "");
expect(await readdir(root)).toEqual(["unexpected"]);
await rmdir(join(root, "unexpected"));
expect(await retained[0]?.stop()).toMatchObject({
status: "verified-cleanup",
});
expect(await readdir(parent)).toEqual([]);
if (!(failure instanceof ProviderCleanupError)) throw failure;
expect(failure.cleanupIncomplete).toBe(false);
retained[0]?.dispose();
});
it("retains root cleanup and the spawn failure when admission fails", async () => {
const runner = await fakeAgentEnvSolves([SOLVE]);
const parent = await createTestTempDirectory("solve-spawn-root-");
const retained: SolveCleanupOwner[] = [];
const failure = await runSolveHelper({
...(await options({
...runner.environment,
AGENT_ENV_LOCAL_SANDBOX_DIR: parent,
})),
signal: new AbortController().signal,
retainCleanup: (owner) => retained.push(owner),
spawn: async (spawnOptions) => {
const root = spawnOptions.env?.AGENT_ENV_LOCAL_SANDBOX_DIR;
if (root === undefined) throw new Error("missing root");
await mkdir(join(root, "unexpected"));
throw new Error("admission failed");
},
}).catch((cause: unknown) => cause);
expect(failure).toBeInstanceOf(ProviderCleanupError);
if (!(failure instanceof ProviderCleanupError)) throw failure;
expect(failure.diagnostics).toMatchObject({
spawn_error: "admission failed",
});
expect(retained).toHaveLength(1);
const roots = await readdir(parent);
await rmdir(join(parent, roots[0] ?? "", "unexpected"));
expect(await retained[0]?.stop()).toMatchObject({
status: "verified-cleanup",
});
expect(failure.cleanupIncomplete).toBe(false);
expect(failure.cleanupResources).toEqual([]);
retained[0]?.dispose();
});