From 0c37c22d512ac55b15fc3ffff6aad09db98f1cd2 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:42:59 -0500 Subject: [PATCH] test: remove copied authentication after command cleanup failure --- scripts/lib/agent-evaluation-cleanup.d.mts | 7 ++++ scripts/lib/agent-evaluation-cleanup.mjs | 24 +++++++++++ scripts/verify-agent-experience.mjs | 24 ++++------- .../agentEvaluationAuthCleanup.test.ts | 42 +++++++++++++++++++ 4 files changed, 82 insertions(+), 15 deletions(-) create mode 100644 scripts/lib/agent-evaluation-cleanup.d.mts create mode 100644 scripts/lib/agent-evaluation-cleanup.mjs create mode 100644 tests/composition/agentEvaluationAuthCleanup.test.ts diff --git a/scripts/lib/agent-evaluation-cleanup.d.mts b/scripts/lib/agent-evaluation-cleanup.d.mts new file mode 100644 index 0000000000..8e99b77794 --- /dev/null +++ b/scripts/lib/agent-evaluation-cleanup.d.mts @@ -0,0 +1,7 @@ +/** Remove copied authentication independently; retain ordinary files until commands release. */ +export function closeAgentEvaluation(options: { + readonly commands: { close(): Promise }; + readonly authPath: string; + readonly evaluationRoot: string; + readonly keepFixtures: boolean; +}): Promise; diff --git a/scripts/lib/agent-evaluation-cleanup.mjs b/scripts/lib/agent-evaluation-cleanup.mjs new file mode 100644 index 0000000000..200167aa41 --- /dev/null +++ b/scripts/lib/agent-evaluation-cleanup.mjs @@ -0,0 +1,24 @@ +import { rm } from "node:fs/promises"; + +import { closeVerifierResources } from "./verifier-cleanup.mjs"; + +/** Remove copied authentication independently; retain ordinary files until commands release. */ +export async function closeAgentEvaluation({ + commands, + authPath, + evaluationRoot, + keepFixtures, +}) { + let commandsReleased = false; + await closeVerifierResources([ + async () => { + await commands.close(); + commandsReleased = true; + }, + () => rm(authPath, { force: true }), + async () => { + if (commandsReleased && !keepFixtures) + await rm(evaluationRoot, { recursive: true, force: true }); + }, + ]); +} diff --git a/scripts/verify-agent-experience.mjs b/scripts/verify-agent-experience.mjs index 7626a4b6a5..f16a489fcd 100644 --- a/scripts/verify-agent-experience.mjs +++ b/scripts/verify-agent-experience.mjs @@ -1,3 +1,4 @@ +import { closeAgentEvaluation } from "./lib/agent-evaluation-cleanup.mjs"; import { importVerifierSource } from "./lib/verifier-source-loader.mjs"; import { createHash } from "node:crypto"; @@ -7,7 +8,6 @@ import { mkdir, mkdtemp, readFile, - rm, writeFile, } from "node:fs/promises"; import { homedir } from "node:os"; @@ -18,10 +18,7 @@ import { tsImport } from "tsx/esm/api"; import { PRODUCT_IDENTITY } from "../dist/identity.js"; import { createAgentEvaluationCommands } from "./lib/agent-evaluation-commands.mjs"; import { assertAgentEvaluationReportPassed } from "./lib/agent-evaluation-report-gate.mjs"; -import { - closeVerifierResources, - withVerifierCleanup, -} from "./lib/verifier-cleanup.mjs"; +import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; const { parseSkillManifestMetadata } = await importVerifierSource( @@ -427,16 +424,13 @@ await withVerifierCleanup( }, // Even a deliberately retained fixture directory must never retain auth. [ - async () => { - // A failed process cleanup retains both its exact owner and its files. - await commands.close(); - await closeVerifierResources([ - () => rm(authPath, { force: true }), - ...(process.env.REA_AGENT_EVAL_KEEP_FIXTURES === "true" - ? [] - : [() => rm(evaluationRoot, { recursive: true, force: true })]), - ]); - }, + () => + closeAgentEvaluation({ + commands, + authPath, + evaluationRoot, + keepFixtures: process.env.REA_AGENT_EVAL_KEEP_FIXTURES === "true", + }), ], ); diff --git a/tests/composition/agentEvaluationAuthCleanup.test.ts b/tests/composition/agentEvaluationAuthCleanup.test.ts new file mode 100644 index 0000000000..73e48a5e6a --- /dev/null +++ b/tests/composition/agentEvaluationAuthCleanup.test.ts @@ -0,0 +1,42 @@ +import { mkdir, readFile, stat, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; + +import { closeAgentEvaluation } from "../../scripts/lib/agent-evaluation-cleanup.mjs"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +it.each([false, true])( + "removes copied auth after command cleanup failure and retains independent failures (auth fails=%s)", + async (authFails) => { + const root = await createTestTempDirectory("rea-evaluation-auth-cleanup-"); + const ordinary = join(root, "retained-observation.txt"); + const auth = join(root, "auth.json"); + await writeFile(ordinary, "retained observation"); + // A directory at the file path gives a real, portable independent rm failure. + if (authFails) await mkdir(auth); + else await writeFile(auth, "inert copied authentication"); + const commandFailure = new Error("owned command did not release"); + const failure = await closeAgentEvaluation({ + commands: { + close: async () => { + throw commandFailure; + }, + }, + authPath: auth, + evaluationRoot: root, + keepFixtures: false, + }).catch((cause: unknown) => cause); + if (!authFails) + await expect(stat(auth)).rejects.toMatchObject({ code: "ENOENT" }); + expect(failure).toBeInstanceOf(AggregateError); + const errors = (failure as AggregateError).errors; + expect(errors[0]).toBe(commandFailure); + expect(errors).toHaveLength(authFails ? 2 : 1); + if (authFails) { + expect(errors[1]).toMatchObject({ code: "ERR_FS_EISDIR", path: auth }); + expect((await stat(auth)).isDirectory()).toBe(true); + } + expect(await readFile(ordinary, "utf8")).toBe("retained observation"); + }, +);