test: remove copied authentication after command cleanup failure

scripts/lib/agent-evaluation-cleanup.d.mts

@@ -0,0 +1,7 @@
+/** Remove copied authentication independently; retain ordinary files until commands release. */
+export function closeAgentEvaluation(options: {
+  readonly commands: { close(): Promise<void> };
+  readonly authPath: string;
+  readonly evaluationRoot: string;
+  readonly keepFixtures: boolean;
+}): Promise<void>;

scripts/lib/agent-evaluation-cleanup.mjs

@@ -0,0 +1,24 @@
+import { rm } from "node:fs/promises";
+
+import { closeVerifierResources } from "./verifier-cleanup.mjs";
+
+/** Remove copied authentication independently; retain ordinary files until commands release. */
+export async function closeAgentEvaluation({
+  commands,
+  authPath,
+  evaluationRoot,
+  keepFixtures,
+}) {
+  let commandsReleased = false;
+  await closeVerifierResources([
+    async () => {
+      await commands.close();
+      commandsReleased = true;
+    },
+    () => rm(authPath, { force: true }),
+    async () => {
+      if (commandsReleased && !keepFixtures)
+        await rm(evaluationRoot, { recursive: true, force: true });
+    },
+  ]);
+}

scripts/verify-agent-experience.mjs

@@ -1,3 +1,4 @@
+import { closeAgentEvaluation } from "./lib/agent-evaluation-cleanup.mjs";
 import { importVerifierSource } from "./lib/verifier-source-loader.mjs";
 
 import { createHash } from "node:crypto";
@@ -7,7 +8,6 @@ import {
   mkdir,
   mkdtemp,
   readFile,
-  rm,
   writeFile,
 } from "node:fs/promises";
 import { homedir } from "node:os";
@@ -18,10 +18,7 @@ import { tsImport } from "tsx/esm/api";
 import { PRODUCT_IDENTITY } from "../dist/identity.js";
 import { createAgentEvaluationCommands } from "./lib/agent-evaluation-commands.mjs";
 import { assertAgentEvaluationReportPassed } from "./lib/agent-evaluation-report-gate.mjs";
-import {
-  closeVerifierResources,
-  withVerifierCleanup,
-} from "./lib/verifier-cleanup.mjs";
+import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
 import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
 
 const { parseSkillManifestMetadata } = await importVerifierSource(
@@ -427,16 +424,13 @@ await withVerifierCleanup(
   },
   // Even a deliberately retained fixture directory must never retain auth.
   [
-    async () => {
-      // A failed process cleanup retains both its exact owner and its files.
-      await commands.close();
-      await closeVerifierResources([
-        () => rm(authPath, { force: true }),
-        ...(process.env.REA_AGENT_EVAL_KEEP_FIXTURES === "true"
-          ? []
-          : [() => rm(evaluationRoot, { recursive: true, force: true })]),
-      ]);
-    },
+    () =>
+      closeAgentEvaluation({
+        commands,
+        authPath,
+        evaluationRoot,
+        keepFixtures: process.env.REA_AGENT_EVAL_KEEP_FIXTURES === "true",
+      }),
   ],
 );
 

tests/composition/agentEvaluationAuthCleanup.test.ts

@@ -0,0 +1,42 @@
+import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { expect, it } from "vitest";
+
+import { closeAgentEvaluation } from "../../scripts/lib/agent-evaluation-cleanup.mjs";
+import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";
+
+it.each([false, true])(
+  "removes copied auth after command cleanup failure and retains independent failures (auth fails=%s)",
+  async (authFails) => {
+    const root = await createTestTempDirectory("rea-evaluation-auth-cleanup-");
+    const ordinary = join(root, "retained-observation.txt");
+    const auth = join(root, "auth.json");
+    await writeFile(ordinary, "retained observation");
+    // A directory at the file path gives a real, portable independent rm failure.
+    if (authFails) await mkdir(auth);
+    else await writeFile(auth, "inert copied authentication");
+    const commandFailure = new Error("owned command did not release");
+    const failure = await closeAgentEvaluation({
+      commands: {
+        close: async () => {
+          throw commandFailure;
+        },
+      },
+      authPath: auth,
+      evaluationRoot: root,
+      keepFixtures: false,
+    }).catch((cause: unknown) => cause);
+    if (!authFails)
+      await expect(stat(auth)).rejects.toMatchObject({ code: "ENOENT" });
+    expect(failure).toBeInstanceOf(AggregateError);
+    const errors = (failure as AggregateError).errors;
+    expect(errors[0]).toBe(commandFailure);
+    expect(errors).toHaveLength(authFails ? 2 : 1);
+    if (authFails) {
+      expect(errors[1]).toMatchObject({ code: "ERR_FS_EISDIR", path: auth });
+      expect((await stat(auth)).isDirectory()).toBe(true);
+    }
+    expect(await readFile(ordinary, "utf8")).toBe("retained observation");
+  },
+);