fix(verify): retain Go fixtures until owned cleanup completes (#10309)

scripts/verify-go-binary.mjs

@@ -1,22 +1,23 @@
 #!/usr/bin/env node
 import assert from "node:assert/strict";
 import { isUtf8 } from "node:buffer";
-import { execFile } from "node:child_process";
 import { createHash } from "node:crypto";
 import { lstat, mkdir, readFile, writeFile } from "node:fs/promises";
 import { isAbsolute, join } from "node:path";
 import { fileURLToPath } from "node:url";
-import { promisify } from "node:util";
 import { Client } from "@modelcontextprotocol/client";
-import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
 import { parseEvidence } from "../dist/domain/evidence.js";
 import { goBinarySchema } from "../dist/domain/go/goBinary.js";
 import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
+import { AnalysisCleanupLease } from "../dist/artifacts/AnalysisCleanupLease.js";
 import { artifactMcpCall } from "./lib/artifact-e2e.mjs";
 import { requireMcpOutput } from "./lib/mcp-verifier-results.mjs";
 import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
+import { createVerifierCommands } from "./lib/verify-commands.mjs";
+import { createOwnedMcpStdioTransport } from "./lib/owned-mcp-stdio.mjs";
+import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
 
-const execute = promisify(execFile);
+const commands = await createVerifierCommands();
 const compiler = process.env.REA_VERIFY_GO_COMMAND ?? "go";
 if (process.env.REA_VERIFY_GO_COMMAND !== undefined && !isAbsolute(compiler))
   throw new Error(
@@ -51,13 +52,16 @@ const goEnvironment = {
   // Go's own verifier seam isolates telemetry; no global configuration is changed.
   TEST_TELEMETRY_DIR: join(root.path, "telemetry"),
 };
-const client = new Client({ name: "go-binary-verifier", version: "1" });
-const transport = new StdioClientTransport({
-  command: process.execPath,
-  args: [entrypoint, "mcp"],
-  env: environment,
-  stderr: "pipe",
-});
+let client;
+let transport;
+const cleanup = new AnalysisCleanupLease(async () => {
+  await closeVerifierResources([
+    () => client?.close(),
+    () => transport?.close(),
+    () => commands.close(),
+  ]);
+  await root.close();
+}, [root.path]);
 const targets = [
   ["linux", "amd64", "elf", 64, "little"],
   ["linux", "386", "elf", 32, "little"],
@@ -73,10 +77,19 @@ const verified = [];
 const omittedCoverage = [];
 const failures = [];
 try {
+  client = new Client({ name: "go-binary-verifier", version: "1" });
+  transport = await createOwnedMcpStdioTransport({
+    command: process.execPath,
+    arguments: [entrypoint, "mcp"],
+    hostEnvironment: environment,
+    cwd: process.cwd(),
+    expectedCommand: null,
+    sweepTokenOwnedProcesses: true,
+  });
   await prepareFixture();
   try {
     compilerVersion = (
-      await execute(compiler, ["version"], {
+      await commands.exec(compiler, ["version"], {
         cwd: root.path,
         env: goEnvironment,
         timeout: 10_000,
@@ -185,7 +198,7 @@ try {
     const path = join(root.path, "linux-amd64-byte-strings");
     const expected = JSON.parse(
       (
-        await execute(oracle, ["--build-byte-fixture", compiler, path], {
+        await commands.exec(oracle, ["--build-byte-fixture", compiler, path], {
           cwd: sourceRoot,
           env: { ...goEnvironment, GOOS: "linux", GOARCH: "amd64" },
           timeout: 120_000,
@@ -198,21 +211,16 @@ try {
 } catch (cause) {
   failures.push(cause);
 } finally {
-  for (const close of [
-    () => client.close(),
-    () => transport.close(),
-    () => root.close(),
-  ]) {
-    try {
-      await close();
-    } catch (cause) {
-      failures.push(cause);
-    }
+  try {
+    await cleanup.release();
+  } catch (cause) {
+    failures.push(cause);
   }
 }
 const verifier = await completeVerifierRun(run);
 try {
-  await assert.rejects(lstat(root.path), { code: "ENOENT" });
+  if (cleanup.released)
+    await assert.rejects(lstat(root.path), { code: "ENOENT" });
   if (verifier.process_lineage.status === "verified")
     assert.deepEqual(verifier.process_lineage.descendants, []);
 } catch (cause) {
@@ -226,15 +234,21 @@ if (failures.length > 0) {
         compiler: compilerVersion,
         verified,
         omitted_coverage: omittedCoverage,
+        retained_root: cleanup.released ? null : root.path,
         verifier,
       },
       null,
       2,
     ),
   );
-  throw new AggregateError(
-    failures,
-    "Real Go metadata verification failed; cleanup failures are retained.",
+  throw Object.assign(
+    new AggregateError(
+      failures,
+      cleanup.released
+        ? "Real Go metadata verification failed; cleanup failures are retained."
+        : `Real Go metadata verification failed; cleanup is incomplete at ${root.path}.`,
+    ),
+    { cleanupLeases: cleanup.released ? [] : [cleanup] },
   );
 }
 console.log(
@@ -256,7 +270,7 @@ console.log(
 );
 
 async function go(args, overrides = {}) {
-  return execute(compiler, args, {
+  return commands.exec(compiler, args, {
     cwd: sourceRoot,
     env: { ...goEnvironment, ...overrides },
     timeout: 120_000,
@@ -356,7 +370,7 @@ async function inspect(mode, path, includeStringLiterals = false) {
       ...(includeStringLiterals ? ["--include-string-literals"] : []),
       "--json",
     ];
-    const { stdout } = await execute(process.execPath, args, {
+    const { stdout } = await commands.exec(process.execPath, args, {
       env: environment,
       timeout: 30_000,
       maxBuffer: 16 * 1024 * 1024,
@@ -424,7 +438,7 @@ async function verify(
     byteExpected ??
     JSON.parse(
       (
-        await execute(oracle, [oraclePath], {
+        await commands.exec(oracle, [oraclePath], {
           timeout: 10_000,
           maxBuffer: 1024 * 1024,
         })

tests/boundary/process/goVerifierLifetime.test.ts

@@ -0,0 +1,84 @@
+import { mkdir } from "node:fs/promises";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { expect } from "vitest";
+import { z } from "zod";
+
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+import { processTest } from "../../support/process/processFixture.js";
+
+const harness = fileURLToPath(
+  new URL("../../fixtures/goVerifierLifetime.mjs", import.meta.url),
+);
+const outcomeSchema = z.object({
+  messages: z.array(z.string()),
+  retained: z.string().nullable(),
+  existedAtFailure: z.boolean(),
+  removedAfterRetry: z.boolean(),
+  rootRemoved: z.boolean(),
+  beforeRetry: z.array(z.string()),
+  events: z.array(z.string()),
+});
+
+processTest
+  .skipIf(process.platform === "win32")
+  .for([
+    "settled",
+    "client-constructor",
+    "transport-constructor",
+    "handshake",
+    "client-close",
+    "transport-close",
+    "client-close+transport-close",
+    "commands-close",
+    "root-close",
+  ])(
+  "owns the Go verifier fixture through %s and cleanup retry",
+  async (scenario, { processes }) => {
+    const directory = await createTestTempDirectory("rea-go-lifetime-");
+    const temporary = join(directory, "temporary");
+    await mkdir(temporary);
+    const result = await processes.run(
+      process.execPath,
+      [harness, scenario, directory],
+      {
+        env: {
+          ...process.env,
+          TMPDIR: temporary,
+          TMP: temporary,
+          TEMP: temporary,
+        },
+        timeoutMs: 40_000,
+      },
+    );
+    expect(result.exitCode, result.stderr + result.stdout).toBe(0);
+    const outcome = outcomeSchema.parse(JSON.parse(result.stdout));
+    const incomplete = scenario.includes("close");
+    if (incomplete) {
+      expect(outcome.retained).not.toBeNull();
+      expect(outcome.existedAtFailure).toBe(true);
+      if (scenario !== "root-close")
+        expect(outcome.beforeRetry).not.toContain("root-close");
+      expect(outcome.messages[0]).toMatch(/ENOENT.*linux-amd64/u);
+    } else {
+      expect(outcome.retained).toBeNull();
+      expect(outcome.messages[0]).toMatch(/fault|ENOENT.*linux-amd64/u);
+    }
+    expect(outcome.removedAfterRetry).toBe(true);
+    expect(outcome.rootRemoved).toBe(true);
+    if (!scenario.includes("constructor")) {
+      expect(outcome.events.indexOf("transport-close")).toBeGreaterThan(
+        outcome.events.indexOf("transport-start"),
+      );
+      expect(
+        outcome.events.lastIndexOf("transport-settled"),
+      ).toBeGreaterThanOrEqual(0);
+      expect(outcome.events.lastIndexOf("root-close")).toBeGreaterThan(
+        outcome.events.lastIndexOf("transport-settled"),
+      );
+    }
+    for (const name of ["client", "transport", "commands", "root"])
+      if (scenario.includes(`${name}-close`))
+        expect(outcome.messages).toContain(`${name} close fault`);
+  },
+);

tests/fixtures/goVerifierLifetime.mjs

@@ -0,0 +1,160 @@
+// Exercise the actual verifier entrypoint with a fixture compiler and authored
+// SDK/transport failures. The fixture root and command owner remain real.
+import assert from "node:assert/strict";
+import { existsSync } from "node:fs";
+import { chmod, writeFile } from "node:fs/promises";
+import { registerHooks } from "node:module";
+import { join } from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+
+import { PrivateRuntimeRoot } from "../../dist/process/PrivateRuntimeRoot.js";
+import { createVerifierCommands } from "../../scripts/lib/verify-commands.mjs";
+
+const scenario = process.argv[2];
+const directory = process.argv[3];
+assert.equal(typeof directory, "string");
+const compiler = join(directory, "go-fixture");
+await writeFile(
+  compiler,
+  '#!/bin/sh\ncase "$1" in\nversion) echo "go version fixture";;\nenv) printf "linux\\namd64\\n";;\nesac\n',
+);
+await chmod(compiler, 0o755);
+process.env.REA_VERIFY_GO_COMMAND = compiler;
+const events = [];
+let retrying = false;
+let transport;
+const fails = (name) => !retrying && scenario.includes(name);
+const state = {
+  Client: class {
+    constructor() {
+      if (fails("client-constructor"))
+        throw new Error("client constructor fault");
+    }
+    async connect(value) {
+      transport = value;
+      await value.start();
+    }
+    async close() {
+      events.push("client-close");
+      if (fails("client-close")) throw new Error("client close fault");
+      await transport?.close();
+    }
+  },
+  createTransport() {
+    if (fails("transport-constructor"))
+      throw new Error("transport constructor fault");
+    let closed = false;
+    return {
+      async start() {
+        if (closed) throw new Error("Transport closed before handshake");
+        events.push("transport-start");
+        if (fails("handshake")) throw new Error("handshake fault");
+      },
+      async close() {
+        events.push("transport-close");
+        if (fails("transport-close")) throw new Error("transport close fault");
+        // A close promise is the completion boundary, not its initial call.
+        await delay(20);
+        closed = true;
+        events.push("transport-settled");
+      },
+    };
+  },
+  async createOwnedMcpStdioTransport() {
+    return state.createTransport();
+  },
+  async createVerifierCommands() {
+    const owner = await createVerifierCommands();
+    const close = owner.close.bind(owner);
+    owner.close = async () => {
+      events.push("commands-close");
+      if (fails("commands-close")) throw new Error("commands close fault");
+      await close();
+    };
+    return owner;
+  },
+};
+globalThis.goVerifierLifetime = state;
+const entrypoint = new URL(
+  "../../scripts/verify-go-binary.mjs",
+  import.meta.url,
+);
+const moduleSources = new Map([
+  [
+    "@modelcontextprotocol/client",
+    "export const Client = globalThis.goVerifierLifetime.Client;",
+  ],
+  [
+    "@modelcontextprotocol/client/stdio",
+    "export class StdioClientTransport { constructor() { return globalThis.goVerifierLifetime.createTransport(); } }",
+  ],
+  [
+    "./lib/owned-mcp-stdio.mjs",
+    "export const createOwnedMcpStdioTransport = globalThis.goVerifierLifetime.createOwnedMcpStdioTransport;",
+  ],
+  [
+    "./lib/verify-commands.mjs",
+    "export const createVerifierCommands = globalThis.goVerifierLifetime.createVerifierCommands;",
+  ],
+]);
+const hook = registerHooks({
+  resolve(specifier, context, next) {
+    const source =
+      context.parentURL === entrypoint.href
+        ? moduleSources.get(specifier)
+        : undefined;
+    return source === undefined
+      ? next(specifier, context)
+      : {
+          shortCircuit: true,
+          url: `data:text/javascript,${encodeURIComponent(source)}`,
+        };
+  },
+});
+const closeRoot = PrivateRuntimeRoot.prototype.close;
+let rootPath;
+PrivateRuntimeRoot.prototype.close = async function () {
+  rootPath = this.path;
+  events.push("root-close");
+  if (fails("root-close")) throw new Error("root close fault");
+  await closeRoot.call(this);
+};
+const reports = [];
+const stderr = console.error;
+console.error = (...values) => {
+  if (typeof values[0] === "string" && values[0].startsWith("{"))
+    reports.push(JSON.parse(values[0]));
+  else stderr(...values);
+};
+let failure;
+try {
+  await import(entrypoint.href);
+} catch (cause) {
+  failure = cause;
+} finally {
+  hook.deregister();
+  console.error = stderr;
+}
+assert.ok(failure, "Fixture compiler intentionally produces no executable");
+const messages = (cause) =>
+  cause instanceof AggregateError
+    ? cause.errors.flatMap(messages)
+    : [cause.message];
+const report = reports.at(-1);
+const retained = report?.retained_root ?? null;
+const existedAtFailure = retained !== null && existsSync(retained);
+const beforeRetry = [...events];
+retrying = true;
+for (const lease of failure.cleanupLeases ?? []) await lease.release();
+console.log(
+  JSON.stringify({
+    messages: messages(failure),
+    report,
+    retained,
+    existedAtFailure,
+    removedAfterRetry: retained === null || !existsSync(retained),
+    rootRemoved: rootPath !== undefined && !existsSync(rootPath),
+    beforeRetry,
+    events,
+  }),
+);