fix(process): retain inherited pipes on output overflow

src/process/ExecFileOutput.ts

@@ -1,6 +1,7 @@
 import type { AnalysisCapturedOutput } from "../domain/analysisErrorBase.js";
 import {
   execFile,
+  spawn,
   type ExecFileOptionsWithBufferEncoding,
 } from "node:child_process";
 
@@ -158,67 +159,149 @@ export const execFileBytes = (
       reject(new ExecFileOutputCancelledError(signal.reason));
       return;
     }
-    let stopped: "abort" | "timeout" | undefined;
-    let deadline: NodeJS.Timeout | undefined;
-    let grace: NodeJS.Timeout | undefined;
-    const child = execFile(
+    const complete = (
+      error: Error | null,
+      stdout: Buffer,
+      stderr: Buffer,
+    ): void => {
+      if (error !== null) {
+        Reflect.set(error, "stdout", stdout.toString("utf8"));
+        Reflect.set(error, "stderr", stderr.toString("utf8"));
+        Reflect.set(error, "stdoutBytes", stdout.byteLength);
+        Reflect.set(error, "stderrBytes", stderr.byteLength);
+        reject(error);
+        return;
+      }
+      resolve({ stdout, stderr });
+    };
+    if (stopSignal === undefined) {
+      execFile(
+        command,
+        [...arguments_],
+        {
+          ...execOptions,
+          ...(signal === undefined ? {} : { signal }),
+          ...(timeout === undefined ? {} : { timeout }),
+          encoding: "buffer",
+          maxBuffer: options.maxBuffer ?? Number.POSITIVE_INFINITY,
+        },
+        (error, stdout, stderr) =>
+          complete(
+            error?.name === "AbortError" && error.code === "ABORT_ERR"
+              ? new ExecFileOutputCancelledError(error.cause)
+              : error,
+            stdout,
+            stderr,
+          ),
+      );
+      return;
+    }
+    startStoppableCommand(
       command,
-      [...arguments_],
-      {
-        ...execOptions,
-        // A stoppable command's abort and timeout are handled below, because
-        // execFile neither waits for the exit on abort nor kills a process
-        // that ignores its timeout signal.
-        ...(stopSignal === undefined
-          ? {
-              ...(signal === undefined ? {} : { signal }),
-              ...(timeout === undefined ? {} : { timeout }),
-            }
-          : { killSignal: stopSignal }),
-        encoding: "buffer",
-        maxBuffer: options.maxBuffer ?? Number.POSITIVE_INFINITY,
-      },
-      (error, stdout, stderr) => {
-        signal?.removeEventListener("abort", onAbort);
-        clearTimeout(deadline);
-        clearTimeout(grace);
-        const failure =
-          stopped === "abort" && signal !== undefined
-            ? new ExecFileOutputCancelledError(signal.reason)
-            : stopped === "timeout" && timeout !== undefined
-              ? timeoutError(command, timeout, error)
-              : error?.name === "AbortError" && error.code === "ABORT_ERR"
-                ? new ExecFileOutputCancelledError(error.cause)
-                : error;
-        if (failure !== null) {
-          Reflect.set(failure, "stdout", stdout.toString("utf8"));
-          Reflect.set(failure, "stderr", stderr.toString("utf8"));
-          Reflect.set(failure, "stdoutBytes", stdout.byteLength);
-          Reflect.set(failure, "stderrBytes", stderr.byteLength);
-          reject(failure);
-          return;
-        }
-        resolve({ stdout, stderr });
-      },
+      arguments_,
+      { ...options, stopSignal },
+      complete,
     );
-    // The first stop request names the failure; a later one is not resent.
-    const stop = (reason: "abort" | "timeout"): void => {
-      if (stopped !== undefined) return;
-      stopped = reason;
-      child.kill(stopSignal);
-      grace = setTimeout(() => child.kill("SIGKILL"), STOP_GRACE_MS);
-    };
-    const onAbort = (): void => {
-      stop("abort");
-    };
-    if (stopSignal === undefined) return;
-    signal?.addEventListener("abort", onAbort, { once: true });
-    if (timeout !== undefined && timeout > 0)
-      deadline = setTimeout(() => {
-        stop("timeout");
-      }, timeout);
   });
 
+/** Own stop requests and bounded capture until the root and inherited pipes close. */
+const startStoppableCommand = (
+  command: string,
+  arguments_: readonly string[],
+  options: ExecFileOutputOptions & { readonly stopSignal: NodeJS.Signals },
+  complete: (error: Error | null, stdout: Buffer, stderr: Buffer) => void,
+): void => {
+  const { stopSignal, signal, timeout, ...execOptions } = options;
+  const maxBuffer = options.maxBuffer ?? Number.POSITIVE_INFINITY;
+  if (typeof maxBuffer !== "number" || !(maxBuffer >= 0)) {
+    const error = new RangeError("maxBuffer must be a nonnegative number");
+    Reflect.set(error, "code", "ERR_OUT_OF_RANGE");
+    throw error;
+  }
+  let stopped: "abort" | "timeout" | "output-limit" | undefined;
+  let processError: Error | null = null;
+  let outputError: Error | undefined;
+  let deadline: NodeJS.Timeout | undefined;
+  let grace: NodeJS.Timeout | undefined;
+  // Node's execFile destroys both pipes when maxBuffer is exceeded. Own
+  // bounded capture here so overflow stops the root without discarding the
+  // pipe lifetime of descendants still using caller-owned temporary files.
+  const child = spawn(command, [...arguments_], {
+    ...execOptions,
+    stdio: "pipe",
+    killSignal: stopSignal,
+  });
+  // The first stop request names the failure; a later one is not resent.
+  const stop = (reason: "abort" | "timeout" | "output-limit"): void => {
+    if (stopped !== undefined) return;
+    stopped = reason;
+    child.kill(stopSignal);
+    grace = setTimeout(() => child.kill("SIGKILL"), STOP_GRACE_MS);
+  };
+  const onAbort = (): void => {
+    stop("abort");
+  };
+  const capture = (stream: typeof child.stdout, name: "stdout" | "stderr") => {
+    const chunks: Buffer[] = [];
+    let retained = 0;
+    stream.on("data", (chunk: Buffer) => {
+      const remaining = Math.max(0, Math.floor(maxBuffer - retained));
+      if (chunk.byteLength <= remaining) {
+        chunks.push(chunk);
+        retained += chunk.byteLength;
+        return;
+      }
+      if (remaining > 0) {
+        chunks.push(Buffer.from(chunk.subarray(0, remaining)));
+        retained += remaining;
+      }
+      if (outputError === undefined) {
+        outputError = new RangeError(`${name} maxBuffer length exceeded`);
+        Reflect.set(outputError, "code", "ERR_CHILD_PROCESS_STDIO_MAXBUFFER");
+        stop("output-limit");
+      }
+    });
+    return () => Buffer.concat(chunks, retained);
+  };
+  const stdout = capture(child.stdout, "stdout");
+  const stderr = capture(child.stderr, "stderr");
+  child.on("error", (error: Error) => {
+    processError ??= error;
+  });
+  child.on("close", (code, exitSignal) => {
+    signal?.removeEventListener("abort", onAbort);
+    clearTimeout(deadline);
+    clearTimeout(grace);
+    const out = stdout();
+    const diagnostic = stderr();
+    const invocation = [command, ...arguments_].join(" ");
+    if (processError === null && (code !== 0 || exitSignal !== null)) {
+      processError = new Error(
+        `Command failed: ${invocation}\n${diagnostic.toString("utf8")}`,
+      );
+      Reflect.set(processError, "code", code);
+      Reflect.set(processError, "signal", exitSignal);
+      Reflect.set(processError, "killed", child.killed);
+    }
+    if (processError !== null) Reflect.set(processError, "cmd", invocation);
+    if (outputError !== undefined) Reflect.set(outputError, "cmd", invocation);
+    const failure =
+      stopped === "abort" && signal !== undefined
+        ? new ExecFileOutputCancelledError(signal.reason)
+        : stopped === "timeout" && timeout !== undefined
+          ? timeoutError(command, timeout, processError)
+          : (outputError ?? processError);
+    if (failure !== null && outputError !== undefined)
+      Reflect.set(failure, "outputTruncated", true);
+    complete(failure, out, diagnostic);
+  });
+  signal?.addEventListener("abort", onAbort, { once: true });
+  if (timeout !== undefined && timeout > 0)
+    deadline = setTimeout(() => {
+      stop("timeout");
+    }, timeout);
+};
+
 /** Run a shell-free command while capturing its complete UTF-8 output. */
 export const execFileOutput = (
   command: string,