fix(process): preserve Darwin snapshot and cleanup failures (#12100)

* fix(process): preserve Darwin opening snapshot failures

* fix(process): retain the exact Darwin helper cleanup owner

Transfer the failed helper-root capability across repeated prepare failures. Reuse that same owner for retirement and reader close so stale cleanup cannot remove a replacement helper.

Co-authored-by: Anandumv <anandumv@gmail.com>
Co-authored-by: nextjohnp7 <nextjohnp7@gmail.com>

* test(process): isolate retained cleanup and compiler acknowledgements

---------

Co-authored-by: Anandumv <anandumv@gmail.com>
Co-authored-by: nextjohnp7 <nextjohnp7@gmail.com>
This commit is contained in:
authored and GitHub committed 2026-11-18 17:34:25 -05:00
1 parent 230f86240f
commit 00a3f4d2b9
6 files changed
+216 -53

No files matched your search

+15 -9
View File
@@ -47,14 +47,14 @@ typealias ProcessTableSysctlCall = (
_ size: inout Int
) -> Int32
private func readProcessSnapshot() -> ProcessSnapshot {
readProcessSnapshot { mib, output, size in
sysctl(&mib, u_int(mib.count), output, &size, nil, 0)
}
private func systemProcessTableSysctl(
_ mib: inout [Int32], _ output: UnsafeMutableRawPointer?, _ size: inout Int
) -> Int32 {
sysctl(&mib, u_int(mib.count), output, &size, nil, 0)
}
func readProcessSnapshot(
sysctlCall: ProcessTableSysctlCall
sysctlCall: ProcessTableSysctlCall = systemProcessTableSysctl
) -> ProcessSnapshot {
for _ in 0..<3 {
var mib = [CTL_KERN, KERN_PROC, KERN_PROC_ALL, 0]
@@ -306,15 +306,21 @@ private func readRunToken(pid: Int32, pointerSize: Int) -> RunTokenObservation {
)
}
func readRunTokens(pids: [Int32]) -> [RunTokenObservation] {
let before = readProcessSnapshot()
func readRunTokens(
pids: [Int32], sysctlCall: ProcessTableSysctlCall = systemProcessTableSysctl
) -> [RunTokenObservation] {
let before = readProcessSnapshot(sysctlCall: sysctlCall)
// A later snapshot cannot establish the identity that opened the bracket.
if let failure = before.failure {
return pids.map { unavailable(pid: $0, reason: failure) }
}
let observations = pids.map { pid -> RunTokenObservation in
guard let metadata = before.metadata[pid] else {
return unavailable(pid: pid, reason: before.failure ?? "process_not_in_snapshot")
return unavailable(pid: pid, reason: "process_not_in_snapshot")
}
return readRunToken(pid: pid, pointerSize: metadata.pointerSize)
}
let after = readProcessSnapshot()
let after = readProcessSnapshot(sysctlCall: sysctlCall)
return zip(pids, observations).map { pid, observation in
guard let initial = before.metadata[pid], let current = after.metadata[pid] else {
return unavailable(pid: pid, reason: after.failure ?? "process_not_in_snapshot")
+90 -17
View File
@@ -4,10 +4,14 @@ import {
readProcessRunTokens,
} from "./ProcessRunTokenObservations.js";
import { ExecFileOutputCancelledError } from "./ExecFileOutput.js";
import { basename, join } from "node:path";
import { basename, dirname, join } from "node:path";
import { tmpdir } from "node:os";
import { writeFile } from "node:fs/promises";
import { access, mkdir, readFile, writeFile } from "node:fs/promises";
import { expect, it, onTestFinished, vi } from "vitest";
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js";
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
import { RetainedCommandCleanup } from "./RetainedCommandCleanup.js";
const fixture = vi.hoisted(() => {
let start!: () => void;
@@ -17,7 +21,7 @@ const fixture = vi.hoisted(() => {
let stopped: Promise<void>;
let gate: Promise<void>;
let mode: "failure" | "abort" | "cleanup" | "abort-error";
let failRemoval = false;
let remainingRemovalFailures = 0;
let calls = 0;
const compilerFailure = new Error("controlled compiler failure");
const compilerAbortError = new DOMException(
@@ -27,8 +31,11 @@ const fixture = vi.hoisted(() => {
return {
compilerFailure,
compilerAbortError,
configure(selected: "failure" | "abort" | "cleanup" | "abort-error") {
failRemoval = selected === "cleanup";
configure(
selected: "failure" | "abort" | "cleanup" | "abort-error",
removalFailures = selected === "cleanup" ? 1 : 0,
) {
remainingRemovalFailures = removalFailures;
mode = selected;
calls = 0;
started = new Promise((resolve) => {
@@ -42,11 +49,14 @@ const fixture = vi.hoisted(() => {
});
},
remove() {
if (failRemoval) {
failRemoval = false;
if (remainingRemovalFailures > 0) {
remainingRemovalFailures--;
throw new Error("controlled cleanup failure");
}
},
allowRemoval() {
remainingRemovalFailures = 0;
},
get started() {
return started;
},
@@ -102,18 +112,21 @@ import {
DarwinProcessOwnershipInspectionError,
} from "./DarwinProcessRunTokenReader.js";
const createFixtureReader = () =>
createDarwinProcessRunTokenReader({
compiler: {
kind: "swiftc",
executable: join(tmpdir(), "fixture-swiftc"),
sdkRoot: tmpdir(),
},
environment: {},
});
it.each(["failure", "abort", "cleanup", "abort-error"] as const)(
"handles a %s outcome for a waiter joining retirement",
async (mode) => {
fixture.configure(mode);
const reader = createDarwinProcessRunTokenReader({
compiler: {
kind: "swiftc",
executable: join(tmpdir(), "fixture-swiftc"),
sdkRoot: tmpdir(),
},
environment: {},
});
const reader = createFixtureReader();
const controller = new AbortController();
const first = reader.prepare(controller.signal);
let second: Promise<string> | undefined;
@@ -121,9 +134,9 @@ it.each(["failure", "abort", "cleanup", "abort-error"] as const)(
fixture.release();
await Promise.allSettled([first, second, reader.close()]);
});
await fixture.started;
await awaitAcknowledgement(fixture.started, first, "compiler startup");
controller.abort(new Error("first caller left"));
await fixture.stopped;
await awaitAcknowledgement(fixture.stopped, first, "compiler cancellation");
second = reader.prepare();
fixture.release();
if (mode === "cleanup")
@@ -196,3 +209,63 @@ it.each(["failure", "abort", "cleanup", "abort-error"] as const)(
}
},
);
it("transfers the same helper root after a second failed cleanup", async () => {
fixture.configure("failure", 2);
const reader = createFixtureReader();
const prepare = reader.prepare();
const first = prepare.catch((cause: unknown) => cause);
onTestFinished(async () => {
fixture.release();
await first;
fixture.allowRemoval();
await reader.close();
});
await awaitAcknowledgement(fixture.started, prepare, "compiler startup");
fixture.release();
const initialFailure = await first;
if (!(initialFailure instanceof DarwinProcessOwnershipInspectionError))
throw new Error("Expected the initial inspection cleanup failure", {
cause: initialFailure,
});
const [root] = initialFailure.cleanupFailure?.cleanupResources ?? [];
if (root === undefined) throw new Error("Missing retained helper root");
expect(initialFailure.cause).toBe(fixture.compilerFailure);
expect(initialFailure.cleanupFailure?.cleanupIncomplete).toBe(true);
await expect(access(root)).resolves.toBeUndefined();
const retryFailure = await reader.prepare().catch((cause: unknown) => cause);
if (!(retryFailure instanceof DarwinProcessOwnershipInspectionError))
throw new Error("Expected the retry inspection cleanup failure", {
cause: retryFailure,
});
const owner = retryFailure.cleanupOwner;
if (owner === undefined) throw new Error("Missing retry cleanup owner");
expect(owner).toBe(initialFailure.cleanupOwner);
expect(retryFailure.cleanupFailure).toBeInstanceOf(ProviderCleanupError);
expect(retryFailure.cleanupFailure?.cleanupResources).toEqual([root]);
expect(retryFailure.cleanupFailure?.cleanupIncomplete).toBe(true);
expect(fixture.calls).toBe(1);
const fixtureDirectory = await createTestTempDirectory(
"rea-retained-command-",
);
const scratch = join(fixtureDirectory, "command");
await mkdir(scratch);
const retained = new RetainedCommandCleanup(scratch, retryFailure);
try {
expect(retained.cleanupResources).toEqual([root, scratch]);
} finally {
await retained.close();
}
await expect(access(fixtureDirectory)).resolves.toBeUndefined();
await expect(access(scratch)).rejects.toMatchObject({ code: "ENOENT" });
await expect(access(root)).rejects.toMatchObject({ code: "ENOENT" });
expect(initialFailure.cleanupFailure?.cleanupIncomplete).toBe(false);
expect(retryFailure.cleanupFailure?.cleanupIncomplete).toBe(false);
const replacement = await reader.prepare();
expect(dirname(replacement)).not.toBe(root);
await owner.close();
await expect(readFile(replacement, "utf8")).resolves.toBe("fixture");
});
+39 -26
View File
@@ -406,6 +406,7 @@ const compilerInspectionError = (
/** One allocated helper root, including its shared finalization and live cleanup status. */
interface ProcessReaderRoot {
readonly path: string;
close(): Promise<void>;
cleanupPromise?: Promise<void> | undefined;
/** Reported by every removal failure until a retry removes the root. */
cleanupRelease?: AnalysisCleanupRelease | undefined;
@@ -533,8 +534,7 @@ export const createDarwinProcessRunTokenReader = (
const cleanupAtExit = () => {
if (root !== undefined) removeHelperRootAtExit(root.path);
};
const cleanupRoot = (target = root): Promise<void> => {
if (target === undefined) return Promise.resolve();
const cleanupRoot = (target: ProcessReaderRoot): Promise<void> => {
target.cleanupPromise ??= Promise.resolve()
.then(async () => {
try {
@@ -567,6 +567,33 @@ export const createDarwinProcessRunTokenReader = (
});
return target.cleanupPromise;
};
const cleanupFailedCompilationRoot = async (
target: ProcessReaderRoot,
primaryFailure?: ProcessReaderCompilation["primaryFailure"],
): Promise<void> => {
try {
await target.close();
} catch (cleanup: unknown) {
if (!(cleanup instanceof ProviderCleanupError)) throw cleanup;
const cause =
primaryFailure === undefined ? cleanup : primaryFailure.cause;
const primary =
cause instanceof DarwinProcessOwnershipInspectionError
? cause
: new DarwinProcessOwnershipInspectionError(
primaryFailure === undefined
? "Darwin process ownership helper root cleanup retry failed"
: "Darwin process ownership helper preparation failed",
{
cause,
...(cause instanceof ProcessOwnershipInspectionCancelled
? { cancellation: { reason: cause.reason } }
: {}),
},
);
throw primary.withCleanupFailure(cleanup, target);
}
};
const runCompilation = async (
operation: ProcessReaderCompilation,
): Promise<string> => {
@@ -592,10 +619,12 @@ export const createDarwinProcessRunTokenReader = (
executable = cached.executable;
return cached.executable;
}
operationRoot = {
const allocatedRoot: ProcessReaderRoot = {
path: await mkdtemp(join(tmpdir(), "rea-process-token-")),
close: () => cleanupRoot(allocatedRoot),
};
root = operationRoot;
operationRoot = allocatedRoot;
root = allocatedRoot;
process.once("exit", cleanupAtExit);
exitCleanupInstalled = true;
if (signal.aborted || closed)
@@ -654,26 +683,10 @@ export const createDarwinProcessRunTokenReader = (
// The extra pipe keeps settlement pending for Swift frontend writers
// that inherit it after the driver and its standard streams close.
// It cannot establish completion of a child that did not inherit it.
try {
await cleanupRoot(operationRoot);
} catch (cleanup: unknown) {
if (!(cleanup instanceof ProviderCleanupError)) throw cleanup;
const primary =
cause instanceof DarwinProcessOwnershipInspectionError
? cause
: new DarwinProcessOwnershipInspectionError(
"Darwin process ownership helper preparation failed",
{
cause,
...(cause instanceof ProcessOwnershipInspectionCancelled
? { cancellation: { reason: cause.reason } }
: {}),
},
);
throw primary.withCleanupFailure(cleanup, {
close: () => cleanupRoot(operationRoot),
});
}
await cleanupFailedCompilationRoot(
operationRoot,
operation.primaryFailure,
);
}
throw cause;
}
@@ -706,7 +719,7 @@ export const createDarwinProcessRunTokenReader = (
assertProcessOwnershipInspectionActive(signal);
if (executable !== undefined) return executable;
if (compilation === undefined && root !== undefined) {
await cleanupRoot();
await cleanupFailedCompilationRoot(root);
continue;
}
let operation = compilation;
@@ -920,7 +933,7 @@ export const createDarwinProcessRunTokenReader = (
// Each owner keeps its own failed cleanup for the next close attempt;
// a failure of both stays incomplete until both owners recover.
const [rootCleanup, cacheCleanup] = await Promise.allSettled([
cleanupRoot(),
root?.close(),
helperCache?.close(),
]);
if (
@@ -48,6 +48,11 @@ it.each([
{ present: false, reason: "unused", availability: "absent" },
{ present: true, reason: "process_not_in_snapshot", availability: "absent" },
{ present: true, reason: "sysctl_failed_13", availability: "unavailable" },
{
present: true,
reason: "process_table_failed_13",
availability: "unavailable",
},
])(
"retains native run-token availability at the host adapter: %j",
async (state) => {
@@ -1482,7 +1482,7 @@ it.skipIf(!onDarwin)(
);
it.skipIf(!onDarwin)(
"retries a growing process table after ENOMEM and preserves other sysctl errors",
"preserves process-table failures and confirms run-token identity",
{ tags: ["long-running"] },
async () => {
const root = await mkdtemp(
@@ -1509,11 +1509,29 @@ it.skipIf(!onDarwin)(
);
const { stdout } = await execFileAsync(executable, [], {
encoding: "utf8",
env: { ...process.env, REA_PROCESS_RUN_ID: "rea-snapshot-probe" },
timeout: 10_000,
});
expect(JSON.parse(stdout)).toEqual({
preservedOtherErrno: true,
retriedAfterFreshSize: true,
openingRunTokenSnapshotReads: 1,
failedOpeningRunTokens: [42, 43].map((pid) => ({
pid,
state: "unavailable",
reason: "process_table_failed_13",
})),
absentRunTokens: [
{ pid: 42, state: "unavailable", reason: "process_not_in_snapshot" },
],
failedConfirmingRunTokens: [
{
state: "unavailable",
reason: "process_table_failed_5",
tokenPresent: false,
},
],
confirmedRunTokens: [{ state: "readable", matchesFixtureToken: true }],
successfulSnapshotMissingPidIsAbsent: true,
failedSnapshotMissingPidIsUnavailable: true,
invalidPidIsUnavailable: true,
+48
View File
@@ -30,6 +30,49 @@ struct ProcessRunTokenSnapshotProbe {
return -1
}
var openingSizeQueries = 0
let failedOpeningTokens = readRunTokens(pids: [42, 43]) { _, output, size in
if let output = output {
output.assumingMemoryBound(to: kinfo_proc.self).pointee.kp_proc.p_pid = 42
size = MemoryLayout<kinfo_proc>.stride
return 0
}
openingSizeQueries += 1
if openingSizeQueries == 1 {
errno = EACCES
return -1
}
size = MemoryLayout<kinfo_proc>.stride
return 0
}
let absentTokens = readRunTokens(pids: [42]) { _, output, size in
size = output == nil ? MemoryLayout<kinfo_proc>.stride : 0
return 0
}
let ownPid = getpid()
var confirmationSizeQueries = 0
let singleProcessTable: ProcessTableSysctlCall = { _, output, size in
if let output = output {
let process = output.assumingMemoryBound(to: kinfo_proc.self)
process.pointee.kp_proc.p_pid = ownPid
process.pointee.kp_proc.p_flag = P_LP64
}
size = MemoryLayout<kinfo_proc>.stride
return 0
}
let failedConfirmingTokens = readRunTokens(pids: [ownPid]) { mib, output, size in
if output == nil {
confirmationSizeQueries += 1
if confirmationSizeQueries == 2 {
errno = EIO
return -1
}
}
return singleProcessTable(&mib, output, &size)
}
let confirmedTokens = readRunTokens(pids: [ownPid], sysctlCall: singleProcessTable)
let absentIdentities = readProcessIdentities(
pids: [42],
snapshot: ProcessSnapshot(metadata: [:], failure: nil)
@@ -53,6 +96,11 @@ struct ProcessRunTokenSnapshotProbe {
let output: [String: Any] = [
"retriedAfterFreshSize": retried.failure == nil && sizeQueries == 2 && readAttempts == 2,
"preservedOtherErrno": otherError.failure == "process_table_failed_\(EACCES)",
"openingRunTokenSnapshotReads": openingSizeQueries,
"failedOpeningRunTokens": failedOpeningTokens.map { ["pid": $0.pid, "state": $0.state, "reason": $0.reason ?? ""] },
"absentRunTokens": absentTokens.map { ["pid": $0.pid, "state": $0.state, "reason": $0.reason ?? ""] },
"failedConfirmingRunTokens": failedConfirmingTokens.map { ["state": $0.state, "reason": $0.reason ?? "", "tokenPresent": $0.run_id != nil] },
"confirmedRunTokens": confirmedTokens.map { ["state": $0.state, "matchesFixtureToken": $0.run_id == "rea-snapshot-probe"] },
"successfulSnapshotMissingPidIsAbsent": absentIdentities.first?.state == "absent" && absentIdentities.first?.reason == "not-in-snapshot",
"failedSnapshotMissingPidIsUnavailable": failedIdentities.first?.state == "unavailable" && failedIdentities.first?.reason == "process_table_failed_13",
"invalidPidIsUnavailable": invalidIdentity.first?.state == "unavailable" && invalidIdentity.first?.reason == "invalid_process_id",