From 00a3f4d2b9ac9a1abc4945178010321877d40f07 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 19 Nov 2026 06:34:25 +0800 Subject: [PATCH] fix(process): preserve Darwin snapshot and cleanup failures (#12100) * fix(process): preserve Darwin opening snapshot failures * fix(process): retain the exact Darwin helper cleanup owner Transfer the failed helper-root capability across repeated prepare failures. Reuse that same owner for retirement and reader close so stale cleanup cannot remove a replacement helper. Co-authored-by: Anandumv Co-authored-by: nextjohnp7 * test(process): isolate retained cleanup and compiler acknowledgements --------- Co-authored-by: Anandumv Co-authored-by: nextjohnp7 --- bridge/process/ProcessRunTokenReader.swift | 24 ++-- .../DarwinCompilationRetirement.test.ts | 107 +++++++++++++++--- src/process/DarwinProcessRunTokenReader.ts | 65 ++++++----- ...ProcessOwnership.tokenAvailability.test.ts | 5 + .../darwinProcessRunTokenReader.test.ts | 20 +++- .../processRunTokenSnapshotProbe.swift | 48 ++++++++ 6 files changed, 216 insertions(+), 53 deletions(-) diff --git a/bridge/process/ProcessRunTokenReader.swift b/bridge/process/ProcessRunTokenReader.swift index 931eb48750..e39405690c 100644 --- a/bridge/process/ProcessRunTokenReader.swift +++ b/bridge/process/ProcessRunTokenReader.swift @@ -47,14 +47,14 @@ typealias ProcessTableSysctlCall = ( _ size: inout Int ) -> Int32 -private func readProcessSnapshot() -> ProcessSnapshot { - readProcessSnapshot { mib, output, size in - sysctl(&mib, u_int(mib.count), output, &size, nil, 0) - } +private func systemProcessTableSysctl( + _ mib: inout [Int32], _ output: UnsafeMutableRawPointer?, _ size: inout Int +) -> Int32 { + sysctl(&mib, u_int(mib.count), output, &size, nil, 0) } func readProcessSnapshot( - sysctlCall: ProcessTableSysctlCall + sysctlCall: ProcessTableSysctlCall = systemProcessTableSysctl ) -> ProcessSnapshot { for _ in 0..<3 { var mib = [CTL_KERN, KERN_PROC, KERN_PROC_ALL, 0] @@ -306,15 +306,21 @@ private func readRunToken(pid: Int32, pointerSize: Int) -> RunTokenObservation { ) } -func readRunTokens(pids: [Int32]) -> [RunTokenObservation] { - let before = readProcessSnapshot() +func readRunTokens( + pids: [Int32], sysctlCall: ProcessTableSysctlCall = systemProcessTableSysctl +) -> [RunTokenObservation] { + let before = readProcessSnapshot(sysctlCall: sysctlCall) + // A later snapshot cannot establish the identity that opened the bracket. + if let failure = before.failure { + return pids.map { unavailable(pid: $0, reason: failure) } + } let observations = pids.map { pid -> RunTokenObservation in guard let metadata = before.metadata[pid] else { - return unavailable(pid: pid, reason: before.failure ?? "process_not_in_snapshot") + return unavailable(pid: pid, reason: "process_not_in_snapshot") } return readRunToken(pid: pid, pointerSize: metadata.pointerSize) } - let after = readProcessSnapshot() + let after = readProcessSnapshot(sysctlCall: sysctlCall) return zip(pids, observations).map { pid, observation in guard let initial = before.metadata[pid], let current = after.metadata[pid] else { return unavailable(pid: pid, reason: after.failure ?? "process_not_in_snapshot") diff --git a/src/process/DarwinCompilationRetirement.test.ts b/src/process/DarwinCompilationRetirement.test.ts index 8e0f05f2e5..9c988a1cc7 100644 --- a/src/process/DarwinCompilationRetirement.test.ts +++ b/src/process/DarwinCompilationRetirement.test.ts @@ -4,10 +4,14 @@ import { readProcessRunTokens, } from "./ProcessRunTokenObservations.js"; import { ExecFileOutputCancelledError } from "./ExecFileOutput.js"; -import { basename, join } from "node:path"; +import { basename, dirname, join } from "node:path"; import { tmpdir } from "node:os"; -import { writeFile } from "node:fs/promises"; +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; import { expect, it, onTestFinished, vi } from "vitest"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { RetainedCommandCleanup } from "./RetainedCommandCleanup.js"; const fixture = vi.hoisted(() => { let start!: () => void; @@ -17,7 +21,7 @@ const fixture = vi.hoisted(() => { let stopped: Promise; let gate: Promise; let mode: "failure" | "abort" | "cleanup" | "abort-error"; - let failRemoval = false; + let remainingRemovalFailures = 0; let calls = 0; const compilerFailure = new Error("controlled compiler failure"); const compilerAbortError = new DOMException( @@ -27,8 +31,11 @@ const fixture = vi.hoisted(() => { return { compilerFailure, compilerAbortError, - configure(selected: "failure" | "abort" | "cleanup" | "abort-error") { - failRemoval = selected === "cleanup"; + configure( + selected: "failure" | "abort" | "cleanup" | "abort-error", + removalFailures = selected === "cleanup" ? 1 : 0, + ) { + remainingRemovalFailures = removalFailures; mode = selected; calls = 0; started = new Promise((resolve) => { @@ -42,11 +49,14 @@ const fixture = vi.hoisted(() => { }); }, remove() { - if (failRemoval) { - failRemoval = false; + if (remainingRemovalFailures > 0) { + remainingRemovalFailures--; throw new Error("controlled cleanup failure"); } }, + allowRemoval() { + remainingRemovalFailures = 0; + }, get started() { return started; }, @@ -102,18 +112,21 @@ import { DarwinProcessOwnershipInspectionError, } from "./DarwinProcessRunTokenReader.js"; +const createFixtureReader = () => + createDarwinProcessRunTokenReader({ + compiler: { + kind: "swiftc", + executable: join(tmpdir(), "fixture-swiftc"), + sdkRoot: tmpdir(), + }, + environment: {}, + }); + it.each(["failure", "abort", "cleanup", "abort-error"] as const)( "handles a %s outcome for a waiter joining retirement", async (mode) => { fixture.configure(mode); - const reader = createDarwinProcessRunTokenReader({ - compiler: { - kind: "swiftc", - executable: join(tmpdir(), "fixture-swiftc"), - sdkRoot: tmpdir(), - }, - environment: {}, - }); + const reader = createFixtureReader(); const controller = new AbortController(); const first = reader.prepare(controller.signal); let second: Promise | undefined; @@ -121,9 +134,9 @@ it.each(["failure", "abort", "cleanup", "abort-error"] as const)( fixture.release(); await Promise.allSettled([first, second, reader.close()]); }); - await fixture.started; + await awaitAcknowledgement(fixture.started, first, "compiler startup"); controller.abort(new Error("first caller left")); - await fixture.stopped; + await awaitAcknowledgement(fixture.stopped, first, "compiler cancellation"); second = reader.prepare(); fixture.release(); if (mode === "cleanup") @@ -196,3 +209,63 @@ it.each(["failure", "abort", "cleanup", "abort-error"] as const)( } }, ); + +it("transfers the same helper root after a second failed cleanup", async () => { + fixture.configure("failure", 2); + const reader = createFixtureReader(); + const prepare = reader.prepare(); + const first = prepare.catch((cause: unknown) => cause); + onTestFinished(async () => { + fixture.release(); + await first; + fixture.allowRemoval(); + await reader.close(); + }); + await awaitAcknowledgement(fixture.started, prepare, "compiler startup"); + fixture.release(); + const initialFailure = await first; + if (!(initialFailure instanceof DarwinProcessOwnershipInspectionError)) + throw new Error("Expected the initial inspection cleanup failure", { + cause: initialFailure, + }); + const [root] = initialFailure.cleanupFailure?.cleanupResources ?? []; + if (root === undefined) throw new Error("Missing retained helper root"); + expect(initialFailure.cause).toBe(fixture.compilerFailure); + expect(initialFailure.cleanupFailure?.cleanupIncomplete).toBe(true); + await expect(access(root)).resolves.toBeUndefined(); + + const retryFailure = await reader.prepare().catch((cause: unknown) => cause); + if (!(retryFailure instanceof DarwinProcessOwnershipInspectionError)) + throw new Error("Expected the retry inspection cleanup failure", { + cause: retryFailure, + }); + const owner = retryFailure.cleanupOwner; + if (owner === undefined) throw new Error("Missing retry cleanup owner"); + expect(owner).toBe(initialFailure.cleanupOwner); + expect(retryFailure.cleanupFailure).toBeInstanceOf(ProviderCleanupError); + expect(retryFailure.cleanupFailure?.cleanupResources).toEqual([root]); + expect(retryFailure.cleanupFailure?.cleanupIncomplete).toBe(true); + expect(fixture.calls).toBe(1); + + const fixtureDirectory = await createTestTempDirectory( + "rea-retained-command-", + ); + const scratch = join(fixtureDirectory, "command"); + await mkdir(scratch); + const retained = new RetainedCommandCleanup(scratch, retryFailure); + try { + expect(retained.cleanupResources).toEqual([root, scratch]); + } finally { + await retained.close(); + } + await expect(access(fixtureDirectory)).resolves.toBeUndefined(); + await expect(access(scratch)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(access(root)).rejects.toMatchObject({ code: "ENOENT" }); + expect(initialFailure.cleanupFailure?.cleanupIncomplete).toBe(false); + expect(retryFailure.cleanupFailure?.cleanupIncomplete).toBe(false); + + const replacement = await reader.prepare(); + expect(dirname(replacement)).not.toBe(root); + await owner.close(); + await expect(readFile(replacement, "utf8")).resolves.toBe("fixture"); +}); diff --git a/src/process/DarwinProcessRunTokenReader.ts b/src/process/DarwinProcessRunTokenReader.ts index ee7809e2e6..ad9cc0965d 100644 --- a/src/process/DarwinProcessRunTokenReader.ts +++ b/src/process/DarwinProcessRunTokenReader.ts @@ -406,6 +406,7 @@ const compilerInspectionError = ( /** One allocated helper root, including its shared finalization and live cleanup status. */ interface ProcessReaderRoot { readonly path: string; + close(): Promise; cleanupPromise?: Promise | undefined; /** Reported by every removal failure until a retry removes the root. */ cleanupRelease?: AnalysisCleanupRelease | undefined; @@ -533,8 +534,7 @@ export const createDarwinProcessRunTokenReader = ( const cleanupAtExit = () => { if (root !== undefined) removeHelperRootAtExit(root.path); }; - const cleanupRoot = (target = root): Promise => { - if (target === undefined) return Promise.resolve(); + const cleanupRoot = (target: ProcessReaderRoot): Promise => { target.cleanupPromise ??= Promise.resolve() .then(async () => { try { @@ -567,6 +567,33 @@ export const createDarwinProcessRunTokenReader = ( }); return target.cleanupPromise; }; + const cleanupFailedCompilationRoot = async ( + target: ProcessReaderRoot, + primaryFailure?: ProcessReaderCompilation["primaryFailure"], + ): Promise => { + try { + await target.close(); + } catch (cleanup: unknown) { + if (!(cleanup instanceof ProviderCleanupError)) throw cleanup; + const cause = + primaryFailure === undefined ? cleanup : primaryFailure.cause; + const primary = + cause instanceof DarwinProcessOwnershipInspectionError + ? cause + : new DarwinProcessOwnershipInspectionError( + primaryFailure === undefined + ? "Darwin process ownership helper root cleanup retry failed" + : "Darwin process ownership helper preparation failed", + { + cause, + ...(cause instanceof ProcessOwnershipInspectionCancelled + ? { cancellation: { reason: cause.reason } } + : {}), + }, + ); + throw primary.withCleanupFailure(cleanup, target); + } + }; const runCompilation = async ( operation: ProcessReaderCompilation, ): Promise => { @@ -592,10 +619,12 @@ export const createDarwinProcessRunTokenReader = ( executable = cached.executable; return cached.executable; } - operationRoot = { + const allocatedRoot: ProcessReaderRoot = { path: await mkdtemp(join(tmpdir(), "rea-process-token-")), + close: () => cleanupRoot(allocatedRoot), }; - root = operationRoot; + operationRoot = allocatedRoot; + root = allocatedRoot; process.once("exit", cleanupAtExit); exitCleanupInstalled = true; if (signal.aborted || closed) @@ -654,26 +683,10 @@ export const createDarwinProcessRunTokenReader = ( // The extra pipe keeps settlement pending for Swift frontend writers // that inherit it after the driver and its standard streams close. // It cannot establish completion of a child that did not inherit it. - try { - await cleanupRoot(operationRoot); - } catch (cleanup: unknown) { - if (!(cleanup instanceof ProviderCleanupError)) throw cleanup; - const primary = - cause instanceof DarwinProcessOwnershipInspectionError - ? cause - : new DarwinProcessOwnershipInspectionError( - "Darwin process ownership helper preparation failed", - { - cause, - ...(cause instanceof ProcessOwnershipInspectionCancelled - ? { cancellation: { reason: cause.reason } } - : {}), - }, - ); - throw primary.withCleanupFailure(cleanup, { - close: () => cleanupRoot(operationRoot), - }); - } + await cleanupFailedCompilationRoot( + operationRoot, + operation.primaryFailure, + ); } throw cause; } @@ -706,7 +719,7 @@ export const createDarwinProcessRunTokenReader = ( assertProcessOwnershipInspectionActive(signal); if (executable !== undefined) return executable; if (compilation === undefined && root !== undefined) { - await cleanupRoot(); + await cleanupFailedCompilationRoot(root); continue; } let operation = compilation; @@ -920,7 +933,7 @@ export const createDarwinProcessRunTokenReader = ( // Each owner keeps its own failed cleanup for the next close attempt; // a failure of both stays incomplete until both owners recover. const [rootCleanup, cacheCleanup] = await Promise.allSettled([ - cleanupRoot(), + root?.close(), helperCache?.close(), ]); if ( diff --git a/src/process/ProcessOwnership.tokenAvailability.test.ts b/src/process/ProcessOwnership.tokenAvailability.test.ts index c525e6b6ff..8f4195b8af 100644 --- a/src/process/ProcessOwnership.tokenAvailability.test.ts +++ b/src/process/ProcessOwnership.tokenAvailability.test.ts @@ -48,6 +48,11 @@ it.each([ { present: false, reason: "unused", availability: "absent" }, { present: true, reason: "process_not_in_snapshot", availability: "absent" }, { present: true, reason: "sysctl_failed_13", availability: "unavailable" }, + { + present: true, + reason: "process_table_failed_13", + availability: "unavailable", + }, ])( "retains native run-token availability at the host adapter: %j", async (state) => { diff --git a/tests/boundary/process/darwinProcessRunTokenReader.test.ts b/tests/boundary/process/darwinProcessRunTokenReader.test.ts index 5bfb4dbe72..07fc708730 100644 --- a/tests/boundary/process/darwinProcessRunTokenReader.test.ts +++ b/tests/boundary/process/darwinProcessRunTokenReader.test.ts @@ -1482,7 +1482,7 @@ it.skipIf(!onDarwin)( ); it.skipIf(!onDarwin)( - "retries a growing process table after ENOMEM and preserves other sysctl errors", + "preserves process-table failures and confirms run-token identity", { tags: ["long-running"] }, async () => { const root = await mkdtemp( @@ -1509,11 +1509,29 @@ it.skipIf(!onDarwin)( ); const { stdout } = await execFileAsync(executable, [], { encoding: "utf8", + env: { ...process.env, REA_PROCESS_RUN_ID: "rea-snapshot-probe" }, timeout: 10_000, }); expect(JSON.parse(stdout)).toEqual({ preservedOtherErrno: true, retriedAfterFreshSize: true, + openingRunTokenSnapshotReads: 1, + failedOpeningRunTokens: [42, 43].map((pid) => ({ + pid, + state: "unavailable", + reason: "process_table_failed_13", + })), + absentRunTokens: [ + { pid: 42, state: "unavailable", reason: "process_not_in_snapshot" }, + ], + failedConfirmingRunTokens: [ + { + state: "unavailable", + reason: "process_table_failed_5", + tokenPresent: false, + }, + ], + confirmedRunTokens: [{ state: "readable", matchesFixtureToken: true }], successfulSnapshotMissingPidIsAbsent: true, failedSnapshotMissingPidIsUnavailable: true, invalidPidIsUnavailable: true, diff --git a/tests/fixtures/processRunTokenSnapshotProbe.swift b/tests/fixtures/processRunTokenSnapshotProbe.swift index 393809c9e8..51a8f9d21e 100644 --- a/tests/fixtures/processRunTokenSnapshotProbe.swift +++ b/tests/fixtures/processRunTokenSnapshotProbe.swift @@ -30,6 +30,49 @@ struct ProcessRunTokenSnapshotProbe { return -1 } + var openingSizeQueries = 0 + let failedOpeningTokens = readRunTokens(pids: [42, 43]) { _, output, size in + if let output = output { + output.assumingMemoryBound(to: kinfo_proc.self).pointee.kp_proc.p_pid = 42 + size = MemoryLayout.stride + return 0 + } + openingSizeQueries += 1 + if openingSizeQueries == 1 { + errno = EACCES + return -1 + } + size = MemoryLayout.stride + return 0 + } + let absentTokens = readRunTokens(pids: [42]) { _, output, size in + size = output == nil ? MemoryLayout.stride : 0 + return 0 + } + + let ownPid = getpid() + var confirmationSizeQueries = 0 + let singleProcessTable: ProcessTableSysctlCall = { _, output, size in + if let output = output { + let process = output.assumingMemoryBound(to: kinfo_proc.self) + process.pointee.kp_proc.p_pid = ownPid + process.pointee.kp_proc.p_flag = P_LP64 + } + size = MemoryLayout.stride + return 0 + } + let failedConfirmingTokens = readRunTokens(pids: [ownPid]) { mib, output, size in + if output == nil { + confirmationSizeQueries += 1 + if confirmationSizeQueries == 2 { + errno = EIO + return -1 + } + } + return singleProcessTable(&mib, output, &size) + } + let confirmedTokens = readRunTokens(pids: [ownPid], sysctlCall: singleProcessTable) + let absentIdentities = readProcessIdentities( pids: [42], snapshot: ProcessSnapshot(metadata: [:], failure: nil) @@ -53,6 +96,11 @@ struct ProcessRunTokenSnapshotProbe { let output: [String: Any] = [ "retriedAfterFreshSize": retried.failure == nil && sizeQueries == 2 && readAttempts == 2, "preservedOtherErrno": otherError.failure == "process_table_failed_\(EACCES)", + "openingRunTokenSnapshotReads": openingSizeQueries, + "failedOpeningRunTokens": failedOpeningTokens.map { ["pid": $0.pid, "state": $0.state, "reason": $0.reason ?? ""] }, + "absentRunTokens": absentTokens.map { ["pid": $0.pid, "state": $0.state, "reason": $0.reason ?? ""] }, + "failedConfirmingRunTokens": failedConfirmingTokens.map { ["state": $0.state, "reason": $0.reason ?? "", "tokenPresent": $0.run_id != nil] }, + "confirmedRunTokens": confirmedTokens.map { ["state": $0.state, "matchesFixtureToken": $0.run_id == "rea-snapshot-probe"] }, "successfulSnapshotMissingPidIsAbsent": absentIdentities.first?.state == "absent" && absentIdentities.first?.reason == "not-in-snapshot", "failedSnapshotMissingPidIsUnavailable": failedIdentities.first?.state == "unavailable" && failedIdentities.first?.reason == "process_table_failed_13", "invalidPidIsUnavailable": invalidIdentity.first?.state == "unavailable" && invalidIdentity.first?.reason == "invalid_process_id",