mirror of
https://github.com/morluto/rea.git
synced 2026-11-19 13:12:26 +00:00
fix(process): preserve Darwin snapshot and cleanup failures (#12100)
* fix(process): preserve Darwin opening snapshot failures * fix(process): retain the exact Darwin helper cleanup owner Transfer the failed helper-root capability across repeated prepare failures. Reuse that same owner for retirement and reader close so stale cleanup cannot remove a replacement helper. Co-authored-by: Anandumv <anandumv@gmail.com> Co-authored-by: nextjohnp7 <nextjohnp7@gmail.com> * test(process): isolate retained cleanup and compiler acknowledgements --------- Co-authored-by: Anandumv <anandumv@gmail.com> Co-authored-by: nextjohnp7 <nextjohnp7@gmail.com>
This commit is contained in:
6 files changed
+216
-53
No files matched your search
@@ -47,14 +47,14 @@ typealias ProcessTableSysctlCall = (
|
||||
_ size: inout Int
|
||||
) -> Int32
|
||||
|
||||
private func readProcessSnapshot() -> ProcessSnapshot {
|
||||
readProcessSnapshot { mib, output, size in
|
||||
sysctl(&mib, u_int(mib.count), output, &size, nil, 0)
|
||||
}
|
||||
private func systemProcessTableSysctl(
|
||||
_ mib: inout [Int32], _ output: UnsafeMutableRawPointer?, _ size: inout Int
|
||||
) -> Int32 {
|
||||
sysctl(&mib, u_int(mib.count), output, &size, nil, 0)
|
||||
}
|
||||
|
||||
func readProcessSnapshot(
|
||||
sysctlCall: ProcessTableSysctlCall
|
||||
sysctlCall: ProcessTableSysctlCall = systemProcessTableSysctl
|
||||
) -> ProcessSnapshot {
|
||||
for _ in 0..<3 {
|
||||
var mib = [CTL_KERN, KERN_PROC, KERN_PROC_ALL, 0]
|
||||
@@ -306,15 +306,21 @@ private func readRunToken(pid: Int32, pointerSize: Int) -> RunTokenObservation {
|
||||
)
|
||||
}
|
||||
|
||||
func readRunTokens(pids: [Int32]) -> [RunTokenObservation] {
|
||||
let before = readProcessSnapshot()
|
||||
func readRunTokens(
|
||||
pids: [Int32], sysctlCall: ProcessTableSysctlCall = systemProcessTableSysctl
|
||||
) -> [RunTokenObservation] {
|
||||
let before = readProcessSnapshot(sysctlCall: sysctlCall)
|
||||
// A later snapshot cannot establish the identity that opened the bracket.
|
||||
if let failure = before.failure {
|
||||
return pids.map { unavailable(pid: $0, reason: failure) }
|
||||
}
|
||||
let observations = pids.map { pid -> RunTokenObservation in
|
||||
guard let metadata = before.metadata[pid] else {
|
||||
return unavailable(pid: pid, reason: before.failure ?? "process_not_in_snapshot")
|
||||
return unavailable(pid: pid, reason: "process_not_in_snapshot")
|
||||
}
|
||||
return readRunToken(pid: pid, pointerSize: metadata.pointerSize)
|
||||
}
|
||||
let after = readProcessSnapshot()
|
||||
let after = readProcessSnapshot(sysctlCall: sysctlCall)
|
||||
return zip(pids, observations).map { pid, observation in
|
||||
guard let initial = before.metadata[pid], let current = after.metadata[pid] else {
|
||||
return unavailable(pid: pid, reason: after.failure ?? "process_not_in_snapshot")
|
||||
|
||||
@@ -4,10 +4,14 @@ import {
|
||||
readProcessRunTokens,
|
||||
} from "./ProcessRunTokenObservations.js";
|
||||
import { ExecFileOutputCancelledError } from "./ExecFileOutput.js";
|
||||
import { basename, join } from "node:path";
|
||||
import { basename, dirname, join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { access, mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { expect, it, onTestFinished, vi } from "vitest";
|
||||
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
|
||||
import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js";
|
||||
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
|
||||
import { RetainedCommandCleanup } from "./RetainedCommandCleanup.js";
|
||||
|
||||
const fixture = vi.hoisted(() => {
|
||||
let start!: () => void;
|
||||
@@ -17,7 +21,7 @@ const fixture = vi.hoisted(() => {
|
||||
let stopped: Promise<void>;
|
||||
let gate: Promise<void>;
|
||||
let mode: "failure" | "abort" | "cleanup" | "abort-error";
|
||||
let failRemoval = false;
|
||||
let remainingRemovalFailures = 0;
|
||||
let calls = 0;
|
||||
const compilerFailure = new Error("controlled compiler failure");
|
||||
const compilerAbortError = new DOMException(
|
||||
@@ -27,8 +31,11 @@ const fixture = vi.hoisted(() => {
|
||||
return {
|
||||
compilerFailure,
|
||||
compilerAbortError,
|
||||
configure(selected: "failure" | "abort" | "cleanup" | "abort-error") {
|
||||
failRemoval = selected === "cleanup";
|
||||
configure(
|
||||
selected: "failure" | "abort" | "cleanup" | "abort-error",
|
||||
removalFailures = selected === "cleanup" ? 1 : 0,
|
||||
) {
|
||||
remainingRemovalFailures = removalFailures;
|
||||
mode = selected;
|
||||
calls = 0;
|
||||
started = new Promise((resolve) => {
|
||||
@@ -42,11 +49,14 @@ const fixture = vi.hoisted(() => {
|
||||
});
|
||||
},
|
||||
remove() {
|
||||
if (failRemoval) {
|
||||
failRemoval = false;
|
||||
if (remainingRemovalFailures > 0) {
|
||||
remainingRemovalFailures--;
|
||||
throw new Error("controlled cleanup failure");
|
||||
}
|
||||
},
|
||||
allowRemoval() {
|
||||
remainingRemovalFailures = 0;
|
||||
},
|
||||
get started() {
|
||||
return started;
|
||||
},
|
||||
@@ -102,18 +112,21 @@ import {
|
||||
DarwinProcessOwnershipInspectionError,
|
||||
} from "./DarwinProcessRunTokenReader.js";
|
||||
|
||||
const createFixtureReader = () =>
|
||||
createDarwinProcessRunTokenReader({
|
||||
compiler: {
|
||||
kind: "swiftc",
|
||||
executable: join(tmpdir(), "fixture-swiftc"),
|
||||
sdkRoot: tmpdir(),
|
||||
},
|
||||
environment: {},
|
||||
});
|
||||
|
||||
it.each(["failure", "abort", "cleanup", "abort-error"] as const)(
|
||||
"handles a %s outcome for a waiter joining retirement",
|
||||
async (mode) => {
|
||||
fixture.configure(mode);
|
||||
const reader = createDarwinProcessRunTokenReader({
|
||||
compiler: {
|
||||
kind: "swiftc",
|
||||
executable: join(tmpdir(), "fixture-swiftc"),
|
||||
sdkRoot: tmpdir(),
|
||||
},
|
||||
environment: {},
|
||||
});
|
||||
const reader = createFixtureReader();
|
||||
const controller = new AbortController();
|
||||
const first = reader.prepare(controller.signal);
|
||||
let second: Promise<string> | undefined;
|
||||
@@ -121,9 +134,9 @@ it.each(["failure", "abort", "cleanup", "abort-error"] as const)(
|
||||
fixture.release();
|
||||
await Promise.allSettled([first, second, reader.close()]);
|
||||
});
|
||||
await fixture.started;
|
||||
await awaitAcknowledgement(fixture.started, first, "compiler startup");
|
||||
controller.abort(new Error("first caller left"));
|
||||
await fixture.stopped;
|
||||
await awaitAcknowledgement(fixture.stopped, first, "compiler cancellation");
|
||||
second = reader.prepare();
|
||||
fixture.release();
|
||||
if (mode === "cleanup")
|
||||
@@ -196,3 +209,63 @@ it.each(["failure", "abort", "cleanup", "abort-error"] as const)(
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("transfers the same helper root after a second failed cleanup", async () => {
|
||||
fixture.configure("failure", 2);
|
||||
const reader = createFixtureReader();
|
||||
const prepare = reader.prepare();
|
||||
const first = prepare.catch((cause: unknown) => cause);
|
||||
onTestFinished(async () => {
|
||||
fixture.release();
|
||||
await first;
|
||||
fixture.allowRemoval();
|
||||
await reader.close();
|
||||
});
|
||||
await awaitAcknowledgement(fixture.started, prepare, "compiler startup");
|
||||
fixture.release();
|
||||
const initialFailure = await first;
|
||||
if (!(initialFailure instanceof DarwinProcessOwnershipInspectionError))
|
||||
throw new Error("Expected the initial inspection cleanup failure", {
|
||||
cause: initialFailure,
|
||||
});
|
||||
const [root] = initialFailure.cleanupFailure?.cleanupResources ?? [];
|
||||
if (root === undefined) throw new Error("Missing retained helper root");
|
||||
expect(initialFailure.cause).toBe(fixture.compilerFailure);
|
||||
expect(initialFailure.cleanupFailure?.cleanupIncomplete).toBe(true);
|
||||
await expect(access(root)).resolves.toBeUndefined();
|
||||
|
||||
const retryFailure = await reader.prepare().catch((cause: unknown) => cause);
|
||||
if (!(retryFailure instanceof DarwinProcessOwnershipInspectionError))
|
||||
throw new Error("Expected the retry inspection cleanup failure", {
|
||||
cause: retryFailure,
|
||||
});
|
||||
const owner = retryFailure.cleanupOwner;
|
||||
if (owner === undefined) throw new Error("Missing retry cleanup owner");
|
||||
expect(owner).toBe(initialFailure.cleanupOwner);
|
||||
expect(retryFailure.cleanupFailure).toBeInstanceOf(ProviderCleanupError);
|
||||
expect(retryFailure.cleanupFailure?.cleanupResources).toEqual([root]);
|
||||
expect(retryFailure.cleanupFailure?.cleanupIncomplete).toBe(true);
|
||||
expect(fixture.calls).toBe(1);
|
||||
|
||||
const fixtureDirectory = await createTestTempDirectory(
|
||||
"rea-retained-command-",
|
||||
);
|
||||
const scratch = join(fixtureDirectory, "command");
|
||||
await mkdir(scratch);
|
||||
const retained = new RetainedCommandCleanup(scratch, retryFailure);
|
||||
try {
|
||||
expect(retained.cleanupResources).toEqual([root, scratch]);
|
||||
} finally {
|
||||
await retained.close();
|
||||
}
|
||||
await expect(access(fixtureDirectory)).resolves.toBeUndefined();
|
||||
await expect(access(scratch)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await expect(access(root)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(initialFailure.cleanupFailure?.cleanupIncomplete).toBe(false);
|
||||
expect(retryFailure.cleanupFailure?.cleanupIncomplete).toBe(false);
|
||||
|
||||
const replacement = await reader.prepare();
|
||||
expect(dirname(replacement)).not.toBe(root);
|
||||
await owner.close();
|
||||
await expect(readFile(replacement, "utf8")).resolves.toBe("fixture");
|
||||
});
|
||||
@@ -406,6 +406,7 @@ const compilerInspectionError = (
|
||||
/** One allocated helper root, including its shared finalization and live cleanup status. */
|
||||
interface ProcessReaderRoot {
|
||||
readonly path: string;
|
||||
close(): Promise<void>;
|
||||
cleanupPromise?: Promise<void> | undefined;
|
||||
/** Reported by every removal failure until a retry removes the root. */
|
||||
cleanupRelease?: AnalysisCleanupRelease | undefined;
|
||||
@@ -533,8 +534,7 @@ export const createDarwinProcessRunTokenReader = (
|
||||
const cleanupAtExit = () => {
|
||||
if (root !== undefined) removeHelperRootAtExit(root.path);
|
||||
};
|
||||
const cleanupRoot = (target = root): Promise<void> => {
|
||||
if (target === undefined) return Promise.resolve();
|
||||
const cleanupRoot = (target: ProcessReaderRoot): Promise<void> => {
|
||||
target.cleanupPromise ??= Promise.resolve()
|
||||
.then(async () => {
|
||||
try {
|
||||
@@ -567,6 +567,33 @@ export const createDarwinProcessRunTokenReader = (
|
||||
});
|
||||
return target.cleanupPromise;
|
||||
};
|
||||
const cleanupFailedCompilationRoot = async (
|
||||
target: ProcessReaderRoot,
|
||||
primaryFailure?: ProcessReaderCompilation["primaryFailure"],
|
||||
): Promise<void> => {
|
||||
try {
|
||||
await target.close();
|
||||
} catch (cleanup: unknown) {
|
||||
if (!(cleanup instanceof ProviderCleanupError)) throw cleanup;
|
||||
const cause =
|
||||
primaryFailure === undefined ? cleanup : primaryFailure.cause;
|
||||
const primary =
|
||||
cause instanceof DarwinProcessOwnershipInspectionError
|
||||
? cause
|
||||
: new DarwinProcessOwnershipInspectionError(
|
||||
primaryFailure === undefined
|
||||
? "Darwin process ownership helper root cleanup retry failed"
|
||||
: "Darwin process ownership helper preparation failed",
|
||||
{
|
||||
cause,
|
||||
...(cause instanceof ProcessOwnershipInspectionCancelled
|
||||
? { cancellation: { reason: cause.reason } }
|
||||
: {}),
|
||||
},
|
||||
);
|
||||
throw primary.withCleanupFailure(cleanup, target);
|
||||
}
|
||||
};
|
||||
const runCompilation = async (
|
||||
operation: ProcessReaderCompilation,
|
||||
): Promise<string> => {
|
||||
@@ -592,10 +619,12 @@ export const createDarwinProcessRunTokenReader = (
|
||||
executable = cached.executable;
|
||||
return cached.executable;
|
||||
}
|
||||
operationRoot = {
|
||||
const allocatedRoot: ProcessReaderRoot = {
|
||||
path: await mkdtemp(join(tmpdir(), "rea-process-token-")),
|
||||
close: () => cleanupRoot(allocatedRoot),
|
||||
};
|
||||
root = operationRoot;
|
||||
operationRoot = allocatedRoot;
|
||||
root = allocatedRoot;
|
||||
process.once("exit", cleanupAtExit);
|
||||
exitCleanupInstalled = true;
|
||||
if (signal.aborted || closed)
|
||||
@@ -654,26 +683,10 @@ export const createDarwinProcessRunTokenReader = (
|
||||
// The extra pipe keeps settlement pending for Swift frontend writers
|
||||
// that inherit it after the driver and its standard streams close.
|
||||
// It cannot establish completion of a child that did not inherit it.
|
||||
try {
|
||||
await cleanupRoot(operationRoot);
|
||||
} catch (cleanup: unknown) {
|
||||
if (!(cleanup instanceof ProviderCleanupError)) throw cleanup;
|
||||
const primary =
|
||||
cause instanceof DarwinProcessOwnershipInspectionError
|
||||
? cause
|
||||
: new DarwinProcessOwnershipInspectionError(
|
||||
"Darwin process ownership helper preparation failed",
|
||||
{
|
||||
cause,
|
||||
...(cause instanceof ProcessOwnershipInspectionCancelled
|
||||
? { cancellation: { reason: cause.reason } }
|
||||
: {}),
|
||||
},
|
||||
);
|
||||
throw primary.withCleanupFailure(cleanup, {
|
||||
close: () => cleanupRoot(operationRoot),
|
||||
});
|
||||
}
|
||||
await cleanupFailedCompilationRoot(
|
||||
operationRoot,
|
||||
operation.primaryFailure,
|
||||
);
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
@@ -706,7 +719,7 @@ export const createDarwinProcessRunTokenReader = (
|
||||
assertProcessOwnershipInspectionActive(signal);
|
||||
if (executable !== undefined) return executable;
|
||||
if (compilation === undefined && root !== undefined) {
|
||||
await cleanupRoot();
|
||||
await cleanupFailedCompilationRoot(root);
|
||||
continue;
|
||||
}
|
||||
let operation = compilation;
|
||||
@@ -920,7 +933,7 @@ export const createDarwinProcessRunTokenReader = (
|
||||
// Each owner keeps its own failed cleanup for the next close attempt;
|
||||
// a failure of both stays incomplete until both owners recover.
|
||||
const [rootCleanup, cacheCleanup] = await Promise.allSettled([
|
||||
cleanupRoot(),
|
||||
root?.close(),
|
||||
helperCache?.close(),
|
||||
]);
|
||||
if (
|
||||
|
||||
@@ -48,6 +48,11 @@ it.each([
|
||||
{ present: false, reason: "unused", availability: "absent" },
|
||||
{ present: true, reason: "process_not_in_snapshot", availability: "absent" },
|
||||
{ present: true, reason: "sysctl_failed_13", availability: "unavailable" },
|
||||
{
|
||||
present: true,
|
||||
reason: "process_table_failed_13",
|
||||
availability: "unavailable",
|
||||
},
|
||||
])(
|
||||
"retains native run-token availability at the host adapter: %j",
|
||||
async (state) => {
|
||||
|
||||
@@ -1482,7 +1482,7 @@ it.skipIf(!onDarwin)(
|
||||
);
|
||||
|
||||
it.skipIf(!onDarwin)(
|
||||
"retries a growing process table after ENOMEM and preserves other sysctl errors",
|
||||
"preserves process-table failures and confirms run-token identity",
|
||||
{ tags: ["long-running"] },
|
||||
async () => {
|
||||
const root = await mkdtemp(
|
||||
@@ -1509,11 +1509,29 @@ it.skipIf(!onDarwin)(
|
||||
);
|
||||
const { stdout } = await execFileAsync(executable, [], {
|
||||
encoding: "utf8",
|
||||
env: { ...process.env, REA_PROCESS_RUN_ID: "rea-snapshot-probe" },
|
||||
timeout: 10_000,
|
||||
});
|
||||
expect(JSON.parse(stdout)).toEqual({
|
||||
preservedOtherErrno: true,
|
||||
retriedAfterFreshSize: true,
|
||||
openingRunTokenSnapshotReads: 1,
|
||||
failedOpeningRunTokens: [42, 43].map((pid) => ({
|
||||
pid,
|
||||
state: "unavailable",
|
||||
reason: "process_table_failed_13",
|
||||
})),
|
||||
absentRunTokens: [
|
||||
{ pid: 42, state: "unavailable", reason: "process_not_in_snapshot" },
|
||||
],
|
||||
failedConfirmingRunTokens: [
|
||||
{
|
||||
state: "unavailable",
|
||||
reason: "process_table_failed_5",
|
||||
tokenPresent: false,
|
||||
},
|
||||
],
|
||||
confirmedRunTokens: [{ state: "readable", matchesFixtureToken: true }],
|
||||
successfulSnapshotMissingPidIsAbsent: true,
|
||||
failedSnapshotMissingPidIsUnavailable: true,
|
||||
invalidPidIsUnavailable: true,
|
||||
|
||||
@@ -30,6 +30,49 @@ struct ProcessRunTokenSnapshotProbe {
|
||||
return -1
|
||||
}
|
||||
|
||||
var openingSizeQueries = 0
|
||||
let failedOpeningTokens = readRunTokens(pids: [42, 43]) { _, output, size in
|
||||
if let output = output {
|
||||
output.assumingMemoryBound(to: kinfo_proc.self).pointee.kp_proc.p_pid = 42
|
||||
size = MemoryLayout<kinfo_proc>.stride
|
||||
return 0
|
||||
}
|
||||
openingSizeQueries += 1
|
||||
if openingSizeQueries == 1 {
|
||||
errno = EACCES
|
||||
return -1
|
||||
}
|
||||
size = MemoryLayout<kinfo_proc>.stride
|
||||
return 0
|
||||
}
|
||||
let absentTokens = readRunTokens(pids: [42]) { _, output, size in
|
||||
size = output == nil ? MemoryLayout<kinfo_proc>.stride : 0
|
||||
return 0
|
||||
}
|
||||
|
||||
let ownPid = getpid()
|
||||
var confirmationSizeQueries = 0
|
||||
let singleProcessTable: ProcessTableSysctlCall = { _, output, size in
|
||||
if let output = output {
|
||||
let process = output.assumingMemoryBound(to: kinfo_proc.self)
|
||||
process.pointee.kp_proc.p_pid = ownPid
|
||||
process.pointee.kp_proc.p_flag = P_LP64
|
||||
}
|
||||
size = MemoryLayout<kinfo_proc>.stride
|
||||
return 0
|
||||
}
|
||||
let failedConfirmingTokens = readRunTokens(pids: [ownPid]) { mib, output, size in
|
||||
if output == nil {
|
||||
confirmationSizeQueries += 1
|
||||
if confirmationSizeQueries == 2 {
|
||||
errno = EIO
|
||||
return -1
|
||||
}
|
||||
}
|
||||
return singleProcessTable(&mib, output, &size)
|
||||
}
|
||||
let confirmedTokens = readRunTokens(pids: [ownPid], sysctlCall: singleProcessTable)
|
||||
|
||||
let absentIdentities = readProcessIdentities(
|
||||
pids: [42],
|
||||
snapshot: ProcessSnapshot(metadata: [:], failure: nil)
|
||||
@@ -53,6 +96,11 @@ struct ProcessRunTokenSnapshotProbe {
|
||||
let output: [String: Any] = [
|
||||
"retriedAfterFreshSize": retried.failure == nil && sizeQueries == 2 && readAttempts == 2,
|
||||
"preservedOtherErrno": otherError.failure == "process_table_failed_\(EACCES)",
|
||||
"openingRunTokenSnapshotReads": openingSizeQueries,
|
||||
"failedOpeningRunTokens": failedOpeningTokens.map { ["pid": $0.pid, "state": $0.state, "reason": $0.reason ?? ""] },
|
||||
"absentRunTokens": absentTokens.map { ["pid": $0.pid, "state": $0.state, "reason": $0.reason ?? ""] },
|
||||
"failedConfirmingRunTokens": failedConfirmingTokens.map { ["state": $0.state, "reason": $0.reason ?? "", "tokenPresent": $0.run_id != nil] },
|
||||
"confirmedRunTokens": confirmedTokens.map { ["state": $0.state, "matchesFixtureToken": $0.run_id == "rea-snapshot-probe"] },
|
||||
"successfulSnapshotMissingPidIsAbsent": absentIdentities.first?.state == "absent" && absentIdentities.first?.reason == "not-in-snapshot",
|
||||
"failedSnapshotMissingPidIsUnavailable": failedIdentities.first?.state == "unavailable" && failedIdentities.first?.reason == "process_table_failed_13",
|
||||
"invalidPidIsUnavailable": invalidIdentity.first?.state == "unavailable" && invalidIdentity.first?.reason == "invalid_process_id",
|
||||
|
||||
Reference in new issue
Block a user