mirror of
https://github.com/morluto/rea.git
synced 2026-11-20 13:45:30 +00:00
fix(process): preserve Darwin snapshot and cleanup failures (#12100)
* fix(process): preserve Darwin opening snapshot failures * fix(process): retain the exact Darwin helper cleanup owner Transfer the failed helper-root capability across repeated prepare failures. Reuse that same owner for retirement and reader close so stale cleanup cannot remove a replacement helper. Co-authored-by: Anandumv <anandumv@gmail.com> Co-authored-by: nextjohnp7 <nextjohnp7@gmail.com> * test(process): isolate retained cleanup and compiler acknowledgements --------- Co-authored-by: Anandumv <anandumv@gmail.com> Co-authored-by: nextjohnp7 <nextjohnp7@gmail.com>
This commit is contained in:
6 files changed
+216
-53
No files matched your search
@@ -4,10 +4,14 @@ import {
|
||||
readProcessRunTokens,
|
||||
} from "./ProcessRunTokenObservations.js";
|
||||
import { ExecFileOutputCancelledError } from "./ExecFileOutput.js";
|
||||
import { basename, join } from "node:path";
|
||||
import { basename, dirname, join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { access, mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { expect, it, onTestFinished, vi } from "vitest";
|
||||
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
|
||||
import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js";
|
||||
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
|
||||
import { RetainedCommandCleanup } from "./RetainedCommandCleanup.js";
|
||||
|
||||
const fixture = vi.hoisted(() => {
|
||||
let start!: () => void;
|
||||
@@ -17,7 +21,7 @@ const fixture = vi.hoisted(() => {
|
||||
let stopped: Promise<void>;
|
||||
let gate: Promise<void>;
|
||||
let mode: "failure" | "abort" | "cleanup" | "abort-error";
|
||||
let failRemoval = false;
|
||||
let remainingRemovalFailures = 0;
|
||||
let calls = 0;
|
||||
const compilerFailure = new Error("controlled compiler failure");
|
||||
const compilerAbortError = new DOMException(
|
||||
@@ -27,8 +31,11 @@ const fixture = vi.hoisted(() => {
|
||||
return {
|
||||
compilerFailure,
|
||||
compilerAbortError,
|
||||
configure(selected: "failure" | "abort" | "cleanup" | "abort-error") {
|
||||
failRemoval = selected === "cleanup";
|
||||
configure(
|
||||
selected: "failure" | "abort" | "cleanup" | "abort-error",
|
||||
removalFailures = selected === "cleanup" ? 1 : 0,
|
||||
) {
|
||||
remainingRemovalFailures = removalFailures;
|
||||
mode = selected;
|
||||
calls = 0;
|
||||
started = new Promise((resolve) => {
|
||||
@@ -42,11 +49,14 @@ const fixture = vi.hoisted(() => {
|
||||
});
|
||||
},
|
||||
remove() {
|
||||
if (failRemoval) {
|
||||
failRemoval = false;
|
||||
if (remainingRemovalFailures > 0) {
|
||||
remainingRemovalFailures--;
|
||||
throw new Error("controlled cleanup failure");
|
||||
}
|
||||
},
|
||||
allowRemoval() {
|
||||
remainingRemovalFailures = 0;
|
||||
},
|
||||
get started() {
|
||||
return started;
|
||||
},
|
||||
@@ -102,18 +112,21 @@ import {
|
||||
DarwinProcessOwnershipInspectionError,
|
||||
} from "./DarwinProcessRunTokenReader.js";
|
||||
|
||||
const createFixtureReader = () =>
|
||||
createDarwinProcessRunTokenReader({
|
||||
compiler: {
|
||||
kind: "swiftc",
|
||||
executable: join(tmpdir(), "fixture-swiftc"),
|
||||
sdkRoot: tmpdir(),
|
||||
},
|
||||
environment: {},
|
||||
});
|
||||
|
||||
it.each(["failure", "abort", "cleanup", "abort-error"] as const)(
|
||||
"handles a %s outcome for a waiter joining retirement",
|
||||
async (mode) => {
|
||||
fixture.configure(mode);
|
||||
const reader = createDarwinProcessRunTokenReader({
|
||||
compiler: {
|
||||
kind: "swiftc",
|
||||
executable: join(tmpdir(), "fixture-swiftc"),
|
||||
sdkRoot: tmpdir(),
|
||||
},
|
||||
environment: {},
|
||||
});
|
||||
const reader = createFixtureReader();
|
||||
const controller = new AbortController();
|
||||
const first = reader.prepare(controller.signal);
|
||||
let second: Promise<string> | undefined;
|
||||
@@ -121,9 +134,9 @@ it.each(["failure", "abort", "cleanup", "abort-error"] as const)(
|
||||
fixture.release();
|
||||
await Promise.allSettled([first, second, reader.close()]);
|
||||
});
|
||||
await fixture.started;
|
||||
await awaitAcknowledgement(fixture.started, first, "compiler startup");
|
||||
controller.abort(new Error("first caller left"));
|
||||
await fixture.stopped;
|
||||
await awaitAcknowledgement(fixture.stopped, first, "compiler cancellation");
|
||||
second = reader.prepare();
|
||||
fixture.release();
|
||||
if (mode === "cleanup")
|
||||
@@ -196,3 +209,63 @@ it.each(["failure", "abort", "cleanup", "abort-error"] as const)(
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("transfers the same helper root after a second failed cleanup", async () => {
|
||||
fixture.configure("failure", 2);
|
||||
const reader = createFixtureReader();
|
||||
const prepare = reader.prepare();
|
||||
const first = prepare.catch((cause: unknown) => cause);
|
||||
onTestFinished(async () => {
|
||||
fixture.release();
|
||||
await first;
|
||||
fixture.allowRemoval();
|
||||
await reader.close();
|
||||
});
|
||||
await awaitAcknowledgement(fixture.started, prepare, "compiler startup");
|
||||
fixture.release();
|
||||
const initialFailure = await first;
|
||||
if (!(initialFailure instanceof DarwinProcessOwnershipInspectionError))
|
||||
throw new Error("Expected the initial inspection cleanup failure", {
|
||||
cause: initialFailure,
|
||||
});
|
||||
const [root] = initialFailure.cleanupFailure?.cleanupResources ?? [];
|
||||
if (root === undefined) throw new Error("Missing retained helper root");
|
||||
expect(initialFailure.cause).toBe(fixture.compilerFailure);
|
||||
expect(initialFailure.cleanupFailure?.cleanupIncomplete).toBe(true);
|
||||
await expect(access(root)).resolves.toBeUndefined();
|
||||
|
||||
const retryFailure = await reader.prepare().catch((cause: unknown) => cause);
|
||||
if (!(retryFailure instanceof DarwinProcessOwnershipInspectionError))
|
||||
throw new Error("Expected the retry inspection cleanup failure", {
|
||||
cause: retryFailure,
|
||||
});
|
||||
const owner = retryFailure.cleanupOwner;
|
||||
if (owner === undefined) throw new Error("Missing retry cleanup owner");
|
||||
expect(owner).toBe(initialFailure.cleanupOwner);
|
||||
expect(retryFailure.cleanupFailure).toBeInstanceOf(ProviderCleanupError);
|
||||
expect(retryFailure.cleanupFailure?.cleanupResources).toEqual([root]);
|
||||
expect(retryFailure.cleanupFailure?.cleanupIncomplete).toBe(true);
|
||||
expect(fixture.calls).toBe(1);
|
||||
|
||||
const fixtureDirectory = await createTestTempDirectory(
|
||||
"rea-retained-command-",
|
||||
);
|
||||
const scratch = join(fixtureDirectory, "command");
|
||||
await mkdir(scratch);
|
||||
const retained = new RetainedCommandCleanup(scratch, retryFailure);
|
||||
try {
|
||||
expect(retained.cleanupResources).toEqual([root, scratch]);
|
||||
} finally {
|
||||
await retained.close();
|
||||
}
|
||||
await expect(access(fixtureDirectory)).resolves.toBeUndefined();
|
||||
await expect(access(scratch)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await expect(access(root)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(initialFailure.cleanupFailure?.cleanupIncomplete).toBe(false);
|
||||
expect(retryFailure.cleanupFailure?.cleanupIncomplete).toBe(false);
|
||||
|
||||
const replacement = await reader.prepare();
|
||||
expect(dirname(replacement)).not.toBe(root);
|
||||
await owner.close();
|
||||
await expect(readFile(replacement, "utf8")).resolves.toBe("fixture");
|
||||
});
|
||||
@@ -406,6 +406,7 @@ const compilerInspectionError = (
|
||||
/** One allocated helper root, including its shared finalization and live cleanup status. */
|
||||
interface ProcessReaderRoot {
|
||||
readonly path: string;
|
||||
close(): Promise<void>;
|
||||
cleanupPromise?: Promise<void> | undefined;
|
||||
/** Reported by every removal failure until a retry removes the root. */
|
||||
cleanupRelease?: AnalysisCleanupRelease | undefined;
|
||||
@@ -533,8 +534,7 @@ export const createDarwinProcessRunTokenReader = (
|
||||
const cleanupAtExit = () => {
|
||||
if (root !== undefined) removeHelperRootAtExit(root.path);
|
||||
};
|
||||
const cleanupRoot = (target = root): Promise<void> => {
|
||||
if (target === undefined) return Promise.resolve();
|
||||
const cleanupRoot = (target: ProcessReaderRoot): Promise<void> => {
|
||||
target.cleanupPromise ??= Promise.resolve()
|
||||
.then(async () => {
|
||||
try {
|
||||
@@ -567,6 +567,33 @@ export const createDarwinProcessRunTokenReader = (
|
||||
});
|
||||
return target.cleanupPromise;
|
||||
};
|
||||
const cleanupFailedCompilationRoot = async (
|
||||
target: ProcessReaderRoot,
|
||||
primaryFailure?: ProcessReaderCompilation["primaryFailure"],
|
||||
): Promise<void> => {
|
||||
try {
|
||||
await target.close();
|
||||
} catch (cleanup: unknown) {
|
||||
if (!(cleanup instanceof ProviderCleanupError)) throw cleanup;
|
||||
const cause =
|
||||
primaryFailure === undefined ? cleanup : primaryFailure.cause;
|
||||
const primary =
|
||||
cause instanceof DarwinProcessOwnershipInspectionError
|
||||
? cause
|
||||
: new DarwinProcessOwnershipInspectionError(
|
||||
primaryFailure === undefined
|
||||
? "Darwin process ownership helper root cleanup retry failed"
|
||||
: "Darwin process ownership helper preparation failed",
|
||||
{
|
||||
cause,
|
||||
...(cause instanceof ProcessOwnershipInspectionCancelled
|
||||
? { cancellation: { reason: cause.reason } }
|
||||
: {}),
|
||||
},
|
||||
);
|
||||
throw primary.withCleanupFailure(cleanup, target);
|
||||
}
|
||||
};
|
||||
const runCompilation = async (
|
||||
operation: ProcessReaderCompilation,
|
||||
): Promise<string> => {
|
||||
@@ -592,10 +619,12 @@ export const createDarwinProcessRunTokenReader = (
|
||||
executable = cached.executable;
|
||||
return cached.executable;
|
||||
}
|
||||
operationRoot = {
|
||||
const allocatedRoot: ProcessReaderRoot = {
|
||||
path: await mkdtemp(join(tmpdir(), "rea-process-token-")),
|
||||
close: () => cleanupRoot(allocatedRoot),
|
||||
};
|
||||
root = operationRoot;
|
||||
operationRoot = allocatedRoot;
|
||||
root = allocatedRoot;
|
||||
process.once("exit", cleanupAtExit);
|
||||
exitCleanupInstalled = true;
|
||||
if (signal.aborted || closed)
|
||||
@@ -654,26 +683,10 @@ export const createDarwinProcessRunTokenReader = (
|
||||
// The extra pipe keeps settlement pending for Swift frontend writers
|
||||
// that inherit it after the driver and its standard streams close.
|
||||
// It cannot establish completion of a child that did not inherit it.
|
||||
try {
|
||||
await cleanupRoot(operationRoot);
|
||||
} catch (cleanup: unknown) {
|
||||
if (!(cleanup instanceof ProviderCleanupError)) throw cleanup;
|
||||
const primary =
|
||||
cause instanceof DarwinProcessOwnershipInspectionError
|
||||
? cause
|
||||
: new DarwinProcessOwnershipInspectionError(
|
||||
"Darwin process ownership helper preparation failed",
|
||||
{
|
||||
cause,
|
||||
...(cause instanceof ProcessOwnershipInspectionCancelled
|
||||
? { cancellation: { reason: cause.reason } }
|
||||
: {}),
|
||||
},
|
||||
);
|
||||
throw primary.withCleanupFailure(cleanup, {
|
||||
close: () => cleanupRoot(operationRoot),
|
||||
});
|
||||
}
|
||||
await cleanupFailedCompilationRoot(
|
||||
operationRoot,
|
||||
operation.primaryFailure,
|
||||
);
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
@@ -706,7 +719,7 @@ export const createDarwinProcessRunTokenReader = (
|
||||
assertProcessOwnershipInspectionActive(signal);
|
||||
if (executable !== undefined) return executable;
|
||||
if (compilation === undefined && root !== undefined) {
|
||||
await cleanupRoot();
|
||||
await cleanupFailedCompilationRoot(root);
|
||||
continue;
|
||||
}
|
||||
let operation = compilation;
|
||||
@@ -920,7 +933,7 @@ export const createDarwinProcessRunTokenReader = (
|
||||
// Each owner keeps its own failed cleanup for the next close attempt;
|
||||
// a failure of both stays incomplete until both owners recover.
|
||||
const [rootCleanup, cacheCleanup] = await Promise.allSettled([
|
||||
cleanupRoot(),
|
||||
root?.close(),
|
||||
helperCache?.close(),
|
||||
]);
|
||||
if (
|
||||
|
||||
@@ -48,6 +48,11 @@ it.each([
|
||||
{ present: false, reason: "unused", availability: "absent" },
|
||||
{ present: true, reason: "process_not_in_snapshot", availability: "absent" },
|
||||
{ present: true, reason: "sysctl_failed_13", availability: "unavailable" },
|
||||
{
|
||||
present: true,
|
||||
reason: "process_table_failed_13",
|
||||
availability: "unavailable",
|
||||
},
|
||||
])(
|
||||
"retains native run-token availability at the host adapter: %j",
|
||||
async (state) => {
|
||||
|
||||
Reference in new issue
Block a user