mirror of
https://github.com/morluto/rea.git
synced 2026-11-16 19:53:50 +00:00
fix(artifacts): run hdiutil under owned supervision with a SIGTERM-first stop (#11632)
* fix(artifacts): run hdiutil under owned supervision with a SIGTERM-first stop runOwnedCommand gains a per-command termination policy: before owned cleanup kills a command's process group, it sends the command's own stop signal to that group while the leader is unreaped and waits a bounded grace for the leader to exit. NativeDmgArtifactReader runs every hdiutil command through runOwnedCommand with SIGTERM and a 5 s grace, so an interrupted attach can roll back as before. An hdiutil whose cleanup fails is retained by the reader and retried on close, which reports it until it is released. Raw command failures are interpreted once in runChecked. Refs #11619 * fix(artifacts): release hdiutil commands retained during DMG reader close Detach and inventory commands that run during close can have their cleanup fail too; close now retries retained hdiutil processes again after detach and reports any that remain instead of resolving. * fix(artifacts): keep hdiutil's exit status when its cleanup fails and budget each output stream * fix(artifacts): admit no hdiutil command while an earlier one's cleanup is unresolved
This commit is contained in:
1 parent
ad4cc7536f
commit
d0dded7d6e
7 files changed
+576
-102
No files matched your search
@@ -8,6 +8,7 @@ import type { Readable } from "node:stream";
|
||||
import type { ArtifactCommand } from "../domain/artifactGraph.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
ArtifactResourceScope,
|
||||
type ArtifactEntry,
|
||||
type ArtifactReader,
|
||||
type ArtifactTraversalOptions,
|
||||
@@ -20,11 +21,16 @@ import {
|
||||
openRegularFile,
|
||||
} from "../filesystem/RegularFile.js";
|
||||
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import {
|
||||
ExecFileOutputCancelledError,
|
||||
execFileOutput,
|
||||
execFileOutputFailure,
|
||||
} from "../process/ExecFileOutput.js";
|
||||
OwnedCommandFailure,
|
||||
prepareOwnedCommand,
|
||||
runOwnedCommand,
|
||||
} from "../process/OwnedCommand.js";
|
||||
import type { ProcessTerminationPolicy } from "../process/ProviderProcess.js";
|
||||
import { launcherFailureOf } from "../process/launchFailure.js";
|
||||
import { capturedProviderOutput } from "../process/capturedProviderOutput.js";
|
||||
import { retainOwnedCommandCleanup } from "../providers/OwnedCommandCleanup.js";
|
||||
import {
|
||||
parseNativeDmgAttachOutput,
|
||||
parseNativeDmgInfo,
|
||||
@@ -34,7 +40,18 @@ import {
|
||||
} from "./NativeDmgOwnership.js";
|
||||
import { invariantViolation } from "../domain/internalInvariant.js";
|
||||
import { errorMessage } from "../kernel/errorMessage.js";
|
||||
const HDIUTIL = "/usr/bin/hdiutil";
|
||||
const HDIUTIL_COMMAND_TIMEOUT_MS = 120_000;
|
||||
/**
|
||||
* hdiutil rolls back an attach it started when asked to stop with SIGTERM, so
|
||||
* owned cleanup asks first and kills what remains only after this grace.
|
||||
*/
|
||||
const HDIUTIL_TERMINATION: ProcessTerminationPolicy = {
|
||||
signal: "SIGTERM",
|
||||
graceMs: 5_000,
|
||||
};
|
||||
/** Retained bytes per stream; attach and info plists are far smaller. */
|
||||
const HDIUTIL_OUTPUT_BYTES = 16 * 1024 * 1024;
|
||||
/** Conditional recovery guidance; a path match never grants attachment ownership. */
|
||||
const DMG_EXISTING_ATTACHMENT_RECOVERY =
|
||||
"hdiutil refused a private read-only DMG attachment after listing an existing attachment of this image path. If it is still attached, ask its owner to release it when safe, then retry. REA does not borrow or detach existing attachments.";
|
||||
@@ -46,7 +63,11 @@ const VERIFY_UNKNOWN_SETTLE_DELAYS_MS = [250, 500] as const;
|
||||
const DETACH_SETTLE_DELAYS_MS = [250, 500, 1000] as const;
|
||||
const delay = (milliseconds: number): Promise<void> =>
|
||||
new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||||
/** Narrow host seam for tested, shell-free hdiutil lifecycle operations. */
|
||||
/**
|
||||
* Narrow host seam for tested, shell-free hdiutil lifecycle operations. `run`
|
||||
* resolves with any ordinary exit and rejects with the command owner's raw
|
||||
* failure, such as an `OwnedCommandFailure` or a launcher's error.
|
||||
*/
|
||||
export interface NativeDmgHost {
|
||||
run(
|
||||
arguments_: readonly string[],
|
||||
@@ -60,6 +81,8 @@ export interface NativeDmgHost {
|
||||
}>;
|
||||
/** Wait between detach rechecks; tests may resolve immediately. */
|
||||
delay?(milliseconds: number): Promise<void>;
|
||||
/** Retry hdiutil processes whose cleanup failed; rejects while any remains. */
|
||||
close?(): Promise<void>;
|
||||
}
|
||||
|
||||
interface NativeDmgFailureClassification {
|
||||
@@ -111,48 +134,70 @@ class NativeDmgCommandFailure extends ArtifactReaderFailure {
|
||||
|
||||
const createSystemHost = (
|
||||
environment: Readonly<NodeJS.ProcessEnv>,
|
||||
): NativeDmgHost => ({
|
||||
delay,
|
||||
async run(arguments_, signal, options) {
|
||||
): NativeDmgHost => {
|
||||
// Owns hdiutil processes whose cleanup failed until the reader closes.
|
||||
const processes = new ArtifactResourceScope();
|
||||
const retained = {
|
||||
retain: (owner: { close(): Promise<void> }, resource: string) => {
|
||||
processes.retain({
|
||||
kind: "lease",
|
||||
resource,
|
||||
lease: { release: () => owner.close() },
|
||||
});
|
||||
},
|
||||
run: <T>(work: () => Promise<T>) => work(),
|
||||
};
|
||||
const runOwned: NativeDmgHost["run"] = async (
|
||||
arguments_,
|
||||
signal,
|
||||
options,
|
||||
) => {
|
||||
try {
|
||||
const { stdout, stderr } = await execFileOutput(
|
||||
"/usr/bin/hdiutil",
|
||||
[...arguments_],
|
||||
await prepareOwnedCommand(signal);
|
||||
const snapshot = await runOwnedCommand(
|
||||
{
|
||||
env: environment,
|
||||
stopSignal: "SIGTERM",
|
||||
...(options === undefined ? {} : { timeout: options.timeoutMs }),
|
||||
...(signal === undefined ? {} : { signal }),
|
||||
command: HDIUTIL,
|
||||
arguments: [...arguments_],
|
||||
runId: `rea-hdiutil-${randomUUID()}`,
|
||||
hostEnvironment: environment,
|
||||
},
|
||||
{
|
||||
timeoutMs: options?.timeoutMs ?? null,
|
||||
diagnosticBytes: {
|
||||
stdout: HDIUTIL_OUTPUT_BYTES,
|
||||
stderr: HDIUTIL_OUTPUT_BYTES,
|
||||
},
|
||||
acceptNonZeroExit: true,
|
||||
termination: HDIUTIL_TERMINATION,
|
||||
},
|
||||
{ signal },
|
||||
);
|
||||
return { stdout, stderr, exitCode: 0 };
|
||||
// An accepted run exited normally, so it has a numeric status.
|
||||
if (typeof snapshot.exitCode !== "number")
|
||||
throw invariantViolation("hdiutil completed without an exit status");
|
||||
const { stdout, stderr } = capturedProviderOutput(snapshot);
|
||||
return { stdout, stderr, exitCode: snapshot.exitCode };
|
||||
} catch (cause: unknown) {
|
||||
const output = execFileOutputFailure(cause);
|
||||
if (cause instanceof ExecFileOutputCancelledError)
|
||||
throw new NativeDmgCommandFailure(
|
||||
{ reason: "cancelled" },
|
||||
"DMG operation cancelled",
|
||||
output?.stdout ?? "",
|
||||
{ cause, cancellation: { reason: cause.cause } },
|
||||
);
|
||||
const exitCode =
|
||||
typeof output?.code === "number" ? output.code : processExitCode(cause);
|
||||
if (exitCode !== undefined)
|
||||
return {
|
||||
stdout: output?.stdout ?? processOutput(cause, "stdout"),
|
||||
stderr: output?.stderr ?? processOutput(cause, "stderr"),
|
||||
exitCode,
|
||||
cause,
|
||||
};
|
||||
throw new NativeDmgCommandFailure(
|
||||
classifyCommandFailure(cause, arguments_[0]),
|
||||
`hdiutil ${arguments_[0] ?? "operation"} failed: ${describeCommandFailure(cause, arguments_)}`,
|
||||
output?.stdout ?? "",
|
||||
{ cause, processNotStarted: processNotStarted(cause) },
|
||||
retainOwnedCommandCleanup(
|
||||
cause,
|
||||
retained,
|
||||
"artifact",
|
||||
`hdiutil ${arguments_[0] ?? "operation"}`,
|
||||
);
|
||||
throw cause;
|
||||
}
|
||||
},
|
||||
});
|
||||
};
|
||||
return {
|
||||
delay,
|
||||
// Admission retries retained cleanup first and stays blocked while it is
|
||||
// unresolved, so no hdiutil starts beside one REA could not stop.
|
||||
run: (arguments_, signal, options) =>
|
||||
processes.run(() => runOwned(arguments_, signal, options)),
|
||||
// The reader releases retained commands before and after its close-time
|
||||
// detach, so this retries them without ending admission.
|
||||
close: () => processes.retryCleanup(),
|
||||
};
|
||||
};
|
||||
|
||||
/** Read-only macOS DMG adapter that owns attachment and reverse-order detach. */
|
||||
export class NativeDmgArtifactReader implements ArtifactReader {
|
||||
@@ -261,6 +306,11 @@ export class NativeDmgArtifactReader implements ArtifactReader {
|
||||
await Promise.allSettled(this.#activeDirectoryOperations);
|
||||
await this.#closeVerificationProbeHandle();
|
||||
await this.#closeMountedDirectory();
|
||||
// An hdiutil whose cleanup failed may still change attachments, so it is
|
||||
// released before detaching.
|
||||
const retained = await this.#releaseRetainedCommands();
|
||||
if (retained !== undefined)
|
||||
throw ArtifactReaderFailure.withCleanup(retained.cause, retained.cleanup);
|
||||
let detachFailure = await this.#detachOwnedDevices();
|
||||
if (
|
||||
this.#devices.length === 0 &&
|
||||
@@ -282,6 +332,13 @@ export class NativeDmgArtifactReader implements ArtifactReader {
|
||||
"hdiutil attachment ownership remains unknown",
|
||||
);
|
||||
}
|
||||
// Detach and inventory commands run during close can be retained too.
|
||||
const retainedDuringClose = await this.#releaseRetainedCommands();
|
||||
if (detachFailure === undefined && retainedDuringClose !== undefined)
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
retainedDuringClose.cause,
|
||||
retainedDuringClose.cleanup,
|
||||
);
|
||||
if (detachFailure !== undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
@@ -300,12 +357,37 @@ export class NativeDmgArtifactReader implements ArtifactReader {
|
||||
...(this.#mountRoot === undefined
|
||||
? []
|
||||
: [`DMG mount root ${this.#mountRoot}`]),
|
||||
...(retainedDuringClose?.cleanup.resources ?? []),
|
||||
],
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/** Retry hdiutil processes whose cleanup failed; report any that remain. */
|
||||
async #releaseRetainedCommands(): Promise<
|
||||
| {
|
||||
readonly cause: unknown;
|
||||
readonly cleanup: ReturnType<
|
||||
typeof ArtifactReaderFailure.cleanupObservation
|
||||
>;
|
||||
}
|
||||
| undefined
|
||||
> {
|
||||
try {
|
||||
await this.host.close?.();
|
||||
return undefined;
|
||||
} catch (cause: unknown) {
|
||||
return {
|
||||
cause,
|
||||
cleanup: ArtifactReaderFailure.cleanupObservation(
|
||||
cause,
|
||||
"hdiutil process",
|
||||
),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async #closeVerificationProbeHandle(): Promise<void> {
|
||||
const owner = this.#verificationProbeHandle;
|
||||
if (owner === undefined) return;
|
||||
@@ -721,18 +803,40 @@ const existingImagePathListed = async (
|
||||
return false;
|
||||
};
|
||||
|
||||
const processNotStarted = (cause: unknown): boolean => {
|
||||
if (!(cause instanceof Error)) return false;
|
||||
const syscall = Reflect.get(cause, "syscall");
|
||||
const code = Reflect.get(cause, "code");
|
||||
const errno = Reflect.get(cause, "errno");
|
||||
return (
|
||||
typeof syscall === "string" &&
|
||||
syscall.startsWith("spawn ") &&
|
||||
(typeof code === "string" ||
|
||||
typeof code === "number" ||
|
||||
typeof errno === "number")
|
||||
);
|
||||
/**
|
||||
* The facts an interrupted or failed owned command reports, in the shape the
|
||||
* hdiutil classification reads: a deadline is ETIMEDOUT, and the stop signal
|
||||
* and retained output come from the command's snapshot. A command that exited
|
||||
* on its own with complete output keeps its exit status even when its cleanup
|
||||
* failed afterwards, so a verify exit is still classified from its diagnostics.
|
||||
*/
|
||||
const commandFailureFacts = (cause: unknown): unknown => {
|
||||
if (!(cause instanceof OwnedCommandFailure)) return cause;
|
||||
const output =
|
||||
cause.snapshot === null
|
||||
? undefined
|
||||
: capturedProviderOutput(cause.snapshot);
|
||||
const exitCode =
|
||||
cause.reason === "process" &&
|
||||
output !== undefined &&
|
||||
!output.truncated &&
|
||||
typeof output.exit_code === "number"
|
||||
? output.exit_code
|
||||
: undefined;
|
||||
return {
|
||||
...(exitCode === undefined ? {} : { exitCode }),
|
||||
code:
|
||||
cause.reason === "timeout"
|
||||
? "ETIMEDOUT"
|
||||
: cause.reason === "output-limit"
|
||||
? "ENOBUFS"
|
||||
: undefined,
|
||||
...(output?.signal == null ? {} : { signal: output.signal }),
|
||||
...(output === undefined
|
||||
? {}
|
||||
: { stdout: output.stdout, stderr: output.stderr }),
|
||||
message: cause.message,
|
||||
};
|
||||
};
|
||||
|
||||
const runChecked = async (
|
||||
@@ -746,22 +850,35 @@ const runChecked = async (
|
||||
result = await host.run(arguments_, signal, options);
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof ArtifactReaderFailure) throw cause;
|
||||
if (cause instanceof ExecFileOutputCancelledError)
|
||||
const stdout =
|
||||
cause instanceof OwnedCommandFailure && cause.snapshot !== null
|
||||
? capturedProviderOutput(cause.snapshot).stdout
|
||||
: "";
|
||||
if (
|
||||
cause instanceof OwnedCommandFailure &&
|
||||
cause.reason === "cancelled" &&
|
||||
cause.cancellation !== undefined
|
||||
)
|
||||
throw new NativeDmgCommandFailure(
|
||||
{ reason: "cancelled" },
|
||||
"DMG operation cancelled",
|
||||
execFileOutputFailure(cause)?.stdout ?? "",
|
||||
{ cause, cancellation: { reason: cause.cause } },
|
||||
stdout,
|
||||
{ cause, cancellation: cause.cancellation },
|
||||
);
|
||||
const output = execFileOutputFailure(cause);
|
||||
const facts = commandFailureFacts(cause);
|
||||
const exitCode = processExitCode(facts);
|
||||
throw new NativeDmgCommandFailure(
|
||||
classifyCommandFailure(cause, arguments_[0]),
|
||||
classifyCommandFailure(facts, arguments_[0]),
|
||||
`hdiutil ${arguments_[0] ?? "operation"} failed: ${describeCommandFailure(
|
||||
cause,
|
||||
facts,
|
||||
arguments_,
|
||||
)}`,
|
||||
output?.stdout ?? "",
|
||||
{ cause, processNotStarted: processNotStarted(cause) },
|
||||
stdout,
|
||||
{
|
||||
cause,
|
||||
processNotStarted: launcherFailureOf(cause) !== undefined,
|
||||
...(exitCode === undefined ? {} : { exitCode }),
|
||||
},
|
||||
);
|
||||
}
|
||||
if (result.exitCode !== 0) {
|
||||
@@ -815,8 +932,11 @@ const describeCommandFailure = (
|
||||
const code = Reflect.get(cause, "code");
|
||||
if (typeof code === "number") fields.exitCode = code;
|
||||
}
|
||||
if (cause instanceof ArtifactReaderFailure) fields.message = cause.message;
|
||||
else if (cause instanceof Error) fields.message = cause.message;
|
||||
if (cause instanceof Error) fields.message = cause.message;
|
||||
else if (typeof cause === "object" && cause !== null) {
|
||||
const message = Reflect.get(cause, "message");
|
||||
if (typeof message === "string") fields.message = message;
|
||||
}
|
||||
return JSON.stringify(fields);
|
||||
};
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { ExecFileOutputCancelledError } from "../process/ExecFileOutput.js";
|
||||
import { OwnedCommandFailure } from "../process/OwnedCommand.js";
|
||||
import { ArtifactReaderFailure } from "../resources/ArtifactReader.js";
|
||||
import { build } from "plist";
|
||||
import { expect, it, vi } from "vitest";
|
||||
@@ -155,7 +155,9 @@ it("uses the typed command cancellation owner for injected hosts and preserves p
|
||||
code: "EIO",
|
||||
});
|
||||
const failureCause = cancelled
|
||||
? new ExecFileOutputCancelledError(reason)
|
||||
? new OwnedCommandFailure("cancelled", "Command cancelled.", null, null, {
|
||||
cancellation: { reason },
|
||||
})
|
||||
: physical;
|
||||
const reader = new NativeDmgArtifactReader(
|
||||
"/fixture/image.dmg",
|
||||
@@ -184,3 +186,79 @@ it("uses the typed command cancellation owner for injected hosts and preserves p
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("reports an hdiutil retained while the reader closes, after its own cleanup commands", async () => {
|
||||
let closes = 0;
|
||||
const retained = new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"hdiutil cleanup withheld",
|
||||
{ cleanup: { reason: "withheld", resources: ["pid:4242"] } },
|
||||
);
|
||||
const host: NativeDmgHost = {
|
||||
run: async () => ({
|
||||
stdout: "",
|
||||
stderr: "hdiutil: verify failed - image not recognized",
|
||||
exitCode: 1,
|
||||
}),
|
||||
// Nothing is retained before close; a command run during close is.
|
||||
close: async () => {
|
||||
closes += 1;
|
||||
if (closes === 2) throw retained;
|
||||
},
|
||||
};
|
||||
const reader = new NativeDmgArtifactReader("/fixture/image.dmg", {}, host);
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const _entry of reader.entries()) {
|
||||
/* consume */
|
||||
}
|
||||
})(),
|
||||
).rejects.toMatchObject({ reason: "format" });
|
||||
const refused = await reader.close().catch((cause: unknown) => cause);
|
||||
expect(refused).toBeInstanceOf(ArtifactReaderFailure);
|
||||
expect((refused as ArtifactReaderFailure).cleanup?.resources).toEqual([
|
||||
"pid:4242",
|
||||
]);
|
||||
await reader.close();
|
||||
expect(closes).toBe(4);
|
||||
});
|
||||
|
||||
it("classifies a verify exit from its diagnostics when the command's cleanup then failed", async () => {
|
||||
const stderr = "hdiutil: verify failed - image not recognized";
|
||||
const exited = new OwnedCommandFailure(
|
||||
"process",
|
||||
"Command cleanup failed.",
|
||||
{
|
||||
stdout: { text: "", bytes: 0, observedBytes: 0 },
|
||||
stderr: {
|
||||
text: stderr,
|
||||
bytes: stderr.length,
|
||||
observedBytes: stderr.length,
|
||||
},
|
||||
exitCode: 1,
|
||||
signal: null,
|
||||
},
|
||||
"process group still present",
|
||||
);
|
||||
const host: NativeDmgHost = {
|
||||
run: async () => {
|
||||
throw exited;
|
||||
},
|
||||
close: async () => {},
|
||||
};
|
||||
const reader = new NativeDmgArtifactReader("/fixture/image.dmg", {}, host);
|
||||
try {
|
||||
const failure = await (async () => {
|
||||
for await (const _entry of reader.entries()) {
|
||||
/* consume */
|
||||
}
|
||||
})().catch((cause: unknown) => cause);
|
||||
expect(failure).toMatchObject({
|
||||
reason: "format",
|
||||
exitCode: 1,
|
||||
cause: exited,
|
||||
});
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
@@ -21,6 +21,7 @@ import type {
|
||||
ProcessDiagnosticBudget,
|
||||
ProviderProcessStopResult,
|
||||
ProviderProcessInspectionCleanup,
|
||||
ProcessTerminationPolicy,
|
||||
SpawnedOwnedProviderProcess,
|
||||
} from "./ProviderProcess.js";
|
||||
import {
|
||||
@@ -120,6 +121,11 @@ interface OwnedCommandLimits {
|
||||
* stays authoritative; a failing stream is a collection failure.
|
||||
*/
|
||||
readonly stdout?: Writable;
|
||||
/**
|
||||
* Ask the command to stop before owned cleanup kills its process group,
|
||||
* for a command that rolls back work it started when signaled.
|
||||
*/
|
||||
readonly termination?: ProcessTerminationPolicy;
|
||||
}
|
||||
|
||||
/** Keep independently bounded macOS helper compilation outside a short command deadline.
|
||||
@@ -485,7 +491,9 @@ const collectOwnedCommand = async (
|
||||
}
|
||||
// A finished sink ignores this; an unfinished one is destroyed.
|
||||
abandoned.abort(OwnerCancellation.noLongerNeeded);
|
||||
const stopped = await supervisor.stop();
|
||||
const stopped = await supervisor.stop(
|
||||
limits.termination === undefined ? {} : { termination: limits.termination },
|
||||
);
|
||||
const snapshot = supervisor.snapshot();
|
||||
// An input error can arrive after finish while process cleanup is still pending.
|
||||
if (failure === undefined && inputFailure !== undefined)
|
||||
|
||||
@@ -80,6 +80,19 @@ export interface SpawnedOwnedProviderProcess {
|
||||
readonly process: ProviderProcessHandle;
|
||||
readonly ownership: OwnedProcessGroup;
|
||||
readonly cleanup?: () => Promise<ProcessCleanupResult>;
|
||||
/** See the owned `ProviderProcessLaunch.requestStop`. */
|
||||
readonly requestStop?: (signal: NodeJS.Signals) => boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* A command's own stop request before owned cleanup: `signal` goes to the
|
||||
* launched process group, and cleanup waits up to `graceMs` for the leader to
|
||||
* exit before it kills what remains. For a command that rolls back work it
|
||||
* started when asked to stop, such as `hdiutil attach` on SIGTERM.
|
||||
*/
|
||||
export interface ProcessTerminationPolicy {
|
||||
readonly signal: NodeJS.Signals;
|
||||
readonly graceMs: number;
|
||||
}
|
||||
|
||||
/** Process events and streams required by provider lifecycle supervision. */
|
||||
@@ -115,6 +128,13 @@ export type ProviderProcessLaunch =
|
||||
readonly ownsProcessLifetime: true;
|
||||
readonly ownership?: OwnedProcessGroup;
|
||||
readonly cleanup?: () => Promise<ProcessCleanupResult>;
|
||||
/**
|
||||
* Signal the launched process group while its leader is unreaped, so the
|
||||
* leader's PID still names the group this launch created. False when
|
||||
* nothing was signaled, such as after the leader exited or on a host
|
||||
* without process groups.
|
||||
*/
|
||||
readonly requestStop?: (signal: NodeJS.Signals) => boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -262,6 +282,8 @@ export interface ProviderProcessStopOptions {
|
||||
readonly cleanupResult?: ProcessCleanupResult;
|
||||
readonly terminationGraceMs?: number;
|
||||
readonly killGraceMs?: number;
|
||||
/** The command's own stop request before owned group cleanup. */
|
||||
readonly termination?: ProcessTerminationPolicy;
|
||||
}
|
||||
|
||||
/** Non-JSON snapshot of the exact inspection facts retained by a stop attempt. */
|
||||
@@ -396,7 +418,29 @@ export const spawnOwnedProviderProcess = async (
|
||||
? {}
|
||||
: { expectedCommand: options.expectedCommand ?? options.command }),
|
||||
};
|
||||
return { process: child, ownership };
|
||||
return {
|
||||
process: child,
|
||||
ownership,
|
||||
requestStop: (signal) => signalUnreapedGroup(child, pid, signal),
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Node sets exitCode or signalCode in the callback that reaps the leader, so
|
||||
* until then the leader's PID still names the group the launch created.
|
||||
*/
|
||||
const signalUnreapedGroup = (
|
||||
child: ChildProcess,
|
||||
processGroupId: number,
|
||||
signal: NodeJS.Signals,
|
||||
): boolean => {
|
||||
if (child.exitCode !== null || child.signalCode !== null) return false;
|
||||
try {
|
||||
process.kill(-processGroupId, signal);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -894,6 +938,15 @@ export class ProviderProcessSupervisor {
|
||||
.join("; "),
|
||||
);
|
||||
if (this.launch.cleanup !== undefined) {
|
||||
// The command's own stop request comes first; owned cleanup then
|
||||
// kills whatever remains, so the grace period only delays it.
|
||||
if (
|
||||
options.termination !== undefined &&
|
||||
options.cleanupResult === undefined &&
|
||||
this.#exitObservation === undefined &&
|
||||
this.launch.requestStop?.(options.termination.signal) === true
|
||||
)
|
||||
await this.waitForExit(options.termination.graceMs);
|
||||
const graceMs = options.killGraceMs ?? DEFAULT_KILL_GRACE_MS;
|
||||
const exitObservedBeforeCleanup = this.#exitObservation !== undefined;
|
||||
const cleanup = await this.#ownedCleanup(options.cleanupResult);
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { Writable } from "node:stream";
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
OwnedCommandFailure,
|
||||
runOwnedCommand,
|
||||
} from "../../../src/process/OwnedCommand.js";
|
||||
import type { ProcessTerminationPolicy } from "../../../src/process/ProviderProcess.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
|
||||
/**
|
||||
* A command that rolls back on SIGTERM (or ignores it) and reports that it is
|
||||
* ready once its handler is installed.
|
||||
*/
|
||||
const rollbackScript = (marker: string, onTerm: "roll-back" | "ignore") => `
|
||||
const { writeFileSync } = require("node:fs");
|
||||
process.on("SIGTERM", () => {
|
||||
${onTerm === "roll-back" ? `writeFileSync(${JSON.stringify(marker)}, "rolled back"); process.exit(0);` : ""}
|
||||
});
|
||||
process.stdout.write("ready\\n");
|
||||
setInterval(() => {}, 1000);
|
||||
`;
|
||||
|
||||
/** Cancel the command once it reports that its SIGTERM handler is installed. */
|
||||
const cancelWhenReady = (controller: AbortController): Writable =>
|
||||
new Writable({
|
||||
write(_chunk, _encoding, callback) {
|
||||
controller.abort("caller stopped");
|
||||
callback();
|
||||
},
|
||||
});
|
||||
|
||||
const run = async (
|
||||
onTerm: "roll-back" | "ignore",
|
||||
termination?: ProcessTerminationPolicy,
|
||||
) => {
|
||||
const root = await createTestTempDirectory("rea-owned-termination-");
|
||||
const marker = join(root, "marker");
|
||||
const controller = new AbortController();
|
||||
const started = performance.now();
|
||||
const failure = await runOwnedCommand(
|
||||
{
|
||||
command: process.execPath,
|
||||
arguments: ["-e", rollbackScript(marker, onTerm)],
|
||||
runId: `owned-command-termination-${onTerm}`,
|
||||
expectedCommand: null,
|
||||
},
|
||||
{
|
||||
timeoutMs: 30_000,
|
||||
diagnosticBytes: 1024,
|
||||
stdout: cancelWhenReady(controller),
|
||||
...(termination === undefined ? {} : { termination }),
|
||||
},
|
||||
{ signal: controller.signal },
|
||||
).catch((cause: unknown) => cause);
|
||||
if (!(failure instanceof OwnedCommandFailure)) throw failure;
|
||||
return {
|
||||
failure,
|
||||
elapsedMs: performance.now() - started,
|
||||
marker: await readFile(marker, "utf8").catch(() => undefined),
|
||||
};
|
||||
};
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"asks a command with a termination policy to stop before owned cleanup",
|
||||
async () => {
|
||||
const { failure, marker } = await run("roll-back", {
|
||||
signal: "SIGTERM",
|
||||
graceMs: 10_000,
|
||||
});
|
||||
expect(failure.reason).toBe("cancelled");
|
||||
expect(failure.cancellation?.reason).toBe("caller stopped");
|
||||
expect(failure.cleanupFailure).toBeNull();
|
||||
expect(failure.snapshot?.signal ?? null).toBeNull();
|
||||
expect(failure.snapshot?.exitCode).toBe(0);
|
||||
expect(marker).toBe("rolled back");
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"kills a command without a termination policy without asking it to stop",
|
||||
async () => {
|
||||
const { failure, marker } = await run("roll-back");
|
||||
expect(failure.reason).toBe("cancelled");
|
||||
expect(failure.cleanupFailure).toBeNull();
|
||||
expect(failure.snapshot?.signal).toBe("SIGKILL");
|
||||
expect(marker).toBeUndefined();
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"kills a command that ignores its stop request once the grace period ends",
|
||||
async () => {
|
||||
const { failure, marker, elapsedMs } = await run("ignore", {
|
||||
signal: "SIGTERM",
|
||||
graceMs: 300,
|
||||
});
|
||||
expect(failure.reason).toBe("cancelled");
|
||||
expect(failure.cleanupFailure).toBeNull();
|
||||
expect(failure.snapshot?.signal).toBe("SIGKILL");
|
||||
expect(marker).toBeUndefined();
|
||||
expect(elapsedMs).toBeGreaterThanOrEqual(300);
|
||||
expect(elapsedMs).toBeLessThan(10_000);
|
||||
},
|
||||
);
|
||||
@@ -13,10 +13,8 @@ import { build } from "plist";
|
||||
import { describe, expect, it, onTestFinished, vi } from "vitest";
|
||||
|
||||
import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js";
|
||||
import {
|
||||
ExecFileOutputCancelledError,
|
||||
execFileOutput,
|
||||
} from "../../../../src/process/ExecFileOutput.js";
|
||||
import { execFileOutput } from "../../../../src/process/ExecFileOutput.js";
|
||||
import { OwnedCommandFailure } from "../../../../src/process/OwnedCommand.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
type ArtifactEntry,
|
||||
@@ -960,7 +958,15 @@ describe("DMG attach cancellation and preexisting mounts", () => {
|
||||
const reason = { request: "cancel DMG attach" };
|
||||
const commandFailure =
|
||||
kind === "owned cancellation"
|
||||
? new ExecFileOutputCancelledError(reason)
|
||||
? new OwnedCommandFailure(
|
||||
"cancelled",
|
||||
"Command cancelled.",
|
||||
null,
|
||||
null,
|
||||
{
|
||||
cancellation: { reason },
|
||||
},
|
||||
)
|
||||
: Object.assign(new Error("attach aborted"), { name: "AbortError" });
|
||||
const mount = createMountFixture();
|
||||
let markAttachStarted = (): void => {};
|
||||
@@ -1681,12 +1687,12 @@ describe("native DMG real verification diagnostics", () => {
|
||||
expect(failure.message).toContain('"exitCode":1');
|
||||
expect(failure.message).toContain('"stdout":""');
|
||||
expect(failure.message).toContain("unable to recognize");
|
||||
expect(failure.cause).toBeInstanceOf(Error);
|
||||
if (!(failure.cause instanceof Error)) continue;
|
||||
expect(Reflect.get(failure.cause, "code")).toBe(1);
|
||||
expect(Reflect.get(failure.cause, "stderr")).toContain(
|
||||
expectedDiagnostic,
|
||||
);
|
||||
// The cause is the exit hdiutil reported, with its diagnostics.
|
||||
expect(failure.cause).toMatchObject({
|
||||
command: "/usr/bin/hdiutil",
|
||||
exitCode: 1,
|
||||
stderr: expect.stringContaining(expectedDiagnostic) as unknown,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
await chmod(unreadablePath, 0o600);
|
||||
@@ -1755,10 +1761,11 @@ describe("native DMG real verification diagnostics", () => {
|
||||
// hdiutil localizes the CRC labels and failure reason, but not this status line.
|
||||
expect(failure.message).toContain("CRC32");
|
||||
expect(failure.message).toContain("is INVALID");
|
||||
expect(failure.cause).toBeInstanceOf(Error);
|
||||
if (!(failure.cause instanceof Error)) return;
|
||||
expect(Reflect.get(failure.cause, "code")).toBe(1);
|
||||
expect(Reflect.get(failure.cause, "stderr")).toContain("is INVALID");
|
||||
expect(failure.cause).toMatchObject({
|
||||
command: "/usr/bin/hdiutil",
|
||||
exitCode: 1,
|
||||
stderr: expect.stringContaining("is INVALID") as unknown,
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
@@ -1,40 +1,65 @@
|
||||
import { access, readFile, rm } from "node:fs/promises";
|
||||
import { access, readFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";
|
||||
|
||||
const state = vi.hoisted(() => ({ marker: "", calls: 0 }));
|
||||
vi.mock("../../src/process/ExecFileOutput.js", async (importOriginal) => {
|
||||
const state = vi.hoisted(() => ({
|
||||
marker: "",
|
||||
calls: 0,
|
||||
/** Run until stopped; otherwise exit at once. */
|
||||
hang: true,
|
||||
/** Withhold owned cleanup until the test allows it. */
|
||||
cleanupAllowed: true,
|
||||
/** When set, exit 1 at once with this hdiutil diagnostic on stderr. */
|
||||
diagnostic: undefined as string | undefined,
|
||||
}));
|
||||
// Stand in for hdiutil at the launch boundary, under production supervision.
|
||||
vi.mock("../../src/process/ProviderProcess.js", async (importOriginal) => {
|
||||
const { cleanupOwnedProcessGroup } =
|
||||
await import("../../src/process/ProcessOwnership.js");
|
||||
const actual =
|
||||
await importOriginal<
|
||||
typeof import("../../src/process/ExecFileOutput.js")
|
||||
typeof import("../../src/process/ProviderProcess.js")
|
||||
>();
|
||||
return {
|
||||
...actual,
|
||||
execFileOutput: (
|
||||
command: string,
|
||||
args: string[],
|
||||
options: Parameters<typeof actual.execFileOutput>[2],
|
||||
spawnOwnedProviderProcess: async (
|
||||
options: Parameters<typeof actual.spawnOwnedProviderProcess>[0],
|
||||
) => {
|
||||
if (command !== "/usr/bin/hdiutil" || args[0] !== "verify")
|
||||
if (
|
||||
options.command !== "/usr/bin/hdiutil" ||
|
||||
options.arguments[0] !== "verify"
|
||||
)
|
||||
throw new Error("Unexpected native command");
|
||||
state.calls += 1;
|
||||
return actual.execFileOutput(
|
||||
process.execPath,
|
||||
[
|
||||
const launched = await actual.spawnOwnedProviderProcess({
|
||||
...options,
|
||||
command: process.execPath,
|
||||
expectedCommand: process.execPath,
|
||||
arguments: [
|
||||
"-e",
|
||||
'require("node:fs").writeFileSync(process.argv[1], String(process.pid)); setInterval(() => {}, 1000)',
|
||||
state.diagnostic !== undefined
|
||||
? `require("node:fs").writeFileSync(process.argv[1], String(process.pid)); process.stderr.write(${JSON.stringify(state.diagnostic)}); process.exitCode = 1`
|
||||
: state.hang
|
||||
? 'const fs = require("node:fs"); process.on("SIGTERM", () => { fs.writeFileSync(process.argv[1] + ".term", "rolled back"); process.exit(0); }); fs.writeFileSync(process.argv[1], String(process.pid)); setInterval(() => {}, 1000)'
|
||||
: 'require("node:fs").writeFileSync(process.argv[1], String(process.pid))',
|
||||
state.marker,
|
||||
],
|
||||
options,
|
||||
);
|
||||
});
|
||||
return {
|
||||
...launched,
|
||||
cleanup: async () =>
|
||||
state.cleanupAllowed
|
||||
? cleanupOwnedProcessGroup(launched.ownership)
|
||||
: { cleaned: false, reason: "cleanup withheld by the test" },
|
||||
};
|
||||
},
|
||||
};
|
||||
});
|
||||
import { NativeDmgArtifactReader } from "../../src/artifacts/NativeDmgArtifactReader.js";
|
||||
import { ArtifactReaderFailure } from "../../src/resources/ArtifactReader.js";
|
||||
import { ExecFileOutputCancelledError } from "../../src/process/ExecFileOutput.js";
|
||||
import { OwnedCommandFailure } from "../../src/process/OwnedCommand.js";
|
||||
import { translateArtifactFailure } from "../../src/resources/index.js";
|
||||
|
||||
it.skipIf(process.platform !== "darwin").each([false, true])(
|
||||
@@ -43,6 +68,8 @@ it.skipIf(process.platform !== "darwin").each([false, true])(
|
||||
const root = await createTestTempDirectory("rea-dmg-cancellation-reason-");
|
||||
state.marker = join(root, "ready");
|
||||
state.calls = 0;
|
||||
state.hang = true;
|
||||
state.cleanupAllowed = true;
|
||||
const reason = { requested: "stop verify" };
|
||||
const controller = new AbortController();
|
||||
if (preAborted) controller.abort(reason);
|
||||
@@ -78,18 +105,92 @@ it.skipIf(process.platform !== "darwin").each([false, true])(
|
||||
);
|
||||
expect(state.calls).toBe(preAborted ? 0 : 1);
|
||||
if (!preAborted) {
|
||||
expect(failure.cause).toBeInstanceOf(ExecFileOutputCancelledError);
|
||||
expect((failure.cause as ExecFileOutputCancelledError).cause).toBe(
|
||||
reason,
|
||||
);
|
||||
expect(failure.cause).toBeInstanceOf(OwnedCommandFailure);
|
||||
expect(
|
||||
(failure.cause as OwnedCommandFailure).cancellation?.reason,
|
||||
).toBe(reason);
|
||||
const pid = Number(await readFile(state.marker, "utf8"));
|
||||
expect(() => process.kill(pid, 0)).toThrow();
|
||||
// hdiutil is asked to stop with SIGTERM before owned cleanup kills it.
|
||||
expect(await readFile(`${state.marker}.term`, "utf8")).toBe(
|
||||
"rolled back",
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
controller.abort(reason);
|
||||
await result;
|
||||
await reader.close();
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(process.platform !== "darwin")(
|
||||
"retains an hdiutil whose cleanup failed until reader close releases it",
|
||||
async () => {
|
||||
const root = await createTestTempDirectory("rea-dmg-retained-cleanup-");
|
||||
state.marker = join(root, "ran");
|
||||
state.calls = 0;
|
||||
state.hang = false;
|
||||
state.cleanupAllowed = false;
|
||||
const reader = new NativeDmgArtifactReader("fixture.dmg", {});
|
||||
try {
|
||||
const failure = await (async () => {
|
||||
for await (const _entry of reader.entries()) {
|
||||
/* consume */
|
||||
}
|
||||
})().catch((cause: unknown) => cause);
|
||||
expect(failure).toBeInstanceOf(ArtifactReaderFailure);
|
||||
expect((failure as ArtifactReaderFailure).cause).toMatchObject({
|
||||
cleanupFailure: "cleanup withheld by the test",
|
||||
});
|
||||
expect(state.calls).toBe(1);
|
||||
const pid = Number(await readFile(state.marker, "utf8"));
|
||||
|
||||
// The retained process blocks close and names its owned identities.
|
||||
const refused = await reader.close().catch((cause: unknown) => cause);
|
||||
expect(refused).toBeInstanceOf(ArtifactReaderFailure);
|
||||
expect((refused as ArtifactReaderFailure).cleanup?.resources).toEqual(
|
||||
expect.arrayContaining([`pid:${String(pid)}`]),
|
||||
);
|
||||
|
||||
state.cleanupAllowed = true;
|
||||
await reader.close();
|
||||
expect(state.calls).toBe(1);
|
||||
} finally {
|
||||
state.cleanupAllowed = true;
|
||||
await reader.close().catch(() => undefined);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(process.platform !== "darwin")(
|
||||
"admits no further hdiutil command while an earlier command's cleanup stays unresolved",
|
||||
async () => {
|
||||
const root = await createTestTempDirectory("rea-dmg-cleanup-admission-");
|
||||
state.marker = join(root, "ran");
|
||||
state.calls = 0;
|
||||
state.hang = false;
|
||||
state.cleanupAllowed = false;
|
||||
// A transient refusal is retried; its exit status survives failed cleanup.
|
||||
state.diagnostic =
|
||||
"hdiutil: verify failed - Resource temporarily unavailable\n";
|
||||
const reader = new NativeDmgArtifactReader("fixture.dmg", {});
|
||||
try {
|
||||
const failure = await (async () => {
|
||||
for await (const _entry of reader.entries()) {
|
||||
/* consume */
|
||||
}
|
||||
})().catch((cause: unknown) => cause);
|
||||
expect(failure).toBeInstanceOf(ArtifactReaderFailure);
|
||||
// The retry waited for the retained cleanup, which is still withheld.
|
||||
expect(state.calls).toBe(1);
|
||||
expect((failure as ArtifactReaderFailure).cleanup?.resources).toEqual(
|
||||
expect.arrayContaining([`pid:${await readFile(state.marker, "utf8")}`]),
|
||||
);
|
||||
} finally {
|
||||
state.diagnostic = undefined;
|
||||
state.cleanupAllowed = true;
|
||||
await reader.close().catch(() => undefined);
|
||||
}
|
||||
},
|
||||
);
|
||||
Reference in new issue
Block a user