fix(process): classify every launcher failure once by errno in the shared launch owner (#11615)

* fix(process): classify every launcher failure once by errno in the shared launch owner

* fix(process): keep the launch owner's syscall, cause and facts in RE and pwntools unavailable reports
This commit is contained in:
morluto authored and GitHub committed 2026-11-16 08:24:39 +08:00
1 parent 0884836abf
commit dbb4bc29f0
29 files changed
+712 -289

No files matched your search

+1
View File
@@ -467,6 +467,7 @@ export class ArtifactAnalysisResources {
budgetParameter?: "producer_limits",
): ProviderOutcomeContext {
return {
providerId: this.providerId,
operation,
capturedOutput: capturedProviderOutput,
timeout: {
+4 -12
View File
@@ -1,5 +1,6 @@
import type { AnalysisCapturedOutput } from "../domain/analysisErrorBase.js";
import { HOST_PLATFORM } from "../platform/HostPlatform.js";
import { launchExhaustedResource } from "./launchFailure.js";
import {
execFile,
spawn,
@@ -136,25 +137,16 @@ export const execFileOutputFailure = (
};
};
/** Spawn errnos reporting exhausted host capacity rather than an unusable command. */
export const HOST_SPAWN_CAPACITY_CODES: ReadonlySet<string> = new Set([
"EAGAIN",
"EMFILE",
"ENFILE",
"ENOMEM",
]);
/**
* Whether the command never started because the host lacked process, memory
* or file-descriptor capacity. The command itself may be usable; a retry after
* capacity is freed can succeed.
* or file-descriptor capacity (classified by `launchExhaustedResource`). The
* command itself may be usable; a retry after capacity is freed can succeed.
*/
export const spawnCapacityExhausted = (
failure: ExecFileOutputFailure | undefined,
): boolean =>
failure?.spawnFailed === true &&
typeof failure.code === "string" &&
HOST_SPAWN_CAPACITY_CODES.has(failure.code);
launchExhaustedResource(failure.code) !== undefined;
/** Project admitted command failure facts without reinterpreting failure policy. */
export const execFileFailureCapturedOutput = (
+2 -3
View File
@@ -38,9 +38,9 @@ import { descendantsOf, liveProcesses } from "./ProcessOwnershipProcessTree.js";
import {
execFileOutput,
execFileOutputFailure,
HOST_SPAWN_CAPACITY_CODES,
spawnCapacityExhausted,
} from "./ExecFileOutput.js";
import { launchExhaustedResource } from "./launchFailure.js";
import {
createDarwinProcessRunTokenReader,
DarwinProcessOwnershipInspectionError,
@@ -1557,8 +1557,7 @@ export const createSystemProcessOwnershipHost = (
spawnCapacityExhausted(failure) ||
(typeof cause === "object" &&
cause !== null &&
typeof Reflect.get(cause, "code") === "string" &&
HOST_SPAWN_CAPACITY_CODES.has(Reflect.get(cause, "code") as string))
launchExhaustedResource(Reflect.get(cause, "code")) !== undefined)
)
throw cause;
const rawCode =
+11 -11
View File
@@ -1,4 +1,4 @@
import { launcherRefusalOf, type LauncherRefusal } from "./launchFailure.js";
import { launcherFailureOf, type LauncherFailure } from "./launchFailure.js";
import { OwnedCommandFailure } from "./OwnedCommand.js";
import type { ProviderProcessSnapshot } from "./ProviderProcess.js";
import type { RuntimeExhaustion } from "./RuntimeExhaustion.js";
@@ -27,10 +27,10 @@ interface ProviderOutcomeFacts {
* `classifyProviderOutcome`.
*/
export type ProviderOutcome =
/** The operating system refused to start the selected executable. */
/** A launcher failed to start the executable; `failure` says why, by errno. */
| (ProviderOutcomeFacts & {
readonly kind: "launch-refused";
readonly refusal: LauncherRefusal;
readonly kind: "launch-failed";
readonly failure: LauncherFailure;
})
/** The caller cancelled; `reason` is the caller's opaque abort reason. */
| (ProviderOutcomeFacts & {
@@ -133,19 +133,19 @@ const ownedCommandOutcome = (failure: OwnedCommandFailure): ProviderOutcome => {
/**
* Read the outcome of an owned command from what `runOwnedCommand` threw.
* Undefined when the value is neither a supervised command failure nor a
* launch refusal.
* Undefined when the value is neither a supervised command failure nor an
* error a launcher raised.
*/
export function providerOutcomeOf(cause: OwnedCommandFailure): ProviderOutcome;
export function providerOutcomeOf(cause: unknown): ProviderOutcome | undefined;
export function providerOutcomeOf(cause: unknown): ProviderOutcome | undefined {
if (cause instanceof OwnedCommandFailure) return ownedCommandOutcome(cause);
// Only a launcher's errno is a launch refusal; one from reading a file is not.
const refusal = launcherRefusalOf(cause);
if (refusal === undefined || !(cause instanceof Error)) return undefined;
// Only a launcher's errno is a launch failure; one from reading a file is not.
const failure = launcherFailureOf(cause);
if (failure === undefined || !(cause instanceof Error)) return undefined;
return {
kind: "launch-refused",
refusal,
kind: "launch-failed",
failure,
cause,
message: cause.message,
output: undefined,
+146 -5
View File
@@ -4,6 +4,7 @@ import type {
AnalysisErrorOptions,
} from "../domain/analysisErrorBase.js";
import {
AnalysisCapabilityUnavailableError,
AnalysisResourceConstraintError,
AnalysisTimeoutError,
cancelledBy,
@@ -12,6 +13,11 @@ import {
} from "../domain/analysisErrorCore.js";
import type { JsonValue } from "../domain/jsonValue.js";
import type { AnalysisLimit } from "../domain/analysisSettings.js";
import {
describeLaunchFailure,
type LaunchExhaustedResource,
type LauncherFailure,
} from "./launchFailure.js";
import type { ProviderOutcome } from "./ProviderOutcome.js";
import type { ProviderProcessSnapshot } from "./ProviderProcess.js";
import {
@@ -35,13 +41,134 @@ const AREA_NAMES = {
/** Outcomes whose meaning depends on the provider's own dialect. */
export type ProviderDialectOutcome = Extract<
ProviderOutcome,
{ readonly kind: "launch-refused" | "exited" | "protocol" }
{ readonly kind: "exited" | "protocol" }
>;
/** A launch failure that leaves the provider unavailable for this call. */
export type LaunchUnavailableFailure = Extract<
LauncherFailure,
{ readonly kind: "missing" | "not_executable" | "failed" }
>;
const EXHAUSTED_RESOURCE_NAMES = {
processes: "process",
memory: "memory",
"file-descriptors": "file-descriptor",
} as const satisfies Record<LaunchExhaustedResource, string>;
/** What one launch failure report needs beyond the launcher's facts. */
export interface LaunchFailureContext {
readonly providerId: string;
readonly operation: string;
/** Options every classified error carries, such as the cause and cleanup. */
readonly details?: AnalysisErrorOptions;
/** The command the caller selected, when the launcher did not name it. */
readonly command?: string;
/** The setting that selected the command; a host constraint leaves it unchanged. */
readonly selection?: string;
/** Bytes of the refused argument list, when the caller measured them. */
readonly argumentBytes?: number;
/**
* The provider's report that it cannot be started, such as a message naming
* its configured executable. Without it the shared report names the
* executable and the errno.
*/
readonly unavailable?: (
failure: LaunchUnavailableFailure,
details: AnalysisErrorOptions,
) => AnalysisError;
}
/**
* The one mapping from a launcher's failure to a typed error. Host capacity
* (EAGAIN, ENOMEM, EMFILE, ENFILE) and an oversized argument list (E2BIG) are
* resource constraints of this call, not provider faults; every other errno
* leaves the provider unavailable, reported with its errno and syscall.
*/
export const classifyLaunchFailure = (
failure: LauncherFailure,
context: LaunchFailureContext,
): AnalysisError => {
const { operation } = context;
const command = failure.executable ?? context.command;
const subject = command ?? "the provider command";
const launchFacts: Readonly<Record<string, JsonValue>> = {
system_code: failure.code ?? null,
...(failure.syscall === undefined ? {} : { syscall: failure.syscall }),
...(command === undefined ? {} : { command }),
};
const details: AnalysisErrorOptions = {
...context.details,
diagnostics: {
...context.details?.diagnostics,
...launchFacts,
boundary:
failure.kind === "host-exhausted" ? "host-capacity" : "provider-launch",
},
};
const unchanged = `${context.selection ?? "the selected command"} does not need to change`;
switch (failure.kind) {
case "host-exhausted": {
const name = EXHAUSTED_RESOURCE_NAMES[failure.resource];
return new AnalysisResourceConstraintError(
operation,
"capacity",
`The host ran out of ${name} capacity (${failure.code}) to start ${subject}.`,
{
spawn_error: failure.code,
exhausted_resource: failure.resource,
...(command === undefined ? {} : { command }),
},
{
...details,
retryable: true,
remediationAction: `The host ran out of ${name} capacity (${failure.code}). Close unused processes or files, or raise the process or open-file limit of the REA process, then retry; ${unchanged}.`,
},
);
}
case "argument-list":
return new AnalysisResourceConstraintError(
operation,
"transport",
`The host rejected ${subject} because its argument list and environment exceed the process launch limit (${failure.code}).`,
{
spawn_error: failure.code,
...(context.argumentBytes === undefined
? {}
: { argument_bytes: context.argumentBytes }),
},
{
...details,
retryable: false,
remediationAction: `Select less input for one call, or pass large values through a file, then retry; ${unchanged}. An unchanged request fails the same way.`,
},
);
case "missing":
case "not_executable":
case "failed": {
if (context.unavailable !== undefined)
return context.unavailable(failure, details);
const reason =
failure.kind === "missing"
? `${subject} was not found (${describeLaunchFailure(failure)}).`
: failure.kind === "not_executable"
? `${subject} exists but could not be executed (${describeLaunchFailure(failure)}).`
: `${subject} could not be launched (${describeLaunchFailure(failure)}).`;
return new AnalysisCapabilityUnavailableError(
context.providerId,
operation,
reason,
{ ...details, userMessage: reason },
);
}
}
};
type ReportedLimits = Readonly<Record<string, JsonValue>> | null;
/** What one provider command needs to turn its outcome into a typed error. */
export interface ProviderOutcomeContext {
readonly providerId: string;
readonly operation: string;
/** The provider's redacted projection of retained output. */
readonly capturedOutput: (
@@ -85,7 +212,13 @@ export interface ProviderOutcomeContext {
readonly limits: ReportedLimits;
readonly recovery: ResourceConstraintRecovery;
};
/** Provider-specific meaning of a launch refusal, an exit, or a stream failure. */
/**
* The provider's report that its command cannot be started; see
* `LaunchFailureContext.unavailable`. Host constraints are not the
* provider's to interpret.
*/
readonly launchUnavailable?: LaunchFailureContext["unavailable"];
/** Provider-specific meaning of an exit or a stream failure. */
readonly dialect: (
outcome: ProviderDialectOutcome,
details: AnalysisErrorOptions,
@@ -94,8 +227,8 @@ export interface ProviderOutcomeContext {
/**
* The one mapping from an owned command's physical outcome to a typed error.
* Owner-observed facts are classified here; only the provider's dialect
* decides what an exit or a launch refusal means for that provider.
* Owner-observed facts, including launch failures, are classified here; only
* the provider's dialect decides what an exit or a stream failure means.
*/
export const classifyProviderOutcome = (
outcome: ProviderOutcome,
@@ -198,7 +331,15 @@ export const classifyProviderOutcome = (
},
);
}
case "launch-refused":
case "launch-failed":
return classifyLaunchFailure(outcome.failure, {
providerId: context.providerId,
operation,
details,
...(context.launchUnavailable === undefined
? {}
: { unavailable: context.launchUnavailable }),
});
case "exited":
case "protocol":
return context.dialect(outcome, details);
+99 -14
View File
@@ -3,7 +3,9 @@ import { expect, it } from "vitest";
import {
classifyLaunchFailure,
classifyLaunchFailureCode,
launcherRefusalOf,
classifyLaunchErrno,
describeLaunchFailure,
launcherFailureOf,
} from "./launchFailure.js";
it.each([
@@ -29,11 +31,39 @@ it.each([["EAGAIN"], ["EMFILE"], ["ETIMEDOUT"], [1], [null], ["toString"]])(
},
);
it.each([
["EAGAIN", { kind: "host-exhausted", code: "EAGAIN", resource: "processes" }],
["ENOMEM", { kind: "host-exhausted", code: "ENOMEM", resource: "memory" }],
[
"EMFILE",
{ kind: "host-exhausted", code: "EMFILE", resource: "file-descriptors" },
],
[
"ENFILE",
{ kind: "host-exhausted", code: "ENFILE", resource: "file-descriptors" },
],
["E2BIG", { kind: "argument-list", code: "E2BIG" }],
["ENOENT", { kind: "missing", code: "ENOENT" }],
["EACCES", { kind: "not_executable", code: "EACCES" }],
["ENOEXEC", { kind: "not_executable", code: "ENOEXEC" }],
["EPERM", { kind: "not_executable", code: "EPERM" }],
["EIO", { kind: "failed", code: "EIO" }],
[undefined, { kind: "failed", code: undefined }],
["toString", { kind: "failed", code: "toString" }],
])("classifies launch errno %j once", (code, expected) => {
expect(classifyLaunchErrno(code)).toEqual(expected);
});
it.each([
[
"Node spawn",
{ code: "ENOENT", syscall: "spawn /tool", path: "/tool" },
{ kind: "missing", code: "ENOENT", executable: "/tool" },
{
kind: "missing",
code: "ENOENT",
executable: "/tool",
syscall: "spawn /tool",
},
],
[
"Windows native launcher",
@@ -46,22 +76,64 @@ it.each([
kind: "not_executable",
code: "EACCES",
executable: "C:\\tools\\tool.exe",
syscall: undefined,
},
],
[
"bare Node spawn",
{ code: "ENOTDIR", syscall: "spawn" },
{ kind: "missing", code: "ENOTDIR", executable: undefined },
{
kind: "missing",
code: "ENOTDIR",
executable: undefined,
syscall: "spawn",
},
],
[
"Node spawn naming another path",
{ code: "ENOENT", syscall: "spawn /other", path: "/tool" },
{ kind: "missing", code: "ENOENT", executable: undefined },
{
kind: "missing",
code: "ENOENT",
executable: undefined,
syscall: "spawn /other",
},
],
])("reads a %s refusal and the executable it names", (_, fields, expected) => {
expect(
launcherRefusalOf(Object.assign(new Error("refused"), fields)),
).toEqual(expected);
[
"host-capacity spawn",
{ code: "EAGAIN", syscall: "spawn /tool", path: "/tool" },
{
kind: "host-exhausted",
code: "EAGAIN",
resource: "processes",
executable: "/tool",
syscall: "spawn /tool",
},
],
[
"argument-list spawn",
{ code: "E2BIG", syscall: "spawn /tool", path: "/tool" },
{
kind: "argument-list",
code: "E2BIG",
executable: "/tool",
syscall: "spawn /tool",
},
],
[
"other errno spawn",
{ code: "EIO", syscall: "spawn /tool", path: "/tool" },
{
kind: "failed",
code: "EIO",
executable: "/tool",
syscall: "spawn /tool",
},
],
])("reads a %s failure and the executable it names", (_, fields, expected) => {
expect(launcherFailureOf(Object.assign(new Error("failed"), fields))).toEqual(
expected,
);
});
it.each([
@@ -75,12 +147,25 @@ it.each([
requestedPath: "C:\\tools\\tool.exe",
},
],
[
"a launcher error whose errno is not a refusal",
{ code: "EAGAIN", syscall: "spawn /tool", path: "/tool" },
],
])("does not read %s as a launcher refusal", (_, fields) => {
])("does not read %s as a launcher failure", (_, fields) => {
expect(
launcherRefusalOf(Object.assign(new Error("refused"), fields)),
launcherFailureOf(Object.assign(new Error("refused"), fields)),
).toBeUndefined();
});
it("names the syscall only for an errno that is neither a refusal nor a constraint", () => {
const read = (code: string) => {
const failure = launcherFailureOf(
Object.assign(new Error("failed"), {
code,
syscall: "spawn /tool",
path: "/tool",
}),
);
if (failure === undefined) throw new Error("Expected a launcher failure");
return describeLaunchFailure(failure);
};
expect(read("EIO")).toBe("EIO from spawn /tool");
expect(read("ENOENT")).toBe("ENOENT");
expect(read("EAGAIN")).toBe("EAGAIN");
});
+84 -10
View File
@@ -23,6 +23,62 @@ const LAUNCH_REFUSALS: Readonly<Record<string, LaunchRefusal["kind"]>> = {
EISDIR: "not_executable",
};
/** The host resource a launch could not obtain. */
export type LaunchExhaustedResource =
| "processes"
| "memory"
| "file-descriptors";
const HOST_EXHAUSTION: Readonly<Record<string, LaunchExhaustedResource>> = {
EAGAIN: "processes",
ENOMEM: "memory",
EMFILE: "file-descriptors",
ENFILE: "file-descriptors",
};
/**
* The host resource a spawn errno reports as exhausted; undefined for any
* other errno. The selected command may be usable once capacity is freed.
*/
export const launchExhaustedResource = (
code: unknown,
): LaunchExhaustedResource | undefined =>
typeof code === "string" && Object.hasOwn(HOST_EXHAUSTION, code)
? HOST_EXHAUSTION[code]
: undefined;
/**
* Why a process did not start, classified once by errno:
*
* - a launch refusal (`missing`, `not_executable`): the selected executable
* cannot be run;
* - `host-exhausted`: the host lacked process, memory or file-descriptor
* capacity (EAGAIN, ENOMEM, EMFILE, ENFILE);
* - `argument-list`: the request's argument list and environment exceed the
* launch limit (E2BIG);
* - `failed`: any other errno, or none; the provider cannot be started.
*/
export type LaunchFailure =
| LaunchRefusal
| {
readonly kind: "host-exhausted";
readonly code: string;
readonly resource: LaunchExhaustedResource;
}
| { readonly kind: "argument-list"; readonly code: "E2BIG" }
| { readonly kind: "failed"; readonly code: string | undefined };
/** Classify any spawn errno; see `LaunchFailure`. */
export const classifyLaunchErrno = (code: unknown): LaunchFailure => {
const refusal = classifyLaunchFailureCode(code);
if (refusal !== undefined) return refusal;
const resource = launchExhaustedResource(code);
if (resource !== undefined && typeof code === "string")
return { kind: "host-exhausted", code, resource };
if (code === "E2BIG") return { kind: "argument-list", code };
return { kind: "failed", code: typeof code === "string" ? code : undefined };
};
/** Classify a spawn errno; undefined when it is not a launch refusal. */
export const classifyLaunchFailureCode = (
code: unknown,
@@ -63,12 +119,14 @@ export const nameSpawnedExecutable = (
};
/**
* A launch refusal a launcher raised; `executable` when the launcher named the
* executable it refused, undefined when it did not (Node's bare `spawn`).
* A launch failure a launcher raised. `executable` names the executable when
* the launcher named it, undefined when it did not (Node's bare `spawn`);
* `syscall` is the launcher's own label, such as `spawn /usr/bin/tool`.
*/
export interface LauncherRefusal extends LaunchRefusal {
export type LauncherFailure = LaunchFailure & {
readonly executable: string | undefined;
}
readonly syscall: string | undefined;
};
/**
* Whether a launcher raised this error while starting a process, and the
@@ -96,15 +154,31 @@ const launcherIdentityOf = (
};
/**
* Classify a launcher's refusal to start an executable. Undefined when the
* errno is not a launch refusal or no launcher raised the error.
* Classify a launcher's failure to start an executable, by errno. Undefined
* when no launcher raised the error, such as an errno from reading a file or
* preparing process ownership.
*/
export const launcherRefusalOf = (
export const launcherFailureOf = (
cause: unknown,
): LauncherRefusal | undefined => {
): LauncherFailure | undefined => {
if (!(cause instanceof Error)) return undefined;
const identity = launcherIdentityOf(cause);
if (identity === undefined) return undefined;
const refusal = classifyLaunchFailure(cause);
return refusal === undefined ? undefined : { ...refusal, ...identity };
const syscall = Reflect.get(cause, "syscall");
return {
...classifyLaunchErrno(Reflect.get(cause, "code")),
...identity,
syscall: typeof syscall === "string" ? syscall : undefined,
};
};
/**
* How a launch failure reads in a report: the errno, and the launcher's
* syscall for a failure that is neither a refusal nor a host constraint.
*/
export const describeLaunchFailure = (failure: LauncherFailure): string => {
const code = failure.code ?? "no system code";
return failure.kind === "failed" && failure.syscall !== undefined
? `${code} from ${failure.syscall}`
: code;
};