mirror of
https://github.com/morluto/rea.git
synced 2026-11-22 14:35:45 +00:00
fix(browser): name the CPU ceiling when mitmproxy dies with SIGXCPU
A tighter inherited RLIMIT_CPU is kept, and that death had no reply. The decoder records the installed ceiling, and SIGXCPU becomes a CPU resource constraint that names it.
This commit is contained in:
1 parent
92f3f28d29
commit
39ade5694b
5 files changed
+284
-2
No files matched your search
@@ -1,4 +1,7 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { expect, it } from "vitest";
|
||||
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
|
||||
import { AnalysisCleanupLease } from "../../resources/AnalysisCleanupLease.js";
|
||||
import { ArtifactReaderFailure } from "../../resources/ArtifactReader.js";
|
||||
import {
|
||||
@@ -9,8 +12,11 @@ import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
|
||||
import { analysisErrorProjectionSchema } from "../../domain/analysisErrorProjectionSchema.js";
|
||||
import { inspectWebNetworkCaptureInputSchema } from "../../domain/webNetworkCapture.js";
|
||||
import {
|
||||
EFFECTIVE_DECODER_CPU_RECORD,
|
||||
historicalCaptureFailure,
|
||||
parseRecordedCpuSeconds,
|
||||
protobufCaptureFailure,
|
||||
recordedDecoderCpuSeconds,
|
||||
} from "./CaptureFailures.js";
|
||||
import {
|
||||
CaptureFormatError,
|
||||
@@ -244,6 +250,102 @@ it.each([
|
||||
},
|
||||
);
|
||||
|
||||
const emptyCommandOutput = { text: "", bytes: 0, observedBytes: 0 };
|
||||
|
||||
const cpuSignalFailure = (signal: "SIGXCPU" | "SIGXFSZ") =>
|
||||
new OwnedCommandFailure("process", "Decoder failed", {
|
||||
stdout: emptyCommandOutput,
|
||||
stderr: emptyCommandOutput,
|
||||
exitCode: null,
|
||||
signal,
|
||||
});
|
||||
|
||||
it("reports SIGXCPU with the CPU ceiling the decoder recorded", () => {
|
||||
const input = inspectWebNetworkCaptureInputSchema.parse({
|
||||
capture_path: "/selected/input.mitm",
|
||||
format: "mitmproxy",
|
||||
});
|
||||
const projected = projectAnalysisError(
|
||||
historicalCaptureFailure(
|
||||
input,
|
||||
cpuSignalFailure("SIGXCPU"),
|
||||
"decoder",
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
1,
|
||||
),
|
||||
);
|
||||
expect(analysisErrorProjectionSchema.parse(projected)).toMatchObject({
|
||||
code: "resource_constraint",
|
||||
retryable: false,
|
||||
details: {
|
||||
resource: "cpu",
|
||||
reported_limits: {
|
||||
cpu_seconds: 1,
|
||||
observed_signal: "SIGXCPU",
|
||||
format: "mitmproxy",
|
||||
},
|
||||
},
|
||||
remediation: {
|
||||
action: expect.stringContaining("RLIMIT_CPU above 1 seconds"),
|
||||
},
|
||||
});
|
||||
expect(JSON.stringify(projected)).not.toContain("768");
|
||||
});
|
||||
|
||||
it("reports SIGXCPU without inventing a CPU ceiling", () => {
|
||||
const input = inspectWebNetworkCaptureInputSchema.parse({
|
||||
capture_path: "/selected/input.mitm",
|
||||
format: "mitmproxy",
|
||||
});
|
||||
const projected = projectAnalysisError(
|
||||
historicalCaptureFailure(input, cpuSignalFailure("SIGXCPU"), "decoder"),
|
||||
);
|
||||
expect(analysisErrorProjectionSchema.parse(projected)).toMatchObject({
|
||||
code: "resource_constraint",
|
||||
details: {
|
||||
resource: "cpu",
|
||||
reported_limits: { observed_signal: "SIGXCPU" },
|
||||
},
|
||||
remediation: { action: expect.stringContaining("was not recorded") },
|
||||
});
|
||||
expect(projected.details?.reported_limits).not.toHaveProperty("cpu_seconds");
|
||||
});
|
||||
|
||||
it("leaves SIGXFSZ on the adapter failure path", () => {
|
||||
const input = inspectWebNetworkCaptureInputSchema.parse({
|
||||
capture_path: "/selected/input.mitm",
|
||||
format: "mitmproxy",
|
||||
});
|
||||
expect(
|
||||
projectAnalysisError(
|
||||
historicalCaptureFailure(input, cpuSignalFailure("SIGXFSZ"), "decoder"),
|
||||
),
|
||||
).toMatchObject({ code: "execution_failure" });
|
||||
});
|
||||
|
||||
it("reads only a single recorded CPU ceiling", async () => {
|
||||
expect(parseRecordedCpuSeconds(Buffer.from('{"seconds":1}'))).toBe(1);
|
||||
expect(parseRecordedCpuSeconds(Buffer.from('{"seconds":1,"other":2}'))).toBe(
|
||||
undefined,
|
||||
);
|
||||
expect(parseRecordedCpuSeconds(Buffer.from('{"seconds":1.5}'))).toBe(
|
||||
undefined,
|
||||
);
|
||||
const root = await createTestTempDirectory("rea-cpu-limit-");
|
||||
await writeFile(
|
||||
join(root, EFFECTIVE_DECODER_CPU_RECORD),
|
||||
'{"seconds":1}',
|
||||
"utf8",
|
||||
);
|
||||
const cause = cpuSignalFailure("SIGXCPU");
|
||||
expect(await recordedDecoderCpuSeconds(root, cause)).toBe(1);
|
||||
expect(
|
||||
await recordedDecoderCpuSeconds(root, cpuSignalFailure("SIGXFSZ")),
|
||||
).toBe(undefined);
|
||||
});
|
||||
|
||||
it.each(["before open", "during read"])(
|
||||
"preserves observed capture changes %s as retryable acquisition integrity failures",
|
||||
(stage) => {
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { open } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { capturedProviderOutput } from "../../process/capturedProviderOutput.js";
|
||||
import { javaScriptWorkerMemoryLimitBytes } from "../../process/JavaScriptWorkerMemoryLimit.js";
|
||||
import { isAbortOf } from "../../process/SignalAbort.js";
|
||||
@@ -98,6 +100,103 @@ export const protobufCaptureFailure = (
|
||||
export const CAPTURE_PROCESS_MEMORY_LIMIT_REASON =
|
||||
"The capture decoder reached its Windows Job Object memory limit";
|
||||
|
||||
/** Sibling of reply.json. The parent reads it after SIGXCPU. */
|
||||
export const EFFECTIVE_DECODER_CPU_RECORD = "effective-cpu.json";
|
||||
|
||||
const MAX_EFFECTIVE_CPU_RECORD_BYTES = 64;
|
||||
|
||||
/** Read the installed CPU ceiling. Undefined when the record is absent or malformed. */
|
||||
export const parseRecordedCpuSeconds = (bytes: Buffer): number | undefined => {
|
||||
if (
|
||||
bytes.byteLength === 0 ||
|
||||
bytes.byteLength > MAX_EFFECTIVE_CPU_RECORD_BYTES
|
||||
)
|
||||
return undefined;
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(bytes.toString("utf8")) as unknown;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed))
|
||||
return undefined;
|
||||
const record = parsed as Record<string, unknown>;
|
||||
const seconds = record.seconds;
|
||||
if (
|
||||
Object.keys(record).length !== 1 ||
|
||||
typeof seconds !== "number" ||
|
||||
!Number.isSafeInteger(seconds) ||
|
||||
seconds < 0
|
||||
)
|
||||
return undefined;
|
||||
return seconds;
|
||||
};
|
||||
|
||||
/** The CPU ceiling the decoder published before it was killed. */
|
||||
export const recordedDecoderCpuSeconds = async (
|
||||
runtimePath: string | undefined,
|
||||
cause: unknown,
|
||||
): Promise<number | undefined> => {
|
||||
if (
|
||||
runtimePath === undefined ||
|
||||
!(cause instanceof OwnedCommandFailure) ||
|
||||
cause.snapshot?.signal !== "SIGXCPU"
|
||||
)
|
||||
return undefined;
|
||||
let descriptor: Awaited<ReturnType<typeof open>> | undefined;
|
||||
try {
|
||||
descriptor = await open(
|
||||
join(runtimePath, EFFECTIVE_DECODER_CPU_RECORD),
|
||||
"r",
|
||||
);
|
||||
const info = await descriptor.stat();
|
||||
if (info.size > MAX_EFFECTIVE_CPU_RECORD_BYTES) return undefined;
|
||||
const bytes = Buffer.alloc(info.size);
|
||||
const read = await descriptor.read(bytes, 0, info.size, 0);
|
||||
if (read.bytesRead !== info.size) return undefined;
|
||||
return parseRecordedCpuSeconds(bytes);
|
||||
} catch {
|
||||
return undefined;
|
||||
} finally {
|
||||
await descriptor?.close();
|
||||
}
|
||||
};
|
||||
|
||||
/** A SIGXCPU death names the installed ceiling when the decoder recorded it. */
|
||||
const captureCpuSignalFailure = (
|
||||
input: InspectWebNetworkCaptureInput,
|
||||
cause: OwnedCommandFailure,
|
||||
effectiveCpuSeconds: number | undefined,
|
||||
): AnalysisResourceConstraintError => {
|
||||
const recorded = effectiveCpuSeconds !== undefined;
|
||||
const reason = recorded
|
||||
? `The effective RLIMIT_CPU of ${String(effectiveCpuSeconds)} seconds stopped the decoder. The requested decoder budget does not raise an inherited limit.`
|
||||
: "The decoder terminated with SIGXCPU. The effective CPU limit was not recorded. A requested decoder budget does not raise an inherited RLIMIT_CPU.";
|
||||
return new AnalysisResourceConstraintError(
|
||||
OPERATION,
|
||||
"cpu",
|
||||
`${reason}: ${input.capture_path}. No complete capture result is available.`,
|
||||
{
|
||||
boundary: "historical-capture-decoder",
|
||||
capture_path: input.capture_path,
|
||||
format: input.format,
|
||||
observed_signal: "SIGXCPU",
|
||||
observed_exit_code: cause.snapshot?.exitCode ?? null,
|
||||
...(recorded ? { cpu_seconds: effectiveCpuSeconds } : {}),
|
||||
},
|
||||
{
|
||||
cause,
|
||||
retryable: false,
|
||||
remediationAction: recorded
|
||||
? `Raise the process RLIMIT_CPU above ${String(effectiveCpuSeconds)} seconds, or select a smaller capture. Repeating the unchanged request can stop at the same inherited limit.`
|
||||
: "Raise the process RLIMIT_CPU, or select a smaller capture. The effective inherited limit was not recorded, so the requested decoder budget is not that ceiling.",
|
||||
...(cause.snapshot === null
|
||||
? {}
|
||||
: { capturedOutput: capturedProviderOutput(cause.snapshot) }),
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Preserve the origin and typed cause of a historical capture failure.
|
||||
* `cleanupLease` is the decoder's lease on `rawCause`'s retained process
|
||||
@@ -110,6 +209,7 @@ export const historicalCaptureFailure = (
|
||||
options?: ExecutionOptions,
|
||||
heap: CaptureDecoderHeap = DEFAULT_CAPTURE_DECODER_HEAP,
|
||||
cleanupLease?: AnalysisCleanupLease,
|
||||
effectiveCpuSeconds?: number,
|
||||
): AnalysisError => {
|
||||
const { cause, origin } = artifactFilePrimaryFacts(rawCause);
|
||||
if (origin === "owned-or-classified" && cause instanceof AnalysisError)
|
||||
@@ -148,7 +248,15 @@ export const historicalCaptureFailure = (
|
||||
new AnalysisCleanupLeaseGroup(cleanup, [cleanupLease]);
|
||||
return cause.cause instanceof OwnedCommandFailure
|
||||
? analysisErrorWithCleanupFailure(
|
||||
historicalCaptureFailure(input, cause.cause, phase, options, heap),
|
||||
historicalCaptureFailure(
|
||||
input,
|
||||
cause.cause,
|
||||
phase,
|
||||
options,
|
||||
heap,
|
||||
undefined,
|
||||
effectiveCpuSeconds,
|
||||
),
|
||||
cleanup,
|
||||
OPERATION,
|
||||
)
|
||||
@@ -201,6 +309,12 @@ export const historicalCaptureFailure = (
|
||||
cause,
|
||||
heap,
|
||||
);
|
||||
if (
|
||||
phase === "decoder" &&
|
||||
outcome.kind === "resource-signal" &&
|
||||
outcome.signal === "SIGXCPU"
|
||||
)
|
||||
return captureCpuSignalFailure(input, cause, effectiveCpuSeconds);
|
||||
}
|
||||
if (
|
||||
phase !== "decoder" &&
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
import {
|
||||
CAPTURE_PROCESS_MEMORY_LIMIT_REASON,
|
||||
historicalCaptureFailure,
|
||||
recordedDecoderCpuSeconds,
|
||||
captureMemoryFailure,
|
||||
protobufCaptureFailure,
|
||||
withCaptureReaderCleanup,
|
||||
@@ -433,6 +434,10 @@ export class HistoricalCaptureDecoder {
|
||||
// Preparation transfers independent helper/descriptor owners. Keep that
|
||||
// typed stage separate from the decoder supervisor lease above: its
|
||||
// historical error already follows the post-launch completion owner.
|
||||
const effectiveCpuSeconds = await recordedDecoderCpuSeconds(
|
||||
runtime?.path,
|
||||
cause,
|
||||
);
|
||||
const error = retainProcessInspectionCleanup(
|
||||
processInspectionFailure(cause),
|
||||
historicalCaptureFailure(
|
||||
@@ -442,6 +447,7 @@ export class HistoricalCaptureDecoder {
|
||||
options,
|
||||
reportedHeap,
|
||||
processLease,
|
||||
effectiveCpuSeconds,
|
||||
),
|
||||
runtimeResources,
|
||||
OPERATION,
|
||||
|
||||
Reference in new issue
Block a user