mirror of
https://github.com/morluto/rea.git
synced 2026-11-22 06:25:44 +00:00
fix(browser): name the CPU ceiling when mitmproxy dies with SIGXCPU
A tighter inherited RLIMIT_CPU is kept, and that death had no reply. The decoder records the installed ceiling, and SIGXCPU becomes a CPU resource constraint that names it.
This commit is contained in:
1 parent
92f3f28d29
commit
39ade5694b
5 files changed
+284
-2
No files matched your search
@@ -10,6 +10,7 @@ import hashlib
|
||||
import ipaddress
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import resource
|
||||
from pathlib import Path
|
||||
@@ -347,6 +348,8 @@ def decode(request):
|
||||
|
||||
|
||||
ADDRESS_BUDGET = 768 * 1024 * 1024
|
||||
CPU_BUDGET = 30
|
||||
EFFECTIVE_CPU_RECORD = "effective-cpu.json"
|
||||
|
||||
|
||||
def failure_reply(error, address_limit=None):
|
||||
@@ -372,6 +375,23 @@ def _clamp_resource_limit(kind, requested):
|
||||
return effective
|
||||
|
||||
|
||||
def _record_cpu_limit(reply_path, effective):
|
||||
"""Publish the installed CPU ceiling before decode.
|
||||
|
||||
SIGXCPU ends the process with no reply. The parent reads this sibling of
|
||||
the reply when it observes that signal.
|
||||
"""
|
||||
target = Path(reply_path).with_name(EFFECTIVE_CPU_RECORD)
|
||||
temporary = Path(reply_path).with_name(EFFECTIVE_CPU_RECORD + ".tmp")
|
||||
payload = json.dumps({"seconds": effective}, separators=(",", ":")).encode("ascii")
|
||||
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
try:
|
||||
os.write(descriptor, payload)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
os.replace(temporary, target)
|
||||
|
||||
|
||||
def _address_limit_reply(effective):
|
||||
"""A memory refusal that names the inherited address-space ceiling."""
|
||||
return {
|
||||
@@ -420,7 +440,7 @@ class OfflineCapture:
|
||||
def running(self):
|
||||
try:
|
||||
address_limit = _clamp_resource_limit(resource.RLIMIT_AS, ADDRESS_BUDGET)
|
||||
_clamp_resource_limit(resource.RLIMIT_CPU, 30)
|
||||
cpu_limit = _clamp_resource_limit(resource.RLIMIT_CPU, CPU_BUDGET)
|
||||
# Zero never raises the inherited hard core limit, and it stops descendants from raising it.
|
||||
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
|
||||
request = json.loads(Path(ctx.options.rea_request_path).read_text(encoding="utf-8"))
|
||||
@@ -428,6 +448,7 @@ class OfflineCapture:
|
||||
if any(type(budgets[key]) is not int or budgets[key] < 1 for key in ("inputBytes", "outputBytes", "depth")):
|
||||
raise ValueError("Invalid admitted capture budgets")
|
||||
output_bytes = budgets["outputBytes"]
|
||||
_record_cpu_limit(request["reply_path"], cpu_limit)
|
||||
file_limit = _clamp_resource_limit(resource.RLIMIT_FSIZE, output_bytes)
|
||||
try:
|
||||
response = {"ok": True, "value": decode(request)}
|
||||
|
||||
@@ -362,6 +362,9 @@ class OfflineLimitTests(unittest.TestCase):
|
||||
)
|
||||
module.ctx.options = types.SimpleNamespace(rea_request_path=request_path)
|
||||
module.addons[0].running()
|
||||
with open(os.path.join(parent, "effective-cpu.json"), encoding="ascii") as handle:
|
||||
cpu_record = json.load(handle)
|
||||
self.assertEqual(cpu_record, {"seconds": 30})
|
||||
self.assertEqual(
|
||||
calls,
|
||||
[
|
||||
@@ -373,6 +376,42 @@ class OfflineLimitTests(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual(shutdowns, ["down"])
|
||||
|
||||
def test_tighter_inherited_cpu_limit_is_recorded(self):
|
||||
infinity = 2 ** 62
|
||||
limits = {kind: (infinity, infinity) for kind in (1, 3, 4)}
|
||||
limits[2] = (1, 100)
|
||||
calls = []
|
||||
|
||||
def setrlimit(kind, pair):
|
||||
_soft, hard = limits[kind]
|
||||
if hard != infinity and pair[1] > hard:
|
||||
raise ValueError("not allowed to raise maximum limit")
|
||||
calls.append((kind, pair))
|
||||
limits[kind] = pair
|
||||
|
||||
module = self.load_limited(limits, setrlimit)
|
||||
shutdowns = []
|
||||
module.ctx.master = types.SimpleNamespace(shutdown=lambda: shutdowns.append("down"))
|
||||
with tempfile.TemporaryDirectory() as parent:
|
||||
request_path = os.path.join(parent, "request.json")
|
||||
reply_path = os.path.join(parent, "reply.json")
|
||||
with open(request_path, "w", encoding="utf-8") as handle:
|
||||
json.dump(
|
||||
{
|
||||
"budgets": {"inputBytes": 8, "outputBytes": 1000, "depth": 2},
|
||||
"reply_path": reply_path,
|
||||
},
|
||||
handle,
|
||||
)
|
||||
module.ctx.options = types.SimpleNamespace(rea_request_path=request_path)
|
||||
module.addons[0].running()
|
||||
with open(os.path.join(parent, "effective-cpu.json"), encoding="ascii") as handle:
|
||||
cpu_record = json.load(handle)
|
||||
self.assertEqual(calls[1], (2, (1, 100)))
|
||||
self.assertEqual(cpu_record, {"seconds": 1})
|
||||
self.assertFalse(os.path.exists(os.path.join(parent, "effective-cpu.json.tmp")))
|
||||
self.assertEqual(shutdowns, ["down"])
|
||||
|
||||
def test_tighter_inherited_file_limit_bounds_the_reply(self):
|
||||
infinity = 2 ** 62
|
||||
file_limit = 4096
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { expect, it } from "vitest";
|
||||
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
|
||||
import { AnalysisCleanupLease } from "../../resources/AnalysisCleanupLease.js";
|
||||
import { ArtifactReaderFailure } from "../../resources/ArtifactReader.js";
|
||||
import {
|
||||
@@ -9,8 +12,11 @@ import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
|
||||
import { analysisErrorProjectionSchema } from "../../domain/analysisErrorProjectionSchema.js";
|
||||
import { inspectWebNetworkCaptureInputSchema } from "../../domain/webNetworkCapture.js";
|
||||
import {
|
||||
EFFECTIVE_DECODER_CPU_RECORD,
|
||||
historicalCaptureFailure,
|
||||
parseRecordedCpuSeconds,
|
||||
protobufCaptureFailure,
|
||||
recordedDecoderCpuSeconds,
|
||||
} from "./CaptureFailures.js";
|
||||
import {
|
||||
CaptureFormatError,
|
||||
@@ -244,6 +250,102 @@ it.each([
|
||||
},
|
||||
);
|
||||
|
||||
const emptyCommandOutput = { text: "", bytes: 0, observedBytes: 0 };
|
||||
|
||||
const cpuSignalFailure = (signal: "SIGXCPU" | "SIGXFSZ") =>
|
||||
new OwnedCommandFailure("process", "Decoder failed", {
|
||||
stdout: emptyCommandOutput,
|
||||
stderr: emptyCommandOutput,
|
||||
exitCode: null,
|
||||
signal,
|
||||
});
|
||||
|
||||
it("reports SIGXCPU with the CPU ceiling the decoder recorded", () => {
|
||||
const input = inspectWebNetworkCaptureInputSchema.parse({
|
||||
capture_path: "/selected/input.mitm",
|
||||
format: "mitmproxy",
|
||||
});
|
||||
const projected = projectAnalysisError(
|
||||
historicalCaptureFailure(
|
||||
input,
|
||||
cpuSignalFailure("SIGXCPU"),
|
||||
"decoder",
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
1,
|
||||
),
|
||||
);
|
||||
expect(analysisErrorProjectionSchema.parse(projected)).toMatchObject({
|
||||
code: "resource_constraint",
|
||||
retryable: false,
|
||||
details: {
|
||||
resource: "cpu",
|
||||
reported_limits: {
|
||||
cpu_seconds: 1,
|
||||
observed_signal: "SIGXCPU",
|
||||
format: "mitmproxy",
|
||||
},
|
||||
},
|
||||
remediation: {
|
||||
action: expect.stringContaining("RLIMIT_CPU above 1 seconds"),
|
||||
},
|
||||
});
|
||||
expect(JSON.stringify(projected)).not.toContain("768");
|
||||
});
|
||||
|
||||
it("reports SIGXCPU without inventing a CPU ceiling", () => {
|
||||
const input = inspectWebNetworkCaptureInputSchema.parse({
|
||||
capture_path: "/selected/input.mitm",
|
||||
format: "mitmproxy",
|
||||
});
|
||||
const projected = projectAnalysisError(
|
||||
historicalCaptureFailure(input, cpuSignalFailure("SIGXCPU"), "decoder"),
|
||||
);
|
||||
expect(analysisErrorProjectionSchema.parse(projected)).toMatchObject({
|
||||
code: "resource_constraint",
|
||||
details: {
|
||||
resource: "cpu",
|
||||
reported_limits: { observed_signal: "SIGXCPU" },
|
||||
},
|
||||
remediation: { action: expect.stringContaining("was not recorded") },
|
||||
});
|
||||
expect(projected.details?.reported_limits).not.toHaveProperty("cpu_seconds");
|
||||
});
|
||||
|
||||
it("leaves SIGXFSZ on the adapter failure path", () => {
|
||||
const input = inspectWebNetworkCaptureInputSchema.parse({
|
||||
capture_path: "/selected/input.mitm",
|
||||
format: "mitmproxy",
|
||||
});
|
||||
expect(
|
||||
projectAnalysisError(
|
||||
historicalCaptureFailure(input, cpuSignalFailure("SIGXFSZ"), "decoder"),
|
||||
),
|
||||
).toMatchObject({ code: "execution_failure" });
|
||||
});
|
||||
|
||||
it("reads only a single recorded CPU ceiling", async () => {
|
||||
expect(parseRecordedCpuSeconds(Buffer.from('{"seconds":1}'))).toBe(1);
|
||||
expect(parseRecordedCpuSeconds(Buffer.from('{"seconds":1,"other":2}'))).toBe(
|
||||
undefined,
|
||||
);
|
||||
expect(parseRecordedCpuSeconds(Buffer.from('{"seconds":1.5}'))).toBe(
|
||||
undefined,
|
||||
);
|
||||
const root = await createTestTempDirectory("rea-cpu-limit-");
|
||||
await writeFile(
|
||||
join(root, EFFECTIVE_DECODER_CPU_RECORD),
|
||||
'{"seconds":1}',
|
||||
"utf8",
|
||||
);
|
||||
const cause = cpuSignalFailure("SIGXCPU");
|
||||
expect(await recordedDecoderCpuSeconds(root, cause)).toBe(1);
|
||||
expect(
|
||||
await recordedDecoderCpuSeconds(root, cpuSignalFailure("SIGXFSZ")),
|
||||
).toBe(undefined);
|
||||
});
|
||||
|
||||
it.each(["before open", "during read"])(
|
||||
"preserves observed capture changes %s as retryable acquisition integrity failures",
|
||||
(stage) => {
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { open } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { capturedProviderOutput } from "../../process/capturedProviderOutput.js";
|
||||
import { javaScriptWorkerMemoryLimitBytes } from "../../process/JavaScriptWorkerMemoryLimit.js";
|
||||
import { isAbortOf } from "../../process/SignalAbort.js";
|
||||
@@ -98,6 +100,103 @@ export const protobufCaptureFailure = (
|
||||
export const CAPTURE_PROCESS_MEMORY_LIMIT_REASON =
|
||||
"The capture decoder reached its Windows Job Object memory limit";
|
||||
|
||||
/** Sibling of reply.json. The parent reads it after SIGXCPU. */
|
||||
export const EFFECTIVE_DECODER_CPU_RECORD = "effective-cpu.json";
|
||||
|
||||
const MAX_EFFECTIVE_CPU_RECORD_BYTES = 64;
|
||||
|
||||
/** Read the installed CPU ceiling. Undefined when the record is absent or malformed. */
|
||||
export const parseRecordedCpuSeconds = (bytes: Buffer): number | undefined => {
|
||||
if (
|
||||
bytes.byteLength === 0 ||
|
||||
bytes.byteLength > MAX_EFFECTIVE_CPU_RECORD_BYTES
|
||||
)
|
||||
return undefined;
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(bytes.toString("utf8")) as unknown;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed))
|
||||
return undefined;
|
||||
const record = parsed as Record<string, unknown>;
|
||||
const seconds = record.seconds;
|
||||
if (
|
||||
Object.keys(record).length !== 1 ||
|
||||
typeof seconds !== "number" ||
|
||||
!Number.isSafeInteger(seconds) ||
|
||||
seconds < 0
|
||||
)
|
||||
return undefined;
|
||||
return seconds;
|
||||
};
|
||||
|
||||
/** The CPU ceiling the decoder published before it was killed. */
|
||||
export const recordedDecoderCpuSeconds = async (
|
||||
runtimePath: string | undefined,
|
||||
cause: unknown,
|
||||
): Promise<number | undefined> => {
|
||||
if (
|
||||
runtimePath === undefined ||
|
||||
!(cause instanceof OwnedCommandFailure) ||
|
||||
cause.snapshot?.signal !== "SIGXCPU"
|
||||
)
|
||||
return undefined;
|
||||
let descriptor: Awaited<ReturnType<typeof open>> | undefined;
|
||||
try {
|
||||
descriptor = await open(
|
||||
join(runtimePath, EFFECTIVE_DECODER_CPU_RECORD),
|
||||
"r",
|
||||
);
|
||||
const info = await descriptor.stat();
|
||||
if (info.size > MAX_EFFECTIVE_CPU_RECORD_BYTES) return undefined;
|
||||
const bytes = Buffer.alloc(info.size);
|
||||
const read = await descriptor.read(bytes, 0, info.size, 0);
|
||||
if (read.bytesRead !== info.size) return undefined;
|
||||
return parseRecordedCpuSeconds(bytes);
|
||||
} catch {
|
||||
return undefined;
|
||||
} finally {
|
||||
await descriptor?.close();
|
||||
}
|
||||
};
|
||||
|
||||
/** A SIGXCPU death names the installed ceiling when the decoder recorded it. */
|
||||
const captureCpuSignalFailure = (
|
||||
input: InspectWebNetworkCaptureInput,
|
||||
cause: OwnedCommandFailure,
|
||||
effectiveCpuSeconds: number | undefined,
|
||||
): AnalysisResourceConstraintError => {
|
||||
const recorded = effectiveCpuSeconds !== undefined;
|
||||
const reason = recorded
|
||||
? `The effective RLIMIT_CPU of ${String(effectiveCpuSeconds)} seconds stopped the decoder. The requested decoder budget does not raise an inherited limit.`
|
||||
: "The decoder terminated with SIGXCPU. The effective CPU limit was not recorded. A requested decoder budget does not raise an inherited RLIMIT_CPU.";
|
||||
return new AnalysisResourceConstraintError(
|
||||
OPERATION,
|
||||
"cpu",
|
||||
`${reason}: ${input.capture_path}. No complete capture result is available.`,
|
||||
{
|
||||
boundary: "historical-capture-decoder",
|
||||
capture_path: input.capture_path,
|
||||
format: input.format,
|
||||
observed_signal: "SIGXCPU",
|
||||
observed_exit_code: cause.snapshot?.exitCode ?? null,
|
||||
...(recorded ? { cpu_seconds: effectiveCpuSeconds } : {}),
|
||||
},
|
||||
{
|
||||
cause,
|
||||
retryable: false,
|
||||
remediationAction: recorded
|
||||
? `Raise the process RLIMIT_CPU above ${String(effectiveCpuSeconds)} seconds, or select a smaller capture. Repeating the unchanged request can stop at the same inherited limit.`
|
||||
: "Raise the process RLIMIT_CPU, or select a smaller capture. The effective inherited limit was not recorded, so the requested decoder budget is not that ceiling.",
|
||||
...(cause.snapshot === null
|
||||
? {}
|
||||
: { capturedOutput: capturedProviderOutput(cause.snapshot) }),
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Preserve the origin and typed cause of a historical capture failure.
|
||||
* `cleanupLease` is the decoder's lease on `rawCause`'s retained process
|
||||
@@ -110,6 +209,7 @@ export const historicalCaptureFailure = (
|
||||
options?: ExecutionOptions,
|
||||
heap: CaptureDecoderHeap = DEFAULT_CAPTURE_DECODER_HEAP,
|
||||
cleanupLease?: AnalysisCleanupLease,
|
||||
effectiveCpuSeconds?: number,
|
||||
): AnalysisError => {
|
||||
const { cause, origin } = artifactFilePrimaryFacts(rawCause);
|
||||
if (origin === "owned-or-classified" && cause instanceof AnalysisError)
|
||||
@@ -148,7 +248,15 @@ export const historicalCaptureFailure = (
|
||||
new AnalysisCleanupLeaseGroup(cleanup, [cleanupLease]);
|
||||
return cause.cause instanceof OwnedCommandFailure
|
||||
? analysisErrorWithCleanupFailure(
|
||||
historicalCaptureFailure(input, cause.cause, phase, options, heap),
|
||||
historicalCaptureFailure(
|
||||
input,
|
||||
cause.cause,
|
||||
phase,
|
||||
options,
|
||||
heap,
|
||||
undefined,
|
||||
effectiveCpuSeconds,
|
||||
),
|
||||
cleanup,
|
||||
OPERATION,
|
||||
)
|
||||
@@ -201,6 +309,12 @@ export const historicalCaptureFailure = (
|
||||
cause,
|
||||
heap,
|
||||
);
|
||||
if (
|
||||
phase === "decoder" &&
|
||||
outcome.kind === "resource-signal" &&
|
||||
outcome.signal === "SIGXCPU"
|
||||
)
|
||||
return captureCpuSignalFailure(input, cause, effectiveCpuSeconds);
|
||||
}
|
||||
if (
|
||||
phase !== "decoder" &&
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
import {
|
||||
CAPTURE_PROCESS_MEMORY_LIMIT_REASON,
|
||||
historicalCaptureFailure,
|
||||
recordedDecoderCpuSeconds,
|
||||
captureMemoryFailure,
|
||||
protobufCaptureFailure,
|
||||
withCaptureReaderCleanup,
|
||||
@@ -433,6 +434,10 @@ export class HistoricalCaptureDecoder {
|
||||
// Preparation transfers independent helper/descriptor owners. Keep that
|
||||
// typed stage separate from the decoder supervisor lease above: its
|
||||
// historical error already follows the post-launch completion owner.
|
||||
const effectiveCpuSeconds = await recordedDecoderCpuSeconds(
|
||||
runtime?.path,
|
||||
cause,
|
||||
);
|
||||
const error = retainProcessInspectionCleanup(
|
||||
processInspectionFailure(cause),
|
||||
historicalCaptureFailure(
|
||||
@@ -442,6 +447,7 @@ export class HistoricalCaptureDecoder {
|
||||
options,
|
||||
reportedHeap,
|
||||
processLease,
|
||||
effectiveCpuSeconds,
|
||||
),
|
||||
runtimeResources,
|
||||
OPERATION,
|
||||
|
||||
Reference in new issue
Block a user