From 39ade5694b680dae47228ff26fcdfaafb74555ff Mon Sep 17 00:00:00 2001 From: Adolanium <94890352+Adolanium@users.noreply.github.com> Date: Fri, 20 Nov 2026 16:59:32 +0200 Subject: [PATCH] fix(browser): name the CPU ceiling when mitmproxy dies with SIGXCPU A tighter inherited RLIMIT_CPU is kept, and that death had no reply. The decoder records the installed ceiling, and SIGXCPU becomes a CPU resource constraint that names it. --- bridge/mitmproxy/capture.py | 23 +++- bridge/mitmproxy/test_capture.py | 39 ++++++ src/browser/history/CaptureFailures.test.ts | 102 +++++++++++++++ src/browser/history/CaptureFailures.ts | 116 +++++++++++++++++- .../history/HistoricalCaptureDecoder.ts | 6 + 5 files changed, 284 insertions(+), 2 deletions(-) diff --git a/bridge/mitmproxy/capture.py b/bridge/mitmproxy/capture.py index dcf0a3e573..a9ae3f4522 100644 --- a/bridge/mitmproxy/capture.py +++ b/bridge/mitmproxy/capture.py @@ -10,6 +10,7 @@ import hashlib import ipaddress import json import math +import os import re import resource from pathlib import Path @@ -347,6 +348,8 @@ def decode(request): ADDRESS_BUDGET = 768 * 1024 * 1024 +CPU_BUDGET = 30 +EFFECTIVE_CPU_RECORD = "effective-cpu.json" def failure_reply(error, address_limit=None): @@ -372,6 +375,23 @@ def _clamp_resource_limit(kind, requested): return effective +def _record_cpu_limit(reply_path, effective): + """Publish the installed CPU ceiling before decode. + + SIGXCPU ends the process with no reply. The parent reads this sibling of + the reply when it observes that signal. + """ + target = Path(reply_path).with_name(EFFECTIVE_CPU_RECORD) + temporary = Path(reply_path).with_name(EFFECTIVE_CPU_RECORD + ".tmp") + payload = json.dumps({"seconds": effective}, separators=(",", ":")).encode("ascii") + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + os.write(descriptor, payload) + finally: + os.close(descriptor) + os.replace(temporary, target) + + def _address_limit_reply(effective): """A memory refusal that names the inherited address-space ceiling.""" return { @@ -420,7 +440,7 @@ class OfflineCapture: def running(self): try: address_limit = _clamp_resource_limit(resource.RLIMIT_AS, ADDRESS_BUDGET) - _clamp_resource_limit(resource.RLIMIT_CPU, 30) + cpu_limit = _clamp_resource_limit(resource.RLIMIT_CPU, CPU_BUDGET) # Zero never raises the inherited hard core limit, and it stops descendants from raising it. resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) request = json.loads(Path(ctx.options.rea_request_path).read_text(encoding="utf-8")) @@ -428,6 +448,7 @@ class OfflineCapture: if any(type(budgets[key]) is not int or budgets[key] < 1 for key in ("inputBytes", "outputBytes", "depth")): raise ValueError("Invalid admitted capture budgets") output_bytes = budgets["outputBytes"] + _record_cpu_limit(request["reply_path"], cpu_limit) file_limit = _clamp_resource_limit(resource.RLIMIT_FSIZE, output_bytes) try: response = {"ok": True, "value": decode(request)} diff --git a/bridge/mitmproxy/test_capture.py b/bridge/mitmproxy/test_capture.py index bc67486bf7..f29891d3ee 100644 --- a/bridge/mitmproxy/test_capture.py +++ b/bridge/mitmproxy/test_capture.py @@ -362,6 +362,9 @@ class OfflineLimitTests(unittest.TestCase): ) module.ctx.options = types.SimpleNamespace(rea_request_path=request_path) module.addons[0].running() + with open(os.path.join(parent, "effective-cpu.json"), encoding="ascii") as handle: + cpu_record = json.load(handle) + self.assertEqual(cpu_record, {"seconds": 30}) self.assertEqual( calls, [ @@ -373,6 +376,42 @@ class OfflineLimitTests(unittest.TestCase): ) self.assertEqual(shutdowns, ["down"]) + def test_tighter_inherited_cpu_limit_is_recorded(self): + infinity = 2 ** 62 + limits = {kind: (infinity, infinity) for kind in (1, 3, 4)} + limits[2] = (1, 100) + calls = [] + + def setrlimit(kind, pair): + _soft, hard = limits[kind] + if hard != infinity and pair[1] > hard: + raise ValueError("not allowed to raise maximum limit") + calls.append((kind, pair)) + limits[kind] = pair + + module = self.load_limited(limits, setrlimit) + shutdowns = [] + module.ctx.master = types.SimpleNamespace(shutdown=lambda: shutdowns.append("down")) + with tempfile.TemporaryDirectory() as parent: + request_path = os.path.join(parent, "request.json") + reply_path = os.path.join(parent, "reply.json") + with open(request_path, "w", encoding="utf-8") as handle: + json.dump( + { + "budgets": {"inputBytes": 8, "outputBytes": 1000, "depth": 2}, + "reply_path": reply_path, + }, + handle, + ) + module.ctx.options = types.SimpleNamespace(rea_request_path=request_path) + module.addons[0].running() + with open(os.path.join(parent, "effective-cpu.json"), encoding="ascii") as handle: + cpu_record = json.load(handle) + self.assertEqual(calls[1], (2, (1, 100))) + self.assertEqual(cpu_record, {"seconds": 1}) + self.assertFalse(os.path.exists(os.path.join(parent, "effective-cpu.json.tmp"))) + self.assertEqual(shutdowns, ["down"]) + def test_tighter_inherited_file_limit_bounds_the_reply(self): infinity = 2 ** 62 file_limit = 4096 diff --git a/src/browser/history/CaptureFailures.test.ts b/src/browser/history/CaptureFailures.test.ts index 4f554af2ca..c7295a0d0a 100644 --- a/src/browser/history/CaptureFailures.test.ts +++ b/src/browser/history/CaptureFailures.test.ts @@ -1,4 +1,7 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; import { expect, it } from "vitest"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; import { AnalysisCleanupLease } from "../../resources/AnalysisCleanupLease.js"; import { ArtifactReaderFailure } from "../../resources/ArtifactReader.js"; import { @@ -9,8 +12,11 @@ import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; import { analysisErrorProjectionSchema } from "../../domain/analysisErrorProjectionSchema.js"; import { inspectWebNetworkCaptureInputSchema } from "../../domain/webNetworkCapture.js"; import { + EFFECTIVE_DECODER_CPU_RECORD, historicalCaptureFailure, + parseRecordedCpuSeconds, protobufCaptureFailure, + recordedDecoderCpuSeconds, } from "./CaptureFailures.js"; import { CaptureFormatError, @@ -244,6 +250,102 @@ it.each([ }, ); +const emptyCommandOutput = { text: "", bytes: 0, observedBytes: 0 }; + +const cpuSignalFailure = (signal: "SIGXCPU" | "SIGXFSZ") => + new OwnedCommandFailure("process", "Decoder failed", { + stdout: emptyCommandOutput, + stderr: emptyCommandOutput, + exitCode: null, + signal, + }); + +it("reports SIGXCPU with the CPU ceiling the decoder recorded", () => { + const input = inspectWebNetworkCaptureInputSchema.parse({ + capture_path: "/selected/input.mitm", + format: "mitmproxy", + }); + const projected = projectAnalysisError( + historicalCaptureFailure( + input, + cpuSignalFailure("SIGXCPU"), + "decoder", + undefined, + undefined, + undefined, + 1, + ), + ); + expect(analysisErrorProjectionSchema.parse(projected)).toMatchObject({ + code: "resource_constraint", + retryable: false, + details: { + resource: "cpu", + reported_limits: { + cpu_seconds: 1, + observed_signal: "SIGXCPU", + format: "mitmproxy", + }, + }, + remediation: { + action: expect.stringContaining("RLIMIT_CPU above 1 seconds"), + }, + }); + expect(JSON.stringify(projected)).not.toContain("768"); +}); + +it("reports SIGXCPU without inventing a CPU ceiling", () => { + const input = inspectWebNetworkCaptureInputSchema.parse({ + capture_path: "/selected/input.mitm", + format: "mitmproxy", + }); + const projected = projectAnalysisError( + historicalCaptureFailure(input, cpuSignalFailure("SIGXCPU"), "decoder"), + ); + expect(analysisErrorProjectionSchema.parse(projected)).toMatchObject({ + code: "resource_constraint", + details: { + resource: "cpu", + reported_limits: { observed_signal: "SIGXCPU" }, + }, + remediation: { action: expect.stringContaining("was not recorded") }, + }); + expect(projected.details?.reported_limits).not.toHaveProperty("cpu_seconds"); +}); + +it("leaves SIGXFSZ on the adapter failure path", () => { + const input = inspectWebNetworkCaptureInputSchema.parse({ + capture_path: "/selected/input.mitm", + format: "mitmproxy", + }); + expect( + projectAnalysisError( + historicalCaptureFailure(input, cpuSignalFailure("SIGXFSZ"), "decoder"), + ), + ).toMatchObject({ code: "execution_failure" }); +}); + +it("reads only a single recorded CPU ceiling", async () => { + expect(parseRecordedCpuSeconds(Buffer.from('{"seconds":1}'))).toBe(1); + expect(parseRecordedCpuSeconds(Buffer.from('{"seconds":1,"other":2}'))).toBe( + undefined, + ); + expect(parseRecordedCpuSeconds(Buffer.from('{"seconds":1.5}'))).toBe( + undefined, + ); + const root = await createTestTempDirectory("rea-cpu-limit-"); + await writeFile( + join(root, EFFECTIVE_DECODER_CPU_RECORD), + '{"seconds":1}', + "utf8", + ); + const cause = cpuSignalFailure("SIGXCPU"); + expect(await recordedDecoderCpuSeconds(root, cause)).toBe(1); + expect( + await recordedDecoderCpuSeconds(root, cpuSignalFailure("SIGXFSZ")), + ).toBe(undefined); +}); + it.each(["before open", "during read"])( "preserves observed capture changes %s as retryable acquisition integrity failures", (stage) => { diff --git a/src/browser/history/CaptureFailures.ts b/src/browser/history/CaptureFailures.ts index b582d39ecd..367217ff57 100644 --- a/src/browser/history/CaptureFailures.ts +++ b/src/browser/history/CaptureFailures.ts @@ -1,3 +1,5 @@ +import { open } from "node:fs/promises"; +import { join } from "node:path"; import { capturedProviderOutput } from "../../process/capturedProviderOutput.js"; import { javaScriptWorkerMemoryLimitBytes } from "../../process/JavaScriptWorkerMemoryLimit.js"; import { isAbortOf } from "../../process/SignalAbort.js"; @@ -98,6 +100,103 @@ export const protobufCaptureFailure = ( export const CAPTURE_PROCESS_MEMORY_LIMIT_REASON = "The capture decoder reached its Windows Job Object memory limit"; +/** Sibling of reply.json. The parent reads it after SIGXCPU. */ +export const EFFECTIVE_DECODER_CPU_RECORD = "effective-cpu.json"; + +const MAX_EFFECTIVE_CPU_RECORD_BYTES = 64; + +/** Read the installed CPU ceiling. Undefined when the record is absent or malformed. */ +export const parseRecordedCpuSeconds = (bytes: Buffer): number | undefined => { + if ( + bytes.byteLength === 0 || + bytes.byteLength > MAX_EFFECTIVE_CPU_RECORD_BYTES + ) + return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(bytes.toString("utf8")) as unknown; + } catch { + return undefined; + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) + return undefined; + const record = parsed as Record; + const seconds = record.seconds; + if ( + Object.keys(record).length !== 1 || + typeof seconds !== "number" || + !Number.isSafeInteger(seconds) || + seconds < 0 + ) + return undefined; + return seconds; +}; + +/** The CPU ceiling the decoder published before it was killed. */ +export const recordedDecoderCpuSeconds = async ( + runtimePath: string | undefined, + cause: unknown, +): Promise => { + if ( + runtimePath === undefined || + !(cause instanceof OwnedCommandFailure) || + cause.snapshot?.signal !== "SIGXCPU" + ) + return undefined; + let descriptor: Awaited> | undefined; + try { + descriptor = await open( + join(runtimePath, EFFECTIVE_DECODER_CPU_RECORD), + "r", + ); + const info = await descriptor.stat(); + if (info.size > MAX_EFFECTIVE_CPU_RECORD_BYTES) return undefined; + const bytes = Buffer.alloc(info.size); + const read = await descriptor.read(bytes, 0, info.size, 0); + if (read.bytesRead !== info.size) return undefined; + return parseRecordedCpuSeconds(bytes); + } catch { + return undefined; + } finally { + await descriptor?.close(); + } +}; + +/** A SIGXCPU death names the installed ceiling when the decoder recorded it. */ +const captureCpuSignalFailure = ( + input: InspectWebNetworkCaptureInput, + cause: OwnedCommandFailure, + effectiveCpuSeconds: number | undefined, +): AnalysisResourceConstraintError => { + const recorded = effectiveCpuSeconds !== undefined; + const reason = recorded + ? `The effective RLIMIT_CPU of ${String(effectiveCpuSeconds)} seconds stopped the decoder. The requested decoder budget does not raise an inherited limit.` + : "The decoder terminated with SIGXCPU. The effective CPU limit was not recorded. A requested decoder budget does not raise an inherited RLIMIT_CPU."; + return new AnalysisResourceConstraintError( + OPERATION, + "cpu", + `${reason}: ${input.capture_path}. No complete capture result is available.`, + { + boundary: "historical-capture-decoder", + capture_path: input.capture_path, + format: input.format, + observed_signal: "SIGXCPU", + observed_exit_code: cause.snapshot?.exitCode ?? null, + ...(recorded ? { cpu_seconds: effectiveCpuSeconds } : {}), + }, + { + cause, + retryable: false, + remediationAction: recorded + ? `Raise the process RLIMIT_CPU above ${String(effectiveCpuSeconds)} seconds, or select a smaller capture. Repeating the unchanged request can stop at the same inherited limit.` + : "Raise the process RLIMIT_CPU, or select a smaller capture. The effective inherited limit was not recorded, so the requested decoder budget is not that ceiling.", + ...(cause.snapshot === null + ? {} + : { capturedOutput: capturedProviderOutput(cause.snapshot) }), + }, + ); +}; + /** * Preserve the origin and typed cause of a historical capture failure. * `cleanupLease` is the decoder's lease on `rawCause`'s retained process @@ -110,6 +209,7 @@ export const historicalCaptureFailure = ( options?: ExecutionOptions, heap: CaptureDecoderHeap = DEFAULT_CAPTURE_DECODER_HEAP, cleanupLease?: AnalysisCleanupLease, + effectiveCpuSeconds?: number, ): AnalysisError => { const { cause, origin } = artifactFilePrimaryFacts(rawCause); if (origin === "owned-or-classified" && cause instanceof AnalysisError) @@ -148,7 +248,15 @@ export const historicalCaptureFailure = ( new AnalysisCleanupLeaseGroup(cleanup, [cleanupLease]); return cause.cause instanceof OwnedCommandFailure ? analysisErrorWithCleanupFailure( - historicalCaptureFailure(input, cause.cause, phase, options, heap), + historicalCaptureFailure( + input, + cause.cause, + phase, + options, + heap, + undefined, + effectiveCpuSeconds, + ), cleanup, OPERATION, ) @@ -201,6 +309,12 @@ export const historicalCaptureFailure = ( cause, heap, ); + if ( + phase === "decoder" && + outcome.kind === "resource-signal" && + outcome.signal === "SIGXCPU" + ) + return captureCpuSignalFailure(input, cause, effectiveCpuSeconds); } if ( phase !== "decoder" && diff --git a/src/browser/history/HistoricalCaptureDecoder.ts b/src/browser/history/HistoricalCaptureDecoder.ts index f9ddc77e3b..61b01e2b8c 100644 --- a/src/browser/history/HistoricalCaptureDecoder.ts +++ b/src/browser/history/HistoricalCaptureDecoder.ts @@ -14,6 +14,7 @@ import { import { CAPTURE_PROCESS_MEMORY_LIMIT_REASON, historicalCaptureFailure, + recordedDecoderCpuSeconds, captureMemoryFailure, protobufCaptureFailure, withCaptureReaderCleanup, @@ -433,6 +434,10 @@ export class HistoricalCaptureDecoder { // Preparation transfers independent helper/descriptor owners. Keep that // typed stage separate from the decoder supervisor lease above: its // historical error already follows the post-launch completion owner. + const effectiveCpuSeconds = await recordedDecoderCpuSeconds( + runtime?.path, + cause, + ); const error = retainProcessInspectionCleanup( processInspectionFailure(cause), historicalCaptureFailure( @@ -442,6 +447,7 @@ export class HistoricalCaptureDecoder { options, reportedHeap, processLease, + effectiveCpuSeconds, ), runtimeResources, OPERATION,