Files
reverse-engineering-anything/bridge/mitmproxy/test_capture.py
T
Adolanium 39ade5694b fix(browser): name the CPU ceiling when mitmproxy dies with SIGXCPU
A tighter inherited RLIMIT_CPU is kept, and that death had no reply. The decoder records the installed ceiling, and SIGXCPU becomes a CPU resource constraint that names it.
2026-11-21 20:08:53 +01:00

539 lines
24 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Source-owned tests for the mitmproxy adapter's record projection."""
import base64
import importlib.util
import json
import os
import sys
import tempfile
import types
import unittest
from unittest import mock
def load_capture(resource_module=None):
"""Load the adapter with fresh mitmproxy stand-ins scoped to its import."""
fake_mitmproxy = types.ModuleType("mitmproxy")
fake_mitmproxy.ctx = types.SimpleNamespace()
fake_mitmproxy.version = types.SimpleNamespace(VERSION="12.2.3")
fake_io = types.ModuleType("mitmproxy.io")
fake_io.tnetstring = types.ModuleType("mitmproxy.io.tnetstring")
if resource_module is None:
try:
import resource as resource_module
except ImportError:
resource_module = types.ModuleType("resource")
resource_module.RLIM_INFINITY = -1
resource_module.RLIMIT_AS = 9
resource_module.RLIMIT_CPU = 0
resource_module.RLIMIT_CORE = 4
resource_module.RLIMIT_FSIZE = 1
resource_module.getrlimit = lambda kind: (resource_module.RLIM_INFINITY, resource_module.RLIM_INFINITY)
resource_module.setrlimit = lambda kind, limits: None
stand_ins = {
"mitmproxy": fake_mitmproxy,
"mitmproxy.io": fake_io,
"mitmproxy.io.tnetstring": fake_io.tnetstring,
"resource": resource_module,
}
spec = importlib.util.spec_from_file_location(
"rea_mitmproxy_capture", os.path.join(os.path.dirname(__file__), "capture.py")
)
module = importlib.util.module_from_spec(spec)
with mock.patch.dict(sys.modules, stand_ins):
spec.loader.exec_module(module)
return module
capture = load_capture()
def location(value, scheme=None):
state = {"response": {"headers": [[b"Location", value]]}}
if scheme is not None:
state["request"] = {"scheme": scheme}
record = capture.project_record(state, [], 64)
sidecar = next(
field for field in record["binary_fields"] if field["pointer"] == "/response/headers/0/1"
)
return record["reported"]["response"]["headers"][0][1], sidecar, record["redactions"]
class TransportUserinfoTests(unittest.TestCase):
def assert_location_redacted(self, value, expected, scheme=None):
text, sidecar, redactions = location(value, scheme)
self.assertEqual(text, expected)
self.assertEqual(sidecar, {
"pointer": "/response/headers/0/1",
"representation": "producer-bytes",
"state": "redacted",
"content_base64": None,
"bytes": None,
"sha256": None,
})
self.assertEqual(
redactions,
[{"pointer": "/response/headers/0/1", "reason": "transport-credential"}],
)
def assert_location_retained(self, value, scheme=None):
text, sidecar, redactions = location(value, scheme)
self.assertEqual(text, value.decode("utf-8"))
self.assertEqual(sidecar["state"], "retained")
self.assertEqual(base64.b64decode(sidecar["content_base64"]), value)
self.assertEqual(redactions, [])
def test_ignorable_controls_cannot_hide_authority_credentials(self):
for control in [b"\t", b"\n", b"\r"]:
for value in [b"https://" + control + b"/user:pass@host/path", b"ht" + control + b"tps://user:" + control + b"pass@host/path"]:
with self.subTest(value=value):
text, sidecar, redactions = location(value, b"https")
self.assertNotIn("user", text)
self.assertNotIn("pass", text)
self.assertEqual(sidecar["state"], "redacted")
self.assertEqual(len(redactions), 1)
def test_mixed_network_separators_remove_credentials(self):
for prefix in [b"\\\\", b"/\\", b"\\/"]:
with self.subTest(prefix=prefix):
self.assert_location_redacted(
prefix + b"user:pass@host/path", prefix.decode("ascii") + "host/path", b"https"
)
def test_non_special_base_keeps_backslashes_inside_network_authorities(self):
text, sidecar, redactions = location(b"//user\\name:pass@host/x", b"foo")
self.assertEqual(text, "//host/x")
self.assertEqual(sidecar["state"], "redacted")
self.assertEqual(len(redactions), 1)
text, sidecar, redactions = location(b"/\\invoice@customer", b"foo")
self.assertEqual(text, "/\\invoice@customer")
self.assertEqual(sidecar["state"], "retained")
self.assertEqual(redactions, [])
def test_native_request_paths_preserve_origin_form_and_redact_absolute_form(self):
for value in [b"/\\invoice@customer", b"//invoice@customer", b"\\\\invoice@customer"]:
with self.subTest(value=value):
record = capture.project_record({"request": {"path": value}}, [], 64)
self.assertEqual(record["reported"]["request"]["path"], value.decode("ascii"))
self.assertEqual(record["binary_fields"][0]["state"], "retained")
self.assertEqual(record["redactions"], [])
record = capture.project_record({"request": {"path": b"https://user:pass@host/x"}}, [], 64)
self.assertEqual(record["reported"]["request"]["path"], "https://host/x")
self.assertEqual(record["binary_fields"][0]["state"], "redacted")
def test_same_scheme_locations_preserve_relative_path_text_and_bytes(self):
for value in [b"https:invoice@customer", b"https:/invoice@customer", b"https:\\invoice@customer"]:
with self.subTest(value=value):
self.assert_location_retained(value, b"https")
def test_location_authorities_still_remove_credentials_with_a_base_scheme(self):
for value, scheme, expected in [
(b"https://user:pass@host/path", b"https", "https://host/path"),
(b"https:\\\\user:pass@host/path", b"https", "https:\\\\host/path"),
(b"https:user:pass@host/path", b"http", "https:host/path"),
]:
with self.subTest(value=value, scheme=scheme):
self.assert_location_redacted(value, expected, scheme)
def test_backup_location_uses_its_own_request_scheme(self):
record = capture.project_record({
"request": {"scheme": b"http"},
"backup": {
"request": {"scheme": b"https"},
"response": {"headers": [[b"Location", b"https:invoice@customer"]]},
},
}, [], 64)
self.assertEqual(record["reported"]["backup"]["response"]["headers"][0][1], "https:invoice@customer")
self.assertEqual(record["redactions"], [])
def test_where_a_special_scheme_authority_ends(self):
self.assert_location_retained(b"https://example.test\\part@tail")
self.assert_location_redacted(
b"https://user:pw@example.test\\path", "https://example.test\\path"
)
self.assert_location_redacted(
b"https://user:pw@example.test/a@b", "https://example.test/a@b"
)
def test_empty_userinfo_marker_is_removed(self):
self.assertEqual(
capture.without_userinfo("https://@host"), "https://host"
)
def test_special_schemes_skip_leading_slashes_before_userinfo(self):
for value, expected in [
(b"https://\\user:pass@host/path", b"https://\\host/path"),
(b"https:\\\\user:pass@host/path", b"https:\\\\host/path"),
(b"https:user:pass@host", b"https:host"),
(b"//\\user:pass@host", b"//\\host"),
]:
with self.subTest(value=value):
self.assertEqual(capture.without_userinfo(value), expected)
text, sidecar, redactions = location(b"https://\\user:pass@host/path")
self.assertEqual(text, "https://\\host/path")
self.assertEqual(sidecar["state"], "redacted")
self.assertEqual(len(redactions), 1)
def test_file_base_keeps_paths_after_the_fixed_network_boundary(self):
for value in [b"///u@h", b"////u@h"]:
with self.subTest(value=value):
self.assert_location_retained(value, b"file")
def test_file_urls_keep_path_text(self):
self.assertEqual(capture.without_userinfo("file:///a@b"), "file:///a@b")
def test_file_urls_reject_userinfo_and_keep_their_source(self):
# WHATWG rejects an "@" in a file host, so nothing parses as credentials.
self.assertEqual(
capture.without_userinfo("file://user:pass@host/x"), "file://user:pass@host/x"
)
for value in [b"\\\\user:pass@host/x", b"/\\user:pass@host/x", b"//user:pass@host/x"]:
with self.subTest(value=value):
self.assert_location_retained(value, b"file")
def test_rejected_special_hosts_keep_their_source(self):
# A rejected host or port fails the whole URL, so it has no userinfo.
for value in [
"https:user@%",
"https://user@",
"https://user:pass@h^st/x",
"https://user@ho st/x",
"https://user@host:99999/x",
"https://user@host:12a",
"https://user@[::1",
]:
with self.subTest(value=value):
self.assertEqual(capture.without_userinfo(value), value)
self.assert_location_retained(b"https://user:pass@h^st/x", b"https")
def test_accepted_special_hosts_still_remove_userinfo(self):
for value, expected in [
("https:user@%41", "https:%41"),
("https://user@[::1]:8443/x", "https://[::1]:8443/x"),
("https://user@host:/x", "https://host:/x"),
("wss://user:pass@host:65535", "wss://host:65535"),
("https://user@0x7f.1/", "https://0x7f.1/"),
("https://user@1.2.3.4./", "https://1.2.3.4./"),
("https://user@4294967295/", "https://4294967295/"),
("https://user@1.2.3.08a/", "https://1.2.3.08a/"),
("https://user@[::ffff:1.2.3.4]/", "https://[::ffff:1.2.3.4]/"),
]:
with self.subTest(value=value):
self.assertEqual(capture.without_userinfo(value), expected)
def test_unicode_hosts_that_may_map_to_accepted_hosts_remove_userinfo(self):
# UTS #46 maps these to 1.2.3.4; mapping is not modeled, so redact.
for host in ["%EF%BC%91.2.3.4", "1。2.3.4", "%C2%AD1.2.3.4", "1.2.3.4"]:
with self.subTest(host=host):
self.assertEqual(
capture.without_userinfo(f"https://user:pw@{host}/"), f"https://{host}/"
)
self.assert_location_redacted(
b"https://user:pw@%EF%BC%91.2.3.4/", "https://%EF%BC%91.2.3.4/", b"https"
)
# ASCII rejections still hold for non-ASCII hosts.
self.assertEqual(capture.without_userinfo("https://user@1^/"), "https://user@1^/")
def test_trailing_controls_and_spaces_do_not_hide_userinfo(self):
# WHATWG strips trailing C0 controls and spaces before parsing.
for suffix in [b" ", b"\x01", b"\x00 \x1f"]:
with self.subTest(suffix=suffix):
value = b"https://user:pw@example.test" + suffix
self.assert_location_redacted(
value, "https://example.test" + suffix.decode("latin-1"), b"https"
)
def test_rejected_ip_literals_keep_their_source(self):
for value in [
"https://user:pass@256.256.256.256/path",
"https://user@1.2.3.4.5/",
"https://user@1.2.3.09/",
"https://user@4294967296/",
"https://user@a.0x/",
"https://user@1..2/",
"https://user@1." + "9" * 5000 + "/",
"https://user@[1::2::3]/",
"https://user@[::1%25eth0]/",
]:
with self.subTest(value=value[:40]):
self.assertEqual(capture.without_userinfo(value), value)
def test_overlong_ports_are_compared_without_numeric_overflow(self):
# Python 3.11+ refuses int() on decimal strings over 4300 digits.
accepted = "https://user@host:" + "0" * 5000 + "80/x"
self.assertEqual(capture.without_userinfo(accepted), "https://host:" + "0" * 5000 + "80/x")
rejected = "https://user@host:" + "9" * 5000 + "/x"
self.assertEqual(capture.without_userinfo(rejected), rejected)
def test_non_special_schemes_keep_backslashes_in_the_authority(self):
self.assertEqual(capture.without_userinfo("foo://a\\b@host/x"), "foo://host/x")
self.assertEqual(capture.without_userinfo(b"//a\\b@host"), b"//a\\b@host")
class OfflineLimitTests(unittest.TestCase):
def load_limited(self, limits, setrlimit):
resource = types.ModuleType("resource")
resource.RLIM_INFINITY = 2 ** 62
resource.RLIMIT_AS = 1
resource.RLIMIT_CPU = 2
resource.RLIMIT_CORE = 3
resource.RLIMIT_FSIZE = 4
resource.getrlimit = lambda kind: limits[kind]
resource.setrlimit = setrlimit
return load_capture(resource)
def test_tighter_hard_limits_are_kept_and_shutdown_still_runs(self):
infinity = 2 ** 62
# Linux never reports a soft limit above its hard limit. Start at that ceiling.
limits = {
1: (1024, 1024),
2: (5, 5),
3: (infinity, infinity),
4: (512, 512),
}
calls = []
def setrlimit(kind, pair):
_soft, hard = limits[kind]
if hard != infinity and pair[1] > hard:
raise ValueError("not allowed to raise maximum limit")
calls.append((kind, pair))
limits[kind] = pair
module = self.load_limited(limits, setrlimit)
shutdowns = []
module.ctx.master = types.SimpleNamespace(shutdown=lambda: shutdowns.append("down"))
with tempfile.TemporaryDirectory() as parent:
request_path = os.path.join(parent, "request.json")
reply_path = os.path.join(parent, "reply.json")
with open(request_path, "w", encoding="utf-8") as handle:
json.dump(
{
"budgets": {"inputBytes": 8, "outputBytes": 1000, "depth": 2},
"reply_path": reply_path,
},
handle,
)
module.ctx.options = types.SimpleNamespace(rea_request_path=request_path)
module.addons[0].running()
with open(reply_path, encoding="ascii") as handle:
reply = json.loads(handle.read())
self.assertEqual(
calls,
[(1, (1024, 1024)), (2, (5, 5)), (3, (0, 0)), (4, (512, 512))],
)
self.assertEqual(shutdowns, ["down"])
self.assertFalse(reply["ok"])
def test_inherited_hard_limit_stays_above_the_new_soft_limit(self):
infinity = 2 ** 62
address_limit = 768 * 1024 * 1024
output_bytes = 1000
# Unlimited CPU, and finite soft limits strictly below their hard limits.
# Writing the new soft value into the hard limit fails these pairs.
limits = {
1: (address_limit * 2, address_limit * 4),
2: (infinity, infinity),
3: (infinity, infinity),
4: (5000, 9000),
}
calls = []
def setrlimit(kind, pair):
_soft, hard = limits[kind]
if hard != infinity and pair[1] > hard:
raise ValueError("not allowed to raise maximum limit")
calls.append((kind, pair))
limits[kind] = pair
module = self.load_limited(limits, setrlimit)
shutdowns = []
module.ctx.master = types.SimpleNamespace(shutdown=lambda: shutdowns.append("down"))
with tempfile.TemporaryDirectory() as parent:
request_path = os.path.join(parent, "request.json")
reply_path = os.path.join(parent, "reply.json")
with open(request_path, "w", encoding="utf-8") as handle:
json.dump(
{
"budgets": {"inputBytes": 8, "outputBytes": output_bytes, "depth": 2},
"reply_path": reply_path,
},
handle,
)
module.ctx.options = types.SimpleNamespace(rea_request_path=request_path)
module.addons[0].running()
with open(os.path.join(parent, "effective-cpu.json"), encoding="ascii") as handle:
cpu_record = json.load(handle)
self.assertEqual(cpu_record, {"seconds": 30})
self.assertEqual(
calls,
[
(1, (address_limit, address_limit * 4)),
(2, (30, infinity)),
(3, (0, 0)),
(4, (output_bytes, 9000)),
],
)
self.assertEqual(shutdowns, ["down"])
def test_tighter_inherited_cpu_limit_is_recorded(self):
infinity = 2 ** 62
limits = {kind: (infinity, infinity) for kind in (1, 3, 4)}
limits[2] = (1, 100)
calls = []
def setrlimit(kind, pair):
_soft, hard = limits[kind]
if hard != infinity and pair[1] > hard:
raise ValueError("not allowed to raise maximum limit")
calls.append((kind, pair))
limits[kind] = pair
module = self.load_limited(limits, setrlimit)
shutdowns = []
module.ctx.master = types.SimpleNamespace(shutdown=lambda: shutdowns.append("down"))
with tempfile.TemporaryDirectory() as parent:
request_path = os.path.join(parent, "request.json")
reply_path = os.path.join(parent, "reply.json")
with open(request_path, "w", encoding="utf-8") as handle:
json.dump(
{
"budgets": {"inputBytes": 8, "outputBytes": 1000, "depth": 2},
"reply_path": reply_path,
},
handle,
)
module.ctx.options = types.SimpleNamespace(rea_request_path=request_path)
module.addons[0].running()
with open(os.path.join(parent, "effective-cpu.json"), encoding="ascii") as handle:
cpu_record = json.load(handle)
self.assertEqual(calls[1], (2, (1, 100)))
self.assertEqual(cpu_record, {"seconds": 1})
self.assertFalse(os.path.exists(os.path.join(parent, "effective-cpu.json.tmp")))
self.assertEqual(shutdowns, ["down"])
def test_tighter_inherited_file_limit_bounds_the_reply(self):
infinity = 2 ** 62
file_limit = 4096
limits = {kind: (infinity, infinity) for kind in (1, 2, 3)}
limits[4] = (file_limit, 8192)
calls = []
def setrlimit(kind, pair):
_soft, hard = limits[kind]
if hard != infinity and pair[1] > hard:
raise ValueError("not allowed to raise maximum limit")
calls.append((kind, pair))
limits[kind] = pair
module = self.load_limited(limits, setrlimit)
module.decode = lambda _request: {"pad": "x" * 5000}
shutdowns = []
module.ctx.master = types.SimpleNamespace(shutdown=lambda: shutdowns.append("down"))
with tempfile.TemporaryDirectory() as parent:
request_path = os.path.join(parent, "request.json")
reply_path = os.path.join(parent, "reply.json")
with open(request_path, "w", encoding="utf-8") as handle:
json.dump(
{
"budgets": {"inputBytes": 8, "outputBytes": 10000, "depth": 2},
"reply_path": reply_path,
},
handle,
)
module.ctx.options = types.SimpleNamespace(rea_request_path=request_path)
module.addons[0].running()
with open(reply_path, "rb") as handle:
reply_bytes = handle.read()
reply = json.loads(reply_bytes.decode("ascii"))
self.assertLessEqual(len(reply_bytes), file_limit)
self.assertEqual(reply["reason"], "fixed-limit")
self.assertEqual(reply["boundary"], "file-size")
self.assertEqual(reply["ceiling"]["limit"], file_limit)
self.assertEqual(calls[-1], (4, (file_limit, 8192)))
self.assertEqual(shutdowns, ["down"])
def _memory_failure(self, address_soft, address_hard):
infinity = 2 ** 62
limits = {kind: (infinity, infinity) for kind in (2, 3, 4)}
limits[1] = (address_soft, address_hard)
calls = []
def setrlimit(kind, pair):
_soft, hard = limits[kind]
if hard != infinity and pair[1] > hard:
raise ValueError("not allowed to raise maximum limit")
calls.append((kind, pair))
limits[kind] = pair
module = self.load_limited(limits, setrlimit)
def exhausted(_request):
raise MemoryError("payload must not enter diagnostics")
module.decode = exhausted
shutdowns = []
module.ctx.master = types.SimpleNamespace(shutdown=lambda: shutdowns.append("down"))
with tempfile.TemporaryDirectory() as parent:
request_path = os.path.join(parent, "request.json")
reply_path = os.path.join(parent, "reply.json")
with open(request_path, "w", encoding="utf-8") as handle:
json.dump(
{
"budgets": {"inputBytes": 8, "outputBytes": 10000, "depth": 2},
"reply_path": reply_path,
},
handle,
)
module.ctx.options = types.SimpleNamespace(rea_request_path=request_path)
module.addons[0].running()
with open(reply_path, "rb") as handle:
reply_bytes = handle.read()
return json.loads(reply_bytes.decode("ascii")), reply_bytes, calls, shutdowns, module
def test_tighter_inherited_address_limit_names_the_memory_ceiling(self):
soft = 512 * 1024 * 1024
hard = 1024 * 1024 * 1024
reply, reply_bytes, calls, shutdowns, _module = self._memory_failure(soft, hard)
self.assertNotIn(b"payload must not enter diagnostics", reply_bytes)
self.assertEqual(reply["reason"], "fixed-limit")
self.assertEqual(reply["boundary"], "memory")
self.assertEqual(reply["ceiling"]["limit"], soft)
self.assertEqual(reply["ceiling"]["kind"], "bytes")
self.assertIn(str(soft), reply["message"])
self.assertNotIn("exhausted its 768 MiB", reply["message"])
self.assertEqual(calls[0], (1, (soft, hard)))
self.assertEqual(shutdowns, ["down"])
def test_requested_address_budget_stays_a_resource_limit(self):
budget = 768 * 1024 * 1024
reply, reply_bytes, calls, shutdowns, module = self._memory_failure(budget * 2, budget * 4)
self.assertNotIn(b"payload must not enter diagnostics", reply_bytes)
self.assertEqual(reply["reason"], "resource-limit")
self.assertEqual(reply["message"], "Native decoder exhausted its 768 MiB address-space budget.")
self.assertNotIn("ceiling", reply)
self.assertEqual(calls[0], (1, (budget, budget * 4)))
self.assertEqual(shutdowns, ["down"])
direct = module.failure_reply(MemoryError("payload must not enter diagnostics"))
self.assertEqual(direct["reason"], "resource-limit")
self.assertNotIn("payload must not enter diagnostics", json.dumps(direct))
def test_limit_setup_failure_still_shuts_down(self):
infinity = 2 ** 62
limits = {kind: (infinity, infinity) for kind in (1, 2, 3, 4)}
def setrlimit(_kind, _pair):
raise ValueError("not allowed to raise maximum limit")
module = self.load_limited(limits, setrlimit)
shutdowns = []
module.ctx.master = types.SimpleNamespace(shutdown=lambda: shutdowns.append("down"))
module.ctx.options = types.SimpleNamespace(rea_request_path="")
with self.assertRaisesRegex(ValueError, "not allowed to raise maximum limit"):
module.addons[0].running()
self.assertEqual(shutdowns, ["down"])
if __name__ == "__main__":
unittest.main()