diff --git a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md index c06025d613..70b0afdff3 100644 --- a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md +++ b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md @@ -296,6 +296,35 @@ Exported C and embedded resources remain useful independent evidence, but a timeout does not establish complete analysis or absence of code. See the [deadline and client guide](https://github.com/morluto/rea/blob/main/docs/mcp-contracts.md#ghidra-first-query-deadlines-and-recovery). +## Bridge request ceiling + +Hopper and Cutter share `REA_BRIDGE_MAX_REQUEST_BYTES`. Unset, both sides use +16777216 bytes. The payload budget excludes the terminating LF. A decimal +integer from 7464 to 536870887 raises or lowers that ceiling. The floor fits +the largest bounded symbol-name command. A tight Cutter status still returns +its session id and up to 64 completion receipts. Its path is included when it +fits, and is unknown when it does not, so an older descriptor path is not +paired with the new generation. The maximum is the encoded payload, one byte under Node's string ceiling so +the framing LF still fits. Cutter measures, signs, and writes a fitting +response in bounded chunks. The bridge searches only the newly read bytes +for the framing LF. The client unescapes each socket chunk into one response +string before parsing it, so this ceiling does not keep the paged wire frame +and another concatenated copy. Invalid values are configuration errors and do not fall back to 16777216. Set the variable in the environment that starts REA, restart REA, +and start Hopper or Cutter from that same environment. An out-of-band bridge +without the setting stays at 16777216 bytes. The client reports that mismatch +during discovery only after the hello proves a live authenticated bridge, and +does not send a status request or the command. A stale descriptor is ignored. +Discovery fails if any such live ceiling differs, even when another live +session matches. A command response discarded for that ceiling stays an +unknown execution with `output_truncated`. A hello +that never sends its authenticated frame does not hold the accept loop. +At most 8 connection threads run; a further accept is closed, and a failure +to start a thread does not stop the listener. `completed_connection` is the +longest prefix of connections that have terminated. Dense scalar replies are +encoded in bounded batches. +The 25000 ms Cutter request deadline and the 256-candidate discovery budget +still have no runtime override. + ## Static memory and original file bytes For Ghidra, `read_bytes` reads initialized analysis memory, including loader diff --git a/bridge/cutter/rea_bridge.py b/bridge/cutter/rea_bridge.py index adaaa20ef9..cc3dd507b5 100644 --- a/bridge/cutter/rea_bridge.py +++ b/bridge/cutter/rea_bridge.py @@ -11,6 +11,7 @@ import json import hashlib import hmac import os +import re import secrets import socket import stat @@ -33,9 +34,151 @@ except ImportError: from PySide2.QtCore import QCoreApplication, QObject, Signal -MAX_FRAME_BYTES = 16 * 1024 * 1024 +# Request and response frame budget in bytes, excluding the trailing LF. +# REA_BRIDGE_MAX_REQUEST_BYTES selects it. Unset stays 16777216. +# 7464 is the authenticated frame of the largest bounded symbol name. +# The 64-receipt recovery status is smaller with current_file null. A path +# that fits is kept. A path that does not fit is null, not the descriptor path. +# 536870887 is Node's 536870888-character string ceiling minus the framing LF. +# The ceiling is the encoded payload; JSON expansion is measured before it. +# A fitting response is signed from bounded JSON pieces and written in 64 KiB +# chunks, so this maximum does not retain several copies of the frame. +_BRIDGE_MAX_REQUEST_BYTES_SETTING = "REA_BRIDGE_MAX_REQUEST_BYTES" +DEFAULT_BRIDGE_MAX_REQUEST_BYTES = 16 * 1024 * 1024 +MINIMUM_BRIDGE_MAX_REQUEST_BYTES = 7464 +MAXIMUM_BRIDGE_MAX_REQUEST_BYTES = 536870887 +# The outer authenticated frame around the escaped JSON payload, excluding LF. +_AUTHENTICATED_PREFIX = b'{"response":"' +_AUTHENTICATED_PROOF_MARK = b'","proof":"' +_PROOF_HEX_LENGTH = 64 +_AUTHENTICATED_FRAME_OVERHEAD = ( + len(_AUTHENTICATED_PREFIX) + + len(_AUTHENTICATED_PROOF_MARK) + + _PROOF_HEX_LENGTH + + len(b'"}') +) +_AUTHENTICATED_WRITE_BUFFER_BYTES = 64 * 1024 +# Raw JSON is produced in pieces no larger than this many characters. +# One character expands to at most six JSON bytes (\u00XX) before the outer +# string escape, so a piece stays far below the frame ceiling. +_JSON_CHUNK_CHARS = 64 * 1024 +# Fixed admission cap. It does not follow REA_BRIDGE_MAX_REQUEST_BYTES. +# A further accept is closed. Starting a worker must not stop the listener. +MAX_CONNECTION_WORKERS = 8 + + +def _close_socket(connection: socket.socket) -> None: + try: + connection.close() + except OSError: + pass + + +def _bridge_max_request_bytes_constraint(): + return ( + "REA_BRIDGE_MAX_REQUEST_BYTES must be a positive decimal integer " + "from %d to %d bytes; the default is %d bytes (16 MiB). " + "A value below %d bytes cannot carry the largest admitted " + "symbol-name command, and a larger value is refused so the encoded " + "payload plus its framing LF stays one string under Node's " + "single-string ceiling." + % ( + MINIMUM_BRIDGE_MAX_REQUEST_BYTES, + MAXIMUM_BRIDGE_MAX_REQUEST_BYTES, + DEFAULT_BRIDGE_MAX_REQUEST_BYTES, + MINIMUM_BRIDGE_MAX_REQUEST_BYTES, + ) + ) + + +def _parse_bridge_max_request_bytes(raw): + """Refuse a non-decimal or out-of-range ceiling. Never fall back.""" + constraint = _bridge_max_request_bytes_constraint() + if isinstance(raw, bool): + raise RuntimeError(constraint) + if isinstance(raw, int): + value = raw + elif isinstance(raw, str): + # ASCII digits only. str.isdigit() accepts numerals such as U+0668 + # that the TypeScript setting rejects. + if re.fullmatch(r"[1-9][0-9]*", raw) is None: + raise RuntimeError(constraint) + value = int(raw) + else: + raise RuntimeError(constraint) + if ( + value < MINIMUM_BRIDGE_MAX_REQUEST_BYTES + or value > MAXIMUM_BRIDGE_MAX_REQUEST_BYTES + ): + raise RuntimeError(constraint) + return value + + +def _resolve_bridge_max_request_bytes(): + """Prefer an injected global, then the process environment, then 16 MiB.""" + injected = globals().get(_BRIDGE_MAX_REQUEST_BYTES_SETTING) + if injected is not None: + return _parse_bridge_max_request_bytes(injected), "setting" + raw = os.environ.get(_BRIDGE_MAX_REQUEST_BYTES_SETTING) + if raw is None: + return DEFAULT_BRIDGE_MAX_REQUEST_BYTES, "default" + return _parse_bridge_max_request_bytes(raw), "setting" + + +MAX_FRAME_BYTES, BRIDGE_MAX_REQUEST_BYTES_SOURCE = ( + _resolve_bridge_max_request_bytes() +) + + +def _bridge_max_request_bytes_recovery(): + return ( + "The effective REA_BRIDGE_MAX_REQUEST_BYTES value is %d bytes " + "(source %s; default %d, floor %d, maximum %d). " + "Set REA_BRIDGE_MAX_REQUEST_BYTES in the environment that starts REA " + "to a decimal integer from %d to %d, then restart REA. " + "Restart Hopper or Cutter from that same environment so the bridge " + "uses this ceiling." + % ( + MAX_FRAME_BYTES, + BRIDGE_MAX_REQUEST_BYTES_SOURCE, + DEFAULT_BRIDGE_MAX_REQUEST_BYTES, + MINIMUM_BRIDGE_MAX_REQUEST_BYTES, + MAXIMUM_BRIDGE_MAX_REQUEST_BYTES, + MINIMUM_BRIDGE_MAX_REQUEST_BYTES, + MAXIMUM_BRIDGE_MAX_REQUEST_BYTES, + ) + ) + + +def _frame_limit_exceeded(): + return ValueError( + "frame-limit-exceeded: Bridge request exceeds the %d-byte payload limit. %s" + % (MAX_FRAME_BYTES, _bridge_max_request_bytes_recovery()) + ) + + SOCKET_TIMEOUT_SECONDS = 20 AUTH_FRAME_TIMEOUT_SECONDS = 1 +# The client request deadline. The authenticated frame read may use up to +# this long; it does not grow past it when the byte ceiling is raised. +REQUEST_DEADLINE_SECONDS = 25 + + +def _authenticated_frame_timeout_seconds(): + """Time to hash and transfer one maximum-sized authenticated frame. + + The hello frame stays at one second. The following command frame is + built only after that challenge, so its read budget follows the + configured payload size, at one second per mebibyte, and never exceeds + the 25 second client deadline. + """ + mebibytes = (MAX_FRAME_BYTES + (1024 * 1024) - 1) // (1024 * 1024) + return min( + REQUEST_DEADLINE_SECONDS, + max(AUTH_FRAME_TIMEOUT_SECONDS, mebibytes), + ) + + MAX_DESCRIPTOR_BYTES = 64 * 1024 MAX_COMPLETION_RECEIPTS = 64 @@ -480,6 +623,17 @@ class _BridgeServer(threading.Thread): self.token = secrets.token_urlsafe(32) self.poisoned = False self.connection_ordinal = 0 + # The client proof is inside the authenticated frame, so that read + # cannot start after proof. It runs off the accept loop. One + # authenticated command still runs at a time. completed_connection + # is the longest prefix of these ordinals whose handlers have + # terminated, not "this status ordinal minus one". + self._command_lock = threading.Lock() + self._connections_lock = threading.Lock() + self._connections: set[socket.socket] = set() + self._workers: list[threading.Thread] = [] + self._finished_through = 0 + self._finished_ordinals: set[int] = set() self.directory = _bridge_directory() self.descriptor = self.directory / f"cutter-{os.getpid()}-{self.session_id}.json" @@ -500,6 +654,7 @@ class _BridgeServer(threading.Thread): "current_file": self.dispatcher.current_file, "cutter_version": self.dispatcher.cutter_version, "identity_status": "partial", + "max_request_bytes": MAX_FRAME_BYTES, } if os.name == "nt": _write_windows_descriptor(self.directory, self.descriptor, json.dumps(payload).encode("utf-8")) @@ -541,8 +696,46 @@ class _BridgeServer(threading.Thread): continue except OSError: break + with self._connections_lock: + self._workers = [ + worker for worker in self._workers if worker.is_alive() + ] + at_capacity = len(self._workers) >= MAX_CONNECTION_WORKERS + if at_capacity: + _close_socket(connection) + continue self.connection_ordinal += 1 connection_ordinal = self.connection_ordinal + worker = threading.Thread( + target=self._serve_connection, + args=(connection, connection_ordinal), + name="rea-cutter-bridge-connection", + daemon=True, + ) + with self._connections_lock: + self._connections.add(connection) + self._workers.append(worker) + try: + worker.start() + except RuntimeError: + with self._connections_lock: + self._connections.discard(connection) + if worker in self._workers: + self._workers.remove(worker) + self._note_connection_finished(connection_ordinal) + _close_socket(connection) + with self._connections_lock: + workers = list(self._workers) + for worker in workers: + worker.join(timeout=1) + self.server.close() + try: + self.descriptor.unlink(missing_ok=True) + except OSError: + pass + + def _serve_connection(self, connection: socket.socket, connection_ordinal: int) -> None: + try: with connection: connection.settimeout(AUTH_FRAME_TIMEOUT_SECONDS) authenticated = False @@ -550,60 +743,73 @@ class _BridgeServer(threading.Thread): hello = self._receive_frame(connection) if not isinstance(hello, dict) or hello.get("kind") != "rea-auth-v2" or not isinstance(hello.get("nonce"), str) or len(hello["nonce"]) != 64: self._send(connection, {"ok": False, "error": "authentication-failed"}) - continue + return client_nonce = hello["nonce"] server_nonce = f"{connection_ordinal:016x}{secrets.token_hex(24)}" server_proof = hmac.new(self.token.encode(), f"rea-cutter-server-v2:{self.session_id}:{client_nonce}:{server_nonce}".encode(), hashlib.sha256).hexdigest() self._send(connection, {"session_id": self.session_id, "nonce": server_nonce, "proof": server_proof}) - authenticated_frame = self._receive_frame(connection) + authenticated_frame = self._receive_frame( + connection, _authenticated_frame_timeout_seconds() + ) request_bytes = authenticated_frame.get("request") if isinstance(authenticated_frame, dict) else None - expected_proof = hmac.new(self.token.encode(), f"rea-cutter-client-v2:{self.session_id}:{client_nonce}:{server_nonce}:{request_bytes}".encode(), hashlib.sha256).hexdigest() - if not isinstance(request_bytes, str) or not isinstance(authenticated_frame.get("proof"), str) or not hmac.compare_digest(authenticated_frame["proof"], expected_proof): + presented_proof = authenticated_frame.get("proof") if isinstance(authenticated_frame, dict) else None + if not isinstance(request_bytes, str) or not isinstance(presented_proof, str): self._send(connection, {"ok": False, "error": "authentication-failed"}) - continue + return + # Update the prefix and the request separately. One joined + # preimage would retain another copy of a maximum-sized command. + expected_proof = hmac.new(self.token.encode(), digestmod=hashlib.sha256) + expected_proof.update( + f"rea-cutter-client-v2:{self.session_id}:{client_nonce}:{server_nonce}:".encode() + ) + expected_proof.update(request_bytes.encode()) + if not hmac.compare_digest(presented_proof, expected_proof.hexdigest()): + self._send(connection, {"ok": False, "error": "authentication-failed"}) + return request = json.loads(request_bytes) if not isinstance(request, dict) or request.get("session_id") != self.session_id: self._send(connection, {"ok": False, "error": "authentication-failed"}) - continue + return authenticated = True - connection.settimeout(SOCKET_TIMEOUT_SECONDS) - if self.poisoned and request.get("kind") not in ("status", "ack"): - self._send_authenticated(connection, { - "ok": False, - "error": "session-poisoned-after-ui-timeout", - "execution_state": "not_started", - }, client_nonce, server_nonce) - continue - if ( - request.get("kind") == "command" - and not self.dispatcher.can_accept_command() - ): - self._send_authenticated(connection, { - "ok": False, - "error": "completion-receipt-capacity-exceeded", - "execution_state": "not_started", - }, client_nonce, server_nonce) - continue - request["session_id"] = self.session_id - event = threading.Event() - cancelled = threading.Event() - reply: dict[str, Any] = {} - request["_event"] = event - request["_cancelled"] = cancelled - self.dispatcher.dispatch.emit(request, reply) - if not event.wait(SOCKET_TIMEOUT_SECONDS): - cancelled.set() - self.poisoned = True - self._send_authenticated(connection, { - "ok": False, - "error": "ui-thread-request-timeout", - "execution_state": "unknown", - "message": "The UI operation may still complete after this timeout; restart Cutter before issuing further commands.", - }, client_nonce, server_nonce) - else: - if request.get("kind") == "status" and reply.get("ok") is True: - reply["completed_connection"] = f"{connection_ordinal - 1:016x}" - self._send_authenticated(connection, reply, client_nonce, server_nonce) + with self._command_lock: + connection.settimeout(SOCKET_TIMEOUT_SECONDS) + if self.poisoned and request.get("kind") not in ("status", "ack"): + self._send_authenticated(connection, { + "ok": False, + "error": "session-poisoned-after-ui-timeout", + "execution_state": "not_started", + }, client_nonce, server_nonce) + return + if ( + request.get("kind") == "command" + and not self.dispatcher.can_accept_command() + ): + self._send_authenticated(connection, { + "ok": False, + "error": "completion-receipt-capacity-exceeded", + "execution_state": "not_started", + }, client_nonce, server_nonce) + return + request["session_id"] = self.session_id + event = threading.Event() + cancelled = threading.Event() + reply: dict[str, Any] = {} + request["_event"] = event + request["_cancelled"] = cancelled + self.dispatcher.dispatch.emit(request, reply) + if not event.wait(SOCKET_TIMEOUT_SECONDS): + cancelled.set() + self.poisoned = True + self._send_authenticated(connection, { + "ok": False, + "error": "ui-thread-request-timeout", + "execution_state": "unknown", + "message": "The UI operation may still complete after this timeout; restart Cutter before issuing further commands.", + }, client_nonce, server_nonce) + else: + if request.get("kind") == "status" and reply.get("ok") is True: + reply["completed_connection"] = self._completion_watermark() + self._send_authenticated(connection, reply, client_nonce, server_nonce) except socket.timeout: if authenticated: self._send_authenticated(connection, { @@ -614,19 +820,41 @@ class _BridgeServer(threading.Thread): }, client_nonce, server_nonce) except Exception as error: self._send(connection, {"ok": False, "error": str(error)}) - self.server.close() - try: - self.descriptor.unlink(missing_ok=True) - except OSError: - pass + finally: + self._note_connection_finished(connection_ordinal) + with self._connections_lock: + self._connections.discard(connection) + + def _note_connection_finished(self, ordinal: int) -> None: + """Record one accepted connection as terminated. + + The watermark is the longest prefix 1..W that has terminated. A later + status cannot report an earlier connection that is still reading or + still waiting to run. + """ + with self._connections_lock: + if ordinal <= self._finished_through or ordinal in self._finished_ordinals: + return + self._finished_ordinals.add(ordinal) + while self._finished_through + 1 in self._finished_ordinals: + self._finished_through += 1 + self._finished_ordinals.discard(self._finished_through) + + def _completion_watermark(self) -> str: + with self._connections_lock: + return f"{self._finished_through:016x}" @staticmethod - def _receive_frame(connection: socket.socket) -> dict[str, Any]: + def _receive_frame( + connection: socket.socket, + timeout_seconds: float = AUTH_FRAME_TIMEOUT_SECONDS, + ) -> dict[str, Any]: data = bytearray() - deadline = time.monotonic() + AUTH_FRAME_TIMEOUT_SECONDS - while b"\n" not in data: + deadline = time.monotonic() + timeout_seconds + newline = -1 + while newline < 0: if len(data) > MAX_FRAME_BYTES: - raise ValueError("frame-limit-exceeded") + raise _frame_limit_exceeded() remaining = deadline - time.monotonic() if remaining <= 0: raise socket.timeout("authentication frame timed out") @@ -636,57 +864,173 @@ class _BridgeServer(threading.Thread): part = connection.recv(min(65536, MAX_FRAME_BYTES + 1 - len(data))) if not part: raise ValueError("incomplete authentication frame") + scanned = len(data) data.extend(part) - line, remainder = bytes(data).split(b"\n", 1) - if len(line) > MAX_FRAME_BYTES or remainder: + # The delimiter is one byte, so it cannot straddle a chunk. + # Search only the bytes just received. + newline = data.find(b"\n", scanned) + if newline > MAX_FRAME_BYTES: + raise _frame_limit_exceeded() + if newline != len(data) - 1: raise ValueError("invalid framed request") - value = json.loads(line) + value = json.loads(bytes(data[:newline])) return value if isinstance(value, dict) else {} def _send_authenticated(self, connection: socket.socket, value: dict[str, Any], client_nonce: str, server_nonce: str) -> None: - encoded = self._encode_authenticated_response(value, client_nonce, server_nonce) - if encoded is None: - value = self._truncated_response(value) - encoded = self._encode_authenticated_response(value, client_nonce, server_nonce) - if encoded is None: - # This fixed-size fallback is checked through the same signed framing path. - value = { - "ok": False, - "error": "response-limit-exceeded", - "execution_state": value.get("execution_state", "unknown"), - "output": None, - "output_truncated": True, - "message": "The result exceeded the bridge frame limit; command completion state is preserved.", - } - encoded = self._encode_authenticated_response(value, client_nonce, server_nonce) - if encoded is None: + if self._deliver_authenticated(connection, value, client_nonce, server_nonce): return - try: - connection.sendall(encoded) - except OSError: - pass + # Keep the recovery payload. Include the observed path when it + # fits. An explicit null path means unknown, so the client does + # not pair this generation with the descriptor's older path. + bounded = self._bounded_status_response(value) + if bounded is not None: + observed_path = value.get("current_file") + if isinstance(observed_path, str) and observed_path: + with_path = dict(bounded) + with_path["current_file"] = observed_path + with_path.pop("message", None) + if self._deliver_authenticated( + connection, with_path, client_nonce, server_nonce + ): + return + if self._deliver_authenticated( + connection, bounded, client_nonce, server_nonce + ): + return + command = self._bounded_command_response(value) + if command is not None: + observed_path = value.get("current_file") + if isinstance(observed_path, str) and observed_path: + with_path = dict(command) + with_path["current_file"] = observed_path + with_path.pop("message", None) + if self._deliver_authenticated( + connection, with_path, client_nonce, server_nonce + ): + return + if self._deliver_authenticated( + connection, command, client_nonce, server_nonce + ): + return + value = self._truncated_response(value) + if self._deliver_authenticated(connection, value, client_nonce, server_nonce): + return + # This fixed-size fallback is checked through the same signed framing path. + self._deliver_authenticated(connection, { + "ok": False, + "error": "response-limit-exceeded", + "execution_state": value.get("execution_state", "unknown"), + "output": None, + "output_truncated": True, + "message": "The result exceeded the bridge frame limit; command completion state is preserved.", + }, client_nonce, server_nonce) - def _encode_authenticated_response(self, value: dict[str, Any], client_nonce: str, server_nonce: str) -> bytes | None: + def _deliver_authenticated( + self, connection: socket.socket, value: dict[str, Any], client_nonce: str, server_nonce: str + ) -> bool: + proof = self._authenticated_proof(value, client_nonce, server_nonce) + if proof is None: + return False + try: + self._write_authenticated_frame(connection, value, proof) + except OSError: + # A disconnected caller must not fall through to another payload. + pass + return True + + def _authenticated_proof( + self, value: dict[str, Any], client_nonce: str, server_nonce: str + ) -> str | None: if not _within_response_limit(value): return None - prefix = b'{"response":"' - suffix = b'","proof":"' + (b"0" * 64) + b'"}\n' - frame_size = len(prefix) + len(suffix) - response_parts: list[bytes] = [] - escaped_parts: list[bytes] = [] - for fragment in json.JSONEncoder(ensure_ascii=False, separators=(",", ":")).iterencode(value): - raw = fragment.encode("utf-8") - escaped = json.dumps(fragment, ensure_ascii=False)[1:-1].encode("utf-8") - frame_size += len(escaped) + # The payload budget excludes the terminating LF. Fragments are + # measured and signed here, then discarded. The write pass encodes + # them again so a maximum-sized reply is not held in several copies. + frame_size = _AUTHENTICATED_FRAME_OVERHEAD + proof = hmac.new(self.token.encode(), digestmod=hashlib.sha256) + proof.update( + f"rea-cutter-response-v2:{self.session_id}:{client_nonce}:{server_nonce}:".encode() + ) + for fragment in _response_fragments(value): + frame_size += len(_escape_json_fragment(fragment)) if frame_size > MAX_FRAME_BYTES: return None - response_parts.append(raw) - escaped_parts.append(escaped) - proof = hmac.new(self.token.encode(), digestmod=hashlib.sha256) - proof.update(f"rea-cutter-response-v2:{self.session_id}:{client_nonce}:{server_nonce}:".encode()) - for part in response_parts: - proof.update(part) - return prefix + b"".join(escaped_parts) + b'","proof":"' + proof.hexdigest().encode() + b'"}\n' + proof.update(fragment.encode("utf-8")) + return proof.hexdigest() + + def _encode_authenticated_response( + self, value: dict[str, Any], client_nonce: str, server_nonce: str + ) -> bytes | None: + """Return one exact frame. The socket path writes the same pieces.""" + proof = self._authenticated_proof(value, client_nonce, server_nonce) + if proof is None: + return None + return b"".join(self._iter_authenticated_frame(value, proof)) + + def _iter_authenticated_frame(self, value: dict[str, Any], proof: str): + yield _AUTHENTICATED_PREFIX + for fragment in _response_fragments(value): + yield _escape_json_fragment(fragment) + yield _AUTHENTICATED_PROOF_MARK + proof.encode("ascii") + b'"}\n' + + def _write_authenticated_frame( + self, connection: socket.socket, value: dict[str, Any], proof: str + ) -> None: + pending = bytearray() + for chunk in self._iter_authenticated_frame(value, proof): + pending.extend(chunk) + if len(pending) >= _AUTHENTICATED_WRITE_BUFFER_BYTES: + connection.sendall(pending) + pending.clear() + if pending: + connection.sendall(pending) + + @staticmethod + def _bounded_status_response(value: dict[str, Any]) -> dict[str, Any] | None: + if "completed_commands" not in value and "completed_connection" not in value: + return None + bounded: dict[str, Any] = { + "ok": value.get("ok") is True, + "session_id": value.get("session_id"), + "document_generation": value.get("document_generation"), + "identity_status": value.get("identity_status", "partial"), + "completed_commands": value.get("completed_commands"), + "current_file": None, + "message": ( + "current_file is unknown because the status frame exceeded " + "REA_BRIDGE_MAX_REQUEST_BYTES. Do not reuse an older descriptor " + "path with this document_generation." + ), + } + if "completed_connection" in value: + bounded["completed_connection"] = value["completed_connection"] + return bounded + + @staticmethod + def _bounded_command_response(value: dict[str, Any]) -> dict[str, Any] | None: + if "execution_state" not in value and "document_generation" not in value: + return None + bounded: dict[str, Any] = { + "ok": value.get("ok") is True, + "output": None, + "execution_state": value.get("execution_state", "unknown"), + "output_truncated": True, + "current_file": None, + "message": ( + "current_file is unknown because the result frame exceeded " + "REA_BRIDGE_MAX_REQUEST_BYTES. The command completion state and " + "document_generation are preserved. Do not reuse an older " + "descriptor path with this document_generation." + ), + } + generation = value.get("document_generation") + if isinstance(generation, int) and not isinstance(generation, bool): + bounded["document_generation"] = generation + if value.get("identity_status") == "partial": + bounded["identity_status"] = "partial" + if value.get("ok") is not True and isinstance(value.get("error"), str): + bounded["error"] = value["error"] + return bounded @staticmethod def _truncated_response(value: dict[str, Any]) -> dict[str, Any]: @@ -744,6 +1088,17 @@ class _BridgeServer(threading.Thread): self.server.close() except OSError: pass + with self._connections_lock: + connections = list(self._connections) + for connection in connections: + try: + connection.shutdown(socket.SHUT_RDWR) + except OSError: + pass + try: + connection.close() + except OSError: + pass class ReaCutterPlugin(cutter.CutterPlugin): @@ -777,6 +1132,128 @@ def create_cutter_plugin() -> ReaCutterPlugin: return ReaCutterPlugin() +def _json_object_key(key: Any) -> str: + """Match CPython's JSON object-key text before quoting.""" + if isinstance(key, str): + return key + if key is True: + return "true" + if key is False: + return "false" + if key is None: + return "null" + if isinstance(key, int): + return str(key) + if isinstance(key, float): + return json.dumps(key, separators=(",", ":")) + raise TypeError("keys must be str, int, float, bool or None") + + +def _response_fragments(value: Any): + """Yield compact JSON pieces whose concatenation is the encoded value. + + String contents are sliced before encoding so a large output never becomes + one frame-sized fragment. Each piece is at most ``_JSON_CHUNK_CHARS`` + characters, or the six-byte ``\\u00XX`` expansion of one such slice. + """ + if isinstance(value, str): + yield '"' + for start in range(0, len(value), _JSON_CHUNK_CHARS): + yield json.dumps( + value[start:start + _JSON_CHUNK_CHARS], ensure_ascii=False + )[1:-1] + yield '"' + return + if isinstance(value, dict): + yield "{" + first = True + for key, item in value.items(): + if not first: + yield "," + first = False + yield from _response_fragments(_json_object_key(key)) + yield ":" + yield from _response_fragments(item) + yield "}" + return + if isinstance(value, (list, tuple)): + yield from _list_fragments(value) + return + token = _scalar_token(value) + if token is None: + raise TypeError("response value is not JSON") + yield token + + +def _scalar_token(value: Any) -> str | None: + """Compact JSON for one scalar. None means the value needs structure.""" + if type(value) is int: + return str(value) + if value is None: + return "null" + if value is True: + return "true" + if value is False: + return "false" + if isinstance(value, float): + return json.dumps(value, separators=(",", ":")) + return None + + +def _list_fragments(value: list[Any] | tuple[Any, ...]): + """Encode a list, batching scalars into bounded pieces. + + One ``json.dumps`` per number makes a dense array miss the client + deadline. A batch is at most ``_JSON_CHUNK_CHARS`` of compact JSON. + """ + yield "[" + first = True + batch: list[str] = [] + batch_len = 0 + + def flush_batch() -> str | None: + nonlocal first, batch, batch_len + if not batch: + return None + piece = ",".join(batch) + batch = [] + batch_len = 0 + if first: + first = False + return piece + return "," + piece + + for item in value: + token = _scalar_token(item) + if token is None: + emitted = flush_batch() + if emitted is not None: + yield emitted + if first: + first = False + else: + yield "," + yield from _response_fragments(item) + continue + extra = len(token) if not batch else len(token) + 1 + if batch and batch_len + extra > _JSON_CHUNK_CHARS: + emitted = flush_batch() + if emitted is not None: + yield emitted + extra = len(token) + batch.append(token) + batch_len += extra + emitted = flush_batch() + if emitted is not None: + yield emitted + yield "]" + + +def _escape_json_fragment(fragment: str) -> bytes: + """Escape one raw JSON piece for the outer response string.""" + return json.dumps(fragment, ensure_ascii=False)[1:-1].encode("utf-8") + + def _within_response_limit(value: Any) -> bool: pending = [value] seen: set[int] = set() @@ -800,13 +1277,22 @@ def _within_response_limit(value: Any) -> bool: # Unencodable output also needs the signed omission fallback. return False size += current.count('"') + current.count("\\") - size += sum(5 for character in current if ord(character) < 0x20) + # A short escape is two JSON bytes and was already counted as one. + # Other controls expand to a six-byte \\u00XX sequence. + size += sum( + 1 if ord(character) in (0x08, 0x09, 0x0A, 0x0C, 0x0D) else 5 + for character in current + if ord(character) < 0x20 + ) elif isinstance(current, dict): identity = id(current) if identity in seen: return False seen.add(identity) - size += 2 + 4 * len(current) + count = len(current) + # Braces, one colon per entry, and commas between entries. + # Keys and values are counted when they are visited. + size += 2 + count + max(0, count - 1) if size > MAX_FRAME_BYTES: return False pending.extend(current.keys()) @@ -816,12 +1302,29 @@ def _within_response_limit(value: Any) -> bool: if identity in seen: return False seen.add(identity) - size += 2 + 2 * len(current) + count = len(current) + # Brackets and commas between items. Each item is counted below. + size += 2 + max(0, count - 1) if size > MAX_FRAME_BYTES: return False pending.extend(current) + elif current is None: + size += 4 + elif current is True: + size += 4 + elif current is False: + size += 5 + elif isinstance(current, int): + size += len(str(current)) + elif isinstance(current, float): + size += len(json.dumps(current, separators=(",", ":"))) else: - size += 16 + try: + size += len( + json.dumps(current, ensure_ascii=False, separators=(",", ":")) + ) + except (TypeError, ValueError): + return False if size > MAX_FRAME_BYTES: return False return True diff --git a/bridge/hopper_bridge.py b/bridge/hopper_bridge.py index 3636c6b440..fa1bb3e546 100644 --- a/bridge/hopper_bridge.py +++ b/bridge/hopper_bridge.py @@ -8,15 +8,100 @@ API access on this thread: moving dispatch to a worker can deadlock Hopper. import json import hmac import os +import re import socket from typing import Any, Optional, Protocol, Sequence BAD_ADDRESSES = (-1, 0xFFFFFFFFFFFFFFFF, None) # The native label setter truncates NSString names at this UTF-16 extent. HOPPER_SYMBOL_NAME_UTF16_UNITS = 1024 -# Maximum request JSON payload size in bytes, excluding its trailing LF. -# Matches the Cutter bridge's MAX_FRAME_BYTES payload ceiling. -MAX_REQUEST_BYTES = 16 * 1024 * 1024 +# Request JSON payload budget in bytes, excluding the trailing LF. +# REA_BRIDGE_MAX_REQUEST_BYTES selects it. Unset stays 16777216. +# 7464 is the authenticated Cutter frame of the largest bounded symbol name. +# Cutter's 64-receipt recovery status is smaller with a null path. A path +# that does not fit is null, not an older descriptor path. +# 536870887 is Node's 536870888-character string ceiling minus the framing LF. +# The ceiling is the encoded payload; JSON expansion is measured before it. +_BRIDGE_MAX_REQUEST_BYTES_SETTING = "REA_BRIDGE_MAX_REQUEST_BYTES" +DEFAULT_BRIDGE_MAX_REQUEST_BYTES = 16 * 1024 * 1024 +MINIMUM_BRIDGE_MAX_REQUEST_BYTES = 7464 +MAXIMUM_BRIDGE_MAX_REQUEST_BYTES = 536870887 + + +def _bridge_max_request_bytes_constraint(): + return ( + "REA_BRIDGE_MAX_REQUEST_BYTES must be a positive decimal integer " + "from %d to %d bytes; the default is %d bytes (16 MiB). " + "A value below %d bytes cannot carry the largest admitted " + "symbol-name command, and a larger value is refused so the encoded " + "payload plus its framing LF stays one string under Node's " + "single-string ceiling." + % ( + MINIMUM_BRIDGE_MAX_REQUEST_BYTES, + MAXIMUM_BRIDGE_MAX_REQUEST_BYTES, + DEFAULT_BRIDGE_MAX_REQUEST_BYTES, + MINIMUM_BRIDGE_MAX_REQUEST_BYTES, + ) + ) + + +def _parse_bridge_max_request_bytes(raw): + """Refuse a non-decimal or out-of-range ceiling. Never fall back.""" + constraint = _bridge_max_request_bytes_constraint() + if isinstance(raw, bool): + raise RuntimeError(constraint) + if isinstance(raw, int): + value = raw + elif isinstance(raw, str): + # ASCII digits only. str.isdigit() accepts numerals such as U+0668 + # that the TypeScript setting rejects. + if re.fullmatch(r"[1-9][0-9]*", raw) is None: + raise RuntimeError(constraint) + value = int(raw) + else: + raise RuntimeError(constraint) + if ( + value < MINIMUM_BRIDGE_MAX_REQUEST_BYTES + or value > MAXIMUM_BRIDGE_MAX_REQUEST_BYTES + ): + raise RuntimeError(constraint) + return value + + +def _resolve_bridge_max_request_bytes(): + """Prefer the bootstrap injection, then the process environment, then 16 MiB.""" + injected = globals().get(_BRIDGE_MAX_REQUEST_BYTES_SETTING) + if injected is not None: + return _parse_bridge_max_request_bytes(injected), "setting" + raw = os.environ.get(_BRIDGE_MAX_REQUEST_BYTES_SETTING) + if raw is None: + return DEFAULT_BRIDGE_MAX_REQUEST_BYTES, "default" + return _parse_bridge_max_request_bytes(raw), "setting" + + +MAX_REQUEST_BYTES, BRIDGE_MAX_REQUEST_BYTES_SOURCE = ( + _resolve_bridge_max_request_bytes() +) + + +def _bridge_max_request_bytes_recovery(): + return ( + "The effective REA_BRIDGE_MAX_REQUEST_BYTES value is %d bytes " + "(source %s; default %d, floor %d, maximum %d). " + "Set REA_BRIDGE_MAX_REQUEST_BYTES in the environment that starts REA " + "to a decimal integer from %d to %d, then restart REA. " + "Restart Hopper or Cutter from that same environment so the bridge " + "uses this ceiling." + % ( + MAX_REQUEST_BYTES, + BRIDGE_MAX_REQUEST_BYTES_SOURCE, + DEFAULT_BRIDGE_MAX_REQUEST_BYTES, + MINIMUM_BRIDGE_MAX_REQUEST_BYTES, + MAXIMUM_BRIDGE_MAX_REQUEST_BYTES, + MINIMUM_BRIDGE_MAX_REQUEST_BYTES, + MAXIMUM_BRIDGE_MAX_REQUEST_BYTES, + ) + ) _rea_session_document = None _search_inventory_cache = {} _pseudocode_cache = {} @@ -743,6 +828,7 @@ def _dispatch(method, params): "version": "1.0.0", "run_id": REA_RUN_ID, "process_id": os.getpid(), + "max_request_bytes": MAX_REQUEST_BYTES, } if method in ("shutdown", "shutdown_document"): document = _session_document() @@ -1071,8 +1157,8 @@ def _serve_connection(connection): break if not line.endswith(b"\n"): message = ( - "Bridge request exceeds the %d-byte payload limit" - % MAX_REQUEST_BYTES + "Bridge request exceeds the %d-byte payload limit. %s" + % (MAX_REQUEST_BYTES, _bridge_max_request_bytes_recovery()) if len(line) > MAX_REQUEST_BYTES else "Incomplete bridge request frame" ) diff --git a/docs/reverse-engineering-tools.md b/docs/reverse-engineering-tools.md index 8b2ad94af1..c54b389a4f 100644 --- a/docs/reverse-engineering-tools.md +++ b/docs/reverse-engineering-tools.md @@ -70,17 +70,48 @@ Captured stdout and stderr are retained up to a 16 MiB combined limit; `output_t Cutter commands use the local REA Python bridge and authenticated loopback messages. The client and plugin must support the current v2 handshake; older peers are rejected before dispatch because they cannot preserve cross-caller recovery proof. -The client checks the final encoded request against the bridge's 16 MiB frame -limit before dispatch; the 16 MiB budget applies to request payload bytes and -excludes the terminating LF. The bridge accepts that full payload size and -rejects larger or incomplete frames while bounding the read; an oversized -command does not retain a cleanup owner. Hopper uses the same request payload -budget. -Each bridge request waits 25000 ms. Discovery admits at most 256 descriptor +The client checks the final encoded request against +`REA_BRIDGE_MAX_REQUEST_BYTES` before dispatch. The budget applies to request +payload bytes and excludes the terminating LF. Unset, the ceiling is 16777216 +bytes (16 MiB). A decimal integer from 7464 to 536870887 is accepted. The floor +fits the largest bounded symbol-name command inside an authenticated Cutter +frame. A Cutter status keeps its session id, connection proof, and up to 64 +completion receipts when the frame is tight. The observed `current_file` is +included when it fits. When it does not, the status reports that path as +unknown, and the client does not reuse the descriptor path with the new +generation. The maximum is one byte under Node's 536870888-character string ceiling +so the framing LF still fits. The ceiling counts the encoded payload. A JSON +escape is measured before serialization and is not divided into the maximum +again. A structured Cutter reply is admitted when that encoded payload fits, +including a dense array of numbers. Cutter hashes the HMAC prefix and the payload as separate updates, so +an exact-limit frame does not build one preimage string. The plugin measures and signs the response +from bounded JSON pieces, then writes the authenticated frame in 64 KiB chunks. The bridge searches only the newly read bytes for the framing LF. The client unescapes each socket chunk into one response string, checks the proof, and then parses that string. A fitting reply keeps +the original output and does not retain the paged wire frame plus another concatenated copy. Zero, negative, non-decimal, and out-of-range +values are `configuration_invalid` and do not fall back to the default. The +bridge accepts that full payload size and rejects a larger payload while +bounding the read. An incomplete frame is still `Incomplete bridge request +frame`. An oversized command does not retain a cleanup owner. Hopper uses the +same request payload budget. A Hopper request is serialized when it reaches +the bridge, so one waiting behind Hopper's serial bridge does not retain +another copy of the encoded frame. REA injects a caller-selected value into the +Hopper bootstrap and the bridge also reads the process environment; an injected +value wins. Cutter reads the environment that starts Cutter and publishes +`max_request_bytes` on its session descriptor. A Hopper or Cutter process +started out of band without the setting uses 16777216 bytes. The client +compares that advertisement with its own ceiling before a discovery status probe and before sending a command. A different ceiling is a mismatch only after the hello proves that descriptor is a live authenticated bridge; a stale descriptor is ignored. That hello does not send a status request. Discovery fails if any such live ceiling differs, even when another live session matches. A hello that +never sends its authenticated frame does not hold the bridge accept loop. +At most 8 connection threads run. A further accept is closed, and a failure to start a thread closes that socket without stopping the listener. A status `completed_connection` is the longest prefix of accepted connections that have terminated. It does not treat an earlier connection as finished because a later status reached the command lock first. Scalar values in one Cutter reply are encoded in bounded batches. +A command reply that does not fit keeps its completion state and document +generation, and reports an oversized path as unknown. Cutter response frames use this same ceiling and +also exclude the terminating LF. A command response the client discards because it exceeds that ceiling stays an unknown execution and sets `output_truncated`, so Evidence records the omitted output. Hopper response reads are unchanged. +Each bridge request waits 25000 ms. The Cutter hello frame still must arrive +within 1 second. The authenticated command frame is read with one second per +mebibyte of `REA_BRIDGE_MAX_REQUEST_BYTES`, and that read never waits longer +than the same 25000 ms. Discovery admits at most 256 descriptor candidates and reads and probes at most eight concurrently. A larger directory reports `bridge_directory_capacity_exceeded`. This release has no runtime -override for the 25000 ms request deadline, the 16777216 byte frame, or the -256-candidate discovery budget. +override for the 25000 ms request deadline or the 256-candidate discovery +budget. Those limits do not follow `REA_BRIDGE_MAX_REQUEST_BYTES`. Terminal receipts are recovered before another command, including from a new CLI invocation. A signed status response also confirms completion of older bridge diff --git a/scripts/lib/test-projects.mjs b/scripts/lib/test-projects.mjs index 2960d6f87e..13026a798f 100644 --- a/scripts/lib/test-projects.mjs +++ b/scripts/lib/test-projects.mjs @@ -17,7 +17,7 @@ export const SOURCE_TEST_PROJECTS = [ group: "source", include: [ "src/*.test.ts", - "src/{android,apkSignature,apktool,jeb,adb,artifacts,browser,config,dotnet,electron,erlang,evm,firmware,flutter,pcap,ghidra,go,gpu,hopper,ida,inspector,javascript,main,native,process,python,reference,rust,server,sqlite,wasm,windows}/**/*.test.ts", + "src/{android,apkSignature,apktool,jeb,adb,artifacts,browser,config,cutter,dotnet,electron,erlang,evm,firmware,flutter,pcap,ghidra,go,gpu,hopper,ida,inspector,javascript,main,native,process,python,reference,rust,server,sqlite,wasm,windows}/**/*.test.ts", ], }, { diff --git a/src/application/Doctor.test.ts b/src/application/Doctor.test.ts index 8f01ccc7ea..0e9df8d6f0 100644 --- a/src/application/Doctor.test.ts +++ b/src/application/Doctor.test.ts @@ -165,6 +165,7 @@ describe("doctor native analysis configuration", () => { it.each([ ["REA_ANALYSIS_PROVIDER", "Hopper"], ["REA_REGEX_MATCHING_DEADLINE_MS", "0"], + ["REA_BRIDGE_MAX_REQUEST_BYTES", "7463"], ["REA_CAPTURE_DECODER_OLD_GENERATION_MIB", "7"], ["REA_REGEX_WORKER_HEAP_REQUEST_MIB", "6"], ["REA_MCP_INITIALIZE_BUDGET_MS", "0"], diff --git a/src/application/SetupRegistrationEnvironment.ts b/src/application/SetupRegistrationEnvironment.ts index 7ffef6da49..7b5ed4e7df 100644 --- a/src/application/SetupRegistrationEnvironment.ts +++ b/src/application/SetupRegistrationEnvironment.ts @@ -3,6 +3,7 @@ import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; const SELECTED_REGISTRATION_SETTINGS = [ "REA_ANALYSIS_PROVIDER", + "REA_BRIDGE_MAX_REQUEST_BYTES", "REA_CAPTURE_DECODER_OLD_GENERATION_MIB", "REA_GHIDRA_STARTUP_TIMEOUT_MS", "REA_HOPPER_STARTUP_TIMEOUT_MS", diff --git a/src/config/bridgeMaxRequestBytes.test.ts b/src/config/bridgeMaxRequestBytes.test.ts new file mode 100644 index 0000000000..1d931cc395 --- /dev/null +++ b/src/config/bridgeMaxRequestBytes.test.ts @@ -0,0 +1,147 @@ +import { expect, it } from "vitest"; + +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { parseEnvironment } from "./environment.js"; +import { parseConfig } from "./parseConfig.js"; +import { + BRIDGE_MAX_REQUEST_BYTES_SETTING, + bridgeMaxRequestBytesConstraint, + bridgeMaxRequestBytesRecovery, + DEFAULT_BRIDGE_MAX_REQUEST, + DEFAULT_BRIDGE_MAX_REQUEST_BYTES, + MAXIMUM_BRIDGE_MAX_REQUEST_BYTES, + MINIMUM_BRIDGE_MAX_REQUEST_BYTES, + parseBridgeMaxRequestBytes, +} from "./bridgeMaxRequestBytes.js"; + +const configured = (maximumBytes: number) => + ({ maximumBytes, source: "setting" }) as const; + +const symbolName = "\u0001".repeat(1024); +const session = "00000000-0000-4000-8000-000000000001"; +const token = "a".repeat(64); + +it("keeps the 16 MiB ceiling when the setting is unset", () => { + expect(parseBridgeMaxRequestBytes(undefined)).toEqual({ + ok: true, + value: DEFAULT_BRIDGE_MAX_REQUEST, + }); + const parsed = parseConfig({}); + if (!parsed.ok) throw new Error("unset bridge ceiling was rejected"); + expect(parsed.value.bridgeMaxRequestBytes).toEqual( + DEFAULT_BRIDGE_MAX_REQUEST, + ); + const environment = parseEnvironment({}); + if (!environment.ok) throw new Error("doctor rejected an unset ceiling"); + expect(environment.value.REA_BRIDGE_MAX_REQUEST_BYTES).toBeUndefined(); + const recovery = bridgeMaxRequestBytesRecovery(DEFAULT_BRIDGE_MAX_REQUEST); + expect(recovery).toContain( + `The effective ${BRIDGE_MAX_REQUEST_BYTES_SETTING} value is ${String(DEFAULT_BRIDGE_MAX_REQUEST_BYTES)} bytes (source default; default ${String(DEFAULT_BRIDGE_MAX_REQUEST_BYTES)}, floor ${String(MINIMUM_BRIDGE_MAX_REQUEST_BYTES)}, maximum ${String(MAXIMUM_BRIDGE_MAX_REQUEST_BYTES)}).`, + ); + expect(recovery).toContain( + `Set ${BRIDGE_MAX_REQUEST_BYTES_SETTING} in the environment that starts REA`, + ); + expect(recovery).not.toContain("no runtime override"); +}); + +it("accepts a caller-selected ceiling inside the floor and maximum", () => { + const parsed = parseConfig({ + [BRIDGE_MAX_REQUEST_BYTES_SETTING]: "33554432", + }); + if (!parsed.ok) throw new Error("valid bridge ceiling was rejected"); + expect(parsed.value.bridgeMaxRequestBytes).toEqual(configured(33_554_432)); + const environment = parseEnvironment({ + [BRIDGE_MAX_REQUEST_BYTES_SETTING]: "8192", + }); + if (!environment.ok) throw new Error("doctor rejected a valid ceiling"); + expect(environment.value.REA_BRIDGE_MAX_REQUEST_BYTES).toBe(8192); + expect( + bridgeMaxRequestBytesRecovery(parsed.value.bridgeMaxRequestBytes), + ).toContain("source setting"); +}); + +it("sets the maximum to the serialized string bound minus the framing LF", () => { + const stringCeiling = 536_870_888; + const hundredMib = 100 * 1024 * 1024; + expect(MAXIMUM_BRIDGE_MAX_REQUEST_BYTES).toBe(stringCeiling - 1); + expect(MAXIMUM_BRIDGE_MAX_REQUEST_BYTES).toBe(536_870_887); + expect(hundredMib).toBeLessThan(MAXIMUM_BRIDGE_MAX_REQUEST_BYTES); + const parsed = parseConfig({ + [BRIDGE_MAX_REQUEST_BYTES_SETTING]: String(hundredMib), + }); + if (!parsed.ok) throw new Error("a 100 MiB encoded ceiling was rejected"); + expect(parsed.value.bridgeMaxRequestBytes).toEqual(configured(hundredMib)); + expect(MAXIMUM_BRIDGE_MAX_REQUEST_BYTES).toBeGreaterThan( + DEFAULT_BRIDGE_MAX_REQUEST_BYTES, + ); +}); + +it("locks the floor to the authenticated frame of the largest bounded symbol name", () => { + const hopper = Buffer.byteLength( + JSON.stringify({ + id: Number.MAX_SAFE_INTEGER, + token, + method: "set_address_name", + params: { address: "0xffffffffffffffff", name: symbolName }, + }), + ); + const command = JSON.stringify({ + kind: "command", + request_id: session, + expected_generation: Number.MAX_SAFE_INTEGER, + command: symbolName, + json: false, + session_id: session, + }); + const framed = Buffer.byteLength( + JSON.stringify({ request: command, proof: "f".repeat(64) }), + ); + let estimated = + Buffer.byteLength(JSON.stringify({ request: "", proof: "0".repeat(64) })) + + Buffer.byteLength(command); + for (const character of command) + if (character === '"' || character === "\\") estimated += 1; + const hello = Buffer.byteLength( + JSON.stringify({ kind: "rea-auth-v2", nonce: "ab".repeat(32) }), + ); + expect(hopper).toBe(6322); + expect(framed).toBe(7464); + expect(estimated).toBe(framed); + expect(hello).toBeLessThan(MINIMUM_BRIDGE_MAX_REQUEST_BYTES); + expect(Math.max(hopper, framed)).toBe(MINIMUM_BRIDGE_MAX_REQUEST_BYTES); + expect(MINIMUM_BRIDGE_MAX_REQUEST_BYTES).toBeLessThan( + DEFAULT_BRIDGE_MAX_REQUEST_BYTES, + ); +}); + +it.each([ + "", + "0", + "-1", + "7463", + "016777216", + "16_777_216", + "1e7", + " 16777216", + "16777216 ", + "1.5", + String(MAXIMUM_BRIDGE_MAX_REQUEST_BYTES + 1), + "9".repeat(40), +])("rejects invalid %j as configuration_invalid", (value) => { + const parsed = parseBridgeMaxRequestBytes(value); + expect(parsed.ok).toBe(false); + if (parsed.ok) return; + expect(parsed.error.settings).toEqual([ + { + setting: BRIDGE_MAX_REQUEST_BYTES_SETTING, + constraint: bridgeMaxRequestBytesConstraint, + }, + ]); + expect(projectAnalysisError(parsed.error).code).toBe("configuration_invalid"); + const startup = parseConfig({ [BRIDGE_MAX_REQUEST_BYTES_SETTING]: value }); + const doctor = parseEnvironment({ + [BRIDGE_MAX_REQUEST_BYTES_SETTING]: value, + }); + expect(startup.ok).toBe(false); + expect(doctor.ok).toBe(false); +}); diff --git a/src/config/bridgeMaxRequestBytes.ts b/src/config/bridgeMaxRequestBytes.ts new file mode 100644 index 0000000000..12857df6a1 --- /dev/null +++ b/src/config/bridgeMaxRequestBytes.ts @@ -0,0 +1,184 @@ +import { z } from "zod"; + +import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; +import { ConfigurationError } from "../domain/configurationErrors.js"; +import { err, ok, type Result } from "../domain/result.js"; + +export const BRIDGE_MAX_REQUEST_BYTES_SETTING = "REA_BRIDGE_MAX_REQUEST_BYTES"; + +/** Request payload ceiling used when the setting is unset. Today's fixed constant. */ +export const DEFAULT_BRIDGE_MAX_REQUEST_BYTES = 16 * 1024 * 1024; + +/** + * Smallest ceiling that still admits the largest bounded bridge request. + * Hopper symbol names stop at 1024 UTF-16 units. A name of U+0001 repeated + * that many times JSON-escapes to six bytes per character. The Hopper + * `set_address_name` request for that name, a max-safe id, and a 64-hex token + * is 6322 bytes. The authenticated Cutter frame of a command carrying the same + * name is 7464 bytes. The required Cutter recovery status, 64 receipts with + * an unknown path, is smaller. A status path that fits is kept. A path that + * does not fit is null, so the client does not reuse the descriptor path with + * the new generation. Hello and a minimal command frame are smaller. + * Comments, search text, and Cutter command text are unbounded; this ceiling + * is what limits them. The floor stays below the 16 MiB default so a caller + * can lower the ceiling. + */ +export const MINIMUM_BRIDGE_MAX_REQUEST_BYTES = 7464; + +/** + * Largest configurable request, in encoded payload bytes. + * + * The ceiling is the final JSON payload, not the raw command. Callers measure + * escaped JSON against it before `JSON.stringify`, so a U+0001 expansion is + * counted once. Node's single-string ceiling is 536870888 UTF-16 code units. + * The payload budget excludes the terminating LF, and that LF is the one code + * unit appended after the check. The maximum leaves room for it. Cutter signs + * that payload from bounded JSON pieces and writes it in 64 KiB chunks. The + * bridge looks for the framing LF only in the bytes just read. The client + * unescapes each socket chunk into one response string, checks the proof, and + * then parses that string. A fitting response does not keep the paged wire + * frame, a second concatenated copy, and an outer JSON string. + */ +const NODE_MAX_STRING_LENGTH = 536_870_888; +const FRAMING_LF_CODE_UNITS = 1; +export const MAXIMUM_BRIDGE_MAX_REQUEST_BYTES = + NODE_MAX_STRING_LENGTH - FRAMING_LF_CODE_UNITS; + +/** Effective bridge request ceiling and where its value came from. */ +export interface BridgeMaxRequestBytes { + readonly maximumBytes: number; + readonly source: "default" | "setting"; +} + +export const DEFAULT_BRIDGE_MAX_REQUEST: BridgeMaxRequestBytes = { + maximumBytes: DEFAULT_BRIDGE_MAX_REQUEST_BYTES, + source: "default", +}; + +export const bridgeMaxRequestBytesConstraint = `${BRIDGE_MAX_REQUEST_BYTES_SETTING} must be a positive decimal integer from ${String(MINIMUM_BRIDGE_MAX_REQUEST_BYTES)} to ${String(MAXIMUM_BRIDGE_MAX_REQUEST_BYTES)} bytes; the default is ${String(DEFAULT_BRIDGE_MAX_REQUEST_BYTES)} bytes (16 MiB). A value below ${String(MINIMUM_BRIDGE_MAX_REQUEST_BYTES)} bytes cannot carry the largest admitted symbol-name command, and a larger value is refused so the encoded payload plus its framing LF stays one string under Node's single-string ceiling.`; + +const decimalInteger = /^[1-9]\d*$/u; + +const withinRequestRange = (value: string): boolean => { + // The pattern check reports a non-decimal value. Refine only bounds integers. + if (!decimalInteger.test(value)) return true; + const parsed = BigInt(value); + return ( + parsed >= BigInt(MINIMUM_BRIDGE_MAX_REQUEST_BYTES) && + parsed <= BigInt(MAXIMUM_BRIDGE_MAX_REQUEST_BYTES) + ); +}; + +/** Optional decimal byte count for the Hopper and Cutter request payload ceiling. */ +export const bridgeMaxRequestBytesSchema = z + .string() + .regex(decimalInteger, { error: bridgeMaxRequestBytesConstraint }) + .refine(withinRequestRange, { error: bridgeMaxRequestBytesConstraint }) + .transform(Number); + +/** Resolve the effective ceiling from one environment value. */ +export const parseBridgeMaxRequestBytes = ( + value: string | undefined, +): Result => { + if (value === undefined) return ok(DEFAULT_BRIDGE_MAX_REQUEST); + const parsed = bridgeMaxRequestBytesSchema.safeParse(value); + return parsed.success + ? ok({ maximumBytes: parsed.data, source: "setting" }) + : err( + new ConfigurationError(bridgeMaxRequestBytesConstraint, { + settings: [ + { + setting: BRIDGE_MAX_REQUEST_BYTES_SETTING, + constraint: bridgeMaxRequestBytesConstraint, + }, + ], + }), + ); +}; + +/** + * Resolve the ceiling from a composition root's selected environment. An + * invalid setting is refused, not ignored. Callers that already resolved it + * at startup pass that value through instead of reading the environment again. + */ +export const resolveBridgeMaxRequestBytes = ( + environment: Readonly>, +): BridgeMaxRequestBytes => { + const parsed = parseBridgeMaxRequestBytes( + environment[BRIDGE_MAX_REQUEST_BYTES_SETTING], + ); + if (!parsed.ok) throw parsed.error; + return parsed.value; +}; + +/** How to read and change the effective bridge request ceiling. */ +export const bridgeMaxRequestBytesRecovery = ( + limit: BridgeMaxRequestBytes, +): string => + [ + `The effective ${BRIDGE_MAX_REQUEST_BYTES_SETTING} value is ${String(limit.maximumBytes)} bytes (source ${limit.source}; default ${String(DEFAULT_BRIDGE_MAX_REQUEST_BYTES)}, floor ${String(MINIMUM_BRIDGE_MAX_REQUEST_BYTES)}, maximum ${String(MAXIMUM_BRIDGE_MAX_REQUEST_BYTES)}).`, + `Set ${BRIDGE_MAX_REQUEST_BYTES_SETTING} in the environment that starts REA to a decimal integer from ${String(MINIMUM_BRIDGE_MAX_REQUEST_BYTES)} to ${String(MAXIMUM_BRIDGE_MAX_REQUEST_BYTES)}, then restart REA.`, + "Restart Hopper or Cutter from that same environment so the bridge uses this ceiling.", + ].join(" "); + +const limitFacts = ( + limit: BridgeMaxRequestBytes, +): { readonly max_request_bytes: number } => ({ + max_request_bytes: limit.maximumBytes, +}); + +/** Refuse one request whose payload is above the effective ceiling. */ +export const bridgeRequestOversizeError = ( + operation: string, + limit: BridgeMaxRequestBytes, + payloadBytes: number, + peer: "Hopper" | "Cutter", +): AnalysisResourceConstraintError => { + const recovery = bridgeMaxRequestBytesRecovery(limit); + const refusal = + peer === "Cutter" + ? `Cutter request exceeds its authenticated frame limit of ${String(limit.maximumBytes)} bytes; no command was sent.` + : `Hopper request exceeds its ${String(limit.maximumBytes)}-byte payload limit; no request was sent.`; + return new AnalysisResourceConstraintError( + operation, + "transport", + `${refusal} ${recovery}`, + { ...limitFacts(limit), payload_bytes: payloadBytes }, + { remediationAction: recovery, retryable: false }, + ); +}; + +/** Refuse a command when the bridge advertisement and the client ceiling differ. */ +export const bridgeRequestLimitMismatchError = ( + operation: string, + limit: BridgeMaxRequestBytes, + bridgeBytes: number, + peer: "Hopper" | "Cutter", + sent = peer === "Cutter" ? "No command was sent." : "No request was sent.", +): AnalysisResourceConstraintError => { + const recovery = bridgeMaxRequestBytesRecovery(limit); + return new AnalysisResourceConstraintError( + operation, + "transport", + `The ${peer} bridge ceiling is ${String(bridgeBytes)} bytes and the REA client ceiling is ${String(limit.maximumBytes)} bytes. ${sent} An out-of-band ${peer} process defaults to ${String(DEFAULT_BRIDGE_MAX_REQUEST_BYTES)} bytes when ${BRIDGE_MAX_REQUEST_BYTES_SETTING} is unset. ${recovery}`, + { + bridge_max_request_bytes: bridgeBytes, + client_max_request_bytes: limit.maximumBytes, + }, + { remediationAction: recovery, retryable: false }, + ); +}; + +/** Refuse a Cutter response that exceeds the same configured frame ceiling. */ +export const bridgeResponseOversizeError = ( + limit: BridgeMaxRequestBytes, +): AnalysisResourceConstraintError => { + const recovery = bridgeMaxRequestBytesRecovery(limit); + return new AnalysisResourceConstraintError( + "cutter_command", + "transport", + `Cutter bridge response exceeded its framed transport limit of ${String(limit.maximumBytes)} bytes. ${recovery}`, + limitFacts(limit), + { remediationAction: recovery, retryable: false }, + ); +}; diff --git a/src/config/environment.ts b/src/config/environment.ts index 72447f8b51..43d8f635fc 100644 --- a/src/config/environment.ts +++ b/src/config/environment.ts @@ -19,6 +19,7 @@ import { mcpInputSchemaProfileSchema } from "./mcpInputSchemaProfile.js"; import { mcpOutputSchemaProfileSchema } from "./mcpOutputSchemaProfile.js"; import { mcpResponseBudgetSchema } from "./mcpResponseBudget.js"; import { asarMaxHeaderBytesSchema } from "./asarHeaderAdmission.js"; +import { bridgeMaxRequestBytesSchema } from "./bridgeMaxRequestBytes.js"; import { captureDecoderOldGenerationMiBSchema } from "./captureDecoderHeap.js"; import { regexMatchingDeadlineMsSchema } from "./regexMatchingDeadline.js"; import { regexWorkerHeapRequestMiBSchema } from "./regexWorkerHeapRequest.js"; @@ -32,6 +33,7 @@ import { parseLoaderArgs, parseStringArray } from "./parsers.js"; const environmentSchema = z.object({ REA_ANALYSIS_PROVIDER: analysisProviderSelectorSchema.default("auto"), + REA_BRIDGE_MAX_REQUEST_BYTES: bridgeMaxRequestBytesSchema.optional(), REA_MCP_MAX_RESPONSE_BYTES: mcpResponseBudgetSchema.optional(), REA_ASAR_MAX_HEADER_BYTES: asarMaxHeaderBytesSchema.optional(), REA_CAPTURE_DECODER_OLD_GENERATION_MIB: diff --git a/src/config/parseConfig.ts b/src/config/parseConfig.ts index 28ff6a705c..c3d4a9fef0 100644 --- a/src/config/parseConfig.ts +++ b/src/config/parseConfig.ts @@ -4,6 +4,7 @@ import type { ConfigurationError } from "../domain/configurationErrors.js"; import { ok, type Result } from "../domain/result.js"; import { parseEnvironment } from "./environment.js"; import { homeDirectoryFromEnvironment } from "./homeDirectory.js"; +import { DEFAULT_BRIDGE_MAX_REQUEST } from "./bridgeMaxRequestBytes.js"; import { DEFAULT_REGEX_MATCHING_DEADLINE } from "./regexMatchingDeadline.js"; import { DEFAULT_REGEX_WORKER_HEAP_REQUEST } from "./regexWorkerHeapRequest.js"; import type { AppConfig } from "./types.js"; @@ -123,6 +124,13 @@ export const parseConfig = ( referenceSourcePolicy: { secretPatterns: env.referenceSecretPatterns, }, + bridgeMaxRequestBytes: + env.REA_BRIDGE_MAX_REQUEST_BYTES === undefined + ? DEFAULT_BRIDGE_MAX_REQUEST + : { + maximumBytes: env.REA_BRIDGE_MAX_REQUEST_BYTES, + source: "setting", + }, regexMatchingDeadline: env.REA_REGEX_MATCHING_DEADLINE_MS === undefined ? DEFAULT_REGEX_MATCHING_DEADLINE diff --git a/src/config/types.ts b/src/config/types.ts index 97858d8b26..0e3ff20192 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -5,6 +5,7 @@ import type { AnalysisProviderSelector } from "../contracts/providerSelection.js import type { GhidraLanguageOverride } from "./ghidraLanguageOverride.js"; import type { McpInputSchemaProfile } from "./mcpInputSchemaProfile.js"; import type { McpOutputSchemaProfile } from "./mcpOutputSchemaProfile.js"; +import type { BridgeMaxRequestBytes } from "./bridgeMaxRequestBytes.js"; import type { RegexMatchingDeadline } from "./regexMatchingDeadline.js"; import type { RegexWorkerHeapRequest } from "./regexWorkerHeapRequest.js"; @@ -29,6 +30,8 @@ export interface AppConfig { readonly hopperLoaderArgs: readonly string[]; readonly logLevel: LogLevel; readonly referenceSourcePolicy: ReferenceSourcePolicy; + /** Hopper and Cutter request payload ceiling resolved at startup. */ + readonly bridgeMaxRequestBytes: BridgeMaxRequestBytes; /** Supervised ECMAScript matching deadline resolved at startup. */ readonly regexMatchingDeadline: RegexMatchingDeadline; /** Regex worker old-generation request resolved at startup. */ diff --git a/src/contracts/cutterToolContracts.ts b/src/contracts/cutterToolContracts.ts index be931a9c70..6d84d22dbe 100644 --- a/src/contracts/cutterToolContracts.ts +++ b/src/contracts/cutterToolContracts.ts @@ -68,7 +68,7 @@ export const CUTTER_TOOL_CONTRACTS = [ ...toolContractMetadata("list_cutter_sessions"), kind: "native-provider", description: - "List live REA bridge sessions hosted by upstream Cutter instances, including observed document identity and bridge-directory security status. Discovery admits at most 256 descriptor candidates (16 MiB at the descriptor byte limit), reads and probes at most eight concurrently, and reports bridge_directory_capacity_exceeded rather than claiming no sessions when that budget is exceeded.", + "List live REA bridge sessions hosted by upstream Cutter instances, including observed document identity and bridge-directory security status. Discovery admits at most 256 descriptor candidates (16 MiB at the descriptor byte limit). That candidate budget is separate from REA_BRIDGE_MAX_REQUEST_BYTES and has no runtime override. Discovery reads and probes at most eight concurrently, and reports bridge_directory_capacity_exceeded rather than claiming no sessions when that budget is exceeded.", inputSchema: cutterListInputSchema, outputSchema: cutterDiscoverySchema, examples: [{ title: "List open Cutter instances", input: {} }], @@ -78,7 +78,7 @@ export const CUTTER_TOOL_CONTRACTS = [ ...toolContractMetadata("cutter_command"), kind: "native-provider", description: - "Run one arbitrary Rizin command in a selected live Cutter session. Commands may edit and persist analysis state or execute plugin-provided effects; completion uncertainty is returned as Evidence and must not be retried automatically. In a shared MCP server, this admitted command closes any active binary-analysis session; after its effects have settled, call open_binary before further analysis. Close an active Ghidra session explicitly first.", + "Run one arbitrary Rizin command in a selected live Cutter session. Commands may edit and persist analysis state or execute plugin-provided effects; completion uncertainty is returned as Evidence and must not be retried automatically. The authenticated request and response frame is bounded by REA_BRIDGE_MAX_REQUEST_BYTES (default 16777216 bytes, floor 7464, maximum 536870887). The payload budget excludes the terminating LF. An out-of-band Cutter process without the setting uses 16777216 bytes; a mismatch is refused before the command is sent. In a shared MCP server, this admitted command closes any active binary-analysis session; after its effects have settled, call open_binary before further analysis. Close an active Ghidra session explicitly first.", inputSchema: cutterCommandInputSchema, outputSchema: evidenceResultOf(cutterCommandResultSchema), examples: [ diff --git a/src/cutter/CutterBridgeClient.ts b/src/cutter/CutterBridgeClient.ts index 1085a829e7..7968656ed7 100644 --- a/src/cutter/CutterBridgeClient.ts +++ b/src/cutter/CutterBridgeClient.ts @@ -23,12 +23,24 @@ import { AnalysisInputError, AnalysisOutputError, AnalysisResourceConstraintError, + resourceConstraintRecovery, } from "../domain/analysisErrorCore.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; +import { jsonValueByteLength } from "../domain/jsonValueByteLength.js"; +import type { JsonValue } from "../domain/jsonValue.js"; import { err, ok, type Result } from "../domain/result.js"; +import { cutterBridgeMac } from "./cutterBridgeMac.js"; +import { CutterBridgeResponseFrames } from "./cutterAuthenticatedResponse.js"; import { requireWindowsNativeAuthority } from "../windows/WindowsNativeLoader.js"; import { readBoundedFileBytes } from "../process/BoundedFileBytes.js"; import { homeDirectoryFromEnvironment } from "../config/homeDirectory.js"; +import { + bridgeRequestLimitMismatchError, + bridgeRequestOversizeError, + DEFAULT_BRIDGE_MAX_REQUEST_BYTES, + resolveBridgeMaxRequestBytes, + type BridgeMaxRequestBytes, +} from "../config/bridgeMaxRequestBytes.js"; const descriptorSchema = z.object({ session_id: z.string().uuid(), @@ -40,6 +52,7 @@ const descriptorSchema = z.object({ current_file: z.string().nullable(), cutter_version: z.string().nullable(), identity_status: z.literal("partial"), + max_request_bytes: z.int().positive().optional(), }); export const CUTTER_PROVIDER_IDENTITY = { @@ -81,105 +94,80 @@ const acknowledgementSchema = z.object({ acknowledged: z.boolean(), }); const MAX_DESCRIPTOR_BYTES = 64 * 1024; -const MAX_FRAME_BYTES = 16 * 1024 * 1024; -const FRAME_PAGE_BYTES = 64 * 1024; const REQUEST_TIMEOUT_MS = 25_000; -// Bound admitted descriptor input bytes and concurrent file/socket handles. -const MAX_DESCRIPTOR_CANDIDATES = MAX_FRAME_BYTES / MAX_DESCRIPTOR_BYTES; +// Discovery stays at the historical 16 MiB descriptor budget. It does not +// follow REA_BRIDGE_MAX_REQUEST_BYTES. +const DISCOVERY_CANDIDATE_BUDGET_BYTES = 16 * 1024 * 1024; +const MAX_DESCRIPTOR_CANDIDATES = + DISCOVERY_CANDIDATE_BUDGET_BYTES / MAX_DESCRIPTOR_BYTES; const DISCOVERY_CONCURRENCY = 8; type AuthenticatedRequestBytes = string & { readonly __authenticatedRequestBytes: unique symbol; }; const serializeRequest = ( + limit: BridgeMaxRequestBytes, descriptor: z.infer, request: Readonly>, ): AuthenticatedRequestBytes => { - const requestBytes = JSON.stringify({ + const requestValue = { ...request, session_id: descriptor.session_id, - }); + } as JsonValue; + // Worst-case JSON expansion (\u00XX) can exceed Node's string ceiling + // before an after-the-fact byte check. Measure within the budget first. + const measured = jsonValueByteLength(requestValue, limit.maximumBytes); + if (measured > limit.maximumBytes) + throw bridgeRequestOversizeError( + "cutter_command", + limit, + measured, + "Cutter", + ); + const requestBytes = JSON.stringify(requestValue); let frameBytes = Buffer.byteLength(JSON.stringify({ request: "", proof: "0".repeat(64) })) + Buffer.byteLength(requestBytes); for (const character of requestBytes) if (character === '"' || character === "\\") frameBytes += 1; - if (frameBytes > MAX_FRAME_BYTES) - throw new AnalysisResourceConstraintError( + if (frameBytes > limit.maximumBytes) + throw bridgeRequestOversizeError( "cutter_command", - "transport", - `Cutter request exceeds its authenticated frame limit of ${String(MAX_FRAME_BYTES)} bytes (16 MiB); no command was sent. This release has no runtime override.`, - { frame_bytes: frameBytes, max_frame_bytes: MAX_FRAME_BYTES }, - { - remediationAction: `Shorten the Cutter command. This ${String(MAX_FRAME_BYTES)} byte frame limit has no runtime override.`, - }, + limit, + frameBytes, + "Cutter", ); return requestBytes as AuthenticatedRequestBytes; }; -class CutterBridgeResponseFrames { - #parts: Buffer[] = []; - #page = Buffer.allocUnsafe(FRAME_PAGE_BYTES); - #pageLength = 0; - #contentBytes = 0; - #framesRead = 0; +const serverChallengeSchema = z.object({ + session_id: z.string().uuid(), + nonce: z.string().regex(/^[0-9a-f]{64}$/), + proof: z.string().regex(/^[0-9a-f]{64}$/), +}); - push(chunk: Buffer, onFrame: (line: Buffer) => boolean): void { - let offset = 0; - while (offset < chunk.length) { - const delimiter = chunk.indexOf(0x0a, offset); - const end = delimiter < 0 ? chunk.length : delimiter; - const segment = chunk.subarray(offset, end); - if ( - this.#contentBytes + segment.length + (delimiter < 0 ? 1 : 0) > - MAX_FRAME_BYTES - ) - throw new Error( - `Cutter bridge response exceeded its framed transport limit of ${String(MAX_FRAME_BYTES)} bytes (16 MiB). This release has no runtime override`, - ); - this.#append(segment); - if (delimiter < 0) return; - if (++this.#contentBytes > MAX_FRAME_BYTES) - throw new Error( - `Cutter bridge response exceeded its framed transport limit of ${String(MAX_FRAME_BYTES)} bytes (16 MiB). This release has no runtime override`, - ); - if (this.#pageLength > 0) - this.#parts.push(this.#page.subarray(0, this.#pageLength)); - const line = Buffer.concat(this.#parts, this.#contentBytes - 1); - this.#reset(); - if (++this.#framesRead > 2) - throw new Error("Cutter bridge sent more than two protocol frames"); - offset = delimiter + 1; - if (!onFrame(line)) return; - } - } - - #append(segment: Buffer): void { - let copied = 0; - while (copied < segment.length) { - const length = Math.min( - segment.length - copied, - FRAME_PAGE_BYTES - this.#pageLength, - ); - segment.copy(this.#page, this.#pageLength, copied, copied + length); - this.#pageLength += length; - this.#contentBytes += length; - copied += length; - if (this.#pageLength === FRAME_PAGE_BYTES) { - this.#parts.push(this.#page); - this.#page = Buffer.allocUnsafe(FRAME_PAGE_BYTES); - this.#pageLength = 0; - } - } - } - - #reset(): void { - this.#parts = []; - this.#page = Buffer.allocUnsafe(FRAME_PAGE_BYTES); - this.#pageLength = 0; - this.#contentBytes = 0; - } -} +/** Nonce from a hello whose proof shows this descriptor's bridge is the peer. */ +const authenticatedServerNonce = ( + descriptor: { readonly session_id: string; readonly token: string }, + clientNonce: string, + value: unknown, +): string => { + const challenge = serverChallengeSchema.parse(value); + if (challenge.session_id !== descriptor.session_id) + throw new Error("unexpected peer session"); + const expected = createHmac("sha256", descriptor.token) + .update( + `rea-cutter-server-v2:${descriptor.session_id}:${clientNonce}:${challenge.nonce}`, + ) + .digest(); + const presented = Buffer.from(challenge.proof, "hex"); + if ( + expected.length !== presented.length || + !timingSafeEqual(expected, presented) + ) + throw new Error("peer proof failed"); + return challenge.nonce; +}; const assertBridgeActive = ( signal: AbortSignal | undefined, @@ -253,6 +241,7 @@ class CutterBridgeRequestFailure extends Error { message: string, readonly requestSent: boolean, readonly cancellation?: { readonly reason: unknown }, + readonly limit?: AnalysisResourceConstraintError, ) { super(message); } @@ -269,7 +258,7 @@ interface CutterBridgeClientOptions { export type CutterBridgeSession = Omit< z.infer, - "token" | "host" | "port" + "token" | "host" | "port" | "max_request_bytes" >; export interface CutterBridgeDiscovery { readonly sessions: readonly CutterBridgeSession[]; @@ -289,6 +278,7 @@ export interface CutterBridgeDiscovery { /** Discovers and communicates with REA Python bridge plugins in live Cutter processes. */ export class CutterBridgeClient { + readonly #maxRequestBytes: BridgeMaxRequestBytes; readonly #directory: string; readonly #descriptorResources = new ArtifactResourceScope(); readonly #platform: NodeJS.Platform; @@ -299,6 +289,7 @@ export class CutterBridgeClient { environment: Readonly = process.env, options: CutterBridgeClientOptions = {}, ) { + this.#maxRequestBytes = resolveBridgeMaxRequestBytes(environment); const configured = environment.REA_CUTTER_BRIDGE_DIR; const localAppData = environment.LOCALAPPDATA; this.#platform = options.platform ?? process.platform; @@ -354,6 +345,7 @@ export class CutterBridgeClient { options: CutterDiscoveryOptions = {}, ): Promise { const { signal } = options; + const mismatches: number[] = []; const loaded = await this.#readDescriptors("list_cutter_sessions", signal); if (loaded.error !== undefined) return { @@ -366,11 +358,21 @@ export class CutterBridgeClient { loaded.descriptors, async (descriptor) => { assertBridgeActive(signal, "list_cutter_sessions"); + const advertised = + descriptor.max_request_bytes ?? DEFAULT_BRIDGE_MAX_REQUEST_BYTES; + if (advertised !== this.#maxRequestBytes.maximumBytes) { + // A crashed process can leave the previous ceiling on disk. Count + // the mismatch only after the hello proves a live authenticated + // bridge. The probe does not send a status request. + if (await this.#liveAuthenticatedBridge(descriptor, signal)) + mismatches.push(advertised); + return undefined; + } try { const status = responseSchema.safeParse( await this.#request( descriptor, - serializeRequest(descriptor, { + serializeRequest(this.#maxRequestBytes, descriptor, { kind: "status", session_id: descriptor.session_id, }), @@ -389,6 +391,8 @@ export class CutterBridgeClient { document_generation: status.data.document_generation ?? descriptor.document_generation, current_file: + // Null is an observed unknown path. Only a missing field may + // fall back to the descriptor, and never beside a newer generation. status.data.current_file === undefined ? descriptor.current_file : status.data.current_file, @@ -397,13 +401,21 @@ export class CutterBridgeClient { identity_status: "partial" as const, }; } catch (cause) { - if ( - cause instanceof CutterBridgeRequestFailure && - cause.cancellation !== undefined - ) - throw new AnalysisCancelledError("list_cutter_sessions", { - cause: cause.cancellation.reason, - }); + if (cause instanceof AnalysisResourceConstraintError) throw cause; + if (cause instanceof CutterBridgeRequestFailure) { + if (cause.cancellation !== undefined) + throw new AnalysisCancelledError("list_cutter_sessions", { + cause: cause.cancellation.reason, + }); + if (cause.limit !== undefined) + throw new AnalysisResourceConstraintError( + "list_cutter_sessions", + cause.limit.resource, + cause.limit.reason, + cause.limit.reportedLimits, + resourceConstraintRecovery(cause.limit), + ); + } return undefined; } }, @@ -414,6 +426,14 @@ export class CutterBridgeClient { const sessions = descriptors.filter( (value): value is CutterBridgeSession => value !== undefined, ); + if (mismatches.length > 0) + throw bridgeRequestLimitMismatchError( + "list_cutter_sessions", + this.#maxRequestBytes, + mismatches[0] ?? DEFAULT_BRIDGE_MAX_REQUEST_BYTES, + "Cutter", + "No status request was sent to the mismatched session.", + ); return { sessions, bridge_directory: this.#directory, @@ -434,14 +454,27 @@ export class CutterBridgeClient { assertBridgeActive(input.signal, "cutter_command"); const descriptor = await this.#find(input.sessionId, input.signal); assertBridgeActive(input.signal, "cutter_command"); + const advertised = + descriptor.max_request_bytes ?? DEFAULT_BRIDGE_MAX_REQUEST_BYTES; + if (advertised !== this.#maxRequestBytes.maximumBytes) + throw bridgeRequestLimitMismatchError( + "cutter_command", + this.#maxRequestBytes, + advertised, + "Cutter", + ); const requestId = randomUUID(); - const commandRequest = serializeRequest(descriptor, { - kind: "command", - request_id: requestId, - expected_generation: input.expectedGeneration, - command: input.command, - json: input.json, - }); + const commandRequest = serializeRequest( + this.#maxRequestBytes, + descriptor, + { + kind: "command", + request_id: requestId, + expected_generation: input.expectedGeneration, + command: input.command, + json: input.json, + }, + ); await this.#recoverCompletedCommands(descriptor, input.signal); await this.#retryPendingAcknowledgements(descriptor); assertBridgeActive(input.signal, "cutter_command"); @@ -472,7 +505,9 @@ export class CutterBridgeClient { executionState: "unknown", error: "transport-response-missing", message: `${cause.message}; the command may have completed; do not retry automatically`, - outputTruncated: false, + // The carried limit is the framed response ceiling. The command was + // sent, and the discarded response is omitted output. + outputTruncated: cause.limit !== undefined, ...(completionToken === undefined ? {} : { completionToken }), }; } @@ -617,7 +652,7 @@ export class CutterBridgeClient { ): Promise> { const raw = await this.#request( descriptor, - serializeRequest(descriptor, { + serializeRequest(this.#maxRequestBytes, descriptor, { kind: "status", session_id: descriptor.session_id, }), @@ -666,7 +701,7 @@ export class CutterBridgeClient { try { const raw = await this.#request( descriptor, - serializeRequest(descriptor, { + serializeRequest(this.#maxRequestBytes, descriptor, { kind: "ack", request_id: requestId, }), @@ -934,6 +969,84 @@ export class CutterBridgeClient { }; } + /** + * Hello only. A matching server proof means this descriptor is a live + * authenticated bridge. The socket closes before any status or command. + */ + #liveAuthenticatedBridge( + descriptor: z.infer, + signal: AbortSignal | undefined, + ): Promise { + assertBridgeActive(signal, "list_cutter_sessions"); + return new Promise((resolve, reject) => { + const connection = createConnection({ + host: descriptor.host, + port: descriptor.port, + }); + const frames = new CutterBridgeResponseFrames(this.#maxRequestBytes); + let settled = false; + const settle = ( + outcome: + | { readonly ok: true; readonly live: boolean } + | { readonly ok: false; readonly cause: unknown }, + ): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + connection.destroy(); + if (outcome.ok) resolve(outcome.live); + else reject(outcome.cause); + }; + const timer = setTimeout( + () => settle({ ok: true, live: false }), + REQUEST_TIMEOUT_MS, + ); + const onAbort = (): void => { + settle({ + ok: false, + cause: new AnalysisCancelledError("list_cutter_sessions", { + cause: signal?.reason, + }), + }); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + if (signal?.aborted) { + onAbort(); + return; + } + const clientNonce = randomBytes(32).toString("hex"); + connection.once("connect", () => { + if (settled) return; + connection.write( + `${JSON.stringify({ kind: "rea-auth-v2", nonce: clientNonce })}\n`, + ); + }); + connection.on("data", (chunk: Buffer) => { + if (settled) return; + try { + frames.push(chunk, (frame) => { + if (frame.kind !== "challenge") { + settle({ ok: true, live: false }); + return false; + } + try { + authenticatedServerNonce(descriptor, clientNonce, frame.value); + settle({ ok: true, live: true }); + } catch { + settle({ ok: true, live: false }); + } + return false; + }); + } catch { + settle({ ok: true, live: false }); + } + }); + connection.once("error", () => settle({ ok: true, live: false })); + connection.once("close", () => settle({ ok: true, live: false })); + }); + } + #request( descriptor: z.infer, requestBytes: AuthenticatedRequestBytes, @@ -945,7 +1058,7 @@ export class CutterBridgeClient { host: descriptor.host, port: descriptor.port, }); - const frames = new CutterBridgeResponseFrames(); + const frames = new CutterBridgeResponseFrames(this.#maxRequestBytes); let completion: (() => void) | undefined; let requestSent = false; let authenticated = false; @@ -963,10 +1076,16 @@ export class CutterBridgeClient { const rejectRequest = ( message: string, cancellation?: { readonly reason: unknown }, + limit?: AnalysisResourceConstraintError, ): void => finish(() => reject( - new CutterBridgeRequestFailure(message, requestSent, cancellation), + new CutterBridgeRequestFailure( + message, + requestSent, + cancellation, + limit, + ), ), ); const timer = setTimeout(() => { @@ -990,63 +1109,47 @@ export class CutterBridgeClient { connection.on("data", (chunk: Buffer) => { if (completion !== undefined) return; try { - frames.push(chunk, (line) => { - const frame: unknown = JSON.parse(line.toString("utf8")); - if (!authenticated) { - const challenge = z - .object({ - session_id: z.string().uuid(), - nonce: z.string().regex(/^[0-9a-f]{64}$/), - proof: z.string().regex(/^[0-9a-f]{64}$/), - }) - .parse(frame); - if (challenge.session_id !== descriptor.session_id) - throw new Error("unexpected peer session"); - serverNonce = challenge.nonce; - connectionOrdinal = challenge.nonce.slice(0, 16); - const expected = createHmac("sha256", descriptor.token) - .update( - `rea-cutter-server-v2:${descriptor.session_id}:${clientNonce}:${challenge.nonce}`, - ) - .digest(); - if ( - !timingSafeEqual(expected, Buffer.from(challenge.proof, "hex")) - ) - throw new Error("peer proof failed"); + frames.push(chunk, (frame) => { + if (frame.kind === "challenge") { + if (authenticated) throw new Error("unexpected challenge"); + serverNonce = authenticatedServerNonce( + descriptor, + clientNonce, + frame.value, + ); + connectionOrdinal = serverNonce.slice(0, 16); onAuthenticated?.(connectionOrdinal); - const proof = createHmac("sha256", descriptor.token) - .update( - `rea-cutter-client-v2:${descriptor.session_id}:${clientNonce}:${challenge.nonce}:${requestBytes}`, - ) - .digest("hex"); + const proof = cutterBridgeMac(descriptor.token, [ + `rea-cutter-client-v2:${descriptor.session_id}:${clientNonce}:${serverNonce}:`, + requestBytes, + ]).toString("hex"); authenticated = true; requestSent = true; connection.write( `${JSON.stringify({ request: requestBytes, proof })}\n`, ); return true; - } else { - const envelope = z - .object({ - response: z.string(), - proof: z.string().regex(/^[0-9a-f]{64}$/), - }) - .parse(frame); - const expected = createHmac("sha256", descriptor.token) - .update( - `rea-cutter-response-v2:${descriptor.session_id}:${clientNonce}:${serverNonce}:${envelope.response}`, - ) - .digest(); - if ( - !timingSafeEqual(expected, Buffer.from(envelope.proof, "hex")) - ) - throw new Error("response proof failed"); - const response: unknown = JSON.parse(envelope.response); - finish(() => resolve(response)); - return false; } + // One decoded payload. The wire pages and outer JSON string are gone. + const expected = cutterBridgeMac(descriptor.token, [ + `rea-cutter-response-v2:${descriptor.session_id}:${clientNonce}:${serverNonce}:`, + frame.response, + ]); + if (!timingSafeEqual(expected, Buffer.from(frame.proof, "hex"))) + throw new Error("response proof failed"); + const response: unknown = JSON.parse(frame.response); + finish(() => resolve(response)); + return false; }); - } catch { + } catch (cause) { + if (cause instanceof AnalysisResourceConstraintError) { + // The command is already on the UI thread. Keep the unknown + // execution so cleanup waits for its receipt, and carry the + // ceiling diagnostic in that result. + if (requestSent) rejectRequest(cause.reason, undefined, cause); + else finish(() => reject(cause)); + return; + } rejectRequest("Cutter bridge peer authentication or response failed"); } }); diff --git a/src/cutter/cutterAuthenticatedResponse.test.ts b/src/cutter/cutterAuthenticatedResponse.test.ts new file mode 100644 index 0000000000..a487f7ea5e --- /dev/null +++ b/src/cutter/cutterAuthenticatedResponse.test.ts @@ -0,0 +1,172 @@ +import { expect, it } from "vitest"; + +import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; +import { + MAXIMUM_BRIDGE_MAX_REQUEST_BYTES, + MINIMUM_BRIDGE_MAX_REQUEST_BYTES, + type BridgeMaxRequestBytes, +} from "../config/bridgeMaxRequestBytes.js"; +import { + CutterAuthenticatedResponseDecoder, + CutterBridgeResponseFrames, +} from "./cutterAuthenticatedResponse.js"; + +const limit = ( + maximumBytes: number, + source: BridgeMaxRequestBytes["source"] = "setting", +): BridgeMaxRequestBytes => ({ maximumBytes, source }); + +const PROOF = "ab".repeat(32); + +const envelope = (response: string, proof = PROOF): Buffer => + Buffer.from(JSON.stringify({ response, proof })); + +const decode = ( + frame: Buffer, + maximumBytes: number, + chunkSize = frame.length, +): { readonly response: string; readonly proof: string } => { + const decoder = new CutterAuthenticatedResponseDecoder(limit(maximumBytes)); + for (let offset = 0; offset < frame.length; offset += chunkSize) + decoder.push( + frame.subarray(offset, Math.min(frame.length, offset + chunkSize)), + ); + return decoder.finish(); +}; + +const splitPoints = (length: number): number[] => + [1, 2, 3, 7, 13, 64, 65].filter((size) => size < length); + +it("decodes an authenticated response split at escape and proof boundaries", () => { + const response = JSON.stringify({ + ok: true, + output: + 'quote " slash \\ newline \n unit \u0001 line \u2028 snow 雪 emoji 😀', + empty: "", + }); + const frame = envelope(response); + const parsed = JSON.parse(frame.toString("utf8")) as { + response: string; + proof: string; + }; + expect(decode(frame, frame.length).response).toBe(parsed.response); + expect(decode(frame, frame.length).proof).toBe(PROOF); + for (const size of splitPoints(frame.length)) + expect(decode(frame, frame.length, size).response).toBe(response); + expect(decode(envelope(""), envelope("").length).response).toBe(""); + const slashes = "\\".repeat(32_000); + expect( + decode(envelope(slashes), envelope(slashes).length, 64 * 1024).response, + ).toBe(slashes); + expect( + decode(Buffer.from(`{"response":"\\u0041","proof":"${PROOF}"}`), 256) + .response, + ).toBe("A"); +}); + +it("accepts a raw UTF-8 line separator inside the response string", () => { + const response = `{"ok":"${String.fromCharCode(0x2028)}"}`; + const body = Buffer.concat( + [...response].map((character) => { + if (character === '"') return Buffer.from('\\"'); + if (character === "\\") return Buffer.from("\\\\"); + return Buffer.from(character, "utf8"); + }), + ); + const frame = Buffer.concat([ + Buffer.from('{"response":"'), + body, + Buffer.from(`","proof":"${PROOF}"}`), + ]); + expect(decode(frame, frame.length, 5).response).toBe(response); + expect( + ( + JSON.parse(frame.toString("utf8")) as { + response: string; + } + ).response, + ).toBe(response); +}); + +it("decodes a surrogate-pair escape into one code point", () => { + const frame = Buffer.from(`{"response":"\\uD83D\\uDE00","proof":"${PROOF}"}`); + expect(decode(frame, frame.length, 3).response).toBe("😀"); +}); + +it("refuses a malformed envelope, a bad escape, and an unfinished frame", () => { + const ceiling = limit(4096); + const refuse = (frame: Buffer): void => { + const decoder = new CutterAuthenticatedResponseDecoder(ceiling); + expect(() => { + decoder.push(frame); + decoder.finish(); + }).toThrow("invalid authenticated response"); + }; + refuse(Buffer.from(`{"ok":false,"proof":"${PROOF}"}`)); + refuse(Buffer.from(`{"response":"\\x","proof":"${PROOF}"}`)); + refuse(Buffer.from(`{"response":"\\uD800","proof":"${PROOF}"}`)); + refuse(Buffer.from(`{"response":"ok","proof":"${"AB".repeat(32)}"}`)); + refuse(Buffer.from(`{"response":"ok","proof":"${PROOF}"} `)); + const decoder = new CutterAuthenticatedResponseDecoder(ceiling); + decoder.push(Buffer.from('{"response":"ok"')); + expect(() => decoder.finish()).toThrow("invalid authenticated response"); +}); + +it("reports the configured ceiling when a response frame does not fit", () => { + const frame = envelope("x".repeat(80)); + const ceiling = limit(64); + const decoder = new CutterAuthenticatedResponseDecoder(ceiling); + let caught: unknown; + try { + decoder.push(frame); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(AnalysisResourceConstraintError); + const error = caught as AnalysisResourceConstraintError; + expect(error._tag).toBe("AnalysisResourceConstraintError"); + expect(error.operation).toBe("cutter_command"); + expect(error.resource).toBe("transport"); + expect(error.reportedLimits).toMatchObject({ max_request_bytes: 64 }); + expect(error.reason).toContain("REA_BRIDGE_MAX_REQUEST_BYTES"); + expect(error.reason).toContain("value is 64 bytes"); + expect(error.reason).toContain("source setting"); + expect(error.reason).toContain(String(MINIMUM_BRIDGE_MAX_REQUEST_BYTES)); + expect(error.reason).toContain(String(MAXIMUM_BRIDGE_MAX_REQUEST_BYTES)); + expect(error.reason).toContain("restart"); +}); + +it("reads one challenge and one response without buffering a third frame", () => { + const challenge = Buffer.from( + `${JSON.stringify({ + session_id: "27d3e3f1-f1e5-49ae-91ec-95af1f343a5a", + nonce: "c".repeat(64), + proof: "d".repeat(64), + })}\n`, + ); + const response = JSON.stringify({ output: "split 雪" }); + const body = Buffer.concat([envelope(response), Buffer.from("\n")]); + const frames = new CutterBridgeResponseFrames(limit(4096, "default")); + const seen: string[] = []; + const combined = Buffer.concat([challenge, body]); + for (let offset = 0; offset < combined.length; offset += 5) + frames.push(combined.subarray(offset, offset + 5), (frame) => { + if (frame.kind === "challenge") { + seen.push("challenge"); + return true; + } + seen.push(frame.response); + return false; + }); + expect(seen).toEqual(["challenge", response]); + expect(() => frames.push(Buffer.from("{"), () => true)).toThrow( + "more than two protocol frames", + ); +}); + +it("refuses an oversized authentication challenge before the response", () => { + const frames = new CutterBridgeResponseFrames(limit(4096)); + expect(() => frames.push(Buffer.alloc(2049, 0x7b), () => true)).toThrow( + "invalid authentication challenge", + ); +}); diff --git a/src/cutter/cutterAuthenticatedResponse.ts b/src/cutter/cutterAuthenticatedResponse.ts new file mode 100644 index 0000000000..e48af01454 --- /dev/null +++ b/src/cutter/cutterAuthenticatedResponse.ts @@ -0,0 +1,340 @@ +import { bridgeResponseOversizeError } from "../config/bridgeMaxRequestBytes.js"; +import type { BridgeMaxRequestBytes } from "../config/bridgeMaxRequestBytes.js"; + +/** + * Decode one authenticated Cutter response frame. + * + * The wire frame is `{"response":"","proof":"<64 hex>"}\n`. + * Socket chunks are unescaped into one payload buffer and then one string. + * The paged wire bytes, a concatenated copy, and an outer `JSON.parse` string + * are not retained. The caller checks the proof and parses that one string. + */ + +const PREFIX = Buffer.from('{"response":"'); +const PROOF_MARK = Buffer.from(',"proof":"'); +const CLOSE = Buffer.from('"}'); +const PROOF_HEX_LENGTH = 64; +const CHALLENGE_FRAME_BYTES = 2048; +const INITIAL_RESPONSE_BYTES = 256; + +const HEX = new Int8Array(256).fill(-1); +for (let digit = 0; digit < 10; digit += 1) HEX[0x30 + digit] = digit; +for (let digit = 0; digit < 6; digit += 1) { + HEX[0x41 + digit] = 10 + digit; + HEX[0x61 + digit] = 10 + digit; +} + +const SIMPLE_ESCAPE = new Int16Array(256).fill(-1); +SIMPLE_ESCAPE[0x22] = 0x22; +SIMPLE_ESCAPE[0x5c] = 0x5c; +SIMPLE_ESCAPE[0x2f] = 0x2f; +SIMPLE_ESCAPE[0x62] = 0x08; +SIMPLE_ESCAPE[0x66] = 0x0c; +SIMPLE_ESCAPE[0x6e] = 0x0a; +SIMPLE_ESCAPE[0x72] = 0x0d; +SIMPLE_ESCAPE[0x74] = 0x09; + +type Stage = "prefix" | "body" | "mark" | "proof" | "close" | "done"; +type Escape = "none" | "slash" | "hex" | "low-slash" | "low-u" | "low-hex"; + +const invalid = (): Error => new Error("invalid authenticated response"); + +export interface DecodedCutterResponse { + readonly response: string; + readonly proof: string; +} + +export class CutterAuthenticatedResponseDecoder { + readonly #limit: BridgeMaxRequestBytes; + #wire = 0; + #stage: Stage = "prefix"; + #literal = 0; + #escape: Escape = "none"; + #hex = 0; + #hexCount = 0; + #high = 0; + #proof = ""; + #out = Buffer.allocUnsafe(INITIAL_RESPONSE_BYTES); + #outLength = 0; + + constructor(limit: BridgeMaxRequestBytes) { + this.#limit = limit; + } + + push(chunk: Buffer): void { + if (chunk.length === 0) return; + if (this.#stage === "done") throw invalid(); + const next = this.#wire + chunk.length; + if (next > this.#limit.maximumBytes) + throw bridgeResponseOversizeError(this.#limit); + this.#wire = next; + let offset = 0; + while (offset < chunk.length) offset = this.#take(chunk, offset); + } + + /** Release the decode buffer and return the one response string. */ + finish(): DecodedCutterResponse { + if (this.#stage !== "done") throw invalid(); + const response = this.#out.toString("utf8", 0, this.#outLength); + this.#out = Buffer.alloc(0); + this.#outLength = 0; + return { response, proof: this.#proof }; + } + + #take(chunk: Buffer, offset: number): number { + switch (this.#stage) { + case "prefix": + return this.#match(chunk, offset, PREFIX, "body"); + case "mark": + return this.#match(chunk, offset, PROOF_MARK, "proof"); + case "close": + return this.#match(chunk, offset, CLOSE, "done"); + case "proof": + return this.#takeProof(chunk, offset); + case "body": + return this.#takeBody(chunk, offset); + case "done": + throw invalid(); + } + } + + #match(chunk: Buffer, offset: number, literal: Buffer, next: Stage): number { + const available = Math.min( + literal.length - this.#literal, + chunk.length - offset, + ); + const actual = chunk.subarray(offset, offset + available); + const expected = literal.subarray(this.#literal, this.#literal + available); + if (!actual.equals(expected)) throw invalid(); + this.#literal += available; + if (this.#literal === literal.length) { + this.#stage = next; + this.#literal = 0; + } + return offset + available; + } + + #takeProof(chunk: Buffer, offset: number): number { + const end = Math.min( + chunk.length, + offset + (PROOF_HEX_LENGTH - this.#proof.length), + ); + for (let index = offset; index < end; index += 1) { + const byte = chunk[index]!; + const digit = + byte >= 0x30 && byte <= 0x39 + ? byte - 0x30 + : byte >= 0x61 && byte <= 0x66 + ? byte - 0x57 + : -1; + if (digit < 0) throw invalid(); + this.#proof += String.fromCharCode(byte); + } + if (this.#proof.length === PROOF_HEX_LENGTH) this.#stage = "close"; + return end; + } + + #takeBody(chunk: Buffer, offset: number): number { + if (this.#escape !== "none") return this.#takeEscape(chunk, offset); + // Stop at the first quote or backslash. Searching the whole tail for a + // quote after every escape repeats that scan for backslash-heavy replies. + const slash = chunk.indexOf(0x5c, offset); + let quote = -1; + if (slash < 0) quote = chunk.indexOf(0x22, offset); + else { + const relative = chunk.subarray(offset, slash).indexOf(0x22); + if (relative >= 0) quote = offset + relative; + } + let end = chunk.length; + if (quote >= 0 && quote < end) end = quote; + if (slash >= 0 && slash < end) end = slash; + if (end > offset) this.#append(chunk.subarray(offset, end)); + if (end === chunk.length) return chunk.length; + if (chunk[end] === 0x22) { + this.#stage = "mark"; + this.#literal = 0; + return end + 1; + } + this.#escape = "slash"; + return end + 1; + } + + #takeEscape(chunk: Buffer, offset: number): number { + const byte = chunk[offset]!; + if (this.#escape === "slash") this.#escaped(byte); + else if (this.#escape === "hex") this.#hexDigit(byte, false); + else if (this.#escape === "low-slash") { + if (byte !== 0x5c) throw invalid(); + this.#escape = "low-u"; + } else if (this.#escape === "low-u") { + if (byte !== 0x75) throw invalid(); + this.#escape = "low-hex"; + this.#hex = 0; + this.#hexCount = 0; + } else this.#hexDigit(byte, true); + return offset + 1; + } + + #escaped(byte: number): void { + if (byte === 0x75) { + this.#escape = "hex"; + this.#hex = 0; + this.#hexCount = 0; + return; + } + const mapped = SIMPLE_ESCAPE[byte] ?? -1; + if (mapped < 0) throw invalid(); + this.#appendByte(mapped); + this.#escape = "none"; + } + + #hexDigit(byte: number, low: boolean): void { + const digit = HEX[byte] ?? -1; + if (digit < 0) throw invalid(); + this.#hex = (this.#hex << 4) | digit; + this.#hexCount += 1; + if (this.#hexCount < 4) return; + const code = this.#hex; + this.#hex = 0; + this.#hexCount = 0; + this.#escape = "none"; + if (low) { + if (code < 0xdc00 || code > 0xdfff) throw invalid(); + this.#appendCodePoint( + 0x10000 + ((this.#high - 0xd800) << 10) + (code - 0xdc00), + ); + return; + } + if (code >= 0xd800 && code <= 0xdbff) { + this.#high = code; + this.#escape = "low-slash"; + return; + } + if (code >= 0xdc00 && code <= 0xdfff) throw invalid(); + this.#appendCodePoint(code); + } + + #appendCodePoint(code: number): void { + if (code < 0x80) { + this.#appendByte(code); + return; + } + if (code < 0x800) { + this.#appendByte(0xc0 | (code >> 6)); + this.#appendByte(0x80 | (code & 0x3f)); + return; + } + if (code < 0x10000) { + this.#appendByte(0xe0 | (code >> 12)); + this.#appendByte(0x80 | ((code >> 6) & 0x3f)); + this.#appendByte(0x80 | (code & 0x3f)); + return; + } + this.#appendByte(0xf0 | (code >> 18)); + this.#appendByte(0x80 | ((code >> 12) & 0x3f)); + this.#appendByte(0x80 | ((code >> 6) & 0x3f)); + this.#appendByte(0x80 | (code & 0x3f)); + } + + #appendByte(value: number): void { + this.#ensure(1); + this.#out[this.#outLength] = value; + this.#outLength += 1; + } + + #append(bytes: Buffer): void { + this.#ensure(bytes.length); + bytes.copy(this.#out, this.#outLength); + this.#outLength += bytes.length; + } + + #ensure(extra: number): void { + if (this.#outLength + extra <= this.#out.length) return; + let next = this.#out.length; + while (this.#outLength + extra > next) next *= 2; + if (next > this.#limit.maximumBytes) next = this.#limit.maximumBytes; + if (this.#outLength + extra > next) throw invalid(); + const grown = Buffer.allocUnsafe(next); + this.#out.copy(grown, 0, 0, this.#outLength); + this.#out = grown; + } +} + +export type CutterInboundFrame = + | { readonly kind: "challenge"; readonly value: unknown } + | { + readonly kind: "response"; + readonly response: string; + readonly proof: string; + }; + +/** Read a Cutter challenge, then one authenticated response, from socket chunks. */ +export class CutterBridgeResponseFrames { + readonly #limit: BridgeMaxRequestBytes; + #challenge: Buffer | undefined; + #challengeLength = 0; + #decoder: CutterAuthenticatedResponseDecoder | undefined; + #frames = 0; + + constructor(limit: BridgeMaxRequestBytes) { + this.#limit = limit; + } + + push(chunk: Buffer, onFrame: (frame: CutterInboundFrame) => boolean): void { + let offset = 0; + while (offset < chunk.length) { + if (this.#frames >= 2) + throw new Error("Cutter bridge sent more than two protocol frames"); + const step = + this.#frames === 0 + ? this.#pushChallenge(chunk, offset, onFrame) + : this.#pushResponse(chunk, offset, onFrame); + if (step < 0) return; + offset = step; + } + } + + #pushChallenge( + chunk: Buffer, + offset: number, + onFrame: (frame: CutterInboundFrame) => boolean, + ): number { + const delimiter = chunk.indexOf(0x0a, offset); + const end = delimiter < 0 ? chunk.length : delimiter; + const length = end - offset; + if (this.#challengeLength + length > CHALLENGE_FRAME_BYTES) + throw new Error("invalid authentication challenge"); + this.#challenge ??= Buffer.allocUnsafe(CHALLENGE_FRAME_BYTES); + chunk.copy(this.#challenge, this.#challengeLength, offset, end); + this.#challengeLength += length; + if (delimiter < 0) return chunk.length; + let value: unknown; + try { + value = JSON.parse( + this.#challenge.toString("utf8", 0, this.#challengeLength), + ) as unknown; + } catch { + throw new Error("invalid authentication challenge"); + } + this.#challenge = undefined; + this.#challengeLength = 0; + this.#frames = 1; + return onFrame({ kind: "challenge", value }) ? delimiter + 1 : -1; + } + + #pushResponse( + chunk: Buffer, + offset: number, + onFrame: (frame: CutterInboundFrame) => boolean, + ): number { + const delimiter = chunk.indexOf(0x0a, offset); + const end = delimiter < 0 ? chunk.length : delimiter; + this.#decoder ??= new CutterAuthenticatedResponseDecoder(this.#limit); + this.#decoder.push(chunk.subarray(offset, end)); + if (delimiter < 0) return chunk.length; + const decoded = this.#decoder.finish(); + this.#decoder = undefined; + this.#frames = 2; + return onFrame({ kind: "response", ...decoded }) ? delimiter + 1 : -1; + } +} diff --git a/src/cutter/cutterBridgeMac.test.ts b/src/cutter/cutterBridgeMac.test.ts new file mode 100644 index 0000000000..f3b555c1bc --- /dev/null +++ b/src/cutter/cutterBridgeMac.test.ts @@ -0,0 +1,42 @@ +import { createHmac } from "node:crypto"; + +import { expect, it } from "vitest"; + +import { cutterBridgeMac } from "./cutterBridgeMac.js"; + +const TOKEN = "bridge-token-0123456789abcdef"; +const SESSION = "00000000-0000-4000-8000-000000000001"; +const NONCE = "a".repeat(64); + +it("matches one update of the joined client and response preimages", () => { + const request = JSON.stringify({ command: "雪" }); + const clientPrefix = `rea-cutter-client-v2:${SESSION}:${NONCE}:${NONCE}:`; + const responsePrefix = `rea-cutter-response-v2:${SESSION}:${NONCE}:${NONCE}:`; + expect(clientPrefix).toHaveLength(188); + expect(responsePrefix).toHaveLength(190); + expect(cutterBridgeMac(TOKEN, [clientPrefix, request]).toString("hex")).toBe( + createHmac("sha256", TOKEN) + .update(`${clientPrefix}${request}`) + .digest("hex"), + ); + expect( + cutterBridgeMac(TOKEN, [responsePrefix, request]).toString("hex"), + ).toBe( + createHmac("sha256", TOKEN) + .update(`${responsePrefix}${request}`) + .digest("hex"), + ); +}); + +// A maximum-sized preimage is about 512 MiB. Routine adapter runs keep the +// split-update check above. Set REA_BRIDGE_MAC_LIMIT_TEST=1 to hash the ceiling. +it.skipIf(process.env.REA_BRIDGE_MAC_LIMIT_TEST !== "1")( + "hashes a preimage that does not fit in one Node string", + () => { + const prefix = `rea-cutter-client-v2:${SESSION}:${NONCE}:${NONCE}:`; + const stringCeiling = 536_870_888; + const body = "x".repeat(stringCeiling - prefix.length + 1); + expect(prefix.length + body.length).toBe(stringCeiling + 1); + expect(cutterBridgeMac(TOKEN, [prefix, body])).toHaveLength(32); + }, +); diff --git a/src/cutter/cutterBridgeMac.ts b/src/cutter/cutterBridgeMac.ts new file mode 100644 index 0000000000..dfc17df6ca --- /dev/null +++ b/src/cutter/cutterBridgeMac.ts @@ -0,0 +1,16 @@ +import { createHmac } from "node:crypto"; + +/** + * SHA-256 over `parts` in order. + * + * Each part is updated on its own. The Cutter proof prefix plus an exact-limit + * request or response does not fit in one Node string. + */ +export const cutterBridgeMac = ( + token: string, + parts: readonly [string, ...string[]], +): Buffer => { + const mac = createHmac("sha256", token); + for (const part of parts) mac.update(part); + return mac.digest(); +}; diff --git a/src/hopper/BridgeLauncher.ts b/src/hopper/BridgeLauncher.ts index bbcac1bdba..c77707af95 100644 --- a/src/hopper/BridgeLauncher.ts +++ b/src/hopper/BridgeLauncher.ts @@ -47,6 +47,11 @@ import { acquireLinuxHopperApplicationLease, } from "../process/HopperLease.js"; import type { HopperTargetLease } from "../process/HopperLease.js"; +import { + MAXIMUM_BRIDGE_MAX_REQUEST_BYTES, + MINIMUM_BRIDGE_MAX_REQUEST_BYTES, + type BridgeMaxRequestBytes, +} from "../config/bridgeMaxRequestBytes.js"; import { HOPPER_SHUTDOWN_TIMEOUT_MS } from "./HopperSessionValues.js"; import { verifyMacHopperNativeHelper } from "./HopperMacPublisher.js"; import { inspectLinuxHopperLauncher } from "../application/LinuxHopper.js"; @@ -129,6 +134,8 @@ interface SharedHopperApplicationLauncherOptions { readonly targetKind: "executable" | "database"; readonly loaderArgs: readonly string[]; readonly bridgeScriptPath: string; + /** Caller-selected request ceiling. Omitted or source default leaves the bridge at 16 MiB. */ + readonly bridgeMaxRequestBytes?: BridgeMaxRequestBytes; readonly preparedImage?: { readonly image: HopperMachOImage; readonly sourceSha256: string; @@ -554,22 +561,33 @@ export class HopperApplicationLauncher implements BridgeLauncher { } } -const bridgeBootstrapSource = ( +/** Bootstrap assignments Hopper execs before the bridge module. Exported for the ceiling regression. */ +export const bridgeBootstrapSource = ( session: BridgeSession, options: SharedHopperApplicationLauncherOptions, ownsProcessLifetime: boolean, loadedPath: string, -): string => - [ +): string => { + const limit = options.bridgeMaxRequestBytes; + const injectedLimit = + limit?.source === "setting" && + Number.isSafeInteger(limit.maximumBytes) && + limit.maximumBytes >= MINIMUM_BRIDGE_MAX_REQUEST_BYTES && + limit.maximumBytes <= MAXIMUM_BRIDGE_MAX_REQUEST_BYTES + ? `REA_BRIDGE_MAX_REQUEST_BYTES = ${String(limit.maximumBytes)}` + : undefined; + return [ `REA_SOCKET = ${JSON.stringify(session.socketPath)}`, `REA_TOKEN = ${JSON.stringify(session.token)}`, `REA_RUN_ID = ${JSON.stringify(session.runId)}`, `REA_TARGET_PATH = ${JSON.stringify(loadedPath)}`, `REA_OWNS_PROCESS_LIFETIME = ${ownsProcessLifetime ? "True" : "False"}`, `REA_RECONNECT_TIMEOUT_SECONDS = ${HOPPER_SHUTDOWN_TIMEOUT_MS / 1_000}`, + ...(injectedLimit === undefined ? [] : [injectedLimit]), `exec(compile(open(${JSON.stringify(options.bridgeScriptPath)}, 'rb').read(), ${JSON.stringify(options.bridgeScriptPath)}, 'exec'))`, "", ].join("\n"); +}; const launchHopperProcess = async (input: { readonly options: HopperApplicationLauncherOptions; diff --git a/src/hopper/HopperClient.ts b/src/hopper/HopperClient.ts index 8159333b28..d6e3a1a187 100644 --- a/src/hopper/HopperClient.ts +++ b/src/hopper/HopperClient.ts @@ -33,6 +33,7 @@ import { import { err, ok, type Result } from "../domain/result.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import type { JsonValue } from "../domain/jsonValue.js"; +import { jsonValueByteLength } from "../domain/jsonValueByteLength.js"; import type { ProgressReporter } from "../application/ProgressReporter.js"; import { silentLogger } from "../logger.js"; import type { Logger } from "pino"; @@ -73,6 +74,12 @@ import { nativeHopperLaunchWasDenied, } from "./HopperProcessDiagnostic.js"; import { DEFAULT_HOPPER_STARTUP_TIMEOUT_MS } from "../config/providerStartupTimeout.js"; +import { + bridgeRequestLimitMismatchError, + bridgeRequestOversizeError, + DEFAULT_BRIDGE_MAX_REQUEST, + type BridgeMaxRequestBytes, +} from "../config/bridgeMaxRequestBytes.js"; import { HopperRequestQueue, type HopperRequestActivity, @@ -94,6 +101,8 @@ export interface HopperClientOptions { readonly launcher: BridgeLauncher; readonly runId?: string; readonly startupTimeoutMs?: number; + /** Effective request ceiling. Omitted uses the 16 MiB default. */ + readonly bridgeMaxRequestBytes?: BridgeMaxRequestBytes; readonly onDiagnostic?: (event: HopperDiagnostic) => void; readonly logger?: Logger; } @@ -122,7 +131,9 @@ type HopperClientCloseOptions = { * Hopper launcher does not confer ownership of the GUI application. */ export class HopperClient { - readonly #options: Required> & + readonly #options: Required< + Pick + > & HopperClientOptions; readonly #requests: HopperRequestQueue; readonly #responses: HopperResponseStream; @@ -197,6 +208,8 @@ export class HopperClient { ...options, startupTimeoutMs: options.startupTimeoutMs ?? DEFAULT_HOPPER_STARTUP_TIMEOUT_MS, + bridgeMaxRequestBytes: + options.bridgeMaxRequestBytes ?? DEFAULT_BRIDGE_MAX_REQUEST, }; this.#requests = new HopperRequestQueue( ({ id, method, params }, failed) => { @@ -206,12 +219,39 @@ export class HopperClient { failed(); return; } - socket.write( - `${JSON.stringify({ id, token, method, params })}\n`, - (cause) => { - if (cause !== undefined && cause !== null) failed(); - }, - ); + // The FIFO keeps the caller's params. The encoded frame exists only + // for the request that has reached the wire. + const payloadValue = { id, token, method, params }; + let payload: string; + try { + payload = JSON.stringify(payloadValue); + } catch (cause: unknown) { + this.#requests.accept( + id, + err( + new HopperProtocolError("Hopper socket write failed", { cause }), + ), + ); + return; + } + const payloadBytes = Buffer.byteLength(payload); + if (payloadBytes > this.#options.bridgeMaxRequestBytes.maximumBytes) { + this.#requests.accept( + id, + err( + bridgeRequestOversizeError( + method, + this.#options.bridgeMaxRequestBytes, + payloadBytes, + "Hopper", + ), + ), + ); + return; + } + socket.write(`${payload}\n`, (cause) => { + if (cause !== undefined && cause !== null) failed(); + }); }, ); this.#responses = new HopperResponseStream({ @@ -511,6 +551,18 @@ export class HopperClient { } const parsed = await this.#authenticateServerInfo(health.value, runId); if (!parsed.ok) return this.#startupFailure(parsed.error); + if ( + parsed.value.maxRequestBytes !== + this.#options.bridgeMaxRequestBytes.maximumBytes + ) + return this.#startupFailure( + bridgeRequestLimitMismatchError( + "hopper_bridge", + this.#options.bridgeMaxRequestBytes, + parsed.value.maxRequestBytes, + "Hopper", + ), + ); try { await this.#lineage.observe(this.#resources.launch); } catch (cause: unknown) { @@ -998,7 +1050,7 @@ export class HopperClient { readonly timeoutMs?: number; readonly progress?: ProgressReporter; }, - ): Promise> { + ): Promise> { const id = this.#nextId++; if ( this.#resources.launch?.providerLifetime === "launcher-process" && @@ -1032,6 +1084,23 @@ export class HopperClient { } if (options.signal?.aborted === true) return err(hopperCancelledBySignal(options.signal)); + const payloadValue = { id, token, method, params }; + // Count escaped JSON before the request can wait in the FIFO. A rejected + // payload is not serialized, and an admitted one is serialized only when + // it reaches the wire. + const measured = jsonValueByteLength( + payloadValue, + this.#options.bridgeMaxRequestBytes.maximumBytes, + ); + if (measured > this.#options.bridgeMaxRequestBytes.maximumBytes) + return err( + bridgeRequestOversizeError( + method, + this.#options.bridgeMaxRequestBytes, + measured, + "Hopper", + ), + ); const startedAt = performance.now(); const result = await this.#requests.run(id, method, params, { ...(options.signal === undefined ? {} : { signal: options.signal }), diff --git a/src/hopper/HopperProvider.ts b/src/hopper/HopperProvider.ts index 77a2dd7b26..ceca59c709 100644 --- a/src/hopper/HopperProvider.ts +++ b/src/hopper/HopperProvider.ts @@ -299,6 +299,7 @@ export class HopperProvider implements AnalysisProviderCandidate { bridgeScriptPath: fileURLToPath( new URL("../../bridge/hopper_bridge.py", import.meta.url), ), + bridgeMaxRequestBytes: this.config.bridgeMaxRequestBytes, ...(this.platform === "linux" ? { launchMode: "verified_linux_demo" as const, @@ -310,6 +311,7 @@ export class HopperProvider implements AnalysisProviderCandidate { }), ...(context === undefined ? {} : { runId: context.runId }), startupTimeoutMs: this.config.hopperStartupTimeoutMs, + bridgeMaxRequestBytes: this.config.bridgeMaxRequestBytes, logger: this.logger.child({ layer: "bridge" }), onDiagnostic: (diagnostic) => this.logger.info(diagnostic, "Hopper bridge reported a diagnostic"), diff --git a/src/hopper/HopperRequestQueue.test.ts b/src/hopper/HopperRequestQueue.test.ts index 2cc32e9785..6e2aa6a950 100644 --- a/src/hopper/HopperRequestQueue.test.ts +++ b/src/hopper/HopperRequestQueue.test.ts @@ -352,3 +352,17 @@ describe("HopperRequestQueue caller abort reasons", () => { expect(sent).toEqual([1]); }); }); + +it("does not hand a waiting request to the sender until it reaches the wire", async () => { + const sent: number[] = []; + const queue = new HopperRequestQueue(({ id }) => sent.push(id)); + const active = queue.run(1, "active", { value: "A" }, {}); + const queued = queue.run(2, "queued", { value: "B" }, {}); + expect(sent).toEqual([1]); + expect(queue.queuedCount()).toBe(1); + queue.accept(1, ok(null)); + expect(sent).toEqual([1, 2]); + queue.accept(2, ok(null)); + await active; + await queued; +}); diff --git a/src/hopper/HopperRequestQueue.ts b/src/hopper/HopperRequestQueue.ts index 9dc9fb500f..f5474fc142 100644 --- a/src/hopper/HopperRequestQueue.ts +++ b/src/hopper/HopperRequestQueue.ts @@ -272,7 +272,11 @@ export class HopperRequestQueue { try { entry.transmitted = true; this.send( - { id: entry.id, method: entry.method, params: entry.params }, + { + id: entry.id, + method: entry.method, + params: entry.params, + }, () => this.accept( entry.id, diff --git a/src/hopper/HopperSessionValues.test.ts b/src/hopper/HopperSessionValues.test.ts new file mode 100644 index 0000000000..b692085115 --- /dev/null +++ b/src/hopper/HopperSessionValues.test.ts @@ -0,0 +1,47 @@ +import { expect, it } from "vitest"; + +import { DEFAULT_BRIDGE_MAX_REQUEST_BYTES } from "../config/bridgeMaxRequestBytes.js"; +import { parseHopperServerInfo } from "./HopperSessionValues.js"; + +const health = { + name: "REA Hopper bridge", + version: "1.0.0", + process_id: 42, + run_id: "run-1", +}; + +it("treats an omitted request ceiling as the 16 MiB default", () => { + const parsed = parseHopperServerInfo(health, "run-1"); + expect(parsed).toEqual({ + ok: true, + value: { + name: "REA Hopper bridge", + version: "1.0.0", + processId: 42, + maxRequestBytes: DEFAULT_BRIDGE_MAX_REQUEST_BYTES, + }, + }); +}); + +it("keeps an advertised ceiling that differs from the client default", () => { + const parsed = parseHopperServerInfo( + { ...health, max_request_bytes: 8192 }, + "run-1", + ); + expect(parsed.ok && parsed.value.maxRequestBytes).toBe(8192); +}); + +it.each([0, -1, 1.5, "16777216", true])( + "rejects an unusable advertised ceiling %j", + (max_request_bytes) => { + const parsed = parseHopperServerInfo( + { ...health, max_request_bytes }, + "run-1", + ); + expect(parsed.ok).toBe(false); + if (!parsed.ok) + expect(parsed.error.message).toBe( + "Hopper bridge health result is invalid", + ); + }, +); diff --git a/src/hopper/HopperSessionValues.ts b/src/hopper/HopperSessionValues.ts index abee06e047..477060081b 100644 --- a/src/hopper/HopperSessionValues.ts +++ b/src/hopper/HopperSessionValues.ts @@ -1,3 +1,4 @@ +import { DEFAULT_BRIDGE_MAX_REQUEST_BYTES } from "../config/bridgeMaxRequestBytes.js"; import { HopperProtocolError } from "../domain/hopperErrors.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { err, ok, type Result } from "../domain/result.js"; @@ -10,6 +11,8 @@ export interface HopperServerInfo { readonly name: string; readonly version: string; readonly processId: number; + /** Bridge request ceiling. An omitted health field means the 16 MiB default. */ + readonly maxRequestBytes: number; } /** Validate the bridge identity and exact per-launch run identifier. */ @@ -27,12 +30,25 @@ export const parseHopperServerInfo = ( Number.isSafeInteger(value.process_id) && value.process_id > 0 && value.run_id === expectedRunId - ) + ) { + const advertised = value.max_request_bytes; + const maxRequestBytes = + advertised === undefined ? DEFAULT_BRIDGE_MAX_REQUEST_BYTES : advertised; + if ( + typeof maxRequestBytes !== "number" || + !Number.isSafeInteger(maxRequestBytes) || + maxRequestBytes <= 0 + ) + return err( + new HopperProtocolError("Hopper bridge health result is invalid"), + ); return ok({ name: value.name, version: value.version, processId: value.process_id, + maxRequestBytes, }); + } return err(new HopperProtocolError("Hopper bridge health result is invalid")); }; diff --git a/src/hopper/bridgeBootstrapLimit.test.ts b/src/hopper/bridgeBootstrapLimit.test.ts new file mode 100644 index 0000000000..5429b09215 --- /dev/null +++ b/src/hopper/bridgeBootstrapLimit.test.ts @@ -0,0 +1,39 @@ +import { expect, it } from "vitest"; + +import { bridgeBootstrapSource } from "./BridgeLauncher.js"; + +const session = { + directory: "/tmp/rea-bootstrap", + socketPath: "/tmp/rea-bootstrap/bridge.sock", + token: "token", + runId: "run", +}; +const options = { + environment: {}, + launcherPath: "/hopper", + targetPath: "/target", + targetKind: "executable" as const, + loaderArgs: [], + bridgeScriptPath: "/bridge.py", +}; + +it("leaves an unset ceiling out of the Hopper bootstrap", () => { + const source = bridgeBootstrapSource(session, options, true, "/target"); + expect(source).not.toContain("REA_BRIDGE_MAX_REQUEST_BYTES"); + expect(source).toContain("exec(compile"); +}); + +it("injects a caller-selected ceiling before the bridge is executed", () => { + const source = bridgeBootstrapSource( + session, + { + ...options, + bridgeMaxRequestBytes: { maximumBytes: 8192, source: "setting" }, + }, + true, + "/target", + ); + const injected = source.indexOf("REA_BRIDGE_MAX_REQUEST_BYTES = 8192"); + expect(injected).toBeGreaterThanOrEqual(0); + expect(injected).toBeLessThan(source.indexOf("exec(compile")); +}); diff --git a/src/setupEnvironment.test.ts b/src/setupEnvironment.test.ts index 02efabcfbf..21ebeb268c 100644 --- a/src/setupEnvironment.test.ts +++ b/src/setupEnvironment.test.ts @@ -182,6 +182,7 @@ it.each([ name: "registration-only settings", environment: { REA_ANALYSIS_PROVIDER: "auto", + REA_BRIDGE_MAX_REQUEST_BYTES: "33554432", REA_GHIDRA_STARTUP_TIMEOUT_MS: "2700000", REA_HOPPER_STARTUP_TIMEOUT_MS: "120000", REA_MCP_INPUT_SCHEMA_PROFILE: "full", @@ -221,6 +222,7 @@ it.each([ if ("REA_MCP_INPUT_SCHEMA_PROFILE" in environment) { expect(initial.providerEnvironment).toMatchObject({ REA_ANALYSIS_PROVIDER: "auto", + REA_BRIDGE_MAX_REQUEST_BYTES: "33554432", REA_GHIDRA_STARTUP_TIMEOUT_MS: "2700000", REA_HOPPER_STARTUP_TIMEOUT_MS: "120000", REA_MCP_INPUT_SCHEMA_PROFILE: "full", @@ -233,6 +235,9 @@ it.each([ REA_MCP_DOCTOR_DEADLINE_MS: "31000", }); } else { + expect(initial.providerEnvironment).not.toHaveProperty( + "REA_BRIDGE_MAX_REQUEST_BYTES", + ); expect(initial.providerEnvironment).not.toHaveProperty( "REA_REGEX_MATCHING_DEADLINE_MS", ); diff --git a/tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts b/tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts index 57e5b85a6d..3e1af5804e 100644 --- a/tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts +++ b/tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts @@ -482,6 +482,7 @@ cliTest( KIMI_CODE_HOME: kimiHome, ...missingHopperAndGhidraEnvironment(home), REA_ANALYSIS_PROVIDER: "auto", + REA_BRIDGE_MAX_REQUEST_BYTES: "33554432", REA_GHIDRA_STARTUP_TIMEOUT_MS: "2700001", REA_HOPPER_STARTUP_TIMEOUT_MS: "2800002", REA_JEB_MCP_TIMEOUT_MS: "2900003", @@ -520,6 +521,7 @@ cliTest( "rea", ); expect(codexRegistration).toMatchObject({ tool_timeout_sec: 3021 }); + expect(codexConfig).toContain('REA_BRIDGE_MAX_REQUEST_BYTES = "33554432"'); expect(codexConfig).toContain('REA_JEB_MCP_TIMEOUT_MS = "2900003"'); expect(codexConfig).toContain( 'REA_CAPTURE_DECODER_OLD_GENERATION_MIB = "256"', @@ -541,6 +543,7 @@ cliTest( toolTimeoutMs: 3_020_003, env: { REA_ANALYSIS_PROVIDER: "auto", + REA_BRIDGE_MAX_REQUEST_BYTES: "33554432", REA_GHIDRA_STARTUP_TIMEOUT_MS: "2700001", REA_HOPPER_STARTUP_TIMEOUT_MS: "2800002", REA_JEB_MCP_TIMEOUT_MS: "2900003", diff --git a/tests/boundary/filesystem/clientConfigurationEnvironment.test.ts b/tests/boundary/filesystem/clientConfigurationEnvironment.test.ts index 2c2f58e36e..35132ecc74 100644 --- a/tests/boundary/filesystem/clientConfigurationEnvironment.test.ts +++ b/tests/boundary/filesystem/clientConfigurationEnvironment.test.ts @@ -506,6 +506,16 @@ it.each([ { REA_REGEX_WORKER_HEAP_REQUEST_MIB: "6" }, "REA_REGEX_WORKER_HEAP_REQUEST_MIB", ], + [ + "bridge request ceiling below its floor", + { REA_BRIDGE_MAX_REQUEST_BYTES: "7463" }, + "REA_BRIDGE_MAX_REQUEST_BYTES", + ], + [ + "bridge request ceiling above its maximum", + { REA_BRIDGE_MAX_REQUEST_BYTES: "536870888" }, + "REA_BRIDGE_MAX_REQUEST_BYTES", + ], [ "MCP initialize budget of zero", { REA_MCP_INITIALIZE_BUDGET_MS: "0" }, diff --git a/tests/boundary/providers/CutterBridgeClient.test.ts b/tests/boundary/providers/CutterBridgeClient.test.ts index 6c5bd40892..539893a940 100644 --- a/tests/boundary/providers/CutterBridgeClient.test.ts +++ b/tests/boundary/providers/CutterBridgeClient.test.ts @@ -919,18 +919,22 @@ describe("CutterBridgeClient: scenario group 6", () => { port, "synthetic-cutter-bridge-token-for-test", ); - await expect( - createClient({ REA_CUTTER_BRIDGE_DIR: directory }).execute({ - sessionId, - expectedGeneration: 0, - command: "pd 1", - json: false, - }), - ).resolves.toMatchObject({ + const execution = await createClient({ + REA_CUTTER_BRIDGE_DIR: directory, + }).execute({ + sessionId, + expectedGeneration: 0, + command: "pd 1", + json: false, + }); + expect(execution).toMatchObject({ executionState: "unknown", error: "transport-response-missing", - message: expect.stringContaining("may have completed"), + message: expect.stringMatching( + /framed transport limit of 16777216 bytes.*REA_BRIDGE_MAX_REQUEST_BYTES.*may have completed/s, + ), }); + expect(execution.message).not.toContain("no runtime override"); }); it("rejects stale sessions and preserves failures before command dispatch", async () => { diff --git a/tests/boundary/providers/cutterBridgeProducer.test.ts b/tests/boundary/providers/cutterBridgeProducer.test.ts index a9a06c283a..721ffc05a1 100644 --- a/tests/boundary/providers/cutterBridgeProducer.test.ts +++ b/tests/boundary/providers/cutterBridgeProducer.test.ts @@ -100,7 +100,7 @@ it("authenticates bridge peers before payloads and permits status recovery after [resolve("tests/fixtures/cutter_bridge_protocol_test.py")], { encoding: "utf8", - timeout: 10_000, + timeout: 20_000, }, ); }); diff --git a/tests/boundary/providers/cutterBridgeRequestLimit.test.ts b/tests/boundary/providers/cutterBridgeRequestLimit.test.ts new file mode 100644 index 0000000000..4baec0470b --- /dev/null +++ b/tests/boundary/providers/cutterBridgeRequestLimit.test.ts @@ -0,0 +1,466 @@ +import { createServer } from "node:net"; +import { chmod, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { expect, it, onTestFinished, vi } from "vitest"; + +import { CutterBridgeClient } from "../../../src/cutter/CutterBridgeClient.js"; +import { CutterBridgeService } from "../../../src/cutter/CutterBridgeService.js"; +import { ConfigurationError } from "../../../src/domain/configurationErrors.js"; +import { AnalysisResourceConstraintError } from "../../../src/domain/analysisErrorCore.js"; +import { + authenticatedServer, + writeDescriptor, +} from "../../fixtures/cutterBridgeProtocol.js"; + +const SESSION = "00000000-0000-4000-8000-000000000001"; +const TOKEN = "bridge-token-0123456789abcdef"; + +const privateDirectory = async (): Promise => { + const directory = await mkdtemp(join(tmpdir(), "rea-cutter-request-limit-")); + await chmod(directory, 0o700); + onTestFinished(async () => { + await rm(directory, { recursive: true, force: true }); + }); + return directory; +}; + +const listen = async ( + server: ReturnType, +): Promise => { + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (address === null || typeof address === "string") + throw new Error("Expected an ephemeral IPv4 listener"); + onTestFinished( + () => + new Promise((resolve) => { + server.close(() => resolve()); + }), + ); + return address.port; +}; + +const clientFor = (directory: string, ceiling: string): CutterBridgeClient => { + const client = new CutterBridgeClient({ + REA_CUTTER_BRIDGE_DIR: directory, + REA_BRIDGE_MAX_REQUEST_BYTES: ceiling, + }); + onTestFinished(() => client.close()); + return client; +}; + +it("rejects a bridge ceiling below the floor at client construction", () => { + expect( + () => + new CutterBridgeClient({ + REA_BRIDGE_MAX_REQUEST_BYTES: "7463", + }), + ).toThrow(ConfigurationError); +}); + +const closedPort = async (): Promise => { + const server = createServer(); + const port = await listen(server); + await new Promise((resolve) => { + server.close(() => resolve()); + }); + return port; +}; + +const ceilingMismatch = (error: unknown): boolean => { + expect(error).toBeInstanceOf(AnalysisResourceConstraintError); + if (!(error instanceof AnalysisResourceConstraintError)) return false; + expect(error.operation).toBe("list_cutter_sessions"); + expect(error.resource).toBe("transport"); + expect(error.reportedLimits).toMatchObject({ + bridge_max_request_bytes: 16_777_216, + client_max_request_bytes: 8192, + }); + expect(error.reason).toContain( + "No status request was sent to the mismatched session.", + ); + expect(error.remediationAction).toContain("Set REA_BRIDGE_MAX_REQUEST_BYTES"); + return true; +}; + +it("ignores a stale Cutter descriptor whose advertised ceiling differs", async () => { + const directory = await privateDirectory(); + await writeDescriptor(directory, SESSION, await closedPort(), TOKEN); + const client = clientFor(directory, "8192"); + await expect(client.listSessions()).resolves.toMatchObject({ + sessions: [], + discovery_status: "no_live_bridge_found", + }); +}); + +it("returns the matching session when the other ceiling descriptor is stale", async () => { + const directory = await privateDirectory(); + const server = authenticatedServer(TOKEN, SESSION, () => undefined); + const port = await listen(server); + const other = "00000000-0000-4000-8000-000000000002"; + await writeDescriptor(directory, SESSION, port, TOKEN, { + maxRequestBytes: 8192, + }); + await writeDescriptor(directory, other, await closedPort(), TOKEN); + const client = clientFor(directory, "8192"); + await expect(client.listSessions()).resolves.toMatchObject({ + discovery_status: "sessions_found", + sessions: [{ session_id: SESSION }], + }); +}); + +it("fails discovery when a live authenticated session has a different ceiling", async () => { + const directory = await privateDirectory(); + const server = authenticatedServer(TOKEN, SESSION, () => undefined); + const port = await listen(server); + const other = "00000000-0000-4000-8000-000000000002"; + const requests: string[] = []; + let connections = 0; + const mismatched = authenticatedServer( + TOKEN, + other, + (request) => { + requests.push(String(request.kind)); + }, + true, + ); + mismatched.on("connection", () => { + connections += 1; + }); + const mismatchedPort = await listen(mismatched); + await writeDescriptor(directory, SESSION, port, TOKEN, { + maxRequestBytes: 8192, + }); + await writeDescriptor(directory, other, mismatchedPort, TOKEN); + const client = clientFor(directory, "8192"); + await expect(client.listSessions()).rejects.toSatisfy(ceilingMismatch); + expect(connections).toBe(1); + expect(requests).toEqual([]); +}); + +it("does not treat an unauthenticated listener as a Cutter ceiling mismatch", async () => { + const directory = await privateDirectory(); + const server = authenticatedServer(TOKEN, SESSION, () => undefined); + const port = await listen(server); + const other = "00000000-0000-4000-8000-000000000002"; + const stranger = createServer((socket) => { + socket.write( + `${JSON.stringify({ + session_id: other, + nonce: "ab".repeat(32), + proof: "00".repeat(32), + })}\n`, + ); + socket.destroy(); + }); + const strangerPort = await listen(stranger); + await writeDescriptor(directory, SESSION, port, TOKEN, { + maxRequestBytes: 8192, + }); + await writeDescriptor(directory, other, strangerPort, TOKEN); + const client = clientFor(directory, "8192"); + await expect(client.listSessions()).resolves.toMatchObject({ + discovery_status: "sessions_found", + sessions: [{ session_id: SESSION }], + }); +}); + +it("reports a Cutter ceiling mismatch before opening a command socket", async () => { + const directory = await privateDirectory(); + await writeDescriptor(directory, SESSION, 1, TOKEN); + const client = clientFor(directory, "33554432"); + await expect( + client.execute({ + sessionId: SESSION, + expectedGeneration: 0, + command: "ij", + json: false, + }), + ).rejects.toSatisfy((error: unknown) => { + expect(error).toBeInstanceOf(AnalysisResourceConstraintError); + if (!(error instanceof AnalysisResourceConstraintError)) return false; + expect(error.reason).toContain("REA_BRIDGE_MAX_REQUEST_BYTES"); + expect(error.reason).toContain("33554432"); + expect(error.reason).toContain("16777216"); + expect(error.reason).toContain("No command was sent."); + expect(error.reason).not.toContain("no runtime override"); + expect(error.remediationAction).toContain( + "Set REA_BRIDGE_MAX_REQUEST_BYTES", + ); + return true; + }); +}); + +it("lowers the Cutter command ceiling for both the refusal and a live command", async () => { + const directory = await privateDirectory(); + let connections = 0; + const server = authenticatedServer(TOKEN, SESSION, (request, respond) => { + respond({ + ok: true, + session_id: SESSION, + output: request.command, + execution_state: "complete", + document_generation: 0, + identity_status: "partial", + }); + }); + server.on("connection", () => { + connections += 1; + }); + const port = await listen(server); + await writeDescriptor(directory, SESSION, port, TOKEN, { + maxRequestBytes: 8192, + }); + const client = clientFor(directory, "8192"); + await expect( + client.execute({ + sessionId: SESSION, + expectedGeneration: 0, + command: "c".repeat(8000), + json: false, + }), + ).rejects.toSatisfy((error: unknown) => { + expect(error).toBeInstanceOf(AnalysisResourceConstraintError); + if (!(error instanceof AnalysisResourceConstraintError)) return false; + expect(error.reason).toContain("authenticated frame limit of 8192 bytes"); + expect(error.reason).toContain("no command was sent"); + expect(error.reason).toContain("REA_BRIDGE_MAX_REQUEST_BYTES"); + expect(error.reason).not.toContain("no runtime override"); + return true; + }); + expect(connections).toBe(0); + await expect( + client.execute({ + sessionId: SESSION, + expectedGeneration: 0, + command: "ij", + json: false, + }), + ).resolves.toMatchObject({ + output: "ij", + executionState: "complete", + }); +}); + +const responseEnvelopeBytes = (output: string): number => { + const response = JSON.stringify({ + ok: true, + output, + execution_state: "complete", + document_generation: 0, + identity_status: "partial", + }); + return Buffer.byteLength(JSON.stringify({ response, proof: "0".repeat(64) })); +}; + +const outputForEnvelope = (target: number): string => { + const count = target - responseEnvelopeBytes(""); + if (responseEnvelopeBytes("A".repeat(count)) !== target) + throw new Error(`expected an exact ${String(target)}-byte response frame`); + return "A".repeat(count); +}; + +it("refuses an escaped Cutter command before serializing it", async () => { + const directory = await privateDirectory(); + let connections = 0; + const server = authenticatedServer(TOKEN, SESSION, (_request, respond) => { + respond({ + ok: true, + output: null, + execution_state: "complete", + document_generation: 0, + identity_status: "partial", + }); + }); + server.on("connection", () => { + connections += 1; + }); + const port = await listen(server); + await writeDescriptor(directory, SESSION, port, TOKEN, { + maxRequestBytes: 8192, + }); + const client = clientFor(directory, "8192"); + const command = "\u0001".repeat(2000); + const stringify = vi.spyOn(JSON, "stringify"); + try { + await expect( + client.execute({ + sessionId: SESSION, + expectedGeneration: 0, + command, + json: false, + }), + ).rejects.toSatisfy((error: unknown) => { + expect(error).toBeInstanceOf(AnalysisResourceConstraintError); + if (!(error instanceof AnalysisResourceConstraintError)) return false; + expect(error.reason).toContain("authenticated frame limit of 8192 bytes"); + expect(error.reason).toContain("no command was sent"); + expect(error.reason).toContain("REA_BRIDGE_MAX_REQUEST_BYTES"); + return true; + }); + expect(connections).toBe(0); + expect( + stringify.mock.calls.some((call) => { + const value: unknown = call[0]; + return ( + typeof value === "object" && + value !== null && + "command" in value && + value.command === command + ); + }), + ).toBe(false); + } finally { + stringify.mockRestore(); + } +}); + +it("accepts a Cutter response at the ceiling and excludes the terminating LF", async () => { + const directory = await privateDirectory(); + const ceiling = 8192; + const output = outputForEnvelope(ceiling); + const server = authenticatedServer(TOKEN, SESSION, (request, respond) => { + if (request.kind === "ack") { + respond({ ok: true, acknowledged: true }); + return; + } + const command = request.command; + const body = { + ok: true, + output: command === "over" ? `${output}A` : output, + execution_state: "complete", + document_generation: 0, + identity_status: "partial", + }; + if (command === "split") respond(body, { chunkBytes: ceiling }); + else respond(body); + }); + const port = await listen(server); + await writeDescriptor(directory, SESSION, port, TOKEN, { + maxRequestBytes: ceiling, + }); + const client = clientFor(directory, String(ceiling)); + await expect( + client.execute({ + sessionId: SESSION, + expectedGeneration: 0, + command: "exact", + json: false, + }), + ).resolves.toMatchObject({ + output, + executionState: "complete", + }); + await expect( + client.execute({ + sessionId: SESSION, + expectedGeneration: 0, + command: "split", + json: false, + }), + ).resolves.toMatchObject({ + output, + executionState: "complete", + }); + const oversize = await new CutterBridgeService(client).execute({ + session_id: SESSION, + expected_generation: 0, + command: "over", + json: false, + }); + expect(oversize).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisOutputError", + partialObservation: { + normalized_result: { + execution_state: "unknown", + error: "transport-response-missing", + output_truncated: true, + message: expect.stringContaining("8192"), + }, + limitations: expect.arrayContaining([ + "The command completed or may have completed, but output exceeded the bridge transport limit and was omitted.", + ]), + }, + }, + }); + expect(oversize).toMatchObject({ + error: { + partialObservation: { + normalized_result: { + message: expect.stringContaining("REA_BRIDGE_MAX_REQUEST_BYTES"), + }, + }, + }, + }); + expect(oversize).toMatchObject({ + error: { + partialObservation: { + normalized_result: { + message: expect.stringContaining("may have completed"), + }, + }, + }, + }); + expect(oversize).toMatchObject({ + error: { + partialObservation: { + normalized_result: { + message: expect.not.stringContaining("no runtime override"), + }, + }, + }, + }); +}); + +it("sends a Cutter command above 16 MiB when both sides raise the ceiling", async () => { + const directory = await privateDirectory(); + const ceiling = 17 * 1024 * 1024; + const command = "h".repeat(16 * 1024 * 1024); + const server = authenticatedServer(TOKEN, SESSION, (request, respond) => { + respond({ + ok: true, + session_id: SESSION, + output: typeof request.command === "string" ? request.command.length : 0, + execution_state: "complete", + document_generation: 0, + identity_status: "partial", + }); + }); + const port = await listen(server); + await writeDescriptor(directory, SESSION, port, TOKEN, { + maxRequestBytes: ceiling, + }); + const raised = clientFor(directory, String(ceiling)); + await expect( + raised.execute({ + sessionId: SESSION, + expectedGeneration: 0, + command, + json: false, + }), + ).resolves.toMatchObject({ + output: command.length, + executionState: "complete", + }); + + const directoryForDefault = await privateDirectory(); + await writeDescriptor(directoryForDefault, SESSION, port, TOKEN, { + maxRequestBytes: ceiling, + }); + const defaults = new CutterBridgeClient({ + REA_CUTTER_BRIDGE_DIR: directoryForDefault, + }); + onTestFinished(() => defaults.close()); + await expect( + defaults.execute({ + sessionId: SESSION, + expectedGeneration: 0, + command: "ij", + json: false, + }), + ).rejects.toBeInstanceOf(AnalysisResourceConstraintError); +}); diff --git a/tests/boundary/providers/hopper/hopperBridgeFrameLimit.test.ts b/tests/boundary/providers/hopper/hopperBridgeFrameLimit.test.ts index f07264b867..e91a9431a0 100644 --- a/tests/boundary/providers/hopper/hopperBridgeFrameLimit.test.ts +++ b/tests/boundary/providers/hopper/hopperBridgeFrameLimit.test.ts @@ -28,6 +28,9 @@ const exchangeSchema = z.strictObject({ const probeResultSchema = z.strictObject({ limit: z.number().int().positive(), + source: z.enum(["default", "setting"]), + minimum: z.literal(7464), + maximum: z.literal(536870887), legit: exchangeSchema, at_limit: exchangeSchema, past_limit: exchangeSchema, @@ -42,6 +45,7 @@ it("serves requests up to the frame limit and refuses a line that exceeds it", a ]); const result = probeResultSchema.parse(JSON.parse(stdout)); expect(result.limit).toBe(MAX_REQUEST_BYTES); + expect(result.source).toBe("default"); // Legitimate traffic is unaffected, including a request that fills the limit. for (const exchange of [result.legit, result.at_limit]) @@ -68,6 +72,11 @@ it("serves requests up to the frame limit and refuses a line that exceeds it", a message: expect.stringContaining(String(MAX_REQUEST_BYTES)), }, }); + expect(result.stalled_line.reply).toMatchObject({ + error: { + message: expect.stringContaining("REA_BRIDGE_MAX_REQUEST_BYTES"), + }, + }); expect(result.stalled_line.next).toEqual({ closed: true }); expect(result.stalled_line.thread_released).toBe(true); @@ -85,3 +94,51 @@ it("serves requests up to the frame limit and refuses a line that exceeds it", a expect(result.eof_partial.dispatch_calls).toBe(0); expect(result.eof_partial.thread_released).toBe(true); }); + +const runProbe = async (...arguments_: readonly string[]): Promise => { + const { stdout } = await execute("python3", [ + fileURLToPath(probePath), + fileURLToPath(bridgePath), + ...arguments_, + ]); + return JSON.parse(stdout) as unknown; +}; + +it.each([8192, 17 * 1024 * 1024])( + "serves a caller-selected ceiling of %d bytes", + async (limit) => { + const result = probeResultSchema.parse( + await runProbe("bytes", String(limit)), + ); + expect(result.limit).toBe(limit); + expect(result.source).toBe("setting"); + expect(result.at_limit.reply).toMatchObject({ + result: { run_id: "owned-run", ready: true }, + }); + expect(result.stalled_line.reply).toMatchObject({ + error: { + type: "invalid_request", + message: expect.stringContaining("REA_BRIDGE_MAX_REQUEST_BYTES"), + }, + }); + expect(result.eof_partial.reply).toMatchObject({ + error: { message: "Incomplete bridge request frame" }, + }); + }, +); + +it("prefers the injected bootstrap ceiling over the process environment", async () => { + const result = probeResultSchema.parse(await runProbe("inject", "8192")); + expect(result).toMatchObject({ limit: 8192, source: "setting" }); +}); + +it.each(["7463", "536870888", "0", "-1", "abc", "016777216", "٨١٩٢"])( + "refuses invalid bridge ceiling %j without falling back", + async (value) => { + const result = z + .object({ error: z.string() }) + .parse(await runProbe("invalid", value)); + expect(result.error).toContain("REA_BRIDGE_MAX_REQUEST_BYTES"); + expect(result.error).not.toContain("was accepted"); + }, +); diff --git a/tests/boundary/providers/hopper/hopperBridgeRequestLimit.test.ts b/tests/boundary/providers/hopper/hopperBridgeRequestLimit.test.ts new file mode 100644 index 0000000000..dc1756d4a6 --- /dev/null +++ b/tests/boundary/providers/hopper/hopperBridgeRequestLimit.test.ts @@ -0,0 +1,216 @@ +import { expect, it, vi } from "vitest"; + +import { AnalysisResourceConstraintError } from "../../../../src/domain/analysisErrorCore.js"; +import { HopperClient } from "../../../../src/hopper/HopperClient.js"; +import { + HopperFixtureLauncher, + trackHopperClient, +} from "./hopperClient.fixture.js"; + +const limit = (maximumBytes: number) => + ({ maximumBytes, source: "setting" }) as const; + +it("reports a Hopper ceiling mismatch and releases the fixture", async () => { + const launcher = new HopperFixtureLauncher(); + const client = trackHopperClient( + new HopperClient({ + launcher, + startupTimeoutMs: 10_000, + bridgeMaxRequestBytes: limit(1_048_576), + }), + ); + const started = await client.start(); + expect(started.ok).toBe(false); + if (started.ok) return; + expect(started.error).toBeInstanceOf(AnalysisResourceConstraintError); + if (!(started.error instanceof AnalysisResourceConstraintError)) return; + expect(started.error.reason).toContain("REA_BRIDGE_MAX_REQUEST_BYTES"); + expect(started.error.reason).toContain("1048576"); + expect(started.error.reason).toContain("16777216"); + expect(started.error.reason).toContain("source setting"); + expect(started.error.remediationAction).toContain( + "Set REA_BRIDGE_MAX_REQUEST_BYTES", + ); + expect(started.error.reason).not.toContain("no runtime override"); + const process = launcher.processes[0]; + expect(process?.exitCode !== null || process?.signalCode !== null).toBe(true); +}); + +it("refuses an oversized Hopper request before the fixture receives it", async () => { + const launcher = new HopperFixtureLauncher( + undefined, + "quiet", + "ready", + "acknowledge", + "none", + 8192, + ); + const client = trackHopperClient( + new HopperClient({ + launcher, + startupTimeoutMs: 10_000, + bridgeMaxRequestBytes: limit(8192), + }), + ); + expect((await client.start()).ok).toBe(true); + const refused = await client.callTool("set_address_name", { + address: "0x1", + name: "n".repeat(9000), + }); + expect(refused.ok).toBe(false); + if (refused.ok) return; + expect(refused.error).toBeInstanceOf(AnalysisResourceConstraintError); + if (!(refused.error instanceof AnalysisResourceConstraintError)) return; + expect(refused.error.reason).toContain("8192"); + expect(refused.error.reason).toContain("no request was sent"); + expect(refused.error.reason).toContain("REA_BRIDGE_MAX_REQUEST_BYTES"); + expect(launcher.requests.map(({ method }) => method)).not.toContain( + "set_address_name", + ); +}); + +it("refuses a Hopper payload whose JSON expansion exceeds the ceiling before serializing it", async () => { + const launcher = new HopperFixtureLauncher( + undefined, + "quiet", + "ready", + "acknowledge", + "none", + 8192, + ); + const client = trackHopperClient( + new HopperClient({ + launcher, + startupTimeoutMs: 10_000, + bridgeMaxRequestBytes: limit(8192), + }), + ); + expect((await client.start()).ok).toBe(true); + const name = "\u0001".repeat(2000); + const stringify = vi.spyOn(JSON, "stringify"); + try { + const refused = await client.callTool("set_address_name", { + address: "0x1", + name, + }); + expect(refused.ok).toBe(false); + if (refused.ok) return; + expect(refused.error).toBeInstanceOf(AnalysisResourceConstraintError); + if (!(refused.error instanceof AnalysisResourceConstraintError)) return; + expect(refused.error.reason).toContain("8192"); + expect(refused.error.reason).toContain("no request was sent"); + expect(refused.error.reason).toContain("REA_BRIDGE_MAX_REQUEST_BYTES"); + expect(launcher.requests.map(({ method }) => method)).not.toContain( + "set_address_name", + ); + expect( + stringify.mock.calls.some((call) => { + const value: unknown = call[0]; + if (typeof value !== "object" || value === null || !("params" in value)) + return false; + const params = value.params; + return ( + typeof params === "object" && + params !== null && + "name" in params && + params.name === name + ); + }), + ).toBe(false); + } finally { + stringify.mockRestore(); + } +}); + +const serializedPayload = ( + calls: readonly unknown[][], + value: string, +): boolean => + calls.some((call) => { + const candidate: unknown = call[0]; + if ( + typeof candidate !== "object" || + candidate === null || + !("params" in candidate) + ) + return false; + const params = candidate.params; + return ( + typeof params === "object" && + params !== null && + "value" in params && + params.value === value + ); + }); + +it("serializes a queued Hopper request only when it reaches the wire", async () => { + const launcher = new HopperFixtureLauncher(); + const client = trackHopperClient( + new HopperClient({ + launcher, + startupTimeoutMs: 10_000, + }), + ); + expect((await client.start()).ok).toBe(true); + const active = client.callTool("echo", { gate: "hold", value: "active" }); + const held = await launcher.waitForRequest("echo", "hold"); + const value = "queued-payload"; + const stringify = vi.spyOn(JSON, "stringify"); + let queuedNotice = false; + const queued = client.callTool( + "echo", + { value }, + { + progress: { + report: (update) => { + if (update.message.includes("queued behind")) queuedNotice = true; + return Promise.resolve(); + }, + }, + }, + ); + await vi.waitFor( + () => { + if (!queuedNotice) throw new Error("echo is not queued yet"); + }, + { timeout: 5_000 }, + ); + expect(serializedPayload(stringify.mock.calls, value)).toBe(false); + await launcher.release(held); + await expect(active).resolves.toMatchObject({ + ok: true, + value: { gate: "hold", value: "active" }, + }); + await expect(queued).resolves.toEqual({ ok: true, value: { value } }); + expect(serializedPayload(stringify.mock.calls, value)).toBe(true); + stringify.mockRestore(); +}); + +it("sends a Hopper request above 16 MiB when both sides raise the ceiling", async () => { + const ceiling = 17 * 1024 * 1024; + const launcher = new HopperFixtureLauncher( + undefined, + "quiet", + "ready", + "acknowledge", + "none", + ceiling, + ); + const client = trackHopperClient( + new HopperClient({ + launcher, + startupTimeoutMs: 10_000, + bridgeMaxRequestBytes: limit(ceiling), + }), + ); + expect((await client.start()).ok).toBe(true); + const value = "h".repeat(16 * 1024 * 1024); + // The fixture echoes the payload. That reply is the proof the pre-dispatch + // check admitted a request above the historical 16 MiB ceiling. The fixture's + // IPC acknowledgement is not used here; a 16 MiB params object is not a + // reliable control message. + await expect(client.callTool("echo", { value })).resolves.toEqual({ + ok: true, + value: { value }, + }); +}); diff --git a/tests/boundary/providers/hopper/hopperClient.fixture.ts b/tests/boundary/providers/hopper/hopperClient.fixture.ts index 9a009b45e0..0c2a464ef1 100644 --- a/tests/boundary/providers/hopper/hopperClient.fixture.ts +++ b/tests/boundary/providers/hopper/hopperClient.fixture.ts @@ -41,6 +41,7 @@ export class HopperFixtureLauncher implements BridgeLauncher { | "none" | "capability_unavailable" | "bridge_exception" = "none", + readonly advertisedRequestBytes?: number, ) {} launch(session: BridgeSession) { @@ -59,6 +60,9 @@ export class HopperFixtureLauncher implements BridgeLauncher { this.diagnosticMode, this.healthMode, this.inventoryFailure, + ...(this.advertisedRequestBytes === undefined + ? [] + : [String(this.advertisedRequestBytes)]), ], { stdio: ["ignore", "pipe", "pipe", "ipc"] }, ); diff --git a/tests/boundary/providers/hopper/hopperClient.restart.test.ts b/tests/boundary/providers/hopper/hopperClient.restart.test.ts index 8e5ed8038d..a6b63bdabb 100644 --- a/tests/boundary/providers/hopper/hopperClient.restart.test.ts +++ b/tests/boundary/providers/hopper/hopperClient.restart.test.ts @@ -34,6 +34,7 @@ describe("HopperClient restart", () => { name: "REA Hopper bridge", version: "1.0.0", processId: expect.any(Number), + maxRequestBytes: 16 * 1024 * 1024, }, }); }); @@ -50,6 +51,7 @@ describe("HopperClient restart", () => { name: "REA Hopper bridge", version: "1.0.0", processId: expect.any(Number), + maxRequestBytes: 16 * 1024 * 1024, }, }); await expect(client.callTool("echo", { value: "fresh" })).resolves.toEqual({ diff --git a/tests/boundary/providers/hopper/hopperClient.startupFailures.test.ts b/tests/boundary/providers/hopper/hopperClient.startupFailures.test.ts index 46f9733f12..6d23a1a86d 100644 --- a/tests/boundary/providers/hopper/hopperClient.startupFailures.test.ts +++ b/tests/boundary/providers/hopper/hopperClient.startupFailures.test.ts @@ -268,6 +268,7 @@ describe("HopperClient startup failures", () => { name: "REA Hopper bridge", version: "1.0.0", processId: retriedProcess?.pid, + maxRequestBytes: 16 * 1024 * 1024, }, }); await expect( @@ -300,6 +301,7 @@ describe("HopperClient startup failures", () => { name: "REA Hopper bridge", version: "1.0.0", processId: launcher.processes[0]?.pid, + maxRequestBytes: 16 * 1024 * 1024, }, }); await expect( diff --git a/tests/fixtures/cutterBridgeProtocol.ts b/tests/fixtures/cutterBridgeProtocol.ts index 2103a87536..091625ec48 100644 --- a/tests/fixtures/cutterBridgeProtocol.ts +++ b/tests/fixtures/cutterBridgeProtocol.ts @@ -118,6 +118,7 @@ export const writeDescriptor = async ( options: { readonly currentFile?: string | null; readonly documentGeneration?: number; + readonly maxRequestBytes?: number; } = {}, ): Promise => writeFile( @@ -132,6 +133,9 @@ export const writeDescriptor = async ( current_file: options.currentFile ?? null, cutter_version: "Cutter fixture version", identity_status: "partial", + ...(options.maxRequestBytes === undefined + ? {} + : { max_request_bytes: options.maxRequestBytes }), }), { mode: 0o600 }, ); diff --git a/tests/fixtures/cutter_bridge_protocol_test.py b/tests/fixtures/cutter_bridge_protocol_test.py index af6b502185..1f8bf76e10 100644 --- a/tests/fixtures/cutter_bridge_protocol_test.py +++ b/tests/fixtures/cutter_bridge_protocol_test.py @@ -253,6 +253,60 @@ class ResponseEncodingTest(unittest.TestCase): "execution_state": "complete", }, "a" * 64, "b" * 64)) + def test_streamed_frame_matches_the_encoded_frame(self): + value = { + "ok": True, + "output": 'say "hi"\\\n雪\u2028', + "execution_state": "complete", + } + encoded = self.server._encode_authenticated_response(value, "a" * 64, "b" * 64) + sender, receiver = socket.socketpair() + try: + receiver.settimeout(2) + self.server._send_authenticated(sender, value, "a" * 64, "b" * 64) + sender.shutdown(socket.SHUT_WR) + streamed = bytearray() + while True: + part = receiver.recv(65536) + if not part: + break + streamed.extend(part) + finally: + sender.close() + receiver.close() + self.assertEqual(bytes(streamed), encoded) + self.assertEqual(json.loads(json.loads(encoded)["response"]), value) + + def test_json_pieces_stay_within_one_chunk(self): + output = 'say "hi"\\\n雪\u0001' * 20000 + self.assertGreater(len(output), bridge._JSON_CHUNK_CHARS) + value = {"ok": True, "output": output, "execution_state": "complete"} + pieces = list(bridge._response_fragments(value)) + self.assertGreater(len(pieces), 4) + for piece in pieces: + self.assertLessEqual(len(piece), bridge._JSON_CHUNK_CHARS * 6) + self.assertEqual( + "".join(pieces), + json.dumps(value, ensure_ascii=False, separators=(",", ":")), + ) + + def test_scalar_lists_are_batched_and_match_json(self): + mixed = [0, True, False, None, -2, 1.5, {"n": 1}, "ab", [3, 4]] + self.assertEqual( + "".join(bridge._response_fragments(mixed)), + json.dumps(mixed, ensure_ascii=False, separators=(",", ":")), + ) + zeroes = [0] * 40000 + pieces = list(bridge._response_fragments(zeroes)) + self.assertEqual( + "".join(pieces), + json.dumps(zeroes, separators=(",", ":")), + ) + self.assertLess(len(pieces), 8) + self.assertGreater(len(pieces), 2) + for piece in pieces: + self.assertLessEqual(len(piece), bridge._JSON_CHUNK_CHARS + 1) + def test_unencodable_output_preserves_authenticated_completion(self): for index, surrogate in ((0, "\ud800"), (5 * 1024 * 1024 - 1, "\udfff")): with self.subTest(index=index): @@ -367,6 +421,112 @@ class HostVersionTest(unittest.TestCase): class ProtocolProducerTest(unittest.TestCase): + def test_stalled_hello_does_not_block_the_next_client(self): + stalled = socket.create_connection(("127.0.0.1", self.server.port), timeout=2) + try: + _send(stalled, {"kind": "rea-auth-v2", "nonce": "b" * 64}) + challenge = _frame(stalled) + self.assertEqual(challenge["session_id"], self.server.session_id) + started = time.monotonic() + response, _nonce = self.exchange({ + "kind": "status", + "session_id": self.server.session_id, + }) + self.assertLess(time.monotonic() - started, 2) + self.assertTrue(response["ok"]) + self.assertEqual(response["session_id"], self.server.session_id) + finally: + stalled.close() + + def test_status_does_not_complete_an_earlier_connection_still_reading(self): + earlier = socket.create_connection(("127.0.0.1", self.server.port), timeout=2) + try: + _send(earlier, {"kind": "rea-auth-v2", "nonce": "c" * 64}) + challenge = _frame(earlier) + earlier_ordinal = challenge["nonce"][:16] + self.assertEqual(earlier_ordinal, "0000000000000001") + response, _nonce = self.exchange({ + "kind": "status", + "session_id": self.server.session_id, + }) + self.assertEqual(response["completed_connection"], "0" * 16) + finally: + earlier.close() + deadline = time.monotonic() + 2 + latest = "0" * 16 + while time.monotonic() < deadline: + response, _nonce = self.exchange({ + "kind": "status", + "session_id": self.server.session_id, + }) + latest = response["completed_connection"] + if latest >= earlier_ordinal: + break + time.sleep(0.02) + self.assertGreaterEqual(latest, earlier_ordinal) + + def test_worker_admission_is_bounded_and_keeps_listening(self): + previous = bridge.MAX_CONNECTION_WORKERS + bridge.MAX_CONNECTION_WORKERS = 1 + stalled = socket.create_connection(("127.0.0.1", self.server.port), timeout=2) + try: + _send(stalled, {"kind": "rea-auth-v2", "nonce": "d" * 64}) + self.assertEqual(_frame(stalled)["session_id"], self.server.session_id) + rejected = socket.create_connection(("127.0.0.1", self.server.port), timeout=2) + try: + rejected.settimeout(1) + try: + observed = rejected.recv(16) + except OSError: + observed = b"" + self.assertEqual(observed, b"") + finally: + rejected.close() + finally: + stalled.close() + bridge.MAX_CONNECTION_WORKERS = previous + deadline = time.monotonic() + 2 + while time.monotonic() < deadline and any( + worker.is_alive() for worker in self.server._workers + ): + time.sleep(0.01) + response, _nonce = self.exchange({ + "kind": "status", + "session_id": self.server.session_id, + }) + self.assertTrue(response["ok"]) + self.assertTrue(self.server.descriptor.exists()) + + def test_failed_worker_start_does_not_stop_the_listener(self): + calls = {"n": 0} + original = threading.Thread.start + + def start(thread): + if thread.name == "rea-cutter-bridge-connection": + calls["n"] += 1 + if calls["n"] == 1: + raise RuntimeError("cannot start new thread") + return original(thread) + + with patch.object(threading.Thread, "start", start): + rejected = socket.create_connection(("127.0.0.1", self.server.port), timeout=2) + try: + rejected.settimeout(1) + try: + observed = rejected.recv(16) + except OSError: + observed = b"" + self.assertEqual(observed, b"") + finally: + rejected.close() + self.assertTrue(self.server.descriptor.exists()) + response, _nonce = self.exchange({ + "kind": "status", + "session_id": self.server.session_id, + }) + self.assertTrue(response["ok"]) + self.assertEqual(response["completed_connection"], "0000000000000001") + def test_rejects_legacy_peers_before_ui_dispatch(self): with socket.create_connection(("127.0.0.1", self.server.port), timeout=2) as connection: connection.sendall((json.dumps({"kind": "rea-auth-v1", "nonce": "a" * 64}) + "\n").encode()) @@ -612,6 +772,260 @@ class RequestFrameLimitTest(unittest.TestCase): with self.assertRaisesRegex(ValueError, "frame-limit-exceeded"): bridge._BridgeServer._receive_frame(BufferedSocket(payload + b" " + b"\n")) + def test_delimiter_in_a_later_chunk_is_accepted(self): + class ShortReads: + def __init__(self, data, size): + self.data = data + self.size = size + + def settimeout(self, _timeout): + pass + + def recv(self, _count): + part, self.data = self.data[: self.size], self.data[self.size :] + return part + + payload = b'{"value":"' + b"A" * 70 + b'"}' + received = bridge._BridgeServer._receive_frame(ShortReads(payload + b"\n", 3)) + self.assertEqual(received["value"], "A" * 70) + with self.assertRaisesRegex(ValueError, "invalid framed request"): + bridge._BridgeServer._receive_frame(ShortReads(payload + b"\n{}", 3)) + + +def _load_cutter_bridge(suffix, raw): + """Import a fresh plugin module under one explicit ceiling.""" + name = "rea_cutter_bridge_limit_%s" % suffix + sys.modules.pop(name, None) + setting = "REA_BRIDGE_MAX_REQUEST_BYTES" + previous = os.environ.get(setting) + if raw is None: + os.environ.pop(setting, None) + else: + os.environ[setting] = raw + try: + spec = importlib.util.spec_from_file_location(name, MODULE_PATH) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + finally: + if previous is None: + os.environ.pop(setting, None) + else: + os.environ[setting] = previous + + +class _BufferedSocket: + def __init__(self, data): + self.data = data + + def settimeout(self, _timeout): + pass + + def recv(self, count): + part, self.data = self.data[:count], self.data[count:] + return part + + +class BridgeRequestCeilingTest(unittest.TestCase): + def test_unset_ceiling_matches_the_historical_16_mib_constant(self): + module = _load_cutter_bridge("default", None) + self.assertEqual(module.MAX_FRAME_BYTES, 16 * 1024 * 1024) + self.assertEqual(module.BRIDGE_MAX_REQUEST_BYTES_SOURCE, "default") + self.assertEqual(module.MINIMUM_BRIDGE_MAX_REQUEST_BYTES, 7464) + self.assertEqual(module.MAXIMUM_BRIDGE_MAX_REQUEST_BYTES, 536870887) + self.assertEqual(module._authenticated_frame_timeout_seconds(), 16) + self.assertEqual(module.AUTH_FRAME_TIMEOUT_SECONDS, 1) + payload = b"{}" + b" " * (module.MAX_FRAME_BYTES - 2) + self.assertEqual( + module._BridgeServer._receive_frame(_BufferedSocket(payload + b"\n")), + {}, + ) + + def test_setting_lowers_and_raises_the_frame_ceiling(self): + lowered = _load_cutter_bridge("lowered", "8192") + self.assertEqual(lowered.MAX_FRAME_BYTES, 8192) + self.assertEqual(lowered.BRIDGE_MAX_REQUEST_BYTES_SOURCE, "setting") + accepted = b"{}" + b" " * (8192 - 2) + self.assertEqual( + lowered._BridgeServer._receive_frame(_BufferedSocket(accepted + b"\n")), + {}, + ) + with self.assertRaisesRegex(ValueError, "frame-limit-exceeded") as lowered_error: + lowered._BridgeServer._receive_frame( + _BufferedSocket(accepted + b" " + b"\n") + ) + self.assertIn("REA_BRIDGE_MAX_REQUEST_BYTES", str(lowered_error.exception)) + raised = _load_cutter_bridge("raised", str(17 * 1024 * 1024)) + self.assertEqual(raised.MAX_FRAME_BYTES, 17 * 1024 * 1024) + self.assertEqual(raised._authenticated_frame_timeout_seconds(), 17) + capped = _load_cutter_bridge("capped", "536870887") + self.assertEqual(capped._authenticated_frame_timeout_seconds(), 25) + above_default = b"{}" + b" " * (16 * 1024 * 1024 - 1) + self.assertGreater(len(above_default), 16 * 1024 * 1024) + self.assertEqual( + raised._BridgeServer._receive_frame(_BufferedSocket(above_default + b"\n")), + {}, + ) + + def test_authenticated_response_budget_excludes_the_line_terminator(self): + module = _load_cutter_bridge("response_lf", "8192") + server = module._BridgeServer.__new__(module._BridgeServer) + server.token = "token" + server.session_id = "00000000-0000-4000-8000-000000000001" + + def encode(length): + return server._encode_authenticated_response( + { + "ok": True, + "output": "A" * length, + "execution_state": "complete", + }, + "a" * 64, + "b" * 64, + ) + + low = 0 + high = module.MAX_FRAME_BYTES + accepted = None + accepted_length = None + while low <= high: + mid = (low + high) // 2 + encoded = encode(mid) + if encoded is None: + high = mid - 1 + else: + accepted = encoded + accepted_length = mid + low = mid + 1 + self.assertIsNotNone(accepted) + self.assertTrue(accepted.endswith(b"\n")) + self.assertEqual(len(accepted) - 1, module.MAX_FRAME_BYTES) + self.assertIsNone(encode(accepted_length + 1)) + + def test_tight_status_keeps_recovery_fields_and_omits_the_path(self): + module = _load_cutter_bridge("status_floor", "7464") + server = module._BridgeServer.__new__(module._BridgeServer) + server.token = "token" + server.session_id = "00000000-0000-4000-8000-000000000001" + receipts = [ + { + "request_id": str(uuid.UUID(int=index + 1)), + "execution_state": "not_started", + } + for index in range(module.MAX_COMPLETION_RECEIPTS) + ] + status = { + "ok": True, + "session_id": server.session_id, + "current_file": "a" * 4095, + "document_generation": 0, + "cutter_version": "2.4.0", + "identity_status": "partial", + "completed_commands": receipts, + "completed_connection": "f" * 16, + } + self.assertIsNone( + server._encode_authenticated_response(status, "a" * 64, "b" * 64) + ) + required = server._bounded_status_response(status) + encoded = server._encode_authenticated_response( + required, "a" * 64, "b" * 64 + ) + self.assertIsNotNone(encoded) + self.assertLessEqual(len(encoded) - 1, module.MINIMUM_BRIDGE_MAX_REQUEST_BYTES) + captured = bytearray() + + class _Capture: + def sendall(self, data): + captured.extend(data) + + server._send_authenticated(_Capture(), status, "a" * 64, "b" * 64) + outer = json.loads(bytes(captured).decode("utf-8")) + response = json.loads(outer["response"]) + self.assertEqual(response["session_id"], server.session_id) + self.assertEqual(response["document_generation"], 0) + self.assertEqual(response["completed_connection"], "f" * 16) + self.assertEqual(response["completed_commands"], receipts) + self.assertIsNone(response["current_file"]) + self.assertNotIn("cutter_version", response) + self.assertIn("unknown", response["message"]) + self.assertIn("REA_BRIDGE_MAX_REQUEST_BYTES", response["message"]) + short = dict(status) + short["current_file"] = "/tmp/fresh.bin" + short["cutter_version"] = "v" * 4000 + captured.clear() + server._send_authenticated(_Capture(), short, "a" * 64, "b" * 64) + preserved = json.loads( + json.loads(bytes(captured).decode("utf-8"))["response"] + ) + self.assertEqual(preserved["current_file"], "/tmp/fresh.bin") + self.assertEqual(preserved["document_generation"], 0) + self.assertNotIn("cutter_version", preserved) + + def test_tight_command_keeps_completion_and_omits_the_path(self): + module = _load_cutter_bridge("command_path", "7464") + server = module._BridgeServer.__new__(module._BridgeServer) + server.token = "token" + server.session_id = "00000000-0000-4000-8000-000000000001" + value = { + "ok": True, + "output": "ok", + "execution_state": "complete", + "current_file": "\\" * 3000, + "document_generation": 7, + "identity_status": "partial", + } + self.assertIsNone( + server._encode_authenticated_response(value, "a" * 64, "b" * 64) + ) + captured = bytearray() + + class _Capture: + def sendall(self, data): + captured.extend(data) + + server._send_authenticated(_Capture(), value, "a" * 64, "b" * 64) + response = json.loads(json.loads(bytes(captured).decode("utf-8"))["response"]) + self.assertEqual(response["execution_state"], "complete") + self.assertEqual(response["document_generation"], 7) + self.assertIsNone(response["current_file"]) + self.assertIsNone(response["output"]) + self.assertTrue(response["output_truncated"]) + self.assertIn("REA_BRIDGE_MAX_REQUEST_BYTES", response["message"]) + short = dict(value) + short["current_file"] = "/tmp/fresh.bin" + short["output"] = "A" * 10000 + captured.clear() + server._send_authenticated(_Capture(), short, "a" * 64, "b" * 64) + preserved = json.loads(json.loads(bytes(captured).decode("utf-8"))["response"]) + self.assertEqual(preserved["current_file"], "/tmp/fresh.bin") + self.assertEqual(preserved["document_generation"], 7) + self.assertEqual(preserved["execution_state"], "complete") + self.assertIsNone(preserved["output"]) + + def test_dense_primitive_array_uses_its_encoded_size(self): + module = _load_cutter_bridge("dense", "8192") + zeros = [0] * 2000 + value = {"ok": True, "output": zeros, "execution_state": "complete"} + self.assertTrue(module._within_response_limit(value)) + server = module._BridgeServer.__new__(module._BridgeServer) + server.token = "token" + server.session_id = "00000000-0000-4000-8000-000000000001" + encoded = server._encode_authenticated_response(value, "a" * 64, "b" * 64) + self.assertIsNotNone(encoded) + response = json.loads(json.loads(encoded)["response"]) + self.assertEqual(response["output"], zeros) + self.assertNotIn("output_truncated", response) + overflow = {"ok": True, "output": [0] * 8000, "execution_state": "complete"} + self.assertFalse(module._within_response_limit(overflow)) + + def test_invalid_ceilings_are_refused(self): + for raw in ("7463", "536870888", "0", "-1", "abc", "016777216", "٨١٩٢"): + with self.assertRaises(RuntimeError) as caught: + _load_cutter_bridge("invalid_%s" % raw, raw) + self.assertIn("REA_BRIDGE_MAX_REQUEST_BYTES", str(caught.exception)) + if __name__ == "__main__": if sys.argv[1:] == ["--serve"]: diff --git a/tests/fixtures/fakeHopper.mjs b/tests/fixtures/fakeHopper.mjs index 14549d55db..f6b650b785 100644 --- a/tests/fixtures/fakeHopper.mjs +++ b/tests/fixtures/fakeHopper.mjs @@ -8,7 +8,12 @@ const [ diagnosticMode = "quiet", healthMode = "ready", inventoryFailure = "none", + advertisedRequestBytes, ] = process.argv.slice(2); +const maxRequestBytes = + advertisedRequestBytes === undefined + ? 16 * 1024 * 1024 + : Number(advertisedRequestBytes); let diagnosticOutputEmitted = false; const heldReplies = new Map(); const sockets = new Set(); @@ -219,6 +224,7 @@ const server = createServer((socket) => { version: "1.0.0", run_id: runId, process_id: process.pid, + max_request_bytes: maxRequestBytes, }, }, true, diff --git a/tests/fixtures/hopperBridgeFacadeProbe.py b/tests/fixtures/hopperBridgeFacadeProbe.py index fa1cacefa5..26b70a8264 100644 --- a/tests/fixtures/hopperBridgeFacadeProbe.py +++ b/tests/fixtures/hopperBridgeFacadeProbe.py @@ -108,11 +108,13 @@ class FakeBlock: def getSuccessorCount(self): return 0 -def load_bridge(path): +def load_bridge(path, namespace_updates=None): namespace = { "__file__": path, "__name__": "rea_hopper_bridge", } + if namespace_updates: + namespace.update(namespace_updates) source = Path(path).read_text(encoding="utf-8") exec(compile(source, path, "exec"), namespace) return namespace diff --git a/tests/fixtures/hopperBridgeFrameLimitProbe.py b/tests/fixtures/hopperBridgeFrameLimitProbe.py index e59b68a561..64c8df630f 100644 --- a/tests/fixtures/hopperBridgeFrameLimitProbe.py +++ b/tests/fixtures/hopperBridgeFrameLimitProbe.py @@ -16,6 +16,7 @@ Four cases share one loaded bridge module: incomplete frame without dispatching it, then close. """ import json +import os import socket import sys import threading @@ -78,8 +79,36 @@ def request_line(bridge, params): return json.dumps(request, separators=(",", ":")).encode("utf-8") +def _load(mode): + """Resolve one ceiling. The default invocation ignores an ambient setting.""" + os.environ.pop("REA_BRIDGE_MAX_REQUEST_BYTES", None) + injected = None + if mode == "bytes": + os.environ["REA_BRIDGE_MAX_REQUEST_BYTES"] = sys.argv[3] + elif mode == "inject": + os.environ["REA_BRIDGE_MAX_REQUEST_BYTES"] = "9000" + injected = int(sys.argv[3]) + elif mode == "invalid": + os.environ["REA_BRIDGE_MAX_REQUEST_BYTES"] = sys.argv[3] + try: + load_bridge(sys.argv[1]) + except RuntimeError as error: + return {"error": str(error)} + return {"error": "invalid setting was accepted"} + elif mode != "serve": + return {"error": "unknown probe mode"} + return load_bridge( + sys.argv[1], + None if injected is None else {"REA_BRIDGE_MAX_REQUEST_BYTES": injected}, + ) + + def main(): - bridge = load_bridge(sys.argv[1]) + mode = sys.argv[2] if len(sys.argv) > 2 else "serve" + bridge = _load(mode) + if "error" in bridge and "MAX_REQUEST_BYTES" not in bridge: + json.dump(bridge, sys.stdout) + return bridge.update(REA_TOKEN="owned-token", REA_RUN_ID="owned-run") bridge["_probe_dispatches"] = [] @@ -118,6 +147,9 @@ def main(): json.dump( { "limit": limit, + "source": bridge.get("BRIDGE_MAX_REQUEST_BYTES_SOURCE"), + "minimum": bridge.get("MINIMUM_BRIDGE_MAX_REQUEST_BYTES"), + "maximum": bridge.get("MAXIMUM_BRIDGE_MAX_REQUEST_BYTES"), "legit": legit, "at_limit": at_limit, "past_limit": past_limit,