diff --git a/scripts/verify/architecture-guard-baselines.json b/scripts/verify/architecture-guard-baselines.json index ab56b1a8e8..32187b16ca 100644 --- a/scripts/verify/architecture-guard-baselines.json +++ b/scripts/verify/architecture-guard-baselines.json @@ -265,7 +265,6 @@ "src/artifacts/nodeSea/NodeSeaBlob.ts": 4, "src/artifacts/nodeSea/NodeSeaLocation.ts": 2, "src/artifacts/pyinstaller/CArchiveLayout.ts": 8, - "src/artifacts/pyinstaller/MarshalSubset.ts": 2, "src/artifacts/pyinstaller/PyzArchive.ts": 1, "src/browser/PngVisualDiff.ts": 7, "src/domain/apkSignature/apkSignatureSchemes.ts": 1, @@ -302,6 +301,7 @@ "src/domain/peOptionalHeader.ts": 12, "src/domain/peSectionHeader.ts": 5, "src/domain/pngHeader.ts": 4, + "src/domain/python/marshalReader.ts": 4, "src/domain/python/pythonBytecodeIdentity.ts": 1, "src/domain/riscvIsaDeclaration.ts": 1, "src/domain/uimageFormat.ts": 10, @@ -375,7 +375,6 @@ "src/native/pwntools/RecordedCrashSourceBindings.ts": 6, "src/native/windows/WindowsShortcutParser.ts": 49, "src/pcap/PcapDnsWire.ts": 5, - "src/python/PythonPycParser.ts": 9, "src/v8/V8CodeCacheParser.ts": 1 }, "range-error-interpretation": { diff --git a/src/artifacts/pyinstaller/MarshalSubset.ts b/src/artifacts/pyinstaller/MarshalSubset.ts index 864ae7799f..54b321a0aa 100644 --- a/src/artifacts/pyinstaller/MarshalSubset.ts +++ b/src/artifacts/pyinstaller/MarshalSubset.ts @@ -1,5 +1,15 @@ import { err, ok, type Result } from "../../domain/result.js"; -import { decodeMarshalUtf8 } from "../../domain/python/marshalUtf8.js"; +import { + MarshalDefectError, + MarshalReader, + type MarshalBytes, + type MarshalInt, + type MarshalList, + type MarshalString, + type MarshalTuple, + type MarshalDefect, + type MarshalLimitExceeded, +} from "../../domain/python/marshalReader.js"; import { pyInstallerFailure, type PyInstallerFailure, @@ -9,7 +19,8 @@ const MAX_DEPTH = 16; const MAX_NODES = 100_000; const MAX_SEQUENCE = 100_000; const MAX_STRING_BYTES = 65_536; -const FLAG_REF = 0x80; +/** Digits of a TYPE_LONG whose magnitude can still be a safe integer. */ +const MAX_LONG_DIGITS = 8; /** Values the PYZ table of contents is allowed to contain. */ export type MarshalValue = @@ -18,14 +29,89 @@ export type MarshalValue = | Uint8Array | readonly MarshalValue[]; -interface MarshalCursor { - readonly bytes: Buffer; - offset: number; - readonly refs: Array; - depth: number; - nodes: number; +/** The shared reader's values for the subset: no other type is decoded. */ +type SubsetValue = + | MarshalInt + | MarshalBytes + | MarshalString + | MarshalTuple + | MarshalList; + +const FORMAT_MESSAGES: Readonly> = { + truncated: "PYZ marshal value is truncated", + "negative-string-length": "PYZ marshal string has a negative length", + "negative-sequence-length": "PYZ marshal sequence has a negative count", + "negative-field": "PYZ marshal value has a negative count", + "invalid-utf8": "PYZ marshal string is not valid UTF-8", + "long-digit-range": "PYZ marshal long digit is out of range", + "long-not-normalized": "PYZ marshal long is not normalized", + "unresolved-reference": + "PYZ marshal reference does not point at a completed value", + "unsupported-type": + "PYZ marshal value uses a type outside the supported subset", +}; + +const LIMIT_MESSAGES: Readonly> = + { + objects: "PYZ marshal value exceeds the depth or object bound", + depth: "PYZ marshal value exceeds the depth or object bound", + sequence: "PYZ marshal sequence exceeds the supported count", + "string-bytes": "PYZ marshal string exceeds the supported length", + "long-bytes": "PYZ marshal integer exceeds the safe range", + "long-digits": "PYZ marshal integer exceeds the safe range", + }; + +/** A PYZ budget refusal, kept apart from the shared reader's format failures. */ +class PyzMarshalLimit { + constructor(readonly message: string) {} } +/** + * The shared CPython marshal reader restricted to list, tuple, str, int and + * bytes. Containers are published only once complete, so the projected + * values cannot be cyclic and a reference into an unfinished container fails. + */ +class PyzMarshalReader extends MarshalReader { + constructor(bytes: Buffer) { + super(bytes, { + format: "pyinstaller-pyz-toc", + limits: { + maxObjects: MAX_NODES, + maxDepth: MAX_DEPTH, + maxStringBytes: MAX_STRING_BYTES, + maxSequence: MAX_SEQUENCE, + maxLongDigits: MAX_LONG_DIGITS, + }, + publishContainers: "after-items", + }); + } + + protected override wrap(value: SubsetValue): SubsetValue { + return value; + } + + protected override limitExceeded(exceeded: MarshalLimitExceeded): never { + throw new PyzMarshalLimit(LIMIT_MESSAGES[exceeded.limit]); + } + + /** A TOC integer must be a safe JavaScript number, checked in stream order. */ + override readTyped( + type: number, + depth: number, + slot: number | undefined, + ): SubsetValue { + const value = super.readTyped(type, depth, slot); + if ( + value.kind === "int" && + (value.value > MAX_SAFE || value.value < -MAX_SAFE) + ) + throw new PyzMarshalLimit(LIMIT_MESSAGES["long-digits"]); + return value; + } +} + +const MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER); + /** * Read one marshal value using only list, tuple, str, int, and bytes. * Container slots are reserved before their elements so later references match @@ -34,250 +120,45 @@ interface MarshalCursor { export const readMarshalSubset = ( bytes: Buffer, ): Result => { - const cursor: MarshalCursor = { - bytes, - offset: 0, - refs: [], - depth: 0, - nodes: 0, - }; - const value = readObject(cursor); - if (!value.ok) return value; - if (cursor.offset !== bytes.length) + const reader = new PyzMarshalReader(bytes); + let value: MarshalValue; + try { + value = project(reader.readObject(0), new WeakMap()); + } catch (cause: unknown) { + if (cause instanceof PyzMarshalLimit) + return err(pyInstallerFailure("limit", cause.message)); + if (cause instanceof MarshalDefectError) + return err(pyInstallerFailure("format", FORMAT_MESSAGES[cause.defect])); + throw cause; + } + if (reader.offset !== bytes.length) return err( pyInstallerFailure( "format", "PYZ marshal value did not consume its table of contents", ), ); - return value; + return ok(value); }; -const readObject = ( - cursor: MarshalCursor, -): Result => { - if (cursor.depth > MAX_DEPTH || cursor.nodes >= MAX_NODES) - return err( - pyInstallerFailure( - "limit", - "PYZ marshal value exceeds the depth or object bound", - ), - ); - cursor.nodes += 1; - const code = readByte(cursor); - if (!code.ok) return code; - const flag = (code.value & FLAG_REF) !== 0; - const type = code.value & ~FLAG_REF; - if (type === 0x72) return readReference(cursor); - if (type === 0x69) return storeScalar(cursor, flag, readInt32(cursor)); - if (type === 0x6c) return storeScalar(cursor, flag, readLong(cursor)); - if (type === 0x7a || type === 0x5a) - return storeScalar(cursor, flag, readShortAscii(cursor)); - if (type === 0x61 || type === 0x41) - return storeScalar(cursor, flag, readCountedAscii(cursor)); - if (type === 0x74 || type === 0x75) - return storeScalar(cursor, flag, readUtf8(cursor)); - if (type === 0x73) return storeScalar(cursor, flag, readByteString(cursor)); - if (type === 0x28) return readSequence(cursor, flag, readInt32(cursor)); - if (type === 0x29) return readSequence(cursor, flag, readByte(cursor)); - if (type === 0x5b) return readSequence(cursor, flag, readInt32(cursor)); - return err( - pyInstallerFailure( - "format", - "PYZ marshal value uses a type outside the supported subset", - ), - ); -}; - -const storeScalar = ( - cursor: MarshalCursor, - flag: boolean, - value: Result, -): Result => { - if (!value.ok) return value; - if (flag) cursor.refs.push(value.value); - return value; -}; - -const readSequence = ( - cursor: MarshalCursor, - flag: boolean, - count: Result, -): Result => { - if (!count.ok) return count; - if (count.value < 0) - return err( - pyInstallerFailure("format", "PYZ marshal sequence has a negative count"), - ); - if (count.value > MAX_SEQUENCE) - return err( - pyInstallerFailure( - "limit", - "PYZ marshal sequence exceeds the supported count", - ), - ); - const slot = flag ? cursor.refs.length : undefined; - if (slot !== undefined) cursor.refs.push(undefined); - const items: MarshalValue[] = []; - cursor.depth += 1; - for (let index = 0; index < count.value; index += 1) { - const item = readObject(cursor); - if (!item.ok) return item; - items.push(item.value); +/** Shared values keep their identity, so one projection serves every reference. */ +const project = ( + value: SubsetValue, + projected: WeakMap, +): MarshalValue => { + switch (value.kind) { + case "int": + return Number(value.value); + case "str": + case "bytes": + return value.value; + case "tuple": + case "list": { + const known = projected.get(value); + if (known !== undefined) return known; + const items = value.items.map((item) => project(item, projected)); + projected.set(value, items); + return items; + } } - cursor.depth -= 1; - if (slot !== undefined) cursor.refs[slot] = items; - return ok(items); -}; - -const readReference = ( - cursor: MarshalCursor, -): Result => { - const index = readInt32(cursor); - if (!index.ok) return index; - const value = cursor.refs[index.value]; - if (value === undefined) - return err( - pyInstallerFailure( - "format", - "PYZ marshal reference does not point at a completed value", - ), - ); - return ok(value); -}; - -const readShortAscii = ( - cursor: MarshalCursor, -): Result => { - const length = readByte(cursor); - if (!length.ok) return length; - return readAsciiBytes(cursor, length.value); -}; - -const readCountedAscii = ( - cursor: MarshalCursor, -): Result => { - const length = readInt32(cursor); - if (!length.ok) return length; - return readAsciiBytes(cursor, length.value); -}; - -const readAsciiBytes = ( - cursor: MarshalCursor, - length: number, -): Result => { - const bytes = readBytes(cursor, length); - if (!bytes.ok) return bytes; - // These tags use CPython's one-byte Unicode constructor, not an ASCII check. - return ok(bytes.value.toString("latin1")); -}; - -const readUtf8 = ( - cursor: MarshalCursor, -): Result => { - const length = readInt32(cursor); - if (!length.ok) return length; - const bytes = readBytes(cursor, length.value); - if (!bytes.ok) return bytes; - const value = decodeMarshalUtf8(bytes.value); - if (!value.ok) - return err( - pyInstallerFailure("format", "PYZ marshal string is not valid UTF-8"), - ); - return ok(value.value.text); -}; - -const readByteString = ( - cursor: MarshalCursor, -): Result => { - const length = readInt32(cursor); - if (!length.ok) return length; - return readBytes(cursor, length.value); -}; - -const readLong = ( - cursor: MarshalCursor, -): Result => { - const digits = readInt32(cursor); - if (!digits.ok) return digits; - const count = Math.abs(digits.value); - if (count > 8) - return err( - pyInstallerFailure("limit", "PYZ marshal integer exceeds the safe range"), - ); - let magnitude = 0n; - let mostSignificantDigit = 0; - for (let index = 0; index < count; index += 1) { - const digit = readUint16(cursor); - if (!digit.ok) return digit; - if (digit.value > 0x7fff) - return err( - pyInstallerFailure("format", "PYZ marshal long digit is out of range"), - ); - mostSignificantDigit = digit.value; - magnitude += BigInt(digit.value) << BigInt(15 * index); - } - if (count > 0 && mostSignificantDigit === 0) - return err( - pyInstallerFailure("format", "PYZ marshal long is not normalized"), - ); - if (magnitude > BigInt(Number.MAX_SAFE_INTEGER)) - return err( - pyInstallerFailure("limit", "PYZ marshal integer exceeds the safe range"), - ); - const signed = digits.value < 0 ? -Number(magnitude) : Number(magnitude); - return ok(signed); -}; - -const readBytes = ( - cursor: MarshalCursor, - length: number, -): Result => { - if (length < 0) - return err( - pyInstallerFailure("format", "PYZ marshal string has a negative length"), - ); - if (!Number.isSafeInteger(length) || length > MAX_STRING_BYTES) - return err( - pyInstallerFailure( - "limit", - "PYZ marshal string exceeds the supported length", - ), - ); - const end = cursor.offset + length; - if (end > cursor.bytes.length) - return err(pyInstallerFailure("format", "PYZ marshal value is truncated")); - const bytes = cursor.bytes.subarray(cursor.offset, end); - cursor.offset = end; - return ok(bytes); -}; - -const readByte = ( - cursor: MarshalCursor, -): Result => { - if (cursor.offset >= cursor.bytes.length) - return err(pyInstallerFailure("format", "PYZ marshal value is truncated")); - const value = cursor.bytes[cursor.offset] ?? 0; - cursor.offset += 1; - return ok(value); -}; - -const readInt32 = ( - cursor: MarshalCursor, -): Result => { - if (cursor.offset + 4 > cursor.bytes.length) - return err(pyInstallerFailure("format", "PYZ marshal value is truncated")); - const value = cursor.bytes.readInt32LE(cursor.offset); - cursor.offset += 4; - return ok(value); -}; - -const readUint16 = ( - cursor: MarshalCursor, -): Result => { - if (cursor.offset + 2 > cursor.bytes.length) - return err(pyInstallerFailure("format", "PYZ marshal value is truncated")); - const value = cursor.bytes.readUInt16LE(cursor.offset); - cursor.offset += 2; - return ok(value); }; diff --git a/src/domain/python/marshalReader.ts b/src/domain/python/marshalReader.ts new file mode 100644 index 0000000000..8f0bd17f4a --- /dev/null +++ b/src/domain/python/marshalReader.ts @@ -0,0 +1,463 @@ +import { decodeMarshalUtf8 } from "./marshalUtf8.js"; +import { ParseFailureError } from "../parseFailure.js"; + +const FLAG_REF = 0x80; +const TYPE_INT = 0x69; +const TYPE_LONG = 0x6c; +const TYPE_BYTES = 0x73; +const TYPE_INTERNED = 0x74; +const TYPE_REF = 0x72; +const TYPE_TUPLE = 0x28; +const TYPE_LIST = 0x5b; +const TYPE_UNICODE = 0x75; +const TYPE_ASCII = 0x61; +const TYPE_ASCII_INTERNED = 0x41; +const TYPE_SMALL_TUPLE = 0x29; +const TYPE_SHORT_ASCII = 0x7a; +const TYPE_SHORT_ASCII_INTERNED = 0x5a; + +/** A marshal integer: TYPE_INT, or a normalized TYPE_LONG. */ +export interface MarshalInt { + readonly kind: "int"; + readonly value: bigint; + readonly bits: number; +} + +export interface MarshalBytes { + readonly kind: "bytes"; + readonly value: Buffer; +} + +export interface MarshalString { + readonly kind: "str"; + readonly value: string; + readonly bytes: number; + /** Marshal bytes. JavaScript strings merge lone surrogates with pairs. */ + readonly raw: Buffer; + readonly previewEnd: number; +} + +/** A tuple. Items are filled after the container is created. */ +export interface MarshalTuple { + readonly kind: "tuple"; + readonly items: V[]; +} + +/** A list. Items are filled after the container is created. */ +export interface MarshalList { + readonly kind: "list"; + readonly items: V[]; +} + +/** The value types every marshal reader decodes. */ +export type MarshalCoreValue = + | MarshalInt + | MarshalBytes + | MarshalString + | MarshalTuple + | MarshalList; + +/** What a marshal stream failed on. Each reader names it in its own words. */ +export type MarshalDefect = + | "truncated" + | "negative-string-length" + | "negative-sequence-length" + | "negative-field" + | "invalid-utf8" + | "long-digit-range" + | "long-not-normalized" + | "unresolved-reference" + | "unsupported-type"; + +/** A malformed marshal stream, naming its defect for each reader's wording. */ +export class MarshalDefectError extends ParseFailureError<"malformed"> { + constructor( + readonly defect: MarshalDefect, + format: string, + message: string, + ) { + super({ kind: "malformed", format, detail: message, field: defect }); + } +} + +/** Budgets a marshal reader enforces while walking the stream. */ +export interface MarshalLimits { + readonly maxObjects: number; + readonly maxDepth: number; + readonly maxStringBytes: number; + /** Largest tuple or list count, when the reader bounds it. */ + readonly maxSequence?: number; + /** Largest TYPE_LONG digit count, checked before any digit is read. */ + readonly maxLongDigits?: number; +} + +/** A budget the stream exceeded, with the amount the stream asked for. */ +export type MarshalLimitExceeded = + | { readonly limit: "objects" | "depth" } + | { + readonly limit: "string-bytes" | "long-bytes" | "long-digits"; + readonly amount: number; + } + | { readonly limit: "sequence"; readonly amount: number }; + +export interface MarshalReaderOptions { + /** The ParseFailure format named in every failure. */ + readonly format: string; + readonly limits: MarshalLimits; + /** UTF-16 units of each string retained as its preview prefix. */ + readonly previewLimit?: number; + /** + * CPython registers a tuple or list before reading its items, so an item + * can refer back to it. A reader whose values cannot be cyclic publishes + * the container only once its items are complete. + */ + readonly publishContainers: "before-items" | "after-items"; +} + +/** + * One CPython marshal value reader. It owns the cursor, the reference table, + * the walk budgets, and the integer, string, bytes, tuple and list types. + * A subclass adds the other types through {@link readExtended}; without it, + * every other type is refused, which is the PyInstaller table subset. + */ +export abstract class MarshalReader { + offset = 0; + objects = 0; + depthReached = 0; + readonly #refs: (V | undefined)[] = []; + + constructor( + readonly bytes: Buffer, + readonly options: MarshalReaderOptions, + ) {} + + /** View a core value as this reader's value type. */ + protected abstract wrap(value: MarshalCoreValue): V; + + /** Report a budget the stream exceeded. */ + protected abstract limitExceeded(exceeded: MarshalLimitExceeded): never; + + /** Read a type outside the core set. The default refuses it. */ + protected readExtended( + type: number, + _depth: number, + _slot: number | undefined, + ): V { + return this.malformed( + "unsupported-type", + `Unsupported marshal type 0x${type.toString(16)}.`, + ); + } + + /** Whether a flagged type reserves a reference slot. */ + protected reservesReference(_type: number): boolean { + return true; + } + + /** Whether a decoded value is no object at all, such as TYPE_NULL. */ + protected isAbsent(_value: V): boolean { + return false; + } + + /** Check one decoded tuple or list item. */ + protected sequenceItem(_item: V): void {} + + /** Note that a tuple or list starts or finishes reading its items. */ + protected sequenceReading(_value: V, _reading: boolean): void {} + + protected malformed(defect: MarshalDefect, message: string): never { + throw new MarshalDefectError(defect, this.options.format, message); + } + + readObject(depth: number): V { + const { limits } = this.options; + this.objects += 1; + if (this.objects > limits.maxObjects) + this.limitExceeded({ limit: "objects" }); + if (depth > limits.maxDepth) this.limitExceeded({ limit: "depth" }); + this.depthReached = Math.max(this.depthReached, depth); + const raw = this.u8(); + const flagged = (raw & FLAG_REF) !== 0; + const type = raw & ~FLAG_REF; + // CPython masks FLAG_REF before dispatch; TYPE_REF returns its existing + // slot directly, and singleton tags bypass reference registration. + if (type === TYPE_REF) return this.readRef(); + const slot = + flagged && this.reservesReference(type) ? this.#refs.length : undefined; + if (slot !== undefined) this.#refs.push(undefined); + const value = this.readTyped(type, depth, slot); + if ( + slot !== undefined && + this.#refs[slot] === undefined && + !this.isAbsent(value) + ) + this.#refs[slot] = value; + return value; + } + + readTyped(type: number, depth: number, slot: number | undefined): V { + switch (type) { + case TYPE_INT: { + const value = BigInt(this.i32()); + return this.wrap({ kind: "int", value, bits: smallBitLength(value) }); + } + case TYPE_LONG: + return this.wrap({ kind: "int", ...this.long() }); + case TYPE_BYTES: + return this.wrap({ + kind: "bytes", + value: this.blob(this.stringLength()), + }); + case TYPE_INTERNED: + case TYPE_UNICODE: + return this.wrap(this.string(this.stringLength(), "utf8")); + case TYPE_ASCII: + case TYPE_ASCII_INTERNED: + return this.wrap(this.string(this.stringLength(), "latin1")); + case TYPE_SHORT_ASCII: + case TYPE_SHORT_ASCII_INTERNED: + return this.wrap(this.string(this.u8(), "latin1")); + case TYPE_TUPLE: + return this.sequenceValue( + "tuple", + this.nonNegativeInt32( + "Marshal tuple length", + "negative-sequence-length", + ), + depth, + slot, + ); + case TYPE_SMALL_TUPLE: + return this.sequenceValue("tuple", this.u8(), depth, slot); + case TYPE_LIST: + return this.sequenceValue( + "list", + this.nonNegativeInt32( + "Marshal list length", + "negative-sequence-length", + ), + depth, + slot, + ); + default: + return this.readExtended(type, depth, slot); + } + } + + readRef(): V { + const index = this.u32(); + const value = this.#refs[index]; + if (value === undefined) + return this.malformed( + "unresolved-reference", + `TYPE_REF index ${index} is missing or still being constructed.`, + ); + return value; + } + + /** Fill a reserved reference slot with the value it stands for. */ + protected publish(slot: number | undefined, value: V): void { + if (slot !== undefined) this.#refs[slot] = value; + } + + sequenceValue( + kind: "tuple" | "list", + count: number, + depth: number, + slot: number | undefined, + ): V { + const { maxSequence } = this.options.limits; + if (maxSequence !== undefined && count > maxSequence) + this.limitExceeded({ limit: "sequence", amount: count }); + const items: V[] = []; + const value = this.wrap( + kind === "tuple" ? { kind: "tuple", items } : { kind: "list", items }, + ); + if (this.options.publishContainers === "before-items") + this.publish(slot, value); + this.sequenceReading(value, true); + for (let index = 0; index < count; index += 1) { + const item = this.readObject(depth + 1); + this.sequenceItem(item); + items.push(item); + } + this.sequenceReading(value, false); + return value; + } + + long(): { value: bigint; bits: number } { + const count = this.i32(); + const size = Math.abs(count); + const { maxLongDigits, maxStringBytes } = this.options.limits; + if (maxLongDigits !== undefined && size > maxLongDigits) + this.limitExceeded({ limit: "long-digits", amount: size }); + if (size * 2 > maxStringBytes) + this.limitExceeded({ limit: "long-bytes", amount: size * 2 }); + if (this.offset > this.bytes.length - size * 2) + this.malformed("truncated", "Marshal long overruns the selected file."); + const nibbles: number[] = []; + let carry = 0; + let carryBits = 0; + let digit = 0; + for (let index = 0; index < size; index += 1) { + digit = this.u16(); + if (digit >= 0x8000) + this.malformed( + "long-digit-range", + "Marshal long digit is out of range.", + ); + carry |= digit << carryBits; + carryBits += 15; + while (carryBits >= 4) { + nibbles.push(carry & 0xf); + carry >>>= 4; + carryBits -= 4; + } + } + if (size > 0 && digit === 0) + this.malformed("long-not-normalized", "Marshal long is not normalized."); + if (carryBits > 0) nibbles.push(carry); + let bits = 0; + if (size > 0) { + let top = digit; + while (top > 0) { + top >>>= 1; + bits += 1; + } + bits += (size - 1) * 15; + } + if (nibbles.length === 0) return { value: 0n, bits: 0 }; + const magnitude = BigInt( + `0x${nibbles + .reverse() + .map((nibble) => nibble.toString(16)) + .join("")}`, + ); + return { value: count < 0 ? -magnitude : magnitude, bits }; + } + + string(length: number, encoding: "latin1" | "utf8"): MarshalString { + const bytes = this.blob(length); + const previewLimit = this.options.previewLimit ?? Infinity; + if (encoding === "latin1") { + // CPython reads the ASCII-tagged forms as one-byte Unicode, even when + // high bytes appear. Identity must agree with the UTF-8 string form. + const value = bytes.toString("latin1"); + return { + kind: "str", + value, + previewEnd: Math.min(length, previewLimit), + bytes: length, + raw: bytes.some((byte) => byte > 0x7f) + ? Buffer.from(value, "utf8") + : bytes, + }; + } + const decoded = decodeMarshalUtf8(bytes, previewLimit); + if (!decoded.ok) + return this.malformed( + "invalid-utf8", + "Marshal string is not valid UTF-8.", + ); + return { + kind: "str", + value: decoded.value.text, + previewEnd: decoded.value.prefixEnd, + bytes: length, + raw: bytes, + }; + } + + /** A string or bytes length, which CPython requires non-negative. */ + stringLength(): number { + return this.nonNegativeInt32( + "Marshal string length", + "negative-string-length", + ); + } + + blob(length: number): Buffer { + if (length > this.options.limits.maxStringBytes) + this.limitExceeded({ limit: "string-bytes", amount: length }); + if (this.offset > this.bytes.length - length) + this.malformed("truncated", "Marshal string overruns the selected file."); + const value = this.bytes.subarray(this.offset, this.offset + length); + this.offset += length; + return Buffer.from(value); + } + + u8(): number { + const value = this.bytes[this.offset]; + if (value === undefined) + return this.malformed( + "truncated", + "Marshal stream ended inside an object.", + ); + this.offset += 1; + return value; + } + + u16(): number { + if (this.offset > this.bytes.length - 2) + this.malformed( + "truncated", + "Marshal stream ended inside a 16-bit field.", + ); + const value = this.bytes.readUInt16LE(this.offset); + this.offset += 2; + return value; + } + + u32(): number { + return this.i32() >>> 0; + } + + i32(): number { + if (this.offset > this.bytes.length - 4) + this.malformed( + "truncated", + "Marshal stream ended inside a 32-bit field.", + ); + const value = this.bytes.readInt32LE(this.offset); + this.offset += 4; + return value; + } + + /** CPython `TYPE_INT64`: a signed little-endian 64-bit integer. */ + i64(): bigint { + if (this.offset > this.bytes.length - 8) + this.malformed( + "truncated", + "Marshal stream ended inside a 64-bit integer.", + ); + const value = this.bytes.readBigInt64LE(this.offset); + this.offset += 8; + return value; + } + + /** CPython `TYPE_BINARY_FLOAT`: a little-endian IEEE 754 double. */ + f64(): number { + if (this.offset > this.bytes.length - 8) + this.malformed("truncated", "Marshal stream ended inside a float."); + const value = this.bytes.readDoubleLE(this.offset); + this.offset += 8; + return value; + } + + /** CPython `r_long` count fields that its constructors require non-negative. */ + nonNegativeInt32( + field: string, + defect: MarshalDefect = "negative-field", + ): number { + const value = this.i32(); + if (value < 0) this.malformed(defect, `${field} is negative.`); + return value; + } +} + +/** Bit length of an integer's magnitude, as CPython's `int.bit_length`. */ +export const smallBitLength = (value: bigint): number => { + if (value === 0n) return 0; + const magnitude = value < 0n ? -value : value; + return magnitude.toString(2).length; +}; diff --git a/src/python/PythonPycParser.test.ts b/src/python/PythonPycParser.test.ts index 5f7ff4eca5..5744181e21 100644 --- a/src/python/PythonPycParser.test.ts +++ b/src/python/PythonPycParser.test.ts @@ -135,6 +135,28 @@ it("rejects truncated marshal without partial success", () => { ); }); +it.each([ + { type: 0x67, message: "Marshal stream ended inside a float." }, + { type: 0x49, message: "Marshal stream ended inside a 64-bit integer." }, +])( + "reports a truncated 8-byte marshal value 0x$type as a malformed truncation", + ({ type, message }) => { + const bytes = Buffer.concat([ + pythonPycHeader(3571), + Buffer.from([type, 0, 0, 0, 0]), + ]); + expect(() => parse(bytes)).toThrow( + expect.objectContaining({ + message, + failure: expect.objectContaining({ + kind: "malformed", + field: "truncated", + }), + }), + ); + }, +); + it.each([ { magic: 3571, layout: "py311", version: "3.13" }, { magic: 3627, layout: "py311", version: "3.14" }, diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts index de53ebbdd3..e54334562c 100644 --- a/src/python/PythonPycParser.ts +++ b/src/python/PythonPycParser.ts @@ -1,6 +1,16 @@ import { observePythonBytecodePrefix } from "../domain/python/pythonBytecodeIdentity.js"; import { createHash } from "node:crypto"; -import { decodeMarshalUtf8 } from "../domain/python/marshalUtf8.js"; +import { + MarshalReader, + smallBitLength, + type MarshalBytes, + type MarshalCoreValue, + type MarshalInt, + type MarshalLimitExceeded, + type MarshalList, + type MarshalString, + type MarshalTuple, +} from "../domain/python/marshalReader.js"; import { PYTHON_PYC_CHECK_SOURCE, PYTHON_PYC_CONST_PREVIEW_LIMIT, @@ -16,41 +26,28 @@ import { type PythonPycConst, } from "../domain/python/pythonPyc.js"; import { malformedInput, unsupportedInput } from "../domain/parseFailure.js"; +import { BinaryReader } from "../native/metadata/BinaryContainer.js"; import { PYTHON_PYC_FORMAT, PythonPycResourceFailure, } from "./PythonPyc.errors.js"; import { pythonPycBytecodeIdentity } from "./PythonPycBytecodeEquality.js"; -const FLAG_REF = 0x80; const TYPE_NULL = 0x30; const TYPE_NONE = 0x4e; const TYPE_FALSE = 0x46; const TYPE_TRUE = 0x54; const TYPE_STOPITER = 0x53; const TYPE_ELLIPSIS = 0x2e; -const TYPE_INT = 0x69; const TYPE_INT64 = 0x49; const TYPE_FLOAT_TEXT = 0x66; const TYPE_FLOAT = 0x67; const TYPE_COMPLEX_TEXT = 0x78; const TYPE_COMPLEX = 0x79; -const TYPE_LONG = 0x6c; -const TYPE_BYTES = 0x73; -const TYPE_INTERNED = 0x74; -const TYPE_REF = 0x72; -const TYPE_TUPLE = 0x28; -const TYPE_LIST = 0x5b; const TYPE_DICT = 0x7b; const TYPE_CODE = 0x63; -const TYPE_UNICODE = 0x75; const TYPE_SET = 0x3c; const TYPE_FROZENSET = 0x3e; -const TYPE_ASCII = 0x61; -const TYPE_ASCII_INTERNED = 0x41; -const TYPE_SMALL_TUPLE = 0x29; -const TYPE_SHORT_ASCII = 0x7a; -const TYPE_SHORT_ASCII_INTERNED = 0x5a; /** CPython 3.14 slice constants; `PYTHON_PYC_SLICE_MAGIC` gates the tag. */ const TYPE_SLICE = 0x3a; const TYPES_WITHOUT_REFERENCE_SLOTS = new Set([ @@ -70,25 +67,18 @@ const CO_NOFREE = 0x40; const CO_FAST_LOCAL = 0x20; type MarshalValue = + | MarshalInt + | MarshalBytes + | MarshalString + | MarshalTuple + | MarshalList | { readonly kind: "null" } | { readonly kind: "none" } | { readonly kind: "bool"; readonly value: boolean } | { readonly kind: "ellipsis" } | { readonly kind: "stopiter" } - | { readonly kind: "int"; readonly value: bigint; readonly bits: number } | { readonly kind: "float"; readonly value: number } | { readonly kind: "complex"; readonly real: number; readonly imag: number } - | { readonly kind: "bytes"; readonly value: Buffer } - | { - readonly kind: "str"; - readonly value: string; - readonly bytes: number; - /** Marshal bytes. JavaScript strings merge lone surrogates with pairs. */ - readonly raw: Buffer; - readonly previewEnd: number; - } - | { readonly kind: "tuple"; readonly items: MarshalValue[] } - | { readonly kind: "list"; readonly items: MarshalValue[] } | { readonly kind: "set"; readonly items: MarshalValue[] } | { readonly kind: "frozenset"; readonly items: MarshalValue[] } /** CPython 3.14+ slice constant: start, stop, and step. */ @@ -231,11 +221,14 @@ export const parsePythonPyc = ( malformed( "Selected file is shorter than a 16-byte CPython 3.7+ pyc header.", ); + // Every CPython 3.7+ header packs flags at 4, then the source hash (8..16) + // or the source mtime (8) and size (12). + const header = new BinaryReader(bytes, true, PYTHON_PYC_FORMAT); if (mapped === null || identity === null) { // A known identity without a pinned code layout is a different outcome from an // unrecognized magic: the header below is still qualified from the same packed // fields every 3.7+ pyc uses, so report it instead of a bare rejection. - const flags = bytes.readUInt32LE(4); + const flags = header.u32(4); const hashBased = (flags & PYTHON_PYC_HASH_BASED) !== 0; const headerFacts = `16-byte header, flags 0x${flags.toString(16)}, ${ hashBased ? "hash-based" : "timestamp-based" @@ -246,9 +239,9 @@ export const parsePythonPyc = ( : `CPython ${identity.pythonVersion} header is recognized (magic ${magic}, ${headerFacts}), but no marshal code layout is pinned for that version, so its code objects cannot be decoded. Standalone marshal decoding is pinned for CPython 3.7–3.13 and the 3.14 and 3.15 magics 3627 and 3666.`, ); } - const flags = bytes.readUInt32LE(4); + const flags = header.u32(4); const hashBased = (flags & PYTHON_PYC_HASH_BASED) !== 0; - const header = { + const headerFields = { magic, python_version: identity.pythonVersion, code_layout: mapped.code_layout, @@ -256,11 +249,11 @@ export const parsePythonPyc = ( hash_based: hashBased, check_source: (flags & PYTHON_PYC_CHECK_SOURCE) !== 0, source_hash_hex: hashBased ? bytes.subarray(8, 16).toString("hex") : null, - source_mtime: hashBased ? null : bytes.readUInt32LE(8), - source_size: hashBased ? null : bytes.readUInt32LE(12), + source_mtime: hashBased ? null : header.u32(8), + source_size: hashBased ? null : header.u32(12), }; const marshal = bytes.subarray(PYTHON_PYC_HEADER_BYTES); - const reader = new MarshalReader( + const reader = new PycMarshalReader( marshal, mapped.code_layout, magic, @@ -283,7 +276,7 @@ export const parsePythonPyc = ( sha256: artifact.sha256, bytes: artifact.bytes, }, - header, + header: headerFields, code_objects: codeObjects, coverage: { status: "complete", @@ -314,11 +307,7 @@ export const parsePythonPyc = ( }; }; -class MarshalReader { - offset = 0; - objects = 0; - depthReached = 0; - readonly #intern: (MarshalValue | undefined)[] = []; +class PycMarshalReader extends MarshalReader { /** Name and identifier bytes the report carries across all code objects. */ #reportedStringBytes = 0; /** Const summaries the report copies, including tables shared by TYPE_REF. */ @@ -357,54 +346,89 @@ class MarshalReader { #nestedRetained = 0; constructor( - readonly bytes: Buffer, + bytes: Buffer, readonly layout: PythonPycCodeLayout, readonly magic: number, readonly input: PythonPycLimits, readonly rejectNegativeCodeFields: boolean, /** Marshal slice constants exist from CPython 3.14a1 magic 3608. */ readonly acceptSlices: boolean, - ) {} + ) { + super(bytes, { + format: PYTHON_PYC_FORMAT, + limits: { + maxObjects: input.max_objects, + maxDepth: input.max_depth, + maxStringBytes: input.max_string_bytes, + }, + previewLimit: PYTHON_PYC_CONST_PREVIEW_LIMIT, + publishContainers: "before-items", + }); + } - readObject(depth: number): MarshalValue { - this.objects += 1; - if (this.objects > this.input.max_objects) - throw new PythonPycResourceFailure( - "objects", - { max_objects: this.input.max_objects }, - `Marshal object count exceeds max_objects ${this.input.max_objects}.`, - ); - if (depth > this.input.max_depth) - throw new PythonPycResourceFailure( - "depth", - { max_depth: this.input.max_depth }, - `Marshal nesting exceeds max_depth ${this.input.max_depth}.`, - ); - this.depthReached = Math.max(this.depthReached, depth); - const raw = this.u8(); - const flagged = (raw & FLAG_REF) !== 0; - const type = raw & ~FLAG_REF; - // CPython masks FLAG_REF before dispatch; TYPE_REF returns its existing - // slot directly, and singleton tags bypass reference registration. - if (type === TYPE_REF) return this.readRef(); - const slot = - flagged && !TYPES_WITHOUT_REFERENCE_SLOTS.has(type) - ? this.#intern.length - : undefined; - if (slot !== undefined) this.#intern.push(undefined); - const value = this.readTyped(type, depth, slot); - // A slice with a TYPE_NULL member yields no object, as in CPython, so - // its reserved slot stays invalid for later references. - if ( - slot !== undefined && - this.#intern[slot] === undefined && - value.kind !== "null" - ) - this.#intern[slot] = value; + protected wrap(value: MarshalCoreValue): MarshalValue { return value; } - readTyped( + protected limitExceeded(exceeded: MarshalLimitExceeded): never { + const { max_objects, max_depth, max_string_bytes } = this.input; + switch (exceeded.limit) { + case "objects": + throw new PythonPycResourceFailure( + "objects", + { max_objects }, + `Marshal object count exceeds max_objects ${max_objects}.`, + ); + case "depth": + throw new PythonPycResourceFailure( + "depth", + { max_depth }, + `Marshal nesting exceeds max_depth ${max_depth}.`, + ); + case "string-bytes": + throw new PythonPycResourceFailure( + "string", + { max_string_bytes }, + `Marshal string of ${exceeded.amount} bytes exceeds max_string_bytes ${max_string_bytes}.`, + ); + default: + // Long digits; this reader sets no sequence or digit-count bound. + throw new PythonPycResourceFailure( + "string", + { max_string_bytes }, + `Marshal long of ${exceeded.amount} digit bytes exceeds max_string_bytes ${max_string_bytes}.`, + ); + } + } + + protected override reservesReference(type: number): boolean { + return !TYPES_WITHOUT_REFERENCE_SLOTS.has(type); + } + + /** + * A slice with a TYPE_NULL member yields no object, as in CPython, so its + * reserved slot stays invalid for later references. + */ + protected override isAbsent(value: MarshalValue): boolean { + return value.kind === "null"; + } + + protected override sequenceItem(item: MarshalValue): void { + if (item.kind === "null") + malformed("NULL object in marshal data for sequence."); + } + + /** Tuples whose items are still being read cannot be hashed yet. */ + protected override sequenceReading( + value: MarshalValue, + reading: boolean, + ): void { + if (value.kind !== "tuple") return; + if (reading) this.#readingTuples.add(value); + else this.#readingTuples.delete(value); + } + + protected override readExtended( type: number, depth: number, slot: number | undefined, @@ -422,10 +446,6 @@ class MarshalReader { return { kind: "stopiter" }; case TYPE_ELLIPSIS: return { kind: "ellipsis" }; - case TYPE_INT: { - const value = BigInt(this.i32()); - return { kind: "int", value, bits: smallBitLength(value) }; - } case TYPE_INT64: { const value = this.i64(); return { kind: "int", value, bits: smallBitLength(value) }; @@ -442,46 +462,6 @@ class MarshalReader { }; case TYPE_COMPLEX: return { kind: "complex", real: this.f64(), imag: this.f64() }; - case TYPE_LONG: { - const decoded = this.long(); - return { kind: "int", value: decoded.value, bits: decoded.bits }; - } - case TYPE_BYTES: - return { - kind: "bytes", - value: this.blob(this.nonNegativeInt32("Marshal string length")), - }; - case TYPE_INTERNED: - case TYPE_UNICODE: - return this.string( - this.nonNegativeInt32("Marshal string length"), - "utf8", - ); - case TYPE_ASCII: - case TYPE_ASCII_INTERNED: - return this.string( - this.nonNegativeInt32("Marshal string length"), - "latin1", - ); - case TYPE_SHORT_ASCII: - case TYPE_SHORT_ASCII_INTERNED: - return this.string(this.u8(), "latin1"); - case TYPE_TUPLE: - return this.sequenceValue( - "tuple", - this.nonNegativeInt32("Marshal tuple length"), - depth, - slot, - ); - case TYPE_SMALL_TUPLE: - return this.sequenceValue("tuple", this.u8(), depth, slot); - case TYPE_LIST: - return this.sequenceValue( - "list", - this.nonNegativeInt32("Marshal list length"), - depth, - slot, - ); case TYPE_SET: return this.setValue( "set", @@ -507,7 +487,7 @@ class MarshalReader { ); return this.slice(depth); default: - return malformed(`Unsupported marshal type 0x${type.toString(16)}.`); + return super.readExtended(type, depth, slot); } } @@ -929,20 +909,6 @@ class MarshalReader { ); } - readRef(): MarshalValue { - const index = this.u32(); - const value = this.#intern[index]; - if (value === undefined) - return malformed( - `TYPE_REF index ${index} is missing or still being constructed.`, - ); - return value; - } - - #publish(slot: number | undefined, value: MarshalValue): void { - if (slot !== undefined) this.#intern[slot] = value; - } - /** * CPython 3.14 reserves a slice's reference slot before reading its start, * stop, and step, so a reference back into the slice is still invalid. A @@ -959,31 +925,6 @@ class MarshalReader { return { kind: "slice", items }; } - /** - * CPython registers tuples and lists before reading items, so a container - * can refer to itself. The same object is what later TYPE_REF entries see. - */ - sequenceValue( - kind: "tuple" | "list", - count: number, - depth: number, - slot: number | undefined, - ): MarshalValue { - const items: MarshalValue[] = []; - const value: MarshalValue = - kind === "tuple" ? { kind: "tuple", items } : { kind: "list", items }; - this.#publish(slot, value); - if (kind === "tuple") this.#readingTuples.add(value); - for (let index = 0; index < count; index += 1) { - const item = this.readObject(depth + 1); - if (item.kind === "null") - malformed("NULL object in marshal data for sequence."); - items.push(item); - } - this.#readingTuples.delete(value); - return value; - } - /** * Sets are registered before their members. Frozensets are registered after, * matching CPython, so a reference inside a frozenset body is still invalid. @@ -998,7 +939,7 @@ class MarshalReader { const items: MarshalValue[] = []; const value: MarshalValue = kind === "set" ? { kind: "set", items } : { kind: "frozenset", items }; - if (kind === "set") this.#publish(slot, value); + if (kind === "set") this.publish(slot, value); for (let index = 0; index < count; index += 1) { const item = this.readObject(depth + 1); if (item.kind === "null") @@ -1020,7 +961,7 @@ class MarshalReader { dict(depth: number, slot: number | undefined): MarshalValue { const items: [MarshalValue, MarshalValue][] = []; const value: MarshalValue = { kind: "dict", items }; - this.#publish(slot, value); + this.publish(slot, value); const indexes = new Map(); for (;;) { const key = this.readObject(depth + 1); @@ -1543,53 +1484,6 @@ class MarshalReader { * the digits against the string budget (they never use another one), reject * the digits CPython rejects, and build the value in one linear pass. */ - long(): { value: bigint; bits: number } { - const count = this.i32(); - const size = Math.abs(count); - if (size * 2 > this.input.max_string_bytes) - throw new PythonPycResourceFailure( - "string", - { max_string_bytes: this.input.max_string_bytes }, - `Marshal long of ${size * 2} digit bytes exceeds max_string_bytes ${this.input.max_string_bytes}.`, - ); - if (this.offset > this.bytes.length - size * 2) - malformed("Marshal long overruns the selected file."); - const nibbles: number[] = []; - let carry = 0; - let carryBits = 0; - let digit = 0; - for (let index = 0; index < size; index += 1) { - digit = this.u16(); - if (digit >= 0x8000) malformed("Marshal long digit is out of range."); - carry |= digit << carryBits; - carryBits += 15; - while (carryBits >= 4) { - nibbles.push(carry & 0xf); - carry >>>= 4; - carryBits -= 4; - } - } - if (size > 0 && digit === 0) malformed("Marshal long is not normalized."); - if (carryBits > 0) nibbles.push(carry); - let bits = 0; - if (size > 0) { - let top = digit; - while (top > 0) { - top >>>= 1; - bits += 1; - } - bits += (size - 1) * 15; - } - if (nibbles.length === 0) return { value: 0n, bits: 0 }; - const magnitude = BigInt( - `0x${nibbles - .reverse() - .map((nibble) => nibble.toString(16)) - .join("")}`, - ); - return { value: count < 0 ? -magnitude : magnitude, bits }; - } - /** Marshal version 0 writes a one-byte length and a CPython float literal. */ textFloat(): number { const bytes = this.blob(this.u8()); @@ -1600,104 +1494,6 @@ class MarshalReader { malformed("Marshal float text is not a CPython float.") ); } - - string(length: number, encoding: "latin1" | "utf8"): MarshalValue { - const bytes = this.blob(length); - if (encoding === "latin1") { - // CPython reads the ASCII-tagged forms as one-byte Unicode, even when - // high bytes appear. Identity must agree with the UTF-8 string form. - const value = bytes.toString("latin1"); - return { - kind: "str", - value, - previewEnd: Math.min(length, PYTHON_PYC_CONST_PREVIEW_LIMIT), - bytes: length, - raw: bytes.some((byte) => byte > 0x7f) - ? Buffer.from(value, "utf8") - : bytes, - }; - } - const decoded = decodeMarshalUtf8(bytes, PYTHON_PYC_CONST_PREVIEW_LIMIT); - if (!decoded.ok) return malformed("Marshal string is not valid UTF-8."); - return { - kind: "str", - value: decoded.value.text, - previewEnd: decoded.value.prefixEnd, - bytes: length, - raw: bytes, - }; - } - - blob(length: number): Buffer { - if (length > this.input.max_string_bytes) - throw new PythonPycResourceFailure( - "string", - { max_string_bytes: this.input.max_string_bytes }, - `Marshal string of ${length} bytes exceeds max_string_bytes ${this.input.max_string_bytes}.`, - ); - if (this.offset > this.bytes.length - length) - malformed("Marshal string overruns the selected file."); - const value = this.bytes.subarray(this.offset, this.offset + length); - this.offset += length; - return Buffer.from(value); - } - - u8(): number { - if (this.offset >= this.bytes.length) - malformed("Marshal stream ended inside an object."); - const value = this.bytes[this.offset]; - this.offset += 1; - if (value === undefined) - return malformed("Marshal stream ended inside an object."); - return value; - } - - u16(): number { - if (this.offset > this.bytes.length - 2) - malformed("Marshal stream ended inside a 16-bit field."); - const value = this.bytes.readUInt16LE(this.offset); - this.offset += 2; - return value; - } - - u32(): number { - if (this.offset > this.bytes.length - 4) - malformed("Marshal stream ended inside a 32-bit field."); - const value = this.bytes.readUInt32LE(this.offset); - this.offset += 4; - return value; - } - - i32(): number { - if (this.offset > this.bytes.length - 4) - malformed("Marshal stream ended inside a 32-bit field."); - const value = this.bytes.readInt32LE(this.offset); - this.offset += 4; - return value; - } - - i64(): bigint { - if (this.offset > this.bytes.length - 8) - malformed("Marshal stream ended inside a 64-bit integer."); - const value = this.bytes.readBigInt64LE(this.offset); - this.offset += 8; - return value; - } - - /** CPython `r_long` count fields that its constructors require non-negative. */ - nonNegativeInt32(field: string): number { - const value = this.i32(); - if (value < 0) malformed(`${field} is negative.`); - return value; - } - - f64(): number { - if (this.offset > this.bytes.length - 8) - malformed("Marshal stream ended inside a float."); - const value = this.bytes.readDoubleLE(this.offset); - this.offset += 8; - return value; - } } /** Keep values whose identity JSON numbers cannot preserve as Python repr. */ @@ -1840,12 +1636,6 @@ const buildConstSummary = (value: MarshalValue): PythonPycConst => { } }; -const smallBitLength = (value: bigint): number => { - if (value === 0n) return 0; - const magnitude = value < 0n ? -value : value; - return magnitude.toString(2).length; -}; - /** * CPython's `PyOS_string_to_double` grammar for marshal version 0: an optional * sign, then `inf`/`infinity`, `nan`, or a decimal literal. No hex floats and