From 1e80a96094ec176d1ca568a0696533ad29f33aca Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:44:54 +0800 Subject: [PATCH] fix(python): handle one-byte marshal strings and deep code references (#9895) * fix(python): decode marshal one-byte strings as Latin-1 * fix(python): hash referenced code objects iteratively --------- Co-authored-by: Cursor Agent --- .../pyinstaller/MarshalSubset.test.ts | 5 + src/artifacts/pyinstaller/MarshalSubset.ts | 10 +- src/python/PythonPycParser.test.ts | 54 ++++++++++ src/python/PythonPycParser.ts | 100 +++++++++++++++--- 4 files changed, 144 insertions(+), 25 deletions(-) diff --git a/src/artifacts/pyinstaller/MarshalSubset.test.ts b/src/artifacts/pyinstaller/MarshalSubset.test.ts index 62832ad7c0..b9ff396f46 100644 --- a/src/artifacts/pyinstaller/MarshalSubset.test.ts +++ b/src/artifacts/pyinstaller/MarshalSubset.test.ts @@ -20,6 +20,11 @@ it("reads a Python 3.9 reference-flagged PYZ table", () => { ]); }); +it("reads marshal one-byte string tags as Latin-1", () => { + for (const hex of ["6101000000e9", "4101000000e9", "7a01e9", "5a01e9"]) + expect(readMarshalSubset(bytes(hex))).toEqual({ ok: true, value: "é" }); +}); + it("decodes CPython marshal Unicode with surrogatepass", () => { expect( readMarshalSubset( diff --git a/src/artifacts/pyinstaller/MarshalSubset.ts b/src/artifacts/pyinstaller/MarshalSubset.ts index 5898aefc77..ad3584eef1 100644 --- a/src/artifacts/pyinstaller/MarshalSubset.ts +++ b/src/artifacts/pyinstaller/MarshalSubset.ts @@ -168,14 +168,8 @@ const readAsciiBytes = ( ): Result => { const bytes = readBytes(cursor, length); if (!bytes.ok) return bytes; - if (bytes.value.some((byte) => byte > 0x7f)) - return err( - pyInstallerFailure( - "format", - "PYZ marshal ASCII string contains a non-ASCII byte", - ), - ); - return ok(bytes.value.toString("ascii")); + // These tags use CPython's one-byte Unicode constructor, not an ASCII check. + return ok(bytes.value.toString("latin1")); }; const readUtf8 = ( diff --git a/src/python/PythonPycParser.test.ts b/src/python/PythonPycParser.test.ts index 6c0bae3ef0..26a81f15db 100644 --- a/src/python/PythonPycParser.test.ts +++ b/src/python/PythonPycParser.test.ts @@ -407,6 +407,30 @@ it("charges long digits to the string budget and keeps large previews bounded", ); }); +it("reads marshal one-byte string tags as Latin-1 with Unicode identity", () => { + for (const tag of [0x61, 0x41, 0x7a, 0x5a]) { + const encoded = Buffer.concat([ + Buffer.from([tag]), + tag === 0x61 || tag === 0x41 ? u32(1) : Buffer.from([1]), + Buffer.from([0xe9]), + ]); + const sameUnicode = marshalUnicode(Buffer.from("é", "utf8")); + const set = Buffer.concat([ + Buffer.from([0x3c]), + u32(2), + encoded, + sameUnicode, + ]); + const report = root(parse(withConsts([encoded, set]))); + expect(report.consts[0]).toMatchObject({ + kind: "str", + value: "é", + bytes: 1, + }); + expect(report.consts[1]).toEqual({ kind: "set", length: 1 }); + } +}); + it("decodes marshal strings with CPython surrogatepass rules", () => { const report = parse( withConsts([marshalUnicode(Buffer.from("efbbbf73eda080efbbbf", "hex"))]), @@ -1534,6 +1558,36 @@ it("hashes a long TYPE_REF tuple chain without recursing", () => { expect(pythonPycSchema.safeParse(report).success).toBe(true); }); +it("hashes shallow TYPE_REF code chains without using the JavaScript stack", () => { + const links = 3000; + const codes = Array.from({ length: links }, (_, index) => + marshalCode("py311", { + flagged: true, + name: "f", + consts: index === 0 ? [Buffer.from([0x4e])] : [marshalRef(index - 1)], + }), + ); + const bytes = Buffer.concat([ + pythonPycHeader(3531), + marshalCode("py311", { + constsObject: Buffer.concat([ + Buffer.from([0x28]), + u32(links + 1), + ...codes, + Buffer.from([0x3c]), + u32(1), + marshalRef(links - 1), + ]), + }), + ]); + const report = parseWith(bytes, { max_objects: 65_536 }); + expect(root(report).consts.at(-1)).toEqual({ kind: "set", length: 1 }); + expect(report.coverage).toMatchObject({ + status: "complete", + code_objects: links + 1, + }); +}); + it("rejects code CPython treats as corrupt instead of complete coverage", () => { const file = ( layout: "py37" | "py38" | "py311", diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts index 85bfd9ae86..d170dc10e6 100644 --- a/src/python/PythonPycParser.ts +++ b/src/python/PythonPycParser.ts @@ -332,7 +332,7 @@ class MarshalReader { readonly #codeKeys = new WeakMap(); readonly #codeIdentities = new WeakMap(); readonly #codeEquality = new WeakMap(); - /** Python hashability per code object. False while that check is in progress. */ + /** Completed Python hashability checks per code object. */ readonly #codeHashable = new WeakMap(); /** Tuples whose items are still being read; CPython cannot hash them yet. */ readonly #readingTuples = new WeakSet(); @@ -436,13 +436,19 @@ class MarshalReader { }; case TYPE_INTERNED: case TYPE_UNICODE: - return this.utf8(this.nonNegativeInt32("Marshal string length"), false); + return this.string( + this.nonNegativeInt32("Marshal string length"), + "utf8", + ); case TYPE_ASCII: case TYPE_ASCII_INTERNED: - return this.utf8(this.nonNegativeInt32("Marshal string length"), true); + return this.string( + this.nonNegativeInt32("Marshal string length"), + "latin1", + ); case TYPE_SHORT_ASCII: case TYPE_SHORT_ASCII_INTERNED: - return this.utf8(this.u8(), true); + return this.string(this.u8(), "latin1"); case TYPE_TUPLE: return this.sequenceValue( "tuple", @@ -1309,13 +1315,70 @@ class MarshalReader { #isCodeHashable(node: CodeNode): boolean { const known = this.#codeHashable.get(node); if (known !== undefined) return known; - this.#codeHashable.set(node, false); - const material = - this.#codeEquality.get(node) ?? - malformed("Code object is missing equality material."); - const hashable = this.#equalityKey(material.constsTuple, "hash") !== null; - this.#codeHashable.set(node, hashable); - return hashable; + const frames: { value: MarshalValue; finish: boolean }[] = [ + { value: { kind: "code", value: node }, finish: false }, + ]; + const gray = new Set(); + const complete = new Set(); + const reject = (): false => { + this.#codeHashable.set(node, false); + return false; + }; + while (frames.length > 0) { + this.#takeKeyStep(); + const frame = frames.pop()!; + const value = frame.value; + const identity = value.kind === "code" ? value.value : value; + if (frame.finish) { + gray.delete(identity); + if (value.kind === "code") { + const material = + this.#codeEquality.get(value.value) ?? + malformed("Code object is missing equality material."); + // Child code checks are now complete, so the existing tuple key + // walker preserves its equality and expansion budgets without recursion. + const hashable = + this.#equalityKey(material.constsTuple, "hash") !== null; + this.#codeHashable.set(value.value, hashable); + if (!hashable) return reject(); + } else complete.add(value); + continue; + } + if (value.kind === "code") { + const cached = this.#codeHashable.get(value.value); + if (cached === false) return reject(); + if (cached === true) continue; + } else { + const cached = this.#keyCache.get(value); + if (cached === null) return reject(); + if (cached !== undefined || complete.has(value)) continue; + } + if ( + gray.has(identity) || + this.#readingTuples.has(value) || + value.kind === "list" || + value.kind === "set" || + value.kind === "dict" || + value.kind === "null" + ) + return reject(); + const children = + value.kind === "code" + ? [ + ( + this.#codeEquality.get(value.value) ?? + malformed("Code object is missing equality material.") + ).constsTuple, + ] + : value.kind === "tuple" || value.kind === "frozenset" + ? value.items + : []; + gray.add(identity); + frames.push({ value, finish: true }); + for (let index = children.length - 1; index >= 0; index -= 1) + frames.push({ value: children[index]!, finish: false }); + } + return true; } #codeIdentity(node: CodeNode): number { @@ -1492,17 +1555,20 @@ class MarshalReader { ); } - utf8(length: number, ascii: boolean): MarshalValue { + string(length: number, encoding: "latin1" | "utf8"): MarshalValue { const bytes = this.blob(length); - if (ascii) { - if (bytes.some((byte) => byte > 0x7f)) - malformed("Marshal ASCII string contains a non-ASCII byte."); + if (encoding === "latin1") { + // CPython reads the ASCII-tagged forms as one-byte Unicode, even when + // high bytes appear. Identity must agree with the UTF-8 string form. + const value = bytes.toString("latin1"); return { kind: "str", - value: bytes.toString("latin1"), + value, previewEnd: Math.min(length, PYTHON_PYC_CONST_PREVIEW_LIMIT), bytes: length, - raw: bytes, + raw: bytes.some((byte) => byte > 0x7f) + ? Buffer.from(value, "utf8") + : bytes, }; } const decoded = decodeMarshalUtf8(bytes, PYTHON_PYC_CONST_PREVIEW_LIMIT);