From 156303f6ede33e79ee2ebf2346bc9f28529149d5 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 19 Nov 2026 09:07:48 -0500 Subject: [PATCH] refactor(python): read marshal scalars and bytecode magic through BinaryReader (#12362) refactor(python): read marshal numbers through BinaryReader --- scripts/verify/baselines/raw-buffer-read.json | 2 -- src/domain/python/marshalReader.ts | 11 +++++--- src/domain/python/pythonBytecodeIdentity.ts | 12 ++++---- src/domain/python/pythonPyc.test.ts | 28 ++++++++++++++++++- 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/scripts/verify/baselines/raw-buffer-read.json b/scripts/verify/baselines/raw-buffer-read.json index de21e8ee72..50357264bc 100644 --- a/scripts/verify/baselines/raw-buffer-read.json +++ b/scripts/verify/baselines/raw-buffer-read.json @@ -38,8 +38,6 @@ "src/domain/pe/peResourceTreeFacts.ts": 9, "src/domain/peSectionHeader.ts": 5, "src/domain/pngVisualDiff.ts": 2, - "src/domain/python/marshalReader.ts": 4, - "src/domain/python/pythonBytecodeIdentity.ts": 1, "src/domain/riscvIsaDeclaration.ts": 1, "src/dotnet/ManagedConstantValue.ts": 12, "src/dotnet/ManagedMemberInstructionDecoder.ts": 30, diff --git a/src/domain/python/marshalReader.ts b/src/domain/python/marshalReader.ts index 7a706d3ae3..93e593ea1f 100644 --- a/src/domain/python/marshalReader.ts +++ b/src/domain/python/marshalReader.ts @@ -1,4 +1,5 @@ import { decodeMarshalUtf8 } from "./marshalUtf8.js"; +import { BinaryReader } from "../binaryContainer.js"; import { Budget } from "../budget.js"; import { ParseFailureError } from "../parseFailure.js"; @@ -126,12 +127,14 @@ export abstract class MarshalReader { depthReached = 0; readonly #refs: (V | undefined)[] = []; readonly #objects: Budget; + readonly #reader: BinaryReader; constructor( readonly bytes: Buffer, readonly options: MarshalReaderOptions, ) { this.#objects = new Budget(options.limits.maxObjects); + this.#reader = new BinaryReader(bytes, true, "python-marshal"); } /** Objects read so far, within `maxObjects`. */ @@ -409,7 +412,7 @@ export abstract class MarshalReader { "truncated", "Marshal stream ended inside a 16-bit field.", ); - const value = this.bytes.readUInt16LE(this.offset); + const value = this.#reader.u16(this.offset); this.offset += 2; return value; } @@ -424,7 +427,7 @@ export abstract class MarshalReader { "truncated", "Marshal stream ended inside a 32-bit field.", ); - const value = this.bytes.readInt32LE(this.offset); + const value = this.#reader.i32(this.offset); this.offset += 4; return value; } @@ -436,7 +439,7 @@ export abstract class MarshalReader { "truncated", "Marshal stream ended inside a 64-bit integer.", ); - const value = this.bytes.readBigInt64LE(this.offset); + const value = this.#reader.i64(this.offset); this.offset += 8; return value; } @@ -445,7 +448,7 @@ export abstract class MarshalReader { f64(): number { if (this.offset > this.bytes.length - 8) this.malformed("truncated", "Marshal stream ended inside a float."); - const value = this.bytes.readDoubleLE(this.offset); + const value = this.#reader.f64(this.offset); this.offset += 8; return value; } diff --git a/src/domain/python/pythonBytecodeIdentity.ts b/src/domain/python/pythonBytecodeIdentity.ts index dd97920cb9..bd10142d53 100644 --- a/src/domain/python/pythonBytecodeIdentity.ts +++ b/src/domain/python/pythonBytecodeIdentity.ts @@ -1,3 +1,5 @@ +import { BinaryReader } from "../binaryContainer.js"; + /** Known CPython source-format identity, independent of decoder coverage. */ export interface PythonBytecodeIdentity { readonly pythonVersion: string; @@ -63,11 +65,11 @@ export const observePythonBytecodePrefix = ( const magic = prefix.byteLength < 2 ? null - : new DataView( - prefix.buffer, - prefix.byteOffset, - prefix.byteLength, - ).getUint16(0, true); + : new BinaryReader( + Buffer.from(prefix.buffer, prefix.byteOffset, prefix.byteLength), + true, + "python-bytecode", + ).u16(0); if (prefix.byteLength < 4 || magic === null) return { state: "incomplete", prefix, magic, identity: null }; if (prefix[2] !== 0x0d || prefix[3] !== 0x0a) diff --git a/src/domain/python/pythonPyc.test.ts b/src/domain/python/pythonPyc.test.ts index f793df872f..6cff9962d8 100644 --- a/src/domain/python/pythonPyc.test.ts +++ b/src/domain/python/pythonPyc.test.ts @@ -5,7 +5,10 @@ import { PYTHON_PYC_MARSHAL_PROFILES, pythonPycHeaderSchema, } from "./pythonPyc.js"; -import { pythonBytecodeIdentity } from "./pythonBytecodeIdentity.js"; +import { + observePythonBytecodePrefix, + pythonBytecodeIdentity, +} from "./pythonBytecodeIdentity.js"; const hashHeader = { magic: 3531, @@ -100,6 +103,29 @@ it("keeps pinned CPython identity separate from standalone marshal coverage", () expect(PYTHON_PYC_MARSHAL_PROFILES.size).toBeGreaterThan(80); }); +it("retains little-endian magic inside a borrowed, partially available prefix", () => { + const bytes = Uint8Array.of(0xff, 0xcb, 0x0d, 0x0d, 0x0a, 0xff).subarray(1); + expect(observePythonBytecodePrefix(bytes.subarray(0, 1))).toMatchObject({ + state: "incomplete", + magic: null, + identity: null, + }); + expect(observePythonBytecodePrefix(bytes.subarray(0, 2))).toMatchObject({ + state: "incomplete", + magic: 3531, + identity: null, + }); + const complete = observePythonBytecodePrefix(bytes); + expect(complete).toMatchObject({ + state: "framed", + magic: 3531, + identity: { pythonVersion: "3.12", headerBytes: 16 }, + }); + expect(complete.prefix.buffer).toBe(bytes.buffer); + expect(complete.prefix.byteOffset).toBe(bytes.byteOffset); + expect(complete.prefix.byteLength).toBe(4); +}); + it("rejects extra inspect arguments", () => { expect( inspectPythonPycInputSchema.safeParse({