refactor(python): read marshal scalars and bytecode magic through BinaryReader (#12362)

refactor(python): read marshal numbers through BinaryReader
This commit is contained in:
morluto authored and GitHub committed 2026-11-19 09:07:48 -05:00
1 parent 7758d03c5d
commit 156303f6ed
4 files changed
+41 -12

No files matched your search

@@ -38,8 +38,6 @@
"src/domain/pe/peResourceTreeFacts.ts": 9,
"src/domain/peSectionHeader.ts": 5,
"src/domain/pngVisualDiff.ts": 2,
"src/domain/python/marshalReader.ts": 4,
"src/domain/python/pythonBytecodeIdentity.ts": 1,
"src/domain/riscvIsaDeclaration.ts": 1,
"src/dotnet/ManagedConstantValue.ts": 12,
"src/dotnet/ManagedMemberInstructionDecoder.ts": 30,
+7 -4
View File
@@ -1,4 +1,5 @@
import { decodeMarshalUtf8 } from "./marshalUtf8.js";
import { BinaryReader } from "../binaryContainer.js";
import { Budget } from "../budget.js";
import { ParseFailureError } from "../parseFailure.js";
@@ -126,12 +127,14 @@ export abstract class MarshalReader<V extends { readonly kind: string }> {
depthReached = 0;
readonly #refs: (V | undefined)[] = [];
readonly #objects: Budget;
readonly #reader: BinaryReader;
constructor(
readonly bytes: Buffer,
readonly options: MarshalReaderOptions,
) {
this.#objects = new Budget(options.limits.maxObjects);
this.#reader = new BinaryReader(bytes, true, "python-marshal");
}
/** Objects read so far, within `maxObjects`. */
@@ -409,7 +412,7 @@ export abstract class MarshalReader<V extends { readonly kind: string }> {
"truncated",
"Marshal stream ended inside a 16-bit field.",
);
const value = this.bytes.readUInt16LE(this.offset);
const value = this.#reader.u16(this.offset);
this.offset += 2;
return value;
}
@@ -424,7 +427,7 @@ export abstract class MarshalReader<V extends { readonly kind: string }> {
"truncated",
"Marshal stream ended inside a 32-bit field.",
);
const value = this.bytes.readInt32LE(this.offset);
const value = this.#reader.i32(this.offset);
this.offset += 4;
return value;
}
@@ -436,7 +439,7 @@ export abstract class MarshalReader<V extends { readonly kind: string }> {
"truncated",
"Marshal stream ended inside a 64-bit integer.",
);
const value = this.bytes.readBigInt64LE(this.offset);
const value = this.#reader.i64(this.offset);
this.offset += 8;
return value;
}
@@ -445,7 +448,7 @@ export abstract class MarshalReader<V extends { readonly kind: string }> {
f64(): number {
if (this.offset > this.bytes.length - 8)
this.malformed("truncated", "Marshal stream ended inside a float.");
const value = this.bytes.readDoubleLE(this.offset);
const value = this.#reader.f64(this.offset);
this.offset += 8;
return value;
}
+7 -5
View File
@@ -1,3 +1,5 @@
import { BinaryReader } from "../binaryContainer.js";
/** Known CPython source-format identity, independent of decoder coverage. */
export interface PythonBytecodeIdentity {
readonly pythonVersion: string;
@@ -63,11 +65,11 @@ export const observePythonBytecodePrefix = (
const magic =
prefix.byteLength < 2
? null
: new DataView(
prefix.buffer,
prefix.byteOffset,
prefix.byteLength,
).getUint16(0, true);
: new BinaryReader(
Buffer.from(prefix.buffer, prefix.byteOffset, prefix.byteLength),
true,
"python-bytecode",
).u16(0);
if (prefix.byteLength < 4 || magic === null)
return { state: "incomplete", prefix, magic, identity: null };
if (prefix[2] !== 0x0d || prefix[3] !== 0x0a)
+27 -1
View File
@@ -5,7 +5,10 @@ import {
PYTHON_PYC_MARSHAL_PROFILES,
pythonPycHeaderSchema,
} from "./pythonPyc.js";
import { pythonBytecodeIdentity } from "./pythonBytecodeIdentity.js";
import {
observePythonBytecodePrefix,
pythonBytecodeIdentity,
} from "./pythonBytecodeIdentity.js";
const hashHeader = {
magic: 3531,
@@ -100,6 +103,29 @@ it("keeps pinned CPython identity separate from standalone marshal coverage", ()
expect(PYTHON_PYC_MARSHAL_PROFILES.size).toBeGreaterThan(80);
});
it("retains little-endian magic inside a borrowed, partially available prefix", () => {
const bytes = Uint8Array.of(0xff, 0xcb, 0x0d, 0x0d, 0x0a, 0xff).subarray(1);
expect(observePythonBytecodePrefix(bytes.subarray(0, 1))).toMatchObject({
state: "incomplete",
magic: null,
identity: null,
});
expect(observePythonBytecodePrefix(bytes.subarray(0, 2))).toMatchObject({
state: "incomplete",
magic: 3531,
identity: null,
});
const complete = observePythonBytecodePrefix(bytes);
expect(complete).toMatchObject({
state: "framed",
magic: 3531,
identity: { pythonVersion: "3.12", headerBytes: 16 },
});
expect(complete.prefix.buffer).toBe(bytes.buffer);
expect(complete.prefix.byteOffset).toBe(bytes.byteOffset);
expect(complete.prefix.byteLength).toBe(4);
});
it("rejects extra inspect arguments", () => {
expect(
inspectPythonPycInputSchema.safeParse({