From d06ca092fb7e9624215c89c621417bf74dab8293 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 27 Oct 2026 02:09:47 +0800 Subject: [PATCH] refactor(python): consolidate bytecode identity across readers Share CPython magic framing, version and header facts while keeping standalone marshal coverage and PyInstaller reconstruction eligibility distinct. Preserve legacy headers, opaque unknowns and source bytes. Validated 272 focused tests across source, CLI and MCP workflows, a 38-test reader recheck, and check:fast. --- .../PyInstallerArtifactReader.test.ts | 64 +++++++++++++++ src/artifacts/pyinstaller/PythonBytecode.ts | 35 ++++---- src/artifacts/pyinstaller/PyzArchive.test.ts | 41 ++++++++++ src/domain/python/pythonBytecodeIdentity.ts | 79 +++++++++++++++++++ src/domain/python/pythonPyc.test.ts | 53 ++++++------- src/domain/python/pythonPyc.ts | 68 +++++++--------- src/python/PythonPycParser.test.ts | 54 +++++++++++++ src/python/PythonPycParser.ts | 25 +++--- 8 files changed, 322 insertions(+), 97 deletions(-) create mode 100644 src/domain/python/pythonBytecodeIdentity.ts diff --git a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts index 069edb298e..63b178d66c 100644 --- a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts +++ b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts @@ -1563,3 +1563,67 @@ it.each([1, 2])( } }, ); + +it.each([ + { prefix: "63000d0a", typeCode: "m" }, + { prefix: "e3000d0a", typeCode: "M" }, + { prefix: "e30d0d0a", typeCode: "m" }, +])( + "preserves CRLF-framed module $prefix instead of deriving a header from its TYPE_CODE byte", + async ({ prefix, typeCode }) => { + const original = Buffer.concat([ + Buffer.from(prefix, "hex"), + Buffer.alloc(12), + Buffer.from("original code"), + ]); + const path = await writeArchive( + "framed-code", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { name: "framed", typeCode, payload: original, compress: true }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }), + compress: false, + }, + ], + }), + ); + const reader = new PyInstallerArtifactReader(path); + try { + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + const framed = entries.find((entry) => entry.path === "framed"); + if (framed === undefined) throw new Error("Missing framed module"); + expect(framed.formatMetadata).toMatchObject({ + bytecode_header: "unknown", + bytecode_magic: null, + }); + expect((await buffer(await reader.open(framed))).equals(original)).toBe( + true, + ); + } finally { + await reader.close(); + } + const inventory = await scanCanonicalArtifactInventory(path); + const occurrence = inventory.occurrences.find( + (entry) => entry.logical_path === "framed", + ); + expect( + inventory.nodes.find( + (node) => node.artifact_id === occurrence?.artifact_id, + ), + ).toMatchObject({ + sha256: sha256(original), + size: original.length, + }); + expect( + inventory.edges.find( + (edge) => edge.occurrence_id === occurrence?.occurrence_id, + )?.relation, + ).toBe("contains"); + }, +); diff --git a/src/artifacts/pyinstaller/PythonBytecode.ts b/src/artifacts/pyinstaller/PythonBytecode.ts index e24c637edc..863a9e5829 100644 --- a/src/artifacts/pyinstaller/PythonBytecode.ts +++ b/src/artifacts/pyinstaller/PythonBytecode.ts @@ -1,26 +1,23 @@ -/** - * Final CPython bytecode magics for the minor versions whose 16-byte `.pyc` - * header this reader can reconstruct. Other magics leave the PYZ opaque. - * Each value is the four-byte little-endian magic, including the trailing CR LF. - */ -const PYTHON_BYTECODE_MAGICS: Readonly> = { - "420d0d0a": "3.7", - "550d0d0a": "3.8", - "610d0d0a": "3.9", - "6f0d0d0a": "3.10", - a70d0d0a: "3.11", - cb0d0d0a: "3.12", - f30d0d0a: "3.13", - "2b0e0d0a": "3.14", -}; +import { observePythonBytecodePrefix } from "../../domain/python/pythonBytecodeIdentity.js"; + +/** Final magics admitted for archive reconstruction, independently of standalone code-layout support. */ +const RECONSTRUCTABLE_PYTHON_MAGICS: ReadonlySet = new Set([ + 3394, 3413, 3425, 3439, 3495, 3531, 3571, 3627, +]); /** Limitation recorded when a member's digest covers a derived `.pyc` header. */ export const RECONSTRUCTED_PYTHON_HEADER_LIMITATION = "Python bytecode header was reconstructed from the archive Python magic; this digest covers the derived file."; -/** Dotted Python version for a supported bytecode magic, when one is known. */ -export const pythonVersionForMagic = (magic: Buffer): string | undefined => - PYTHON_BYTECODE_MAGICS[magic.toString("hex")]; +/** Resolve an archive-eligible magic without treating every known family as reconstructable. */ +export const pythonVersionForMagic = (magic: Buffer): string | undefined => { + if (magic.length !== 4) return undefined; + const observed = observePythonBytecodePrefix(magic); + return observed.state === "framed" && + RECONSTRUCTABLE_PYTHON_MAGICS.has(observed.magic) + ? observed.identity?.pythonVersion + : undefined; +}; /** * 16-byte `.pyc` header used since Python 3.7: magic, zero flags, zero @@ -35,7 +32,7 @@ export const reconstructedPycHeader = (magic: Buffer): Buffer => { /** TYPE_CODE with or without FLAG_REF; a pyc magic takes precedence. */ export const isHeaderlessPythonCode = (prefix: Buffer): boolean => prefix.length === 4 && - !(prefix[2] === 0x0d && prefix[3] === 0x0a) && + observePythonBytecodePrefix(prefix).state !== "framed" && (prefix[0] === 0x63 || prefix[0] === 0xe3); export interface CArchiveBytecodeHeader { diff --git a/src/artifacts/pyinstaller/PyzArchive.test.ts b/src/artifacts/pyinstaller/PyzArchive.test.ts index 3aff9f4614..d1f3948127 100644 --- a/src/artifacts/pyinstaller/PyzArchive.test.ts +++ b/src/artifacts/pyinstaller/PyzArchive.test.ts @@ -1,3 +1,6 @@ +import { pythonPycFixture } from "../../python/PythonPyc.fixture.js"; +import { parsePythonPyc } from "../../python/PythonPycParser.js"; +import { inspectPythonPycInputSchema } from "../../domain/python/pythonPyc.js"; import { createHash } from "node:crypto"; import { open, writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -182,3 +185,41 @@ it("reports the signature after completing a readable table and image hash", asy error: { reason: "format", message: "PYZ archive magic does not match" }, }); }); + +it.each([ + { magic: 3412, layout: "py38", version: "3.8", reconstructable: false }, + { magic: 3571, layout: "py311", version: "3.13", reconstructable: true }, +] as const)( + "keeps standalone identity separate from PYZ reconstruction for magic $magic", + async ({ magic, layout, version, reconstructable }) => { + const pyc = pythonPycFixture(layout, { magic }); + const standalone = parsePythonPyc( + pyc, + { + path: "/fixtures/module.pyc", + bytes: pyc.length, + sha256: createHash("sha256").update(pyc).digest("hex"), + }, + inspectPythonPycInputSchema.parse({ path: "/fixtures/module.pyc" }), + ); + expect(standalone.header.python_version).toBe(version); + const image = pyzArchive({ + magic: pyc.subarray(0, 4), + modules: [{ name: "module", typeCode: 0, body: pyc.subarray(16) }], + }); + const parsed = await readImage(image); + expect(parsed).toMatchObject({ + ok: true, + value: { + kind: reconstructable ? "entries" : "opaque", + pythonMagic: pyc.subarray(0, 4), + pythonVersion: reconstructable ? version : undefined, + sha256: createHash("sha256").update(image).digest("hex"), + }, + }); + if (parsed.ok && parsed.value.kind === "entries") + expect(parsed.value.entries).toMatchObject([ + { name: "module", typeCode: 0, role: "module" }, + ]); + }, +); diff --git a/src/domain/python/pythonBytecodeIdentity.ts b/src/domain/python/pythonBytecodeIdentity.ts new file mode 100644 index 0000000000..f41e6b0b51 --- /dev/null +++ b/src/domain/python/pythonBytecodeIdentity.ts @@ -0,0 +1,79 @@ +/** Known CPython source-format identity, independent of decoder coverage. */ +export interface PythonBytecodeIdentity { + readonly pythonVersion: string; + readonly headerBytes: 12 | 16; +} + +const identities = ( + values: readonly number[], + pythonVersion: string, + headerBytes: 12 | 16 = 16, +): ReadonlyArray => + values.map((magic) => [magic, { pythonVersion, headerBytes }]); + +const range = (start: number, end: number): readonly number[] => + Array.from({ length: end - start + 1 }, (_, offset) => start + offset); + +/** Pinned identities include archive reconstruction and standalone decoder inputs. */ +const PYTHON_BYTECODE_IDENTITIES: ReadonlyMap = + new Map([ + ...identities([3390, 3391], "3.7", 12), + ...identities([3392, 3393, 3394], "3.7"), + ...identities([3400, 3401, 3410, 3411, 3412, 3413], "3.8"), + ...identities(range(3420, 3425), "3.9"), + ...identities(range(3430, 3439), "3.10"), + ...identities([3494, 3495], "3.11"), + ...identities(range(3500, 3531), "3.12"), + ...identities(range(3550, 3571), "3.13"), + ...identities([3627], "3.14"), + ]); + +export const pythonBytecodeIdentity = ( + magic: number, +): PythonBytecodeIdentity | null => + PYTHON_BYTECODE_IDENTITIES.get(magic) ?? null; + +export type PythonBytecodePrefix = + | { + readonly state: "incomplete"; + readonly prefix: Uint8Array; + readonly magic: number | null; + readonly identity: null; + } + | { + readonly state: "invalid-crlf"; + readonly prefix: Uint8Array; + readonly magic: number; + readonly identity: null; + } + | { + readonly state: "framed"; + readonly prefix: Uint8Array; + readonly magic: number; + readonly identity: PythonBytecodeIdentity | null; + }; + +/** Borrow the source prefix; a CR LF trailer establishes framing even for unknown magic. */ +export const observePythonBytecodePrefix = ( + bytes: Uint8Array, +): PythonBytecodePrefix => { + const prefix = bytes.subarray(0, 4); + const magic = + prefix.byteLength < 2 + ? null + : new DataView( + prefix.buffer, + prefix.byteOffset, + prefix.byteLength, + ).getUint16(0, true); + if (prefix.byteLength < 4 || magic === null) + return { state: "incomplete", prefix, magic, identity: null }; + if (prefix[2] !== 0x0d || prefix[3] !== 0x0a) + return { state: "invalid-crlf", prefix, magic, identity: null }; + return { + state: "framed", + prefix, + magic, + identity: pythonBytecodeIdentity(magic), + }; +}; diff --git a/src/domain/python/pythonPyc.test.ts b/src/domain/python/pythonPyc.test.ts index 4345d0427b..5d75c654c1 100644 --- a/src/domain/python/pythonPyc.test.ts +++ b/src/domain/python/pythonPyc.test.ts @@ -1,51 +1,50 @@ import { expect, it } from "vitest"; import { inspectPythonPycInputSchema, - pythonPycMagicEntry, - PYTHON_PYC_MAGIC_TABLE, + pythonPycMarshalProfile, + PYTHON_PYC_MARSHAL_PROFILES, } from "./pythonPyc.js"; +import { pythonBytecodeIdentity } from "./pythonBytecodeIdentity.js"; -it("maps pinned CPython 3.7–3.13 magics and leaves gaps unknown", () => { - expect(pythonPycMagicEntry(3394)).toEqual({ - python_version: "3.7", +it("keeps pinned CPython identity separate from standalone marshal coverage", () => { + expect(pythonPycMarshalProfile(3394)).toEqual({ code_layout: "py37", - header_bytes: 16, negative_code_fields: "accepted", }); - expect(pythonPycMagicEntry(3390)).toMatchObject({ - python_version: "3.7", - header_bytes: 12, + expect(pythonBytecodeIdentity(3390)).toEqual({ + pythonVersion: "3.7", + headerBytes: 12, }); - expect(pythonPycMagicEntry(3391)).toMatchObject({ - python_version: "3.7", - header_bytes: 12, + expect(pythonBytecodeIdentity(3391)).toEqual({ + pythonVersion: "3.7", + headerBytes: 12, }); - expect(pythonPycMagicEntry(3413)).toEqual({ - python_version: "3.8", + expect(pythonPycMarshalProfile(3413)).toEqual({ code_layout: "py38", - header_bytes: 16, negative_code_fields: "rejected", }); - expect(pythonPycMagicEntry(3401)?.negative_code_fields).toBe("accepted"); - expect(pythonPycMagicEntry(3410)?.negative_code_fields).toBe( + expect(pythonPycMarshalProfile(3401)?.negative_code_fields).toBe("accepted"); + expect(pythonPycMarshalProfile(3410)?.negative_code_fields).toBe( "revision-dependent", ); - expect(pythonPycMagicEntry(3411)?.negative_code_fields).toBe("rejected"); - expect(pythonPycMagicEntry(3531)).toEqual({ - python_version: "3.12", + expect(pythonPycMarshalProfile(3411)?.negative_code_fields).toBe("rejected"); + expect(pythonPycMarshalProfile(3531)).toEqual({ code_layout: "py311", - header_bytes: 16, negative_code_fields: "rejected", }); - expect(pythonPycMagicEntry(3571)).toEqual({ - python_version: "3.13", + expect(pythonPycMarshalProfile(3571)).toEqual({ code_layout: "py311", - header_bytes: 16, negative_code_fields: "rejected", }); - expect(pythonPycMagicEntry(3450)).toBeNull(); - expect(pythonPycMagicEntry(62171)).toBeNull(); - expect(PYTHON_PYC_MAGIC_TABLE.size).toBeGreaterThan(80); + // Recognizing a newer CPython identity does not authorize its code layout. + expect(pythonBytecodeIdentity(3627)).toEqual({ + pythonVersion: "3.14", + headerBytes: 16, + }); + expect(pythonPycMarshalProfile(3627)).toBeNull(); + expect(pythonPycMarshalProfile(3450)).toBeNull(); + expect(pythonPycMarshalProfile(62171)).toBeNull(); + expect(PYTHON_PYC_MARSHAL_PROFILES.size).toBeGreaterThan(80); }); it("rejects extra inspect arguments", () => { diff --git a/src/domain/python/pythonPyc.ts b/src/domain/python/pythonPyc.ts index adb8b2ca11..780926e9d1 100644 --- a/src/domain/python/pythonPyc.ts +++ b/src/domain/python/pythonPyc.ts @@ -16,66 +16,52 @@ const nonNegativeInt32 = z.number().int().min(0).max(0x7fffffff); export const PYTHON_PYC_CODE_LAYOUTS = ["py37", "py38", "py311"] as const; export type PythonPycCodeLayout = (typeof PYTHON_PYC_CODE_LAYOUTS)[number]; -type MagicEntry = { - readonly python_version: string; +interface PythonPycMarshalProfile { readonly code_layout: PythonPycCodeLayout; - readonly header_bytes: 12 | 16; /** CPython constructor policy for negative co_stacksize/co_flags fields. */ readonly negative_code_fields: "accepted" | "revision-dependent" | "rejected"; -}; +} -const magics = ( +const profiles = ( values: readonly number[], - python_version: string, code_layout: PythonPycCodeLayout, - header_bytes: 12 | 16 = PYTHON_PYC_HEADER_BYTES, - negative_code_fields: MagicEntry["negative_code_fields"] = "rejected", -): ReadonlyArray => - values.map((magic) => [ - magic, - { python_version, code_layout, header_bytes, negative_code_fields }, - ]); + negative_code_fields: PythonPycMarshalProfile["negative_code_fields"] = "rejected", +): ReadonlyArray => + values.map((magic) => [magic, { code_layout, negative_code_fields }]); -const consecutive = ( +const consecutiveProfiles = ( start: number, end: number, - python_version: string, code_layout: PythonPycCodeLayout, -): ReadonlyArray => - magics( +): ReadonlyArray => + profiles( Array.from({ length: end - start + 1 }, (_, offset) => start + offset), - python_version, code_layout, ); -/** - * Pinned CPython pyc magics for 3.7–3.13. Identity is the 16-bit little-endian - * field; `\r\n` is required separately. Layout follows marshal TYPE_CODE, not - * bytecode opcode tables. - */ -export const PYTHON_PYC_MAGIC_TABLE: ReadonlyMap = new Map([ - ...magics([3390, 3391], "3.7", "py37", 12, "accepted"), - ...magics([3392, 3393, 3394], "3.7", "py37", 16, "accepted"), - ...magics([3400, 3401], "3.8", "py37", 16, "accepted"), +/** Standalone marshal coverage; source identity and header shape have a separate owner. */ +export const PYTHON_PYC_MARSHAL_PROFILES: ReadonlyMap< + number, + PythonPycMarshalProfile +> = new Map([ + ...profiles([3390, 3391, 3392, 3393, 3394], "py37", "accepted"), + ...profiles([3400, 3401], "py37", "accepted"), // 3410 spans 3.8.0a4 and b1; a9f05d69 added constructor sign checks between. - ...magics([3410], "3.8", "py38", 16, "revision-dependent"), - ...magics([3411, 3412, 3413], "3.8", "py38", 16, "rejected"), - ...magics([3420, 3421, 3422, 3423, 3424, 3425], "3.9", "py38"), - ...magics( - [3430, 3431, 3432, 3433, 3434, 3435, 3436, 3437, 3438, 3439], - "3.10", - "py38", - ), + ...profiles([3410], "py38", "revision-dependent"), + ...profiles([3411, 3412, 3413], "py38"), + ...consecutiveProfiles(3420, 3425, "py38"), + ...consecutiveProfiles(3430, 3439, "py38"), // 3.11 alphas before 3494 marshal other code layouts: no co_qualname // before 3460, and PEP 657 end-line and column tables until 3494. - ...consecutive(3494, 3495, "3.11", "py311"), - ...consecutive(3500, 3531, "3.12", "py311"), - ...consecutive(3550, 3571, "3.13", "py311"), + ...consecutiveProfiles(3494, 3495, "py311"), + ...consecutiveProfiles(3500, 3531, "py311"), + ...consecutiveProfiles(3550, 3571, "py311"), ]); -/** Map a pyc magic integer to its pinned CPython family, or null when unknown. */ -export const pythonPycMagicEntry = (magic: number): MagicEntry | null => - PYTHON_PYC_MAGIC_TABLE.get(magic) ?? null; +export const pythonPycMarshalProfile = ( + magic: number, +): PythonPycMarshalProfile | null => + PYTHON_PYC_MARSHAL_PROFILES.get(magic) ?? null; export const PYTHON_PYC_LIMIT_MAXIMA = { max_file_bytes: 64 * 1024 * 1024, diff --git a/src/python/PythonPycParser.test.ts b/src/python/PythonPycParser.test.ts index da2fec700d..246d6cb569 100644 --- a/src/python/PythonPycParser.test.ts +++ b/src/python/PythonPycParser.test.ts @@ -139,6 +139,54 @@ it("fails unknown magics and truncated marshal without partial success", () => { ); }); +it.each([ + { magic: 3571, layout: "py311", version: "3.13" }, + { magic: 3401, layout: "py37", version: "3.8" }, +] as const)( + "decodes supported standalone magic $magic with its own marshal layout", + ({ magic, layout, version }) => { + const result = parse( + Buffer.concat([pythonPycHeader(magic), marshalCode(layout)]), + ); + expect(result.header).toMatchObject({ + magic, + python_version: version, + code_layout: layout, + }); + expect(root(result).name).toBe(""); + }, +); + +it("keeps a known newer CPython identity outside standalone decoder coverage", () => { + expect(() => parse(pythonPycHeader(3627))).toThrow( + expect.objectContaining({ + kind: "unsupported", + message: expect.stringContaining( + "outside the pinned CPython 3.7–3.13 table", + ), + }), + ); +}); + +it("reports a short unknown header before unsupported magic, and bad CR LF before either", () => { + const unknown = pythonPycHeader(1).subarray(0, 4); + expect(() => parse(unknown)).toThrow( + expect.objectContaining({ + kind: "malformed", + message: + "Selected file is shorter than a 16-byte CPython 3.7+ pyc header.", + }), + ); + const badTrailer = Buffer.from(unknown); + badTrailer[3] = 0; + expect(() => parse(badTrailer)).toThrow( + expect.objectContaining({ + kind: "malformed", + message: "Pyc magic is not a little-endian integer followed by CR LF.", + }), + ); +}); + it("classifies the 12-byte 3.7 alpha header as unsupported", () => { for (const magic of [3390, 3391]) { const header = Buffer.alloc(12); @@ -151,6 +199,12 @@ it("classifies the 12-byte 3.7 alpha header as unsupported", () => { message: expect.stringContaining("unsupported 12-byte CPython header"), }), ); + expect(() => parse(header.subarray(0, 4))).toThrow( + expect.objectContaining({ + kind: "unsupported", + message: expect.stringContaining("unsupported 12-byte CPython header"), + }), + ); } }); diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts index af6b76394c..4d39a2544e 100644 --- a/src/python/PythonPycParser.ts +++ b/src/python/PythonPycParser.ts @@ -1,3 +1,4 @@ +import { observePythonBytecodePrefix } from "../domain/python/pythonBytecodeIdentity.js"; import { createHash } from "node:crypto"; import { decodeMarshalUtf8 } from "../domain/python/marshalUtf8.js"; import { @@ -6,7 +7,7 @@ import { PYTHON_PYC_HASH_BASED, PYTHON_PYC_HEADER_BYTES, PYTHON_PYC_INT_PREVIEW_BITS, - pythonPycMagicEntry, + pythonPycMarshalProfile, type InspectPythonPycInput, type PythonPyc, type PythonPycCodeLayout, @@ -205,20 +206,24 @@ export const parsePythonPyc = ( ): PythonPyc => { if (bytes.length !== artifact.bytes) malformed("Selected artifact length disagrees with the retained bytes."); - if (bytes.length < 4) malformed("Selected file is shorter than a pyc magic."); - if (bytes[2] !== 0x0d || bytes[3] !== 0x0a) - malformed("Pyc magic is not a little-endian integer followed by CR LF."); - const magic = bytes.readUInt16LE(0); - const mapped = pythonPycMagicEntry(magic); - if (mapped !== null && mapped.header_bytes !== PYTHON_PYC_HEADER_BYTES) + const prefix = observePythonBytecodePrefix(bytes); + if (prefix.state === "incomplete") + return malformed("Selected file is shorter than a pyc magic."); + if (prefix.state === "invalid-crlf") + return malformed( + "Pyc magic is not a little-endian integer followed by CR LF.", + ); + const { magic, identity } = prefix; + const mapped = pythonPycMarshalProfile(magic); + if (identity !== null && identity.headerBytes !== PYTHON_PYC_HEADER_BYTES) return unsupported( - `Pyc magic ${magic} uses an unsupported ${mapped.header_bytes}-byte CPython header.`, + `Pyc magic ${magic} uses an unsupported ${identity.headerBytes}-byte CPython header.`, ); if (bytes.length < PYTHON_PYC_HEADER_BYTES) malformed( "Selected file is shorter than a 16-byte CPython 3.7+ pyc header.", ); - if (mapped === null) + if (mapped === null || identity === null) return unsupported( `Pyc magic ${magic} is outside the pinned CPython 3.7–3.13 table.`, ); @@ -226,7 +231,7 @@ export const parsePythonPyc = ( const hashBased = (flags & PYTHON_PYC_HASH_BASED) !== 0; const header = { magic, - python_version: mapped.python_version, + python_version: identity.pythonVersion, code_layout: mapped.code_layout, flags, hash_based: hashBased,