diff --git a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts index 069edb298e..63b178d66c 100644 --- a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts +++ b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts @@ -1563,3 +1563,67 @@ it.each([1, 2])( } }, ); + +it.each([ + { prefix: "63000d0a", typeCode: "m" }, + { prefix: "e3000d0a", typeCode: "M" }, + { prefix: "e30d0d0a", typeCode: "m" }, +])( + "preserves CRLF-framed module $prefix instead of deriving a header from its TYPE_CODE byte", + async ({ prefix, typeCode }) => { + const original = Buffer.concat([ + Buffer.from(prefix, "hex"), + Buffer.alloc(12), + Buffer.from("original code"), + ]); + const path = await writeArchive( + "framed-code", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { name: "framed", typeCode, payload: original, compress: true }, + { + name: "PYZ.pyz", + typeCode: "z", + payload: pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }), + compress: false, + }, + ], + }), + ); + const reader = new PyInstallerArtifactReader(path); + try { + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + const framed = entries.find((entry) => entry.path === "framed"); + if (framed === undefined) throw new Error("Missing framed module"); + expect(framed.formatMetadata).toMatchObject({ + bytecode_header: "unknown", + bytecode_magic: null, + }); + expect((await buffer(await reader.open(framed))).equals(original)).toBe( + true, + ); + } finally { + await reader.close(); + } + const inventory = await scanCanonicalArtifactInventory(path); + const occurrence = inventory.occurrences.find( + (entry) => entry.logical_path === "framed", + ); + expect( + inventory.nodes.find( + (node) => node.artifact_id === occurrence?.artifact_id, + ), + ).toMatchObject({ + sha256: sha256(original), + size: original.length, + }); + expect( + inventory.edges.find( + (edge) => edge.occurrence_id === occurrence?.occurrence_id, + )?.relation, + ).toBe("contains"); + }, +); diff --git a/src/artifacts/pyinstaller/PythonBytecode.ts b/src/artifacts/pyinstaller/PythonBytecode.ts index e24c637edc..863a9e5829 100644 --- a/src/artifacts/pyinstaller/PythonBytecode.ts +++ b/src/artifacts/pyinstaller/PythonBytecode.ts @@ -1,26 +1,23 @@ -/** - * Final CPython bytecode magics for the minor versions whose 16-byte `.pyc` - * header this reader can reconstruct. Other magics leave the PYZ opaque. - * Each value is the four-byte little-endian magic, including the trailing CR LF. - */ -const PYTHON_BYTECODE_MAGICS: Readonly> = { - "420d0d0a": "3.7", - "550d0d0a": "3.8", - "610d0d0a": "3.9", - "6f0d0d0a": "3.10", - a70d0d0a: "3.11", - cb0d0d0a: "3.12", - f30d0d0a: "3.13", - "2b0e0d0a": "3.14", -}; +import { observePythonBytecodePrefix } from "../../domain/python/pythonBytecodeIdentity.js"; + +/** Final magics admitted for archive reconstruction, independently of standalone code-layout support. */ +const RECONSTRUCTABLE_PYTHON_MAGICS: ReadonlySet = new Set([ + 3394, 3413, 3425, 3439, 3495, 3531, 3571, 3627, +]); /** Limitation recorded when a member's digest covers a derived `.pyc` header. */ export const RECONSTRUCTED_PYTHON_HEADER_LIMITATION = "Python bytecode header was reconstructed from the archive Python magic; this digest covers the derived file."; -/** Dotted Python version for a supported bytecode magic, when one is known. */ -export const pythonVersionForMagic = (magic: Buffer): string | undefined => - PYTHON_BYTECODE_MAGICS[magic.toString("hex")]; +/** Resolve an archive-eligible magic without treating every known family as reconstructable. */ +export const pythonVersionForMagic = (magic: Buffer): string | undefined => { + if (magic.length !== 4) return undefined; + const observed = observePythonBytecodePrefix(magic); + return observed.state === "framed" && + RECONSTRUCTABLE_PYTHON_MAGICS.has(observed.magic) + ? observed.identity?.pythonVersion + : undefined; +}; /** * 16-byte `.pyc` header used since Python 3.7: magic, zero flags, zero @@ -35,7 +32,7 @@ export const reconstructedPycHeader = (magic: Buffer): Buffer => { /** TYPE_CODE with or without FLAG_REF; a pyc magic takes precedence. */ export const isHeaderlessPythonCode = (prefix: Buffer): boolean => prefix.length === 4 && - !(prefix[2] === 0x0d && prefix[3] === 0x0a) && + observePythonBytecodePrefix(prefix).state !== "framed" && (prefix[0] === 0x63 || prefix[0] === 0xe3); export interface CArchiveBytecodeHeader { diff --git a/src/artifacts/pyinstaller/PyzArchive.test.ts b/src/artifacts/pyinstaller/PyzArchive.test.ts index 3aff9f4614..d1f3948127 100644 --- a/src/artifacts/pyinstaller/PyzArchive.test.ts +++ b/src/artifacts/pyinstaller/PyzArchive.test.ts @@ -1,3 +1,6 @@ +import { pythonPycFixture } from "../../python/PythonPyc.fixture.js"; +import { parsePythonPyc } from "../../python/PythonPycParser.js"; +import { inspectPythonPycInputSchema } from "../../domain/python/pythonPyc.js"; import { createHash } from "node:crypto"; import { open, writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -182,3 +185,41 @@ it("reports the signature after completing a readable table and image hash", asy error: { reason: "format", message: "PYZ archive magic does not match" }, }); }); + +it.each([ + { magic: 3412, layout: "py38", version: "3.8", reconstructable: false }, + { magic: 3571, layout: "py311", version: "3.13", reconstructable: true }, +] as const)( + "keeps standalone identity separate from PYZ reconstruction for magic $magic", + async ({ magic, layout, version, reconstructable }) => { + const pyc = pythonPycFixture(layout, { magic }); + const standalone = parsePythonPyc( + pyc, + { + path: "/fixtures/module.pyc", + bytes: pyc.length, + sha256: createHash("sha256").update(pyc).digest("hex"), + }, + inspectPythonPycInputSchema.parse({ path: "/fixtures/module.pyc" }), + ); + expect(standalone.header.python_version).toBe(version); + const image = pyzArchive({ + magic: pyc.subarray(0, 4), + modules: [{ name: "module", typeCode: 0, body: pyc.subarray(16) }], + }); + const parsed = await readImage(image); + expect(parsed).toMatchObject({ + ok: true, + value: { + kind: reconstructable ? "entries" : "opaque", + pythonMagic: pyc.subarray(0, 4), + pythonVersion: reconstructable ? version : undefined, + sha256: createHash("sha256").update(image).digest("hex"), + }, + }); + if (parsed.ok && parsed.value.kind === "entries") + expect(parsed.value.entries).toMatchObject([ + { name: "module", typeCode: 0, role: "module" }, + ]); + }, +); diff --git a/src/domain/python/pythonBytecodeIdentity.ts b/src/domain/python/pythonBytecodeIdentity.ts new file mode 100644 index 0000000000..f41e6b0b51 --- /dev/null +++ b/src/domain/python/pythonBytecodeIdentity.ts @@ -0,0 +1,79 @@ +/** Known CPython source-format identity, independent of decoder coverage. */ +export interface PythonBytecodeIdentity { + readonly pythonVersion: string; + readonly headerBytes: 12 | 16; +} + +const identities = ( + values: readonly number[], + pythonVersion: string, + headerBytes: 12 | 16 = 16, +): ReadonlyArray => + values.map((magic) => [magic, { pythonVersion, headerBytes }]); + +const range = (start: number, end: number): readonly number[] => + Array.from({ length: end - start + 1 }, (_, offset) => start + offset); + +/** Pinned identities include archive reconstruction and standalone decoder inputs. */ +const PYTHON_BYTECODE_IDENTITIES: ReadonlyMap = + new Map([ + ...identities([3390, 3391], "3.7", 12), + ...identities([3392, 3393, 3394], "3.7"), + ...identities([3400, 3401, 3410, 3411, 3412, 3413], "3.8"), + ...identities(range(3420, 3425), "3.9"), + ...identities(range(3430, 3439), "3.10"), + ...identities([3494, 3495], "3.11"), + ...identities(range(3500, 3531), "3.12"), + ...identities(range(3550, 3571), "3.13"), + ...identities([3627], "3.14"), + ]); + +export const pythonBytecodeIdentity = ( + magic: number, +): PythonBytecodeIdentity | null => + PYTHON_BYTECODE_IDENTITIES.get(magic) ?? null; + +export type PythonBytecodePrefix = + | { + readonly state: "incomplete"; + readonly prefix: Uint8Array; + readonly magic: number | null; + readonly identity: null; + } + | { + readonly state: "invalid-crlf"; + readonly prefix: Uint8Array; + readonly magic: number; + readonly identity: null; + } + | { + readonly state: "framed"; + readonly prefix: Uint8Array; + readonly magic: number; + readonly identity: PythonBytecodeIdentity | null; + }; + +/** Borrow the source prefix; a CR LF trailer establishes framing even for unknown magic. */ +export const observePythonBytecodePrefix = ( + bytes: Uint8Array, +): PythonBytecodePrefix => { + const prefix = bytes.subarray(0, 4); + const magic = + prefix.byteLength < 2 + ? null + : new DataView( + prefix.buffer, + prefix.byteOffset, + prefix.byteLength, + ).getUint16(0, true); + if (prefix.byteLength < 4 || magic === null) + return { state: "incomplete", prefix, magic, identity: null }; + if (prefix[2] !== 0x0d || prefix[3] !== 0x0a) + return { state: "invalid-crlf", prefix, magic, identity: null }; + return { + state: "framed", + prefix, + magic, + identity: pythonBytecodeIdentity(magic), + }; +}; diff --git a/src/domain/python/pythonPyc.test.ts b/src/domain/python/pythonPyc.test.ts index 4345d0427b..5d75c654c1 100644 --- a/src/domain/python/pythonPyc.test.ts +++ b/src/domain/python/pythonPyc.test.ts @@ -1,51 +1,50 @@ import { expect, it } from "vitest"; import { inspectPythonPycInputSchema, - pythonPycMagicEntry, - PYTHON_PYC_MAGIC_TABLE, + pythonPycMarshalProfile, + PYTHON_PYC_MARSHAL_PROFILES, } from "./pythonPyc.js"; +import { pythonBytecodeIdentity } from "./pythonBytecodeIdentity.js"; -it("maps pinned CPython 3.7–3.13 magics and leaves gaps unknown", () => { - expect(pythonPycMagicEntry(3394)).toEqual({ - python_version: "3.7", +it("keeps pinned CPython identity separate from standalone marshal coverage", () => { + expect(pythonPycMarshalProfile(3394)).toEqual({ code_layout: "py37", - header_bytes: 16, negative_code_fields: "accepted", }); - expect(pythonPycMagicEntry(3390)).toMatchObject({ - python_version: "3.7", - header_bytes: 12, + expect(pythonBytecodeIdentity(3390)).toEqual({ + pythonVersion: "3.7", + headerBytes: 12, }); - expect(pythonPycMagicEntry(3391)).toMatchObject({ - python_version: "3.7", - header_bytes: 12, + expect(pythonBytecodeIdentity(3391)).toEqual({ + pythonVersion: "3.7", + headerBytes: 12, }); - expect(pythonPycMagicEntry(3413)).toEqual({ - python_version: "3.8", + expect(pythonPycMarshalProfile(3413)).toEqual({ code_layout: "py38", - header_bytes: 16, negative_code_fields: "rejected", }); - expect(pythonPycMagicEntry(3401)?.negative_code_fields).toBe("accepted"); - expect(pythonPycMagicEntry(3410)?.negative_code_fields).toBe( + expect(pythonPycMarshalProfile(3401)?.negative_code_fields).toBe("accepted"); + expect(pythonPycMarshalProfile(3410)?.negative_code_fields).toBe( "revision-dependent", ); - expect(pythonPycMagicEntry(3411)?.negative_code_fields).toBe("rejected"); - expect(pythonPycMagicEntry(3531)).toEqual({ - python_version: "3.12", + expect(pythonPycMarshalProfile(3411)?.negative_code_fields).toBe("rejected"); + expect(pythonPycMarshalProfile(3531)).toEqual({ code_layout: "py311", - header_bytes: 16, negative_code_fields: "rejected", }); - expect(pythonPycMagicEntry(3571)).toEqual({ - python_version: "3.13", + expect(pythonPycMarshalProfile(3571)).toEqual({ code_layout: "py311", - header_bytes: 16, negative_code_fields: "rejected", }); - expect(pythonPycMagicEntry(3450)).toBeNull(); - expect(pythonPycMagicEntry(62171)).toBeNull(); - expect(PYTHON_PYC_MAGIC_TABLE.size).toBeGreaterThan(80); + // Recognizing a newer CPython identity does not authorize its code layout. + expect(pythonBytecodeIdentity(3627)).toEqual({ + pythonVersion: "3.14", + headerBytes: 16, + }); + expect(pythonPycMarshalProfile(3627)).toBeNull(); + expect(pythonPycMarshalProfile(3450)).toBeNull(); + expect(pythonPycMarshalProfile(62171)).toBeNull(); + expect(PYTHON_PYC_MARSHAL_PROFILES.size).toBeGreaterThan(80); }); it("rejects extra inspect arguments", () => { diff --git a/src/domain/python/pythonPyc.ts b/src/domain/python/pythonPyc.ts index adb8b2ca11..780926e9d1 100644 --- a/src/domain/python/pythonPyc.ts +++ b/src/domain/python/pythonPyc.ts @@ -16,66 +16,52 @@ const nonNegativeInt32 = z.number().int().min(0).max(0x7fffffff); export const PYTHON_PYC_CODE_LAYOUTS = ["py37", "py38", "py311"] as const; export type PythonPycCodeLayout = (typeof PYTHON_PYC_CODE_LAYOUTS)[number]; -type MagicEntry = { - readonly python_version: string; +interface PythonPycMarshalProfile { readonly code_layout: PythonPycCodeLayout; - readonly header_bytes: 12 | 16; /** CPython constructor policy for negative co_stacksize/co_flags fields. */ readonly negative_code_fields: "accepted" | "revision-dependent" | "rejected"; -}; +} -const magics = ( +const profiles = ( values: readonly number[], - python_version: string, code_layout: PythonPycCodeLayout, - header_bytes: 12 | 16 = PYTHON_PYC_HEADER_BYTES, - negative_code_fields: MagicEntry["negative_code_fields"] = "rejected", -): ReadonlyArray => - values.map((magic) => [ - magic, - { python_version, code_layout, header_bytes, negative_code_fields }, - ]); + negative_code_fields: PythonPycMarshalProfile["negative_code_fields"] = "rejected", +): ReadonlyArray => + values.map((magic) => [magic, { code_layout, negative_code_fields }]); -const consecutive = ( +const consecutiveProfiles = ( start: number, end: number, - python_version: string, code_layout: PythonPycCodeLayout, -): ReadonlyArray => - magics( +): ReadonlyArray => + profiles( Array.from({ length: end - start + 1 }, (_, offset) => start + offset), - python_version, code_layout, ); -/** - * Pinned CPython pyc magics for 3.7–3.13. Identity is the 16-bit little-endian - * field; `\r\n` is required separately. Layout follows marshal TYPE_CODE, not - * bytecode opcode tables. - */ -export const PYTHON_PYC_MAGIC_TABLE: ReadonlyMap = new Map([ - ...magics([3390, 3391], "3.7", "py37", 12, "accepted"), - ...magics([3392, 3393, 3394], "3.7", "py37", 16, "accepted"), - ...magics([3400, 3401], "3.8", "py37", 16, "accepted"), +/** Standalone marshal coverage; source identity and header shape have a separate owner. */ +export const PYTHON_PYC_MARSHAL_PROFILES: ReadonlyMap< + number, + PythonPycMarshalProfile +> = new Map([ + ...profiles([3390, 3391, 3392, 3393, 3394], "py37", "accepted"), + ...profiles([3400, 3401], "py37", "accepted"), // 3410 spans 3.8.0a4 and b1; a9f05d69 added constructor sign checks between. - ...magics([3410], "3.8", "py38", 16, "revision-dependent"), - ...magics([3411, 3412, 3413], "3.8", "py38", 16, "rejected"), - ...magics([3420, 3421, 3422, 3423, 3424, 3425], "3.9", "py38"), - ...magics( - [3430, 3431, 3432, 3433, 3434, 3435, 3436, 3437, 3438, 3439], - "3.10", - "py38", - ), + ...profiles([3410], "py38", "revision-dependent"), + ...profiles([3411, 3412, 3413], "py38"), + ...consecutiveProfiles(3420, 3425, "py38"), + ...consecutiveProfiles(3430, 3439, "py38"), // 3.11 alphas before 3494 marshal other code layouts: no co_qualname // before 3460, and PEP 657 end-line and column tables until 3494. - ...consecutive(3494, 3495, "3.11", "py311"), - ...consecutive(3500, 3531, "3.12", "py311"), - ...consecutive(3550, 3571, "3.13", "py311"), + ...consecutiveProfiles(3494, 3495, "py311"), + ...consecutiveProfiles(3500, 3531, "py311"), + ...consecutiveProfiles(3550, 3571, "py311"), ]); -/** Map a pyc magic integer to its pinned CPython family, or null when unknown. */ -export const pythonPycMagicEntry = (magic: number): MagicEntry | null => - PYTHON_PYC_MAGIC_TABLE.get(magic) ?? null; +export const pythonPycMarshalProfile = ( + magic: number, +): PythonPycMarshalProfile | null => + PYTHON_PYC_MARSHAL_PROFILES.get(magic) ?? null; export const PYTHON_PYC_LIMIT_MAXIMA = { max_file_bytes: 64 * 1024 * 1024, diff --git a/src/python/PythonPycParser.test.ts b/src/python/PythonPycParser.test.ts index da2fec700d..246d6cb569 100644 --- a/src/python/PythonPycParser.test.ts +++ b/src/python/PythonPycParser.test.ts @@ -139,6 +139,54 @@ it("fails unknown magics and truncated marshal without partial success", () => { ); }); +it.each([ + { magic: 3571, layout: "py311", version: "3.13" }, + { magic: 3401, layout: "py37", version: "3.8" }, +] as const)( + "decodes supported standalone magic $magic with its own marshal layout", + ({ magic, layout, version }) => { + const result = parse( + Buffer.concat([pythonPycHeader(magic), marshalCode(layout)]), + ); + expect(result.header).toMatchObject({ + magic, + python_version: version, + code_layout: layout, + }); + expect(root(result).name).toBe(""); + }, +); + +it("keeps a known newer CPython identity outside standalone decoder coverage", () => { + expect(() => parse(pythonPycHeader(3627))).toThrow( + expect.objectContaining({ + kind: "unsupported", + message: expect.stringContaining( + "outside the pinned CPython 3.7–3.13 table", + ), + }), + ); +}); + +it("reports a short unknown header before unsupported magic, and bad CR LF before either", () => { + const unknown = pythonPycHeader(1).subarray(0, 4); + expect(() => parse(unknown)).toThrow( + expect.objectContaining({ + kind: "malformed", + message: + "Selected file is shorter than a 16-byte CPython 3.7+ pyc header.", + }), + ); + const badTrailer = Buffer.from(unknown); + badTrailer[3] = 0; + expect(() => parse(badTrailer)).toThrow( + expect.objectContaining({ + kind: "malformed", + message: "Pyc magic is not a little-endian integer followed by CR LF.", + }), + ); +}); + it("classifies the 12-byte 3.7 alpha header as unsupported", () => { for (const magic of [3390, 3391]) { const header = Buffer.alloc(12); @@ -151,6 +199,12 @@ it("classifies the 12-byte 3.7 alpha header as unsupported", () => { message: expect.stringContaining("unsupported 12-byte CPython header"), }), ); + expect(() => parse(header.subarray(0, 4))).toThrow( + expect.objectContaining({ + kind: "unsupported", + message: expect.stringContaining("unsupported 12-byte CPython header"), + }), + ); } }); diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts index af6b76394c..4d39a2544e 100644 --- a/src/python/PythonPycParser.ts +++ b/src/python/PythonPycParser.ts @@ -1,3 +1,4 @@ +import { observePythonBytecodePrefix } from "../domain/python/pythonBytecodeIdentity.js"; import { createHash } from "node:crypto"; import { decodeMarshalUtf8 } from "../domain/python/marshalUtf8.js"; import { @@ -6,7 +7,7 @@ import { PYTHON_PYC_HASH_BASED, PYTHON_PYC_HEADER_BYTES, PYTHON_PYC_INT_PREVIEW_BITS, - pythonPycMagicEntry, + pythonPycMarshalProfile, type InspectPythonPycInput, type PythonPyc, type PythonPycCodeLayout, @@ -205,20 +206,24 @@ export const parsePythonPyc = ( ): PythonPyc => { if (bytes.length !== artifact.bytes) malformed("Selected artifact length disagrees with the retained bytes."); - if (bytes.length < 4) malformed("Selected file is shorter than a pyc magic."); - if (bytes[2] !== 0x0d || bytes[3] !== 0x0a) - malformed("Pyc magic is not a little-endian integer followed by CR LF."); - const magic = bytes.readUInt16LE(0); - const mapped = pythonPycMagicEntry(magic); - if (mapped !== null && mapped.header_bytes !== PYTHON_PYC_HEADER_BYTES) + const prefix = observePythonBytecodePrefix(bytes); + if (prefix.state === "incomplete") + return malformed("Selected file is shorter than a pyc magic."); + if (prefix.state === "invalid-crlf") + return malformed( + "Pyc magic is not a little-endian integer followed by CR LF.", + ); + const { magic, identity } = prefix; + const mapped = pythonPycMarshalProfile(magic); + if (identity !== null && identity.headerBytes !== PYTHON_PYC_HEADER_BYTES) return unsupported( - `Pyc magic ${magic} uses an unsupported ${mapped.header_bytes}-byte CPython header.`, + `Pyc magic ${magic} uses an unsupported ${identity.headerBytes}-byte CPython header.`, ); if (bytes.length < PYTHON_PYC_HEADER_BYTES) malformed( "Selected file is shorter than a 16-byte CPython 3.7+ pyc header.", ); - if (mapped === null) + if (mapped === null || identity === null) return unsupported( `Pyc magic ${magic} is outside the pinned CPython 3.7–3.13 table.`, ); @@ -226,7 +231,7 @@ export const parsePythonPyc = ( const hashBased = (flags & PYTHON_PYC_HASH_BASED) !== 0; const header = { magic, - python_version: mapped.python_version, + python_version: identity.pythonVersion, code_layout: mapped.code_layout, flags, hash_based: hashBased,