diff --git a/docs/cli.md b/docs/cli.md index ba49b0e3a9..3c743cd68a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -473,7 +473,8 @@ applicable. gzip, xz, and zstd containers are unsupported and name utility, then inspect the decompressed file. See [Linux kernel modules](kernel-modules.md). -Inspect a CPython 3.7–3.13 `.pyc` header and nested marshal code objects +Inspect a CPython 3.7–3.13 `.pyc` header and nested marshal code objects (3.14 and +3.15 headers are qualified but their marshal layout is not pinned) without executing bytecode or installing Python: ```bash diff --git a/docs/python-pyc.md b/docs/python-pyc.md index be09af8cd3..c5b69976ee 100644 --- a/docs/python-pyc.md +++ b/docs/python-pyc.md @@ -81,9 +81,14 @@ The inline Evidence result contains the artifact path, SHA-256, and byte length. - `magic`, the 16-bit little-endian field; - `python_version` and `code_layout`, mapped from the pinned CPython 3.7–3.13 - table (`py37`, `py38`, or `py311`); 3.11 alpha magics before 3494 used + marshal table (`py37`, `py38`, or `py311`); 3.11 alpha magics before 3494 used other code layouts and are unsupported. The 3.7.0a1/a2 magics 3390/3391 - used 12-byte headers and are outside this reader's supported format; + used 12-byte headers and are outside this reader's supported format. The + identity table also recognizes 3.14 (3627) and 3.15 (3666): those magics + carry the same packed 16-byte header, so the typed refusal names the observed + version and flags, and says that no marshal code layout is pinned for it. + Without a pinned layout the code objects cannot be decoded, so a recognized + version is still not a successful decode; - `flags`, `hash_based`, and `check_source` from PEP 552; - `source_hash_hex` for hash-based pycs, or `source_mtime` and `source_size` for timestamp pycs. @@ -180,9 +185,11 @@ name-table entry (`co_names`, varnames, freevars, cellvars, and Copied constant tables and each reported nested code relationship count against that same object budget. Integer digits and expanded set or dictionary keys, including a shared tuple repeated at each level, count -against the string budget. Magics outside the pinned 3.7–3.13 table, -truncated headers, trailing marshal bytes, and a non-code top-level object are -typed failures. +against the string budget. Magics outside the pinned identity table, truncated +headers, trailing marshal bytes, and a non-code top-level object are typed +failures. A recognized 3.14/3.15 magic is a typed unsupported failure too: the +header is qualified from its packed 16-byte fields, but no marshal code layout is +pinned, so no code object is reported and the result is never partial success. A decoded header does not prove the matching `.py` is present. `python_version` is the magic table, not a running interpreter. This workflow does not recover diff --git a/src/contracts/python/pythonPycToolContracts.ts b/src/contracts/python/pythonPycToolContracts.ts index 858f7db18e..eb2f065a8a 100644 --- a/src/contracts/python/pythonPycToolContracts.ts +++ b/src/contracts/python/pythonPycToolContracts.ts @@ -31,7 +31,7 @@ export const PYTHON_PYC_TOOL_CONTRACTS = [ kind: "artifact-provider", requiresActiveTarget: true, description: - "Inspect one explicit local CPython 3.7–3.13 .pyc file with a supported 16-byte header without executing bytecode, invoking CPython or recovering source. Returns the header (magic, pinned python_version, PEP 552 flags, source hash or mtime/size), the top-level code object with nested functions, names, const summaries, and bytecode length/SHA-256. Complete output is the default. detail summary retains the complete Evidence and returns header/coverage. Include an optional python-pyc code_objects view to return selected names/indices/fields in the first call; inspect_analysis_view pages the same retained source later. Every matching display name survives under its parent-scoped source_index; nested_code represents code-constant references, not runtime calls. Marshal is walked with object, depth and string budgets; unsupported magics/layouts, truncated streams and exhausted budgets return no partial success. A decoded header is not proof the matching .py is present. Portable on supported Node hosts; no Python installation required.", + "Inspect one explicit local CPython 3.7–3.13 .pyc file with a supported 16-byte header without executing bytecode, invoking CPython or recovering source. Returns the header (magic, pinned python_version, PEP 552 flags, source hash or mtime/size), the top-level code object with nested functions, names, const summaries, and bytecode length/SHA-256. Complete output is the default. detail summary retains the complete Evidence and returns header/coverage. Include an optional python-pyc code_objects view to return selected names/indices/fields in the first call; inspect_analysis_view pages the same retained source later. Every matching display name survives under its parent-scoped source_index; nested_code represents code-constant references, not runtime calls. Marshal is walked with object, depth and string budgets; unsupported magics/layouts, truncated streams and exhausted budgets return no partial success. A recognized 3.14 (3627) or 3.15 (3666) magic is qualified from its packed 16-byte header — the refusal names the observed version and flags — but no marshal code layout is pinned for those versions, so no code object is reported. A decoded header is not proof the matching .py is present. Portable on supported Node hosts; no Python installation required.", inputSchema: inspectPythonPycRequestSchema, outputSchema: analysisDetailOutputSchemaOf( pythonPycSchema, diff --git a/src/domain/python/pythonBytecodeIdentity.ts b/src/domain/python/pythonBytecodeIdentity.ts index f41e6b0b51..dd97920cb9 100644 --- a/src/domain/python/pythonBytecodeIdentity.ts +++ b/src/domain/python/pythonBytecodeIdentity.ts @@ -26,6 +26,8 @@ const PYTHON_BYTECODE_IDENTITIES: ReadonlyMap = ...identities(range(3500, 3531), "3.12"), ...identities(range(3550, 3571), "3.13"), ...identities([3627], "3.14"), + // Reported against Python 3.15, the default `python3` on current Homebrew. + ...identities([3666], "3.15"), ]); export const pythonBytecodeIdentity = ( diff --git a/src/domain/python/pythonPyc.test.ts b/src/domain/python/pythonPyc.test.ts index 5d75c654c1..69b9d0485c 100644 --- a/src/domain/python/pythonPyc.test.ts +++ b/src/domain/python/pythonPyc.test.ts @@ -41,7 +41,12 @@ it("keeps pinned CPython identity separate from standalone marshal coverage", () pythonVersion: "3.14", headerBytes: 16, }); + expect(pythonBytecodeIdentity(3666)).toEqual({ + pythonVersion: "3.15", + headerBytes: 16, + }); expect(pythonPycMarshalProfile(3627)).toBeNull(); + expect(pythonPycMarshalProfile(3666)).toBeNull(); expect(pythonPycMarshalProfile(3450)).toBeNull(); expect(pythonPycMarshalProfile(62171)).toBeNull(); expect(PYTHON_PYC_MARSHAL_PROFILES.size).toBeGreaterThan(80); diff --git a/src/python/PythonPycParser.test.ts b/src/python/PythonPycParser.test.ts index 72a67a0f7e..fb1642ce45 100644 --- a/src/python/PythonPycParser.test.ts +++ b/src/python/PythonPycParser.test.ts @@ -157,12 +157,97 @@ it.each([ }, ); -it("keeps a known newer CPython identity outside standalone decoder coverage", () => { - expect(() => parse(pythonPycHeader(3627))).toThrow( +it.each([ + { magic: 3627, version: "3.14", flags: 0x0, mode: "timestamp-based" }, + { magic: 3666, version: "3.15", flags: 0x0, mode: "timestamp-based" }, + // A non-zero flag word and a PEP 552 hash-based header, so a refusal that hardcoded + // `flags 0x0, timestamp-based` could not pass. + { magic: 3666, version: "3.15", flags: 0x3, mode: "hash-based" }, + { magic: 3627, version: "3.14", flags: 0x1, mode: "hash-based" }, +] as const)( + "qualifies the recognized CPython $version header without a pinned code layout", + ({ magic, version, flags, mode }) => { + // The header is still a packed 16-byte 3.7+ header, so its identity and flags are + // reported even though no marshal layout is pinned for this version. + const header = pythonPycHeader(magic, { + flags, + ...(flags === 0 ? {} : { sourceHash: Buffer.alloc(8, 0xab) }), + }); + expect(() => parse(header)).toThrow( + expect.objectContaining({ + kind: "unsupported", + message: expect.stringContaining( + `CPython ${version} header is recognized`, + ), + }), + ); + expect(() => parse(header)).toThrow( + expect.objectContaining({ + message: expect.stringContaining(`magic ${magic}, 16-byte header`), + }), + ); + // The reported facts are the ones packed into this header. + expect(() => parse(header)).toThrow( + expect.objectContaining({ + message: expect.stringContaining(`flags 0x${flags.toString(16)}`), + }), + ); + expect(() => parse(header)).toThrow( + expect.objectContaining({ + message: expect.stringContaining(mode), + }), + ); + expect(() => parse(header)).toThrow( + expect.objectContaining({ + message: expect.stringContaining( + "no marshal code layout is pinned for that version", + ), + }), + ); + expect(() => parse(header)).toThrow( + expect.objectContaining({ + message: expect.stringContaining( + "marshal decoding is pinned for CPython 3.7", + ), + }), + ); + }, +); + +it("reports timestamp-based for a pyc without the hash flag and hash-based with it", () => { + // Same magic, opposite header mode: the two refusals must differ only in that fact. + const timestamp = () => + parse(pythonPycHeader(3666, { mtime: 7, sourceSize: 9 })); + const hash = () => + parse( + pythonPycHeader(3666, { flags: 0x1, sourceHash: Buffer.alloc(8, 1) }), + ); + expect(timestamp).toThrow( + expect.objectContaining({ + message: expect.stringContaining("timestamp-based"), + }), + ); + expect(timestamp).toThrow( + expect.not.objectContaining({ + message: expect.stringContaining("hash-based"), + }), + ); + expect(hash).toThrow( + expect.objectContaining({ message: expect.stringContaining("hash-based") }), + ); + expect(hash).toThrow( + expect.not.objectContaining({ + message: expect.stringContaining("timestamp-based"), + }), + ); +}); + +it("does not qualify a magic outside the pinned identity table", () => { + expect(() => parse(pythonPycHeader(1))).toThrow( expect.objectContaining({ kind: "unsupported", message: expect.stringContaining( - "outside the pinned CPython 3.7–3.13 table", + "is not a pinned CPython identity, so its header cannot be qualified", ), }), ); diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts index 4d39a2544e..0931ff47bc 100644 --- a/src/python/PythonPycParser.ts +++ b/src/python/PythonPycParser.ts @@ -223,10 +223,21 @@ export const parsePythonPyc = ( malformed( "Selected file is shorter than a 16-byte CPython 3.7+ pyc header.", ); - if (mapped === null || identity === null) + if (mapped === null || identity === null) { + // A known identity without a pinned code layout is a different outcome from an + // unrecognized magic: the header below is still qualified from the same packed + // fields every 3.7+ pyc uses, so report it instead of a bare rejection. + const flags = bytes.readUInt32LE(4); + const hashBased = (flags & PYTHON_PYC_HASH_BASED) !== 0; + const headerFacts = `16-byte header, flags 0x${flags.toString(16)}, ${ + hashBased ? "hash-based" : "timestamp-based" + }`; return unsupported( - `Pyc magic ${magic} is outside the pinned CPython 3.7–3.13 table.`, + identity === null + ? `Pyc magic ${magic} is not a pinned CPython identity, so its header cannot be qualified; standalone marshal decoding is pinned for CPython 3.7–3.13.` + : `CPython ${identity.pythonVersion} header is recognized (magic ${magic}, ${headerFacts}), but no marshal code layout is pinned for that version, so its code objects cannot be decoded. Standalone marshal decoding is pinned for CPython 3.7–3.13.`, ); + } const flags = bytes.readUInt32LE(4); const hashBased = (flags & PYTHON_PYC_HASH_BASED) !== 0; const header = {