From e46ace974464d86261448200cb33dbaea750412a Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 19 Nov 2026 15:20:31 -0500 Subject: [PATCH] fix(python): disclose inactive pyc check-source flags (#12508) --- docs/python-pyc.md | 4 +- src/domain/python/pythonPyc.ts | 40 ++++++++++++++----- src/python/PythonPycParser.ts | 27 ++++++------- tests/acceptance/pythonPycCli.test.ts | 55 +++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 24 deletions(-) diff --git a/docs/python-pyc.md b/docs/python-pyc.md index 457b5bbdcd..d63fea9dd5 100644 --- a/docs/python-pyc.md +++ b/docs/python-pyc.md @@ -88,7 +88,9 @@ The inline Evidence result contains the artifact path, SHA-256, and byte length. magics before 3494 used other code layouts and are unsupported. The 3.7.0a1/a2 magics 3390/3391 used 12-byte headers and are outside this reader's supported format; -- `flags`, `hash_based`, and `check_source` from PEP 552; +- `flags`, `hash_based`, and the stored `check_source` bit from PEP 552. + `check_source` applies only to hash-based pycs. A timestamp-based header + with that bit set retains it and includes a limitation explaining its scope; - `source_hash_hex` for hash-based pycs, or `source_mtime` and `source_size` for timestamp pycs. diff --git a/src/domain/python/pythonPyc.ts b/src/domain/python/pythonPyc.ts index f5ab78a47f..ccb3623c45 100644 --- a/src/domain/python/pythonPyc.ts +++ b/src/domain/python/pythonPyc.ts @@ -6,11 +6,24 @@ import { } from "../localPath.js"; export const PYTHON_PYC_HEADER_BYTES = 16; -export const PYTHON_PYC_HASH_BASED = 0x1; -export const PYTHON_PYC_CHECK_SOURCE = 0x2; +const PYTHON_PYC_HASH_BASED = 0x1; +const PYTHON_PYC_CHECK_SOURCE = 0x2; /** CPython 3.14a1 magic 3608 added TYPE_SLICE to marshal (gh-125063). */ export const PYTHON_PYC_SLICE_MAGIC = 3608; +/** PEP 552 stores the check-source bit even where hash validation is inapplicable. */ +export const interpretPythonPycHeaderFlags = (flags: number) => { + const hashBased = (flags & PYTHON_PYC_HASH_BASED) !== 0; + const checkSource = (flags & PYTHON_PYC_CHECK_SOURCE) !== 0; + return { + hashBased, + checkSource, + inactiveCheckSource: checkSource && !hashBased, + hasUnknownFlags: + (flags & ~(PYTHON_PYC_HASH_BASED | PYTHON_PYC_CHECK_SOURCE)) !== 0, + }; +}; + const uint = z.number().int().min(0).max(0xffffffff); const index = z.number().int().nonnegative(); const signedInt32 = z.number().int().min(-0x80000000).max(0x7fffffff); @@ -220,7 +233,11 @@ export const pythonPycHeaderSchema = z code_layout: z.enum(PYTHON_PYC_CODE_LAYOUTS), flags: uint, hash_based: z.boolean(), - check_source: z.boolean(), + check_source: z + .boolean() + .describe( + "Stored PEP 552 check-source bit. It applies only when hash_based is true.", + ), source_hash_hex: z .string() .regex(/^[0-9a-f]{16}$/u) @@ -229,30 +246,33 @@ export const pythonPycHeaderSchema = z source_size: uint.nullable(), }) .superRefine((header, context) => { - for (const [field, flag] of [ - ["hash_based", PYTHON_PYC_HASH_BASED], - ["check_source", PYTHON_PYC_CHECK_SOURCE], + const { hashBased, checkSource } = interpretPythonPycHeaderFlags( + header.flags, + ); + for (const [field, value] of [ + ["hash_based", hashBased], + ["check_source", checkSource], ] as const) - if (header[field] !== ((header.flags & flag) !== 0)) + if (header[field] !== value) context.addIssue({ code: "custom", path: [field], message: `${field} disagrees with its recorded header flag.`, }); - if (header.hash_based === (header.source_hash_hex === null)) + if (hashBased === (header.source_hash_hex === null)) context.addIssue({ code: "custom", path: ["source_hash_hex"], message: "Hash-based pycs require an 8-byte source hash; others must omit it.", }); - if (header.hash_based === (header.source_mtime !== null)) + if (hashBased === (header.source_mtime !== null)) context.addIssue({ code: "custom", path: ["source_mtime"], message: "Timestamp pycs require mtime; hash-based pycs must omit it.", }); - if (header.hash_based === (header.source_size !== null)) + if (hashBased === (header.source_size !== null)) context.addIssue({ code: "custom", path: ["source_size"], diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts index 73b1c406cb..5df711def1 100644 --- a/src/python/PythonPycParser.ts +++ b/src/python/PythonPycParser.ts @@ -11,12 +11,11 @@ import { type MarshalTuple, } from "../domain/python/marshalReader.js"; import { - PYTHON_PYC_CHECK_SOURCE, PYTHON_PYC_CONST_PREVIEW_LIMIT, - PYTHON_PYC_HASH_BASED, PYTHON_PYC_HEADER_BYTES, PYTHON_PYC_INT_PREVIEW_BITS, PYTHON_PYC_SLICE_MAGIC, + interpretPythonPycHeaderFlags, pythonPycMarshalProfile, type PythonPycLimits, type PythonPyc, @@ -256,12 +255,13 @@ export const parsePythonPyc = ( // Every CPython 3.7+ header packs flags at 4, then the source hash (8..16) // or the source mtime (8) and size (12). const header = new BinaryReader(bytes, true, PYTHON_PYC_FORMAT); + const flags = header.u32(4); + const { hashBased, checkSource, inactiveCheckSource, hasUnknownFlags } = + interpretPythonPycHeaderFlags(flags); if (mapped === null || identity === null) { // A known identity without a pinned code layout is a different outcome from an // unrecognized magic: the header below is still qualified from the same packed // fields every 3.7+ pyc uses, so report it instead of a bare rejection. - const flags = header.u32(4); - const hashBased = (flags & PYTHON_PYC_HASH_BASED) !== 0; const headerFacts = `16-byte header, flags 0x${flags.toString(16)}, ${ hashBased ? "hash-based" : "timestamp-based" }`; @@ -271,15 +271,13 @@ export const parsePythonPyc = ( : `CPython ${identity.pythonVersion} header is recognized (magic ${magic}, ${headerFacts}), but no marshal code layout is pinned for that version, so its code objects cannot be decoded. Standalone marshal decoding is pinned for CPython 3.7–3.13 and the 3.14 and 3.15 magics 3627 and 3666.`, ); } - const flags = header.u32(4); - const hashBased = (flags & PYTHON_PYC_HASH_BASED) !== 0; const headerFields = { magic, python_version: identity.pythonVersion, code_layout: mapped.code_layout, flags, hash_based: hashBased, - check_source: (flags & PYTHON_PYC_CHECK_SOURCE) !== 0, + check_source: checkSource, source_hash_hex: hashBased ? bytes.subarray(8, 16).toString("hex") : null, source_mtime: hashBased ? null : header.u32(8), source_size: hashBased ? null : header.u32(12), @@ -301,8 +299,6 @@ export const parsePythonPyc = ( return malformed("Top-level marshal object is not a code object."); reader.finishCodes(); const codeObjects = flattenCode(root.value, work); - const unknownFlags = - flags & ~(PYTHON_PYC_HASH_BASED | PYTHON_PYC_CHECK_SOURCE); return { artifact: { path: artifact.path, @@ -331,11 +327,16 @@ export const parsePythonPyc = ( "Magic 3410 is shared by CPython revisions with different negative co_stacksize/co_flags constructor checks; these raw values are preserved, but this header cannot establish constructor acceptance.", ] : []), - ...(unknownFlags === 0 - ? [] - : [ + ...(inactiveCheckSource + ? [ + "The stored check_source bit is set but applies only to hash-based pycs. This pyc carries timestamp metadata.", + ] + : []), + ...(hasUnknownFlags + ? [ `Header flags ${flags} include unrecognized bits; only hash-based and check-source bits are decoded.`, - ]), + ] + : []), ], }; }; diff --git a/tests/acceptance/pythonPycCli.test.ts b/tests/acceptance/pythonPycCli.test.ts index e8011b440d..d2783a6f86 100644 --- a/tests/acceptance/pythonPycCli.test.ts +++ b/tests/acceptance/pythonPycCli.test.ts @@ -43,6 +43,61 @@ cliTest( }, ); +cliTest( + "preserves the check-source bit and discloses when it is inactive", + async ({ cli }) => { + const directory = await createTestTempDirectory("rea-pyc-header-flags-"); + const call = await cli.sharedMcpCall?.(); + if (call === undefined) throw new Error("Expected file MCP session"); + + for (const flags of [0x02, 0x03]) { + const hashBased = flags === 0x03; + const path = join(directory, `flags-${flags}.pyc`); + await writeFile( + path, + pythonPycFixture("py311", { + header: { + flags, + mtime: 7, + sourceSize: 11, + sourceHash: Buffer.from("0102030405060708", "hex"), + }, + }), + ); + + const response = await cli.run({ + arguments: ["inspect-python-pyc", path, "--json"], + }); + expect(response.exitCode, response.stderr).toBe(0); + const report = completePycEvidenceSchema.parse( + response.json, + ).normalized_result; + expect(report.header).toMatchObject({ + flags, + check_source: true, + hash_based: hashBased, + source_hash_hex: hashBased ? "0102030405060708" : null, + source_mtime: hashBased ? null : 7, + source_size: hashBased ? null : 11, + }); + expect( + report.limitations.some((note) => + note.includes( + "check_source bit is set but applies only to hash-based", + ), + ), + ).toBe(!hashBased); + + const mcp = await call("inspect_python_pyc", { path }); + expect(mcp.isError).not.toBe(true); + expect( + completePycEvidenceSchema.parse(mcp.structuredContent) + .normalized_result, + ).toEqual(report); + } + }, +); + const marshalUtf8 = (bytes: Buffer): Buffer => { const length = Buffer.alloc(4); length.writeUInt32LE(bytes.length);