mirror of
https://github.com/morluto/rea.git
synced 2026-11-20 05:32:44 +00:00
fix(python): disclose inactive pyc check-source flags (#12508)
This commit is contained in:
1 parent
762909f72c
commit
e46ace9744
4 files changed
+102
-24
No files matched your search
+3
-1
@@ -88,7 +88,9 @@ The inline Evidence result contains the artifact path, SHA-256, and byte length.
|
||||
magics before 3494 used other code layouts and are unsupported. The
|
||||
3.7.0a1/a2 magics 3390/3391 used 12-byte headers and are outside this
|
||||
reader's supported format;
|
||||
- `flags`, `hash_based`, and `check_source` from PEP 552;
|
||||
- `flags`, `hash_based`, and the stored `check_source` bit from PEP 552.
|
||||
`check_source` applies only to hash-based pycs. A timestamp-based header
|
||||
with that bit set retains it and includes a limitation explaining its scope;
|
||||
- `source_hash_hex` for hash-based pycs, or `source_mtime` and `source_size`
|
||||
for timestamp pycs.
|
||||
|
||||
|
||||
@@ -6,11 +6,24 @@ import {
|
||||
} from "../localPath.js";
|
||||
|
||||
export const PYTHON_PYC_HEADER_BYTES = 16;
|
||||
export const PYTHON_PYC_HASH_BASED = 0x1;
|
||||
export const PYTHON_PYC_CHECK_SOURCE = 0x2;
|
||||
const PYTHON_PYC_HASH_BASED = 0x1;
|
||||
const PYTHON_PYC_CHECK_SOURCE = 0x2;
|
||||
/** CPython 3.14a1 magic 3608 added TYPE_SLICE to marshal (gh-125063). */
|
||||
export const PYTHON_PYC_SLICE_MAGIC = 3608;
|
||||
|
||||
/** PEP 552 stores the check-source bit even where hash validation is inapplicable. */
|
||||
export const interpretPythonPycHeaderFlags = (flags: number) => {
|
||||
const hashBased = (flags & PYTHON_PYC_HASH_BASED) !== 0;
|
||||
const checkSource = (flags & PYTHON_PYC_CHECK_SOURCE) !== 0;
|
||||
return {
|
||||
hashBased,
|
||||
checkSource,
|
||||
inactiveCheckSource: checkSource && !hashBased,
|
||||
hasUnknownFlags:
|
||||
(flags & ~(PYTHON_PYC_HASH_BASED | PYTHON_PYC_CHECK_SOURCE)) !== 0,
|
||||
};
|
||||
};
|
||||
|
||||
const uint = z.number().int().min(0).max(0xffffffff);
|
||||
const index = z.number().int().nonnegative();
|
||||
const signedInt32 = z.number().int().min(-0x80000000).max(0x7fffffff);
|
||||
@@ -220,7 +233,11 @@ export const pythonPycHeaderSchema = z
|
||||
code_layout: z.enum(PYTHON_PYC_CODE_LAYOUTS),
|
||||
flags: uint,
|
||||
hash_based: z.boolean(),
|
||||
check_source: z.boolean(),
|
||||
check_source: z
|
||||
.boolean()
|
||||
.describe(
|
||||
"Stored PEP 552 check-source bit. It applies only when hash_based is true.",
|
||||
),
|
||||
source_hash_hex: z
|
||||
.string()
|
||||
.regex(/^[0-9a-f]{16}$/u)
|
||||
@@ -229,30 +246,33 @@ export const pythonPycHeaderSchema = z
|
||||
source_size: uint.nullable(),
|
||||
})
|
||||
.superRefine((header, context) => {
|
||||
for (const [field, flag] of [
|
||||
["hash_based", PYTHON_PYC_HASH_BASED],
|
||||
["check_source", PYTHON_PYC_CHECK_SOURCE],
|
||||
const { hashBased, checkSource } = interpretPythonPycHeaderFlags(
|
||||
header.flags,
|
||||
);
|
||||
for (const [field, value] of [
|
||||
["hash_based", hashBased],
|
||||
["check_source", checkSource],
|
||||
] as const)
|
||||
if (header[field] !== ((header.flags & flag) !== 0))
|
||||
if (header[field] !== value)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: [field],
|
||||
message: `${field} disagrees with its recorded header flag.`,
|
||||
});
|
||||
if (header.hash_based === (header.source_hash_hex === null))
|
||||
if (hashBased === (header.source_hash_hex === null))
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["source_hash_hex"],
|
||||
message:
|
||||
"Hash-based pycs require an 8-byte source hash; others must omit it.",
|
||||
});
|
||||
if (header.hash_based === (header.source_mtime !== null))
|
||||
if (hashBased === (header.source_mtime !== null))
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["source_mtime"],
|
||||
message: "Timestamp pycs require mtime; hash-based pycs must omit it.",
|
||||
});
|
||||
if (header.hash_based === (header.source_size !== null))
|
||||
if (hashBased === (header.source_size !== null))
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["source_size"],
|
||||
|
||||
@@ -11,12 +11,11 @@ import {
|
||||
type MarshalTuple,
|
||||
} from "../domain/python/marshalReader.js";
|
||||
import {
|
||||
PYTHON_PYC_CHECK_SOURCE,
|
||||
PYTHON_PYC_CONST_PREVIEW_LIMIT,
|
||||
PYTHON_PYC_HASH_BASED,
|
||||
PYTHON_PYC_HEADER_BYTES,
|
||||
PYTHON_PYC_INT_PREVIEW_BITS,
|
||||
PYTHON_PYC_SLICE_MAGIC,
|
||||
interpretPythonPycHeaderFlags,
|
||||
pythonPycMarshalProfile,
|
||||
type PythonPycLimits,
|
||||
type PythonPyc,
|
||||
@@ -256,12 +255,13 @@ export const parsePythonPyc = (
|
||||
// Every CPython 3.7+ header packs flags at 4, then the source hash (8..16)
|
||||
// or the source mtime (8) and size (12).
|
||||
const header = new BinaryReader(bytes, true, PYTHON_PYC_FORMAT);
|
||||
const flags = header.u32(4);
|
||||
const { hashBased, checkSource, inactiveCheckSource, hasUnknownFlags } =
|
||||
interpretPythonPycHeaderFlags(flags);
|
||||
if (mapped === null || identity === null) {
|
||||
// A known identity without a pinned code layout is a different outcome from an
|
||||
// unrecognized magic: the header below is still qualified from the same packed
|
||||
// fields every 3.7+ pyc uses, so report it instead of a bare rejection.
|
||||
const flags = header.u32(4);
|
||||
const hashBased = (flags & PYTHON_PYC_HASH_BASED) !== 0;
|
||||
const headerFacts = `16-byte header, flags 0x${flags.toString(16)}, ${
|
||||
hashBased ? "hash-based" : "timestamp-based"
|
||||
}`;
|
||||
@@ -271,15 +271,13 @@ export const parsePythonPyc = (
|
||||
: `CPython ${identity.pythonVersion} header is recognized (magic ${magic}, ${headerFacts}), but no marshal code layout is pinned for that version, so its code objects cannot be decoded. Standalone marshal decoding is pinned for CPython 3.7–3.13 and the 3.14 and 3.15 magics 3627 and 3666.`,
|
||||
);
|
||||
}
|
||||
const flags = header.u32(4);
|
||||
const hashBased = (flags & PYTHON_PYC_HASH_BASED) !== 0;
|
||||
const headerFields = {
|
||||
magic,
|
||||
python_version: identity.pythonVersion,
|
||||
code_layout: mapped.code_layout,
|
||||
flags,
|
||||
hash_based: hashBased,
|
||||
check_source: (flags & PYTHON_PYC_CHECK_SOURCE) !== 0,
|
||||
check_source: checkSource,
|
||||
source_hash_hex: hashBased ? bytes.subarray(8, 16).toString("hex") : null,
|
||||
source_mtime: hashBased ? null : header.u32(8),
|
||||
source_size: hashBased ? null : header.u32(12),
|
||||
@@ -301,8 +299,6 @@ export const parsePythonPyc = (
|
||||
return malformed("Top-level marshal object is not a code object.");
|
||||
reader.finishCodes();
|
||||
const codeObjects = flattenCode(root.value, work);
|
||||
const unknownFlags =
|
||||
flags & ~(PYTHON_PYC_HASH_BASED | PYTHON_PYC_CHECK_SOURCE);
|
||||
return {
|
||||
artifact: {
|
||||
path: artifact.path,
|
||||
@@ -331,11 +327,16 @@ export const parsePythonPyc = (
|
||||
"Magic 3410 is shared by CPython revisions with different negative co_stacksize/co_flags constructor checks; these raw values are preserved, but this header cannot establish constructor acceptance.",
|
||||
]
|
||||
: []),
|
||||
...(unknownFlags === 0
|
||||
? []
|
||||
: [
|
||||
...(inactiveCheckSource
|
||||
? [
|
||||
"The stored check_source bit is set but applies only to hash-based pycs. This pyc carries timestamp metadata.",
|
||||
]
|
||||
: []),
|
||||
...(hasUnknownFlags
|
||||
? [
|
||||
`Header flags ${flags} include unrecognized bits; only hash-based and check-source bits are decoded.`,
|
||||
]),
|
||||
]
|
||||
: []),
|
||||
],
|
||||
};
|
||||
};
|
||||
|
||||
@@ -43,6 +43,61 @@ cliTest(
|
||||
},
|
||||
);
|
||||
|
||||
cliTest(
|
||||
"preserves the check-source bit and discloses when it is inactive",
|
||||
async ({ cli }) => {
|
||||
const directory = await createTestTempDirectory("rea-pyc-header-flags-");
|
||||
const call = await cli.sharedMcpCall?.();
|
||||
if (call === undefined) throw new Error("Expected file MCP session");
|
||||
|
||||
for (const flags of [0x02, 0x03]) {
|
||||
const hashBased = flags === 0x03;
|
||||
const path = join(directory, `flags-${flags}.pyc`);
|
||||
await writeFile(
|
||||
path,
|
||||
pythonPycFixture("py311", {
|
||||
header: {
|
||||
flags,
|
||||
mtime: 7,
|
||||
sourceSize: 11,
|
||||
sourceHash: Buffer.from("0102030405060708", "hex"),
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const response = await cli.run({
|
||||
arguments: ["inspect-python-pyc", path, "--json"],
|
||||
});
|
||||
expect(response.exitCode, response.stderr).toBe(0);
|
||||
const report = completePycEvidenceSchema.parse(
|
||||
response.json,
|
||||
).normalized_result;
|
||||
expect(report.header).toMatchObject({
|
||||
flags,
|
||||
check_source: true,
|
||||
hash_based: hashBased,
|
||||
source_hash_hex: hashBased ? "0102030405060708" : null,
|
||||
source_mtime: hashBased ? null : 7,
|
||||
source_size: hashBased ? null : 11,
|
||||
});
|
||||
expect(
|
||||
report.limitations.some((note) =>
|
||||
note.includes(
|
||||
"check_source bit is set but applies only to hash-based",
|
||||
),
|
||||
),
|
||||
).toBe(!hashBased);
|
||||
|
||||
const mcp = await call("inspect_python_pyc", { path });
|
||||
expect(mcp.isError).not.toBe(true);
|
||||
expect(
|
||||
completePycEvidenceSchema.parse(mcp.structuredContent)
|
||||
.normalized_result,
|
||||
).toEqual(report);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
const marshalUtf8 = (bytes: Buffer): Buffer => {
|
||||
const length = Buffer.alloc(4);
|
||||
length.writeUInt32LE(bytes.length);
|
||||
|
||||
Reference in new issue
Block a user