mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
1 parent
c6ab965062
commit
3784a8e547
4 files changed
+552
-10
No files matched your search
+8
-4
@@ -49,12 +49,16 @@ filename, the signed 32-bit `co_firstlineno` value, argument counts, flags,
|
||||
names, locals (or `localsplusnames` on 3.11+), const
|
||||
summaries, and bytecode length plus SHA-256.
|
||||
|
||||
Code-object equality used by sets and dictionaries follows the selected layout:
|
||||
Code-object equality used by sets and dictionaries follows the selected CPython format:
|
||||
3.7–3.10 derive `CO_NOFREE` from the free/cell tables for equality, while
|
||||
3.11+ compare raw flags; raw `co_flags` remain in reported facts. The older
|
||||
layouts ignore line tables, while 3.11+ include line and exception tables. The
|
||||
artifact SHA-256 identifies the exact selected `.pyc` bytes independently of
|
||||
this decoded object graph. Reference slots follow each tag's CPython behavior:
|
||||
layouts ignore line tables, while 3.11+ include line and exception tables and
|
||||
skip inline-cache code units according to the cache table for the exact magic.
|
||||
Reported bytecode length and SHA-256 still describe the original code bytes.
|
||||
This does not emulate CPython's deoptimization of specialized or instrumented
|
||||
opcode bytes in handcrafted pyc files.
|
||||
The artifact SHA-256 identifies the exact selected `.pyc` bytes independently
|
||||
of this decoded object graph. Reference slots follow each tag's CPython behavior:
|
||||
singleton values and `TYPE_REF` do not reserve new slots even when their tag
|
||||
has `FLAG_REF` set.
|
||||
|
||||
|
||||
@@ -0,0 +1,468 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
type InlineCacheEntry = readonly [opcode: number, codeUnits: number];
|
||||
|
||||
const inlineCacheProfiles = (
|
||||
magics: readonly number[],
|
||||
entries: readonly InlineCacheEntry[],
|
||||
): ReadonlyArray<readonly [number, ReadonlyMap<number, number>]> => {
|
||||
const profile = new Map(entries);
|
||||
return magics.map((magic) => [magic, profile]);
|
||||
};
|
||||
|
||||
/**
|
||||
* CPython equality skips cache code units after comparing original code size.
|
||||
* These tables are pinned to CPython bytecode snapshots, not the shared marshal
|
||||
* layout. The 3.11 profile is from v3.11.9; commit
|
||||
* 616622cab7200bac476bf753d6cc98f4ee48808c adds magic 3495 with the same
|
||||
* opcode/cache table. 3.12 magic/profile transitions
|
||||
* follow CPython's v3.12.12 magic history and these cache-changing source
|
||||
* commits: f012df706cf3a9e63625b8166fe3f419855e9a94 (3503),
|
||||
* b083450f8896bb4a29ac522e4474d91c056b9f32 (3504),
|
||||
* 5fcfdd87c9b5066a581d3ccb4b2fede938f343ec (3505),
|
||||
* 6e4e14d98fe0868981f29701496d57a8223c5407 (3515),
|
||||
* 7b14c2ef194b6eed79670aa9d7e29ab8e2256a56 (3516),
|
||||
* 160f2fe2b90ed5ec7838cb4141dd35768422891f (3519),
|
||||
* 08b67fb34f4519be1b0bb4673643a2c761c7ae92 (3521),
|
||||
* 0444ae24875489dc290d8efdb9ee6440ee60dac8 (3523),
|
||||
* 121057aa3600c4d4d392539aeb79e1b09fd5659d (3524),
|
||||
* b4978ff872be5102117b4e25d93dbbb4e04c8292 (3525),
|
||||
* ef25febcf2ede92a03c5ea00a13e167e0b5cb274 (3528), and
|
||||
* 77262458fe3fe9f3f0266bbf578675a906b1e353 (3530). The 3.13 magic history
|
||||
* is recorded in CPython's Lib/importlib/_bootstrap_external.py; opcode/cache sources include
|
||||
* 1d857da7f0e4858e561223f319ae5afe737d5657 (3553/3554),
|
||||
* 665a4391e10167dad1c854fb604c86f336fcd331 (3559),
|
||||
* 3b2ecbc1275bd05534885cee9ac1389987238561 (3560),
|
||||
* bcce5e271815c0bdbe894964e853210d2c75949b (3561),
|
||||
* 8b55adfa8ff05477b4be7def36db7b66c73f181d (3562),
|
||||
* 22e65eecaad3f5337862319687047afe9861e1ef (3563), and
|
||||
* 52cc4af6ae9002f11605f91b672746c127494efd (3564). Adjacent magics share a
|
||||
* profile only where their source changes do not alter cache widths or IDs.
|
||||
*/
|
||||
const INLINE_CACHE_ENTRIES_BY_MAGIC: ReadonlyMap<
|
||||
number,
|
||||
ReadonlyMap<number, number>
|
||||
> = new Map([
|
||||
...inlineCacheProfiles(
|
||||
[3494, 3495],
|
||||
[
|
||||
[25, 4],
|
||||
[60, 1],
|
||||
[92, 1],
|
||||
[95, 4],
|
||||
[106, 4],
|
||||
[107, 2],
|
||||
[116, 5],
|
||||
[122, 1],
|
||||
[160, 10],
|
||||
[166, 1],
|
||||
[171, 4],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3500, 3501, 3502],
|
||||
[
|
||||
[25, 4],
|
||||
[60, 1],
|
||||
[92, 1],
|
||||
[95, 4],
|
||||
[106, 4],
|
||||
[107, 2],
|
||||
[116, 5],
|
||||
[122, 1],
|
||||
[160, 10],
|
||||
[171, 4],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3503],
|
||||
[
|
||||
[25, 4],
|
||||
[60, 1],
|
||||
[92, 1],
|
||||
[95, 4],
|
||||
[106, 4],
|
||||
[107, 2],
|
||||
[116, 5],
|
||||
[122, 1],
|
||||
[160, 9],
|
||||
[171, 4],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3504],
|
||||
[
|
||||
[25, 4],
|
||||
[60, 1],
|
||||
[92, 1],
|
||||
[95, 4],
|
||||
[106, 9],
|
||||
[107, 2],
|
||||
[116, 5],
|
||||
[122, 1],
|
||||
[171, 4],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3505, 3506, 3507, 3508, 3509, 3510, 3511, 3512, 3513, 3514],
|
||||
[
|
||||
[25, 4],
|
||||
[60, 1],
|
||||
[92, 1],
|
||||
[93, 1],
|
||||
[95, 4],
|
||||
[106, 9],
|
||||
[107, 2],
|
||||
[116, 5],
|
||||
[122, 1],
|
||||
[171, 4],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3515],
|
||||
[
|
||||
[25, 4],
|
||||
[60, 1],
|
||||
[92, 1],
|
||||
[93, 1],
|
||||
[95, 4],
|
||||
[106, 9],
|
||||
[107, 1],
|
||||
[116, 5],
|
||||
[122, 1],
|
||||
[171, 4],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3516, 3517, 3518],
|
||||
[
|
||||
[25, 4],
|
||||
[60, 1],
|
||||
[92, 1],
|
||||
[93, 1],
|
||||
[95, 4],
|
||||
[106, 9],
|
||||
[107, 1],
|
||||
[116, 5],
|
||||
[122, 1],
|
||||
[141, 1],
|
||||
[171, 4],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3519, 3520],
|
||||
[
|
||||
[25, 4],
|
||||
[60, 1],
|
||||
[92, 1],
|
||||
[93, 1],
|
||||
[95, 4],
|
||||
[106, 9],
|
||||
[107, 1],
|
||||
[116, 5],
|
||||
[122, 1],
|
||||
[123, 1],
|
||||
[141, 1],
|
||||
[171, 4],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3521, 3522],
|
||||
[
|
||||
[25, 4],
|
||||
[60, 1],
|
||||
[92, 1],
|
||||
[93, 1],
|
||||
[95, 4],
|
||||
[106, 9],
|
||||
[107, 1],
|
||||
[116, 4],
|
||||
[122, 1],
|
||||
[123, 1],
|
||||
[141, 1],
|
||||
[171, 4],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3523],
|
||||
[
|
||||
[25, 4],
|
||||
[60, 1],
|
||||
[92, 1],
|
||||
[93, 1],
|
||||
[95, 4],
|
||||
[106, 9],
|
||||
[107, 1],
|
||||
[116, 4],
|
||||
[122, 1],
|
||||
[123, 1],
|
||||
[171, 4],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3524],
|
||||
[
|
||||
[25, 1],
|
||||
[60, 1],
|
||||
[92, 1],
|
||||
[93, 1],
|
||||
[95, 4],
|
||||
[106, 9],
|
||||
[107, 1],
|
||||
[116, 4],
|
||||
[122, 1],
|
||||
[123, 1],
|
||||
[171, 4],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3525, 3526, 3527],
|
||||
[
|
||||
[25, 1],
|
||||
[60, 1],
|
||||
[92, 1],
|
||||
[93, 1],
|
||||
[95, 4],
|
||||
[106, 9],
|
||||
[107, 1],
|
||||
[116, 4],
|
||||
[122, 1],
|
||||
[123, 1],
|
||||
[171, 3],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3528, 3529],
|
||||
[
|
||||
[25, 1],
|
||||
[60, 1],
|
||||
[92, 1],
|
||||
[93, 1],
|
||||
[95, 4],
|
||||
[106, 9],
|
||||
[107, 1],
|
||||
[116, 4],
|
||||
[122, 1],
|
||||
[123, 1],
|
||||
[141, 9],
|
||||
[171, 3],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3530, 3531],
|
||||
[
|
||||
[25, 1],
|
||||
[60, 1],
|
||||
[92, 1],
|
||||
[93, 1],
|
||||
[95, 4],
|
||||
[106, 9],
|
||||
[107, 1],
|
||||
[116, 4],
|
||||
[122, 1],
|
||||
[123, 1],
|
||||
[141, 1],
|
||||
[171, 3],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3550, 3551, 3552, 3553, 3554, 3555, 3556, 3557, 3558],
|
||||
[
|
||||
[25, 1],
|
||||
[60, 1],
|
||||
[92, 1],
|
||||
[93, 1],
|
||||
[95, 4],
|
||||
[106, 9],
|
||||
[107, 1],
|
||||
[116, 4],
|
||||
[122, 1],
|
||||
[123, 1],
|
||||
[140, 1],
|
||||
[141, 1],
|
||||
[171, 3],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3559],
|
||||
[
|
||||
[12, 1],
|
||||
[53, 1],
|
||||
[56, 3],
|
||||
[67, 1],
|
||||
[75, 3],
|
||||
[97, 1],
|
||||
[114, 1],
|
||||
[123, 1],
|
||||
[129, 9],
|
||||
[150, 4],
|
||||
[154, 1],
|
||||
[168, 1],
|
||||
[173, 4],
|
||||
[183, 1],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3560],
|
||||
[
|
||||
[12, 1],
|
||||
[54, 1],
|
||||
[57, 3],
|
||||
[68, 1],
|
||||
[76, 3],
|
||||
[98, 1],
|
||||
[115, 1],
|
||||
[124, 1],
|
||||
[130, 9],
|
||||
[151, 4],
|
||||
[155, 1],
|
||||
[169, 1],
|
||||
[174, 4],
|
||||
[184, 1],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3561],
|
||||
[
|
||||
[12, 1],
|
||||
[54, 1],
|
||||
[57, 3],
|
||||
[68, 1],
|
||||
[76, 3],
|
||||
[98, 1],
|
||||
[115, 1],
|
||||
[124, 1],
|
||||
[130, 9],
|
||||
[151, 4],
|
||||
[155, 1],
|
||||
[161, 1],
|
||||
[162, 1],
|
||||
[163, 1],
|
||||
[164, 1],
|
||||
[169, 1],
|
||||
[174, 4],
|
||||
[184, 1],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3562],
|
||||
[
|
||||
[5, 1],
|
||||
[39, 1],
|
||||
[40, 3],
|
||||
[45, 1],
|
||||
[53, 3],
|
||||
[57, 1],
|
||||
[71, 1],
|
||||
[76, 1],
|
||||
[82, 9],
|
||||
[91, 4],
|
||||
[93, 1],
|
||||
[97, 1],
|
||||
[98, 1],
|
||||
[99, 1],
|
||||
[100, 1],
|
||||
[104, 1],
|
||||
[108, 4],
|
||||
[117, 1],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3563, 3564, 3565, 3566, 3567, 3568],
|
||||
[
|
||||
[5, 1],
|
||||
[39, 1],
|
||||
[40, 3],
|
||||
[45, 1],
|
||||
[53, 3],
|
||||
[58, 1],
|
||||
[72, 1],
|
||||
[77, 1],
|
||||
[82, 9],
|
||||
[91, 4],
|
||||
[93, 1],
|
||||
[97, 1],
|
||||
[98, 1],
|
||||
[99, 1],
|
||||
[100, 1],
|
||||
[104, 1],
|
||||
[108, 4],
|
||||
[117, 1],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3569, 3570],
|
||||
[
|
||||
[5, 1],
|
||||
[39, 1],
|
||||
[40, 3],
|
||||
[45, 1],
|
||||
[53, 3],
|
||||
[58, 1],
|
||||
[59, 1],
|
||||
[72, 1],
|
||||
[77, 1],
|
||||
[82, 9],
|
||||
[91, 4],
|
||||
[93, 1],
|
||||
[97, 1],
|
||||
[98, 1],
|
||||
[99, 1],
|
||||
[100, 1],
|
||||
[104, 1],
|
||||
[108, 4],
|
||||
[117, 1],
|
||||
],
|
||||
),
|
||||
...inlineCacheProfiles(
|
||||
[3571],
|
||||
[
|
||||
[5, 1],
|
||||
[38, 1],
|
||||
[39, 3],
|
||||
[44, 1],
|
||||
[52, 3],
|
||||
[57, 1],
|
||||
[58, 1],
|
||||
[71, 1],
|
||||
[76, 1],
|
||||
[81, 9],
|
||||
[91, 4],
|
||||
[93, 1],
|
||||
[97, 1],
|
||||
[98, 1],
|
||||
[99, 1],
|
||||
[100, 1],
|
||||
[104, 1],
|
||||
[108, 4],
|
||||
[117, 1],
|
||||
],
|
||||
),
|
||||
]);
|
||||
|
||||
/** Preserve CPython's bytecode-size check and omit only cache code units. */
|
||||
export const pythonPycBytecodeIdentity = (
|
||||
bytecode: Buffer,
|
||||
magic: number,
|
||||
rawSha256: string,
|
||||
): string => {
|
||||
const caches = INLINE_CACHE_ENTRIES_BY_MAGIC.get(magic);
|
||||
if (caches === undefined) return `${bytecode.length}:${rawSha256}`;
|
||||
|
||||
const digest = createHash("sha256");
|
||||
let activeStart = 0;
|
||||
for (let offset = 0; offset < bytecode.length;) {
|
||||
const opcode = bytecode[offset] ?? 0;
|
||||
const afterOpcode = offset + 2;
|
||||
const cacheBytes = (caches.get(opcode) ?? 0) * 2;
|
||||
offset = afterOpcode + cacheBytes;
|
||||
if (cacheBytes > 0) {
|
||||
digest.update(bytecode.subarray(activeStart, afterOpcode));
|
||||
activeStart = offset;
|
||||
}
|
||||
}
|
||||
if (activeStart < bytecode.length) {
|
||||
digest.update(bytecode.subarray(activeStart));
|
||||
}
|
||||
return `${bytecode.length}:${digest.digest("hex")}`;
|
||||
};
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
PythonPycFormatFailure,
|
||||
PythonPycResourceFailure,
|
||||
} from "./PythonPyc.errors.js";
|
||||
import { pythonPycBytecodeIdentity } from "./PythonPycBytecodeEquality.js";
|
||||
|
||||
const FLAG_REF = 0x80;
|
||||
const TYPE_NULL = 0x30;
|
||||
@@ -154,6 +155,7 @@ type CodeEquality = {
|
||||
/** The co_consts tuple itself, which may be a reference to an unfinished tuple. */
|
||||
readonly constsTuple: MarshalValue;
|
||||
readonly effectiveFlags: number;
|
||||
readonly bytecode: Buffer;
|
||||
readonly name: Buffer;
|
||||
readonly nameTables: readonly (readonly Buffer[])[];
|
||||
readonly linetable: string;
|
||||
@@ -234,6 +236,7 @@ export const parsePythonPyc = (
|
||||
const reader = new MarshalReader(
|
||||
marshal,
|
||||
mapped.code_layout,
|
||||
magic,
|
||||
input,
|
||||
mapped.negative_code_fields === "rejected",
|
||||
);
|
||||
@@ -328,6 +331,7 @@ class MarshalReader {
|
||||
constructor(
|
||||
readonly bytes: Buffer,
|
||||
readonly layout: PythonPycCodeLayout,
|
||||
readonly magic: number,
|
||||
readonly input: InspectPythonPycInput,
|
||||
readonly rejectNegativeCodeFields: boolean,
|
||||
) {}
|
||||
@@ -558,6 +562,7 @@ class MarshalReader {
|
||||
consts,
|
||||
constsTuple,
|
||||
effectiveFlags,
|
||||
bytecode,
|
||||
name: codeName.raw,
|
||||
nameTables:
|
||||
localsplusnames !== null
|
||||
@@ -1148,7 +1153,7 @@ class MarshalReader {
|
||||
":",
|
||||
);
|
||||
case "str":
|
||||
return this.#stringKey(value.raw);
|
||||
return this.#marshalStringKey(value.raw);
|
||||
case "bytes": {
|
||||
this.#reserveKey(value.value.length * 2 + 8);
|
||||
const prefix = mode === "hash" ? "b" : "y";
|
||||
@@ -1227,7 +1232,11 @@ class MarshalReader {
|
||||
node.nlocals === null ? "" : String(node.nlocals),
|
||||
String(material.effectiveFlags),
|
||||
String(node.firstlineno),
|
||||
`${node.bytecode.bytes}:${node.bytecode.sha256}`,
|
||||
pythonPycBytecodeIdentity(
|
||||
material.bytecode,
|
||||
this.magic,
|
||||
node.bytecode.sha256,
|
||||
),
|
||||
this.#constKeyItems(material.consts),
|
||||
...material.nameTables.map((items) => this.#nameTableKey(items)),
|
||||
// CPython started including line and exception tables in code equality
|
||||
@@ -1307,10 +1316,6 @@ class MarshalReader {
|
||||
}
|
||||
|
||||
/** Python string identity is the marshal bytes, not the JavaScript string. */
|
||||
#stringKey(raw: Buffer): string {
|
||||
return this.#marshalStringKey(raw);
|
||||
}
|
||||
|
||||
#marshalStringKey(raw: Buffer): string {
|
||||
this.#reserveKey(this.#marshalStringKeyLength(raw.length));
|
||||
return `s${raw.length}:${raw.toString("hex")}`;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, symlink, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { expect } from "vitest";
|
||||
@@ -16,6 +17,70 @@ const marshalUtf8 = (bytes: Buffer): Buffer => {
|
||||
return Buffer.concat([Buffer.from([0x75]), length, bytes]);
|
||||
};
|
||||
|
||||
cliTest(
|
||||
"uses magic-specific cache tables for synthetic pyc bytecode equality",
|
||||
async ({ cli }) => {
|
||||
const root = await createTestTempDirectory("rea-python-pyc-inline-cache-");
|
||||
const cases = [
|
||||
{ magic: 3495, opcode: 166, cacheUnits: 1 },
|
||||
{ magic: 3500, opcode: 25, cacheUnits: 4 },
|
||||
{ magic: 3524, opcode: 25, cacheUnits: 1 },
|
||||
{ magic: 3531, opcode: 106, cacheUnits: 9 },
|
||||
{ magic: 3550, opcode: 25, cacheUnits: 1 },
|
||||
{ magic: 3553, opcode: 25, cacheUnits: 1 },
|
||||
{ magic: 3564, opcode: 58, cacheUnits: 1 },
|
||||
{ magic: 3571, opcode: 81, cacheUnits: 9 },
|
||||
];
|
||||
for (const { magic, opcode, cacheUnits } of cases) {
|
||||
const cache = Buffer.alloc(cacheUnits * 2);
|
||||
const bytecode = (arg: number, cachePayload: Buffer) =>
|
||||
Buffer.concat([
|
||||
Buffer.from([opcode, arg]),
|
||||
cachePayload,
|
||||
Buffer.from([83, 0]),
|
||||
]);
|
||||
const changedCache = Buffer.from(cache);
|
||||
changedCache[0] = 1;
|
||||
const frozenSet = Buffer.concat([
|
||||
Buffer.from([0x3e, 4, 0, 0, 0]),
|
||||
...[
|
||||
bytecode(0, cache),
|
||||
bytecode(0, changedCache),
|
||||
bytecode(1, cache),
|
||||
Buffer.concat([bytecode(0, cache), Buffer.from([0, 0])]),
|
||||
].map((code) => marshalCode("py311", { bytecode: code })),
|
||||
]);
|
||||
const pyc = Buffer.concat([
|
||||
pythonPycHeader(magic),
|
||||
marshalCode("py311", { consts: [frozenSet] }),
|
||||
]);
|
||||
const path = join(root, `inline-caches-${magic}.pyc`);
|
||||
await writeFile(path, pyc);
|
||||
|
||||
const result = await cli.run({
|
||||
arguments: ["inspect-python-pyc", path, "--json"],
|
||||
});
|
||||
expect(
|
||||
result.exitCode,
|
||||
`${result.stderr}\n${JSON.stringify(result.json)}`,
|
||||
).toBe(0);
|
||||
const evidence = toolContract("inspect_python_pyc").outputSchema.parse(
|
||||
result.json,
|
||||
);
|
||||
expect(evidence.normalized_result.code_objects[0]?.consts).toEqual([
|
||||
expect.objectContaining({ kind: "frozenset", length: 3 }),
|
||||
]);
|
||||
expect(evidence.normalized_result.code_objects[1]?.bytecode).toEqual({
|
||||
bytes: bytecode(0, cache).length,
|
||||
sha256: createHash("sha256").update(bytecode(0, cache)).digest("hex"),
|
||||
});
|
||||
expect(evidence.normalized_result.artifact.sha256).toBe(
|
||||
createHash("sha256").update(pyc).digest("hex"),
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
cliTest(
|
||||
"keeps distinct Python code names distinct when JavaScript display strings collide",
|
||||
async ({ cli }) => {
|
||||
|
||||
Reference in new issue
Block a user