feat(python): decode CPython 3.14 and 3.15 pyc code objects

Decode marshal slice constants from magic 3608 onward and pin the 3.11 code
layout and inline-cache tables for the 3.14 and 3.15 magics.

Refs #8206

src/python/PythonPycBytecodeEquality.test.ts

@@ -0,0 +1,21 @@
+import { createHash } from "node:crypto";
+import { expect, it } from "vitest";
+
+import { pythonPycBytecodeIdentity } from "./PythonPycBytecodeEquality.js";
+
+const raw = (bytes: Buffer): string =>
+  createHash("sha256").update(bytes).digest("hex");
+const identity = (bytes: Buffer, magic: number): string =>
+  pythonPycBytecodeIdentity(bytes, magic, raw(bytes));
+
+it("skips inline-cache units only under the pinned 3.14 and 3.15 magics", () => {
+  // Opcode 80 carries nine cache code units in the 3627 table; only the cache
+  // bytes differ between the two bytecodes.
+  const first = Buffer.from([80, 0, ...Array(18).fill(1), 83, 0]);
+  const second = Buffer.from([80, 0, ...Array(18).fill(9), 83, 0]);
+  expect(identity(first, 3627)).toBe(identity(second, 3627));
+  // A magic with no pinned table keeps the raw bytecode digest, so code
+  // objects that differ only in cache units stay distinct.
+  expect(identity(first, 3619)).toBe(`${String(first.length)}:${raw(first)}`);
+  expect(identity(first, 3619)).not.toBe(identity(second, 3619));
+});

src/python/PythonPycBytecodeEquality.ts

@@ -40,6 +40,16 @@ const inlineCacheProfiles = (
  * profile only where their source changes do not alter cache widths or IDs.
  * Final 3.13 magic 3571 matches the opcode IDs and cache widths in CPython
  * v3.13.17's Lib/_opcode_metadata.py and Lib/opcode.py.
+ * The 3.14 and 3.15 tables are pinned only to magics checked against real
+ * CPython: 3627 (final 3.14, matches v3.14.4) and 3663 (3.15.0a8) were read
+ * back from those builds; 3666 (final 3.15, matches v3.15.0) was checked with
+ * real 3.15.2 release bytes, not by running 3.15.2. Each is the generated
+ * _PyOpcode_Caches and opmap from CPython's Include/internal/pycore_magic_number.h
+ * history at the last cache-changing bump commit at or before it:
+ * 60202609a2c2d0971aadfa4729ba30b50e89c6ea (3623, shared by 3624 to 3627),
+ * 879c85f6e4ca8424ff6b4fddfbc946889572bf13 (3663) and
+ * 600f4dbd54bb35f56ecf1430134d2072b0520aa3 (3665, shared by 3666). Other 3.14
+ * and 3.15 pre-release magics have no table here and are not decoded.
  */
 const INLINE_CACHE_ENTRIES_BY_MAGIC: ReadonlyMap<
   number,
@@ -416,6 +426,83 @@ const INLINE_CACHE_ENTRIES_BY_MAGIC: ReadonlyMap<
       [117, 1],
     ],
   ),
+  ...inlineCacheProfiles(
+    [3627],
+    [
+      [38, 1],
+      [39, 3],
+      [44, 5],
+      [52, 3],
+      [55, 3],
+      [56, 1],
+      [57, 1],
+      [70, 1],
+      [75, 1],
+      [80, 9],
+      [92, 4],
+      [96, 1],
+      [100, 1],
+      [101, 1],
+      [102, 1],
+      [103, 1],
+      [106, 1],
+      [110, 4],
+      [119, 1],
+    ],
+  ),
+  ...inlineCacheProfiles(
+    [3663],
+    [
+      [4, 1],
+      [36, 1],
+      [37, 3],
+      [42, 5],
+      [50, 3],
+      [53, 3],
+      [54, 1],
+      [55, 1],
+      [68, 1],
+      [74, 1],
+      [79, 9],
+      [91, 4],
+      [95, 1],
+      [99, 1],
+      [100, 1],
+      [101, 1],
+      [102, 1],
+      [105, 1],
+      [109, 4],
+      [118, 1],
+      [128, 1],
+    ],
+  ),
+  ...inlineCacheProfiles(
+    [3666],
+    [
+      [4, 1],
+      [36, 1],
+      [37, 3],
+      [42, 5],
+      [50, 3],
+      [53, 3],
+      [54, 1],
+      [55, 1],
+      [68, 1],
+      [70, 1],
+      [74, 1],
+      [79, 9],
+      [91, 4],
+      [95, 1],
+      [99, 1],
+      [100, 1],
+      [101, 1],
+      [102, 1],
+      [105, 1],
+      [109, 4],
+      [118, 1],
+      [128, 1],
+    ],
+  ),
 ]);
 
 /** Preserve CPython's bytecode-size check and omit only cache code units. */

src/python/PythonPycParser.test.ts

@@ -139,6 +139,8 @@ it("rejects truncated marshal without partial success", () => {
 
 it.each([
   { magic: 3571, layout: "py311", version: "3.13" },
+  { magic: 3627, layout: "py311", version: "3.14" },
+  { magic: 3666, layout: "py311", version: "3.15" },
   { magic: 3401, layout: "py37", version: "3.8" },
 ] as const)(
   "decodes supported standalone magic $magic with its own marshal layout",
@@ -155,44 +157,14 @@ it.each([
   },
 );
 
-it.each([
-  { magic: 3627, version: "3.14", flags: 0x0, mode: "timestamp-based" },
-  { magic: 3666, version: "3.15", flags: 0x0, mode: "timestamp-based" },
-  // A non-zero flag word and a PEP 552 hash-based header, so a refusal that hardcoded
-  // `flags 0x0, timestamp-based` could not pass.
-  { magic: 3666, version: "3.15", flags: 0x3, mode: "hash-based" },
-  { magic: 3627, version: "3.14", flags: 0x1, mode: "hash-based" },
-] as const)(
-  "qualifies the recognized CPython $version header without a pinned code layout",
-  ({ magic, version, flags, mode }) => {
-    // The header is still a packed 16-byte 3.7+ header, so its identity and flags are
-    // reported even though no marshal layout is pinned for this version.
-    const header = pythonPycHeader(magic, {
-      flags,
-      ...(flags === 0 ? {} : { sourceHash: Buffer.alloc(8, 0xab) }),
-    });
-    let failure: unknown;
-    try {
-      parse(header);
-    } catch (error) {
-      failure = error;
-    }
-    expect(failure).toBeInstanceOf(PythonPycFormatFailure);
-    expect(failure).toMatchObject({ kind: "unsupported" });
-    if (!(failure instanceof PythonPycFormatFailure)) throw failure;
-    expect(failure.message).toContain(
-      `CPython ${version} header is recognized`,
-    );
-    expect(failure.message).toContain(`magic ${magic}, 16-byte header`);
-    expect(failure.message).toContain(`flags 0x${flags.toString(16)}`);
-    expect(failure.message).toContain(mode);
-    expect(failure.message).not.toContain(
-      mode === "hash-based" ? "timestamp-based" : "hash-based",
-    );
-    expect(failure.message).toMatch(/no marshal code layout.*pinned/iu);
-    expect(failure.message).toMatch(/marshal decoding.*CPython 3\.7/iu);
-  },
-);
+it("keeps CPython 3.16 pre-release magics outside standalone decoder coverage", () => {
+  expect(() => parse(pythonPycHeader(3706))).toThrow(
+    expect.objectContaining({
+      kind: "unsupported",
+      message: expect.stringMatching(/3706.*3\.7.*3\.15/),
+    }),
+  );
+});
 
 it("does not qualify a magic outside the pinned identity table", () => {
   expect(() => parse(pythonPycHeader(1))).toThrow(
@@ -203,6 +175,115 @@ it("does not qualify a magic outside the pinned identity table", () => {
   );
 });
 
+const withSliceConsts = (magic: number, consts: Buffer[]): Buffer =>
+  Buffer.concat([pythonPycHeader(magic), marshalCode("py311", { consts })]);
+const marshalSlice = (
+  start: Buffer,
+  stop: Buffer,
+  step: Buffer,
+  flagged = false,
+): Buffer =>
+  Buffer.concat([Buffer.from([flagged ? 0xba : 0x3a]), start, stop, step]);
+const marshalInt = (value: number): Buffer =>
+  Buffer.concat([Buffer.from([0x69]), i32(value)]);
+const marshalFrozenset = (...members: Buffer[]): Buffer =>
+  Buffer.concat([Buffer.from([0x3e]), u32(members.length), ...members]);
+const MARSHAL_NONE = Buffer.from([0x4e]);
+
+it("decodes CPython 3.14 slice constants with Python slice equality", () => {
+  const report = parse(
+    withSliceConsts(3627, [
+      marshalSlice(marshalInt(1), marshalInt(10), marshalInt(2)),
+      // 1 == True, so the first two slices are one member; the third differs
+      // in its step.
+      marshalFrozenset(
+        marshalSlice(marshalInt(1), marshalInt(2), MARSHAL_NONE),
+        marshalSlice(Buffer.from([0x54]), marshalInt(2), MARSHAL_NONE),
+        marshalSlice(marshalInt(1), marshalInt(2), marshalInt(1)),
+      ),
+      // A slice never equals the tuple of its fields.
+      marshalFrozenset(
+        marshalSlice(marshalInt(1), marshalInt(2), MARSHAL_NONE),
+        Buffer.concat([
+          Buffer.from([0x29, 3]),
+          marshalInt(1),
+          marshalInt(2),
+          MARSHAL_NONE,
+        ]),
+      ),
+    ]),
+  );
+  expect(report.header).toMatchObject({ magic: 3627, python_version: "3.14" });
+  expect(root(report).consts).toEqual([
+    { kind: "slice" },
+    { kind: "frozenset", length: 2 },
+    { kind: "frozenset", length: 2 },
+  ]);
+  expect(pythonPycSchema.safeParse(report).success).toBe(true);
+});
+
+it("walks slice members for hashability and nested code", () => {
+  const emptyList = Buffer.concat([Buffer.from([0x5b]), u32(0)]);
+  expect(() =>
+    parse(
+      withSliceConsts(3627, [
+        marshalFrozenset(marshalSlice(emptyList, MARSHAL_NONE, MARSHAL_NONE)),
+      ]),
+    ),
+  ).toThrow(/Unhashable set member/);
+  const report = parse(
+    withSliceConsts(3627, [
+      marshalSlice(
+        marshalCode("py311", {
+          name: "inner",
+          qualname: "inner",
+          firstlineno: 2,
+        }),
+        MARSHAL_NONE,
+        MARSHAL_NONE,
+      ),
+    ]),
+  );
+  expect(root(report).nested_code).toEqual([1]);
+  expect(report.code_objects[1]?.name).toBe("inner");
+});
+
+it("rejects slice tags before magic 3608 and follows CPython on null members", () => {
+  const sliced = marshalSlice(marshalInt(1), marshalInt(2), MARSHAL_NONE);
+  expect(() => parse(withSliceConsts(3571, [sliced]))).toThrow(
+    expect.objectContaining({
+      kind: "malformed",
+      message: expect.stringMatching(/0x3a.*3608/),
+    }),
+  );
+  expect(root(parse(withSliceConsts(3627, [sliced]))).consts).toEqual([
+    { kind: "slice" },
+  ]);
+  // A TYPE_NULL member leaves no slice object; the enclosing tuple reports it.
+  expect(() =>
+    parse(
+      withSliceConsts(3627, [
+        marshalSlice(marshalInt(1), Buffer.from([0x30]), MARSHAL_NONE),
+      ]),
+    ),
+  ).toThrow(/NULL object in marshal data for sequence/);
+  // A dictionary ends at that null key, and the slice's reserved slot stays
+  // invalid for a later reference, as in CPython.
+  expect(() =>
+    parse(
+      withSliceConsts(3627, [Buffer.from([0x7b, 0xba, 0x30]), marshalRef(0)]),
+    ),
+  ).toThrow(/TYPE_REF index 0 is missing/);
+  // A reference back into a slice still being read is invalid too.
+  expect(() =>
+    parse(
+      withSliceConsts(3627, [
+        marshalSlice(marshalRef(0), MARSHAL_NONE, MARSHAL_NONE, true),
+      ]),
+    ),
+  ).toThrow(/TYPE_REF index 0 is missing/);
+});
+
 it("reports a short unknown header before unsupported magic, and bad CR LF before either", () => {
   const unknown = pythonPycHeader(1).subarray(0, 4);
   expect(() => parse(unknown)).toThrow(

src/python/PythonPycParser.ts

@@ -7,6 +7,7 @@ import {
   PYTHON_PYC_HASH_BASED,
   PYTHON_PYC_HEADER_BYTES,
   PYTHON_PYC_INT_PREVIEW_BITS,
+  PYTHON_PYC_SLICE_MAGIC,
   pythonPycMarshalProfile,
   type PythonPycLimits,
   type PythonPyc,
@@ -49,6 +50,8 @@ const TYPE_ASCII_INTERNED = 0x41;
 const TYPE_SMALL_TUPLE = 0x29;
 const TYPE_SHORT_ASCII = 0x7a;
 const TYPE_SHORT_ASCII_INTERNED = 0x5a;
+/** CPython 3.14 slice constants; `PYTHON_PYC_SLICE_MAGIC` gates the tag. */
+const TYPE_SLICE = 0x3a;
 const TYPES_WITHOUT_REFERENCE_SLOTS = new Set([
   TYPE_NULL,
   TYPE_NONE,
@@ -87,6 +90,8 @@ type MarshalValue =
   | { readonly kind: "list"; readonly items: MarshalValue[] }
   | { readonly kind: "set"; readonly items: MarshalValue[] }
   | { readonly kind: "frozenset"; readonly items: MarshalValue[] }
+  /** CPython 3.14+ slice constant: start, stop, and step. */
+  | { readonly kind: "slice"; readonly items: MarshalValue[] }
   | {
       readonly kind: "dict";
       readonly items: [MarshalValue, MarshalValue][];
@@ -131,7 +136,8 @@ const isContainer = (value: MarshalValue): boolean =>
   value.kind === "list" ||
   value.kind === "set" ||
   value.kind === "frozenset" ||
-  value.kind === "dict";
+  value.kind === "dict" ||
+  value.kind === "slice";
 
 const containerChildren = (value: MarshalValue): readonly MarshalValue[] => {
   if (value.kind === "dict") {
@@ -143,7 +149,8 @@ const containerChildren = (value: MarshalValue): readonly MarshalValue[] => {
     value.kind === "tuple" ||
     value.kind === "list" ||
     value.kind === "set" ||
-    value.kind === "frozenset"
+    value.kind === "frozenset" ||
+    value.kind === "slice"
   )
     return value.items;
   return [];
@@ -198,7 +205,7 @@ const sha256 = (bytes: Buffer): string => {
   return digest;
 };
 
-/** Decode one bounded CPython 3.7–3.13 pyc without executing bytecode. */
+/** Decode one bounded CPython 3.7–3.15 pyc without executing bytecode. */
 export const parsePythonPyc = (
   bytes: Buffer,
   artifact: { path: string; sha256: string; bytes: number },
@@ -234,8 +241,8 @@ export const parsePythonPyc = (
     }`;
     return unsupported(
       identity === null
-        ? `Pyc magic ${magic} is not a pinned CPython identity, so its header cannot be qualified; standalone marshal decoding is pinned for CPython 3.7–3.13.`
-        : `CPython ${identity.pythonVersion} header is recognized (magic ${magic}, ${headerFacts}), but no marshal code layout is pinned for that version, so its code objects cannot be decoded. Standalone marshal decoding is pinned for CPython 3.7–3.13.`,
+        ? `Pyc magic ${magic} is not a pinned CPython identity, so its header cannot be qualified; standalone marshal decoding is pinned for CPython 3.7–3.13 and the 3.14 and 3.15 magics 3627, 3663 and 3666.`
+        : `CPython ${identity.pythonVersion} header is recognized (magic ${magic}, ${headerFacts}), but no marshal code layout is pinned for that version, so its code objects cannot be decoded. Standalone marshal decoding is pinned for CPython 3.7–3.13 and the 3.14 and 3.15 magics 3627, 3663 and 3666.`,
     );
   }
   const flags = bytes.readUInt32LE(4);
@@ -258,6 +265,7 @@ export const parsePythonPyc = (
     magic,
     input,
     mapped.negative_code_fields === "rejected",
+    magic >= PYTHON_PYC_SLICE_MAGIC,
   );
   const root = reader.readObject(0);
   if (reader.offset !== marshal.length)
@@ -353,6 +361,8 @@ class MarshalReader {
     readonly magic: number,
     readonly input: PythonPycLimits,
     readonly rejectNegativeCodeFields: boolean,
+    /** Marshal slice constants exist from CPython 3.14a1 magic 3608. */
+    readonly acceptSlices: boolean,
   ) {}
 
   readObject(depth: number): MarshalValue {
@@ -382,7 +392,13 @@ class MarshalReader {
         : undefined;
     if (slot !== undefined) this.#intern.push(undefined);
     const value = this.readTyped(type, depth, slot);
-    if (slot !== undefined && this.#intern[slot] === undefined)
+    // A slice with a TYPE_NULL member yields no object, as in CPython, so
+    // its reserved slot stays invalid for later references.
+    if (
+      slot !== undefined &&
+      this.#intern[slot] === undefined &&
+      value.kind !== "null"
+    )
       this.#intern[slot] = value;
     return value;
   }
@@ -483,6 +499,12 @@ class MarshalReader {
         return this.dict(depth, slot);
       case TYPE_CODE:
         return { kind: "code", value: this.code(depth) };
+      case TYPE_SLICE:
+        if (!this.acceptSlices)
+          return malformed(
+            `Unsupported marshal type 0x3a before CPython magic ${PYTHON_PYC_SLICE_MAGIC}.`,
+          );
+        return this.slice(depth);
       default:
         return malformed(`Unsupported marshal type 0x${type.toString(16)}.`);
     }
@@ -920,6 +942,22 @@ class MarshalReader {
     if (slot !== undefined) this.#intern[slot] = value;
   }
 
+  /**
+   * CPython 3.14 reserves a slice's reference slot before reading its start,
+   * stop, and step, so a reference back into the slice is still invalid. A
+   * null member leaves the slice without a value instead of failing here;
+   * the enclosing container or top-level read reports it, as in CPython.
+   */
+  slice(depth: number): MarshalValue {
+    const items: MarshalValue[] = [];
+    for (let index = 0; index < 3; index += 1) {
+      const item = this.readObject(depth + 1);
+      if (item.kind === "null") return item;
+      items.push(item);
+    }
+    return { kind: "slice", items };
+  }
+
   /**
    * CPython registers tuples and lists before reading items, so a container
    * can refer to itself. The same object is what later TYPE_REF entries see.
@@ -1058,8 +1096,12 @@ class MarshalReader {
       // A tuple still being read has missing items; CPython cannot hash it.
       if (mode === "hash" && this.#readingTuples.has(frame.value))
         frame.failed = true;
+      // Slices hash and compare through start, stop, and step since CPython
+      // 3.12, so they walk like a tuple but never equal one.
       const items =
-        frame.value.kind === "tuple" || frame.value.kind === "frozenset"
+        frame.value.kind === "tuple" ||
+        frame.value.kind === "frozenset" ||
+        frame.value.kind === "slice"
           ? frame.value.items
           : undefined;
       if (items === undefined) {
@@ -1233,7 +1275,8 @@ class MarshalReader {
     const expanded = this.#expandedContainerSize(childExpanded);
     if (expanded > this.input.max_string_bytes) this.#keyTooLarge(expanded);
     const ordered = value.kind === "frozenset" ? [...parts].sort() : parts;
-    const kind = value.kind === "tuple" ? "T" : "F";
+    const kind =
+      value.kind === "tuple" ? "T" : value.kind === "slice" ? "L" : "F";
     const descriptor = `${kind}:${ordered.join(",")}`;
     // Charge the descriptor that is stored, not the unmaterialized expansion.
     // A linear reference chain stays within the file budget; repeating one
@@ -1781,6 +1824,8 @@ const buildConstSummary = (value: MarshalValue): PythonPycConst => {
       return { kind: "dict", length: value.items.length };
     case "ellipsis":
       return { kind: "ellipsis" };
+    case "slice":
+      return { kind: "slice" };
     case "complex":
       return {
         kind: "complex",