From a80a47f143c4119e647ee114620fe0e05be33a51 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:45:43 +0800 Subject: [PATCH] test(python): bound pyc parser work by counts instead of elapsed time (#11608) PythonPycParser takes an optional work counter (digested bytes, constant summaries, listed container children, nested and key walk steps, key bytes and report flattening visits). The eight pyc tests that bounded wall-clock time now assert those counts, so each regression they guard (repeated hashing, per-reference summaries, per-step child copies, shared walks, exponential key expansion) fails deterministically on any machine. Refs #10513 --- src/python/PythonPycParser.test.ts | 69 ++++++++++++++---------- src/python/PythonPycParser.ts | 84 +++++++++++++++++++++++++----- 2 files changed, 112 insertions(+), 41 deletions(-) diff --git a/src/python/PythonPycParser.test.ts b/src/python/PythonPycParser.test.ts index 5744181e21..f4ac778ea6 100644 --- a/src/python/PythonPycParser.test.ts +++ b/src/python/PythonPycParser.test.ts @@ -10,7 +10,11 @@ import { pythonPycFixture, pythonPycHeader, } from "./PythonPyc.fixture.js"; -import { parsePythonPyc } from "./PythonPycParser.js"; +import { + emptyPythonPycWork, + parsePythonPyc, + type PythonPycWork, +} from "./PythonPycParser.js"; import { PythonPycResourceFailure } from "./PythonPyc.errors.js"; import { ParseFailureError } from "../domain/parseFailure.js"; @@ -24,7 +28,7 @@ const root = (report: PythonPyc) => { return code; }; -const parse = (bytes: Buffer) => +const parse = (bytes: Buffer, work?: PythonPycWork) => parsePythonPyc( bytes, { @@ -33,8 +37,15 @@ const parse = (bytes: Buffer) => bytes: bytes.length, }, defaults, + work, ); +/** The report and the work the parser counted while building it. */ +const measure = (bytes: Buffer) => { + const work = emptyPythonPycWork(); + return { report: parse(bytes, work), work }; +}; + it("reads 3.7, 3.10 and 3.12 layouts including nested code", () => { const py37 = parse(pythonPycFixture("py37")); expect(py37.header).toMatchObject({ @@ -410,9 +421,10 @@ it("lists a code object shared through TYPE_REF once and refers to it by index", ]); expect(small.coverage.code_objects).toBe(4); expect(pythonPycSchema.safeParse(small).success).toBe(true); - const started = Date.now(); - const deep = parse(sharedChain(25)); - expect(Date.now() - started).toBeLessThan(1000); + const { report: deep, work } = measure(sharedChain(25)); + // Each level references its child three times; every step stays one per level. + expect(work.codeVisits).toBe(26); + expect(work.constSummaries).toBe(26); expect(deep.coverage.code_objects).toBe(26); expect(JSON.stringify(deep).length).toBeLessThan(64 * 1024); }); @@ -633,9 +645,9 @@ it("hashes one shared byte string no matter how many times it is referenced", () ]), }), ]); - const started = Date.now(); - const report = parse(bytes); - expect(Date.now() - started).toBeLessThan(1000); + const { report, work } = measure(bytes); + expect(work.constSummaries).toBe(1); + expect(work.digestedBytes).toBeLessThan(2 * payload.length); expect(root(report).consts).toHaveLength(refs + 1); expect(root(report).consts[refs]).toEqual(root(report).consts[0]); expect(root(report).consts[0]).toMatchObject({ @@ -803,9 +815,9 @@ it("reuses a shared integer's bit length instead of converting every reference", ]), }), ]); - const started = Date.now(); - const report = parse(bytes); - expect(Date.now() - started).toBeLessThan(1000); + const { report, work } = measure(bytes); + // One summary carries the bit length for the integer and all its references. + expect(work.constSummaries).toBe(1); expect(root(report).consts[0]).toEqual({ kind: "int", bits: digits.length * 15, @@ -1036,9 +1048,11 @@ it("rejects a shared tuple key before its expansion doubles without bound", () = const bytes = withConsts([ Buffer.concat([Buffer.from([0x3c]), u32(1), body]), ]); - const started = Date.now(); - expect(() => parse(bytes)).toThrow(PythonPycResourceFailure); - expect(Date.now() - started).toBeLessThan(1000); + const work = emptyPythonPycWork(); + expect(() => parse(bytes, work)).toThrow(PythonPycResourceFailure); + // Refused while the walk and its keys are still linear in the depth. + expect(work.keySteps).toBeLessThanOrEqual(depth * 4); + expect(work.keyBytes).toBeLessThanOrEqual(depth * 8); let shallow = Buffer.concat([Buffer.from([0xe9]), i32(1)]); for (let index = 7; index >= 0; index -= 1) shallow = Buffer.concat([ @@ -1070,9 +1084,10 @@ it("lists a shared container's code objects once per owner and charges each copy consts: [container, ...Array.from({ length: 7 }, () => marshalRef(0))], }), ]); - const started = Date.now(); - const shared = parse(repeated); - expect(Date.now() - started).toBeLessThan(1000); + const { report: shared, work } = measure(repeated); + // Eight owners share one walk of the container and its children. + expect(work.listedChildren).toBe(copies); + expect(work.nestedSteps).toBe(copies + 1); expect(shared.code_objects.map((code) => code.name)).toEqual([ "", "child", @@ -1299,10 +1314,10 @@ it("walks a wide dictionary once per child instead of copying it each step", () ); pieces.push(Buffer.from([0x30])); const bytes = withConsts([Buffer.concat(pieces)]); - const started = Date.now(); - const report = parse(bytes); - // Rebuilding the child list on every key took about 700ms for 8,000 entries. - expect(Date.now() - started).toBeLessThan(250); + const { report, work } = measure(bytes); + // The dictionary's key and value slots are listed once, not once per step. + expect(work.listedChildren).toBe(2 * keys); + expect(work.nestedSteps).toBe(2 * keys + 1); expect(root(report).consts[0]).toEqual({ kind: "dict", length: keys }); expect(root(report).nested_code).toEqual([1]); expect(report.code_objects[1]?.name).toBe("inside"); @@ -1509,9 +1524,9 @@ it("follows a long TYPE_REF chain without a recursive walk", () => { pythonPycHeader(3531), marshalCode("py311", { constsObject: Buffer.concat(pieces) }), ]); - const started = Date.now(); - const report = parse(bytes); - expect(Date.now() - started).toBeLessThan(1000); + const { report, work } = measure(bytes); + expect(work.listedChildren).toBe(links); + expect(work.nestedSteps).toBe(2 * links); expect(report.code_objects.map((code) => code.name)).toEqual([ "", "hidden", @@ -1652,9 +1667,9 @@ it("hashes a long TYPE_REF tuple chain without recursing", () => { marshalCode("py311", { constsObject: Buffer.concat(pieces) }), ]); expect(bytes.length).toBeLessThan(64_000); - const started = Date.now(); - const report = parse(bytes); - expect(Date.now() - started).toBeLessThan(1000); + const { report, work } = measure(bytes); + // Two key steps per link: each tuple and its reference to the previous one. + expect(work.keySteps).toBe(2 * links + 1); expect(root(report).consts[0]).toEqual({ kind: "tuple", length: links + 1 }); expect(root(report).consts[1]).toEqual({ kind: "set", length: 1 }); expect(report.coverage.status).toBe("complete"); diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts index a7c5f75d7f..22d15f0dd7 100644 --- a/src/python/PythonPycParser.ts +++ b/src/python/PythonPycParser.ts @@ -188,10 +188,43 @@ const unsupported = (message: string): never => { throw unsupportedInput(PYTHON_PYC_FORMAT, message); }; +/** + * Work one parse performs, counted where it happens, so a caller can bound an + * algorithm by exact counts rather than elapsed time. Counts accumulate until + * the parse returns or throws. + */ +export type PythonPycWork = { + /** Bytes hashed with SHA-256. */ + digestedBytes: number; + /** Constant summaries built; every TYPE_REF copy reuses one. */ + constSummaries: number; + /** Container child slots listed for the nested code walk. */ + listedChildren: number; + /** Nested code walk steps. */ + nestedSteps: number; + /** Equality-key walk steps. */ + keySteps: number; + /** Bytes of joined equality keys. */ + keyBytes: number; + /** Code objects the report flattening visits, including repeat references. */ + codeVisits: number; +}; + +export const emptyPythonPycWork = (): PythonPycWork => ({ + digestedBytes: 0, + constSummaries: 0, + listedChildren: 0, + nestedSteps: 0, + keySteps: 0, + keyBytes: 0, + codeVisits: 0, +}); + const digests = new WeakMap(); -const sha256 = (bytes: Buffer): string => { +const sha256 = (bytes: Buffer, work: PythonPycWork): string => { const cached = digests.get(bytes); if (cached !== undefined) return cached; + work.digestedBytes += bytes.length; const digest = createHash("sha256").update(bytes).digest("hex"); digests.set(bytes, digest); return digest; @@ -202,6 +235,7 @@ export const parsePythonPyc = ( bytes: Buffer, artifact: { path: string; sha256: string; bytes: number }, input: PythonPycLimits, + work: PythonPycWork = emptyPythonPycWork(), ): PythonPyc => { if (bytes.length !== artifact.bytes) malformed("Selected artifact length disagrees with the retained bytes."); @@ -261,6 +295,7 @@ export const parsePythonPyc = ( input, mapped.negative_code_fields === "rejected", magic >= PYTHON_PYC_SLICE_MAGIC, + work, ); const root = reader.readObject(0); if (reader.offset !== marshal.length) @@ -268,7 +303,7 @@ export const parsePythonPyc = ( if (root.kind !== "code") return malformed("Top-level marshal object is not a code object."); reader.finishCodes(); - const codeObjects = flattenCode(root.value); + const codeObjects = flattenCode(root.value, work); const unknownFlags = flags & ~(PYTHON_PYC_HASH_BASED | PYTHON_PYC_CHECK_SOURCE); return { @@ -358,6 +393,7 @@ class PycMarshalReader extends MarshalReader { readonly rejectNegativeCodeFields: boolean, /** Marshal slice constants exist from CPython 3.14a1 magic 3608. */ readonly acceptSlices: boolean, + readonly work: PythonPycWork, ) { super(bytes, { format: PYTHON_PYC_FORMAT, @@ -613,7 +649,7 @@ class PycMarshalReader extends MarshalReader { cellvars: cellvars?.display ?? null, localsplusnames: localsplusnames?.display ?? null, localspluskinds_hex, - bytecode: { bytes: bytecode.length, sha256: sha256(bytecode) }, + bytecode: { bytes: bytecode.length, sha256: sha256(bytecode, this.work) }, consts: [], nested: [], }; @@ -639,8 +675,8 @@ class PycMarshalReader extends MarshalReader { freevars?.equality ?? [], cellvars?.equality ?? [], ], - linetable: `${linetable.length}:${sha256(linetable)}`, - exceptiontable: `${exceptiontable.length}:${sha256(exceptiontable)}`, + linetable: `${linetable.length}:${sha256(linetable, this.work)}`, + exceptiontable: `${exceptiontable.length}:${sha256(exceptiontable, this.work)}`, }); this.#codes.push(node); return node; @@ -669,7 +705,7 @@ class PycMarshalReader extends MarshalReader { const summaries: PythonPycConst[] = []; for (const item of items) { if (item.kind === "code") this.#reportStrings([item.value.name]); - summaries.push(summarizeConst(item)); + summaries.push(summarizeConst(item, this.work)); } return summaries; } @@ -796,7 +832,7 @@ class PycMarshalReader extends MarshalReader { index: number; /** Built once. A dictionary child list is not copied on every step. */ children: readonly MarshalValue[]; - }[] = [{ value: root, index: 0, children: containerChildren(root) }]; + }[] = [{ value: root, index: 0, children: this.#children(root) }]; gray.add(root); while (frames.length > 0) { this.#takeNestedStep(); @@ -813,7 +849,7 @@ class PycMarshalReader extends MarshalReader { frames.push({ value: child, index: 0, - children: containerChildren(child), + children: this.#children(child), }); continue; } @@ -851,7 +887,7 @@ class PycMarshalReader extends MarshalReader { continue; } visited.add(current); - const children = containerChildren(current); + const children = this.#children(current); for (let index = children.length - 1; index >= 0; index -= 1) { const child = children[index]; if (child !== undefined) stack.push(child); @@ -910,6 +946,13 @@ class PycMarshalReader extends MarshalReader { `Nested code walk exceeds ${String(this.#nestedSteps.maximum)} steps (max_objects * max_depth).`, ), ); + this.work.nestedSteps += 1; + } + + #children(value: MarshalValue): readonly MarshalValue[] { + const children = containerChildren(value); + this.work.listedChildren += children.length; + return children; } #retainNested(codes: readonly CodeNode[]): void { @@ -1156,6 +1199,7 @@ class PycMarshalReader extends MarshalReader { `Marshal key walk exceeds ${this.#keySteps.maximum} steps (max_objects * 8).`, ), ); + this.work.keySteps += 1; } #leafKey(value: MarshalValue, mode: KeyMode): ResolvedKey | null { @@ -1473,6 +1517,7 @@ class PycMarshalReader extends MarshalReader { `Marshal key expansion exceeds max_file_bytes ${this.input.max_file_bytes}.`, ), ); + this.work.keyBytes += size; } #keyTooLarge(size: number): never { @@ -1534,10 +1579,14 @@ const jsonFloat = (value: number): number | "inf" | "-inf" | "nan" | "-0.0" => * shared through TYPE_REF is listed once and referenced by index, so the * report never repeats a subtree. */ -const flattenCode = (root: CodeNode): PythonPycCodeObject[] => { +const flattenCode = ( + root: CodeNode, + work: PythonPycWork, +): PythonPycCodeObject[] => { const indexes = new Map(); const order: CodeNode[] = []; const visit = (node: CodeNode): void => { + work.codeVisits += 1; if (indexes.has(node)) return; indexes.set(node, order.length); order.push(node); @@ -1586,15 +1635,22 @@ const asTuple = ( const constSummaries = new WeakMap(); /** One summary per decoded object, so TYPE_REF does not repeat the work. */ -const summarizeConst = (value: MarshalValue): PythonPycConst => { +const summarizeConst = ( + value: MarshalValue, + work: PythonPycWork, +): PythonPycConst => { const cached = constSummaries.get(value); if (cached !== undefined) return cached; - const summary = buildConstSummary(value); + work.constSummaries += 1; + const summary = buildConstSummary(value, work); constSummaries.set(value, summary); return summary; }; -const buildConstSummary = (value: MarshalValue): PythonPycConst => { +const buildConstSummary = ( + value: MarshalValue, + work: PythonPycWork, +): PythonPycConst => { switch (value.kind) { case "none": return { kind: "none" }; @@ -1624,7 +1680,7 @@ const buildConstSummary = (value: MarshalValue): PythonPycConst => { return { kind: "bytes", bytes: value.value.length, - sha256: sha256(value.value), + sha256: sha256(value.value, work), }; case "code": return {