diff --git a/src/python/PythonPycParser.test.ts b/src/python/PythonPycParser.test.ts index 5744181e21..f4ac778ea6 100644 --- a/src/python/PythonPycParser.test.ts +++ b/src/python/PythonPycParser.test.ts @@ -10,7 +10,11 @@ import { pythonPycFixture, pythonPycHeader, } from "./PythonPyc.fixture.js"; -import { parsePythonPyc } from "./PythonPycParser.js"; +import { + emptyPythonPycWork, + parsePythonPyc, + type PythonPycWork, +} from "./PythonPycParser.js"; import { PythonPycResourceFailure } from "./PythonPyc.errors.js"; import { ParseFailureError } from "../domain/parseFailure.js"; @@ -24,7 +28,7 @@ const root = (report: PythonPyc) => { return code; }; -const parse = (bytes: Buffer) => +const parse = (bytes: Buffer, work?: PythonPycWork) => parsePythonPyc( bytes, { @@ -33,8 +37,15 @@ const parse = (bytes: Buffer) => bytes: bytes.length, }, defaults, + work, ); +/** The report and the work the parser counted while building it. */ +const measure = (bytes: Buffer) => { + const work = emptyPythonPycWork(); + return { report: parse(bytes, work), work }; +}; + it("reads 3.7, 3.10 and 3.12 layouts including nested code", () => { const py37 = parse(pythonPycFixture("py37")); expect(py37.header).toMatchObject({ @@ -410,9 +421,10 @@ it("lists a code object shared through TYPE_REF once and refers to it by index", ]); expect(small.coverage.code_objects).toBe(4); expect(pythonPycSchema.safeParse(small).success).toBe(true); - const started = Date.now(); - const deep = parse(sharedChain(25)); - expect(Date.now() - started).toBeLessThan(1000); + const { report: deep, work } = measure(sharedChain(25)); + // Each level references its child three times; every step stays one per level. + expect(work.codeVisits).toBe(26); + expect(work.constSummaries).toBe(26); expect(deep.coverage.code_objects).toBe(26); expect(JSON.stringify(deep).length).toBeLessThan(64 * 1024); }); @@ -633,9 +645,9 @@ it("hashes one shared byte string no matter how many times it is referenced", () ]), }), ]); - const started = Date.now(); - const report = parse(bytes); - expect(Date.now() - started).toBeLessThan(1000); + const { report, work } = measure(bytes); + expect(work.constSummaries).toBe(1); + expect(work.digestedBytes).toBeLessThan(2 * payload.length); expect(root(report).consts).toHaveLength(refs + 1); expect(root(report).consts[refs]).toEqual(root(report).consts[0]); expect(root(report).consts[0]).toMatchObject({ @@ -803,9 +815,9 @@ it("reuses a shared integer's bit length instead of converting every reference", ]), }), ]); - const started = Date.now(); - const report = parse(bytes); - expect(Date.now() - started).toBeLessThan(1000); + const { report, work } = measure(bytes); + // One summary carries the bit length for the integer and all its references. + expect(work.constSummaries).toBe(1); expect(root(report).consts[0]).toEqual({ kind: "int", bits: digits.length * 15, @@ -1036,9 +1048,11 @@ it("rejects a shared tuple key before its expansion doubles without bound", () = const bytes = withConsts([ Buffer.concat([Buffer.from([0x3c]), u32(1), body]), ]); - const started = Date.now(); - expect(() => parse(bytes)).toThrow(PythonPycResourceFailure); - expect(Date.now() - started).toBeLessThan(1000); + const work = emptyPythonPycWork(); + expect(() => parse(bytes, work)).toThrow(PythonPycResourceFailure); + // Refused while the walk and its keys are still linear in the depth. + expect(work.keySteps).toBeLessThanOrEqual(depth * 4); + expect(work.keyBytes).toBeLessThanOrEqual(depth * 8); let shallow = Buffer.concat([Buffer.from([0xe9]), i32(1)]); for (let index = 7; index >= 0; index -= 1) shallow = Buffer.concat([ @@ -1070,9 +1084,10 @@ it("lists a shared container's code objects once per owner and charges each copy consts: [container, ...Array.from({ length: 7 }, () => marshalRef(0))], }), ]); - const started = Date.now(); - const shared = parse(repeated); - expect(Date.now() - started).toBeLessThan(1000); + const { report: shared, work } = measure(repeated); + // Eight owners share one walk of the container and its children. + expect(work.listedChildren).toBe(copies); + expect(work.nestedSteps).toBe(copies + 1); expect(shared.code_objects.map((code) => code.name)).toEqual([ "", "child", @@ -1299,10 +1314,10 @@ it("walks a wide dictionary once per child instead of copying it each step", () ); pieces.push(Buffer.from([0x30])); const bytes = withConsts([Buffer.concat(pieces)]); - const started = Date.now(); - const report = parse(bytes); - // Rebuilding the child list on every key took about 700ms for 8,000 entries. - expect(Date.now() - started).toBeLessThan(250); + const { report, work } = measure(bytes); + // The dictionary's key and value slots are listed once, not once per step. + expect(work.listedChildren).toBe(2 * keys); + expect(work.nestedSteps).toBe(2 * keys + 1); expect(root(report).consts[0]).toEqual({ kind: "dict", length: keys }); expect(root(report).nested_code).toEqual([1]); expect(report.code_objects[1]?.name).toBe("inside"); @@ -1509,9 +1524,9 @@ it("follows a long TYPE_REF chain without a recursive walk", () => { pythonPycHeader(3531), marshalCode("py311", { constsObject: Buffer.concat(pieces) }), ]); - const started = Date.now(); - const report = parse(bytes); - expect(Date.now() - started).toBeLessThan(1000); + const { report, work } = measure(bytes); + expect(work.listedChildren).toBe(links); + expect(work.nestedSteps).toBe(2 * links); expect(report.code_objects.map((code) => code.name)).toEqual([ "", "hidden", @@ -1652,9 +1667,9 @@ it("hashes a long TYPE_REF tuple chain without recursing", () => { marshalCode("py311", { constsObject: Buffer.concat(pieces) }), ]); expect(bytes.length).toBeLessThan(64_000); - const started = Date.now(); - const report = parse(bytes); - expect(Date.now() - started).toBeLessThan(1000); + const { report, work } = measure(bytes); + // Two key steps per link: each tuple and its reference to the previous one. + expect(work.keySteps).toBe(2 * links + 1); expect(root(report).consts[0]).toEqual({ kind: "tuple", length: links + 1 }); expect(root(report).consts[1]).toEqual({ kind: "set", length: 1 }); expect(report.coverage.status).toBe("complete"); diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts index a7c5f75d7f..22d15f0dd7 100644 --- a/src/python/PythonPycParser.ts +++ b/src/python/PythonPycParser.ts @@ -188,10 +188,43 @@ const unsupported = (message: string): never => { throw unsupportedInput(PYTHON_PYC_FORMAT, message); }; +/** + * Work one parse performs, counted where it happens, so a caller can bound an + * algorithm by exact counts rather than elapsed time. Counts accumulate until + * the parse returns or throws. + */ +export type PythonPycWork = { + /** Bytes hashed with SHA-256. */ + digestedBytes: number; + /** Constant summaries built; every TYPE_REF copy reuses one. */ + constSummaries: number; + /** Container child slots listed for the nested code walk. */ + listedChildren: number; + /** Nested code walk steps. */ + nestedSteps: number; + /** Equality-key walk steps. */ + keySteps: number; + /** Bytes of joined equality keys. */ + keyBytes: number; + /** Code objects the report flattening visits, including repeat references. */ + codeVisits: number; +}; + +export const emptyPythonPycWork = (): PythonPycWork => ({ + digestedBytes: 0, + constSummaries: 0, + listedChildren: 0, + nestedSteps: 0, + keySteps: 0, + keyBytes: 0, + codeVisits: 0, +}); + const digests = new WeakMap(); -const sha256 = (bytes: Buffer): string => { +const sha256 = (bytes: Buffer, work: PythonPycWork): string => { const cached = digests.get(bytes); if (cached !== undefined) return cached; + work.digestedBytes += bytes.length; const digest = createHash("sha256").update(bytes).digest("hex"); digests.set(bytes, digest); return digest; @@ -202,6 +235,7 @@ export const parsePythonPyc = ( bytes: Buffer, artifact: { path: string; sha256: string; bytes: number }, input: PythonPycLimits, + work: PythonPycWork = emptyPythonPycWork(), ): PythonPyc => { if (bytes.length !== artifact.bytes) malformed("Selected artifact length disagrees with the retained bytes."); @@ -261,6 +295,7 @@ export const parsePythonPyc = ( input, mapped.negative_code_fields === "rejected", magic >= PYTHON_PYC_SLICE_MAGIC, + work, ); const root = reader.readObject(0); if (reader.offset !== marshal.length) @@ -268,7 +303,7 @@ export const parsePythonPyc = ( if (root.kind !== "code") return malformed("Top-level marshal object is not a code object."); reader.finishCodes(); - const codeObjects = flattenCode(root.value); + const codeObjects = flattenCode(root.value, work); const unknownFlags = flags & ~(PYTHON_PYC_HASH_BASED | PYTHON_PYC_CHECK_SOURCE); return { @@ -358,6 +393,7 @@ class PycMarshalReader extends MarshalReader { readonly rejectNegativeCodeFields: boolean, /** Marshal slice constants exist from CPython 3.14a1 magic 3608. */ readonly acceptSlices: boolean, + readonly work: PythonPycWork, ) { super(bytes, { format: PYTHON_PYC_FORMAT, @@ -613,7 +649,7 @@ class PycMarshalReader extends MarshalReader { cellvars: cellvars?.display ?? null, localsplusnames: localsplusnames?.display ?? null, localspluskinds_hex, - bytecode: { bytes: bytecode.length, sha256: sha256(bytecode) }, + bytecode: { bytes: bytecode.length, sha256: sha256(bytecode, this.work) }, consts: [], nested: [], }; @@ -639,8 +675,8 @@ class PycMarshalReader extends MarshalReader { freevars?.equality ?? [], cellvars?.equality ?? [], ], - linetable: `${linetable.length}:${sha256(linetable)}`, - exceptiontable: `${exceptiontable.length}:${sha256(exceptiontable)}`, + linetable: `${linetable.length}:${sha256(linetable, this.work)}`, + exceptiontable: `${exceptiontable.length}:${sha256(exceptiontable, this.work)}`, }); this.#codes.push(node); return node; @@ -669,7 +705,7 @@ class PycMarshalReader extends MarshalReader { const summaries: PythonPycConst[] = []; for (const item of items) { if (item.kind === "code") this.#reportStrings([item.value.name]); - summaries.push(summarizeConst(item)); + summaries.push(summarizeConst(item, this.work)); } return summaries; } @@ -796,7 +832,7 @@ class PycMarshalReader extends MarshalReader { index: number; /** Built once. A dictionary child list is not copied on every step. */ children: readonly MarshalValue[]; - }[] = [{ value: root, index: 0, children: containerChildren(root) }]; + }[] = [{ value: root, index: 0, children: this.#children(root) }]; gray.add(root); while (frames.length > 0) { this.#takeNestedStep(); @@ -813,7 +849,7 @@ class PycMarshalReader extends MarshalReader { frames.push({ value: child, index: 0, - children: containerChildren(child), + children: this.#children(child), }); continue; } @@ -851,7 +887,7 @@ class PycMarshalReader extends MarshalReader { continue; } visited.add(current); - const children = containerChildren(current); + const children = this.#children(current); for (let index = children.length - 1; index >= 0; index -= 1) { const child = children[index]; if (child !== undefined) stack.push(child); @@ -910,6 +946,13 @@ class PycMarshalReader extends MarshalReader { `Nested code walk exceeds ${String(this.#nestedSteps.maximum)} steps (max_objects * max_depth).`, ), ); + this.work.nestedSteps += 1; + } + + #children(value: MarshalValue): readonly MarshalValue[] { + const children = containerChildren(value); + this.work.listedChildren += children.length; + return children; } #retainNested(codes: readonly CodeNode[]): void { @@ -1156,6 +1199,7 @@ class PycMarshalReader extends MarshalReader { `Marshal key walk exceeds ${this.#keySteps.maximum} steps (max_objects * 8).`, ), ); + this.work.keySteps += 1; } #leafKey(value: MarshalValue, mode: KeyMode): ResolvedKey | null { @@ -1473,6 +1517,7 @@ class PycMarshalReader extends MarshalReader { `Marshal key expansion exceeds max_file_bytes ${this.input.max_file_bytes}.`, ), ); + this.work.keyBytes += size; } #keyTooLarge(size: number): never { @@ -1534,10 +1579,14 @@ const jsonFloat = (value: number): number | "inf" | "-inf" | "nan" | "-0.0" => * shared through TYPE_REF is listed once and referenced by index, so the * report never repeats a subtree. */ -const flattenCode = (root: CodeNode): PythonPycCodeObject[] => { +const flattenCode = ( + root: CodeNode, + work: PythonPycWork, +): PythonPycCodeObject[] => { const indexes = new Map(); const order: CodeNode[] = []; const visit = (node: CodeNode): void => { + work.codeVisits += 1; if (indexes.has(node)) return; indexes.set(node, order.length); order.push(node); @@ -1586,15 +1635,22 @@ const asTuple = ( const constSummaries = new WeakMap(); /** One summary per decoded object, so TYPE_REF does not repeat the work. */ -const summarizeConst = (value: MarshalValue): PythonPycConst => { +const summarizeConst = ( + value: MarshalValue, + work: PythonPycWork, +): PythonPycConst => { const cached = constSummaries.get(value); if (cached !== undefined) return cached; - const summary = buildConstSummary(value); + work.constSummaries += 1; + const summary = buildConstSummary(value, work); constSummaries.set(value, summary); return summary; }; -const buildConstSummary = (value: MarshalValue): PythonPycConst => { +const buildConstSummary = ( + value: MarshalValue, + work: PythonPycWork, +): PythonPycConst => { switch (value.kind) { case "none": return { kind: "none" }; @@ -1624,7 +1680,7 @@ const buildConstSummary = (value: MarshalValue): PythonPycConst => { return { kind: "bytes", bytes: value.value.length, - sha256: sha256(value.value), + sha256: sha256(value.value, work), }; case "code": return {