mirror of
https://github.com/morluto/rea.git
synced 2026-11-19 13:12:26 +00:00
test(python): bound pyc parser work by counts instead of elapsed time (#11608)
PythonPycParser takes an optional work counter (digested bytes, constant summaries, listed container children, nested and key walk steps, key bytes and report flattening visits). The eight pyc tests that bounded wall-clock time now assert those counts, so each regression they guard (repeated hashing, per-reference summaries, per-step child copies, shared walks, exponential key expansion) fails deterministically on any machine. Refs #10513
This commit is contained in:
1 parent
ad4f72dd68
commit
a80a47f143
2 files changed
+112
-41
No files matched your search
@@ -188,10 +188,43 @@ const unsupported = (message: string): never => {
|
||||
throw unsupportedInput(PYTHON_PYC_FORMAT, message);
|
||||
};
|
||||
|
||||
/**
|
||||
* Work one parse performs, counted where it happens, so a caller can bound an
|
||||
* algorithm by exact counts rather than elapsed time. Counts accumulate until
|
||||
* the parse returns or throws.
|
||||
*/
|
||||
export type PythonPycWork = {
|
||||
/** Bytes hashed with SHA-256. */
|
||||
digestedBytes: number;
|
||||
/** Constant summaries built; every TYPE_REF copy reuses one. */
|
||||
constSummaries: number;
|
||||
/** Container child slots listed for the nested code walk. */
|
||||
listedChildren: number;
|
||||
/** Nested code walk steps. */
|
||||
nestedSteps: number;
|
||||
/** Equality-key walk steps. */
|
||||
keySteps: number;
|
||||
/** Bytes of joined equality keys. */
|
||||
keyBytes: number;
|
||||
/** Code objects the report flattening visits, including repeat references. */
|
||||
codeVisits: number;
|
||||
};
|
||||
|
||||
export const emptyPythonPycWork = (): PythonPycWork => ({
|
||||
digestedBytes: 0,
|
||||
constSummaries: 0,
|
||||
listedChildren: 0,
|
||||
nestedSteps: 0,
|
||||
keySteps: 0,
|
||||
keyBytes: 0,
|
||||
codeVisits: 0,
|
||||
});
|
||||
|
||||
const digests = new WeakMap<Buffer, string>();
|
||||
const sha256 = (bytes: Buffer): string => {
|
||||
const sha256 = (bytes: Buffer, work: PythonPycWork): string => {
|
||||
const cached = digests.get(bytes);
|
||||
if (cached !== undefined) return cached;
|
||||
work.digestedBytes += bytes.length;
|
||||
const digest = createHash("sha256").update(bytes).digest("hex");
|
||||
digests.set(bytes, digest);
|
||||
return digest;
|
||||
@@ -202,6 +235,7 @@ export const parsePythonPyc = (
|
||||
bytes: Buffer,
|
||||
artifact: { path: string; sha256: string; bytes: number },
|
||||
input: PythonPycLimits,
|
||||
work: PythonPycWork = emptyPythonPycWork(),
|
||||
): PythonPyc => {
|
||||
if (bytes.length !== artifact.bytes)
|
||||
malformed("Selected artifact length disagrees with the retained bytes.");
|
||||
@@ -261,6 +295,7 @@ export const parsePythonPyc = (
|
||||
input,
|
||||
mapped.negative_code_fields === "rejected",
|
||||
magic >= PYTHON_PYC_SLICE_MAGIC,
|
||||
work,
|
||||
);
|
||||
const root = reader.readObject(0);
|
||||
if (reader.offset !== marshal.length)
|
||||
@@ -268,7 +303,7 @@ export const parsePythonPyc = (
|
||||
if (root.kind !== "code")
|
||||
return malformed("Top-level marshal object is not a code object.");
|
||||
reader.finishCodes();
|
||||
const codeObjects = flattenCode(root.value);
|
||||
const codeObjects = flattenCode(root.value, work);
|
||||
const unknownFlags =
|
||||
flags & ~(PYTHON_PYC_HASH_BASED | PYTHON_PYC_CHECK_SOURCE);
|
||||
return {
|
||||
@@ -358,6 +393,7 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
|
||||
readonly rejectNegativeCodeFields: boolean,
|
||||
/** Marshal slice constants exist from CPython 3.14a1 magic 3608. */
|
||||
readonly acceptSlices: boolean,
|
||||
readonly work: PythonPycWork,
|
||||
) {
|
||||
super(bytes, {
|
||||
format: PYTHON_PYC_FORMAT,
|
||||
@@ -613,7 +649,7 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
|
||||
cellvars: cellvars?.display ?? null,
|
||||
localsplusnames: localsplusnames?.display ?? null,
|
||||
localspluskinds_hex,
|
||||
bytecode: { bytes: bytecode.length, sha256: sha256(bytecode) },
|
||||
bytecode: { bytes: bytecode.length, sha256: sha256(bytecode, this.work) },
|
||||
consts: [],
|
||||
nested: [],
|
||||
};
|
||||
@@ -639,8 +675,8 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
|
||||
freevars?.equality ?? [],
|
||||
cellvars?.equality ?? [],
|
||||
],
|
||||
linetable: `${linetable.length}:${sha256(linetable)}`,
|
||||
exceptiontable: `${exceptiontable.length}:${sha256(exceptiontable)}`,
|
||||
linetable: `${linetable.length}:${sha256(linetable, this.work)}`,
|
||||
exceptiontable: `${exceptiontable.length}:${sha256(exceptiontable, this.work)}`,
|
||||
});
|
||||
this.#codes.push(node);
|
||||
return node;
|
||||
@@ -669,7 +705,7 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
|
||||
const summaries: PythonPycConst[] = [];
|
||||
for (const item of items) {
|
||||
if (item.kind === "code") this.#reportStrings([item.value.name]);
|
||||
summaries.push(summarizeConst(item));
|
||||
summaries.push(summarizeConst(item, this.work));
|
||||
}
|
||||
return summaries;
|
||||
}
|
||||
@@ -796,7 +832,7 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
|
||||
index: number;
|
||||
/** Built once. A dictionary child list is not copied on every step. */
|
||||
children: readonly MarshalValue[];
|
||||
}[] = [{ value: root, index: 0, children: containerChildren(root) }];
|
||||
}[] = [{ value: root, index: 0, children: this.#children(root) }];
|
||||
gray.add(root);
|
||||
while (frames.length > 0) {
|
||||
this.#takeNestedStep();
|
||||
@@ -813,7 +849,7 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
|
||||
frames.push({
|
||||
value: child,
|
||||
index: 0,
|
||||
children: containerChildren(child),
|
||||
children: this.#children(child),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
@@ -851,7 +887,7 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
|
||||
continue;
|
||||
}
|
||||
visited.add(current);
|
||||
const children = containerChildren(current);
|
||||
const children = this.#children(current);
|
||||
for (let index = children.length - 1; index >= 0; index -= 1) {
|
||||
const child = children[index];
|
||||
if (child !== undefined) stack.push(child);
|
||||
@@ -910,6 +946,13 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
|
||||
`Nested code walk exceeds ${String(this.#nestedSteps.maximum)} steps (max_objects * max_depth).`,
|
||||
),
|
||||
);
|
||||
this.work.nestedSteps += 1;
|
||||
}
|
||||
|
||||
#children(value: MarshalValue): readonly MarshalValue[] {
|
||||
const children = containerChildren(value);
|
||||
this.work.listedChildren += children.length;
|
||||
return children;
|
||||
}
|
||||
|
||||
#retainNested(codes: readonly CodeNode[]): void {
|
||||
@@ -1156,6 +1199,7 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
|
||||
`Marshal key walk exceeds ${this.#keySteps.maximum} steps (max_objects * 8).`,
|
||||
),
|
||||
);
|
||||
this.work.keySteps += 1;
|
||||
}
|
||||
|
||||
#leafKey(value: MarshalValue, mode: KeyMode): ResolvedKey | null {
|
||||
@@ -1473,6 +1517,7 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
|
||||
`Marshal key expansion exceeds max_file_bytes ${this.input.max_file_bytes}.`,
|
||||
),
|
||||
);
|
||||
this.work.keyBytes += size;
|
||||
}
|
||||
|
||||
#keyTooLarge(size: number): never {
|
||||
@@ -1534,10 +1579,14 @@ const jsonFloat = (value: number): number | "inf" | "-inf" | "nan" | "-0.0" =>
|
||||
* shared through TYPE_REF is listed once and referenced by index, so the
|
||||
* report never repeats a subtree.
|
||||
*/
|
||||
const flattenCode = (root: CodeNode): PythonPycCodeObject[] => {
|
||||
const flattenCode = (
|
||||
root: CodeNode,
|
||||
work: PythonPycWork,
|
||||
): PythonPycCodeObject[] => {
|
||||
const indexes = new Map<CodeNode, number>();
|
||||
const order: CodeNode[] = [];
|
||||
const visit = (node: CodeNode): void => {
|
||||
work.codeVisits += 1;
|
||||
if (indexes.has(node)) return;
|
||||
indexes.set(node, order.length);
|
||||
order.push(node);
|
||||
@@ -1586,15 +1635,22 @@ const asTuple = (
|
||||
const constSummaries = new WeakMap<MarshalValue, PythonPycConst>();
|
||||
|
||||
/** One summary per decoded object, so TYPE_REF does not repeat the work. */
|
||||
const summarizeConst = (value: MarshalValue): PythonPycConst => {
|
||||
const summarizeConst = (
|
||||
value: MarshalValue,
|
||||
work: PythonPycWork,
|
||||
): PythonPycConst => {
|
||||
const cached = constSummaries.get(value);
|
||||
if (cached !== undefined) return cached;
|
||||
const summary = buildConstSummary(value);
|
||||
work.constSummaries += 1;
|
||||
const summary = buildConstSummary(value, work);
|
||||
constSummaries.set(value, summary);
|
||||
return summary;
|
||||
};
|
||||
|
||||
const buildConstSummary = (value: MarshalValue): PythonPycConst => {
|
||||
const buildConstSummary = (
|
||||
value: MarshalValue,
|
||||
work: PythonPycWork,
|
||||
): PythonPycConst => {
|
||||
switch (value.kind) {
|
||||
case "none":
|
||||
return { kind: "none" };
|
||||
@@ -1624,7 +1680,7 @@ const buildConstSummary = (value: MarshalValue): PythonPycConst => {
|
||||
return {
|
||||
kind: "bytes",
|
||||
bytes: value.value.length,
|
||||
sha256: sha256(value.value),
|
||||
sha256: sha256(value.value, work),
|
||||
};
|
||||
case "code":
|
||||
return {
|
||||
|
||||
Reference in new issue
Block a user