refactor(python): report pyc parser limits from LimitRef (#12213)

This commit is contained in:
morluto authored and GitHub committed 2026-11-19 05:32:28 +00:00
1 parent 954a2511b7
commit 84551888ec
7 files changed
+182 -118

No files matched your search

+25 -14
View File
@@ -1,18 +1,29 @@
import type { PYTHON_PYC_LIMIT_MAXIMA } from "../domain/python/pythonPyc.js";
import { decodeLimit } from "../domain/parseFailure.js";
import type { PythonPycLimits } from "../domain/python/pythonPyc.js";
/** The ParseFailure format of a standalone CPython pyc. */
export const PYTHON_PYC_FORMAT = "python-pyc";
/** Resource boundary exceeded while walking marshal objects. */
export class PythonPycResourceFailure extends Error {
constructor(
readonly boundary: "objects" | "depth" | "string",
readonly reported_limits: Readonly<
Partial<Record<keyof typeof PYTHON_PYC_LIMIT_MAXIMA, number>>
>,
message: string,
) {
super(message);
this.name = "PythonPycResourceFailure";
}
}
/** Caller controls that determine each parser-owned allowance. */
const LIMIT_CONTROLS = {
max_objects: ["max_objects"],
max_depth: ["max_depth"],
max_string_bytes: ["max_string_bytes"],
reported_string_bytes: ["max_file_bytes"],
reported_rows: ["max_objects"],
nested_code_steps: ["max_objects", "max_depth"],
nested_code_references: ["max_objects", "max_depth"],
marshal_key_steps: ["max_objects"],
marshal_key_bytes: ["max_file_bytes"],
} as const satisfies Record<string, readonly (keyof PythonPycLimits)[]>;
export const pythonPycLimit = (
name: keyof typeof LIMIT_CONTROLS,
maximum: number,
detail: string,
) => decodeLimit(PYTHON_PYC_FORMAT, detail, { name, maximum });
export const pythonPycLimitControls = (
boundary: string,
): readonly (keyof PythonPycLimits)[] =>
Object.entries(LIMIT_CONTROLS).find(([name]) => name === boundary)?.[1] ?? [];
+18 -7
View File
@@ -15,9 +15,17 @@ import {
parsePythonPyc,
type PythonPycWork,
} from "./PythonPycParser.js";
import { PythonPycResourceFailure } from "./PythonPyc.errors.js";
import { ParseFailureError } from "../domain/parseFailure.js";
const parserLimit = (name: string) =>
expect.objectContaining({
failure: expect.objectContaining({
kind: "limit",
format: "python-pyc",
limit: expect.objectContaining({ name }),
}),
});
const defaults = inspectPythonPycInputSchema.parse({
path: "/artifacts/sample.pyc",
});
@@ -452,7 +460,7 @@ it("bounds names repeated through TYPE_REF by what the report carries", () => {
},
{ ...defaults, max_file_bytes: bytes.length },
),
).toThrow(PythonPycResourceFailure);
).toThrow(parserLimit("reported_string_bytes"));
});
it("charges shared empty name tables against the object budget", () => {
@@ -503,7 +511,7 @@ it("charges shared empty name tables against the object budget", () => {
it("charges long digits to the string budget and keeps large previews bounded", () => {
expect(() =>
parse(withConsts([Buffer.concat([Buffer.from([0x6c]), i32(600_000)])])),
).toThrow(PythonPycResourceFailure);
).toThrow(parserLimit("max_string_bytes"));
const report = parse(
withConsts([
marshalLong([1, 2]),
@@ -792,7 +800,7 @@ it("charges every reported code-constant name against the file budget", () => {
});
expect(pythonPycSchema.safeParse(once).success).toBe(true);
expect(() => parseLimited(file(30), 20_000)).toThrow(
PythonPycResourceFailure,
parserLimit("reported_string_bytes"),
);
});
@@ -1049,7 +1057,7 @@ it("rejects a shared tuple key before its expansion doubles without bound", () =
Buffer.concat([Buffer.from([0x3c]), u32(1), body]),
]);
const work = emptyPythonPycWork();
expect(() => parse(bytes, work)).toThrow(PythonPycResourceFailure);
expect(() => parse(bytes, work)).toThrow(parserLimit("max_string_bytes"));
// Refused while the walk and its keys are still linear in the depth.
expect(work.keySteps).toBeLessThanOrEqual(depth * 4);
expect(work.keyBytes).toBeLessThanOrEqual(depth * 8);
@@ -1147,7 +1155,7 @@ it("lists a shared container's code objects once per owner and charges each copy
};
expect(() =>
parsePythonPyc(bytes, artifact, { ...defaults, max_objects: constants }),
).toThrow(PythonPycResourceFailure);
).toThrow(parserLimit("reported_rows"));
expect(
parsePythonPyc(bytes, artifact, {
...defaults,
@@ -1355,7 +1363,10 @@ it("preflights expanded shared code-name keys before building their hex table",
expect(() => parseWith(bytes, { max_string_bytes: 4096 })).toThrow(
expect.objectContaining({
boundary: "string",
failure: expect.objectContaining({
kind: "limit",
limit: { name: "max_string_bytes", maximum: 4096 },
}),
message: expect.stringContaining("max_string_bytes 4096"),
}),
);
+52 -73
View File
@@ -27,10 +27,7 @@ import {
import { Budget } from "../domain/budget.js";
import { malformedInput, unsupportedInput } from "../domain/parseFailure.js";
import { BinaryReader } from "../domain/binaryContainer.js";
import {
PYTHON_PYC_FORMAT,
PythonPycResourceFailure,
} from "./PythonPyc.errors.js";
import { PYTHON_PYC_FORMAT, pythonPycLimit } from "./PythonPyc.errors.js";
import { pythonPycBytecodeIdentity } from "./PythonPycBytecodeEquality.js";
import { sha256Hex } from "../kernel/sha256Hex.js";
@@ -421,28 +418,28 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
const { max_objects, max_depth, max_string_bytes } = this.input;
switch (exceeded.limit) {
case "objects":
throw new PythonPycResourceFailure(
"objects",
{ max_objects },
throw pythonPycLimit(
"max_objects",
max_objects,
`Marshal object count exceeds max_objects ${max_objects}.`,
);
case "depth":
throw new PythonPycResourceFailure(
"depth",
{ max_depth },
throw pythonPycLimit(
"max_depth",
max_depth,
`Marshal nesting exceeds max_depth ${max_depth}.`,
);
case "string-bytes":
throw new PythonPycResourceFailure(
"string",
{ max_string_bytes },
throw pythonPycLimit(
"max_string_bytes",
max_string_bytes,
`Marshal string of ${exceeded.amount} bytes exceeds max_string_bytes ${max_string_bytes}.`,
);
default:
// Long digits; this reader sets no sequence or digit-count bound.
throw new PythonPycResourceFailure(
"string",
{ max_string_bytes },
throw pythonPycLimit(
"max_string_bytes",
max_string_bytes,
`Marshal long of ${exceeded.amount} digit bytes exceeds max_string_bytes ${max_string_bytes}.`,
);
}
@@ -717,14 +714,12 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
#reportStrings(values: readonly string[]): void {
let bytes = 0;
for (const value of values) bytes += Buffer.byteLength(value, "utf8");
this.#reportedStringBytes.charge(
bytes,
() =>
new PythonPycResourceFailure(
"string",
{ max_file_bytes: this.input.max_file_bytes },
`Reported code-object strings exceed max_file_bytes ${this.input.max_file_bytes}.`,
),
this.#reportedStringBytes.charge(bytes, () =>
pythonPycLimit(
"reported_string_bytes",
this.#reportedStringBytes.maximum,
`Reported code-object strings exceed max_file_bytes ${this.#reportedStringBytes.maximum}.`,
),
);
}
@@ -763,14 +758,12 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
count: number,
what: "constants" | "names" | "nested code relationships",
): void {
this.#reportedCopies.charge(
count,
() =>
new PythonPycResourceFailure(
"objects",
{ max_objects: this.input.max_objects },
`Reported ${what} exceed max_objects ${this.input.max_objects}.`,
),
this.#reportedCopies.charge(count, () =>
pythonPycLimit(
"reported_rows",
this.#reportedCopies.maximum,
`Reported ${what} exceed max_objects ${this.#reportedCopies.maximum}.`,
),
);
}
@@ -934,17 +927,12 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
}
#takeNestedStep(): void {
this.#nestedSteps.charge(
1,
() =>
new PythonPycResourceFailure(
"objects",
{
max_objects: this.input.max_objects,
max_depth: this.input.max_depth,
},
`Nested code walk exceeds ${String(this.#nestedSteps.maximum)} steps (max_objects * max_depth).`,
),
this.#nestedSteps.charge(1, () =>
pythonPycLimit(
"nested_code_steps",
this.#nestedSteps.maximum,
`Nested code walk exceeds ${String(this.#nestedSteps.maximum)} steps (max_objects * max_depth).`,
),
);
this.work.nestedSteps += 1;
}
@@ -956,17 +944,12 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
}
#retainNested(codes: readonly CodeNode[]): void {
this.#nestedRetained.charge(
codes.length,
() =>
new PythonPycResourceFailure(
"objects",
{
max_objects: this.input.max_objects,
max_depth: this.input.max_depth,
},
`Nested code walk retains more than ${String(this.#nestedRetained.maximum)} code references.`,
),
this.#nestedRetained.charge(codes.length, () =>
pythonPycLimit(
"nested_code_references",
this.#nestedRetained.maximum,
`Nested code walk retains more than ${String(this.#nestedRetained.maximum)} code references.`,
),
);
}
@@ -1190,14 +1173,12 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
* not `max_depth`, because a reference chain is shallow in the stream.
*/
#takeKeyStep(): void {
this.#keySteps.charge(
1,
() =>
new PythonPycResourceFailure(
"objects",
{ max_objects: this.input.max_objects },
`Marshal key walk exceeds ${this.#keySteps.maximum} steps (max_objects * 8).`,
),
this.#keySteps.charge(1, () =>
pythonPycLimit(
"marshal_key_steps",
this.#keySteps.maximum,
`Marshal key walk exceeds ${this.#keySteps.maximum} steps (max_objects * 8).`,
),
);
this.work.keySteps += 1;
}
@@ -1508,22 +1489,20 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
#reserveKey(size: number): void {
if (size > this.input.max_string_bytes) this.#keyTooLarge(size);
this.#keyBytes.charge(
size,
() =>
new PythonPycResourceFailure(
"string",
{ max_file_bytes: this.input.max_file_bytes },
`Marshal key expansion exceeds max_file_bytes ${this.input.max_file_bytes}.`,
),
this.#keyBytes.charge(size, () =>
pythonPycLimit(
"marshal_key_bytes",
this.#keyBytes.maximum,
`Marshal key expansion exceeds max_file_bytes ${this.#keyBytes.maximum}.`,
),
);
this.work.keyBytes += size;
}
#keyTooLarge(size: number): never {
throw new PythonPycResourceFailure(
"string",
{ max_string_bytes: this.input.max_string_bytes },
throw pythonPycLimit(
"max_string_bytes",
this.input.max_string_bytes,
`Marshal key of ${size} bytes exceeds max_string_bytes ${this.input.max_string_bytes}.`,
);
}
+25 -22
View File
@@ -19,33 +19,35 @@ import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"
import {
PYTHON_PYC_LIMIT_MAXIMA,
type InspectPythonPycInput,
type PythonPycLimits,
} from "../domain/python/pythonPyc.js";
import { ProviderAdapterError } from "../domain/providerAdapterError.js";
import { err, ok } from "../domain/result.js";
import { PythonPycResourceFailure } from "./PythonPyc.errors.js";
import { parseFailureAnalysisError } from "../domain/parseFailureAnalysisError.js";
import {
PYTHON_PYC_FORMAT,
pythonPycLimitControls,
} from "./PythonPyc.errors.js";
import {
isParseLimit,
parseFailureAnalysisError,
parseLimitConstraint,
} from "../domain/parseFailureAnalysisError.js";
import { parsePythonPyc } from "./PythonPycParser.js";
import { errorMessage } from "../kernel/errorMessage.js";
const OPERATION = "inspect_python_pyc";
const parserLimitRecovery = (
limits: PythonPycResourceFailure["reported_limits"],
input: InspectPythonPycInput,
fields: readonly (keyof PythonPycLimits)[],
) => {
const actions = (
Object.keys(
PYTHON_PYC_LIMIT_MAXIMA,
) as (keyof typeof PYTHON_PYC_LIMIT_MAXIMA)[]
).flatMap((field) => {
const selected = limits[field];
if (selected === undefined) return [];
const actions = fields.map((field) => {
const selected = input[field];
const maximum = PYTHON_PYC_LIMIT_MAXIMA[field];
const option = `--${field.replaceAll("_", "-")}`;
return [
selected < maximum
? `Raise ${field} (${option}) above ${selected}, up to ${maximum}, on the next call.`
: `${field} (${option}) is at this operation's maximum of ${maximum}; select a complete artifact within that bound.`,
];
return selected < maximum
? `Raise ${field} (${option}) above ${selected}, up to ${maximum}, on the next call.`
: `${field} (${option}) is at this operation's maximum of ${maximum}; select a complete artifact within that bound.`;
});
return {
retryable: false as const,
@@ -66,14 +68,15 @@ const failureForPrimary = (
input.path,
);
if (identityFailure !== undefined) return identityFailure;
if (cause instanceof PythonPycResourceFailure)
return new AnalysisResourceConstraintError(
if (isParseLimit(cause, PYTHON_PYC_FORMAT)) {
const { name } = cause.failure.limit;
return parseLimitConstraint(
OPERATION,
cause.boundary === "depth" ? "capacity" : "memory",
cause.message,
cause.reported_limits,
{ cause, ...parserLimitRecovery(cause.reported_limits) },
name === "max_depth" ? "capacity" : "memory",
cause,
parserLimitRecovery(input, pythonPycLimitControls(name)),
);
}
const formatFailure = parseFailureAnalysisError(OPERATION, input.path, cause);
if (formatFailure !== undefined) return formatFailure;
if (
@@ -88,7 +91,7 @@ const failureForPrimary = (
{ max_file_bytes: input.max_file_bytes },
{
cause,
...parserLimitRecovery({ max_file_bytes: input.max_file_bytes }),
...parserLimitRecovery(input, ["max_file_bytes"]),
},
);
if (cause.reason === "path" || cause.reason === "format")