mirror of
https://github.com/morluto/rea.git
synced 2026-11-19 13:12:26 +00:00
refactor(python): report pyc parser limits from LimitRef (#12213)
This commit is contained in:
1 parent
954a2511b7
commit
84551888ec
7 files changed
+182
-118
No files matched your search
@@ -1,18 +1,29 @@
|
||||
import type { PYTHON_PYC_LIMIT_MAXIMA } from "../domain/python/pythonPyc.js";
|
||||
import { decodeLimit } from "../domain/parseFailure.js";
|
||||
import type { PythonPycLimits } from "../domain/python/pythonPyc.js";
|
||||
|
||||
/** The ParseFailure format of a standalone CPython pyc. */
|
||||
export const PYTHON_PYC_FORMAT = "python-pyc";
|
||||
|
||||
/** Resource boundary exceeded while walking marshal objects. */
|
||||
export class PythonPycResourceFailure extends Error {
|
||||
constructor(
|
||||
readonly boundary: "objects" | "depth" | "string",
|
||||
readonly reported_limits: Readonly<
|
||||
Partial<Record<keyof typeof PYTHON_PYC_LIMIT_MAXIMA, number>>
|
||||
>,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "PythonPycResourceFailure";
|
||||
}
|
||||
}
|
||||
/** Caller controls that determine each parser-owned allowance. */
|
||||
const LIMIT_CONTROLS = {
|
||||
max_objects: ["max_objects"],
|
||||
max_depth: ["max_depth"],
|
||||
max_string_bytes: ["max_string_bytes"],
|
||||
reported_string_bytes: ["max_file_bytes"],
|
||||
reported_rows: ["max_objects"],
|
||||
nested_code_steps: ["max_objects", "max_depth"],
|
||||
nested_code_references: ["max_objects", "max_depth"],
|
||||
marshal_key_steps: ["max_objects"],
|
||||
marshal_key_bytes: ["max_file_bytes"],
|
||||
} as const satisfies Record<string, readonly (keyof PythonPycLimits)[]>;
|
||||
|
||||
export const pythonPycLimit = (
|
||||
name: keyof typeof LIMIT_CONTROLS,
|
||||
maximum: number,
|
||||
detail: string,
|
||||
) => decodeLimit(PYTHON_PYC_FORMAT, detail, { name, maximum });
|
||||
|
||||
export const pythonPycLimitControls = (
|
||||
boundary: string,
|
||||
): readonly (keyof PythonPycLimits)[] =>
|
||||
Object.entries(LIMIT_CONTROLS).find(([name]) => name === boundary)?.[1] ?? [];
|
||||
@@ -15,9 +15,17 @@ import {
|
||||
parsePythonPyc,
|
||||
type PythonPycWork,
|
||||
} from "./PythonPycParser.js";
|
||||
import { PythonPycResourceFailure } from "./PythonPyc.errors.js";
|
||||
import { ParseFailureError } from "../domain/parseFailure.js";
|
||||
|
||||
const parserLimit = (name: string) =>
|
||||
expect.objectContaining({
|
||||
failure: expect.objectContaining({
|
||||
kind: "limit",
|
||||
format: "python-pyc",
|
||||
limit: expect.objectContaining({ name }),
|
||||
}),
|
||||
});
|
||||
|
||||
const defaults = inspectPythonPycInputSchema.parse({
|
||||
path: "/artifacts/sample.pyc",
|
||||
});
|
||||
@@ -452,7 +460,7 @@ it("bounds names repeated through TYPE_REF by what the report carries", () => {
|
||||
},
|
||||
{ ...defaults, max_file_bytes: bytes.length },
|
||||
),
|
||||
).toThrow(PythonPycResourceFailure);
|
||||
).toThrow(parserLimit("reported_string_bytes"));
|
||||
});
|
||||
|
||||
it("charges shared empty name tables against the object budget", () => {
|
||||
@@ -503,7 +511,7 @@ it("charges shared empty name tables against the object budget", () => {
|
||||
it("charges long digits to the string budget and keeps large previews bounded", () => {
|
||||
expect(() =>
|
||||
parse(withConsts([Buffer.concat([Buffer.from([0x6c]), i32(600_000)])])),
|
||||
).toThrow(PythonPycResourceFailure);
|
||||
).toThrow(parserLimit("max_string_bytes"));
|
||||
const report = parse(
|
||||
withConsts([
|
||||
marshalLong([1, 2]),
|
||||
@@ -792,7 +800,7 @@ it("charges every reported code-constant name against the file budget", () => {
|
||||
});
|
||||
expect(pythonPycSchema.safeParse(once).success).toBe(true);
|
||||
expect(() => parseLimited(file(30), 20_000)).toThrow(
|
||||
PythonPycResourceFailure,
|
||||
parserLimit("reported_string_bytes"),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -1049,7 +1057,7 @@ it("rejects a shared tuple key before its expansion doubles without bound", () =
|
||||
Buffer.concat([Buffer.from([0x3c]), u32(1), body]),
|
||||
]);
|
||||
const work = emptyPythonPycWork();
|
||||
expect(() => parse(bytes, work)).toThrow(PythonPycResourceFailure);
|
||||
expect(() => parse(bytes, work)).toThrow(parserLimit("max_string_bytes"));
|
||||
// Refused while the walk and its keys are still linear in the depth.
|
||||
expect(work.keySteps).toBeLessThanOrEqual(depth * 4);
|
||||
expect(work.keyBytes).toBeLessThanOrEqual(depth * 8);
|
||||
@@ -1147,7 +1155,7 @@ it("lists a shared container's code objects once per owner and charges each copy
|
||||
};
|
||||
expect(() =>
|
||||
parsePythonPyc(bytes, artifact, { ...defaults, max_objects: constants }),
|
||||
).toThrow(PythonPycResourceFailure);
|
||||
).toThrow(parserLimit("reported_rows"));
|
||||
expect(
|
||||
parsePythonPyc(bytes, artifact, {
|
||||
...defaults,
|
||||
@@ -1355,7 +1363,10 @@ it("preflights expanded shared code-name keys before building their hex table",
|
||||
|
||||
expect(() => parseWith(bytes, { max_string_bytes: 4096 })).toThrow(
|
||||
expect.objectContaining({
|
||||
boundary: "string",
|
||||
failure: expect.objectContaining({
|
||||
kind: "limit",
|
||||
limit: { name: "max_string_bytes", maximum: 4096 },
|
||||
}),
|
||||
message: expect.stringContaining("max_string_bytes 4096"),
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -27,10 +27,7 @@ import {
|
||||
import { Budget } from "../domain/budget.js";
|
||||
import { malformedInput, unsupportedInput } from "../domain/parseFailure.js";
|
||||
import { BinaryReader } from "../domain/binaryContainer.js";
|
||||
import {
|
||||
PYTHON_PYC_FORMAT,
|
||||
PythonPycResourceFailure,
|
||||
} from "./PythonPyc.errors.js";
|
||||
import { PYTHON_PYC_FORMAT, pythonPycLimit } from "./PythonPyc.errors.js";
|
||||
import { pythonPycBytecodeIdentity } from "./PythonPycBytecodeEquality.js";
|
||||
import { sha256Hex } from "../kernel/sha256Hex.js";
|
||||
|
||||
@@ -421,28 +418,28 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
|
||||
const { max_objects, max_depth, max_string_bytes } = this.input;
|
||||
switch (exceeded.limit) {
|
||||
case "objects":
|
||||
throw new PythonPycResourceFailure(
|
||||
"objects",
|
||||
{ max_objects },
|
||||
throw pythonPycLimit(
|
||||
"max_objects",
|
||||
max_objects,
|
||||
`Marshal object count exceeds max_objects ${max_objects}.`,
|
||||
);
|
||||
case "depth":
|
||||
throw new PythonPycResourceFailure(
|
||||
"depth",
|
||||
{ max_depth },
|
||||
throw pythonPycLimit(
|
||||
"max_depth",
|
||||
max_depth,
|
||||
`Marshal nesting exceeds max_depth ${max_depth}.`,
|
||||
);
|
||||
case "string-bytes":
|
||||
throw new PythonPycResourceFailure(
|
||||
"string",
|
||||
{ max_string_bytes },
|
||||
throw pythonPycLimit(
|
||||
"max_string_bytes",
|
||||
max_string_bytes,
|
||||
`Marshal string of ${exceeded.amount} bytes exceeds max_string_bytes ${max_string_bytes}.`,
|
||||
);
|
||||
default:
|
||||
// Long digits; this reader sets no sequence or digit-count bound.
|
||||
throw new PythonPycResourceFailure(
|
||||
"string",
|
||||
{ max_string_bytes },
|
||||
throw pythonPycLimit(
|
||||
"max_string_bytes",
|
||||
max_string_bytes,
|
||||
`Marshal long of ${exceeded.amount} digit bytes exceeds max_string_bytes ${max_string_bytes}.`,
|
||||
);
|
||||
}
|
||||
@@ -717,14 +714,12 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
|
||||
#reportStrings(values: readonly string[]): void {
|
||||
let bytes = 0;
|
||||
for (const value of values) bytes += Buffer.byteLength(value, "utf8");
|
||||
this.#reportedStringBytes.charge(
|
||||
bytes,
|
||||
() =>
|
||||
new PythonPycResourceFailure(
|
||||
"string",
|
||||
{ max_file_bytes: this.input.max_file_bytes },
|
||||
`Reported code-object strings exceed max_file_bytes ${this.input.max_file_bytes}.`,
|
||||
),
|
||||
this.#reportedStringBytes.charge(bytes, () =>
|
||||
pythonPycLimit(
|
||||
"reported_string_bytes",
|
||||
this.#reportedStringBytes.maximum,
|
||||
`Reported code-object strings exceed max_file_bytes ${this.#reportedStringBytes.maximum}.`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -763,14 +758,12 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
|
||||
count: number,
|
||||
what: "constants" | "names" | "nested code relationships",
|
||||
): void {
|
||||
this.#reportedCopies.charge(
|
||||
count,
|
||||
() =>
|
||||
new PythonPycResourceFailure(
|
||||
"objects",
|
||||
{ max_objects: this.input.max_objects },
|
||||
`Reported ${what} exceed max_objects ${this.input.max_objects}.`,
|
||||
),
|
||||
this.#reportedCopies.charge(count, () =>
|
||||
pythonPycLimit(
|
||||
"reported_rows",
|
||||
this.#reportedCopies.maximum,
|
||||
`Reported ${what} exceed max_objects ${this.#reportedCopies.maximum}.`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -934,17 +927,12 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
|
||||
}
|
||||
|
||||
#takeNestedStep(): void {
|
||||
this.#nestedSteps.charge(
|
||||
1,
|
||||
() =>
|
||||
new PythonPycResourceFailure(
|
||||
"objects",
|
||||
{
|
||||
max_objects: this.input.max_objects,
|
||||
max_depth: this.input.max_depth,
|
||||
},
|
||||
`Nested code walk exceeds ${String(this.#nestedSteps.maximum)} steps (max_objects * max_depth).`,
|
||||
),
|
||||
this.#nestedSteps.charge(1, () =>
|
||||
pythonPycLimit(
|
||||
"nested_code_steps",
|
||||
this.#nestedSteps.maximum,
|
||||
`Nested code walk exceeds ${String(this.#nestedSteps.maximum)} steps (max_objects * max_depth).`,
|
||||
),
|
||||
);
|
||||
this.work.nestedSteps += 1;
|
||||
}
|
||||
@@ -956,17 +944,12 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
|
||||
}
|
||||
|
||||
#retainNested(codes: readonly CodeNode[]): void {
|
||||
this.#nestedRetained.charge(
|
||||
codes.length,
|
||||
() =>
|
||||
new PythonPycResourceFailure(
|
||||
"objects",
|
||||
{
|
||||
max_objects: this.input.max_objects,
|
||||
max_depth: this.input.max_depth,
|
||||
},
|
||||
`Nested code walk retains more than ${String(this.#nestedRetained.maximum)} code references.`,
|
||||
),
|
||||
this.#nestedRetained.charge(codes.length, () =>
|
||||
pythonPycLimit(
|
||||
"nested_code_references",
|
||||
this.#nestedRetained.maximum,
|
||||
`Nested code walk retains more than ${String(this.#nestedRetained.maximum)} code references.`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1190,14 +1173,12 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
|
||||
* not `max_depth`, because a reference chain is shallow in the stream.
|
||||
*/
|
||||
#takeKeyStep(): void {
|
||||
this.#keySteps.charge(
|
||||
1,
|
||||
() =>
|
||||
new PythonPycResourceFailure(
|
||||
"objects",
|
||||
{ max_objects: this.input.max_objects },
|
||||
`Marshal key walk exceeds ${this.#keySteps.maximum} steps (max_objects * 8).`,
|
||||
),
|
||||
this.#keySteps.charge(1, () =>
|
||||
pythonPycLimit(
|
||||
"marshal_key_steps",
|
||||
this.#keySteps.maximum,
|
||||
`Marshal key walk exceeds ${this.#keySteps.maximum} steps (max_objects * 8).`,
|
||||
),
|
||||
);
|
||||
this.work.keySteps += 1;
|
||||
}
|
||||
@@ -1508,22 +1489,20 @@ class PycMarshalReader extends MarshalReader<MarshalValue> {
|
||||
|
||||
#reserveKey(size: number): void {
|
||||
if (size > this.input.max_string_bytes) this.#keyTooLarge(size);
|
||||
this.#keyBytes.charge(
|
||||
size,
|
||||
() =>
|
||||
new PythonPycResourceFailure(
|
||||
"string",
|
||||
{ max_file_bytes: this.input.max_file_bytes },
|
||||
`Marshal key expansion exceeds max_file_bytes ${this.input.max_file_bytes}.`,
|
||||
),
|
||||
this.#keyBytes.charge(size, () =>
|
||||
pythonPycLimit(
|
||||
"marshal_key_bytes",
|
||||
this.#keyBytes.maximum,
|
||||
`Marshal key expansion exceeds max_file_bytes ${this.#keyBytes.maximum}.`,
|
||||
),
|
||||
);
|
||||
this.work.keyBytes += size;
|
||||
}
|
||||
|
||||
#keyTooLarge(size: number): never {
|
||||
throw new PythonPycResourceFailure(
|
||||
"string",
|
||||
{ max_string_bytes: this.input.max_string_bytes },
|
||||
throw pythonPycLimit(
|
||||
"max_string_bytes",
|
||||
this.input.max_string_bytes,
|
||||
`Marshal key of ${size} bytes exceeds max_string_bytes ${this.input.max_string_bytes}.`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -19,33 +19,35 @@ import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"
|
||||
import {
|
||||
PYTHON_PYC_LIMIT_MAXIMA,
|
||||
type InspectPythonPycInput,
|
||||
type PythonPycLimits,
|
||||
} from "../domain/python/pythonPyc.js";
|
||||
import { ProviderAdapterError } from "../domain/providerAdapterError.js";
|
||||
import { err, ok } from "../domain/result.js";
|
||||
import { PythonPycResourceFailure } from "./PythonPyc.errors.js";
|
||||
import { parseFailureAnalysisError } from "../domain/parseFailureAnalysisError.js";
|
||||
import {
|
||||
PYTHON_PYC_FORMAT,
|
||||
pythonPycLimitControls,
|
||||
} from "./PythonPyc.errors.js";
|
||||
import {
|
||||
isParseLimit,
|
||||
parseFailureAnalysisError,
|
||||
parseLimitConstraint,
|
||||
} from "../domain/parseFailureAnalysisError.js";
|
||||
import { parsePythonPyc } from "./PythonPycParser.js";
|
||||
import { errorMessage } from "../kernel/errorMessage.js";
|
||||
|
||||
const OPERATION = "inspect_python_pyc";
|
||||
|
||||
const parserLimitRecovery = (
|
||||
limits: PythonPycResourceFailure["reported_limits"],
|
||||
input: InspectPythonPycInput,
|
||||
fields: readonly (keyof PythonPycLimits)[],
|
||||
) => {
|
||||
const actions = (
|
||||
Object.keys(
|
||||
PYTHON_PYC_LIMIT_MAXIMA,
|
||||
) as (keyof typeof PYTHON_PYC_LIMIT_MAXIMA)[]
|
||||
).flatMap((field) => {
|
||||
const selected = limits[field];
|
||||
if (selected === undefined) return [];
|
||||
const actions = fields.map((field) => {
|
||||
const selected = input[field];
|
||||
const maximum = PYTHON_PYC_LIMIT_MAXIMA[field];
|
||||
const option = `--${field.replaceAll("_", "-")}`;
|
||||
return [
|
||||
selected < maximum
|
||||
? `Raise ${field} (${option}) above ${selected}, up to ${maximum}, on the next call.`
|
||||
: `${field} (${option}) is at this operation's maximum of ${maximum}; select a complete artifact within that bound.`,
|
||||
];
|
||||
return selected < maximum
|
||||
? `Raise ${field} (${option}) above ${selected}, up to ${maximum}, on the next call.`
|
||||
: `${field} (${option}) is at this operation's maximum of ${maximum}; select a complete artifact within that bound.`;
|
||||
});
|
||||
return {
|
||||
retryable: false as const,
|
||||
@@ -66,14 +68,15 @@ const failureForPrimary = (
|
||||
input.path,
|
||||
);
|
||||
if (identityFailure !== undefined) return identityFailure;
|
||||
if (cause instanceof PythonPycResourceFailure)
|
||||
return new AnalysisResourceConstraintError(
|
||||
if (isParseLimit(cause, PYTHON_PYC_FORMAT)) {
|
||||
const { name } = cause.failure.limit;
|
||||
return parseLimitConstraint(
|
||||
OPERATION,
|
||||
cause.boundary === "depth" ? "capacity" : "memory",
|
||||
cause.message,
|
||||
cause.reported_limits,
|
||||
{ cause, ...parserLimitRecovery(cause.reported_limits) },
|
||||
name === "max_depth" ? "capacity" : "memory",
|
||||
cause,
|
||||
parserLimitRecovery(input, pythonPycLimitControls(name)),
|
||||
);
|
||||
}
|
||||
const formatFailure = parseFailureAnalysisError(OPERATION, input.path, cause);
|
||||
if (formatFailure !== undefined) return formatFailure;
|
||||
if (
|
||||
@@ -88,7 +91,7 @@ const failureForPrimary = (
|
||||
{ max_file_bytes: input.max_file_bytes },
|
||||
{
|
||||
cause,
|
||||
...parserLimitRecovery({ max_file_bytes: input.max_file_bytes }),
|
||||
...parserLimitRecovery(input, ["max_file_bytes"]),
|
||||
},
|
||||
);
|
||||
if (cause.reason === "path" || cause.reason === "format")
|
||||
|
||||
Reference in new issue
Block a user