diff --git a/docs/python-pyc.md b/docs/python-pyc.md index 30bd8d86bb..457b5bbdcd 100644 --- a/docs/python-pyc.md +++ b/docs/python-pyc.md @@ -147,7 +147,13 @@ and a null dictionary value ends the dict without that entry. The selected file is limited to 16 MiB by default (64 MiB maximum). Marshal object count, nesting depth, and per-string size are bounded; exceeding a -budget fails without partial success. Decode-limit failures are deterministic: +budget fails without partial success. Parser refusals name the exhausted +allowance and its effective bound in `details.reported_limits.boundary` and +`details.reported_limits.maximum`. A derived budget reports its own name and +maximum. For example, `nested_code_steps` reports the product of +`max_objects` and `max_depth`, and recovery names both caller controls. + +Decode-limit failures are deterministic: the recovery action names the selected field and CLI option, its effective value, and its accepted maximum. Raise that field on the next call when the maximum permits it; waiting for another analysis or increasing host memory does diff --git a/scripts/verify/baselines/resource-constraint-constructor.json b/scripts/verify/baselines/resource-constraint-constructor.json index 64eee26ffd..c6d5ece2c5 100644 --- a/scripts/verify/baselines/resource-constraint-constructor.json +++ b/scripts/verify/baselines/resource-constraint-constructor.json @@ -132,7 +132,7 @@ "src/process/classifyProviderOutcome.ts": 6, "src/process/ownedDebuggerSession.ts": 1, "src/process/privateWorkspaceFailure.ts": 1, - "src/python/PythonPycProvider.ts": 2, + "src/python/PythonPycProvider.ts": 1, "src/rust/RustBinaryProvider.ts": 3, "src/server/EvidenceMcpServerCore.ts": 2, "src/server/ProgressiveToolCatalog.ts": 1, diff --git a/src/python/PythonPyc.errors.ts b/src/python/PythonPyc.errors.ts index 5d4772657d..ace8b42979 100644 --- a/src/python/PythonPyc.errors.ts +++ b/src/python/PythonPyc.errors.ts @@ -1,18 +1,29 @@ -import type { PYTHON_PYC_LIMIT_MAXIMA } from "../domain/python/pythonPyc.js"; +import { decodeLimit } from "../domain/parseFailure.js"; +import type { PythonPycLimits } from "../domain/python/pythonPyc.js"; /** The ParseFailure format of a standalone CPython pyc. */ export const PYTHON_PYC_FORMAT = "python-pyc"; -/** Resource boundary exceeded while walking marshal objects. */ -export class PythonPycResourceFailure extends Error { - constructor( - readonly boundary: "objects" | "depth" | "string", - readonly reported_limits: Readonly< - Partial> - >, - message: string, - ) { - super(message); - this.name = "PythonPycResourceFailure"; - } -} +/** Caller controls that determine each parser-owned allowance. */ +const LIMIT_CONTROLS = { + max_objects: ["max_objects"], + max_depth: ["max_depth"], + max_string_bytes: ["max_string_bytes"], + reported_string_bytes: ["max_file_bytes"], + reported_rows: ["max_objects"], + nested_code_steps: ["max_objects", "max_depth"], + nested_code_references: ["max_objects", "max_depth"], + marshal_key_steps: ["max_objects"], + marshal_key_bytes: ["max_file_bytes"], +} as const satisfies Record; + +export const pythonPycLimit = ( + name: keyof typeof LIMIT_CONTROLS, + maximum: number, + detail: string, +) => decodeLimit(PYTHON_PYC_FORMAT, detail, { name, maximum }); + +export const pythonPycLimitControls = ( + boundary: string, +): readonly (keyof PythonPycLimits)[] => + Object.entries(LIMIT_CONTROLS).find(([name]) => name === boundary)?.[1] ?? []; diff --git a/src/python/PythonPycParser.test.ts b/src/python/PythonPycParser.test.ts index f4ac778ea6..27a0a97002 100644 --- a/src/python/PythonPycParser.test.ts +++ b/src/python/PythonPycParser.test.ts @@ -15,9 +15,17 @@ import { parsePythonPyc, type PythonPycWork, } from "./PythonPycParser.js"; -import { PythonPycResourceFailure } from "./PythonPyc.errors.js"; import { ParseFailureError } from "../domain/parseFailure.js"; +const parserLimit = (name: string) => + expect.objectContaining({ + failure: expect.objectContaining({ + kind: "limit", + format: "python-pyc", + limit: expect.objectContaining({ name }), + }), + }); + const defaults = inspectPythonPycInputSchema.parse({ path: "/artifacts/sample.pyc", }); @@ -452,7 +460,7 @@ it("bounds names repeated through TYPE_REF by what the report carries", () => { }, { ...defaults, max_file_bytes: bytes.length }, ), - ).toThrow(PythonPycResourceFailure); + ).toThrow(parserLimit("reported_string_bytes")); }); it("charges shared empty name tables against the object budget", () => { @@ -503,7 +511,7 @@ it("charges shared empty name tables against the object budget", () => { it("charges long digits to the string budget and keeps large previews bounded", () => { expect(() => parse(withConsts([Buffer.concat([Buffer.from([0x6c]), i32(600_000)])])), - ).toThrow(PythonPycResourceFailure); + ).toThrow(parserLimit("max_string_bytes")); const report = parse( withConsts([ marshalLong([1, 2]), @@ -792,7 +800,7 @@ it("charges every reported code-constant name against the file budget", () => { }); expect(pythonPycSchema.safeParse(once).success).toBe(true); expect(() => parseLimited(file(30), 20_000)).toThrow( - PythonPycResourceFailure, + parserLimit("reported_string_bytes"), ); }); @@ -1049,7 +1057,7 @@ it("rejects a shared tuple key before its expansion doubles without bound", () = Buffer.concat([Buffer.from([0x3c]), u32(1), body]), ]); const work = emptyPythonPycWork(); - expect(() => parse(bytes, work)).toThrow(PythonPycResourceFailure); + expect(() => parse(bytes, work)).toThrow(parserLimit("max_string_bytes")); // Refused while the walk and its keys are still linear in the depth. expect(work.keySteps).toBeLessThanOrEqual(depth * 4); expect(work.keyBytes).toBeLessThanOrEqual(depth * 8); @@ -1147,7 +1155,7 @@ it("lists a shared container's code objects once per owner and charges each copy }; expect(() => parsePythonPyc(bytes, artifact, { ...defaults, max_objects: constants }), - ).toThrow(PythonPycResourceFailure); + ).toThrow(parserLimit("reported_rows")); expect( parsePythonPyc(bytes, artifact, { ...defaults, @@ -1355,7 +1363,10 @@ it("preflights expanded shared code-name keys before building their hex table", expect(() => parseWith(bytes, { max_string_bytes: 4096 })).toThrow( expect.objectContaining({ - boundary: "string", + failure: expect.objectContaining({ + kind: "limit", + limit: { name: "max_string_bytes", maximum: 4096 }, + }), message: expect.stringContaining("max_string_bytes 4096"), }), ); diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts index a045b7a2f2..73b1c406cb 100644 --- a/src/python/PythonPycParser.ts +++ b/src/python/PythonPycParser.ts @@ -27,10 +27,7 @@ import { import { Budget } from "../domain/budget.js"; import { malformedInput, unsupportedInput } from "../domain/parseFailure.js"; import { BinaryReader } from "../domain/binaryContainer.js"; -import { - PYTHON_PYC_FORMAT, - PythonPycResourceFailure, -} from "./PythonPyc.errors.js"; +import { PYTHON_PYC_FORMAT, pythonPycLimit } from "./PythonPyc.errors.js"; import { pythonPycBytecodeIdentity } from "./PythonPycBytecodeEquality.js"; import { sha256Hex } from "../kernel/sha256Hex.js"; @@ -421,28 +418,28 @@ class PycMarshalReader extends MarshalReader { const { max_objects, max_depth, max_string_bytes } = this.input; switch (exceeded.limit) { case "objects": - throw new PythonPycResourceFailure( - "objects", - { max_objects }, + throw pythonPycLimit( + "max_objects", + max_objects, `Marshal object count exceeds max_objects ${max_objects}.`, ); case "depth": - throw new PythonPycResourceFailure( - "depth", - { max_depth }, + throw pythonPycLimit( + "max_depth", + max_depth, `Marshal nesting exceeds max_depth ${max_depth}.`, ); case "string-bytes": - throw new PythonPycResourceFailure( - "string", - { max_string_bytes }, + throw pythonPycLimit( + "max_string_bytes", + max_string_bytes, `Marshal string of ${exceeded.amount} bytes exceeds max_string_bytes ${max_string_bytes}.`, ); default: // Long digits; this reader sets no sequence or digit-count bound. - throw new PythonPycResourceFailure( - "string", - { max_string_bytes }, + throw pythonPycLimit( + "max_string_bytes", + max_string_bytes, `Marshal long of ${exceeded.amount} digit bytes exceeds max_string_bytes ${max_string_bytes}.`, ); } @@ -717,14 +714,12 @@ class PycMarshalReader extends MarshalReader { #reportStrings(values: readonly string[]): void { let bytes = 0; for (const value of values) bytes += Buffer.byteLength(value, "utf8"); - this.#reportedStringBytes.charge( - bytes, - () => - new PythonPycResourceFailure( - "string", - { max_file_bytes: this.input.max_file_bytes }, - `Reported code-object strings exceed max_file_bytes ${this.input.max_file_bytes}.`, - ), + this.#reportedStringBytes.charge(bytes, () => + pythonPycLimit( + "reported_string_bytes", + this.#reportedStringBytes.maximum, + `Reported code-object strings exceed max_file_bytes ${this.#reportedStringBytes.maximum}.`, + ), ); } @@ -763,14 +758,12 @@ class PycMarshalReader extends MarshalReader { count: number, what: "constants" | "names" | "nested code relationships", ): void { - this.#reportedCopies.charge( - count, - () => - new PythonPycResourceFailure( - "objects", - { max_objects: this.input.max_objects }, - `Reported ${what} exceed max_objects ${this.input.max_objects}.`, - ), + this.#reportedCopies.charge(count, () => + pythonPycLimit( + "reported_rows", + this.#reportedCopies.maximum, + `Reported ${what} exceed max_objects ${this.#reportedCopies.maximum}.`, + ), ); } @@ -934,17 +927,12 @@ class PycMarshalReader extends MarshalReader { } #takeNestedStep(): void { - this.#nestedSteps.charge( - 1, - () => - new PythonPycResourceFailure( - "objects", - { - max_objects: this.input.max_objects, - max_depth: this.input.max_depth, - }, - `Nested code walk exceeds ${String(this.#nestedSteps.maximum)} steps (max_objects * max_depth).`, - ), + this.#nestedSteps.charge(1, () => + pythonPycLimit( + "nested_code_steps", + this.#nestedSteps.maximum, + `Nested code walk exceeds ${String(this.#nestedSteps.maximum)} steps (max_objects * max_depth).`, + ), ); this.work.nestedSteps += 1; } @@ -956,17 +944,12 @@ class PycMarshalReader extends MarshalReader { } #retainNested(codes: readonly CodeNode[]): void { - this.#nestedRetained.charge( - codes.length, - () => - new PythonPycResourceFailure( - "objects", - { - max_objects: this.input.max_objects, - max_depth: this.input.max_depth, - }, - `Nested code walk retains more than ${String(this.#nestedRetained.maximum)} code references.`, - ), + this.#nestedRetained.charge(codes.length, () => + pythonPycLimit( + "nested_code_references", + this.#nestedRetained.maximum, + `Nested code walk retains more than ${String(this.#nestedRetained.maximum)} code references.`, + ), ); } @@ -1190,14 +1173,12 @@ class PycMarshalReader extends MarshalReader { * not `max_depth`, because a reference chain is shallow in the stream. */ #takeKeyStep(): void { - this.#keySteps.charge( - 1, - () => - new PythonPycResourceFailure( - "objects", - { max_objects: this.input.max_objects }, - `Marshal key walk exceeds ${this.#keySteps.maximum} steps (max_objects * 8).`, - ), + this.#keySteps.charge(1, () => + pythonPycLimit( + "marshal_key_steps", + this.#keySteps.maximum, + `Marshal key walk exceeds ${this.#keySteps.maximum} steps (max_objects * 8).`, + ), ); this.work.keySteps += 1; } @@ -1508,22 +1489,20 @@ class PycMarshalReader extends MarshalReader { #reserveKey(size: number): void { if (size > this.input.max_string_bytes) this.#keyTooLarge(size); - this.#keyBytes.charge( - size, - () => - new PythonPycResourceFailure( - "string", - { max_file_bytes: this.input.max_file_bytes }, - `Marshal key expansion exceeds max_file_bytes ${this.input.max_file_bytes}.`, - ), + this.#keyBytes.charge(size, () => + pythonPycLimit( + "marshal_key_bytes", + this.#keyBytes.maximum, + `Marshal key expansion exceeds max_file_bytes ${this.#keyBytes.maximum}.`, + ), ); this.work.keyBytes += size; } #keyTooLarge(size: number): never { - throw new PythonPycResourceFailure( - "string", - { max_string_bytes: this.input.max_string_bytes }, + throw pythonPycLimit( + "max_string_bytes", + this.input.max_string_bytes, `Marshal key of ${size} bytes exceeds max_string_bytes ${this.input.max_string_bytes}.`, ); } diff --git a/src/python/PythonPycProvider.ts b/src/python/PythonPycProvider.ts index cc56c571b4..cec3d9bcc7 100644 --- a/src/python/PythonPycProvider.ts +++ b/src/python/PythonPycProvider.ts @@ -19,33 +19,35 @@ import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js" import { PYTHON_PYC_LIMIT_MAXIMA, type InspectPythonPycInput, + type PythonPycLimits, } from "../domain/python/pythonPyc.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { err, ok } from "../domain/result.js"; -import { PythonPycResourceFailure } from "./PythonPyc.errors.js"; -import { parseFailureAnalysisError } from "../domain/parseFailureAnalysisError.js"; +import { + PYTHON_PYC_FORMAT, + pythonPycLimitControls, +} from "./PythonPyc.errors.js"; +import { + isParseLimit, + parseFailureAnalysisError, + parseLimitConstraint, +} from "../domain/parseFailureAnalysisError.js"; import { parsePythonPyc } from "./PythonPycParser.js"; import { errorMessage } from "../kernel/errorMessage.js"; const OPERATION = "inspect_python_pyc"; const parserLimitRecovery = ( - limits: PythonPycResourceFailure["reported_limits"], + input: InspectPythonPycInput, + fields: readonly (keyof PythonPycLimits)[], ) => { - const actions = ( - Object.keys( - PYTHON_PYC_LIMIT_MAXIMA, - ) as (keyof typeof PYTHON_PYC_LIMIT_MAXIMA)[] - ).flatMap((field) => { - const selected = limits[field]; - if (selected === undefined) return []; + const actions = fields.map((field) => { + const selected = input[field]; const maximum = PYTHON_PYC_LIMIT_MAXIMA[field]; const option = `--${field.replaceAll("_", "-")}`; - return [ - selected < maximum - ? `Raise ${field} (${option}) above ${selected}, up to ${maximum}, on the next call.` - : `${field} (${option}) is at this operation's maximum of ${maximum}; select a complete artifact within that bound.`, - ]; + return selected < maximum + ? `Raise ${field} (${option}) above ${selected}, up to ${maximum}, on the next call.` + : `${field} (${option}) is at this operation's maximum of ${maximum}; select a complete artifact within that bound.`; }); return { retryable: false as const, @@ -66,14 +68,15 @@ const failureForPrimary = ( input.path, ); if (identityFailure !== undefined) return identityFailure; - if (cause instanceof PythonPycResourceFailure) - return new AnalysisResourceConstraintError( + if (isParseLimit(cause, PYTHON_PYC_FORMAT)) { + const { name } = cause.failure.limit; + return parseLimitConstraint( OPERATION, - cause.boundary === "depth" ? "capacity" : "memory", - cause.message, - cause.reported_limits, - { cause, ...parserLimitRecovery(cause.reported_limits) }, + name === "max_depth" ? "capacity" : "memory", + cause, + parserLimitRecovery(input, pythonPycLimitControls(name)), ); + } const formatFailure = parseFailureAnalysisError(OPERATION, input.path, cause); if (formatFailure !== undefined) return formatFailure; if ( @@ -88,7 +91,7 @@ const failureForPrimary = ( { max_file_bytes: input.max_file_bytes }, { cause, - ...parserLimitRecovery({ max_file_bytes: input.max_file_bytes }), + ...parserLimitRecovery(input, ["max_file_bytes"]), }, ); if (cause.reason === "path" || cause.reason === "format") diff --git a/tests/acceptance/pythonPycCli.test.ts b/tests/acceptance/pythonPycCli.test.ts index 85cdbf663b..e8011b440d 100644 --- a/tests/acceptance/pythonPycCli.test.ts +++ b/tests/acceptance/pythonPycCli.test.ts @@ -1010,3 +1010,57 @@ cliTest( ).toEqual(report); }, ); + +cliTest( + "reports the pyc parser allowance that stopped inspection", + async ({ cli }) => { + const directory = await createTestTempDirectory("rea-pyc-parser-budget-"); + const path = join(directory, "limits.pyc"); + const name = Buffer.alloc(1000, 0x61); + const size = Buffer.alloc(4); + size.writeUInt32LE(name.length); + const namesObject = Buffer.concat([ + Buffer.from([0x29, 101, 0x61 | 0x80]), + size, + name, + ...Array.from({ length: 100 }, () => Buffer.from([0x72, 0, 0, 0, 0])), + ]); + const bytes = Buffer.concat([ + pythonPycHeader(3531), + marshalCode("py311", { namesObject }), + ]); + await writeFile(path, bytes); + + for (const [flag, field, boundary, maximum, resource] of [ + ["--max-depth", "max_depth", "max_depth", 1, "capacity"], + [ + "--max-file-bytes", + "max_file_bytes", + "reported_string_bytes", + bytes.length, + "memory", + ], + ] as const) { + const result = await cli.run({ + arguments: [ + "inspect-python-pyc", + path, + flag, + String(maximum), + "--json", + ], + }); + expect(result.exitCode).toBe(1); + expect.soft(result.json, flag).toMatchObject({ + code: "resource_constraint", + retryable: false, + details: { resource, reported_limits: { boundary, maximum } }, + remediation: { + action: expect.stringContaining( + `Raise ${field} (${flag}) above ${maximum}`, + ), + }, + }); + } + }, +);