From 28332d43ca276ecb0cf04b35fd7256181f833672 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 16 Nov 2026 03:34:31 +0800 Subject: [PATCH] refactor(javascript): charge the worker's retained result bytes through an over-committing Budget (#11399) * refactor(javascript): charge the worker's retained result bytes through an over-committing Budget Static checkpoints may retain past the result budget up to the static capacity, and the worker reports that exhaustion itself. Budget.chargeOverCommitting records the excess (overCommitted) and reports no remainder until refunds bring use back under the maximum. The budget-arithmetic baseline is now empty. * refactor(domain): charge hand-written allowances through Budget and count limit-checked fields (#11405) * refactor(domain): charge hand-written allowances through Budget and count limit-checked fields Pyc report and key-walk tallies, reverse-engineering capture admission, Playwright trace fact retention and semantic alias selections now charge a Budget. The budget-arithmetic guard also counts a field outside Budget classes that is adjusted by hand and compared with a limit-named value; the eleven remaining sites are a counted baseline. * fix(verify): scope limit-checked fields to their class and match limit names in any case --- .../verify/baselines/budget-arithmetic.json | 12 ++- scripts/verify/budget-arithmetic-guard.mjs | 65 +++++++++++--- .../reverse/ReverseEngineeringService.ts | 11 +-- .../history/playwrightTrace/TraceInventory.ts | 29 ++++--- src/domain/budget.test.ts | 16 ++++ src/domain/budget.ts | 29 ++++++- .../javascriptSemanticAliasSelections.ts | 9 +- .../analysis/JavaScriptAnalysisWorker.ts | 44 +++++----- src/python/PythonPycParser.ts | 85 +++++++++++-------- .../evaluation/budgetArithmeticGuard.test.ts | 65 +++++++++++++- 10 files changed, 269 insertions(+), 96 deletions(-) diff --git a/scripts/verify/baselines/budget-arithmetic.json b/scripts/verify/baselines/budget-arithmetic.json index dd218c51ee..a3866e17b6 100644 --- a/scripts/verify/baselines/budget-arithmetic.json +++ b/scripts/verify/baselines/budget-arithmetic.json @@ -1,3 +1,13 @@ { - "src/javascript/analysis/JavaScriptAnalysisWorker.ts": 4 + "src/application/environment/environmentStateStore.ts": 1, + "src/artifacts/AsarHeader.ts": 1, + "src/domain/javascript/javascriptRouteScan.ts": 1, + "src/domain/python/marshalReader.ts": 1, + "src/kuna/KunaJsonStream.ts": 1, + "src/kuna/KunaRunSlots.ts": 1, + "src/process/BoundedOutputExcerpt.ts": 1, + "src/process/OwnedMcpStdioTransport.ts": 1, + "src/process/ProviderProcess.ts": 1, + "src/process/capture/ProcessSensitiveOutput.ts": 1, + "src/server/http/NodeHttpRequestOwner.ts": 1 } diff --git a/scripts/verify/budget-arithmetic-guard.mjs b/scripts/verify/budget-arithmetic-guard.mjs index 8c328fefbe..65ef0c0ce1 100644 --- a/scripts/verify/budget-arithmetic-guard.mjs +++ b/scripts/verify/budget-arithmetic-guard.mjs @@ -7,15 +7,15 @@ // by hand (`+=`, `-=`, `++`, `--`) and compared with `<` or `>`: that is a // limit check, whatever the field is called (`#bytes`, `#visits`). // -// Write cursors (`#arenaWriteOffset`) and observation tallies are not -// allowances; outside a `…Budget` class they are not counted unless named as -// one. +// 3. Anywhere else, a field adjusted by hand and compared with `<` or `>` +// (or `<=`/`>=`) against a value named as a limit (`…max…`, `…limit…`, +// `…allowance…`, `…budget…`, `…capacity…`): a hand-written allowance. +// +// Write cursors (`#arenaWriteOffset`) and observation tallies that are never +// compared with a limit are not allowances and are not counted. const namedAllowance = /this\.#?\w*(?:[Rr]emaining|[Rr]etained|[Uu]sed)\w*\s*[-+]=/gu; -const budgetClass = - /^(?:export\s+)?class\s+\w*Budget\b[^\n]*\{\n([\s\S]*?)^\}/gmu; - const adjusted = /(?:\+\+|--)this\.(#?\w+)|this\.(#?\w+)\s*(?:[-+]=|\+\+|--)/gu; const escape = (name) => name.replaceAll("$", "\\$"); @@ -30,16 +30,53 @@ const comparedIn = (body, name) => { ).test(body); }; +// Limit names in any case: `maxBytes`, `MAX_BYTES`, `FACT_BUDGET`, `capacity`. +const limitName = + "\\b\\w*(?:[Mm][Aa][Xx]|[Ll][Ii][Mm][Ii][Tt]|[Aa][Ll][Ll][Oo][Ww][Aa][Nn][Cc][Ee]|[Bb][Uu][Dd][Gg][Ee][Tt]|[Cc][Aa][Pp][Aa][Cc][Ii][Tt][Yy])\\w*"; + +// `<`, `>`, `<=` or `>=` between the field and a limit-named value, one line. +const comparedWithLimit = (scope, name) => { + const field = `this\\.${escape(name)}\\b`; + const comparison = "(?])[<>]=?(?![<>])"; + return new RegExp( + `${field}[^;\\n]*${comparison}[^;\\n]*${limitName}|${limitName}[^;\\n]*${comparison}[^;\\n]*${field}`, + "u", + ).test(scope); +}; + +const isNamedAllowance = (text) => + new RegExp(`^${namedAllowance.source}$`, "u").test(text); + +// Top-level classes (declaration and closing brace at column 0). A field +// belongs to its class, so each class is its own scope; code outside any +// class is one more scope. +const topLevelClass = + /^(?:export\s+)?(?:default\s+)?(?:abstract\s+)?class\s+(\w+)[^\n]*\{\n([\s\S]*?)^\}/gmu; + +/** Distinct limit-compared fields adjusted by hand within one scope. */ +const limitedFields = (scope) => { + const limited = new Set(); + for (const match of scope.matchAll(adjusted)) { + const name = match[1] ?? match[2]; + if (isNamedAllowance(match[0]) || limited.has(name)) continue; + if (comparedWithLimit(scope, name)) limited.add(name); + } + return limited.size; +}; + /** @param {string} content comment-stripped source */ export const budgetArithmeticCount = (content) => { let count = content.match(namedAllowance)?.length ?? 0; - for (const [, body] of content.matchAll(budgetClass)) { - for (const match of body.matchAll(adjusted)) { - // Already counted as a named allowance. - if (new RegExp(`^${namedAllowance.source}$`, "u").test(match[0])) - continue; - if (comparedIn(body, match[1] ?? match[2])) count += 1; - } + let outside = content; + for (const [whole, name, body] of content.matchAll(topLevelClass)) { + outside = outside.replace(whole, ""); + if (name.endsWith("Budget")) { + for (const match of body.matchAll(adjusted)) { + // Already counted as a named allowance. + if (isNamedAllowance(match[0])) continue; + if (comparedIn(body, match[1] ?? match[2])) count += 1; + } + } else count += limitedFields(body); } - return count; + return count + limitedFields(outside); }; diff --git a/src/application/reverse/ReverseEngineeringService.ts b/src/application/reverse/ReverseEngineeringService.ts index 6abbb9f81f..812f608df5 100644 --- a/src/application/reverse/ReverseEngineeringService.ts +++ b/src/application/reverse/ReverseEngineeringService.ts @@ -1,5 +1,6 @@ import { HOST_PLATFORM } from "../../platform/HostPlatform.js"; import { MAX_NODE_TIMER_DELAY_MS } from "../../domain/analysisResourceLimits.js"; +import { Budget } from "../../domain/budget.js"; import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; import { bindArtifactFileCleanup } from "../../artifacts/ArtifactFileOperation.js"; import { @@ -237,7 +238,8 @@ export class ReverseEngineeringService { AbortController, Promise> >(); - #reservedOutputBytes = 0; + /** Output capture admitted across active operations. */ + readonly #reservedOutput = new Budget(MAX_ACTIVE_CAPTURE_BYTES); #closing = false; #closePromise: Promise | undefined; #dispatched = 0; @@ -514,7 +516,7 @@ export class ReverseEngineeringService { if ( this.#activeOperations.size >= MAX_CONCURRENT_REVERSE_ENGINEERING_OPERATIONS || - this.#reservedOutputBytes + outputBudgetBytes > MAX_ACTIVE_CAPTURE_BYTES + !this.#reservedOutput.tryCharge(outputBudgetBytes) ) return Promise.resolve( err( @@ -527,7 +529,7 @@ export class ReverseEngineeringService { MAX_CONCURRENT_REVERSE_ENGINEERING_OPERATIONS, active_operations: this.#activeOperations.size, max_active_capture_bytes: MAX_ACTIVE_CAPTURE_BYTES, - reserved_output_bytes: this.#reservedOutputBytes, + reserved_output_bytes: this.#reservedOutput.used, requested_output_bytes: outputBudgetBytes, }, { @@ -547,7 +549,6 @@ export class ReverseEngineeringService { signal === undefined || signal instanceof AbortSignal ? controller.signal : signal; - this.#reservedOutputBytes += outputBudgetBytes; const running = this.#observeProvider({ ...request, profile: selection.value, @@ -567,7 +568,7 @@ export class ReverseEngineeringService { .finally(() => { signal?.removeEventListener("abort", relayAbort); this.#activeOperations.delete(controller); - this.#reservedOutputBytes -= outputBudgetBytes; + this.#reservedOutput.refund(outputBudgetBytes); }); this.#activeOperations.set(controller, tracked); return tracked; diff --git a/src/browser/history/playwrightTrace/TraceInventory.ts b/src/browser/history/playwrightTrace/TraceInventory.ts index 1f23b612c7..c581cea854 100644 --- a/src/browser/history/playwrightTrace/TraceInventory.ts +++ b/src/browser/history/playwrightTrace/TraceInventory.ts @@ -1,4 +1,5 @@ import { digestCanonicalValue } from "../../../domain/canonicalDigest.js"; +import { Budget } from "../../../domain/budget.js"; import { AnalysisResourceConstraintError } from "../../../domain/analysisErrorCore.js"; import type { ArtifactEntry } from "../../../artifacts/ArtifactReader.js"; import { @@ -20,21 +21,23 @@ export class TraceInventory { readonly events: TraceEvent[] = []; readonly diagnostics: PlaywrightTrace["diagnostics"] = []; readonly #paths = new Map(); - #bytes = 0; + readonly #facts = new Budget(PLAYWRIGHT_TRACE_LIMITS.factsBytes); constructor(readonly artifact: PlaywrightTrace["artifact"]) {} reserve(bytes: number): void { - this.#bytes += bytes; - if (this.#bytes > PLAYWRIGHT_TRACE_LIMITS.factsBytes) - throw new AnalysisResourceConstraintError( - PLAYWRIGHT_TRACE_OPERATION, - "memory", - "Trace facts exceed the 32 MiB retained representation allowance.", - { retained_fact_allowance: PLAYWRIGHT_TRACE_LIMITS.factsBytes }, - { - remediationAction: - "Select a smaller complete recorder chunk; fields and reply pagination do not increase parser allowances.", - }, - ); + this.#facts.charge( + bytes, + () => + new AnalysisResourceConstraintError( + PLAYWRIGHT_TRACE_OPERATION, + "memory", + "Trace facts exceed the 32 MiB retained representation allowance.", + { retained_fact_allowance: PLAYWRIGHT_TRACE_LIMITS.factsBytes }, + { + remediationAction: + "Select a smaller complete recorder chunk; fields and reply pagination do not increase parser allowances.", + }, + ), + ); } retain(value: unknown): void { this.reserve(512 + Buffer.byteLength(JSON.stringify(value)) * 4); diff --git a/src/domain/budget.test.ts b/src/domain/budget.test.ts index cb2622bd83..ddfae85f72 100644 --- a/src/domain/budget.test.ts +++ b/src/domain/budget.test.ts @@ -37,6 +37,22 @@ describe("Budget", () => { }).toThrow(/refund/u); }); + it("records an over-committing charge and refuses ordinary charges until refunded", () => { + const budget = new Budget(10); + expect(budget.chargeOverCommitting(6)).toBe(0); + expect(budget.chargeOverCommitting(7)).toBe(3); + expect(budget.used).toBe(13); + expect(budget.overCommitted).toBe(3); + expect(budget.remaining).toBe(0); + expect(budget.tryCharge(0)).toBe(true); + expect(budget.tryCharge(1)).toBe(false); + budget.refund(5); + expect(budget.overCommitted).toBe(0); + expect(budget.remaining).toBe(2); + expect(() => budget.chargeOverCommitting(1.5)).toThrow(/charge/u); + expect(budget.used).toBe(8); + }); + it("evicts the oldest entries until a charge fits", () => { const budget = new Budget(10); const entries = [4, 3, 2]; diff --git a/src/domain/budget.ts b/src/domain/budget.ts index 2f989fdf53..23c83b029b 100644 --- a/src/domain/budget.ts +++ b/src/domain/budget.ts @@ -11,6 +11,11 @@ const isAmount = (value: number): boolean => * that fits what remains, a refused charge changes nothing, and a refund never * raises the remainder above the maximum. Each owner keeps its own limit * failure, passed to `charge` as `exhausted`. + * + * An owner whose ceiling is advisory may charge past it with + * `chargeOverCommitting`; the budget then records the excess as + * `overCommitted`, reports no remainder, and refuses `tryCharge` until refunds + * bring use back under the maximum. */ export class Budget { #used = 0; @@ -27,9 +32,14 @@ export class Budget { return this.#used; } - /** Amount still available. */ + /** Amount still available; zero while over-committed. */ get remaining(): number { - return this.maximum - this.#used; + return Math.max(0, this.maximum - this.#used); + } + + /** Amount charged past the maximum by `chargeOverCommitting`. */ + get overCommitted(): number { + return Math.max(0, this.#used - this.maximum); } /** Charge `amount` when it fits; otherwise charge nothing and return false. */ @@ -44,6 +54,21 @@ export class Budget { if (!this.tryCharge(amount)) throw exhausted(); } + /** + * Charge `amount` whether or not it fits, for an owner that reports the + * excess itself rather than refusing it. Returns the shortfall: the part of + * `amount` beyond what remained, zero when it fit. + */ + chargeOverCommitting(amount: number): number { + if (!isAmount(amount)) + throw invariantViolation( + `Budget charge must be a non-negative safe integer, got ${String(amount)}`, + ); + const shortfall = Math.max(0, amount - this.remaining); + this.#used += amount; + return shortfall; + } + /** * Charge `amount` into a cache, evicting entries until it fits. `evict` * discards the oldest entry and returns its charged amount, or `undefined` diff --git a/src/domain/javascript/javascriptSemanticAliasSelections.ts b/src/domain/javascript/javascriptSemanticAliasSelections.ts index 427bf621e8..b85be17691 100644 --- a/src/domain/javascript/javascriptSemanticAliasSelections.ts +++ b/src/domain/javascript/javascriptSemanticAliasSelections.ts @@ -1,5 +1,6 @@ import type { JavaScriptSemanticPropertyPath } from "./javascriptSemanticPropertyPaths.js"; import { JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS } from "./javascriptSemanticPrecision.js"; +import { Budget } from "../budget.js"; type PropertyKey = JavaScriptSemanticPropertyPath[number]; @@ -73,13 +74,15 @@ const joinSelections = ( /** Preserve correlated selector alternatives while coalescing complete grids. */ export class SemanticAliasSelections { #paths = new WeakMap>(); - #calls = 0; + readonly #calls: Budget; #limited = false; constructor( readonly onLimit: () => void, readonly maxSelections: number = JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS.alias_selections, - ) {} + ) { + this.#calls = new Budget(maxSelections); + } get limited(): boolean { return this.#limited; @@ -98,7 +101,7 @@ export class SemanticAliasSelections { ); const covered = paths.find((prior) => pathCovers(prior, candidate)); if (covered !== undefined) return covered; - if (existing !== undefined && ++this.#calls > this.maxSelections) { + if (existing !== undefined && !this.#calls.tryCharge(1)) { this.#limited = true; this.#paths = new WeakMap(); this.onLimit(); diff --git a/src/javascript/analysis/JavaScriptAnalysisWorker.ts b/src/javascript/analysis/JavaScriptAnalysisWorker.ts index 89d76c3d31..c776117042 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorker.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorker.ts @@ -15,6 +15,7 @@ import { } from "../../providers/OwnedCommandCleanup.js"; import { analysisErrorWithCancellation } from "../../domain/analysisErrorCancellation.js"; import { safeParseUtf8Json } from "../../domain/safeJson.js"; +import { Budget } from "../../domain/budget.js"; import { analysisSettingLimit } from "../../domain/analysisSettingDefinitions.js"; import { WorkerFrameSignal } from "../../process/WorkerFrameSignal.js"; import type { @@ -146,7 +147,6 @@ export class JavaScriptAnalysisWorker { #terminalResponseId: string | undefined; #phase = "startup"; #lastFailure: AnalysisError | undefined; - #estimatedRetainedBytes = 0; #staticProjectionReservations: { sourceIndex: number; path: string; @@ -158,7 +158,12 @@ export class JavaScriptAnalysisWorker { #parentCapacityExhaustion: Readonly> | undefined; #sourcesRemaining: number | undefined; #deadline: ProviderStartupDeadline | undefined; - #resultBudgetBytes = 0; + /** + * Estimated retained result bytes against the result budget. Static + * checkpoints may over-commit it up to the static capacity; the worker + * reports that exhaustion itself rather than refusing the checkpoint. + */ + #retained = new Budget(0); #staticCapacityBytes = 0; /** Null when the runtime enforces the V8 heap flags instead. */ readonly #processMemoryLimitBytes: number | null; @@ -278,7 +283,6 @@ export class JavaScriptAnalysisWorker { "JavaScript worker source count must be a nonnegative safe integer", ); this.#sourcesRemaining = sourceCount; - this.#estimatedRetainedBytes = 0; this.#staticProjectionReservations = []; this.#revokedStaticProjections = []; this.#nextSourceIndex = 0; @@ -308,7 +312,7 @@ export class JavaScriptAnalysisWorker { this.#remainingParentHeapBytes() - 24 * JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES, ); - this.#resultBudgetBytes = Math.floor(capacity / 2); + this.#retained = new Budget(Math.floor(capacity / 2)); this.#staticCapacityBytes = Math.floor(capacity * STATIC_CAPACITY_SHARE); } @@ -515,7 +519,7 @@ export class JavaScriptAnalysisWorker { `Completed module facts for ${file.path} exceed the parent's remaining result budget; static facts were retained`, { ...this.#limits(), - parent_result_budget_bytes: this.#resultBudgetBytes, + parent_result_budget_bytes: this.#retained.maximum, parent_available_result_budget_bytes: remaining, module_transfer_bytes: descriptor.module_bytes, module_retention_estimate_bytes: moduleEstimate, @@ -538,8 +542,9 @@ export class JavaScriptAnalysisWorker { ); module = decoded.module; projection = decoded.projection; - this.#estimatedRetainedBytes += - moduleEstimate + (retainGraph ? graphEstimate : 0); + this.#retained.chargeOverCommitting( + moduleEstimate + (retainGraph ? graphEstimate : 0), + ); result = retainGraph ? ok({ javascript, module, projection: decoded.projection }) : err({ @@ -552,7 +557,7 @@ export class JavaScriptAnalysisWorker { `Semantic graph for ${file.path} exceeds the parent's remaining result budget; static and module facts were retained`, { ...this.#limits(), - parent_result_budget_bytes: this.#resultBudgetBytes, + parent_result_budget_bytes: this.#retained.maximum, parent_available_result_budget_bytes: remaining, module_retention_estimate_bytes: moduleEstimate, graph_retention_estimate_bytes: graphEstimate, @@ -885,9 +890,8 @@ export class JavaScriptAnalysisWorker { const staticCeiling = retainedBytes + used_heap_size <= liveAvailable ? this.#staticCapacityBytes - : this.#resultBudgetBytes; - const needed = - retainedBytes - (staticCeiling - this.#estimatedRetainedBytes); + : this.#retained.maximum; + const needed = retainedBytes - (staticCeiling - this.#retained.used); if (needed > 0 && retainedBytes <= liveAvailable) { // Plan all revocations first so an insufficient set leaves the earlier // relationships intact rather than sacrificing them for a failed read. @@ -900,14 +904,14 @@ export class JavaScriptAnalysisWorker { if (reclaimable >= needed) { const revoked = this.#staticProjectionReservations.splice(start); for (const reservation of revoked) { - this.#estimatedRetainedBytes -= reservation.bytes; + this.#retained.refund(reservation.bytes); this.#revokedStaticProjections.push({ sourceIndex: reservation.sourceIndex, error: new AnalysisResourceConstraintError( OPERATION, "memory", `Expanded application relationships for ${reservation.path} were omitted to retain static facts for later sources`, - { parent_result_budget_bytes: this.#resultBudgetBytes }, + { parent_result_budget_bytes: this.#retained.maximum }, { remediationAction: `Read the retained static facts or analyze fewer source files. ${JAVASCRIPT_PARENT_RESULT_BUDGET_REMEDIATION}`, }, @@ -920,7 +924,7 @@ export class JavaScriptAnalysisWorker { // admission so a second read cannot reject after revocations commit. const remaining = Math.max( 0, - Math.min(staticCeiling - this.#estimatedRetainedBytes, liveAvailable), + Math.min(staticCeiling - this.#retained.used, liveAvailable), ); this.#parentCapacityExhaustion = retainedBytes > remaining @@ -931,7 +935,7 @@ export class JavaScriptAnalysisWorker { static_retention_estimate_bytes: retainedBytes, } : undefined; - this.#estimatedRetainedBytes += retainedBytes; + this.#retained.chargeOverCommitting(retainedBytes); const expandedBytes = estimateJavaScriptStaticApplicationProjection( facts, bytes, @@ -939,7 +943,7 @@ export class JavaScriptAnalysisWorker { ); const availableResultBudget = this.#remainingResultBudget(); if (expandedBytes <= availableResultBudget) { - this.#estimatedRetainedBytes += expandedBytes; + this.#retained.chargeOverCommitting(expandedBytes); this.#staticProjectionReservations.push({ sourceIndex, path: sourcePath, @@ -951,7 +955,7 @@ export class JavaScriptAnalysisWorker { "memory", `Expanded application relationships for ${sourcePath} exceed the parent's result budget; complete static facts were retained on the file node`, { - parent_result_budget_bytes: this.#resultBudgetBytes, + parent_result_budget_bytes: this.#retained.maximum, parent_available_result_budget_bytes: availableResultBudget, application_projection_required_bytes: expandedBytes, }, @@ -972,7 +976,7 @@ export class JavaScriptAnalysisWorker { `The parent's result capacity was exhausted after retaining static facts for ${path}; remaining sources were not analyzed`, { ...this.#limits(), - parent_result_budget_bytes: this.#resultBudgetBytes, + parent_result_budget_bytes: this.#retained.maximum, ...this.#parentCapacityExhaustion, parent_result_budget_exhausted: true, }, @@ -1024,7 +1028,7 @@ export class JavaScriptAnalysisWorker { #remainingResultBudget(): number { // One sequential worker produces one next checkpoint at a time. Reserve // its maximum protocol size at the same retained expansion used on receipt; - // earlier checkpoints are already charged to estimatedRetainedBytes. + // earlier checkpoints are already charged to the retained budget. // Reserving every unexamined source's maximum would reject useful results // merely because many small files remain. Actual retained growth and live // heap capacity are checked again for every checkpoint and transfer. @@ -1040,7 +1044,7 @@ export class JavaScriptAnalysisWorker { return Math.max( 0, Math.min( - this.#resultBudgetBytes - this.#estimatedRetainedBytes, + this.#retained.remaining, this.#remainingParentHeapBytes() - 24 * JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES, ), diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts index a82b7745dd..80eb5e1cca 100644 --- a/src/python/PythonPycParser.ts +++ b/src/python/PythonPycParser.ts @@ -310,11 +310,11 @@ export const parsePythonPyc = ( class PycMarshalReader extends MarshalReader { /** Name and identifier bytes the report carries across all code objects. */ - #reportedStringBytes = 0; + readonly #reportedStringBytes: Budget; /** Const summaries the report copies, including tables shared by TYPE_REF. */ - #reportedConsts = 0; + readonly #reportedCopies: Budget; /** Bytes of hash-key descriptors materialized for set and dict rules. */ - #keyBytes = 0; + readonly #keyBytes: Budget; /** Identity keys for NaN values, which are not equal to themselves. */ readonly #identities = new WeakMap(); #identityNext = 0; @@ -325,8 +325,11 @@ class PycMarshalReader extends MarshalReader { /** Interned descriptors so equal values share one token within this walk. */ readonly #keyTokens = new Map(); #keyTokenNext = 0; - /** Explicit key-walk steps. Stream `max_depth` does not bound reference chains. */ - #keySteps = 0; + /** + * Explicit key-walk steps, `max_objects * 8`. Stream `max_depth` does not + * bound reference chains. + */ + readonly #keySteps: Budget; /** One code-equality key per code object. Empty while that key is in progress. */ readonly #codeKeys = new WeakMap(); readonly #codeIdentities = new WeakMap(); @@ -368,6 +371,10 @@ class PycMarshalReader extends MarshalReader { }); this.#nestedSteps = new Budget(input.max_objects * input.max_depth); this.#nestedRetained = new Budget(input.max_objects * input.max_depth); + this.#reportedStringBytes = new Budget(input.max_file_bytes); + this.#reportedCopies = new Budget(input.max_objects); + this.#keyBytes = new Budget(input.max_file_bytes); + this.#keySteps = new Budget(input.max_objects * 8); } protected wrap(value: MarshalCoreValue): MarshalValue { @@ -672,14 +679,17 @@ class PycMarshalReader extends MarshalReader { * in the report, so bound what the report carries, not the stream. */ #reportStrings(values: readonly string[]): void { - for (const value of values) - this.#reportedStringBytes += Buffer.byteLength(value, "utf8"); - if (this.#reportedStringBytes > this.input.max_file_bytes) - throw new PythonPycResourceFailure( - "string", - { max_file_bytes: this.input.max_file_bytes }, - `Reported code-object strings exceed max_file_bytes ${this.input.max_file_bytes}.`, - ); + let bytes = 0; + for (const value of values) bytes += Buffer.byteLength(value, "utf8"); + this.#reportedStringBytes.charge( + bytes, + () => + new PythonPycResourceFailure( + "string", + { max_file_bytes: this.input.max_file_bytes }, + `Reported code-object strings exceed max_file_bytes ${this.input.max_file_bytes}.`, + ), + ); } /** @@ -717,13 +727,15 @@ class PycMarshalReader extends MarshalReader { count: number, what: "constants" | "names" | "nested code relationships", ): void { - this.#reportedConsts += count; - if (this.#reportedConsts > this.input.max_objects) - throw new PythonPycResourceFailure( - "objects", - { max_objects: this.input.max_objects }, - `Reported ${what} exceed max_objects ${this.input.max_objects}.`, - ); + this.#reportedCopies.charge( + count, + () => + new PythonPycResourceFailure( + "objects", + { max_objects: this.input.max_objects }, + `Reported ${what} exceed max_objects ${this.input.max_objects}.`, + ), + ); } /** @@ -1135,14 +1147,15 @@ class PycMarshalReader extends MarshalReader { * not `max_depth`, because a reference chain is shallow in the stream. */ #takeKeyStep(): void { - this.#keySteps += 1; - const limit = this.input.max_objects * 8; - if (this.#keySteps > limit) - throw new PythonPycResourceFailure( - "objects", - { max_objects: this.input.max_objects }, - `Marshal key walk exceeds ${limit} steps (max_objects * 8).`, - ); + this.#keySteps.charge( + 1, + () => + new PythonPycResourceFailure( + "objects", + { max_objects: this.input.max_objects }, + `Marshal key walk exceeds ${this.#keySteps.maximum} steps (max_objects * 8).`, + ), + ); } #leafKey(value: MarshalValue, mode: KeyMode): ResolvedKey | null { @@ -1451,13 +1464,15 @@ class PycMarshalReader extends MarshalReader { #reserveKey(size: number): void { if (size > this.input.max_string_bytes) this.#keyTooLarge(size); - this.#keyBytes += size; - if (this.#keyBytes > this.input.max_file_bytes) - throw new PythonPycResourceFailure( - "string", - { max_file_bytes: this.input.max_file_bytes }, - `Marshal key expansion exceeds max_file_bytes ${this.input.max_file_bytes}.`, - ); + this.#keyBytes.charge( + size, + () => + new PythonPycResourceFailure( + "string", + { max_file_bytes: this.input.max_file_bytes }, + `Marshal key expansion exceeds max_file_bytes ${this.input.max_file_bytes}.`, + ), + ); } #keyTooLarge(size: number): never { diff --git a/tests/evaluation/budgetArithmeticGuard.test.ts b/tests/evaluation/budgetArithmeticGuard.test.ts index 914a19aab4..254d6b9a39 100644 --- a/tests/evaluation/budgetArithmeticGuard.test.ts +++ b/tests/evaluation/budgetArithmeticGuard.test.ts @@ -24,7 +24,7 @@ it("counts hand-checked fields inside a hand-written Budget class", () => { expect(budgetArithmeticCount(cuda)).toBe(2); }); -it("ignores unchecked tallies and fields outside Budget classes", () => { +it("ignores unchecked tallies and fields not compared with a limit", () => { const tallies = [ "class ProtobufBudget {", " async checkpoint(): Promise {", @@ -35,10 +35,69 @@ it("ignores unchecked tallies and fields outside Budget classes", () => { "}", "class Reader {", " read(size: number): void {", - " if (this.#bytes + size > LIMIT) throw failure();", - " this.#bytes += size;", + " this.#observed += size;", + " if (this.#observed > this.#previous) this.#grew = true;", " }", "}", ].join("\n"); expect(budgetArithmeticCount(tallies)).toBe(0); }); + +it("counts a field outside Budget classes compared with a limit once", () => { + const reader = [ + "class Reader {", + " read(size: number): void {", + " if (this.#bytes + size > LIMIT) throw failure();", + " this.#bytes += size;", + " }", + " key(size: number): void {", + " this.#keyBytes += size;", + " if (this.#keyBytes > this.input.max_file_bytes) throw failure();", + " this.#keyBytes += 1;", + " }", + "}", + ].join("\n"); + expect(budgetArithmeticCount(reader)).toBe(2); +}); + +it("counts the same field name separately in each class", () => { + const readers = [ + "class First {", + " read(size: number): void {", + " if (this.#bytes + size > LIMIT) throw failure();", + " this.#bytes += size;", + " }", + "}", + "class SecondBudget {", + " #bytes = 0;", + " observe(size: number): void {", + " this.#bytes += size;", + " }", + "}", + "class Third {", + " read(size: number): void {", + " this.#bytes += size;", + " if (this.#bytes > LIMIT) throw failure();", + " }", + "}", + ].join("\n"); + expect(budgetArithmeticCount(readers)).toBe(2); +}); + +it.each([ + "CAPACITY", + "FACT_ALLOWANCE", + "FACT_BUDGET", + "MAX_BYTES", + "byteLimit", +])("counts a field compared with %s", (limit) => { + const reader = [ + "class Reader {", + " read(size: number): void {", + " this.#facts += size;", + ` if (this.#facts > ${limit}) throw failure();`, + " }", + "}", + ].join("\n"); + expect(budgetArithmeticCount(reader)).toBe(1); +});