diff --git a/scripts/verify/baselines/budget-arithmetic.json b/scripts/verify/baselines/budget-arithmetic.json index dd218c51ee..a3866e17b6 100644 --- a/scripts/verify/baselines/budget-arithmetic.json +++ b/scripts/verify/baselines/budget-arithmetic.json @@ -1,3 +1,13 @@ { - "src/javascript/analysis/JavaScriptAnalysisWorker.ts": 4 + "src/application/environment/environmentStateStore.ts": 1, + "src/artifacts/AsarHeader.ts": 1, + "src/domain/javascript/javascriptRouteScan.ts": 1, + "src/domain/python/marshalReader.ts": 1, + "src/kuna/KunaJsonStream.ts": 1, + "src/kuna/KunaRunSlots.ts": 1, + "src/process/BoundedOutputExcerpt.ts": 1, + "src/process/OwnedMcpStdioTransport.ts": 1, + "src/process/ProviderProcess.ts": 1, + "src/process/capture/ProcessSensitiveOutput.ts": 1, + "src/server/http/NodeHttpRequestOwner.ts": 1 } diff --git a/scripts/verify/budget-arithmetic-guard.mjs b/scripts/verify/budget-arithmetic-guard.mjs index 8c328fefbe..65ef0c0ce1 100644 --- a/scripts/verify/budget-arithmetic-guard.mjs +++ b/scripts/verify/budget-arithmetic-guard.mjs @@ -7,15 +7,15 @@ // by hand (`+=`, `-=`, `++`, `--`) and compared with `<` or `>`: that is a // limit check, whatever the field is called (`#bytes`, `#visits`). // -// Write cursors (`#arenaWriteOffset`) and observation tallies are not -// allowances; outside a `…Budget` class they are not counted unless named as -// one. +// 3. Anywhere else, a field adjusted by hand and compared with `<` or `>` +// (or `<=`/`>=`) against a value named as a limit (`…max…`, `…limit…`, +// `…allowance…`, `…budget…`, `…capacity…`): a hand-written allowance. +// +// Write cursors (`#arenaWriteOffset`) and observation tallies that are never +// compared with a limit are not allowances and are not counted. const namedAllowance = /this\.#?\w*(?:[Rr]emaining|[Rr]etained|[Uu]sed)\w*\s*[-+]=/gu; -const budgetClass = - /^(?:export\s+)?class\s+\w*Budget\b[^\n]*\{\n([\s\S]*?)^\}/gmu; - const adjusted = /(?:\+\+|--)this\.(#?\w+)|this\.(#?\w+)\s*(?:[-+]=|\+\+|--)/gu; const escape = (name) => name.replaceAll("$", "\\$"); @@ -30,16 +30,53 @@ const comparedIn = (body, name) => { ).test(body); }; +// Limit names in any case: `maxBytes`, `MAX_BYTES`, `FACT_BUDGET`, `capacity`. +const limitName = + "\\b\\w*(?:[Mm][Aa][Xx]|[Ll][Ii][Mm][Ii][Tt]|[Aa][Ll][Ll][Oo][Ww][Aa][Nn][Cc][Ee]|[Bb][Uu][Dd][Gg][Ee][Tt]|[Cc][Aa][Pp][Aa][Cc][Ii][Tt][Yy])\\w*"; + +// `<`, `>`, `<=` or `>=` between the field and a limit-named value, one line. +const comparedWithLimit = (scope, name) => { + const field = `this\\.${escape(name)}\\b`; + const comparison = "(?])[<>]=?(?![<>])"; + return new RegExp( + `${field}[^;\\n]*${comparison}[^;\\n]*${limitName}|${limitName}[^;\\n]*${comparison}[^;\\n]*${field}`, + "u", + ).test(scope); +}; + +const isNamedAllowance = (text) => + new RegExp(`^${namedAllowance.source}$`, "u").test(text); + +// Top-level classes (declaration and closing brace at column 0). A field +// belongs to its class, so each class is its own scope; code outside any +// class is one more scope. +const topLevelClass = + /^(?:export\s+)?(?:default\s+)?(?:abstract\s+)?class\s+(\w+)[^\n]*\{\n([\s\S]*?)^\}/gmu; + +/** Distinct limit-compared fields adjusted by hand within one scope. */ +const limitedFields = (scope) => { + const limited = new Set(); + for (const match of scope.matchAll(adjusted)) { + const name = match[1] ?? match[2]; + if (isNamedAllowance(match[0]) || limited.has(name)) continue; + if (comparedWithLimit(scope, name)) limited.add(name); + } + return limited.size; +}; + /** @param {string} content comment-stripped source */ export const budgetArithmeticCount = (content) => { let count = content.match(namedAllowance)?.length ?? 0; - for (const [, body] of content.matchAll(budgetClass)) { - for (const match of body.matchAll(adjusted)) { - // Already counted as a named allowance. - if (new RegExp(`^${namedAllowance.source}$`, "u").test(match[0])) - continue; - if (comparedIn(body, match[1] ?? match[2])) count += 1; - } + let outside = content; + for (const [whole, name, body] of content.matchAll(topLevelClass)) { + outside = outside.replace(whole, ""); + if (name.endsWith("Budget")) { + for (const match of body.matchAll(adjusted)) { + // Already counted as a named allowance. + if (isNamedAllowance(match[0])) continue; + if (comparedIn(body, match[1] ?? match[2])) count += 1; + } + } else count += limitedFields(body); } - return count; + return count + limitedFields(outside); }; diff --git a/src/application/reverse/ReverseEngineeringService.ts b/src/application/reverse/ReverseEngineeringService.ts index 6abbb9f81f..812f608df5 100644 --- a/src/application/reverse/ReverseEngineeringService.ts +++ b/src/application/reverse/ReverseEngineeringService.ts @@ -1,5 +1,6 @@ import { HOST_PLATFORM } from "../../platform/HostPlatform.js"; import { MAX_NODE_TIMER_DELAY_MS } from "../../domain/analysisResourceLimits.js"; +import { Budget } from "../../domain/budget.js"; import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; import { bindArtifactFileCleanup } from "../../artifacts/ArtifactFileOperation.js"; import { @@ -237,7 +238,8 @@ export class ReverseEngineeringService { AbortController, Promise> >(); - #reservedOutputBytes = 0; + /** Output capture admitted across active operations. */ + readonly #reservedOutput = new Budget(MAX_ACTIVE_CAPTURE_BYTES); #closing = false; #closePromise: Promise | undefined; #dispatched = 0; @@ -514,7 +516,7 @@ export class ReverseEngineeringService { if ( this.#activeOperations.size >= MAX_CONCURRENT_REVERSE_ENGINEERING_OPERATIONS || - this.#reservedOutputBytes + outputBudgetBytes > MAX_ACTIVE_CAPTURE_BYTES + !this.#reservedOutput.tryCharge(outputBudgetBytes) ) return Promise.resolve( err( @@ -527,7 +529,7 @@ export class ReverseEngineeringService { MAX_CONCURRENT_REVERSE_ENGINEERING_OPERATIONS, active_operations: this.#activeOperations.size, max_active_capture_bytes: MAX_ACTIVE_CAPTURE_BYTES, - reserved_output_bytes: this.#reservedOutputBytes, + reserved_output_bytes: this.#reservedOutput.used, requested_output_bytes: outputBudgetBytes, }, { @@ -547,7 +549,6 @@ export class ReverseEngineeringService { signal === undefined || signal instanceof AbortSignal ? controller.signal : signal; - this.#reservedOutputBytes += outputBudgetBytes; const running = this.#observeProvider({ ...request, profile: selection.value, @@ -567,7 +568,7 @@ export class ReverseEngineeringService { .finally(() => { signal?.removeEventListener("abort", relayAbort); this.#activeOperations.delete(controller); - this.#reservedOutputBytes -= outputBudgetBytes; + this.#reservedOutput.refund(outputBudgetBytes); }); this.#activeOperations.set(controller, tracked); return tracked; diff --git a/src/browser/history/playwrightTrace/TraceInventory.ts b/src/browser/history/playwrightTrace/TraceInventory.ts index 1f23b612c7..c581cea854 100644 --- a/src/browser/history/playwrightTrace/TraceInventory.ts +++ b/src/browser/history/playwrightTrace/TraceInventory.ts @@ -1,4 +1,5 @@ import { digestCanonicalValue } from "../../../domain/canonicalDigest.js"; +import { Budget } from "../../../domain/budget.js"; import { AnalysisResourceConstraintError } from "../../../domain/analysisErrorCore.js"; import type { ArtifactEntry } from "../../../artifacts/ArtifactReader.js"; import { @@ -20,21 +21,23 @@ export class TraceInventory { readonly events: TraceEvent[] = []; readonly diagnostics: PlaywrightTrace["diagnostics"] = []; readonly #paths = new Map(); - #bytes = 0; + readonly #facts = new Budget(PLAYWRIGHT_TRACE_LIMITS.factsBytes); constructor(readonly artifact: PlaywrightTrace["artifact"]) {} reserve(bytes: number): void { - this.#bytes += bytes; - if (this.#bytes > PLAYWRIGHT_TRACE_LIMITS.factsBytes) - throw new AnalysisResourceConstraintError( - PLAYWRIGHT_TRACE_OPERATION, - "memory", - "Trace facts exceed the 32 MiB retained representation allowance.", - { retained_fact_allowance: PLAYWRIGHT_TRACE_LIMITS.factsBytes }, - { - remediationAction: - "Select a smaller complete recorder chunk; fields and reply pagination do not increase parser allowances.", - }, - ); + this.#facts.charge( + bytes, + () => + new AnalysisResourceConstraintError( + PLAYWRIGHT_TRACE_OPERATION, + "memory", + "Trace facts exceed the 32 MiB retained representation allowance.", + { retained_fact_allowance: PLAYWRIGHT_TRACE_LIMITS.factsBytes }, + { + remediationAction: + "Select a smaller complete recorder chunk; fields and reply pagination do not increase parser allowances.", + }, + ), + ); } retain(value: unknown): void { this.reserve(512 + Buffer.byteLength(JSON.stringify(value)) * 4); diff --git a/src/domain/budget.test.ts b/src/domain/budget.test.ts index cb2622bd83..ddfae85f72 100644 --- a/src/domain/budget.test.ts +++ b/src/domain/budget.test.ts @@ -37,6 +37,22 @@ describe("Budget", () => { }).toThrow(/refund/u); }); + it("records an over-committing charge and refuses ordinary charges until refunded", () => { + const budget = new Budget(10); + expect(budget.chargeOverCommitting(6)).toBe(0); + expect(budget.chargeOverCommitting(7)).toBe(3); + expect(budget.used).toBe(13); + expect(budget.overCommitted).toBe(3); + expect(budget.remaining).toBe(0); + expect(budget.tryCharge(0)).toBe(true); + expect(budget.tryCharge(1)).toBe(false); + budget.refund(5); + expect(budget.overCommitted).toBe(0); + expect(budget.remaining).toBe(2); + expect(() => budget.chargeOverCommitting(1.5)).toThrow(/charge/u); + expect(budget.used).toBe(8); + }); + it("evicts the oldest entries until a charge fits", () => { const budget = new Budget(10); const entries = [4, 3, 2]; diff --git a/src/domain/budget.ts b/src/domain/budget.ts index 2f989fdf53..23c83b029b 100644 --- a/src/domain/budget.ts +++ b/src/domain/budget.ts @@ -11,6 +11,11 @@ const isAmount = (value: number): boolean => * that fits what remains, a refused charge changes nothing, and a refund never * raises the remainder above the maximum. Each owner keeps its own limit * failure, passed to `charge` as `exhausted`. + * + * An owner whose ceiling is advisory may charge past it with + * `chargeOverCommitting`; the budget then records the excess as + * `overCommitted`, reports no remainder, and refuses `tryCharge` until refunds + * bring use back under the maximum. */ export class Budget { #used = 0; @@ -27,9 +32,14 @@ export class Budget { return this.#used; } - /** Amount still available. */ + /** Amount still available; zero while over-committed. */ get remaining(): number { - return this.maximum - this.#used; + return Math.max(0, this.maximum - this.#used); + } + + /** Amount charged past the maximum by `chargeOverCommitting`. */ + get overCommitted(): number { + return Math.max(0, this.#used - this.maximum); } /** Charge `amount` when it fits; otherwise charge nothing and return false. */ @@ -44,6 +54,21 @@ export class Budget { if (!this.tryCharge(amount)) throw exhausted(); } + /** + * Charge `amount` whether or not it fits, for an owner that reports the + * excess itself rather than refusing it. Returns the shortfall: the part of + * `amount` beyond what remained, zero when it fit. + */ + chargeOverCommitting(amount: number): number { + if (!isAmount(amount)) + throw invariantViolation( + `Budget charge must be a non-negative safe integer, got ${String(amount)}`, + ); + const shortfall = Math.max(0, amount - this.remaining); + this.#used += amount; + return shortfall; + } + /** * Charge `amount` into a cache, evicting entries until it fits. `evict` * discards the oldest entry and returns its charged amount, or `undefined` diff --git a/src/domain/javascript/javascriptSemanticAliasSelections.ts b/src/domain/javascript/javascriptSemanticAliasSelections.ts index 427bf621e8..b85be17691 100644 --- a/src/domain/javascript/javascriptSemanticAliasSelections.ts +++ b/src/domain/javascript/javascriptSemanticAliasSelections.ts @@ -1,5 +1,6 @@ import type { JavaScriptSemanticPropertyPath } from "./javascriptSemanticPropertyPaths.js"; import { JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS } from "./javascriptSemanticPrecision.js"; +import { Budget } from "../budget.js"; type PropertyKey = JavaScriptSemanticPropertyPath[number]; @@ -73,13 +74,15 @@ const joinSelections = ( /** Preserve correlated selector alternatives while coalescing complete grids. */ export class SemanticAliasSelections { #paths = new WeakMap>(); - #calls = 0; + readonly #calls: Budget; #limited = false; constructor( readonly onLimit: () => void, readonly maxSelections: number = JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS.alias_selections, - ) {} + ) { + this.#calls = new Budget(maxSelections); + } get limited(): boolean { return this.#limited; @@ -98,7 +101,7 @@ export class SemanticAliasSelections { ); const covered = paths.find((prior) => pathCovers(prior, candidate)); if (covered !== undefined) return covered; - if (existing !== undefined && ++this.#calls > this.maxSelections) { + if (existing !== undefined && !this.#calls.tryCharge(1)) { this.#limited = true; this.#paths = new WeakMap(); this.onLimit(); diff --git a/src/javascript/analysis/JavaScriptAnalysisWorker.ts b/src/javascript/analysis/JavaScriptAnalysisWorker.ts index 89d76c3d31..c776117042 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorker.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorker.ts @@ -15,6 +15,7 @@ import { } from "../../providers/OwnedCommandCleanup.js"; import { analysisErrorWithCancellation } from "../../domain/analysisErrorCancellation.js"; import { safeParseUtf8Json } from "../../domain/safeJson.js"; +import { Budget } from "../../domain/budget.js"; import { analysisSettingLimit } from "../../domain/analysisSettingDefinitions.js"; import { WorkerFrameSignal } from "../../process/WorkerFrameSignal.js"; import type { @@ -146,7 +147,6 @@ export class JavaScriptAnalysisWorker { #terminalResponseId: string | undefined; #phase = "startup"; #lastFailure: AnalysisError | undefined; - #estimatedRetainedBytes = 0; #staticProjectionReservations: { sourceIndex: number; path: string; @@ -158,7 +158,12 @@ export class JavaScriptAnalysisWorker { #parentCapacityExhaustion: Readonly> | undefined; #sourcesRemaining: number | undefined; #deadline: ProviderStartupDeadline | undefined; - #resultBudgetBytes = 0; + /** + * Estimated retained result bytes against the result budget. Static + * checkpoints may over-commit it up to the static capacity; the worker + * reports that exhaustion itself rather than refusing the checkpoint. + */ + #retained = new Budget(0); #staticCapacityBytes = 0; /** Null when the runtime enforces the V8 heap flags instead. */ readonly #processMemoryLimitBytes: number | null; @@ -278,7 +283,6 @@ export class JavaScriptAnalysisWorker { "JavaScript worker source count must be a nonnegative safe integer", ); this.#sourcesRemaining = sourceCount; - this.#estimatedRetainedBytes = 0; this.#staticProjectionReservations = []; this.#revokedStaticProjections = []; this.#nextSourceIndex = 0; @@ -308,7 +312,7 @@ export class JavaScriptAnalysisWorker { this.#remainingParentHeapBytes() - 24 * JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES, ); - this.#resultBudgetBytes = Math.floor(capacity / 2); + this.#retained = new Budget(Math.floor(capacity / 2)); this.#staticCapacityBytes = Math.floor(capacity * STATIC_CAPACITY_SHARE); } @@ -515,7 +519,7 @@ export class JavaScriptAnalysisWorker { `Completed module facts for ${file.path} exceed the parent's remaining result budget; static facts were retained`, { ...this.#limits(), - parent_result_budget_bytes: this.#resultBudgetBytes, + parent_result_budget_bytes: this.#retained.maximum, parent_available_result_budget_bytes: remaining, module_transfer_bytes: descriptor.module_bytes, module_retention_estimate_bytes: moduleEstimate, @@ -538,8 +542,9 @@ export class JavaScriptAnalysisWorker { ); module = decoded.module; projection = decoded.projection; - this.#estimatedRetainedBytes += - moduleEstimate + (retainGraph ? graphEstimate : 0); + this.#retained.chargeOverCommitting( + moduleEstimate + (retainGraph ? graphEstimate : 0), + ); result = retainGraph ? ok({ javascript, module, projection: decoded.projection }) : err({ @@ -552,7 +557,7 @@ export class JavaScriptAnalysisWorker { `Semantic graph for ${file.path} exceeds the parent's remaining result budget; static and module facts were retained`, { ...this.#limits(), - parent_result_budget_bytes: this.#resultBudgetBytes, + parent_result_budget_bytes: this.#retained.maximum, parent_available_result_budget_bytes: remaining, module_retention_estimate_bytes: moduleEstimate, graph_retention_estimate_bytes: graphEstimate, @@ -885,9 +890,8 @@ export class JavaScriptAnalysisWorker { const staticCeiling = retainedBytes + used_heap_size <= liveAvailable ? this.#staticCapacityBytes - : this.#resultBudgetBytes; - const needed = - retainedBytes - (staticCeiling - this.#estimatedRetainedBytes); + : this.#retained.maximum; + const needed = retainedBytes - (staticCeiling - this.#retained.used); if (needed > 0 && retainedBytes <= liveAvailable) { // Plan all revocations first so an insufficient set leaves the earlier // relationships intact rather than sacrificing them for a failed read. @@ -900,14 +904,14 @@ export class JavaScriptAnalysisWorker { if (reclaimable >= needed) { const revoked = this.#staticProjectionReservations.splice(start); for (const reservation of revoked) { - this.#estimatedRetainedBytes -= reservation.bytes; + this.#retained.refund(reservation.bytes); this.#revokedStaticProjections.push({ sourceIndex: reservation.sourceIndex, error: new AnalysisResourceConstraintError( OPERATION, "memory", `Expanded application relationships for ${reservation.path} were omitted to retain static facts for later sources`, - { parent_result_budget_bytes: this.#resultBudgetBytes }, + { parent_result_budget_bytes: this.#retained.maximum }, { remediationAction: `Read the retained static facts or analyze fewer source files. ${JAVASCRIPT_PARENT_RESULT_BUDGET_REMEDIATION}`, }, @@ -920,7 +924,7 @@ export class JavaScriptAnalysisWorker { // admission so a second read cannot reject after revocations commit. const remaining = Math.max( 0, - Math.min(staticCeiling - this.#estimatedRetainedBytes, liveAvailable), + Math.min(staticCeiling - this.#retained.used, liveAvailable), ); this.#parentCapacityExhaustion = retainedBytes > remaining @@ -931,7 +935,7 @@ export class JavaScriptAnalysisWorker { static_retention_estimate_bytes: retainedBytes, } : undefined; - this.#estimatedRetainedBytes += retainedBytes; + this.#retained.chargeOverCommitting(retainedBytes); const expandedBytes = estimateJavaScriptStaticApplicationProjection( facts, bytes, @@ -939,7 +943,7 @@ export class JavaScriptAnalysisWorker { ); const availableResultBudget = this.#remainingResultBudget(); if (expandedBytes <= availableResultBudget) { - this.#estimatedRetainedBytes += expandedBytes; + this.#retained.chargeOverCommitting(expandedBytes); this.#staticProjectionReservations.push({ sourceIndex, path: sourcePath, @@ -951,7 +955,7 @@ export class JavaScriptAnalysisWorker { "memory", `Expanded application relationships for ${sourcePath} exceed the parent's result budget; complete static facts were retained on the file node`, { - parent_result_budget_bytes: this.#resultBudgetBytes, + parent_result_budget_bytes: this.#retained.maximum, parent_available_result_budget_bytes: availableResultBudget, application_projection_required_bytes: expandedBytes, }, @@ -972,7 +976,7 @@ export class JavaScriptAnalysisWorker { `The parent's result capacity was exhausted after retaining static facts for ${path}; remaining sources were not analyzed`, { ...this.#limits(), - parent_result_budget_bytes: this.#resultBudgetBytes, + parent_result_budget_bytes: this.#retained.maximum, ...this.#parentCapacityExhaustion, parent_result_budget_exhausted: true, }, @@ -1024,7 +1028,7 @@ export class JavaScriptAnalysisWorker { #remainingResultBudget(): number { // One sequential worker produces one next checkpoint at a time. Reserve // its maximum protocol size at the same retained expansion used on receipt; - // earlier checkpoints are already charged to estimatedRetainedBytes. + // earlier checkpoints are already charged to the retained budget. // Reserving every unexamined source's maximum would reject useful results // merely because many small files remain. Actual retained growth and live // heap capacity are checked again for every checkpoint and transfer. @@ -1040,7 +1044,7 @@ export class JavaScriptAnalysisWorker { return Math.max( 0, Math.min( - this.#resultBudgetBytes - this.#estimatedRetainedBytes, + this.#retained.remaining, this.#remainingParentHeapBytes() - 24 * JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES, ), diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts index a82b7745dd..80eb5e1cca 100644 --- a/src/python/PythonPycParser.ts +++ b/src/python/PythonPycParser.ts @@ -310,11 +310,11 @@ export const parsePythonPyc = ( class PycMarshalReader extends MarshalReader { /** Name and identifier bytes the report carries across all code objects. */ - #reportedStringBytes = 0; + readonly #reportedStringBytes: Budget; /** Const summaries the report copies, including tables shared by TYPE_REF. */ - #reportedConsts = 0; + readonly #reportedCopies: Budget; /** Bytes of hash-key descriptors materialized for set and dict rules. */ - #keyBytes = 0; + readonly #keyBytes: Budget; /** Identity keys for NaN values, which are not equal to themselves. */ readonly #identities = new WeakMap(); #identityNext = 0; @@ -325,8 +325,11 @@ class PycMarshalReader extends MarshalReader { /** Interned descriptors so equal values share one token within this walk. */ readonly #keyTokens = new Map(); #keyTokenNext = 0; - /** Explicit key-walk steps. Stream `max_depth` does not bound reference chains. */ - #keySteps = 0; + /** + * Explicit key-walk steps, `max_objects * 8`. Stream `max_depth` does not + * bound reference chains. + */ + readonly #keySteps: Budget; /** One code-equality key per code object. Empty while that key is in progress. */ readonly #codeKeys = new WeakMap(); readonly #codeIdentities = new WeakMap(); @@ -368,6 +371,10 @@ class PycMarshalReader extends MarshalReader { }); this.#nestedSteps = new Budget(input.max_objects * input.max_depth); this.#nestedRetained = new Budget(input.max_objects * input.max_depth); + this.#reportedStringBytes = new Budget(input.max_file_bytes); + this.#reportedCopies = new Budget(input.max_objects); + this.#keyBytes = new Budget(input.max_file_bytes); + this.#keySteps = new Budget(input.max_objects * 8); } protected wrap(value: MarshalCoreValue): MarshalValue { @@ -672,14 +679,17 @@ class PycMarshalReader extends MarshalReader { * in the report, so bound what the report carries, not the stream. */ #reportStrings(values: readonly string[]): void { - for (const value of values) - this.#reportedStringBytes += Buffer.byteLength(value, "utf8"); - if (this.#reportedStringBytes > this.input.max_file_bytes) - throw new PythonPycResourceFailure( - "string", - { max_file_bytes: this.input.max_file_bytes }, - `Reported code-object strings exceed max_file_bytes ${this.input.max_file_bytes}.`, - ); + let bytes = 0; + for (const value of values) bytes += Buffer.byteLength(value, "utf8"); + this.#reportedStringBytes.charge( + bytes, + () => + new PythonPycResourceFailure( + "string", + { max_file_bytes: this.input.max_file_bytes }, + `Reported code-object strings exceed max_file_bytes ${this.input.max_file_bytes}.`, + ), + ); } /** @@ -717,13 +727,15 @@ class PycMarshalReader extends MarshalReader { count: number, what: "constants" | "names" | "nested code relationships", ): void { - this.#reportedConsts += count; - if (this.#reportedConsts > this.input.max_objects) - throw new PythonPycResourceFailure( - "objects", - { max_objects: this.input.max_objects }, - `Reported ${what} exceed max_objects ${this.input.max_objects}.`, - ); + this.#reportedCopies.charge( + count, + () => + new PythonPycResourceFailure( + "objects", + { max_objects: this.input.max_objects }, + `Reported ${what} exceed max_objects ${this.input.max_objects}.`, + ), + ); } /** @@ -1135,14 +1147,15 @@ class PycMarshalReader extends MarshalReader { * not `max_depth`, because a reference chain is shallow in the stream. */ #takeKeyStep(): void { - this.#keySteps += 1; - const limit = this.input.max_objects * 8; - if (this.#keySteps > limit) - throw new PythonPycResourceFailure( - "objects", - { max_objects: this.input.max_objects }, - `Marshal key walk exceeds ${limit} steps (max_objects * 8).`, - ); + this.#keySteps.charge( + 1, + () => + new PythonPycResourceFailure( + "objects", + { max_objects: this.input.max_objects }, + `Marshal key walk exceeds ${this.#keySteps.maximum} steps (max_objects * 8).`, + ), + ); } #leafKey(value: MarshalValue, mode: KeyMode): ResolvedKey | null { @@ -1451,13 +1464,15 @@ class PycMarshalReader extends MarshalReader { #reserveKey(size: number): void { if (size > this.input.max_string_bytes) this.#keyTooLarge(size); - this.#keyBytes += size; - if (this.#keyBytes > this.input.max_file_bytes) - throw new PythonPycResourceFailure( - "string", - { max_file_bytes: this.input.max_file_bytes }, - `Marshal key expansion exceeds max_file_bytes ${this.input.max_file_bytes}.`, - ); + this.#keyBytes.charge( + size, + () => + new PythonPycResourceFailure( + "string", + { max_file_bytes: this.input.max_file_bytes }, + `Marshal key expansion exceeds max_file_bytes ${this.input.max_file_bytes}.`, + ), + ); } #keyTooLarge(size: number): never { diff --git a/tests/evaluation/budgetArithmeticGuard.test.ts b/tests/evaluation/budgetArithmeticGuard.test.ts index 914a19aab4..254d6b9a39 100644 --- a/tests/evaluation/budgetArithmeticGuard.test.ts +++ b/tests/evaluation/budgetArithmeticGuard.test.ts @@ -24,7 +24,7 @@ it("counts hand-checked fields inside a hand-written Budget class", () => { expect(budgetArithmeticCount(cuda)).toBe(2); }); -it("ignores unchecked tallies and fields outside Budget classes", () => { +it("ignores unchecked tallies and fields not compared with a limit", () => { const tallies = [ "class ProtobufBudget {", " async checkpoint(): Promise {", @@ -35,10 +35,69 @@ it("ignores unchecked tallies and fields outside Budget classes", () => { "}", "class Reader {", " read(size: number): void {", - " if (this.#bytes + size > LIMIT) throw failure();", - " this.#bytes += size;", + " this.#observed += size;", + " if (this.#observed > this.#previous) this.#grew = true;", " }", "}", ].join("\n"); expect(budgetArithmeticCount(tallies)).toBe(0); }); + +it("counts a field outside Budget classes compared with a limit once", () => { + const reader = [ + "class Reader {", + " read(size: number): void {", + " if (this.#bytes + size > LIMIT) throw failure();", + " this.#bytes += size;", + " }", + " key(size: number): void {", + " this.#keyBytes += size;", + " if (this.#keyBytes > this.input.max_file_bytes) throw failure();", + " this.#keyBytes += 1;", + " }", + "}", + ].join("\n"); + expect(budgetArithmeticCount(reader)).toBe(2); +}); + +it("counts the same field name separately in each class", () => { + const readers = [ + "class First {", + " read(size: number): void {", + " if (this.#bytes + size > LIMIT) throw failure();", + " this.#bytes += size;", + " }", + "}", + "class SecondBudget {", + " #bytes = 0;", + " observe(size: number): void {", + " this.#bytes += size;", + " }", + "}", + "class Third {", + " read(size: number): void {", + " this.#bytes += size;", + " if (this.#bytes > LIMIT) throw failure();", + " }", + "}", + ].join("\n"); + expect(budgetArithmeticCount(readers)).toBe(2); +}); + +it.each([ + "CAPACITY", + "FACT_ALLOWANCE", + "FACT_BUDGET", + "MAX_BYTES", + "byteLimit", +])("counts a field compared with %s", (limit) => { + const reader = [ + "class Reader {", + " read(size: number): void {", + " this.#facts += size;", + ` if (this.#facts > ${limit}) throw failure();`, + " }", + "}", + ].join("\n"); + expect(budgetArithmeticCount(reader)).toBe(1); +});