From 4a01211f1142b4d20d155e9934e9fc5232c21e37 Mon Sep 17 00:00:00 2001 From: Kaoru0822 Date: Mon, 2 Nov 2026 05:16:16 -0800 Subject: [PATCH] fix(javascript): keep static facts after semantic results use the parent budget (#8648) Static checkpoints were admitted against the same budget as semantic and application results, half of the parent's free heap. Once those charges used it, the next checkpoint stopped the analysis and every remaining source lost its static facts, with most of the parent heap still free. Semantic and application results still stop at that budget. Static checkpoints may now continue to three quarters of the free heap while the live heap also has room for another copy of what it holds. Without that room, admission falls back to the result budget. The live condition is needed because the limit V8 reports counts young-generation capacity that retained objects cannot use: without it, the parent aborted on heap exhaustion at small heap limits. Reserved relationships are given up only when the larger share runs short. A stop still fails with a typed error, which now reports the limit that applied, what was available, the live heap in use and what the checkpoint needed. Co-authored-by: Claude Opus 5.5 --- docs/javascript-artifact-reconstruction.md | 18 +- .../analysis/JavaScriptAnalysisWorker.ts | 97 ++++++--- .../javascriptCheckpointReserveCli.test.ts | 71 ++++++ ...ascriptStaticProjectionReclamation.test.ts | 205 +++++++++++++++--- 4 files changed, 332 insertions(+), 59 deletions(-) diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.md index 7ec2febebb..1dd3c99d58 100644 --- a/docs/javascript-artifact-reconstruction.md +++ b/docs/javascript-artifact-reconstruction.md @@ -876,8 +876,22 @@ and Evidence. If that expansion does not fit, the file node retains `static_analysis` and any admitted `semantic_module_analysis` observations, including their exact values and source locations. Use the public modules item view to read them. The graph marks those unexpanded relationships as unknown. -If cumulative result capacity is exhausted, analysis stops; inventoried but -unexamined files remain present with explicit unknown scopes. + +The parent charges everything it retains against the heap that was free when +analysis started, less a transfer reserve. Semantic facts and expanded +relationships stop at half of that capacity, and a source refused there keeps +its static facts with the refusal reported. Static facts can continue to three +quarters of it while the live heap also has room for another copy of what it +holds, so results charged for earlier sources do not cost later sources their +static facts when the heap can still hold them. When that share runs short, +relationships that were reserved but not yet expanded are given up before any +static facts, and each affected file reports the omission. + +If static facts themselves exhaust that share, or the parent's live heap cannot +hold the next checkpoint, analysis stops; inventoried but unexamined files remain +present with explicit unknown scopes. The failure reports the limit that applied +in bytes, what was still available, the live heap in use and what the checkpoint +needed. ### Large local JSON inputs diff --git a/src/javascript/analysis/JavaScriptAnalysisWorker.ts b/src/javascript/analysis/JavaScriptAnalysisWorker.ts index 056a23332b..f6a5e92553 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorker.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorker.ts @@ -83,6 +83,12 @@ const OPERATION = "analyze_javascript_application"; const PROVIDER = "javascript-static-analysis"; const MIB = 1024 * 1024; const STATIC_CHECKPOINT_RETENTION_EXPANSION = 6; +/** + * Share of the parent's free heap that static checkpoints may fill once the + * result budget is used. Unlike the live heap, the charges also cover application + * relationships that are reserved but not yet materialized. + */ +const STATIC_CAPACITY_SHARE = 0.75; /** The worker runs on this executable; Bun accepts but ignores V8 heap flags. */ const WORKER_HEAP_FLAGS_ENFORCED = javaScriptRuntime(process.versions).id === "node"; @@ -130,10 +136,11 @@ export class JavaScriptAnalysisWorker { #revokedStaticProjections: JavaScriptRevokedStaticProjection[] = []; #nextSourceIndex = 0; #applicationProjectionFailure: AnalysisResourceConstraintError | undefined; - #parentCapacityExhausted = false; + #parentCapacityExhaustion: Readonly> | undefined; #sourcesRemaining: number | undefined; #deadline: ProviderStartupDeadline | undefined; #resultBudgetBytes = 0; + #staticCapacityBytes = 0; /** Null when the runtime enforces the V8 heap flags instead. */ readonly #processMemoryLimitBytes: number | null; readonly #onStdout = (chunk: Buffer | string): void => { @@ -248,7 +255,7 @@ export class JavaScriptAnalysisWorker { this.#staticProjectionReservations = []; this.#revokedStaticProjections = []; this.#nextSourceIndex = 0; - this.#parentCapacityExhausted = false; + this.#parentCapacityExhaustion = undefined; this.#applicationProjectionFailure = undefined; this.#refreshResultBudget(); return true; @@ -262,17 +269,20 @@ export class JavaScriptAnalysisWorker { /** * Reserve host-side control, schema/Evidence work, and a maximum record's - * transient copies. Admission also accounts for retained earlier files. + * transient copies. Everything retained is charged against what is left. + * + * Semantic and application results stop at half of it. Static checkpoints + * may go on to a larger share while the live heap has room for them, so a + * charge for earlier results cannot cost a later source its static facts. */ #refreshResultBudget(): void { - this.#resultBudgetBytes = Math.max( + const capacity = Math.max( 0, - Math.floor( - (this.#remainingParentHeapBytes() - - 24 * JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES) / - 2, - ), + this.#remainingParentHeapBytes() - + 24 * JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES, ); + this.#resultBudgetBytes = Math.floor(capacity / 2); + this.#staticCapacityBytes = Math.floor(capacity * STATIC_CAPACITY_SHARE); } /** Analyze inert source; hard worker failure does not terminate this process. */ @@ -290,7 +300,7 @@ export class JavaScriptAnalysisWorker { const id = randomUUID(); const sourceIndex = this.#nextSourceIndex++; this.#applicationProjectionFailure = undefined; - this.#parentCapacityExhausted = false; + this.#parentCapacityExhaustion = undefined; this.#revokedStaticProjections = []; if (this.#sourcesRemaining !== undefined) this.#sourcesRemaining = Math.max(0, this.#sourcesRemaining - 1); @@ -388,20 +398,8 @@ export class JavaScriptAnalysisWorker { sourceIndex, response, ); - if (this.#parentCapacityExhausted) - throw new AnalysisResourceConstraintError( - OPERATION, - "memory", - `The parent's result capacity was exhausted after retaining static facts for ${file.path}; remaining sources were not analyzed`, - { - ...this.#limits(), - parent_result_budget_bytes: this.#resultBudgetBytes, - parent_result_budget_exhausted: true, - }, - { - remediationAction: JAVASCRIPT_PARENT_RESULT_BUDGET_REMEDIATION, - }, - ); + const exhausted = this.#parentCapacityFailure(file.path); + if (exhausted !== undefined) throw exhausted; offeredResultBudget = this.#applicationProjectionFailure !== undefined && this.#sourcesRemaining !== undefined && @@ -799,11 +797,21 @@ export class JavaScriptAnalysisWorker { // Application relationships are materialized only after all source files // have been examined. Reclaim their pending reservations to keep room for // later static observations, without relaxing the actual retained bound. - const needed = - retainedBytes - (this.#resultBudgetBytes - this.#estimatedRetainedBytes); + const { heap_size_limit, used_heap_size } = getHeapStatistics(); const liveAvailable = - this.#remainingParentHeapBytes() - + heap_size_limit - + used_heap_size - 24 * JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES; + // Past the result budget, the live heap must also keep room for another + // copy of what it holds, which final graph and Evidence work needs. The + // limit read here counts young-generation capacity that retained objects + // cannot use, so being merely below it does not protect the parent. + const staticCeiling = + retainedBytes + used_heap_size <= liveAvailable + ? this.#staticCapacityBytes + : this.#resultBudgetBytes; + const needed = + retainedBytes - (staticCeiling - this.#estimatedRetainedBytes); if (needed > 0 && retainedBytes <= liveAvailable) { // Plan all revocations first so an insufficient set leaves the earlier // relationships intact rather than sacrificing them for a failed read. @@ -836,12 +844,17 @@ export class JavaScriptAnalysisWorker { // admission so a second read cannot reject after revocations commit. const remaining = Math.max( 0, - Math.min( - this.#resultBudgetBytes - this.#estimatedRetainedBytes, - liveAvailable, - ), + Math.min(staticCeiling - this.#estimatedRetainedBytes, liveAvailable), ); - this.#parentCapacityExhausted = retainedBytes > remaining; + this.#parentCapacityExhaustion = + retainedBytes > remaining + ? { + parent_static_capacity_bytes: staticCeiling, + parent_available_static_capacity_bytes: remaining, + parent_used_heap_bytes: used_heap_size, + static_retention_estimate_bytes: retainedBytes, + } + : undefined; this.#estimatedRetainedBytes += retainedBytes; const expandedBytes = estimateJavaScriptStaticApplicationProjection( facts, @@ -873,6 +886,26 @@ export class JavaScriptAnalysisWorker { return facts; } + #parentCapacityFailure( + path: string, + ): AnalysisResourceConstraintError | undefined { + if (this.#parentCapacityExhaustion === undefined) return undefined; + return new AnalysisResourceConstraintError( + OPERATION, + "memory", + `The parent's result capacity was exhausted after retaining static facts for ${path}; remaining sources were not analyzed`, + { + ...this.#limits(), + parent_result_budget_bytes: this.#resultBudgetBytes, + ...this.#parentCapacityExhaustion, + parent_result_budget_exhausted: true, + }, + { + remediationAction: JAVASCRIPT_PARENT_RESULT_BUDGET_REMEDIATION, + }, + ); + } + #limits() { return { ...(WORKER_HEAP_FLAGS_ENFORCED diff --git a/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts b/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts index 5e612caf9e..51935ae178 100644 --- a/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts +++ b/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts @@ -170,3 +170,74 @@ cliTest.skipIf(process.versions.bun !== undefined).for([ expect(response.stderr).not.toMatch(/FATAL ERROR:.*heap/iu); }, ); + +cliTest.skipIf(process.versions.bun !== undefined)( + "keeps every source's static facts after semantic results use the parent result budget", + { timeout: 90_000 }, + async ({ cli }) => { + const root = await createTestTempDirectory("rea-js-static-capacity-"); + // Plain functions produce semantic graphs large enough to use the result + // budget of this heap. The later sources' storage findings then produce + // static checkpoints that exceed its reserve for the next checkpoint. + const vendors = Array.from({ length: 12 }, (_, vendor) => [ + `a-vendor-${String(vendor)}.mjs`, + Array.from( + { length: 100 }, + (_, index) => + `export function v${String(vendor)}_${String(index)}(input) { const value = input + ${String(index)}; const pair = { value }; return pair.value; }`, + ).join("\n"), + ]); + const callsPerSource = 12_000; + const sources = Array.from({ length: 5 }, (_, source) => [ + `z-source-${String(source)}.mjs`, + Array.from( + { length: callsPerSource }, + (_, index) => + `localStorage.getItem("k${String(source)}_${String(index)}");`, + ).join("\n"), + ]); + for (const [name, text] of [...vendors, ...sources]) + await writeFile(join(root, name!), text!); + const response = await cli.run({ + arguments: ["analyze", root, "--format", "json"], + environment: { + NODE_OPTIONS: "--max-old-space-size=640 --max-semi-space-size=8", + }, + timeoutMs: 75_000, + }); + expect(response.stderr).not.toMatch(/FATAL ERROR:.*heap/iu); + expect( + response.exitCode, + JSON.stringify(response.json)?.slice(0, 2000), + ).toBe(0); + const result = javascriptApplicationAnalysisResultSchema.parse( + parseEvidence(response.json).normalized_result, + ); + expect(result.statistics).toMatchObject({ + parsed_javascript_files: vendors.length + sources.length, + skipped_javascript_files: 0, + parse_failures: 0, + }); + // The omission is reported where a reader finds the retained facts. + for (const [name, text] of sources) { + const digest = createHash("sha256").update(text!).digest("hex"); + const node = result.graph.nodes.find( + ({ kind, identity }) => + kind === "javascript-asset" && + identity.strategy === "content-digest" && + identity.sha256 === digest, + ); + const observation = node?.observations.find( + ({ properties }) => + properties.application_projection_failure !== undefined, + ); + expect(observation?.label, name).toBe(name); + expect( + z + .object({ storage: z.array(z.unknown()) }) + .parse(observation?.properties.static_analysis).storage, + ).toHaveLength(callsPerSource); + } + expect(result.graph.coverage.status).toBe("partial"); + }, +); diff --git a/tests/boundary/javascript/javascriptStaticProjectionReclamation.test.ts b/tests/boundary/javascript/javascriptStaticProjectionReclamation.test.ts index ef1c99e458..11b78f9819 100644 --- a/tests/boundary/javascript/javascriptStaticProjectionReclamation.test.ts +++ b/tests/boundary/javascript/javascriptStaticProjectionReclamation.test.ts @@ -1,7 +1,9 @@ import { createHash } from "node:crypto"; import { expect, it, vi } from "vitest"; +import { z } from "zod"; const heap = vi.hoisted(() => ({ + usedBytes: 1024 * 1024, remainingBytes: 200 * 1024 * 1024, readings: [] as number[], })); @@ -13,9 +15,9 @@ vi.mock("node:v8", async (load) => { const current = actual.getHeapStatistics(); return { ...current, + used_heap_size: heap.usedBytes, heap_size_limit: - current.used_heap_size + - (heap.readings.shift() ?? heap.remainingBytes), + heap.usedBytes + (heap.readings.shift() ?? heap.remainingBytes), }; }, }; @@ -37,32 +39,56 @@ vi.mock( import { JavaScriptAnalysisWorker } from "../../../src/javascript/analysis/JavaScriptAnalysisWorker.js"; import { AnalysisResourceConstraintError } from "../../../src/domain/analysisErrorCore.js"; import type { JavaScriptArtifactFile } from "../../../src/domain/javascript/javascriptArtifactFiles.js"; -const file = (path: string, count: number): JavaScriptArtifactFile => { +const sourceText = (text: string) => ({ + sha256: createHash("sha256").update(text).digest("hex"), + bytes: Buffer.byteLength(text), + text: { included: true as const, value: text }, +}); +const file = (path: string, count: number): JavaScriptArtifactFile => ({ + path, + container_sha256: "a".repeat(64), + inventory_artifact_id: path, + kind: "javascript", + unpacked: false, + ...sourceText( + Array.from( + { length: count }, + (_, i) => `localStorage.setItem('k${i}', 'v');`, + ).join("\n"), + ), +}); +const expectCapacityRemains = ( + result: Awaited>, +): void => { + if ( + !result.ok && + result.error.error instanceof AnalysisResourceConstraintError + ) + expect( + result.error.error.reportedLimits?.parent_result_budget_exhausted, + ).not.toBe(true); +}; +const functions = (count: number): JavaScriptArtifactFile => { const text = Array.from( { length: count }, - (_, i) => `localStorage.setItem('k${i}', 'v');`, + (_, i) => `export function f${i}(a) { const b = a + ${i}; return b; }`, ).join("\n"); - return { - path, - container_sha256: "a".repeat(64), - sha256: createHash("sha256").update(text).digest("hex"), - bytes: Buffer.byteLength(text), - inventory_artifact_id: path, - kind: "javascript", - unpacked: false, - text: { included: true, value: text }, - }; + return { ...file("functions.js", 0), ...sourceText(text) }; }; +// The mocked heap leaves a 4 MiB result budget and 6 MiB of static capacity. +// A 50-line source reserves about 1.1 MiB of relationships; each later 500-line +// source is charged about 0.5 MiB, so reclamation starts at the eleventh. +const SOURCES_UNTIL_RECLAMATION = 12; it("uses one live heap observation for revocation and static admission", async () => { const run = async (changeHeapAt?: number) => { const worker = new JavaScriptAnalysisWorker( { heapMb: 512, timeoutMs: 30_000 }, undefined, - 8, + SOURCES_UNTIL_RECLAMATION, ); const results = []; try { - for (let i = 0; i < 8; i++) { + for (let i = 0; i < SOURCES_UNTIL_RECLAMATION; i++) { const index = i; results.push( await worker.analyze( @@ -108,15 +134,15 @@ it("uses one live heap observation for revocation and static admission", async ( ).not.toBe(true); }); -it("reclaims prior unmaterialized projections only when a complete static checkpoint would otherwise exhaust the result budget", async () => { +it("reclaims prior unmaterialized projections only when a complete static checkpoint would otherwise exhaust the static capacity", async () => { const worker = new JavaScriptAnalysisWorker( { heapMb: 512, timeoutMs: 30_000 }, undefined, - 8, + SOURCES_UNTIL_RECLAMATION, ); const results = []; try { - for (let i = 0; i < 8; i++) + for (let i = 0; i < SOURCES_UNTIL_RECLAMATION; i++) results.push( await worker.analyze(file(`${i}.js`, i === 0 ? 50 : 500), { nodes: 1, @@ -149,13 +175,13 @@ it("keeps earlier projections when reclaiming them cannot admit the checkpoint", const worker = new JavaScriptAnalysisWorker( { heapMb: 512, timeoutMs: 30_000 }, undefined, - 5, + 6, ); try { const results = []; - for (let i = 0; i < 5; i++) + for (let i = 0; i < 6; i++) results.push( - await worker.analyze(file(`${i}.js`, i === 0 ? 10 : 1000), { + await worker.analyze(file(`${i}.js`, i === 0 ? 10 : 1400), { nodes: 1, unknowns: 1, }), @@ -173,9 +199,136 @@ it("keeps earlier projections when reclaiming them cannot admit the checkpoint", if (last === undefined || last.ok) throw new Error("Expected capacity refusal"); expect(last.error.error).toMatchObject({ - reportedLimits: { parent_result_budget_exhausted: true }, + reportedLimits: { + parent_result_budget_bytes: 4 * 1024 * 1024, + parent_static_capacity_bytes: 6 * 1024 * 1024, + parent_result_budget_exhausted: true, + }, + }); + const { + static_retention_estimate_bytes: required, + parent_available_static_capacity_bytes: available, + } = z + .object({ + static_retention_estimate_bytes: z.number(), + parent_available_static_capacity_bytes: z.number(), + }) + .parse( + last.error.error instanceof AnalysisResourceConstraintError + ? last.error.error.reportedLimits + : null, + ); + expect(required).toBeGreaterThan(available); + } finally { + await worker.close(); + } +}); + +it("retains static facts past the result budget, keeping earlier relationships while the static capacity lasts", async () => { + const sources = 9; + const worker = new JavaScriptAnalysisWorker( + { heapMb: 512, timeoutMs: 30_000 }, + undefined, + sources, + ); + try { + const details = []; + for (let i = 0; i < sources; i++) { + const result = await worker.analyze(file(`${i}.js`, i === 0 ? 50 : 500), { + nodes: 1, + unknowns: 1, + }); + expectCapacityRemains(result); + details.push(result.ok ? result.value : result.error); + } + expect(details.map((detail) => detail.javascript?.storage.length)).toEqual( + details.map((_, i) => (i === 0 ? 50 : 500)), + ); + expect( + details.flatMap((detail) => detail.revokedStaticProjections ?? []), + ).toEqual([]); + // Relationships that do not fit the budget are refused, explicitly. + expect(details[0]?.applicationProjectionFailure).toBeUndefined(); + for (const detail of details.slice(1)) + expect(detail.applicationProjectionFailure).toMatchObject({ + reason: expect.stringContaining("exceed the parent's result budget"), + }); + } finally { + await worker.close(); + } +}); + +it("retains static facts for later sources after semantic results use the result budget", async () => { + const sources = 4; + const worker = new JavaScriptAnalysisWorker( + { heapMb: 512, timeoutMs: 30_000 }, + undefined, + sources, + ); + try { + // Semantic results are already adopted, so nothing can be reclaimed from them. + const first = await worker.analyze(functions(10), { + nodes: 20_000, + unknowns: 20_000, + }); + if (!first.ok) throw first.error.error; + expect(first.value.module?.callables).toHaveLength(10); + expect(first.value.projection).not.toBeNull(); + const later = []; + for (let i = 1; i < sources; i++) { + const result = await worker.analyze(file(`${i}.js`, 1000), { + nodes: 20_000, + unknowns: 20_000, + }); + expectCapacityRemains(result); + later.push(result.ok ? result.value : result.error); + } + expect(later.map((detail) => detail.javascript?.storage.length)).toEqual( + later.map(() => 1000), + ); + expect( + later.flatMap((detail) => detail.revokedStaticProjections ?? []), + ).toEqual([]); + } finally { + await worker.close(); + } +}); + +it("stays within the result budget when the live heap has no room for another copy", async () => { + // Each later checkpoint is charged about 1 MiB, against 8 MiB of live room. + heap.usedBytes = 7.5 * 1024 * 1024; + const sources = 9; + const worker = new JavaScriptAnalysisWorker( + { heapMb: 512, timeoutMs: 30_000 }, + undefined, + sources, + ); + try { + const results = []; + for (let i = 0; i < sources; i++) + results.push( + await worker.analyze(file(`${i}.js`, i === 0 ? 10 : 1000), { + nodes: 1, + unknowns: 1, + }), + ); + const stopped = results + .flatMap((result) => (result.ok ? [] : [result.error.error])) + .find( + (error) => + error instanceof AnalysisResourceConstraintError && + error.reportedLimits?.parent_result_budget_exhausted === true, + ); + expect(stopped).toMatchObject({ + reportedLimits: { + parent_result_budget_bytes: 4 * 1024 * 1024, + parent_static_capacity_bytes: 4 * 1024 * 1024, + parent_used_heap_bytes: 7.5 * 1024 * 1024, + parent_result_budget_exhausted: true, + }, }); } finally { + heap.usedBytes = 1024 * 1024; await worker.close(); } }); @@ -192,9 +345,11 @@ it("resets projection reservations and source indices when an idle worker is reu unknowns: 1, }); expect(first.ok).toBe(true); - expect(worker.prepareForAnotherAnalysis(8)).toBe(true); + expect(worker.prepareForAnotherAnalysis(SOURCES_UNTIL_RECLAMATION)).toBe( + true, + ); const results = []; - for (let i = 0; i < 7; i++) + for (let i = 0; i < SOURCES_UNTIL_RECLAMATION; i++) results.push( await worker.analyze(file(`new-${i}.js`, i === 0 ? 50 : 500), { nodes: 1,