diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.md index 7ec2febebb..1dd3c99d58 100644 --- a/docs/javascript-artifact-reconstruction.md +++ b/docs/javascript-artifact-reconstruction.md @@ -876,8 +876,22 @@ and Evidence. If that expansion does not fit, the file node retains `static_analysis` and any admitted `semantic_module_analysis` observations, including their exact values and source locations. Use the public modules item view to read them. The graph marks those unexpanded relationships as unknown. -If cumulative result capacity is exhausted, analysis stops; inventoried but -unexamined files remain present with explicit unknown scopes. + +The parent charges everything it retains against the heap that was free when +analysis started, less a transfer reserve. Semantic facts and expanded +relationships stop at half of that capacity, and a source refused there keeps +its static facts with the refusal reported. Static facts can continue to three +quarters of it while the live heap also has room for another copy of what it +holds, so results charged for earlier sources do not cost later sources their +static facts when the heap can still hold them. When that share runs short, +relationships that were reserved but not yet expanded are given up before any +static facts, and each affected file reports the omission. + +If static facts themselves exhaust that share, or the parent's live heap cannot +hold the next checkpoint, analysis stops; inventoried but unexamined files remain +present with explicit unknown scopes. The failure reports the limit that applied +in bytes, what was still available, the live heap in use and what the checkpoint +needed. ### Large local JSON inputs diff --git a/src/javascript/analysis/JavaScriptAnalysisWorker.ts b/src/javascript/analysis/JavaScriptAnalysisWorker.ts index 056a23332b..f6a5e92553 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorker.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorker.ts @@ -83,6 +83,12 @@ const OPERATION = "analyze_javascript_application"; const PROVIDER = "javascript-static-analysis"; const MIB = 1024 * 1024; const STATIC_CHECKPOINT_RETENTION_EXPANSION = 6; +/** + * Share of the parent's free heap that static checkpoints may fill once the + * result budget is used. Unlike the live heap, the charges also cover application + * relationships that are reserved but not yet materialized. + */ +const STATIC_CAPACITY_SHARE = 0.75; /** The worker runs on this executable; Bun accepts but ignores V8 heap flags. */ const WORKER_HEAP_FLAGS_ENFORCED = javaScriptRuntime(process.versions).id === "node"; @@ -130,10 +136,11 @@ export class JavaScriptAnalysisWorker { #revokedStaticProjections: JavaScriptRevokedStaticProjection[] = []; #nextSourceIndex = 0; #applicationProjectionFailure: AnalysisResourceConstraintError | undefined; - #parentCapacityExhausted = false; + #parentCapacityExhaustion: Readonly> | undefined; #sourcesRemaining: number | undefined; #deadline: ProviderStartupDeadline | undefined; #resultBudgetBytes = 0; + #staticCapacityBytes = 0; /** Null when the runtime enforces the V8 heap flags instead. */ readonly #processMemoryLimitBytes: number | null; readonly #onStdout = (chunk: Buffer | string): void => { @@ -248,7 +255,7 @@ export class JavaScriptAnalysisWorker { this.#staticProjectionReservations = []; this.#revokedStaticProjections = []; this.#nextSourceIndex = 0; - this.#parentCapacityExhausted = false; + this.#parentCapacityExhaustion = undefined; this.#applicationProjectionFailure = undefined; this.#refreshResultBudget(); return true; @@ -262,17 +269,20 @@ export class JavaScriptAnalysisWorker { /** * Reserve host-side control, schema/Evidence work, and a maximum record's - * transient copies. Admission also accounts for retained earlier files. + * transient copies. Everything retained is charged against what is left. + * + * Semantic and application results stop at half of it. Static checkpoints + * may go on to a larger share while the live heap has room for them, so a + * charge for earlier results cannot cost a later source its static facts. */ #refreshResultBudget(): void { - this.#resultBudgetBytes = Math.max( + const capacity = Math.max( 0, - Math.floor( - (this.#remainingParentHeapBytes() - - 24 * JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES) / - 2, - ), + this.#remainingParentHeapBytes() - + 24 * JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES, ); + this.#resultBudgetBytes = Math.floor(capacity / 2); + this.#staticCapacityBytes = Math.floor(capacity * STATIC_CAPACITY_SHARE); } /** Analyze inert source; hard worker failure does not terminate this process. */ @@ -290,7 +300,7 @@ export class JavaScriptAnalysisWorker { const id = randomUUID(); const sourceIndex = this.#nextSourceIndex++; this.#applicationProjectionFailure = undefined; - this.#parentCapacityExhausted = false; + this.#parentCapacityExhaustion = undefined; this.#revokedStaticProjections = []; if (this.#sourcesRemaining !== undefined) this.#sourcesRemaining = Math.max(0, this.#sourcesRemaining - 1); @@ -388,20 +398,8 @@ export class JavaScriptAnalysisWorker { sourceIndex, response, ); - if (this.#parentCapacityExhausted) - throw new AnalysisResourceConstraintError( - OPERATION, - "memory", - `The parent's result capacity was exhausted after retaining static facts for ${file.path}; remaining sources were not analyzed`, - { - ...this.#limits(), - parent_result_budget_bytes: this.#resultBudgetBytes, - parent_result_budget_exhausted: true, - }, - { - remediationAction: JAVASCRIPT_PARENT_RESULT_BUDGET_REMEDIATION, - }, - ); + const exhausted = this.#parentCapacityFailure(file.path); + if (exhausted !== undefined) throw exhausted; offeredResultBudget = this.#applicationProjectionFailure !== undefined && this.#sourcesRemaining !== undefined && @@ -799,11 +797,21 @@ export class JavaScriptAnalysisWorker { // Application relationships are materialized only after all source files // have been examined. Reclaim their pending reservations to keep room for // later static observations, without relaxing the actual retained bound. - const needed = - retainedBytes - (this.#resultBudgetBytes - this.#estimatedRetainedBytes); + const { heap_size_limit, used_heap_size } = getHeapStatistics(); const liveAvailable = - this.#remainingParentHeapBytes() - + heap_size_limit - + used_heap_size - 24 * JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES; + // Past the result budget, the live heap must also keep room for another + // copy of what it holds, which final graph and Evidence work needs. The + // limit read here counts young-generation capacity that retained objects + // cannot use, so being merely below it does not protect the parent. + const staticCeiling = + retainedBytes + used_heap_size <= liveAvailable + ? this.#staticCapacityBytes + : this.#resultBudgetBytes; + const needed = + retainedBytes - (staticCeiling - this.#estimatedRetainedBytes); if (needed > 0 && retainedBytes <= liveAvailable) { // Plan all revocations first so an insufficient set leaves the earlier // relationships intact rather than sacrificing them for a failed read. @@ -836,12 +844,17 @@ export class JavaScriptAnalysisWorker { // admission so a second read cannot reject after revocations commit. const remaining = Math.max( 0, - Math.min( - this.#resultBudgetBytes - this.#estimatedRetainedBytes, - liveAvailable, - ), + Math.min(staticCeiling - this.#estimatedRetainedBytes, liveAvailable), ); - this.#parentCapacityExhausted = retainedBytes > remaining; + this.#parentCapacityExhaustion = + retainedBytes > remaining + ? { + parent_static_capacity_bytes: staticCeiling, + parent_available_static_capacity_bytes: remaining, + parent_used_heap_bytes: used_heap_size, + static_retention_estimate_bytes: retainedBytes, + } + : undefined; this.#estimatedRetainedBytes += retainedBytes; const expandedBytes = estimateJavaScriptStaticApplicationProjection( facts, @@ -873,6 +886,26 @@ export class JavaScriptAnalysisWorker { return facts; } + #parentCapacityFailure( + path: string, + ): AnalysisResourceConstraintError | undefined { + if (this.#parentCapacityExhaustion === undefined) return undefined; + return new AnalysisResourceConstraintError( + OPERATION, + "memory", + `The parent's result capacity was exhausted after retaining static facts for ${path}; remaining sources were not analyzed`, + { + ...this.#limits(), + parent_result_budget_bytes: this.#resultBudgetBytes, + ...this.#parentCapacityExhaustion, + parent_result_budget_exhausted: true, + }, + { + remediationAction: JAVASCRIPT_PARENT_RESULT_BUDGET_REMEDIATION, + }, + ); + } + #limits() { return { ...(WORKER_HEAP_FLAGS_ENFORCED diff --git a/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts b/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts index 5e612caf9e..51935ae178 100644 --- a/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts +++ b/tests/acceptance/applications/javascriptCheckpointReserveCli.test.ts @@ -170,3 +170,74 @@ cliTest.skipIf(process.versions.bun !== undefined).for([ expect(response.stderr).not.toMatch(/FATAL ERROR:.*heap/iu); }, ); + +cliTest.skipIf(process.versions.bun !== undefined)( + "keeps every source's static facts after semantic results use the parent result budget", + { timeout: 90_000 }, + async ({ cli }) => { + const root = await createTestTempDirectory("rea-js-static-capacity-"); + // Plain functions produce semantic graphs large enough to use the result + // budget of this heap. The later sources' storage findings then produce + // static checkpoints that exceed its reserve for the next checkpoint. + const vendors = Array.from({ length: 12 }, (_, vendor) => [ + `a-vendor-${String(vendor)}.mjs`, + Array.from( + { length: 100 }, + (_, index) => + `export function v${String(vendor)}_${String(index)}(input) { const value = input + ${String(index)}; const pair = { value }; return pair.value; }`, + ).join("\n"), + ]); + const callsPerSource = 12_000; + const sources = Array.from({ length: 5 }, (_, source) => [ + `z-source-${String(source)}.mjs`, + Array.from( + { length: callsPerSource }, + (_, index) => + `localStorage.getItem("k${String(source)}_${String(index)}");`, + ).join("\n"), + ]); + for (const [name, text] of [...vendors, ...sources]) + await writeFile(join(root, name!), text!); + const response = await cli.run({ + arguments: ["analyze", root, "--format", "json"], + environment: { + NODE_OPTIONS: "--max-old-space-size=640 --max-semi-space-size=8", + }, + timeoutMs: 75_000, + }); + expect(response.stderr).not.toMatch(/FATAL ERROR:.*heap/iu); + expect( + response.exitCode, + JSON.stringify(response.json)?.slice(0, 2000), + ).toBe(0); + const result = javascriptApplicationAnalysisResultSchema.parse( + parseEvidence(response.json).normalized_result, + ); + expect(result.statistics).toMatchObject({ + parsed_javascript_files: vendors.length + sources.length, + skipped_javascript_files: 0, + parse_failures: 0, + }); + // The omission is reported where a reader finds the retained facts. + for (const [name, text] of sources) { + const digest = createHash("sha256").update(text!).digest("hex"); + const node = result.graph.nodes.find( + ({ kind, identity }) => + kind === "javascript-asset" && + identity.strategy === "content-digest" && + identity.sha256 === digest, + ); + const observation = node?.observations.find( + ({ properties }) => + properties.application_projection_failure !== undefined, + ); + expect(observation?.label, name).toBe(name); + expect( + z + .object({ storage: z.array(z.unknown()) }) + .parse(observation?.properties.static_analysis).storage, + ).toHaveLength(callsPerSource); + } + expect(result.graph.coverage.status).toBe("partial"); + }, +); diff --git a/tests/boundary/javascript/javascriptStaticProjectionReclamation.test.ts b/tests/boundary/javascript/javascriptStaticProjectionReclamation.test.ts index ef1c99e458..11b78f9819 100644 --- a/tests/boundary/javascript/javascriptStaticProjectionReclamation.test.ts +++ b/tests/boundary/javascript/javascriptStaticProjectionReclamation.test.ts @@ -1,7 +1,9 @@ import { createHash } from "node:crypto"; import { expect, it, vi } from "vitest"; +import { z } from "zod"; const heap = vi.hoisted(() => ({ + usedBytes: 1024 * 1024, remainingBytes: 200 * 1024 * 1024, readings: [] as number[], })); @@ -13,9 +15,9 @@ vi.mock("node:v8", async (load) => { const current = actual.getHeapStatistics(); return { ...current, + used_heap_size: heap.usedBytes, heap_size_limit: - current.used_heap_size + - (heap.readings.shift() ?? heap.remainingBytes), + heap.usedBytes + (heap.readings.shift() ?? heap.remainingBytes), }; }, }; @@ -37,32 +39,56 @@ vi.mock( import { JavaScriptAnalysisWorker } from "../../../src/javascript/analysis/JavaScriptAnalysisWorker.js"; import { AnalysisResourceConstraintError } from "../../../src/domain/analysisErrorCore.js"; import type { JavaScriptArtifactFile } from "../../../src/domain/javascript/javascriptArtifactFiles.js"; -const file = (path: string, count: number): JavaScriptArtifactFile => { +const sourceText = (text: string) => ({ + sha256: createHash("sha256").update(text).digest("hex"), + bytes: Buffer.byteLength(text), + text: { included: true as const, value: text }, +}); +const file = (path: string, count: number): JavaScriptArtifactFile => ({ + path, + container_sha256: "a".repeat(64), + inventory_artifact_id: path, + kind: "javascript", + unpacked: false, + ...sourceText( + Array.from( + { length: count }, + (_, i) => `localStorage.setItem('k${i}', 'v');`, + ).join("\n"), + ), +}); +const expectCapacityRemains = ( + result: Awaited>, +): void => { + if ( + !result.ok && + result.error.error instanceof AnalysisResourceConstraintError + ) + expect( + result.error.error.reportedLimits?.parent_result_budget_exhausted, + ).not.toBe(true); +}; +const functions = (count: number): JavaScriptArtifactFile => { const text = Array.from( { length: count }, - (_, i) => `localStorage.setItem('k${i}', 'v');`, + (_, i) => `export function f${i}(a) { const b = a + ${i}; return b; }`, ).join("\n"); - return { - path, - container_sha256: "a".repeat(64), - sha256: createHash("sha256").update(text).digest("hex"), - bytes: Buffer.byteLength(text), - inventory_artifact_id: path, - kind: "javascript", - unpacked: false, - text: { included: true, value: text }, - }; + return { ...file("functions.js", 0), ...sourceText(text) }; }; +// The mocked heap leaves a 4 MiB result budget and 6 MiB of static capacity. +// A 50-line source reserves about 1.1 MiB of relationships; each later 500-line +// source is charged about 0.5 MiB, so reclamation starts at the eleventh. +const SOURCES_UNTIL_RECLAMATION = 12; it("uses one live heap observation for revocation and static admission", async () => { const run = async (changeHeapAt?: number) => { const worker = new JavaScriptAnalysisWorker( { heapMb: 512, timeoutMs: 30_000 }, undefined, - 8, + SOURCES_UNTIL_RECLAMATION, ); const results = []; try { - for (let i = 0; i < 8; i++) { + for (let i = 0; i < SOURCES_UNTIL_RECLAMATION; i++) { const index = i; results.push( await worker.analyze( @@ -108,15 +134,15 @@ it("uses one live heap observation for revocation and static admission", async ( ).not.toBe(true); }); -it("reclaims prior unmaterialized projections only when a complete static checkpoint would otherwise exhaust the result budget", async () => { +it("reclaims prior unmaterialized projections only when a complete static checkpoint would otherwise exhaust the static capacity", async () => { const worker = new JavaScriptAnalysisWorker( { heapMb: 512, timeoutMs: 30_000 }, undefined, - 8, + SOURCES_UNTIL_RECLAMATION, ); const results = []; try { - for (let i = 0; i < 8; i++) + for (let i = 0; i < SOURCES_UNTIL_RECLAMATION; i++) results.push( await worker.analyze(file(`${i}.js`, i === 0 ? 50 : 500), { nodes: 1, @@ -149,13 +175,13 @@ it("keeps earlier projections when reclaiming them cannot admit the checkpoint", const worker = new JavaScriptAnalysisWorker( { heapMb: 512, timeoutMs: 30_000 }, undefined, - 5, + 6, ); try { const results = []; - for (let i = 0; i < 5; i++) + for (let i = 0; i < 6; i++) results.push( - await worker.analyze(file(`${i}.js`, i === 0 ? 10 : 1000), { + await worker.analyze(file(`${i}.js`, i === 0 ? 10 : 1400), { nodes: 1, unknowns: 1, }), @@ -173,9 +199,136 @@ it("keeps earlier projections when reclaiming them cannot admit the checkpoint", if (last === undefined || last.ok) throw new Error("Expected capacity refusal"); expect(last.error.error).toMatchObject({ - reportedLimits: { parent_result_budget_exhausted: true }, + reportedLimits: { + parent_result_budget_bytes: 4 * 1024 * 1024, + parent_static_capacity_bytes: 6 * 1024 * 1024, + parent_result_budget_exhausted: true, + }, + }); + const { + static_retention_estimate_bytes: required, + parent_available_static_capacity_bytes: available, + } = z + .object({ + static_retention_estimate_bytes: z.number(), + parent_available_static_capacity_bytes: z.number(), + }) + .parse( + last.error.error instanceof AnalysisResourceConstraintError + ? last.error.error.reportedLimits + : null, + ); + expect(required).toBeGreaterThan(available); + } finally { + await worker.close(); + } +}); + +it("retains static facts past the result budget, keeping earlier relationships while the static capacity lasts", async () => { + const sources = 9; + const worker = new JavaScriptAnalysisWorker( + { heapMb: 512, timeoutMs: 30_000 }, + undefined, + sources, + ); + try { + const details = []; + for (let i = 0; i < sources; i++) { + const result = await worker.analyze(file(`${i}.js`, i === 0 ? 50 : 500), { + nodes: 1, + unknowns: 1, + }); + expectCapacityRemains(result); + details.push(result.ok ? result.value : result.error); + } + expect(details.map((detail) => detail.javascript?.storage.length)).toEqual( + details.map((_, i) => (i === 0 ? 50 : 500)), + ); + expect( + details.flatMap((detail) => detail.revokedStaticProjections ?? []), + ).toEqual([]); + // Relationships that do not fit the budget are refused, explicitly. + expect(details[0]?.applicationProjectionFailure).toBeUndefined(); + for (const detail of details.slice(1)) + expect(detail.applicationProjectionFailure).toMatchObject({ + reason: expect.stringContaining("exceed the parent's result budget"), + }); + } finally { + await worker.close(); + } +}); + +it("retains static facts for later sources after semantic results use the result budget", async () => { + const sources = 4; + const worker = new JavaScriptAnalysisWorker( + { heapMb: 512, timeoutMs: 30_000 }, + undefined, + sources, + ); + try { + // Semantic results are already adopted, so nothing can be reclaimed from them. + const first = await worker.analyze(functions(10), { + nodes: 20_000, + unknowns: 20_000, + }); + if (!first.ok) throw first.error.error; + expect(first.value.module?.callables).toHaveLength(10); + expect(first.value.projection).not.toBeNull(); + const later = []; + for (let i = 1; i < sources; i++) { + const result = await worker.analyze(file(`${i}.js`, 1000), { + nodes: 20_000, + unknowns: 20_000, + }); + expectCapacityRemains(result); + later.push(result.ok ? result.value : result.error); + } + expect(later.map((detail) => detail.javascript?.storage.length)).toEqual( + later.map(() => 1000), + ); + expect( + later.flatMap((detail) => detail.revokedStaticProjections ?? []), + ).toEqual([]); + } finally { + await worker.close(); + } +}); + +it("stays within the result budget when the live heap has no room for another copy", async () => { + // Each later checkpoint is charged about 1 MiB, against 8 MiB of live room. + heap.usedBytes = 7.5 * 1024 * 1024; + const sources = 9; + const worker = new JavaScriptAnalysisWorker( + { heapMb: 512, timeoutMs: 30_000 }, + undefined, + sources, + ); + try { + const results = []; + for (let i = 0; i < sources; i++) + results.push( + await worker.analyze(file(`${i}.js`, i === 0 ? 10 : 1000), { + nodes: 1, + unknowns: 1, + }), + ); + const stopped = results + .flatMap((result) => (result.ok ? [] : [result.error.error])) + .find( + (error) => + error instanceof AnalysisResourceConstraintError && + error.reportedLimits?.parent_result_budget_exhausted === true, + ); + expect(stopped).toMatchObject({ + reportedLimits: { + parent_result_budget_bytes: 4 * 1024 * 1024, + parent_static_capacity_bytes: 4 * 1024 * 1024, + parent_used_heap_bytes: 7.5 * 1024 * 1024, + parent_result_budget_exhausted: true, + }, }); } finally { + heap.usedBytes = 1024 * 1024; await worker.close(); } }); @@ -192,9 +345,11 @@ it("resets projection reservations and source indices when an idle worker is reu unknowns: 1, }); expect(first.ok).toBe(true); - expect(worker.prepareForAnotherAnalysis(8)).toBe(true); + expect(worker.prepareForAnotherAnalysis(SOURCES_UNTIL_RECLAMATION)).toBe( + true, + ); const results = []; - for (let i = 0; i < 7; i++) + for (let i = 0; i < SOURCES_UNTIL_RECLAMATION; i++) results.push( await worker.analyze(file(`new-${i}.js`, i === 0 ? 50 : 500), { nodes: 1,