diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.md index 3381815cc5..2cd393c378 100644 --- a/docs/javascript-artifact-reconstruction.md +++ b/docs/javascript-artifact-reconstruction.md @@ -347,6 +347,23 @@ counterparts while resolving to inventoried source files. This substitution requires import or require semantics. Literal Electron preload, renderer, and utility-process runtime paths retain the runtime filename selected by the caller. +Application `.js` and `.ts` sources use the nearest readable, same-container +`package.json` type; explicit module/CommonJS suffixes keep their own meaning. +An unavailable nearer manifest leaves the package type unknown. When a module +package contains `.js` sources, a preliminary pass through the bounded worker +collects BrowserWindow preload declarations. Exact same-container `.js` +preloads use the script-loader context even when their only operation is a +helper `require`. Renderer API names alone never select that context. Dynamic, +unresolved, excluded-source, and cross-container declarations do not establish +a preload identity; this static association is not proof that Electron loaded +the file. Discovery performs another static pass, while semantic projection +runs once and existing heap, deadline, cancellation, and cleanup controls remain. +A source that exceeds its worker limits during discovery contributes no +declarations, and any other discovery failure ends discovery; source analysis +then records that failure and retains partial Evidence as usual. Either gap is +also listed in the Evidence limitations, because source analysis may still read +that source after a declared preload has kept its package scope. + Selected bounded text is then parsed as inert data to recover: - `package.json` metadata and declared main or renderer entrypoints; diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.packageType.test.ts b/src/application/javascript/JavaScriptArtifactAnalysis.packageType.test.ts new file mode 100644 index 0000000000..920769b4e6 --- /dev/null +++ b/src/application/javascript/JavaScriptArtifactAnalysis.packageType.test.ts @@ -0,0 +1,83 @@ +import { expect, it } from "vitest"; +import { analyzeAndAdoptJavaScriptArtifactFiles } from "./JavaScriptArtifactAnalysis.js"; +import { createJavaScriptSemanticGraphProjection } from "./JavaScriptSemanticGraphBuilder.js"; +import { analyzeJavaScriptStaticSource } from "../../domain/javascript/javascriptStaticAnalysis.js"; +import type { + JavaScriptArtifactFile, + JavaScriptUnanalyzedArtifactFile, +} from "../../domain/javascript/javascriptArtifactFiles.js"; +import type { JavaScriptPackageType } from "../../domain/javascript/javascriptSourceParser.js"; +import { ok } from "../../domain/result.js"; + +const container = "a".repeat(64); + +const included = ( + path: string, + kind: JavaScriptArtifactFile["kind"], + text: string, +): JavaScriptArtifactFile => ({ + path, + container_sha256: container, + sha256: "b".repeat(64), + bytes: Buffer.byteLength(text), + inventory_artifact_id: path, + kind, + unpacked: false, + text: { included: true, value: text }, +}); + +it("leaves the package type unknown behind a nearer unavailable manifest", async () => { + const files = [ + included("package.json", "package-json", '{"type":"module"}'), + included("main.js", "javascript", 'require("./dep.cjs");'), + included("nested/main.js", "javascript", 'require("../dep.cjs");'), + included( + "nested/inner/package.json", + "package-json", + '{"type":"commonjs"}', + ), + included("nested/inner/main.js", "javascript", 'require("../../dep.cjs");'), + ]; + const unavailable: JavaScriptUnanalyzedArtifactFile = { + path: "nested/package.json", + container_sha256: container, + sha256: null, + bytes: null, + inventory_artifact_id: null, + kind: "package-json", + limitations: ["Inventoried package bytes are unavailable."], + }; + const observed = new Map(); + const javascript = analyzeJavaScriptStaticSource(""); + const worker = { + analyze: async ( + file: JavaScriptArtifactFile, + _capacity: unknown, + _signal?: AbortSignal, + _onPhase?: unknown, + packageType?: JavaScriptPackageType, + ) => { + observed.set(file.path, packageType); + return ok({ javascript, module: null, projection: null }); + }, + }; + await analyzeAndAdoptJavaScriptArtifactFiles( + { + files, + unavailable_relevant_files: [unavailable], + containers: [], + text_bytes_read: 0, + invalid_utf8_files: 0, + }, + worker, + createJavaScriptSemanticGraphProjection(), + async () => {}, + ); + // The nearest package.json owns a source's scope even when its bytes are + // unavailable, so the root type must not reach nested/main.js. + expect(Object.fromEntries(observed)).toEqual({ + "main.js": "module", + "nested/main.js": "unknown", + "nested/inner/main.js": "commonjs", + }); +}); diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.ts index 52f7858c4e..9c863211aa 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysis.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts @@ -1,7 +1,10 @@ import { createHash } from "node:crypto"; +import { posix } from "node:path"; import { setImmediate } from "node:timers/promises"; +import type { JavaScriptPackageType } from "../../domain/javascript/javascriptSourceParser.js"; + import { resolveJavaScriptSourceMapReferences } from "../../domain/javascript/javascriptSourceMapPaths.js"; import { hasValidSourceMapContents } from "../../domain/sourceMapContents.js"; import { @@ -77,12 +80,19 @@ export const analyzeAndAdoptJavaScriptArtifactFiles = async ( ) => Promise, signal?: AbortSignal, onPhase?: (file: JavaScriptArtifactFile, phase: string) => Promise, + preloadPaths: ReadonlySet = new Set(), + /** Discovery gaps that may leave a declared preload in its package scope. */ + preloadLimitations: readonly string[] = [], ): Promise<{ readonly analysis: JavaScriptArtifactAnalysis; readonly interruption: AnalysisError | null; }> => { const state = emptyArtifactAnalysis(); const sourceIndices: number[] = []; + const packageTypeFor = createJavaScriptPackageTypeLookup( + fileSet, + state.jsonDocuments, + ); let interruption: AnalysisError | null = null; const selectedFileCount = fileSet.files.length - (fileSet.source_selection?.unselected_files ?? 0); @@ -118,6 +128,10 @@ export const analyzeAndAdoptJavaScriptArtifactFiles = async ( projection.remainingFileCapacity(), signal, onPhase === undefined ? undefined : (phase) => onPhase(file, phase), + preloadPaths.has(`${file.container_sha256}:${file.path}`) && + /\.js$/iu.test(file.path) + ? "commonjs" + : packageTypeFor(file), ); const facts = result.ok ? result.value : result.error; for (const revoked of facts.revokedStaticProjections ?? []) { @@ -294,11 +308,15 @@ export const analyzeAndAdoptJavaScriptArtifactFiles = async ( ); } } - return { analysis: finalizeArtifactAnalysis(state), interruption }; + return { + analysis: finalizeArtifactAnalysis(state, preloadLimitations), + interruption, + }; }; const finalizeArtifactAnalysis = ( state: MutableArtifactAnalysis, + preloadLimitations: readonly string[], ): JavaScriptArtifactAnalysis => { return { jsonDocuments: state.jsonDocuments, @@ -328,6 +346,7 @@ const finalizeArtifactAnalysis = ( (failure) => (failure === undefined ? [] : [failure.reason]), ), ), + ...preloadLimitations, ], }; }; @@ -386,6 +405,54 @@ const addStructuredObservations = ( if (json.status === "invalid") state.parseFailures += 1; }; +/** + * Node's package scope for a source is its nearest package.json; the lookup + * stops at a `node_modules` directory (Node's package scope resolution). Only + * a same-container, readable manifest object establishes the type; any other + * nearest manifest, including one whose bytes are unavailable, or none in the + * inventory, leaves the scope unknown. + */ +export const createJavaScriptPackageTypeLookup = ( + fileSet: JavaScriptArtifactFileSet, + jsonDocuments: JavaScriptJsonDocuments, +): ((file: JavaScriptArtifactFile) => JavaScriptPackageType) => { + const manifests = new Map(); + for (const file of fileSet.files) + if (posix.basename(file.path) === "package.json") + manifests.set( + `${file.container_sha256}:${posix.dirname(file.path)}`, + file, + ); + const unavailableManifests = new Set( + fileSet.unavailable_relevant_files + .filter(({ path }) => posix.basename(path) === "package.json") + .map( + ({ path, container_sha256 }) => + `${container_sha256}:${posix.dirname(path)}`, + ), + ); + return (file) => { + for ( + let directory = posix.dirname(file.path); + posix.basename(directory) !== "node_modules"; + directory = posix.dirname(directory) + ) { + const key = `${file.container_sha256}:${directory}`; + if (unavailableManifests.has(key)) return "unknown"; + const manifest = manifests.get(key); + if (manifest !== undefined) { + const document = jsonDocuments.read(manifest); + if (document.status !== "included" || document.packageFields === null) + return "unknown"; + const type = document.packageFields.type; + return type === "module" || type === "commonjs" ? type : "unspecified"; + } + if (directory === "." || directory === "" || directory === "/") break; + } + return "unknown"; + }; +}; + const parsePackage = ( file: JavaScriptArtifactFile, document: JavaScriptJsonDocumentFacts, diff --git a/src/application/javascript/JavaScriptArtifactReconstruction.ts b/src/application/javascript/JavaScriptArtifactReconstruction.ts index b035acfea6..11e993d88e 100644 --- a/src/application/javascript/JavaScriptArtifactReconstruction.ts +++ b/src/application/javascript/JavaScriptArtifactReconstruction.ts @@ -1,3 +1,4 @@ +import { discoverJavaScriptPreloadPaths } from "./JavaScriptPreloadDiscovery.js"; import { artifactNameForClassification } from "../../domain/artifactNameSelection.js"; import type { AnalysisSettingValues } from "../../domain/analysisSettings.js"; import { absoluteCallerPath } from "../../filesystem/callerPath.js"; @@ -180,17 +181,24 @@ export const reconstructJavaScriptArtifact = async ( ? {} : { fileNodes: input.max_semantic_nodes_per_file }), }); + const sourceLimits = { + heapMb: + input.max_heap_mb ?? + overrides?.["javascript.max_heap_mb"] ?? + JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS.heapMb, + timeoutMs: + input.analysis_timeout_ms ?? + overrides?.["javascript.analysis_timeout_ms"] ?? + JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS.timeoutMs, + }; + const preloads = await discoverJavaScriptPreloadPaths( + files, + sourceLimits, + scope, + signal, + ); const worker = await createJavaScriptSourceAnalysis( - { - heapMb: - input.max_heap_mb ?? - overrides?.["javascript.max_heap_mb"] ?? - JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS.heapMb, - timeoutMs: - input.analysis_timeout_ms ?? - overrides?.["javascript.analysis_timeout_ms"] ?? - JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS.timeoutMs, - }, + sourceLimits, files.files.filter( (file) => file.kind === "javascript" && file.text.included, ).length, @@ -229,6 +237,8 @@ export const reconstructJavaScriptArtifact = async ( }); abortIfNeeded(signal); }, + preloads.paths, + preloads.limitations, ); } finally { try { diff --git a/src/application/javascript/JavaScriptJsonDocuments.ts b/src/application/javascript/JavaScriptJsonDocuments.ts index 3700b29004..a4f96d1d45 100644 --- a/src/application/javascript/JavaScriptJsonDocuments.ts +++ b/src/application/javascript/JavaScriptJsonDocuments.ts @@ -54,6 +54,7 @@ const PACKAGE_PROPERTIES = [ "browser", "exports", "imports", + "type", ] as const; /** diff --git a/src/application/javascript/JavaScriptPreloadDiscovery.ts b/src/application/javascript/JavaScriptPreloadDiscovery.ts new file mode 100644 index 0000000000..ab2ab67425 --- /dev/null +++ b/src/application/javascript/JavaScriptPreloadDiscovery.ts @@ -0,0 +1,135 @@ +import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorCleanup.js"; +import { + AnalysisCancelledError, + AnalysisResourceConstraintError, +} from "../../domain/analysisErrorCore.js"; +import { translateArtifactFailure } from "../../artifacts/ArtifactOperationFailure.js"; +import type { JavaScriptArtifactFileSet } from "../../domain/javascript/javascriptArtifactFiles.js"; +import type { JavaScriptSourceAnalysisLimits } from "../../domain/javascript/javascriptSourceAnalysis.js"; +import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; +import { createJavaScriptSourceAnalysis } from "../../artifacts/javascript/JavaScriptSourceAnalysis.js"; +import { createJavaScriptPackageTypeLookup } from "./JavaScriptArtifactAnalysis.js"; +import { JavaScriptJsonDocuments } from "./JavaScriptJsonDocuments.js"; +import { resolveArtifactPathByContext } from "./JavaScriptArtifactPathResolution.js"; + +/** Declared preload identities, with the discovery gaps that may hide others. */ +export interface JavaScriptPreloadDiscovery { + readonly paths: ReadonlySet; + readonly limitations: readonly string[]; +} + +/** Resolve declared preload identities before choosing any .js file's loader. + * Discovery uses the same bounded worker and path owner as normal analysis. + * It never infers a loader from renderer API text in the selected file. + */ +export const discoverJavaScriptPreloadPaths = async ( + fileSet: JavaScriptArtifactFileSet, + limits: JavaScriptSourceAnalysisLimits, + scope: ArtifactResourceScope, + signal?: AbortSignal, +): Promise => { + const paths = new Set(); + const limitations: string[] = []; + const jsonDocuments = new JavaScriptJsonDocuments(); + const packageTypeFor = createJavaScriptPackageTypeLookup( + fileSet, + jsonDocuments, + ); + if ( + !fileSet.files.some( + (file) => + file.kind === "javascript" && + file.text.included && + /\.js$/iu.test(file.path) && + packageTypeFor(file) === "module", + ) + ) + return { paths, limitations }; + const sources = fileSet.files.filter( + (file) => file.kind === "javascript" && file.text.included, + ); + const files = new Map(fileSet.files.map((file) => [file.path, file])); + const worker = await createJavaScriptSourceAnalysis(limits, sources.length, { + scope, + }); + let outcome: { kind: "completed" } | { kind: "failed"; cause: unknown } = { + kind: "completed", + }; + try { + for (const [index, file] of sources.entries()) { + const result = await worker.analyze( + file, + { nodes: 0, unknowns: 0 }, + signal, + undefined, + packageTypeFor(file), + "preload-discovery", + ); + if (!result.ok) { + const { error } = result.error; + if (signal?.aborted === true || error instanceof AnalysisCancelledError) + throw error; + // Normal analysis meets the same source and owns its failure and the + // partial Evidence. A per-source limit only loses this source's + // declarations; any other failure would stop that pass here too. The + // gap stays reported even if normal analysis then reads the source. + const skipped = + error instanceof AnalysisResourceConstraintError && + error.reportedLimits?.parent_result_budget_exhausted !== true; + const later = sources.length - index - 1; + limitations.push( + skipped || later === 0 + ? `Preload discovery could not read ${file.path} (${error.message}); a .js preload declared only there keeps its package scope.` + : `Preload discovery stopped at ${file.path} (${error.message}); a .js preload declared only there or in ${String(later)} later sources keeps its package scope.`, + ); + if (skipped) continue; + break; + } + for (const window of result.value.javascript.electron.browser_windows) { + if (window.preload_path === null) continue; + const resolution = resolveArtifactPathByContext({ + declaredPath: window.preload_path, + sourcePath: file.path, + context: window.preload_resolution_context, + files, + jsonDocuments, + unanalyzedFiles: fileSet.unanalyzed_files, + runtimePathKind: "exact-file", + }); + if ( + resolution.resolution_status !== "resolved" || + resolution.resolved_path === null + ) + continue; + const target = files.get(resolution.resolved_path); + if (target?.container_sha256 === file.container_sha256) + paths.add(`${target.container_sha256}:${target.path}`); + } + } + } catch (cause: unknown) { + outcome = { kind: "failed", cause }; + throw cause; + } finally { + // The scoped port parks a healthy process; the following checkout resets + // its per-analysis accounting, and scope cleanup still owns failed closes. + const cleanup = await scope.release({ + kind: "javascript-source-analysis", + analysis: worker, + resource: "JavaScript preload discovery", + }); + if (cleanup.kind === "failed") { + if (outcome.kind === "failed" && outcome.cause instanceof AnalysisError) + throw analysisErrorWithCleanupFailure( + outcome.cause, + translateArtifactFailure( + "analyze_javascript_application", + scope.cleanupFailure({ kind: "completed" }, [cleanup]), + ), + "analyze_javascript_application", + ); + throw scope.cleanupFailure(outcome, [cleanup]); + } + } + return { paths, limitations }; +}; diff --git a/src/composition/javascriptAnalysisWorkerProcess.ts b/src/composition/javascriptAnalysisWorkerProcess.ts index 540cbadd6b..8155d37eb1 100644 --- a/src/composition/javascriptAnalysisWorkerProcess.ts +++ b/src/composition/javascriptAnalysisWorkerProcess.ts @@ -20,11 +20,13 @@ const analyze: JavaScriptSourceWorkerAnalyzer = async ( file, capacity, callbacks, + packageType, + purpose, ) => { if (!file.text.included) throw new TypeError("JavaScript worker requires decoded source text"); await callbacks.phase("parse"); - let parsed = parseJavaScriptSource(file.text.value, file.path); + let parsed = parseJavaScriptSource(file.text.value, file.path, packageType); if (parsed === null) { await callbacks.checkpoint(failedJavaScriptStaticAnalysis()); return null; @@ -33,6 +35,26 @@ const analyze: JavaScriptSourceWorkerAnalyzer = async ( const facts = await completeJavaScriptAnalysisSteps( analyzeParsedJavaScriptStaticSourceSteps(file.text.value, parsed), ); + if (purpose === "preload-discovery") { + // Only declarations cross this preliminary pass. The normal pass owns + // full static/semantic facts and their result reservations. + const empty = failedJavaScriptStaticAnalysis(); + await callbacks.checkpoint({ + ...empty, + parse_status: facts.parse_status, + parse_error_count: facts.parse_error_count, + ...(facts.analysis_coverage === undefined + ? {} + : { analysis_coverage: facts.analysis_coverage }), + visited_ast_nodes: facts.visited_ast_nodes, + limitations: facts.limitations, + electron: { + ...empty.electron, + browser_windows: facts.electron.browser_windows, + }, + }); + return null; + } const resultBudgetBytes = await callbacks.checkpoint(facts); if (facts.parse_status === "failed") return null; if (resultBudgetBytes === 0) diff --git a/src/domain/javascript/javascriptSourceAnalysis.ts b/src/domain/javascript/javascriptSourceAnalysis.ts index 9a588c7f32..f7e02df35b 100644 --- a/src/domain/javascript/javascriptSourceAnalysis.ts +++ b/src/domain/javascript/javascriptSourceAnalysis.ts @@ -8,6 +8,7 @@ import type { Result } from "../result.js"; import type { JavaScriptArtifactFile } from "./javascriptArtifactFiles.js"; import type { JavaScriptStaticAnalysis } from "./javascriptStaticAnalysisTypes.js"; import type { JavaScriptModuleSemanticIr } from "./javascriptModuleSemanticIr.js"; +import type { JavaScriptPackageType } from "./javascriptSourceParser.js"; /** An earlier static projection displaced to retain later static observations. */ export interface JavaScriptRevokedStaticProjection { @@ -48,6 +49,10 @@ export interface JavaScriptSourceAnalysisPort { capacity: JavaScriptSemanticProjectionCapacity, signal?: AbortSignal, onPhase?: (phase: string) => Promise, + /** Effective package/loader context; declared .js preloads use commonjs. */ + packageType?: JavaScriptPackageType, + /** Discover loader declarations without retaining semantic projections. */ + purpose?: "analysis" | "preload-discovery", ): Promise>; close(): Promise; } diff --git a/src/domain/javascript/javascriptSourceParser.ts b/src/domain/javascript/javascriptSourceParser.ts index 9fc1d9a1b9..43c3be3ba5 100644 --- a/src/domain/javascript/javascriptSourceParser.ts +++ b/src/domain/javascript/javascriptSourceParser.ts @@ -154,6 +154,19 @@ const runtimeSourceTypeForPath = ( ? "module" : "unambiguous"; +/** + * Node package facts apply to .js/.ts sources, not explicit suffixes, + * declaration wrappers, or an unknown JSX compilation pipeline. + */ +const packageContextForPath = ( + sourcePath: string | undefined, + observedPackageType: JavaScriptPackageType | undefined, +): JavaScriptPackageType | undefined => + /\.(?:js|ts)$/iu.test(sourcePath ?? "") && + !/\.d\.ts$/iu.test(sourcePath ?? "") + ? observedPackageType + : undefined; + export interface RecoveredJavaScriptParse { readonly file: ParsedJavaScriptSource; /** Errors that apply only if a source of unknown mode is an ES module. */ @@ -173,13 +186,7 @@ export const parseRecoveredJavaScript = ( observedPackageType?: JavaScriptPackageType, ): RecoveredJavaScriptParse => { const filenameType = runtimeSourceTypeForPath(sourcePath); - // Node package facts apply to .js/.ts sources, not explicit suffixes, - // declaration wrappers, or an unknown JSX compilation pipeline. - const packageContext = - /\.(?:js|ts)$/iu.test(sourcePath ?? "") && - !/\.d\.ts$/iu.test(sourcePath ?? "") - ? observedPackageType - : undefined; + const packageContext = packageContextForPath(sourcePath, observedPackageType); // Retain TypeScript import syntax for later CommonJS compilation. const sourceType = filenameType !== "unambiguous" @@ -246,36 +253,55 @@ export const parseRecoveredJavaScript = ( }; }; -/** Parse inert JavaScript or TypeScript facts without attaching comments to AST nodes. */ +/** + * Parse inert JavaScript or TypeScript facts without attaching comments to AST + * nodes. An observed package type selects the runtime context of .js/.ts + * sources as it does for reference imports; syntax recovery is unchanged. + */ export const parseJavaScriptSource = ( source: string, sourcePath?: string, + observedPackageType?: JavaScriptPackageType, ): ParsedJavaScriptSource | null => { try { const runtimeSourceType = runtimeSourceTypeForPath(sourcePath); const commonjs = runtimeSourceType === "commonjs"; + const packageContext = packageContextForPath( + sourcePath, + observedPackageType, + ); const options = { errorRecovery: true, attachComment: false, plugins: parserPluginsForPath(sourcePath), }; - const file = parseWithRuntimeContext(source, sourcePath, { - ...options, - // Inert artifact analysis retains module declarations even in .cjs assets. - // Infer their syntax here; reference imports keep filename-mode diagnostics. - sourceType: commonjs ? "unambiguous" : runtimeSourceType, - allowReturnOutsideFunction: commonjs, - allowNewTargetOutsideFunction: commonjs, - }); + const file = parseWithRuntimeContext( + source, + sourcePath, + { + ...options, + // Inert artifact analysis retains module declarations even in .cjs assets. + // Infer their syntax here; reference imports keep filename-mode diagnostics. + sourceType: commonjs ? "unambiguous" : runtimeSourceType, + allowReturnOutsideFunction: commonjs, + allowNewTargetOutsideFunction: commonjs, + }, + packageContext, + ); // Unambiguous mode retains module-only diagnostics after choosing a script. // The .cjs extension establishes wrapper mode, so obtain its own diagnostics. return commonjs && file.program.sourceType === "script" && file.errors.length > 0 - ? parseWithRuntimeContext(source, sourcePath, { - ...options, - sourceType: "commonjs", - }) + ? parseWithRuntimeContext( + source, + sourcePath, + { + ...options, + sourceType: "commonjs", + }, + packageContext, + ) : file; } catch (cause: unknown) { // Unparseable source is represented by the null return. diff --git a/src/javascript/analysis/JavaScriptAnalysisWorker.ts b/src/javascript/analysis/JavaScriptAnalysisWorker.ts index 72af9e850e..ca7f2daea4 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorker.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorker.ts @@ -24,6 +24,7 @@ import type { JavaScriptSourceAnalysisLimits, JavaScriptRevokedStaticProjection, } from "../../domain/javascript/javascriptSourceAnalysis.js"; +import type { JavaScriptPackageType } from "../../domain/javascript/javascriptSourceParser.js"; import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { AnalysisCancelledError, @@ -292,6 +293,8 @@ export class JavaScriptAnalysisWorker { capacity: JavaScriptSemanticProjectionCapacity, signal?: AbortSignal, onPhase?: (phase: string) => Promise, + packageType: JavaScriptPackageType = "unknown", + purpose: "analysis" | "preload-discovery" = "analysis", ): Promise< Result > { @@ -347,6 +350,8 @@ export class JavaScriptAnalysisWorker { unknown_budget: capacity.unknowns, text_sha256: createHash("sha256").update(file.text.value).digest("hex"), file: metadata, + package_type: packageType, + purpose, }); const frame = `${JSON.stringify(request)}\n`; if (Buffer.byteLength(frame) > DEFAULTS.protocolBytes) diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts index ffcfb873ee..4af0dacfce 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts @@ -37,6 +37,9 @@ export const javaScriptWorkerRequestSchema = z.strictObject({ kind: z.literal("javascript"), unpacked: z.boolean(), }), + /** Type of the source's inventoried package scope, observed by the parent. */ + package_type: z.enum(["module", "commonjs", "unspecified", "unknown"]), + purpose: z.enum(["analysis", "preload-discovery"]), }); /** Parent admission after validated static observations have been retained. */ diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerServer.test.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerServer.test.ts index 7834a47c2a..5e9d968ff4 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerServer.test.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerServer.test.ts @@ -81,6 +81,8 @@ const startServer = async () => { kind: "javascript", unpacked: false, }, + package_type: "unknown", + purpose: "analysis", }); }, close: async () => { diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts index ae219d4012..ae46284020 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts @@ -16,6 +16,7 @@ import { import { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js"; import { javaScriptRuntime } from "../../domain/runtimeVersion.js"; import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; +import type { JavaScriptPackageType } from "../../domain/javascript/javascriptSourceParser.js"; import type { JavaScriptStaticAnalysis } from "../../domain/javascript/javascriptStaticAnalysisTypes.js"; import type { JavaScriptModuleSemanticIr } from "../../domain/javascript/javascriptModuleSemanticIr.js"; import { @@ -43,6 +44,8 @@ export interface JavaScriptSourceWorkerAnalyzer { ) => Promise; readonly checkpoint: (facts: JavaScriptStaticAnalysis) => Promise; }, + packageType: JavaScriptPackageType, + purpose: "analysis" | "preload-discovery", ): Promise<{ readonly module: JavaScriptModuleSemanticIr; readonly projection: JavaScriptSemanticFileProjection; @@ -186,6 +189,8 @@ const analyzeJob = async ( return resultBudgetBytes; }, }, + request.package_type, + request.purpose, ); await send({ kind: "phase", id: request.id, phase: "transfer" }); const descriptor = diff --git a/tests/acceptance/applications/javascriptAnalysisLimits.test.ts b/tests/acceptance/applications/javascriptAnalysisLimits.test.ts index ccebeb6db5..b7a9215604 100644 --- a/tests/acceptance/applications/javascriptAnalysisLimits.test.ts +++ b/tests/acceptance/applications/javascriptAnalysisLimits.test.ts @@ -259,6 +259,41 @@ cliTest( 60_000, ); +cliTest( + "keeps partial Evidence when a module package source times out", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-js-limit-module-timeout-"); + await Promise.all([ + writeFile(join(root, "package.json"), '{"type":"module"}'), + writeFile(join(root, "a.js"), "export const observed = 7;\n"), + ]); + // A module package with .js sources runs preload discovery first, which + // must leave the timeout to the source pass that retains partial facts. + const result = await cli.run({ + arguments: [ + "analyze-javascript-application", + root, + "--analysis-timeout-ms", + "1", + "--format", + "json", + ], + }); + expect(result.exitCode).toBe(1); + const failure = z + .object({ + code: z.literal("provider_timeout"), + details: z.object({ partial_observation: z.unknown() }), + }) + .parse(result.json); + const partial = parseEvidence(failure.details.partial_observation); + expect(partial.operation).toBe("analyze_javascript_application"); + expect(partial.limitations).toContainEqual( + expect.stringMatching(/^Preload discovery could not read a\.js \(/u), + ); + }, +); + async function getBundle(call: SharedMcpCall) { const response = await call("get_evidence_bundle", { detail: "complete" }); expect(response.isError).not.toBe(true); diff --git a/tests/acceptance/applications/unboundRequires.test.ts b/tests/acceptance/applications/unboundRequires.test.ts index 0e85f0a9a7..b5425ba886 100644 --- a/tests/acceptance/applications/unboundRequires.test.ts +++ b/tests/acceptance/applications/unboundRequires.test.ts @@ -325,6 +325,194 @@ const expectImportEqualsRelationships = ( ).toHaveLength(0); }; +/** Source-module facts and semantic require edges of one analyzed source. */ +const sourceRequireFacts = ( + graph: ReturnType< + typeof javascriptApplicationAnalysisResultSchema.parse + >["graph"], + path: string, +) => { + const module = graph.nodes + .flatMap(({ observations }) => observations) + .find( + ({ properties }) => + properties.semantic_role === "source-module" && + properties.path === path, + )?.properties; + const requires = graph.edges.filter( + ({ relation, properties, evidence }) => + relation === "imports" && + properties.module_link_kind === "require" && + evidence.location.available && + evidence.location.value.kind === "source-range" && + evidence.location.value.source === path, + ); + return { + module_format: module?.module_format, + import_relationships: module?.import_relationships, + require_targets: requires.map(({ properties }) => [ + properties.specifier, + properties.resolved_path, + properties.resolution_status, + ]), + }; +}; + +cliTest( + "follows the nearest package type for an unbound require in .js sources", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-package-type-requires-"); + const source = (dependency: string) => + `try { console.log(require(${JSON.stringify(dependency)})); } catch (error) { console.log(error.name); }\n`; + // The nearest package.json owns each .js file's scope (Node packages.md, + // "Determining module system"). Requiring Electron, directly, through a + // local loader, or in a bundler factory default that never runs, does not + // give a Node-loaded module a require. + await writeFixtureFiles(root, { + "package.json": JSON.stringify({ type: "module" }), + "main.js": source("./dep.cjs"), + "electron-main.js": + 'try { require("electron"); console.log(require("./dep.cjs")); } catch (error) { console.log(error.name); }\n', + "renderer-looking-main.js": + 'try { require("electron").ipcRenderer.send("ready"); require("./dep.cjs"); } catch (error) { console.log(error.name); }', + "shadowed.js": `function load(require) { return require("electron").contextBridge; }\n${source("./dep.cjs")}`, + "dead-default.js": [ + source("./dep.cjs"), + "globalThis.webpackChunkApp = [];", + "globalThis.webpackChunkApp.push([[1], {", + ' 7(module = require("electron").ipcRenderer.send("unused"), exports, load) {}', + "}]);", + ].join("\n"), + "dep.cjs": "module.exports = 1;\n", + "legacy/package.json": JSON.stringify({ type: "commonjs" }), + "legacy/main.js": source("../dep.cjs"), + "untyped/package.json": JSON.stringify({}), + "untyped/main.js": source("../dep.cjs"), + }); + for (const [path, expected] of [ + ["main.js", "ReferenceError"], + ["electron-main.js", "ReferenceError"], + ["renderer-looking-main.js", "ReferenceError"], + ["shadowed.js", "ReferenceError"], + ["dead-default.js", "ReferenceError"], + ["legacy/main.js", "1"], + ["untyped/main.js", "1"], + ] as const) { + const control = await execute(process.execPath, [join(root, path)], { + cwd: root, + timeout: 5_000, + }); + expect(control.stdout.trim(), path).toBe(expected); + } + + const response = await cli.run({ + arguments: ["analyze-javascript-application", root, "--format", "json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + + expect(response.exitCode, response.stderr).toBe(0); + const { graph } = javascriptApplicationAnalysisResultSchema.parse( + parseEvidence(response.json).normalized_result, + ); + for (const path of [ + "main.js", + "electron-main.js", + "renderer-looking-main.js", + "shadowed.js", + "dead-default.js", + ]) + expect(sourceRequireFacts(graph, path), path).toEqual({ + module_format: "unknown", + import_relationships: 0, + require_targets: [], + }); + for (const path of ["legacy/main.js", "untyped/main.js"]) + expect(sourceRequireFacts(graph, path), path).toEqual({ + module_format: "commonjs", + import_relationships: 1, + require_targets: [["../dep.cjs", "dep.cjs", "resolved"]], + }); + }, +); + +cliTest( + "keeps Electron's script loader for a .js preload in a module package scope", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-package-type-preload-"); + // Electron's preload loader ignores "type": "module" ("ES Modules (ESM) + // in Electron"); main.mjs is an ES module by its suffix. + await writeFixtureFiles(root, { + "package.json": JSON.stringify({ type: "module", main: "main.mjs" }), + "main.mjs": [ + 'import { app, BrowserWindow, ipcMain } from "electron";', + 'import { fileURLToPath } from "node:url";', + 'ipcMain.handle("ping", () => "pong");', + "app.whenReady().then(() => {", + " const window = new BrowserWindow({", + ' webPreferences: { sandbox: false, preload: fileURLToPath(new URL("./preload.js", import.meta.url)) },', + " });", + ' window.loadFile("index.html");', + "});", + ].join("\n"), + "preload.js": [ + 'const { contextBridge, ipcRenderer } = require("electron");', + 'function expose() { contextBridge.exposeInMainWorld("api", { ping: () => ipcRenderer.invoke("ping") }); } expose();', + ].join("\n"), + "index.html": "app\n", + }); + + const response = await cli.run({ + arguments: ["analyze-javascript-application", root, "--format", "json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + + expect(response.exitCode, response.stderr).toBe(0); + const { graph, summary } = javascriptApplicationAnalysisResultSchema.parse( + parseEvidence(response.json).normalized_result, + ); + expect(sourceRequireFacts(graph, "preload.js")).toEqual({ + module_format: "commonjs", + import_relationships: 2, + require_targets: [ + ["electron", null, "external"], + ["electron", null, "external"], + ], + }); + expect(summary.ipc).toMatchObject({ + main_handlers: 1, + renderer_transmissions: 1, + paired_renderer_transmissions: 1, + }); + }, +); + +cliTest( + "keeps a declared preload's helper require without renderer API text", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-helper-only-preload-"); + await writeFixtureFiles(root, { + "package.json": JSON.stringify({ type: "module", main: "main.mjs" }), + "main.mjs": + 'import { BrowserWindow } from "electron"; import { fileURLToPath } from "node:url"; new BrowserWindow({webPreferences:{sandbox:false,preload:fileURLToPath(new URL("./aaa-preload.js",import.meta.url))}});', + "aaa-preload.js": 'require("./helper.cjs");', + "helper.cjs": "module.exports = 1;", + }); + const response = await cli.run({ + arguments: ["analyze-javascript-application", root, "--format", "json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(response.exitCode, response.stderr).toBe(0); + const { graph } = javascriptApplicationAnalysisResultSchema.parse( + parseEvidence(response.json).normalized_result, + ); + expect(sourceRequireFacts(graph, "aaa-preload.js")).toEqual({ + module_format: "commonjs", + import_relationships: 1, + require_targets: [["./helper.cjs", "helper.cjs", "resolved"]], + }); + }, +); + cliTest.for(["CLI", "stdio MCP"] as const)( "retains unbound CommonJS semantic relationships through %s", async (surface, { cli }) => { diff --git a/tests/boundary/javascript/javascriptPreloadDiscovery.test.ts b/tests/boundary/javascript/javascriptPreloadDiscovery.test.ts new file mode 100644 index 0000000000..2947dab782 --- /dev/null +++ b/tests/boundary/javascript/javascriptPreloadDiscovery.test.ts @@ -0,0 +1,208 @@ +import { beforeEach, expect, it, vi } from "vitest"; +import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js"; +import { + AnalysisCancelledError, + AnalysisResourceConstraintError, +} from "../../../src/domain/analysisErrorCore.js"; +import type { + JavaScriptArtifactFile, + JavaScriptArtifactFileSet, +} from "../../../src/domain/javascript/javascriptArtifactFiles.js"; +import { analyzeParsedJavaScriptStaticSource } from "../../../src/domain/javascript/javascriptStaticAnalysis.js"; +import { parseJavaScriptSource } from "../../../src/domain/javascript/javascriptSourceParser.js"; +import { ok, err } from "../../../src/domain/result.js"; +import { discoverJavaScriptPreloadPaths } from "../../../src/application/javascript/JavaScriptPreloadDiscovery.js"; +const acquire = vi.hoisted(() => vi.fn()); +vi.mock( + "../../../src/artifacts/javascript/JavaScriptSourceAnalysis.js", + () => ({ + createJavaScriptSourceAnalysis: acquire, + }), +); +const file = ( + path: string, + text: string, + container = "a".repeat(64), +): JavaScriptArtifactFile => ({ + path, + container_sha256: container, + sha256: "b".repeat(64), + bytes: text.length, + inventory_artifact_id: path, + kind: path.endsWith("json") ? "package-json" : "javascript", + unpacked: false, + text: { included: true, value: text }, +}); +const declaration = + 'import { BrowserWindow } from "electron"; import { fileURLToPath } from "node:url"; new BrowserWindow({webPreferences:{sandbox:false,preload:fileURLToPath(new URL("./preload.js",import.meta.url))}});'; +const fixture = (): JavaScriptArtifactFileSet => ({ + files: [ + file("package.json", '{"type":"module"}'), + file("preload.js", 'require("./helper.cjs");'), + file("main.mjs", declaration), + ], + unavailable_relevant_files: [], + containers: [], + text_bytes_read: 0, + invalid_utf8_files: 0, +}); +const limits = { heapMb: 128, timeoutMs: 5000 }; +const close = vi.fn(async () => {}); +const analyze = vi.fn( + async ( + source: JavaScriptArtifactFile, + _capacity: unknown, + _signal: AbortSignal | undefined, + _phase: unknown, + packageType: "module" | "commonjs" | "unknown" | "unspecified", + ) => { + const parsed = parseJavaScriptSource( + source.text.included ? source.text.value : "", + source.path, + packageType, + )!; + return ok({ + javascript: analyzeParsedJavaScriptStaticSource( + source.text.included ? source.text.value : "", + parsed, + ), + module: null, + projection: null, + }); + }, +); +beforeEach(() => { + close.mockReset(); + analyze.mockClear(); + acquire.mockReset(); + acquire.mockResolvedValue({ analyze, close }); +}); +it("resolves an exact preload before its declaring source regardless of inventory order", async () => { + const result = await discoverJavaScriptPreloadPaths( + fixture(), + limits, + new ArtifactResourceScope(), + ); + expect(result).toEqual({ + paths: new Set(["a".repeat(64) + ":preload.js"]), + limitations: [], + }); + expect(close).toHaveBeenCalledOnce(); + expect(analyze.mock.calls.every((call) => call[2] === undefined)).toBe(true); +}); +it("keeps discovering after one source exceeds its worker limit", async () => { + const files = fixture(); + const limited = vi.fn( + async (source: JavaScriptArtifactFile, ...rest: unknown[]) => + source.path === "big.js" + ? err({ + error: new AnalysisResourceConstraintError( + "analyze_javascript_application", + "memory", + "big.js exhausted the worker heap", + { max_heap_mb: 128 }, + ), + javascript: null, + module: null, + projection: null, + }) + : analyze(source, ...(rest as [unknown, undefined, unknown, "module"])), + ); + acquire.mockResolvedValue({ analyze: limited, close }); + const result = await discoverJavaScriptPreloadPaths( + { ...files, files: [file("big.js", ""), ...files.files] }, + limits, + new ArtifactResourceScope(), + ); + // The gap stays reported even if source analysis later reads big.js. + expect(result).toEqual({ + paths: new Set(["a".repeat(64) + ":preload.js"]), + limitations: [ + expect.stringMatching( + /^Preload discovery could not read big\.js \(.*big\.js exhausted the worker heap/u, + ), + ], + }); + expect(limited.mock.calls.map(([source]) => source.path)).toEqual([ + "big.js", + "preload.js", + "main.mjs", + ]); +}); +it("does not turn an invalid ESM require declaration into a preload identity", async () => { + const files = fixture(); + const source = file( + "main.js", + 'const { BrowserWindow } = require("electron"); new BrowserWindow({webPreferences:{preload:"preload.js"}});', + ); + expect( + ( + await discoverJavaScriptPreloadPaths( + { ...files, files: [...files.files.slice(0, 2), source] }, + limits, + new ArtifactResourceScope(), + ) + ).paths, + ).toEqual(new Set()); +}); +it("does not transfer preload identity across containers", async () => { + const files = fixture(); + expect( + ( + await discoverJavaScriptPreloadPaths( + { + ...files, + files: files.files.map((source) => + source.path === "preload.js" + ? { ...source, container_sha256: "c".repeat(64) } + : source, + ), + }, + limits, + new ArtifactResourceScope(), + ) + ).paths, + ).toEqual(new Set()); +}); +it("skips discovery when package scope cannot change a .js loader", async () => { + const files = fixture(); + await discoverJavaScriptPreloadPaths( + { + ...files, + files: [ + file("package.json", '{"type":"commonjs"}'), + ...files.files.slice(1), + ], + }, + limits, + new ArtifactResourceScope(), + ); + expect(acquire).not.toHaveBeenCalled(); +}); +it("retains a failed close and the original cancellation until scope cleanup retries", async () => { + const controller = new AbortController(); + const reason = new Error("caller cancelled"); + controller.abort(reason); + const failure = new AnalysisCancelledError("analyze_javascript_application", { + cause: reason, + }); + const failingAnalyze = vi.fn(async () => + err({ error: failure, javascript: null, module: null, projection: null }), + ); + close.mockRejectedValueOnce(new Error("close denied")); + acquire.mockResolvedValue({ analyze: failingAnalyze, close }); + const scope = new ArtifactResourceScope(); + await expect( + discoverJavaScriptPreloadPaths(fixture(), limits, scope, controller.signal), + ).rejects.toMatchObject({ + cleanupIncomplete: true, + cause: expect.objectContaining({ + errors: expect.arrayContaining([failure]), + }), + }); + expect(scope.hasPending).toBe(true); + expect(failingAnalyze.mock.calls).toHaveLength(1); + await scope.retryCleanup(); + expect(scope.hasPending).toBe(false); + expect(close).toHaveBeenCalledTimes(2); +});