diff --git a/docs/guides/javascript/javascript-artifact-reconstruction.md b/docs/guides/javascript/javascript-artifact-reconstruction.md index baf90aa6df..c5b46be897 100644 --- a/docs/guides/javascript/javascript-artifact-reconstruction.md +++ b/docs/guides/javascript/javascript-artifact-reconstruction.md @@ -227,7 +227,11 @@ Each newly analyzed JavaScript file retains `analysis_coverage` in its graph observation. `syntax` distinguishes complete parsing, recovered syntax errors, failed parsing and unexamined source. `semantics` records the number of observed unknown findings and typed uncertainty, including returned-reference widening -and unavailable type-only compiler options. `examination` reports static semantic +and unavailable type-only compiler options. `conditional-loader` records a +BOM/hashbang source recovered with an uncertain package mode or loader +invocation. Its overall `parse_status` stays partial without inventing a syntax +error. Dependency and worker Evidence retains the loader limitation. +`examination` reports static semantic resource limits and Electron wrapper expansion limits independently. A completed pass does not establish complete runtime semantics. Route examination and result capacity remain in the separate route inventory. @@ -353,6 +357,20 @@ execute them. JavaScript and HTML source ranges retain an initial UTF-8 BOM as one UTF-16 code unit, matching the original bytes identified by the artifact digest. +A BOM before a hashbang is accepted for TypeScript and JSX compiler inputs, +explicit ES modules, and JavaScript in an observed module package. +For this prefix, suffix-only source names, including `.cjs` and case variants such as `.MJS`, +retain a conditional interpretation: direct entrypoint and module import can +select different loaders. A known CommonJS package can load these files +through an ES module import using CommonJS grammar. +An unknown package scope or a package without a `type` field retains a +conditional interpretation for ordinary `.js` and suffix-only JavaScript files. Application analysis and +reference import can recover either script or ES module syntax without +establishing the runtime loader. +Source indexes and later-line columns still refer to the original text. +CommonJS sources with this prefix retain a conditional interpretation through +ES module import. Direct execution and require can reject the same source. +A plain hashbang stays valid. Plain `.ts` artifact sources use TypeScript syntax without JSX, including angle-bracket type assertions. `.tsx` and `.jsx` retain JSX parsing. diff --git a/src/application/javascript/JavaScriptArtifactGraphAccumulator.ts b/src/application/javascript/JavaScriptArtifactGraphAccumulator.ts index 25378a9f15..a4a6166f3b 100644 --- a/src/application/javascript/JavaScriptArtifactGraphAccumulator.ts +++ b/src/application/javascript/JavaScriptArtifactGraphAccumulator.ts @@ -1,3 +1,7 @@ +import { + qualifyJavaScriptLoaderEvidence, + type ApplicationGraphEvidence, +} from "../../domain/javascript/javascriptApplicationEvidenceSchemas.js"; import { createJavaScriptApplicationEdge, createJavaScriptApplicationNode, @@ -17,9 +21,43 @@ export class JavaScriptArtifactGraphAccumulator { readonly #nodes = new Map(); readonly #edges = new Map(); + constructor( + private readonly loaderConditions: ReadonlyMap< + string, + { readonly sha256: string; readonly condition: string } + > = new Map(), + ) {} + + readonly #qualifyEvidence = ( + evidence: ApplicationGraphEvidence, + ): ApplicationGraphEvidence => { + if ( + !evidence.artifact.available || + !evidence.location.available || + (evidence.authority !== "ast-static-analysis" && + evidence.authority !== "static-relationship-inference") + ) + return evidence; + const location = evidence.location.value; + const path = + location.kind === "source-range" + ? location.source + : location.kind === "artifact-path" + ? location.path + : undefined; + const origin = + path === undefined ? undefined : this.loaderConditions.get(path); + return origin?.sha256 === evidence.artifact.sha256 + ? qualifyJavaScriptLoaderEvidence(evidence, origin.condition) + : evidence; + }; + /** Merge observations into a stable live node; order is finalized by nodes(). */ addNode(input: Input): ApplicationNode { - const created = createJavaScriptApplicationNode(displayableNode(input)); + const created = createJavaScriptApplicationNode( + displayableNode(input), + this.loaderConditions.size === 0 ? undefined : this.#qualifyEvidence, + ); const existing = this.#nodes.get(created.node_id); if (existing === undefined) { this.#nodes.set(created.node_id, { @@ -66,7 +104,10 @@ export class JavaScriptArtifactGraphAccumulator { /** Create or replace one canonical edge by semantic identifier. */ addEdge(input: unknown): ApplicationEdge { - const edge = createJavaScriptApplicationEdge(input); + const edge = createJavaScriptApplicationEdge( + input, + this.loaderConditions.size === 0 ? undefined : this.#qualifyEvidence, + ); this.#edges.set(edge.edge_id, edge); return edge; } diff --git a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts index 1e1e5ecab5..bf24af4a3e 100644 --- a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts +++ b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts @@ -92,7 +92,16 @@ function* buildJavaScriptArtifactGraphInputSteps( void, { readonly graph: unknown; readonly electronSummary: ElectronBoundarySummary } > { - const accumulator = new JavaScriptArtifactGraphAccumulator(); + const loaderConditions = new Map< + string, + { sha256: string; condition: string } + >(); + for (const { file, javascript } of analysis.files) { + const condition = javascript?.loaderCondition; + if (condition !== undefined) + loaderConditions.set(file.path, { sha256: file.sha256, condition }); + } + const accumulator = new JavaScriptArtifactGraphAccumulator(loaderConditions); const root = createJavaScriptArtifactRootNode(accumulator, snapshot); const parsedPackages = new Set( analysis.packages diff --git a/src/application/javascript/JavaScriptArtifactGraphFindings.ts b/src/application/javascript/JavaScriptArtifactGraphFindings.ts index 8534e20595..02ad688057 100644 --- a/src/application/javascript/JavaScriptArtifactGraphFindings.ts +++ b/src/application/javascript/JavaScriptArtifactGraphFindings.ts @@ -209,6 +209,18 @@ const addReference = ( input.value.loader_authority === "candidate" ? null : resolveReference(context, input.file.path, input.value, input.coverage); + const candidateLoader = input.value.loader_authority === "candidate"; + const loaderEvidence = { + confidence: candidateLoader ? ("low" as const) : ("high" as const), + limitations: [ + ...(resolved?.limitations ?? + (candidateLoader + ? [ + "The module loader is unknown, so this dependency cannot be resolved against the source file.", + ] + : [])), + ], + }; if ( input.value.operation === "load" && (input.value.kind === "static-import" || @@ -261,7 +273,7 @@ const addReference = ( range: input.value.location, operation: "discover-worker", coverage: input.coverage, - limitations: resolved?.limitations ?? [], + ...loaderEvidence, }), }, ], @@ -278,7 +290,7 @@ const addReference = ( resolution_status: resolved?.resolution?.resolution_status ?? null, ...artifactPathResolutionProperties(resolved?.resolution), }, - limitations: resolved?.limitations ?? [], + ...loaderEvidence, }); if (resolved !== null && resolved.path !== null && resolved.node !== null) addStaticInferenceEdge(context, { @@ -289,7 +301,7 @@ const addReference = ( coverage: input.coverage, relation: "maps_to", properties: { resolved_path: resolved.path }, - limitations: resolved.limitations, + ...loaderEvidence, }); return; } @@ -338,18 +350,7 @@ const addReference = ( resolution_status: resolved?.resolution?.resolution_status ?? null, ...artifactPathResolutionProperties(resolved?.resolution), }, - ...(input.value.loader_authority === "candidate" - ? { confidence: "low" as const } - : {}), - ...(resolved === null - ? input.value.loader_authority === "candidate" - ? { - limitations: [ - "The module loader is unknown, so this dependency cannot be resolved against the source file.", - ], - } - : {} - : { limitations: resolved.limitations }), + ...loaderEvidence, }); }; diff --git a/src/application/javascript/JavaScriptArtifactGraphStructure.ts b/src/application/javascript/JavaScriptArtifactGraphStructure.ts index 7a441117f8..4bc6d88ec3 100644 --- a/src/application/javascript/JavaScriptArtifactGraphStructure.ts +++ b/src/application/javascript/JavaScriptArtifactGraphStructure.ts @@ -690,6 +690,10 @@ const createFileTarget = ( path: file.path, operation: "inventory-relevant-file", coverage: completeApplicationCoverage(), + limitations: + javascript?.loaderCondition === undefined + ? [] + : [javascript.loaderCondition], }), }, ], diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts index 3e64f9caf9..7292faf247 100644 --- a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts +++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts @@ -600,6 +600,26 @@ export function* projectJavaScriptSemanticFileSteps( } function* projectFileSteps( + file: JavaScriptArtifactFile, + ir: JavaScriptSemanticIr, + state: BuilderState, + paths: readonly ElectronCallPath[], +): Generator { + if (ir.loaderCondition === undefined) + return yield* projectFileWithEvidenceSteps(file, ir, state, paths); + state.evidenceContexts.sourceLoaderCondition = { + path: file.path, + sha256: file.sha256, + limitation: ir.loaderCondition, + }; + try { + return yield* projectFileWithEvidenceSteps(file, ir, state, paths); + } finally { + state.evidenceContexts.sourceLoaderCondition = undefined; + } +} + +function* projectFileWithEvidenceSteps( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, state: BuilderState, diff --git a/src/domain/javascript/javascriptApplicationEvidenceSchemas.ts b/src/domain/javascript/javascriptApplicationEvidenceSchemas.ts index b27176bfbb..b1a7b40065 100644 --- a/src/domain/javascript/javascriptApplicationEvidenceSchemas.ts +++ b/src/domain/javascript/javascriptApplicationEvidenceSchemas.ts @@ -528,3 +528,20 @@ export type ApplicationGraphEvidenceContext = z.infer< export type ApplicationNodeIdentity = z.infer< typeof applicationNodeIdentitySchema >; + +/** Qualify a source interpretation while preserving its location and artifact identity. */ +export const qualifyJavaScriptLoaderEvidence = ( + evidence: ApplicationGraphEvidence, + condition: string, +): ApplicationGraphEvidence => ({ + ...evidence, + confidence: + evidence.confidence === "exact" || evidence.confidence === "high" + ? "medium" + : evidence.confidence, + coverage: + evidence.coverage.status === "complete" + ? { ...evidence.coverage, status: "partial" } + : evidence.coverage, + limitations: [...evidence.limitations, condition], +}); diff --git a/src/domain/javascript/javascriptApplicationGraph.ts b/src/domain/javascript/javascriptApplicationGraph.ts index 3cdaa82420..c619e223a2 100644 --- a/src/domain/javascript/javascriptApplicationGraph.ts +++ b/src/domain/javascript/javascriptApplicationGraph.ts @@ -473,6 +473,9 @@ export type JavaScriptApplicationGraph = z.infer< /** Build one normalized application entity and derive all semantic IDs. */ export const createJavaScriptApplicationNode = ( input: unknown, + qualifyEvidence?: ( + evidence: ApplicationGraphEvidence, + ) => ApplicationGraphEvidence, ): ApplicationNode => { const parsed = applicationNodeInputSchema.parse(input); // These properties are newly parsed and exclusively owned. Authenticate @@ -485,7 +488,9 @@ export const createJavaScriptApplicationNode = ( .map((observation) => { const semantic = { ...observation, - evidence: normalizeEvidence(observation.evidence), + evidence: normalizeEvidence( + qualifyEvidence?.(observation.evidence) ?? observation.evidence, + ), identifier_strategy: OBSERVATION_IDENTIFIER_STRATEGY, }; return applicationNodeObservationSchema.parse({ @@ -552,12 +557,17 @@ export const mergeJavaScriptApplicationNodes = ( /** Build one normalized directed relationship and derive its semantic ID. */ export const createJavaScriptApplicationEdge = ( input: unknown, + qualifyEvidence?: ( + evidence: ApplicationGraphEvidence, + ) => ApplicationGraphEvidence, ): ApplicationEdge => { const parsed = applicationEdgeInputSchema.parse(input); freezeJsonSnapshot(parsed.properties); const semantic = { ...parsed, - evidence: normalizeEvidence(parsed.evidence), + evidence: normalizeEvidence( + qualifyEvidence?.(parsed.evidence) ?? parsed.evidence, + ), identifier_strategy: EDGE_IDENTIFIER_STRATEGY, }; return applicationEdgeSchema.parse({ diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index 9fa98409c1..d8dc8b986f 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -130,6 +130,7 @@ import { } from "./javascriptSemanticCoverage.js"; import { semanticResourceLimitReason } from "./javascriptSemanticResourceLimits.js"; import { + javaScriptSourceLoaderCondition, javaScriptStrictnessAt, parseJavaScriptSource, type ParsedJavaScriptSource, @@ -328,8 +329,9 @@ export function* analyzeParsedJavaScriptSemanticsSteps( parserPartial, expressionNodeBudget, ); + const loaderCondition = javaScriptSourceLoaderCondition(file.program); const coverage = semanticCoverage( - parserPartial, + parserPartial || loaderCondition !== undefined, [ ...new Set([...state.valueResourceLimits, ...state.getterResourceLimits]), ].sort(), @@ -347,7 +349,9 @@ export function* analyzeParsedJavaScriptSemanticsSteps( coverage: derived.expressionFlowsTruncated ? { ...coverage, status: "partial", omittedCount: null } : coverage, + ...(loaderCondition === undefined ? {} : { loaderCondition }), limitations: [ + ...(loaderCondition === undefined ? [] : [loaderCondition]), ...(derived.expressionFlowsTruncated ? [ "The offered semantic node capacity bounded expression dependencies; omitted influence remains unknown.", diff --git a/src/domain/javascript/javascriptSemanticGraph.ts b/src/domain/javascript/javascriptSemanticGraph.ts index dbb509b969..76acdce437 100644 --- a/src/domain/javascript/javascriptSemanticGraph.ts +++ b/src/domain/javascript/javascriptSemanticGraph.ts @@ -23,6 +23,7 @@ import { applicationGraphEvidenceSchema, isCompatibleApplicationGraphEvidenceLocation, isValidApplicationGraphEvidenceContext, + qualifyJavaScriptLoaderEvidence, } from "./javascriptApplicationEvidenceSchemas.js"; import { JAVASCRIPT_SEMANTIC_RELATION_FAMILIES, @@ -134,12 +135,23 @@ export class JavaScriptSemanticEvidenceContextRegistry { // distinct context once instead of on every fact. readonly #idsByCanonicalContext = new Map(); readonly #frozenEntries: FrozenInternedEvidenceEntry[] = []; + sourceLoaderCondition: + | { + readonly path: string; + readonly sha256: string; + readonly limitation: string; + } + | undefined; /** Fast-path interning when shared context objects match an already-validated frozen entry. */ [tryInternValidatedInputSymbol]( rawEvidence: unknown, ): JavaScriptSemanticEvidenceReference | undefined { - if (!hasExactEvidenceKeys(rawEvidence)) return undefined; + if ( + this.sourceLoaderCondition !== undefined || + !hasExactEvidenceKeys(rawEvidence) + ) + return undefined; const match = this.#frozenEntries.find( (entry) => entry.authority === rawEvidence.authority && @@ -180,7 +192,22 @@ export class JavaScriptSemanticEvidenceContextRegistry { ): JavaScriptSemanticEvidenceReference { const fast = this[tryInternValidatedInputSymbol](originalInput ?? evidence); if (fast !== undefined) return fast; - const normalized = normalizeEvidence(evidence); + const condition = this.sourceLoaderCondition; + const locationValue = evidence.location.available + ? evidence.location.value + : undefined; + const normalized = normalizeEvidence( + condition !== undefined && + evidence.artifact.available && + evidence.artifact.sha256 === condition.sha256 && + (locationValue?.kind === "source-range" + ? locationValue.source + : locationValue?.kind === "artifact-path" + ? locationValue.path + : undefined) === condition.path + ? qualifyJavaScriptLoaderEvidence(evidence, condition.limitation) + : evidence, + ); const { location, ...context } = normalized; const canonicalContext = canonicalJson( context, @@ -202,7 +229,8 @@ export class JavaScriptSemanticEvidenceContextRegistry { this.#contexts.set(contextId, sealSemanticEvidenceContext(parsedContext)); this.#idsByCanonicalContext.set(canonicalContext, contextId); } - const candidate = originalInput ?? evidence; + const candidate = + condition === undefined ? (originalInput ?? evidence) : normalized; if ( this.#frozenEntries.length < 16 && hasReusableFrozenContext(candidate) && diff --git a/src/domain/javascript/javascriptSemanticIr.ts b/src/domain/javascript/javascriptSemanticIr.ts index 2fa8e85d7c..73f27286b2 100644 --- a/src/domain/javascript/javascriptSemanticIr.ts +++ b/src/domain/javascript/javascriptSemanticIr.ts @@ -506,6 +506,7 @@ export interface JavaScriptSemanticIr { readonly functionFingerprints: readonly JavaScriptSemanticFunctionFingerprint[]; readonly frontiers: readonly JavaScriptSemanticFrontier[]; readonly coverage: JavaScriptSemanticCoverage; + readonly loaderCondition?: string; readonly limitations: readonly string[]; } diff --git a/src/domain/javascript/javascriptSourceParser.ts b/src/domain/javascript/javascriptSourceParser.ts index cd119c84b0..1ff60e3340 100644 --- a/src/domain/javascript/javascriptSourceParser.ts +++ b/src/domain/javascript/javascriptSourceParser.ts @@ -1,6 +1,7 @@ import { posix } from "node:path"; import { parse, type ParserOptions, type ParserPlugin } from "@babel/parser"; import * as t from "@babel/types"; +import { isJavaScriptSourcePath } from "./javascriptArtifactFiles.js"; import { javaScriptDefinitionExpression } from "./javascriptAstValues.js"; import { explicitJavaScriptLoaderMode, @@ -28,16 +29,30 @@ export type JavaScriptPackageType = | "unspecified" | "unknown"; -const runtimeContextByProgram = new WeakMap< +const contextByProgram = new WeakMap< t.Program, - JavaScriptRuntimeContext + { + readonly runtime: JavaScriptRuntimeContext; + readonly loaderCondition?: string; + } >(); /** Only parser-owned facts establish module or CommonJS wrapper isolation. */ export const javaScriptRuntimeContext = ( program: t.Program, ): JavaScriptRuntimeContext => - runtimeContextByProgram.get(program) ?? "unknown"; + contextByProgram.get(program)?.runtime ?? "unknown"; + +/** A recovered grammar does not prove which loader applies to the source. */ +export const javaScriptSourceLoaderCondition = ( + program: t.Program, +): string | undefined => contextByProgram.get(program)?.loaderCondition; + +/** A suffix-only source name has no extension in Node's module-import dispatch. */ +const isSuffixOnlySourcePath = (path: string | undefined): boolean => + path !== undefined && + posix.extname(path) === "" && + isJavaScriptSourcePath(path); /** Runtime strictness follows parser-owned mode, not erased TypeScript syntax. */ export const javaScriptProgramStrictness = ( @@ -102,35 +117,108 @@ const parseWithRuntimeContext = ( source: string, sourcePath: string | undefined, options: ParserOptions, - packageContext?: JavaScriptPackageType, + observedPackageType?: JavaScriptPackageType, ): ParsedJavaScriptSource => { - const file = parse(source, options); const filenameMode = explicitJavaScriptLoaderMode(sourcePath); - runtimeContextByProgram.set( - file.program, - filenameMode === "commonjs" - ? sourcePath?.toLowerCase().endsWith(".d.cts") - ? "unknown" - : "commonjs" - : filenameMode === "module" - ? "module" - : packageContext === "module" || packageContext === "commonjs" - ? packageContext - : packageContext === "unspecified" && - posix.extname(sourcePath ?? "") === ".js" - ? file.program.sourceType === "module" - ? "module" - : "commonjs" - : hasCaseVariantJavaScriptLoaderSuffix(sourcePath) - ? "unknown" - : file.program.body.some((node) => t.isTSExportAssignment(node)) - ? // CommonJS isolation does not establish strict ESM semantics. - "commonjs" - : file.program.sourceType === "module" && - hasRuntimeModuleDeclaration(file.program) - ? "module" - : "unknown", + const packageContext = packageContextForPath(sourcePath, observedPackageType); + const suffixOnlySource = isSuffixOnlySourcePath(sourcePath); + const ambiguousHashbangPath = + posix.extname(sourcePath ?? "") === ".js" || suffixOnlySource; + // Dotfile entrypoints have no extname. Their package can explain signature + // preprocessing, but must not grant wrapper/module authority to ordinary data. + const signaturePackageContext = ambiguousHashbangPath + ? observedPackageType + : packageContext; + // ESM and TypeScript file loading remove the UTF-8 signature before a + // hashbang; JavaScript CommonJS loading does not. Preserve original indexes + // through Babel's origin options rather than repairing downstream ranges. + const compilerSource = /\.(?:jsx|ts|tsx|mts|cts)$/iu.test(sourcePath ?? ""); + const signedHashbang = source.startsWith("\uFEFF#!"); + const conditionalCommonJs = + signedHashbang && + !compilerSource && + (filenameMode === "commonjs" || signaturePackageContext === "commonjs"); + const compilerCommonJsDotfile = + compilerSource && + suffixOnlySource && + signaturePackageContext === "commonjs"; + const observedModuleDotfile = + suffixOnlySource && signaturePackageContext === "module"; + const ambiguousLoader = + signedHashbang && + ambiguousHashbangPath && + (signaturePackageContext === undefined || + signaturePackageContext === "unknown" || + signaturePackageContext === "unspecified" || + observedModuleDotfile || + compilerCommonJsDotfile); + const loaderCondition = conditionalCommonJs + ? `Source in ${sourcePath ?? ""} is recovered as CommonJS when imported through the ES module loader. Direct execution and require can reject this BOM/hashbang.` + : ambiguousLoader + ? observedModuleDotfile + ? `Source in ${sourcePath ?? ""} is recovered under an ES module interpretation. Its loader invocation is unknown; direct entrypoint and module import can differ.` + : compilerCommonJsDotfile + ? `Source in ${sourcePath ?? ""} is recovered with compiler syntax. Its loader invocation is conditional: direct CommonJS loading can reject this BOM/hashbang.` + : `Source in ${sourcePath ?? ""} is recovered under a conditional loader interpretation. Its package loader mode is unknown.` + : undefined; + const hashbangSignature = + signedHashbang && + (compilerSource || + conditionalCommonJs || + ambiguousLoader || + filenameMode === "module" || + (filenameMode === undefined && signaturePackageContext === "module")); + const parserOptions = { ...options }; + if (conditionalCommonJs) { + parserOptions.sourceType = "commonjs"; + delete parserOptions.allowAwaitOutsideFunction; + delete parserOptions.allowReturnOutsideFunction; + delete parserOptions.allowNewTargetOutsideFunction; + } + const file = parse( + hashbangSignature ? source.slice(1) : source, + hashbangSignature + ? { + ...parserOptions, + ...(observedModuleDotfile ? { sourceType: "module" as const } : {}), + startIndex: (options.startIndex ?? 0) + 1, + startColumn: (options.startColumn ?? 0) + 1, + } + : options, ); + const runtime = + conditionalCommonJs && suffixOnlySource + ? "commonjs" + : ambiguousLoader + ? "unknown" + : filenameMode === "commonjs" + ? sourcePath?.toLowerCase().endsWith(".d.cts") + ? "unknown" + : "commonjs" + : filenameMode === "module" + ? "module" + : packageContext === "module" || packageContext === "commonjs" + ? packageContext + : packageContext === "unspecified" && + posix.extname(sourcePath ?? "") === ".js" + ? file.program.sourceType === "module" + ? "module" + : "commonjs" + : hasCaseVariantJavaScriptLoaderSuffix(sourcePath) + ? "unknown" + : file.program.body.some((node) => + t.isTSExportAssignment(node), + ) + ? // CommonJS isolation does not establish strict ESM semantics. + "commonjs" + : file.program.sourceType === "module" && + hasRuntimeModuleDeclaration(file.program) + ? "module" + : "unknown"; + contextByProgram.set(file.program, { + runtime, + ...(loaderCondition === undefined ? {} : { loaderCondition }), + }); return file; }; @@ -242,7 +330,7 @@ export const parseRecoveredJavaScript = ( sourceType, errorRecovery: true, }, - packageContext, + observedPackageType, ), moduleOnlyErrors: [], }; @@ -255,7 +343,7 @@ export const parseRecoveredJavaScript = ( sourceType: "unambiguous", errorRecovery: true, }, - packageContext, + observedPackageType, ); if (file.program.sourceType !== "script" || file.errors.length === 0) return { file, moduleOnlyErrors: [] }; @@ -267,7 +355,7 @@ export const parseRecoveredJavaScript = ( sourceType: "script", errorRecovery: true, }, - packageContext, + observedPackageType, ); const scriptErrors = new Set(script.errors.map((error) => error.message)); return { @@ -294,10 +382,6 @@ export const parseJavaScriptSource = ( try { const runtimeSourceType = runtimeSourceTypeForPath(sourcePath); const commonjs = runtimeSourceType === "commonjs"; - const packageContext = packageContextForPath( - sourcePath, - observedPackageType, - ); const options = { errorRecovery: true, attachComment: false, @@ -314,10 +398,23 @@ export const parseJavaScriptSource = ( allowReturnOutsideFunction: commonjs, allowNewTargetOutsideFunction: commonjs, }, - packageContext, + observedPackageType, ); // Unambiguous mode retains module-only diagnostics after choosing a script. // The .cjs extension establishes wrapper mode, so obtain its own diagnostics. + if ( + !commonjs && + file.program.sourceType === "script" && + file.errors.length > 0 && + javaScriptSourceLoaderCondition(file.program) !== undefined && + javaScriptRuntimeContext(file.program) === "unknown" + ) + return parseWithRuntimeContext( + source, + sourcePath, + { ...options, sourceType: "script" }, + observedPackageType, + ); return commonjs && file.program.sourceType === "script" && file.errors.length > 0 @@ -328,7 +425,7 @@ export const parseJavaScriptSource = ( ...options, sourceType: "commonjs", }, - packageContext, + observedPackageType, ) : file; } catch (cause: unknown) { diff --git a/src/domain/javascript/javascriptStaticAnalysis.ts b/src/domain/javascript/javascriptStaticAnalysis.ts index d61dc7ca49..0e818ec783 100644 --- a/src/domain/javascript/javascriptStaticAnalysis.ts +++ b/src/domain/javascript/javascriptStaticAnalysis.ts @@ -73,7 +73,10 @@ import { indexJavaScriptElectronIdentitySteps, indexJavaScriptBindingsWithReceiversSteps, } from "./javascriptSemanticAnalysis.js"; -import type { ParsedJavaScriptSource } from "./javascriptSourceParser.js"; +import { + javaScriptSourceLoaderCondition, + type ParsedJavaScriptSource, +} from "./javascriptSourceParser.js"; import { createJavaScriptRpcScan } from "./javascriptRpcEndpoints.js"; import { @@ -99,6 +102,7 @@ export const memberMutationRefusedJavaScriptStaticAnalysis = ( refusal: AnalysisResourceConstraintError, ): JavaScriptStaticAnalysis => { const parserErrors = file.errors.length; + const loaderCondition = javaScriptSourceLoaderCondition(file.program); const accumulator = createJavaScriptAnalysisAccumulator(source.length); addSourceMapDirectives(source, file.comments ?? [], accumulator); const reason = `The member-mutation phase was refused (${refusal.reason}); findings that depend on member-mutation facts were not examined.`; @@ -107,7 +111,10 @@ export const memberMutationRefusedJavaScriptStaticAnalysis = ( parserErrors === 0 ? { status: "complete", error_count: 0 } : { status: "recovered", error_count: parserErrors }, - semantics: { unknown_findings: null, uncertainty: [] }, + semantics: { + unknown_findings: null, + uncertainty: loaderCondition === undefined ? [] : ["conditional-loader"], + }, examination: { status: "not-examined", semantic_resource_limits: [], @@ -124,7 +131,9 @@ export const memberMutationRefusedJavaScriptStaticAnalysis = ( parse_error_count: parserErrors, source_map_urls: accumulator.sourceMaps, vendors: detectVendors(source), + ...(loaderCondition === undefined ? {} : { loaderCondition }), limitations: [ + ...(loaderCondition === undefined ? [] : [loaderCondition]), `${reason} Absence of findings is not evidence of absence.`, ...(parserErrors > 0 ? [ @@ -280,6 +289,7 @@ const finalizeStaticAnalysis = ( ...accumulator.protocols, ...accumulator.appPathChanges, ].some(({ value }) => value.loader_authority === "candidate"); + const loaderCondition = javaScriptSourceLoaderCondition(file.program); const coverage: JavaScriptStaticCoverage = { syntax: parserErrors === 0 @@ -288,6 +298,9 @@ const finalizeStaticAnalysis = ( semantics: { unknown_findings: accumulator.unknownFindings, uncertainty: [ + ...(loaderCondition === undefined + ? [] + : ["conditional-loader" as const]), ...(accumulator.unknownFindings > 0 ? ["dynamic-findings" as const] : []), @@ -316,6 +329,7 @@ const finalizeStaticAnalysis = ( applicability: { electron: electronObserved ? "observed" : "unknown" }, }; const limitations = [ + ...(loaderCondition === undefined ? [] : [loaderCondition]), ...(bindings.valueResourceLimits.has("alias-selections") ? [semanticResourceLimitReason("alias-selections", bindings.precision)] : []), @@ -429,6 +443,7 @@ const finalizeStaticAnalysis = ( accumulator.moduleRangeIndex, ), }, + ...(loaderCondition === undefined ? {} : { loaderCondition }), limitations, }; }; diff --git a/src/domain/javascript/javascriptStaticAnalysisTypes.ts b/src/domain/javascript/javascriptStaticAnalysisTypes.ts index b249be50f7..958068e083 100644 --- a/src/domain/javascript/javascriptStaticAnalysisTypes.ts +++ b/src/domain/javascript/javascriptStaticAnalysisTypes.ts @@ -165,6 +165,7 @@ export interface JavaScriptStaticAnalysis { }[]; readonly vendors: readonly string[]; readonly electron: ElectronStaticFindings; + readonly loaderCondition?: string; readonly limitations: readonly string[]; } diff --git a/src/domain/javascript/javascriptStaticCoverage.test.ts b/src/domain/javascript/javascriptStaticCoverage.test.ts index 884d1b24ac..214b4a31d2 100644 --- a/src/domain/javascript/javascriptStaticCoverage.test.ts +++ b/src/domain/javascript/javascriptStaticCoverage.test.ts @@ -1,11 +1,80 @@ import { expect, it } from "vitest"; -import { analyzeJavaScriptStaticSource } from "../../../tests/fixtures/javascriptStaticAnalysis.js"; +import { + analyzeJavaScriptStaticSource, + analyzeParsedJavaScriptStaticSource, +} from "../../../tests/fixtures/javascriptStaticAnalysis.js"; import { analyzeCommonJsStaticFixture } from "../../../tests/fixtures/javascriptCommonJsAnalysis.js"; import { javaScriptStaticParseStatus, unexaminedJavaScriptStaticCoverage, } from "./javascriptStaticCoverage.js"; import { aliasSelectionBudgetSource } from "../../../tests/fixtures/javascriptAliasSelectionBudget.js"; +import { + javaScriptRuntimeContext, + javaScriptProgramStrictness, + parseJavaScriptSource, + parseRecoveredJavaScript, + parserPluginsForPath, +} from "./javascriptSourceParser.js"; + +it.each(["main.js", ".js", ".mjs", ".cjs", ".JS", ".MJS"])( + "retains the unknown loader condition without inventing syntax errors for %s", + (path) => { + const source = '\uFEFF#!/usr/bin/env node\nimport "./dep.mjs";'; + const application = parseJavaScriptSource(source, path, "unknown"); + expect(application).not.toBeNull(); + const historical = parseRecoveredJavaScript( + source, + path, + { plugins: parserPluginsForPath(path) }, + "unknown", + ).file; + for (const file of [application!, historical]) { + expect(file.errors).toEqual([]); + expect(javaScriptRuntimeContext(file.program)).toBe("unknown"); + expect(javaScriptProgramStrictness(file.program)).toBe("unknown"); + const result = analyzeParsedJavaScriptStaticSource(source, file); + expect(result.parse_status).toBe("partial"); + expect(result.analysis_coverage).toMatchObject({ + syntax: { status: "complete", error_count: 0 }, + semantics: { unknown_findings: 0, uncertainty: ["conditional-loader"] }, + }); + expect(result.references).toMatchObject([ + { + kind: "static-import", + specifier: "./dep.mjs", + location: { + start: { line: 2, column: 0 }, + end: { line: 2, column: 19 }, + }, + }, + ]); + expect(result.limitations).toContainEqual( + expect.stringContaining("package loader mode is unknown"), + ); + } + }, +); + +it.each(["unknown", "unspecified"] as const)( + "keeps script grammar and conditional coverage for a BOM/hashbang in a %s package", + (packageType) => { + const source = + '\uFEFF#!/usr/bin/env node\nwith ({}) {} module.exports = require("./dep.cjs");'; + const file = parseJavaScriptSource(source, "main.js", packageType); + expect(file).not.toBeNull(); + expect(file!.program.sourceType).toBe("script"); + expect(javaScriptRuntimeContext(file!.program)).toBe("unknown"); + expect(file!.errors).toEqual([]); + const result = analyzeParsedJavaScriptStaticSource(source, file!); + expect(result.analysis_coverage).toMatchObject({ + syntax: { status: "complete", error_count: 0 }, + semantics: { + uncertainty: expect.arrayContaining(["conditional-loader"]), + }, + }); + }, +); it("separates dynamic semantic uncertainty from completed syntax and examination", () => { const result = analyzeCommonJsStaticFixture( diff --git a/src/domain/javascript/javascriptStaticCoverage.ts b/src/domain/javascript/javascriptStaticCoverage.ts index e35007994e..040625e98e 100644 --- a/src/domain/javascript/javascriptStaticCoverage.ts +++ b/src/domain/javascript/javascriptStaticCoverage.ts @@ -28,6 +28,7 @@ export const javaScriptStaticCoverageSchema = z.strictObject({ "dynamic-findings", "returned-reference-widening", "type-only-module-side-effect", + "conditional-loader", ]), ), }), @@ -61,6 +62,7 @@ export const javaScriptStaticParseStatus = ( coverage.examination.status === "not-examined" || // An unknown finding count cannot establish that none were found. coverage.semantics.unknown_findings !== 0 || + coverage.semantics.uncertainty.includes("conditional-loader") || coverage.examination.semantic_resource_limits.includes( "alias-selections", ) diff --git a/src/domain/referenceSourceImportParsing.test.ts b/src/domain/referenceSourceImportParsing.test.ts index 4983440770..6113188c9b 100644 --- a/src/domain/referenceSourceImportParsing.test.ts +++ b/src/domain/referenceSourceImportParsing.test.ts @@ -57,3 +57,87 @@ it("keeps declaration-level imports when the member-mutation phase is refused", ), ).toHaveLength(1); }); + +it.each([ + ["main.mjs", "commonjs", true], + ["main.js", "module", true], + ["main.cjs", "module", true], + ["main.js", "commonjs", true], + ["main.mts", "commonjs", true], + ["main.cts", "module", true], + ["main.ts", "commonjs", true], + ["main.tsx", "module", true], + ["main.TSX", "module", true], + ["main.jsx", "commonjs", true], + [".js", "module", true], + [".mjs", "module", true], + [".mjs", "commonjs", true], + [".js", "commonjs", true], + [".cjs", "module", true], + [".CJS", "module", true], + [".JS", "module", true], + [".MJS", "module", true], + [".cjs", "commonjs", true], +] as const)( + "retains the loader's BOM/hashbang semantics for %s (%s)", + (path, packageType, valid) => { + const esm = + (path.endsWith(".mjs") && path !== ".mjs") || + path.endsWith(".mts") || + path.endsWith(".jsx") || + (path.startsWith(".") && packageType === "module") || + (!path.endsWith(".cjs") && + !path.endsWith(".cts") && + packageType === "module"); + const statement = esm ? 'import "./dep.mjs";' : 'require("./dep.cjs");'; + const parsed = parseReferenceSourceImports( + path, + Buffer.from(`\uFEFF#!/usr/bin/env node\n${statement}`), + path.endsWith("js") ? "JavaScript" : "TypeScript", + packageType, + ); + if (valid) { + expect(parsed.parse_failures).toEqual([]); + expect(parsed.relationships).toMatchObject([ + { + to: esm ? "./dep.mjs" : "./dep.cjs", + parse_state: + path.startsWith(".") || + path === "main.cjs" || + (path === "main.js" && packageType === "commonjs") + ? "partial" + : "parsed", + }, + ]); + } else { + expect(parsed.parse_failures).toHaveLength(1); + expect(parsed.relationships).toEqual([]); + } + }, +); + +it.each([ + ["main.js", "unknown"], + ["main.js", "unspecified"], + [".cjs", "unknown"], + [".js", "unspecified"], +] as const)( + "retains a conditional dependency without a strict-mode error for %s in a %s package", + (path, mode) => { + const parsed = parseReferenceSourceImports( + path, + Buffer.from( + '\uFEFF#!/usr/bin/env node\nwith ({}) {}\nrequire("./dep.cjs");', + ), + "JavaScript", + mode, + ); + expect(parsed.parse_failures).toEqual([]); + expect(parsed.relationships).toMatchObject([ + { to: "./dep.cjs", parse_state: "partial" }, + ]); + expect(parsed.limitations).toContainEqual( + expect.stringContaining("package loader mode is unknown"), + ); + }, +); diff --git a/src/domain/referenceSourceImportParsing.ts b/src/domain/referenceSourceImportParsing.ts index 814be060b3..03cd8a7758 100644 --- a/src/domain/referenceSourceImportParsing.ts +++ b/src/domain/referenceSourceImportParsing.ts @@ -8,6 +8,7 @@ import { readExactJavaScriptLiteral } from "./javascript/javascriptAstValues.js" import { parseRecoveredJavaScript, parserPluginsForPath, + javaScriptSourceLoaderCondition, type JavaScriptPackageType, } from "./javascript/javascriptSourceParser.js"; import { @@ -319,7 +320,8 @@ export const parseReferenceSourceImports = ( let source: string; try { - source = decodeStrictUtf8(bytes, "strip-signature"); + // The shared parser owns loader-specific BOM/hashbang preprocessing. + source = decodeStrictUtf8(bytes, "preserve"); } catch { return { relationships: [], @@ -339,12 +341,17 @@ export const parseReferenceSourceImports = ( source, packageType, ); + const loaderCondition = + ast === undefined + ? undefined + : javaScriptSourceLoaderCondition(ast.program); const limitations = moduleOnlyErrors.length === 0 ? [] : [ `Strict-mode errors apply only if this file is an ES module: ${moduleOnlyErrors.join("; ")}`, ]; + if (loaderCondition !== undefined) limitations.push(loaderCondition); const parseFailures = reasons.map((reason) => ({ path, parser: "babel", @@ -372,7 +379,8 @@ export const parseReferenceSourceImports = ( relationships: relationships.map((relationship) => ({ ...relationship, parse_state: - parseFailures.length > 0 && relationship.parse_state === "parsed" + (parseFailures.length > 0 || loaderCondition !== undefined) && + relationship.parse_state === "parsed" ? "partial" : relationship.parse_state, })), diff --git a/src/domain/webModuleImports.test.ts b/src/domain/webModuleImports.test.ts index 0a601f0f87..75a3b6dd96 100644 --- a/src/domain/webModuleImports.test.ts +++ b/src/domain/webModuleImports.test.ts @@ -213,3 +213,32 @@ describe("captured module candidates", () => { ]); }); }); + +it.each(["main.mjs", "main.js", "view.jsx"])( + "retains original expression offsets for a BOM-prefixed hashbang in %s", + (path) => { + const source = + '\uFEFF#!/usr/bin/env node\r\nconst face = "😀"; import "./dep.mjs"; import(`./${part}.mjs`);'; + const result = collectWebModuleImports(source, path); + expect(result.state).toBe("parsed"); + expect(result.diagnostics).toEqual( + path === "main.js" + ? [expect.stringContaining("package loader mode is unknown")] + : [], + ); + expect(result.imports.map(({ expression }) => expression)).toEqual([ + '"./dep.mjs"', + "`./${part}.mjs`", + ]); + for (const item of result.imports) { + expect(item.start.offset).toBe(source.indexOf(item.expression)); + expect(source.slice(item.start.offset, item.end.offset)).toBe( + item.expression, + ); + expect(item.start.line).toBe(2); + expect(item.start.column).toBe( + source.split("\r\n")[1]!.indexOf(item.expression), + ); + } + }, +); diff --git a/src/domain/webModuleImports.ts b/src/domain/webModuleImports.ts index e96355aa18..3c6f528b91 100644 --- a/src/domain/webModuleImports.ts +++ b/src/domain/webModuleImports.ts @@ -1,7 +1,10 @@ import type { Node } from "@babel/types"; import { readExactJavaScriptLiteral } from "./javascript/javascriptAstValues.js"; import { traverseJavaScriptAst } from "./javascript/javascriptSemanticTraversal.js"; -import { parseJavaScriptSource } from "./javascript/javascriptSourceParser.js"; +import { + javaScriptSourceLoaderCondition, + parseJavaScriptSource, +} from "./javascript/javascriptSourceParser.js"; import type { WebModuleImport } from "./webModuleTrace.js"; import type { ExportedWebScript } from "./webScriptExport.js"; import { @@ -69,10 +72,12 @@ export const collectWebModuleImports = ( }, }); imports.sort((a, b) => a.start.offset - b.start.offset); + const loaderCondition = javaScriptSourceLoaderCondition(ast.program); return { imports, state: "parsed", diagnostics: [ + ...(loaderCondition === undefined ? [] : [loaderCondition]), ...(ast.errors?.map((error) => error.message) ?? []), ...(hasUncertainTypeOnlyModuleSideEffect(ast.program) ? [TYPE_ONLY_MODULE_SIDE_EFFECT_LIMITATION] diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts index 7b5569b424..4f2e319132 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts @@ -477,6 +477,7 @@ export const javaScriptWorkerStaticAnalysisSchema = z ), vendors: strings, electron, + loaderCondition: z.string().min(1).exactOptional(), limitations: strings, }) .transform((analysis) => ({ diff --git a/tests/acceptance/applications/javascriptBomHashbangCli.test.ts b/tests/acceptance/applications/javascriptBomHashbangCli.test.ts new file mode 100644 index 0000000000..ec5a1702bb --- /dev/null +++ b/tests/acceptance/applications/javascriptBomHashbangCli.test.ts @@ -0,0 +1,520 @@ +import { join } from "node:path"; +import { createHash } from "node:crypto"; +import { expect } from "vitest"; + +import { historicalSourceGraphSchema } from "../../../src/domain/referenceSourceGraph.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { + analyzeJavaScriptDirectory, + runNodeJsonControl, +} from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +const importedDotfiles = [ + ".cjs", + ".CJS", + ".JS", + ".MJS", + ".jsx", + ".JSX", + ".ts", + ".TS", + ".mts", + ".MTS", + ".cts", + ".CTS", + ".tsx", + ".TSX", +].map((name, index) => ({ + path: `esm/dot-${index}/${name}`, + specifier: `./dot-${index}/${name}`, +})); + +const fixture = async () => { + const root = await createTestTempDirectory("rea-bom-hashbang-"); + const files: Record = { + "package.json": JSON.stringify({ type: "commonjs", main: "module.mjs" }), + "module.mjs": + '\uFEFF#!/usr/bin/env node\nconst face="😀"; import {marker} from "./dep.mjs"; console.log(JSON.stringify({marker}));', + "esm/package.json": JSON.stringify({ type: "module" }), + "esm/main.js": + '\uFEFF#!/usr/bin/env node\r\nconst face="😀"; import {marker} from "../dep.mjs"; console.log(JSON.stringify({marker}));', + "esm/.js": + '\uFEFF#!/usr/bin/env node\nimport {marker} from "../dep.mjs"; console.log(JSON.stringify({marker}));', + "esm/.mjs": + '\uFEFF#!/usr/bin/env node\nimport {marker} from "../dep.mjs"; console.log(JSON.stringify({marker}));', + "view.jsx": + '\uFEFF#!/usr/bin/env node\nconst element =
; import "./dep.mjs";', + "typed.mts": + '\uFEFF#!/usr/bin/env node\nconst face: string="😀"; import {marker} from "./dep.mjs"; console.log(JSON.stringify({marker}));', + "typed.cts": + '\uFEFF#!/usr/bin/env node\nconst face: string="😀"; const {marker}=require("./dep.cjs"); console.log(JSON.stringify({marker}));', + "rejected.cjs": + '\uFEFF#!/usr/bin/env node\nconst {marker}=require("./dep.cjs"); console.log(JSON.stringify({marker}));', + "rejected.js": + '\uFEFF#!/usr/bin/env node\nconst {marker}=require("./dep.cjs"); console.log(JSON.stringify({marker}));', + ".cjs": + '\uFEFF#!/usr/bin/env node\nwith ({}) {} const {marker}=require("./dep.cjs"); console.log(JSON.stringify({marker}));', + "invoke-commonjs.mjs": + 'const print = console.log; console.log = () => {}; await import("./rejected.cjs"); await import("./rejected.js"); await import("./.cjs"); print(JSON.stringify({loaded:true}));', + "plain.cjs": + '#!/usr/bin/env node\nconst {marker}=require("./dep.cjs"); console.log(JSON.stringify({marker}));', + "dep.mjs": "export const marker=42;", + "dep.cjs": "exports.marker=42;", + ...Object.fromEntries( + importedDotfiles.map(({ path }) => [ + path, + '\uFEFF#!/usr/bin/env node\r\nconst face="😀"; import {marker} from "../../dep.mjs"; export {marker};', + ]), + ), + "esm/invoke.mjs": `const modules = await Promise.all([${importedDotfiles.map(({ specifier }) => `import(${JSON.stringify(specifier)})`).join(",")}]); console.log(JSON.stringify(modules.map(({marker}) => marker)));`, + }; + await writeFixtureFiles(root, files); + return { root, files }; +}; + +cliTest.for(["CLI", "stdio MCP"] as const)( + "analyzes BOM/hashbang modules with original Evidence through %s", + async (surface, { cli }) => { + const { root, files } = await fixture(); + for (const path of [ + "module.mjs", + "esm/main.js", + "esm/.js", + "esm/.mjs", + "typed.mts", + "typed.cts", + "plain.cjs", + ]) + expect(await runNodeJsonControl(root, path)).toEqual({ marker: 42 }); + for (const path of ["rejected.cjs", "rejected.js", ".cjs"]) + await expect(runNodeJsonControl(root, path)).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("SyntaxError"), + }); + expect(await runNodeJsonControl(root, "invoke-commonjs.mjs")).toEqual({ + loaded: true, + }); + expect(await runNodeJsonControl(root, "esm/invoke.mjs")).toEqual( + importedDotfiles.map(() => 42), + ); + await expect( + runNodeJsonControl(root, "esm/dot-0/.cjs"), + ).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("SyntaxError"), + }); + const { graph, statistics } = await analyzeJavaScriptDirectory( + surface, + cli, + root, + ); + expect(statistics.parse_failures).toBe(0); + for (const [path, kind, target] of [ + ["module.mjs", "static-import", "dep.mjs"], + ["esm/main.js", "static-import", "dep.mjs"], + ["esm/.js", "static-import", "dep.mjs"], + ["esm/.mjs", "static-import", "dep.mjs"], + ["view.jsx", "static-import", "dep.mjs"], + ["typed.mts", "static-import", "dep.mjs"], + ["typed.cts", "require", "dep.cjs"], + ["plain.cjs", "require", "dep.cjs"], + ["rejected.cjs", "require", "dep.cjs"], + ["rejected.js", "require", "dep.cjs"], + [".cjs", "require", "dep.cjs"], + ...importedDotfiles.map( + ({ path }) => [path, "static-import", "dep.mjs"] as const, + ), + ] as const) { + const source = files[path]!; + const line = source.split(/\r?\n/u)[1]!; + const start = line.indexOf(kind === "require" ? "require" : "import"); + const end = line.indexOf(kind === "require" ? ")" : ";", start) + 1; + const edges = graph.edges.filter( + ({ relation, properties, evidence }) => + relation === "imports" && + properties.kind === kind && + evidence.location.available && + evidence.location.value.kind === "source-range" && + evidence.location.value.source === path, + ); + expect(edges).toHaveLength(1); + if ( + path === "rejected.cjs" || + path === "rejected.js" || + path === ".cjs" + ) { + expect(edges[0]?.evidence).toMatchObject({ + coverage: { status: "partial" }, + limitations: expect.arrayContaining([ + expect.stringContaining("imported through the ES module loader"), + ]), + }); + } + + expect(edges[0]).toMatchObject({ + properties: { resolved_path: target }, + evidence: { + artifact: { + available: true, + sha256: createHash("sha256").update(source).digest("hex"), + }, + location: { + available: true, + value: { + kind: "source-range", + source: path, + start: { line: 2, column: start }, + end: { line: 2, column: end }, + }, + }, + }, + }); + } + const dotfile = graph.nodes + .flatMap(({ observations }) => observations) + .find( + ({ properties }) => + properties.path === "esm/dot-0/.cjs" && + properties.text_status !== undefined, + ); + expect(dotfile).toMatchObject({ + properties: { + parse_status: "partial", + analysis_coverage: { + syntax: { status: "complete", error_count: 0 }, + semantics: { uncertainty: ["conditional-loader"] }, + }, + }, + evidence: { + limitations: expect.arrayContaining([ + expect.stringContaining("loader invocation is unknown"), + ]), + }, + }); + for (const path of ["rejected.cjs", "rejected.js", ".cjs"]) { + expect( + graph.nodes.flatMap(({ observations }) => observations), + ).toContainEqual( + expect.objectContaining({ + properties: expect.objectContaining({ + path, + parse_status: "partial", + analysis_coverage: expect.objectContaining({ + semantics: expect.objectContaining({ + uncertainty: ["conditional-loader"], + }), + }), + }), + }), + ); + } + }, +); + +cliTest.runIf(process.platform !== "win32")( + "reference import retains conditional CommonJS BOM/hashbang dependencies", + async ({ cli }) => { + const { root, files } = await fixture(); + const output = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + }); + const subtree = join(root, "esm", "src"); + await writeFixtureFiles(subtree, { + "main.js": '\uFEFF#!/usr/bin/env node\nimport "./dep.mjs";', + "dep.mjs": "export const marker=42;", + }); + const narrowed = await cli.run({ + arguments: ["import-reference-source", subtree, "--json"], + }); + expect(narrowed.exitCode, narrowed.stderr).toBe(0); + const partial = historicalSourceGraphSchema.parse(narrowed.json); + expect(partial.parse_failures).toEqual([]); + expect(partial.relationships).toContainEqual( + expect.objectContaining({ + from_path: "main.js", + to: "dep.mjs", + kind: "imports", + parse_state: "partial", + resolution: "internal", + }), + ); + expect(partial.limitations).toContainEqual( + expect.stringContaining("package loader mode is unknown"), + ); + expect(output.exitCode, output.stderr).toBe(0); + const graph = historicalSourceGraphSchema.parse(output.json); + expect( + graph.parse_failures + .map(({ path }) => path) + .sort((a, b) => a.localeCompare(b)), + ).toEqual([]); + for (const [path, parseState] of [ + ["module.mjs", "parsed"], + ["esm/main.js", "parsed"], + ["esm/.js", "partial"], + ["esm/.mjs", "partial"], + ["view.jsx", "parsed"], + ["typed.mts", "parsed"], + ...importedDotfiles.map(({ path }) => [path, "partial"] as const), + ] as const) { + expect(graph.entries).toContainEqual( + expect.objectContaining({ + path, + sha256: createHash("sha256").update(files[path]!).digest("hex"), + }), + ); + expect(graph.relationships).toContainEqual( + expect.objectContaining({ + from_path: path, + to: "dep.mjs", + kind: "imports", + parse_state: parseState, + resolution: "internal", + }), + ); + } + expect(graph.relationships).toContainEqual( + expect.objectContaining({ + from_path: "typed.cts", + to: "dep.cjs", + kind: "requires", + parse_state: "parsed", + }), + ); + for (const path of ["rejected.cjs", "rejected.js", ".cjs"]) { + expect(graph.relationships).toContainEqual( + expect.objectContaining({ + from_path: path, + to: "dep.cjs", + kind: "requires", + parse_state: "partial", + resolution: "internal", + }), + ); + } + }, +); + +cliTest.for(["CLI", "stdio MCP"] as const)( + "retains conditional BOM/hashbang imports when analyzing a package subtree through %s", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-bom-subtree-"); + const source = + '\uFEFF#!/usr/bin/env node\r\nconst face="😀"; import {marker} from "./dep.mjs"; console.log(JSON.stringify({marker}));'; + await writeFixtureFiles(root, { + "package.json": JSON.stringify({ type: "module" }), + "src/main.js": source, + "src/dep.mjs": "export const marker=42;", + }); + expect(await runNodeJsonControl(root, "src/main.js")).toEqual({ + marker: 42, + }); + const result = await analyzeJavaScriptDirectory( + surface, + cli, + join(root, "src"), + ); + expect(result.statistics.parse_failures).toBe(0); + const properties = result.graph.nodes + .flatMap(({ observations }) => + observations.map(({ properties }) => properties), + ) + .find( + (properties) => + properties.path === "main.js" && properties.text_status !== undefined, + ); + expect(properties).toMatchObject({ + parse_status: "partial", + analysis_coverage: { + syntax: { status: "complete", error_count: 0 }, + semantics: { uncertainty: ["conditional-loader"] }, + }, + }); + const edge = result.graph.edges.find( + ({ relation, properties, evidence }) => + relation === "imports" && + properties.kind === "static-import" && + evidence.location.available && + evidence.location.value.kind === "source-range" && + evidence.location.value.source === "main.js", + ); + expect(edge).toMatchObject({ + properties: { kind: "static-import", resolved_path: "dep.mjs" }, + evidence: { + artifact: { + available: true, + sha256: createHash("sha256").update(source).digest("hex"), + }, + location: { + available: true, + value: { + kind: "source-range", + source: "main.js", + start: { line: 2, column: 17 }, + end: { line: 2, column: 50 }, + }, + }, + }, + }); + const sourceObservation = result.graph.nodes + .flatMap(({ observations }) => observations) + .find( + ({ properties }) => + properties.path === "main.js" && properties.text_status !== undefined, + ); + expect(sourceObservation?.evidence.limitations).toContainEqual( + expect.stringContaining("package loader mode is unknown"), + ); + }, +); + +cliTest.for(["CLI", "stdio MCP"] as const)( + "retains conditional loader evidence on worker references through %s", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-bom-worker-"); + const source = + '\uFEFF#!/usr/bin/env node\nnew Worker(new URL("./worker.js", import.meta.url));\nnavigator.serviceWorker.register(new URL("./worker.js", import.meta.url));'; + await writeFixtureFiles(root, { + "main.js": source, + "worker.js": "export {};", + }); + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); + const workerIds = new Set( + graph.nodes + .filter(({ kind }) => kind === "worker" || kind === "service-worker") + .map(({ node_id }) => node_id), + ); + const workers = graph.nodes.filter(({ node_id }) => workerIds.has(node_id)); + expect(workers.map(({ kind }) => kind).sort()).toEqual([ + "service-worker", + "worker", + ]); + const loads = graph.edges.filter( + ({ relation, target_node_id }) => + relation === "loads" && workerIds.has(target_node_id), + ); + const mappings = graph.edges.filter( + ({ relation, source_node_id }) => + relation === "maps_to" && workerIds.has(source_node_id), + ); + expect(loads).toHaveLength(2); + expect(mappings).toHaveLength(2); + for (const { evidence } of [ + ...workers.flatMap(({ observations }) => observations), + ...loads, + ...mappings, + ]) { + expect(evidence).toMatchObject({ + confidence: "medium", + coverage: { status: "partial" }, + artifact: { + available: true, + sha256: createHash("sha256").update(source).digest("hex"), + }, + limitations: expect.arrayContaining([ + expect.stringContaining("package loader mode is unknown"), + ]), + }); + } + }, +); + +cliTest.for(["CLI", "stdio MCP"] as const)( + "retains loader evidence for compiler dotfiles and semantic calls through %s", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-bom-semantic-"); + const callSource = + 'function marker(value) { return value; } marker(42); fetch("https://example.test");'; + await writeFixtureFiles(root, { + "package.json": '{"type":"commonjs"}', + ".jsx": '\uFEFF#!/usr/bin/env node\nimport("./dep.mjs");', + ".tsx": '\uFEFF#!/usr/bin/env node\nimport("./dep.mjs");', + "main.cjs": `\uFEFF#!/usr/bin/env node\n${callSource}`, + "plain.cjs": callSource, + "dep.mjs": "export {};", + }); + const { graph, semantic_graph: semantic } = + await analyzeJavaScriptDirectory(surface, cli, root); + for (const path of [".jsx", ".tsx"]) { + const edge = graph.edges.find( + ({ relation, evidence }) => + relation === "imports" && + evidence.location.available && + evidence.location.value.kind === "source-range" && + evidence.location.value.source === path, + ); + expect.soft(edge?.evidence).toMatchObject({ + coverage: { status: "partial" }, + confidence: "medium", + limitations: expect.arrayContaining([ + expect.stringContaining("BOM/hashbang"), + ]), + }); + } + for (const path of ["main.cjs", "plain.cjs"]) { + const atSource = ({ + evidence, + }: { + evidence: (typeof graph.edges)[number]["evidence"]; + }) => + evidence.location.available && + evidence.location.value.kind === "source-range" && + evidence.location.value.source === path; + const endpoint = graph.nodes.find( + ({ kind, observations }) => + kind === "endpoint" && observations.some(atSource), + ); + const observation = endpoint?.observations.find(atSource); + const edge = graph.edges.find( + (edge) => + edge.relation === "calls" && + edge.target_node_id === endpoint?.node_id && + atSource(edge), + ); + expect(observation).toBeDefined(); + expect(edge).toBeDefined(); + if (path === "main.cjs") { + for (const fact of [observation, edge]) + expect.soft(fact?.evidence).toMatchObject({ + coverage: { status: "partial" }, + confidence: "medium", + limitations: expect.arrayContaining([ + expect.stringContaining("BOM/hashbang"), + ]), + }); + } else { + expect.soft(edge?.evidence).toMatchObject({ + confidence: "high", + coverage: { status: "complete" }, + }); + expect + .soft(observation?.evidence.limitations) + .not.toContainEqual(expect.stringContaining("BOM/hashbang")); + } + } + for (const { path, status } of [ + { path: "main.cjs", status: "partial" }, + { path: "plain.cjs", status: "complete" }, + ]) { + const relations = semantic.relations.filter( + ({ evidence }) => + evidence.location.available && + evidence.location.value.kind === "source-range" && + evidence.location.value.source === path, + ); + expect(relations.length).toBeGreaterThan(0); + for (const { evidence } of relations) { + const context = semantic.evidence_contexts.find( + ({ context_id }) => context_id === evidence.context_id, + ); + expect.soft(context?.coverage.status).toBe(status); + if (status === "partial") + expect + .soft(context?.limitations) + .toContainEqual(expect.stringContaining("BOM/hashbang")); + } + } + }, +);