Files
reverse-engineering-anything/src/domain/referenceSourceImportParsing.ts
T
morluto cabd56ea71 refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy (#10728)
* refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy

* fix(domain): keep the leading-BOM behaviour every migrated UTF-8 decoder had

* refactor(domain): read the domain layer's strict UTF-8 through the BOM-policy owner (#10803)

Twelve domain decoders built their own fatal TextDecoder with ignoreBOM: true. They now use strictUtf8Decoder("preserve"), which keeps that behaviour: each reads bytes where a leading U+FEFF is content (binary-format names and fields, JSON and XML read byte-for-byte, recorded bodies, registry seeds whose kept bytes are re-emitted). The counted baseline drops from 74 to 62.

* refactor(artifacts): read artifact readers' strict UTF-8 through the BOM-policy owner (#10871)

Fourteen artifact reader modules (17 sites) built their own fatal TextDecoder. They now take it from strictUtf8Decoder with the policy each had: every one kept a leading BOM (ignoreBOM: true), so they use "preserve"; JavaScriptArtifactFiles keeps it for JavaScript and HTML entries and strips it otherwise, as before. The strict-utf8-decoder baseline drops those 17 sites.

* refactor(domain): drop TextDecoder imports left by the strict UTF-8 migration

* refactor(application): read MCP registration metadata through the strict UTF-8 owner

* refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner (#10903)

* refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner

* refactor(formats): drop the model readers' unused TextDecoder imports

* refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner (#10935)

* refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner

* refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner (#10936)

* refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner

* refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner (#10937)

* refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner

* refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner (#10938)

* refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner

* refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner (#10939)

* refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner

* refactor(evm): read the hex bytecode carrier's strict UTF-8 through the owner and refuse new local decoders (#10940)

* refactor(evm): read the hex bytecode carrier's strict UTF-8 through the BOM-policy owner

* chore(architecture): match local strict UTF-8 decoders in any option order
2026-11-15 19:54:39 +08:00

356 lines
11 KiB
TypeScript

import type { File, Node } from "@babel/types";
import { traverseJavaScriptAst } from "./javascript/javascriptSemanticTraversal.js";
import { readExactJavaScriptLiteral } from "./javascript/javascriptAstValues.js";
import {
parseRecoveredJavaScript,
parserPluginsForPath,
type JavaScriptPackageType,
} from "./javascript/javascriptSourceParser.js";
import {
referenceSourceRequireCalls,
type ReferenceSourceRequireCall,
type ReferenceSourceRequireInvocation,
} from "./referenceSourceRequireBindings.js";
import { isRelativeJavaScriptModuleSpecifier } from "./javascript/javascriptModuleSpecifier.js";
import {
isCallExpression,
isExportAllDeclaration,
isExportNamedDeclaration,
isImportDeclaration,
isImportExpression,
isNewExpression,
isOptionalCallExpression,
isTSImportType,
isStringLiteral,
isTSExternalModuleReference,
isTSImportEqualsDeclaration,
isTSModuleDeclaration,
} from "@babel/types";
import { decodeStrictUtf8 } from "./strictUtf8.js";
type ReferenceSourceImportKind =
| "imports"
| "requires"
| "references"
| "declares-module";
type ReferenceSourceImportResolution =
| "internal"
| "external"
| "unresolved"
| "unknown";
type ReferenceSourceImportParseState = "parsed" | "partial" | "unknown";
interface ReferenceSourceImportRelationship {
readonly from_path: string;
readonly to: string;
readonly kind: ReferenceSourceImportKind;
readonly resolution: ReferenceSourceImportResolution;
readonly parse_state: ReferenceSourceImportParseState;
/** Distinguishes call-form CommonJS require from TS import-equals `requires`. */
readonly resolutionStrategy?: "runtime-require";
/** Internal lookup origin for a proven createRequire URL base. */
readonly resolutionBasePath?: string;
}
interface ReferenceSourceParseFailure {
readonly path: string;
readonly parser: string;
readonly reason: string;
}
export interface ReferenceSourceImportParseResult {
readonly relationships: readonly ReferenceSourceImportRelationship[];
readonly parse_failures: readonly ReferenceSourceParseFailure[];
readonly limitations: readonly string[];
}
export const PARSEABLE_REFERENCE_SOURCE_LANGUAGES: ReadonlySet<string> =
new Set(["JavaScript", "TypeScript", "JSX", "TSX"]);
const safeModuleName = (node: Node | null | undefined): string | undefined => {
if (isStringLiteral(node)) return node.value;
return undefined;
};
const exactStringModuleName = (
node: Node | null | undefined,
): string | undefined => {
if (node === null || node === undefined) return undefined;
const literal = readExactJavaScriptLiteral(node);
return literal.found && typeof literal.value === "string"
? literal.value
: undefined;
};
const resolveSpecifier = (
specifier: string,
): ReferenceSourceImportResolution => {
if (
isRelativeJavaScriptModuleSpecifier(specifier) ||
specifier.startsWith("/")
)
return "internal";
return "external";
};
const appendRelationship = (
relationships: ReferenceSourceImportRelationship[],
limitations: string[],
input: {
readonly fromPath: string;
readonly to: string | undefined;
readonly kind: ReferenceSourceImportKind;
readonly parseState: ReferenceSourceImportParseState;
readonly resolution?: ReferenceSourceImportResolution;
readonly resolutionStrategy?: "runtime-require";
readonly resolutionBasePath?: string;
},
): void => {
if (input.to === undefined) return;
if (input.to.length === 0) {
limitations.push(
`Empty module specifier in ${input.fromPath} was not recorded as a relationship.`,
);
return;
}
relationships.push({
from_path: input.fromPath,
to: input.to,
kind: input.kind,
resolution: input.resolution ?? resolveSpecifier(input.to),
parse_state: input.parseState,
...(input.resolutionStrategy === undefined
? {}
: { resolutionStrategy: input.resolutionStrategy }),
...(input.resolutionBasePath === undefined
? {}
: { resolutionBasePath: input.resolutionBasePath }),
});
};
const appendImportEquals = (
statement: Node,
from_path: string,
relationships: ReferenceSourceImportRelationship[],
limitations: string[],
): void => {
if (!isTSImportEqualsDeclaration(statement)) return;
const reference = statement.moduleReference;
if (!isTSExternalModuleReference(reference)) return;
appendRelationship(relationships, limitations, {
fromPath: from_path,
to: safeModuleName(reference.expression),
kind: "requires",
parseState: "parsed",
});
};
/** One AST owner retains historical declarations, types and runtime requests. */
const extractImports = (
ast: File,
from_path: string,
relationships: ReferenceSourceImportRelationship[],
limitations: string[],
requireCalls: ReadonlyMap<
ReferenceSourceRequireInvocation,
ReferenceSourceRequireCall
>,
): void => {
let unresolvedRequireTarget = false;
traverseJavaScriptAst(ast.program, {
enter: (node) => {
if (
isImportDeclaration(node) ||
isExportAllDeclaration(node) ||
(isExportNamedDeclaration(node) && node.source != null)
) {
appendRelationship(relationships, limitations, {
fromPath: from_path,
to: safeModuleName(node.source),
kind: "imports",
parseState: "parsed",
});
} else if (isTSImportEqualsDeclaration(node)) {
appendImportEquals(node, from_path, relationships, limitations);
} else if (isTSModuleDeclaration(node)) {
if (node.declare === true && isStringLiteral(node.id))
appendRelationship(relationships, limitations, {
fromPath: from_path,
to: node.id.value,
kind: "declares-module",
parseState: "parsed",
resolution: "unknown",
});
} else if (isImportExpression(node) || isTSImportType(node)) {
const specifier = exactStringModuleName(node.source);
appendRelationship(relationships, limitations, {
fromPath: from_path,
to: specifier ?? "<dynamic-import>",
kind: "imports",
parseState: specifier === undefined ? "partial" : "parsed",
...(specifier === undefined ? { resolution: "unknown" } : {}),
});
} else if (
isCallExpression(node) ||
isOptionalCallExpression(node) ||
isNewExpression(node)
) {
const requireCall = requireCalls.get(node);
if (requireCall === undefined) return;
// Undefined arguments are an opaque normalized list, such as
// `require.apply(null, args)`. An empty list is an explicit call
// with no target.
if (requireCall.arguments === undefined) {
unresolvedRequireTarget = true;
return;
}
const target = requireCall.arguments[0];
if (target === undefined) return;
const literal = readExactJavaScriptLiteral(target);
if (!literal.found) {
unresolvedRequireTarget = true;
return;
}
if (typeof literal.value !== "string") return;
if (
requireCall.limitation !== undefined &&
!limitations.includes(requireCall.limitation)
)
limitations.push(requireCall.limitation);
const uncertainResolution = requireCall.resolutionBase === "unknown";
appendRelationship(relationships, limitations, {
fromPath: from_path,
to: literal.value,
kind: "requires",
parseState: uncertainResolution ? "partial" : "parsed",
...(uncertainResolution ? { resolution: "unknown" } : {}),
...(uncertainResolution
? {}
: { resolutionStrategy: "runtime-require" as const }),
...(typeof requireCall.resolutionBase === "object"
? { resolutionBasePath: requireCall.resolutionBase.path }
: {}),
});
}
},
});
if (unresolvedRequireTarget)
limitations.push(
`A CommonJS module target in ${from_path} could not be determined statically.`,
);
};
const parseWithBabel = (
path: string,
source: string,
packageType: JavaScriptPackageType | undefined,
): {
ast: File | undefined;
reasons: readonly string[];
moduleOnlyErrors: readonly string[];
} => {
// The shared parser owns filename precedence and observed package context.
const plugins = parserPluginsForPath(path);
try {
const { file: ast, moduleOnlyErrors } = parseRecoveredJavaScript(
source,
path,
{
allowImportExportEverywhere: false,
allowReturnOutsideFunction: true,
allowUndeclaredExports: true,
plugins,
},
packageType,
);
return {
ast,
reasons: ast.errors.map((error) => error.message),
moduleOnlyErrors,
};
} catch (cause: unknown) {
return {
ast: undefined,
reasons: [cause instanceof Error ? cause.message : "Parse failed"],
moduleOnlyErrors: [],
};
}
};
/** Parse ESM/CommonJS imports and dynamic imports from source bytes. */
export const parseReferenceSourceImports = (
path: string,
bytes: Uint8Array,
language: string | null,
packageType?: JavaScriptPackageType,
): ReferenceSourceImportParseResult => {
if (language !== null && !PARSEABLE_REFERENCE_SOURCE_LANGUAGES.has(language))
return { relationships: [], parse_failures: [], limitations: [] };
let source: string;
try {
source = decodeStrictUtf8(bytes, "strip-signature");
} catch {
return {
relationships: [],
parse_failures: [
{
path,
parser: "utf-8",
reason:
"Source bytes are not valid UTF-8; import targets were not parsed.",
},
],
limitations: [],
};
}
const { ast, reasons, moduleOnlyErrors } = parseWithBabel(
path,
source,
packageType,
);
const limitations =
moduleOnlyErrors.length === 0
? []
: [
`Strict-mode errors apply only if this file is an ES module: ${moduleOnlyErrors.join("; ")}`,
];
const parseFailures = reasons.map((reason) => ({
path,
parser: "babel",
reason,
}));
if (ast === undefined) {
return {
relationships: [],
parse_failures: parseFailures,
limitations,
};
}
const relationships: ReferenceSourceImportRelationship[] = [];
extractImports(
ast,
path,
relationships,
limitations,
referenceSourceRequireCalls(ast, path),
);
return {
relationships: relationships.map((relationship) => ({
...relationship,
parse_state:
parseFailures.length > 0 && relationship.parse_state === "parsed"
? "partial"
: relationship.parse_state,
})),
parse_failures: parseFailures,
limitations,
};
};