diff --git a/docs/cli.md b/docs/cli.md index 87792e7c0a..51a45c4628 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -147,6 +147,10 @@ from observations of the current app. File names do not automatically exclude files. Set `REA_REFERENCE_SECRET_PATTERNS_JSON` to a JSON array of ignore patterns when you want to exclude selected paths. +JavaScript and TypeScript import parsing requires valid UTF-8. Malformed source +bytes retain their original hashes and sizes with a decoding diagnostic; REA +does not infer module targets from replacement characters. + Historical-source import requires safe no-follow file opens on Linux or macOS. Native Windows returns `unsupported_host`; use Linux REA inside WSL or another supported host. See diff --git a/src/domain/referenceSourceImportParsing.ts b/src/domain/referenceSourceImportParsing.ts index 70097297a8..7f9ad996af 100644 --- a/src/domain/referenceSourceImportParsing.ts +++ b/src/domain/referenceSourceImportParsing.ts @@ -331,7 +331,25 @@ export const parseReferenceSourceImports = ( if (language !== null && !codeSourceLanguages.has(language)) return { relationships: [], parse_failures: [] }; - const source = new TextDecoder("utf-8", { fatal: false }).decode(bytes); + let source: string; + try { + source = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch (cause: unknown) { + // The import still records the original bytes; lossy text cannot supply + // observed module names or relationships. + void cause; + return { + relationships: [], + parse_failures: [ + { + path, + parser: "utf-8", + reason: + "Source bytes are not valid UTF-8; import targets were not parsed.", + }, + ], + }; + } const { ast, reasons } = parseWithBabel(path, source, language); const parseFailures = reasons.map((reason) => ({ path, diff --git a/tests/boundary/cli/referenceSourceUtf8.test.ts b/tests/boundary/cli/referenceSourceUtf8.test.ts new file mode 100644 index 0000000000..06093f427e --- /dev/null +++ b/tests/boundary/cli/referenceSourceUtf8.test.ts @@ -0,0 +1,77 @@ +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect } from "vitest"; + +import { historicalSourceGraphSchema } from "../../../src/domain/referenceSourceGraph.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "retains malformed source bytes without inventing decoded import targets", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-reference-utf8-"); + const sources = new Map([ + [ + "invalid.mjs", + Buffer.concat([ + Buffer.from('import "./caf'), + Buffer.from([0xff]), + Buffer.from('.js";\n'), + ]), + ], + [ + "comment.ts", + Buffer.concat([ + Buffer.from('import "./dep.js"; // '), + Buffer.from([0xc0, 0xaf]), + ]), + ], + ["valid.mjs", Buffer.from('import "./caf�.js";\n')], + ["caf�.js", Buffer.from("export const value = 1;\n")], + ["dep.js", Buffer.from("export const value = 2;\n")], + ]); + await Promise.all( + [...sources].map(([path, bytes]) => writeFile(join(root, path), bytes)), + ); + const result = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + environment: { HOME: root, XDG_CONFIG_HOME: root, XDG_CACHE_HOME: root }, + }); + expect(result.exitCode).toBe(0); + const graph = historicalSourceGraphSchema.parse(result.json); + expect(graph.parse_failures).toEqual([ + { + path: "comment.ts", + parser: "utf-8", + reason: + "Source bytes are not valid UTF-8; import targets were not parsed.", + }, + { + path: "invalid.mjs", + parser: "utf-8", + reason: + "Source bytes are not valid UTF-8; import targets were not parsed.", + }, + ]); + expect(graph.relationships).toEqual([ + { + from_path: "valid.mjs", + to: "caf�.js", + kind: "imports", + resolution: "internal", + parse_state: "parsed", + }, + ]); + for (const [path, bytes] of sources) + expect(graph.entries).toContainEqual( + expect.objectContaining({ + path, + sha256: createHash("sha256").update(bytes).digest("hex"), + size: bytes.length, + content_state: "hashed", + }), + ); + }, +);