refactor(parsers): share strict UTF-8 byte validation (#12281)

This commit is contained in:
morluto authored and GitHub committed 2026-11-19 11:41:02 +00:00
1 parent 6fdfdc4af1
commit 510301969e
5 files changed
+24 -12

No files matched your search

+3 -2
View File
@@ -1,10 +1,11 @@
import { isStrictUtf8 } from "./strictUtf8.js";
/** Preserve stored bytes; undecodable targets have no invented text spelling. */
export const observedSymlinkTarget = (
raw: Buffer,
): { text: string | null; hex: string } => {
const decoded = raw.toString("utf8");
return {
text: Buffer.from(decoded, "utf8").equals(raw) ? decoded : null,
text: isStrictUtf8(raw) ? raw.toString("utf8") : null,
hex: raw.toString("hex"),
};
};
+4 -3
View File
@@ -1,3 +1,5 @@
import { isStrictUtf8 } from "../domain/strictUtf8.js";
/** A filesystem basename is usable as a logical path only after UTF-8 validity is established. */
export type DirectoryEntryName =
| { readonly kind: "decoded"; readonly name: string }
@@ -8,8 +10,7 @@ export const decodeDirectoryEntryName = (
raw: string | Buffer,
): DirectoryEntryName => {
if (typeof raw === "string") return { kind: "decoded", name: raw };
const name = raw.toString("utf8");
return Buffer.from(name, "utf8").equals(raw)
? { kind: "decoded", name }
return isStrictUtf8(raw)
? { kind: "decoded", name: raw.toString("utf8") }
: { kind: "unrepresentable", rawHex: raw.toString("hex") };
};
+3 -3
View File
@@ -11,6 +11,7 @@ import {
} from "../../domain/native/dwarfUnits.js";
import { DwarfAbbrevCache, type DwarfAbbrevAttribute } from "./DwarfAbbrev.js";
import { DwarfCursor, readInitialLength } from "./DwarfReader.js";
import { isStrictUtf8 } from "../../domain/strictUtf8.js";
const DW_AT_NAME = 0x03;
const DW_AT_STMT_LIST = 0x10;
@@ -620,9 +621,8 @@ const presentString = (
issue: (code: DwarfUnitIssueCode, detail: string) => void,
): DwarfStringFact => {
if (raw === null) return unresolvedString();
const utf8 = raw.toString("utf8");
if (Buffer.from(utf8, "utf8").equals(raw))
return { state: "present", value: utf8 };
if (isStrictUtf8(raw))
return { state: "present", value: raw.toString("utf8") };
issue(
"string_not_utf8",
"String is not valid UTF-8; the value is the raw bytes as Latin-1",
+3 -4
View File
@@ -4,6 +4,7 @@ import {
Asn1BerReader,
type Asn1BerNode,
} from "../../domain/asn1Ber.js";
import { isStrictUtf8 } from "../../domain/strictUtf8.js";
import { byteToHex } from "../../kernel/byteToHex.js";
const OID_NAMES: Readonly<Record<string, string>> = {
@@ -188,11 +189,9 @@ const decodeUtf32Be = (bytes: Buffer, budget: DecodedNameBudget): string => {
const decodeUtf8 = (bytes: Buffer, budget: DecodedNameBudget): string => {
chargeDecodedName(budget, bytes.length);
const decoded = bytes.toString("utf8");
// U+FFFD would hide the original bytes and collapse distinct invalid names.
if (!Buffer.from(decoded, "utf8").equals(bytes))
return fail("DirectoryString is not valid UTF-8.");
return decoded;
if (!isStrictUtf8(bytes)) return fail("DirectoryString is not valid UTF-8.");
return bytes.toString("utf8");
};
const decodeDirectoryString = (
@@ -6,6 +6,17 @@ import { expect, it } from "vitest";
import { scanArtifactInventory } from "../../fixtures/artifactInventory.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
it("keeps a leading UTF-8 signature as part of a directory entry name", async () => {
const root = await createTestTempDirectory("rea-artifact-filename-bom-");
await writeFile(join(root, "\uFEFFname.txt"), "SIGNATURE_IN_NAME");
await writeFile(join(root, "name.txt"), "PLAIN_NAME");
const inventory = await scanArtifactInventory(root);
expect(
inventory.occurrences.map(({ logical_path }) => logical_path).sort(),
).toEqual([".", "name.txt", "\uFEFFname.txt"]);
});
it.skipIf(process.platform !== "linux")(
"keeps invalid UTF-8 filenames distinct from a valid U+FFFD sibling",
async () => {