refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy (#10728)

* refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy

* fix(domain): keep the leading-BOM behaviour every migrated UTF-8 decoder had

* refactor(domain): read the domain layer's strict UTF-8 through the BOM-policy owner (#10803)

Twelve domain decoders built their own fatal TextDecoder with ignoreBOM: true. They now use strictUtf8Decoder("preserve"), which keeps that behaviour: each reads bytes where a leading U+FEFF is content (binary-format names and fields, JSON and XML read byte-for-byte, recorded bodies, registry seeds whose kept bytes are re-emitted). The counted baseline drops from 74 to 62.

* refactor(artifacts): read artifact readers' strict UTF-8 through the BOM-policy owner (#10871)

Fourteen artifact reader modules (17 sites) built their own fatal TextDecoder. They now take it from strictUtf8Decoder with the policy each had: every one kept a leading BOM (ignoreBOM: true), so they use "preserve"; JavaScriptArtifactFiles keeps it for JavaScript and HTML entries and strips it otherwise, as before. The strict-utf8-decoder baseline drops those 17 sites.

* refactor(domain): drop TextDecoder imports left by the strict UTF-8 migration

* refactor(application): read MCP registration metadata through the strict UTF-8 owner

* refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner (#10903)

* refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner

* refactor(formats): drop the model readers' unused TextDecoder imports

* refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner (#10935)

* refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner

* refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner (#10936)

* refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner

* refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner (#10937)

* refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner

* refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner (#10938)

* refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner

* refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner (#10939)

* refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner

* refactor(evm): read the hex bytecode carrier's strict UTF-8 through the owner and refuse new local decoders (#10940)

* refactor(evm): read the hex bytecode carrier's strict UTF-8 through the BOM-policy owner

* chore(architecture): match local strict UTF-8 decoders in any option order
This commit is contained in:
morluto authored and GitHub committed 2026-11-15 19:54:39 +08:00
1 parent abd36dd250
commit cabd56ea71
103 files changed
+322 -251

No files matched your search

+2 -3
View File
@@ -17,6 +17,7 @@ import {
MAXIMUM_ASAR_MAX_HEADER_BYTES,
} from "../domain/analysisResourceLimits.js";
import { parseAsarListingAdmission } from "../config/asarListingAdmission.js";
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
/**
* Resolve the admission from a composition root's selected environment; an
@@ -435,9 +436,7 @@ export const readAsarHeader = async (
throw new ArtifactReaderFailure("format", "Truncated ASAR JSON header");
let json: string;
try {
json = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes,
);
json = strictUtf8Decoder("preserve").decode(bytes);
} catch (cause: unknown) {
throw new ArtifactReaderFailure(
"format",
+2 -1
View File
@@ -23,13 +23,14 @@ import {
type PositionalArtifactEntries,
type PositionalArtifactEntry,
} from "./PositionalArtifactReader.js";
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
const PACK_DIR_ENCRYPTED = 1;
const PACK_REL_FILEBASE = 2;
const PACK_SPARSE_BUNDLE = 4;
const PACK_FILE_ENCRYPTED = 1;
const DIRECTORY_CHUNK_BYTES = 64 * 1024;
const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const UTF8 = strictUtf8Decoder("preserve");
/** One descriptor, one validated directory, and bounded positional member reads. */
export class GodotPckArtifactReader extends PositionalArtifactReader {
+2 -1
View File
@@ -21,6 +21,7 @@ import {
UNITYFS_SIGNATURE,
unityFsLayoutFlags,
} from "../domain/unityfsFormat.js";
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
/** Per-decode resource limits, not format limits. Plain blocks are streamed. */
const MAX_BLOCK_UNCOMPRESSED_BYTES = 64 * 1024 * 1024;
@@ -49,7 +50,7 @@ class UnityFsDirectoryBudget {
}
/** Signature, version, two bounded strings and the fixed tail; reads are bounded. */
const HEADER_PREFIX_MAX = 8 + 4 + UNITYFS_HEADER_STRING_MAX * 2 + 20;
const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const UTF8 = strictUtf8Decoder("preserve");
export interface UnityFsHeader {
readonly formatVersion: number;
+2 -1
View File
@@ -15,6 +15,7 @@ import {
type WxapkgIndexLayout,
type WxapkgIndexLayoutFailure,
} from "../domain/wxapkgFormat.js";
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
/** The member table is a flat name/offset/size row list. */
const MAX_TABLE_BYTES = 16 * 1024 * 1024;
@@ -25,7 +26,7 @@ const MAX_MEMBER_NAME_BYTES = 64 * 1024;
/** Detecting prefix covers the fixed header, table head and encrypted magic. */
const HEADER_PREFIX_BYTES = 64;
const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const UTF8 = strictUtf8Decoder("preserve");
export const wxapkgFormatFailure = (detail: string): ArtifactReaderFailure =>
new ArtifactReaderFailure("format", `wxapkg is malformed: ${detail}`);
+2 -4
View File
@@ -52,12 +52,10 @@ import { artifactReaderLimitFailure } from "./ArtifactReaderLimits.js";
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { readFileHandleRange } from "../filesystem/readFileHandleRange.js";
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
import { invariantViolation } from "../domain/internalInvariant.js";
const ZIP_UTF8_DECODER = new TextDecoder("utf-8", {
fatal: true,
ignoreBOM: true,
});
const ZIP_UTF8_DECODER = strictUtf8Decoder("preserve");
/** Fixed method-93 decoder window: 2^27 bytes, matching the ELF/GPU decoders. */
const ZIP_ZSTANDARD_WINDOW_LOG_MAX = 27;
@@ -3,7 +3,6 @@ import {
isHostLimitFailure,
ParseFailureError,
} from "../../domain/parseFailure.js";
import { TextDecoder } from "node:util";
import { InternalInvariantError } from "../../domain/internalInvariant.js";
import { decodeBinaryPropertyList } from "./BinaryPlistMaterializer.js";
@@ -72,6 +71,7 @@ import {
hasBinaryPlistSignature,
hasNibArchiveSignature,
} from "../../domain/apple/appleArchiveFormat.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
const MAX_AGGREGATE_INPUT_BYTES = 32 * 1024 * 1024;
const MAX_AGGREGATE_DECODE_BYTES = 256 * 1024 * 1024;
@@ -730,7 +730,7 @@ const projectNibArchive = (
try {
// NSString's NS.bytes representation is UTF-8. A failed conversion
// does not establish a replacement-character class, title or selector.
value = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
value = strictUtf8Decoder("preserve").decode(
Buffer.from(encoded, "base64"),
);
} catch {
@@ -45,6 +45,7 @@ import {
type FatSliceDeclaration,
type FatSliceRangeIssue,
} from "../../domain/apple/machoContainer.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
/** Read `length` bytes at `offset`; shorter results mean end of file. */
export type ReadAt = (offset: number, length: number) => Promise<Uint8Array>;
@@ -1043,9 +1044,7 @@ const payloadString = (
chargeLazyString(work, end - offset + 1);
let text: string;
try {
text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
payload.subarray(offset, end),
);
text = strictUtf8Decoder("preserve").decode(payload.subarray(offset, end));
} catch {
throw machoIssue(
"malformed",
@@ -1116,9 +1115,7 @@ const throwCommandStringIssue = (
const decodeCommandString = (bytes: Uint8Array, index: number): string => {
try {
// lc_str contains name/path data, so a leading U+FEFF is part of its identity.
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes,
);
return strictUtf8Decoder("preserve").decode(bytes);
} catch {
throw machoIssue(
"malformed",
+2 -4
View File
@@ -1,7 +1,6 @@
import type { JsonValue } from "../../domain/jsonValue.js";
import { hasNibArchiveSignature } from "../../domain/apple/appleArchiveFormat.js";
import { isInterfaceBuilderConnectionClass } from "../../domain/apple/interfaceBuilderKeyedArchive.js";
import { TextDecoder } from "node:util";
import {
malformedInput,
truncatedInput,
@@ -11,6 +10,7 @@ import {
InterfaceBuilderDecodeBudget,
InterfaceBuilderDecodeBudgetExceeded,
} from "./InterfaceBuilderDecodeBudget.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
const DEFAULT_DECODE_BUDGET_BYTES = 128 * 1024 * 1024;
@@ -629,9 +629,7 @@ const decodeUtf8 = (
): string => {
try {
// These are length-prefixed names, not standalone text documents.
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes.subarray(start, end),
);
return strictUtf8Decoder("preserve").decode(bytes.subarray(start, end));
} catch (cause: unknown) {
throw malformedInput(
"nib-archive",
+3 -7
View File
@@ -1,10 +1,9 @@
import { TextDecoder } from "node:util";
import { fixedAllowanceRemediation } from "../../domain/fixedAllowanceRemediation.js";
import { jsonValueSchema, type JsonValue } from "../../domain/jsonValue.js";
import { abortIfNeeded } from "../ArtifactHash.js";
import { ArtifactReaderFailure } from "../ArtifactReader.js";
import { walkDenoVirtualFs, type DenoVfsWalk } from "./DenoVirtualFs.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
/** Opening and closing marker of a Deno compile data section. */
export const DENO_SECTION_MAGIC = Buffer.from("d3n0l4nd", "latin1");
@@ -20,11 +19,8 @@ const FLAG_TRANSPILED = 1 << 0;
const FLAG_SOURCE_MAP = 1 << 1;
const FLAG_CJS = 1 << 2;
const FLAG_UTF8 = 1 << 3;
const specifierDecoder = new TextDecoder("utf-8", {
fatal: true,
ignoreBOM: true,
});
const jsonDecoder = new TextDecoder("utf-8", { fatal: true });
const specifierDecoder = strictUtf8Decoder("preserve");
const jsonDecoder = strictUtf8Decoder("strip-signature");
const MEDIA_TYPE_NAMES = [
"JavaScript",
+2 -4
View File
@@ -15,6 +15,7 @@ import {
ArtifactReaderFailure,
artifactFixedLimitFailure,
} from "../ArtifactReader.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
/** SHA-256 marker embedded by Microsoft's apphost; the preceding UInt64 is the header offset. */
export const DOTNET_BUNDLE_SIGNATURE = Buffer.from(
@@ -285,10 +286,7 @@ class BundleCursor {
// decoder boundary so I/O and cancellation retain their original cause.
const bytes = await this.take(length);
try {
return new TextDecoder("utf-8", {
fatal: true,
ignoreBOM: true,
}).decode(bytes);
return strictUtf8Decoder("preserve").decode(bytes);
} catch (cause) {
throw new ArtifactReaderFailure(
"format",
+2 -1
View File
@@ -21,6 +21,7 @@ import {
} from "../../firmware/devicetree/DevicetreeParser.js";
import { BinaryFormatFailure } from "../../native/metadata/BinaryContainer.js";
import { ArtifactReaderFailure } from "../ArtifactReader.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
// Fixed resource bounds; configurable ceilings are tracked in #6176.
const FIT_MAX_IMAGES = 1024;
@@ -115,7 +116,7 @@ const valueOf = (fdt: Buffer, property: FitProperty): Buffer =>
fdt.subarray(property.offset, property.offset + property.bytes);
// A property value is data: a leading U+FEFF is not a file encoding signature.
const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const utf8 = strictUtf8Decoder("preserve");
const nulCount = (raw: Buffer): number => {
let count = 0;
for (const byte of raw) if (byte === 0) count += 1;
@@ -30,6 +30,7 @@ import type {
JavaScriptUnanalyzedArtifactLookup,
JavaScriptArtifactFileSet,
} from "../../domain/javascript/javascriptArtifactFiles.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
interface ExpectedFile {
readonly path: string;
@@ -388,12 +389,11 @@ const readText = async (
try {
return {
included: true,
value: new TextDecoder("utf-8", {
fatal: true,
ignoreBOM:
input.expected.kind === "javascript" ||
input.expected.kind === "html",
}).decode(bytes),
value: strictUtf8Decoder(
input.expected.kind === "javascript" || input.expected.kind === "html"
? "preserve"
: "strip-signature",
).decode(bytes),
};
} catch (cause: unknown) {
// Non-UTF8 bytes are counted; the fixed reason preserves the schema.
+2 -3
View File
@@ -7,6 +7,7 @@ import {
type NodeSeaReaderLimits,
} from "../../domain/artifactReaderLimits.js";
import { artifactReaderLimitFailure } from "../ArtifactReaderLimits.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
/** Node `SeaBlob::kMagic` from `src/node_sea.h` (v24). */
const NODE_SEA_MAGIC = 0x143da20;
@@ -231,9 +232,7 @@ class BlobCursor {
const raw = this.#demand(length);
let text: string;
try {
text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
raw,
);
text = strictUtf8Decoder("preserve").decode(raw);
} catch (cause: unknown) {
if (cause instanceof RangeError)
throw new ArtifactReaderFailure(
+3 -6
View File
@@ -6,6 +6,7 @@ import {
pyInstallerFailure,
type PyInstallerFailure,
} from "./PyInstallerFailure.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
/** Magic at the start of a CArchive cookie. */
const PYINSTALLER_COOKIE_MAGIC = Buffer.from("MEI\x0c\x0b\x0a\x0b\x0e");
@@ -299,9 +300,7 @@ const libraryName = (
);
if (text.length === 0) return ok(null);
try {
return ok(
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(text),
);
return ok(strictUtf8Decoder("preserve").decode(text));
} catch {
return err(
pyInstallerFailure(
@@ -318,9 +317,7 @@ const tocName = (bytes: Buffer): Result<string, PyInstallerFailure> => {
if (text.length === 0)
return err(pyInstallerFailure("format", "CArchive entry name is empty"));
try {
return ok(
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(text),
);
return ok(strictUtf8Decoder("preserve").decode(text));
} catch {
return err(
pyInstallerFailure("format", "CArchive entry name is not valid UTF-8"),