mirror of
https://github.com/morluto/rea.git
synced 2026-11-16 03:33:25 +00:00
refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy (#10728)
* refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy * fix(domain): keep the leading-BOM behaviour every migrated UTF-8 decoder had * refactor(domain): read the domain layer's strict UTF-8 through the BOM-policy owner (#10803) Twelve domain decoders built their own fatal TextDecoder with ignoreBOM: true. They now use strictUtf8Decoder("preserve"), which keeps that behaviour: each reads bytes where a leading U+FEFF is content (binary-format names and fields, JSON and XML read byte-for-byte, recorded bodies, registry seeds whose kept bytes are re-emitted). The counted baseline drops from 74 to 62. * refactor(artifacts): read artifact readers' strict UTF-8 through the BOM-policy owner (#10871) Fourteen artifact reader modules (17 sites) built their own fatal TextDecoder. They now take it from strictUtf8Decoder with the policy each had: every one kept a leading BOM (ignoreBOM: true), so they use "preserve"; JavaScriptArtifactFiles keeps it for JavaScript and HTML entries and strips it otherwise, as before. The strict-utf8-decoder baseline drops those 17 sites. * refactor(domain): drop TextDecoder imports left by the strict UTF-8 migration * refactor(application): read MCP registration metadata through the strict UTF-8 owner * refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner (#10903) * refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner * refactor(formats): drop the model readers' unused TextDecoder imports * refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner (#10935) * refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner * refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner (#10936) * refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner * refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner (#10937) * refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner * refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner (#10938) * refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner * refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner (#10939) * refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner * refactor(evm): read the hex bytecode carrier's strict UTF-8 through the owner and refuse new local decoders (#10940) * refactor(evm): read the hex bytecode carrier's strict UTF-8 through the BOM-policy owner * chore(architecture): match local strict UTF-8 decoders in any option order
This commit is contained in:
1 parent
abd36dd250
commit
cabd56ea71
103 files changed
+322
-251
No files matched your search
@@ -17,6 +17,7 @@ import {
|
||||
MAXIMUM_ASAR_MAX_HEADER_BYTES,
|
||||
} from "../domain/analysisResourceLimits.js";
|
||||
import { parseAsarListingAdmission } from "../config/asarListingAdmission.js";
|
||||
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
|
||||
|
||||
/**
|
||||
* Resolve the admission from a composition root's selected environment; an
|
||||
@@ -435,9 +436,7 @@ export const readAsarHeader = async (
|
||||
throw new ArtifactReaderFailure("format", "Truncated ASAR JSON header");
|
||||
let json: string;
|
||||
try {
|
||||
json = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
|
||||
bytes,
|
||||
);
|
||||
json = strictUtf8Decoder("preserve").decode(bytes);
|
||||
} catch (cause: unknown) {
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
|
||||
@@ -23,13 +23,14 @@ import {
|
||||
type PositionalArtifactEntries,
|
||||
type PositionalArtifactEntry,
|
||||
} from "./PositionalArtifactReader.js";
|
||||
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
|
||||
|
||||
const PACK_DIR_ENCRYPTED = 1;
|
||||
const PACK_REL_FILEBASE = 2;
|
||||
const PACK_SPARSE_BUNDLE = 4;
|
||||
const PACK_FILE_ENCRYPTED = 1;
|
||||
const DIRECTORY_CHUNK_BYTES = 64 * 1024;
|
||||
const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
||||
const UTF8 = strictUtf8Decoder("preserve");
|
||||
|
||||
/** One descriptor, one validated directory, and bounded positional member reads. */
|
||||
export class GodotPckArtifactReader extends PositionalArtifactReader {
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
UNITYFS_SIGNATURE,
|
||||
unityFsLayoutFlags,
|
||||
} from "../domain/unityfsFormat.js";
|
||||
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
|
||||
|
||||
/** Per-decode resource limits, not format limits. Plain blocks are streamed. */
|
||||
const MAX_BLOCK_UNCOMPRESSED_BYTES = 64 * 1024 * 1024;
|
||||
@@ -49,7 +50,7 @@ class UnityFsDirectoryBudget {
|
||||
}
|
||||
/** Signature, version, two bounded strings and the fixed tail; reads are bounded. */
|
||||
const HEADER_PREFIX_MAX = 8 + 4 + UNITYFS_HEADER_STRING_MAX * 2 + 20;
|
||||
const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
||||
const UTF8 = strictUtf8Decoder("preserve");
|
||||
|
||||
export interface UnityFsHeader {
|
||||
readonly formatVersion: number;
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
type WxapkgIndexLayout,
|
||||
type WxapkgIndexLayoutFailure,
|
||||
} from "../domain/wxapkgFormat.js";
|
||||
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
|
||||
|
||||
/** The member table is a flat name/offset/size row list. */
|
||||
const MAX_TABLE_BYTES = 16 * 1024 * 1024;
|
||||
@@ -25,7 +26,7 @@ const MAX_MEMBER_NAME_BYTES = 64 * 1024;
|
||||
/** Detecting prefix covers the fixed header, table head and encrypted magic. */
|
||||
const HEADER_PREFIX_BYTES = 64;
|
||||
|
||||
const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
||||
const UTF8 = strictUtf8Decoder("preserve");
|
||||
|
||||
export const wxapkgFormatFailure = (detail: string): ArtifactReaderFailure =>
|
||||
new ArtifactReaderFailure("format", `wxapkg is malformed: ${detail}`);
|
||||
|
||||
@@ -52,12 +52,10 @@ import { artifactReaderLimitFailure } from "./ArtifactReaderLimits.js";
|
||||
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { readFileHandleRange } from "../filesystem/readFileHandleRange.js";
|
||||
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
|
||||
import { invariantViolation } from "../domain/internalInvariant.js";
|
||||
|
||||
const ZIP_UTF8_DECODER = new TextDecoder("utf-8", {
|
||||
fatal: true,
|
||||
ignoreBOM: true,
|
||||
});
|
||||
const ZIP_UTF8_DECODER = strictUtf8Decoder("preserve");
|
||||
|
||||
/** Fixed method-93 decoder window: 2^27 bytes, matching the ELF/GPU decoders. */
|
||||
const ZIP_ZSTANDARD_WINDOW_LOG_MAX = 27;
|
||||
|
||||
@@ -3,7 +3,6 @@ import {
|
||||
isHostLimitFailure,
|
||||
ParseFailureError,
|
||||
} from "../../domain/parseFailure.js";
|
||||
import { TextDecoder } from "node:util";
|
||||
import { InternalInvariantError } from "../../domain/internalInvariant.js";
|
||||
|
||||
import { decodeBinaryPropertyList } from "./BinaryPlistMaterializer.js";
|
||||
@@ -72,6 +71,7 @@ import {
|
||||
hasBinaryPlistSignature,
|
||||
hasNibArchiveSignature,
|
||||
} from "../../domain/apple/appleArchiveFormat.js";
|
||||
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
|
||||
|
||||
const MAX_AGGREGATE_INPUT_BYTES = 32 * 1024 * 1024;
|
||||
const MAX_AGGREGATE_DECODE_BYTES = 256 * 1024 * 1024;
|
||||
@@ -730,7 +730,7 @@ const projectNibArchive = (
|
||||
try {
|
||||
// NSString's NS.bytes representation is UTF-8. A failed conversion
|
||||
// does not establish a replacement-character class, title or selector.
|
||||
value = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
|
||||
value = strictUtf8Decoder("preserve").decode(
|
||||
Buffer.from(encoded, "base64"),
|
||||
);
|
||||
} catch {
|
||||
|
||||
@@ -45,6 +45,7 @@ import {
|
||||
type FatSliceDeclaration,
|
||||
type FatSliceRangeIssue,
|
||||
} from "../../domain/apple/machoContainer.js";
|
||||
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
|
||||
|
||||
/** Read `length` bytes at `offset`; shorter results mean end of file. */
|
||||
export type ReadAt = (offset: number, length: number) => Promise<Uint8Array>;
|
||||
@@ -1043,9 +1044,7 @@ const payloadString = (
|
||||
chargeLazyString(work, end - offset + 1);
|
||||
let text: string;
|
||||
try {
|
||||
text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
|
||||
payload.subarray(offset, end),
|
||||
);
|
||||
text = strictUtf8Decoder("preserve").decode(payload.subarray(offset, end));
|
||||
} catch {
|
||||
throw machoIssue(
|
||||
"malformed",
|
||||
@@ -1116,9 +1115,7 @@ const throwCommandStringIssue = (
|
||||
const decodeCommandString = (bytes: Uint8Array, index: number): string => {
|
||||
try {
|
||||
// lc_str contains name/path data, so a leading U+FEFF is part of its identity.
|
||||
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
|
||||
bytes,
|
||||
);
|
||||
return strictUtf8Decoder("preserve").decode(bytes);
|
||||
} catch {
|
||||
throw machoIssue(
|
||||
"malformed",
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import type { JsonValue } from "../../domain/jsonValue.js";
|
||||
import { hasNibArchiveSignature } from "../../domain/apple/appleArchiveFormat.js";
|
||||
import { isInterfaceBuilderConnectionClass } from "../../domain/apple/interfaceBuilderKeyedArchive.js";
|
||||
import { TextDecoder } from "node:util";
|
||||
import {
|
||||
malformedInput,
|
||||
truncatedInput,
|
||||
@@ -11,6 +10,7 @@ import {
|
||||
InterfaceBuilderDecodeBudget,
|
||||
InterfaceBuilderDecodeBudgetExceeded,
|
||||
} from "./InterfaceBuilderDecodeBudget.js";
|
||||
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
|
||||
|
||||
const DEFAULT_DECODE_BUDGET_BYTES = 128 * 1024 * 1024;
|
||||
|
||||
@@ -629,9 +629,7 @@ const decodeUtf8 = (
|
||||
): string => {
|
||||
try {
|
||||
// These are length-prefixed names, not standalone text documents.
|
||||
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
|
||||
bytes.subarray(start, end),
|
||||
);
|
||||
return strictUtf8Decoder("preserve").decode(bytes.subarray(start, end));
|
||||
} catch (cause: unknown) {
|
||||
throw malformedInput(
|
||||
"nib-archive",
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
import { TextDecoder } from "node:util";
|
||||
|
||||
import { fixedAllowanceRemediation } from "../../domain/fixedAllowanceRemediation.js";
|
||||
import { jsonValueSchema, type JsonValue } from "../../domain/jsonValue.js";
|
||||
import { abortIfNeeded } from "../ArtifactHash.js";
|
||||
import { ArtifactReaderFailure } from "../ArtifactReader.js";
|
||||
import { walkDenoVirtualFs, type DenoVfsWalk } from "./DenoVirtualFs.js";
|
||||
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
|
||||
|
||||
/** Opening and closing marker of a Deno compile data section. */
|
||||
export const DENO_SECTION_MAGIC = Buffer.from("d3n0l4nd", "latin1");
|
||||
@@ -20,11 +19,8 @@ const FLAG_TRANSPILED = 1 << 0;
|
||||
const FLAG_SOURCE_MAP = 1 << 1;
|
||||
const FLAG_CJS = 1 << 2;
|
||||
const FLAG_UTF8 = 1 << 3;
|
||||
const specifierDecoder = new TextDecoder("utf-8", {
|
||||
fatal: true,
|
||||
ignoreBOM: true,
|
||||
});
|
||||
const jsonDecoder = new TextDecoder("utf-8", { fatal: true });
|
||||
const specifierDecoder = strictUtf8Decoder("preserve");
|
||||
const jsonDecoder = strictUtf8Decoder("strip-signature");
|
||||
|
||||
const MEDIA_TYPE_NAMES = [
|
||||
"JavaScript",
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
ArtifactReaderFailure,
|
||||
artifactFixedLimitFailure,
|
||||
} from "../ArtifactReader.js";
|
||||
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
|
||||
|
||||
/** SHA-256 marker embedded by Microsoft's apphost; the preceding UInt64 is the header offset. */
|
||||
export const DOTNET_BUNDLE_SIGNATURE = Buffer.from(
|
||||
@@ -285,10 +286,7 @@ class BundleCursor {
|
||||
// decoder boundary so I/O and cancellation retain their original cause.
|
||||
const bytes = await this.take(length);
|
||||
try {
|
||||
return new TextDecoder("utf-8", {
|
||||
fatal: true,
|
||||
ignoreBOM: true,
|
||||
}).decode(bytes);
|
||||
return strictUtf8Decoder("preserve").decode(bytes);
|
||||
} catch (cause) {
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
} from "../../firmware/devicetree/DevicetreeParser.js";
|
||||
import { BinaryFormatFailure } from "../../native/metadata/BinaryContainer.js";
|
||||
import { ArtifactReaderFailure } from "../ArtifactReader.js";
|
||||
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
|
||||
|
||||
// Fixed resource bounds; configurable ceilings are tracked in #6176.
|
||||
const FIT_MAX_IMAGES = 1024;
|
||||
@@ -115,7 +116,7 @@ const valueOf = (fdt: Buffer, property: FitProperty): Buffer =>
|
||||
fdt.subarray(property.offset, property.offset + property.bytes);
|
||||
|
||||
// A property value is data: a leading U+FEFF is not a file encoding signature.
|
||||
const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
||||
const utf8 = strictUtf8Decoder("preserve");
|
||||
const nulCount = (raw: Buffer): number => {
|
||||
let count = 0;
|
||||
for (const byte of raw) if (byte === 0) count += 1;
|
||||
|
||||
@@ -30,6 +30,7 @@ import type {
|
||||
JavaScriptUnanalyzedArtifactLookup,
|
||||
JavaScriptArtifactFileSet,
|
||||
} from "../../domain/javascript/javascriptArtifactFiles.js";
|
||||
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
|
||||
|
||||
interface ExpectedFile {
|
||||
readonly path: string;
|
||||
@@ -388,12 +389,11 @@ const readText = async (
|
||||
try {
|
||||
return {
|
||||
included: true,
|
||||
value: new TextDecoder("utf-8", {
|
||||
fatal: true,
|
||||
ignoreBOM:
|
||||
input.expected.kind === "javascript" ||
|
||||
input.expected.kind === "html",
|
||||
}).decode(bytes),
|
||||
value: strictUtf8Decoder(
|
||||
input.expected.kind === "javascript" || input.expected.kind === "html"
|
||||
? "preserve"
|
||||
: "strip-signature",
|
||||
).decode(bytes),
|
||||
};
|
||||
} catch (cause: unknown) {
|
||||
// Non-UTF8 bytes are counted; the fixed reason preserves the schema.
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
type NodeSeaReaderLimits,
|
||||
} from "../../domain/artifactReaderLimits.js";
|
||||
import { artifactReaderLimitFailure } from "../ArtifactReaderLimits.js";
|
||||
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
|
||||
|
||||
/** Node `SeaBlob::kMagic` from `src/node_sea.h` (v24). */
|
||||
const NODE_SEA_MAGIC = 0x143da20;
|
||||
@@ -231,9 +232,7 @@ class BlobCursor {
|
||||
const raw = this.#demand(length);
|
||||
let text: string;
|
||||
try {
|
||||
text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
|
||||
raw,
|
||||
);
|
||||
text = strictUtf8Decoder("preserve").decode(raw);
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof RangeError)
|
||||
throw new ArtifactReaderFailure(
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
pyInstallerFailure,
|
||||
type PyInstallerFailure,
|
||||
} from "./PyInstallerFailure.js";
|
||||
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
|
||||
|
||||
/** Magic at the start of a CArchive cookie. */
|
||||
const PYINSTALLER_COOKIE_MAGIC = Buffer.from("MEI\x0c\x0b\x0a\x0b\x0e");
|
||||
@@ -299,9 +300,7 @@ const libraryName = (
|
||||
);
|
||||
if (text.length === 0) return ok(null);
|
||||
try {
|
||||
return ok(
|
||||
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(text),
|
||||
);
|
||||
return ok(strictUtf8Decoder("preserve").decode(text));
|
||||
} catch {
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
@@ -318,9 +317,7 @@ const tocName = (bytes: Buffer): Result<string, PyInstallerFailure> => {
|
||||
if (text.length === 0)
|
||||
return err(pyInstallerFailure("format", "CArchive entry name is empty"));
|
||||
try {
|
||||
return ok(
|
||||
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(text),
|
||||
);
|
||||
return ok(strictUtf8Decoder("preserve").decode(text));
|
||||
} catch {
|
||||
return err(
|
||||
pyInstallerFailure("format", "CArchive entry name is not valid UTF-8"),
|
||||
|
||||
Reference in new issue
Block a user