refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy (#10728)

* refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy

* fix(domain): keep the leading-BOM behaviour every migrated UTF-8 decoder had

* refactor(domain): read the domain layer's strict UTF-8 through the BOM-policy owner (#10803)

Twelve domain decoders built their own fatal TextDecoder with ignoreBOM: true. They now use strictUtf8Decoder("preserve"), which keeps that behaviour: each reads bytes where a leading U+FEFF is content (binary-format names and fields, JSON and XML read byte-for-byte, recorded bodies, registry seeds whose kept bytes are re-emitted). The counted baseline drops from 74 to 62.

* refactor(artifacts): read artifact readers' strict UTF-8 through the BOM-policy owner (#10871)

Fourteen artifact reader modules (17 sites) built their own fatal TextDecoder. They now take it from strictUtf8Decoder with the policy each had: every one kept a leading BOM (ignoreBOM: true), so they use "preserve"; JavaScriptArtifactFiles keeps it for JavaScript and HTML entries and strips it otherwise, as before. The strict-utf8-decoder baseline drops those 17 sites.

* refactor(domain): drop TextDecoder imports left by the strict UTF-8 migration

* refactor(application): read MCP registration metadata through the strict UTF-8 owner

* refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner (#10903)

* refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner

* refactor(formats): drop the model readers' unused TextDecoder imports

* refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner (#10935)

* refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner

* refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner (#10936)

* refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner

* refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner (#10937)

* refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner

* refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner (#10938)

* refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner

* refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner (#10939)

* refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner

* refactor(evm): read the hex bytecode carrier's strict UTF-8 through the owner and refuse new local decoders (#10940)

* refactor(evm): read the hex bytecode carrier's strict UTF-8 through the BOM-policy owner

* chore(architecture): match local strict UTF-8 decoders in any option order
This commit is contained in:
morluto authored and GitHub committed 2026-11-15 19:54:39 +08:00
1 parent abd36dd250
commit cabd56ea71
103 files changed
+322 -251

No files matched your search

+2 -3
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import {
malformedInput,
parseFailureFromCause,
@@ -165,9 +166,7 @@ export function* decodeAppleDispatchMetadataSteps(
);
offset(address, end - start + 1);
// A metadata C string owns every code point, including an initial U+FEFF.
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes.subarray(start, end),
);
return decodeStrictUtf8(bytes.subarray(start, end), "preserve");
};
const result = objcSwiftMetadataSchema.parse({
db_save_result: null,
+2 -3
View File
@@ -1,12 +1,11 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
/** A dyld byte name projected losslessly as UTF-8, or retained exactly. */
export type MachoName = string | { readonly bytes_base64: string };
/** NUL terminators and source bounds are validated by the owning format reader. */
export const decodeMachoName = (bytes: Buffer): MachoName => {
try {
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes,
);
return decodeStrictUtf8(bytes, "preserve");
} catch {
return { bytes_base64: bytes.toString("base64") };
}
+2 -3
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import { bindArtifactFileCleanup } from "../artifacts/ArtifactFileOperation.js";
import { HOST_PLATFORM } from "../platform/HostPlatform.js";
import { invariantViolation } from "../domain/internalInvariant.js";
@@ -1198,9 +1199,7 @@ const readTracerOutput = async (
if (!bytes.value.ok) return bytes.value;
let text: string;
try {
text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes.value.value,
);
text = decodeStrictUtf8(bytes.value.value, "preserve");
} catch (cause: unknown) {
return err(cause);
}
+2 -1
View File
@@ -1,3 +1,4 @@
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
import {
NATIVE_CALL_CAPTURE_DEFAULTS,
type NativeCallCaptureLimits,
@@ -108,7 +109,7 @@ const parseLldbObservationJournal = (bytes: Uint8Array): ParsedLldbJournal => {
const events: ParsedLldbJournal["events"] = [];
const otherStops: string[] = [];
const limitations: string[] = [];
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const decoder = strictUtf8Decoder("preserve");
let malformedCount = 0;
let firstMalformedRow: number | undefined;
let start = 0;
+2 -3
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import { legacyMachoPlatform } from "../domain/apple/machoPlatform.js";
import {
LC,
@@ -88,9 +89,7 @@ const bytesText = (input: Uint8Array, fixed: boolean) => {
const bytes = end < 0 ? input : input.subarray(0, end);
try {
return {
text: new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes,
),
text: decodeStrictUtf8(bytes, "preserve"),
bytesHex: null,
};
} catch {
+4 -2
View File
@@ -1,6 +1,9 @@
import { MAX_NODE_TIMER_DELAY_MS } from "../domain/analysisResourceLimits.js";
import { withArtifactTargetIdentity } from "../artifacts/withArtifactTargetIdentity.js";
import { invariantViolation } from "../domain/internalInvariant.js";
import {
invariantViolation,
InternalInvariantError,
} from "../domain/internalInvariant.js";
import { translateNativeCommandFailure } from "./NativeCommandAnalysisError.js";
import {
plistSourceFormat,
@@ -114,7 +117,6 @@ import {
type NativeCommandInvocation,
} from "../domain/native/nativeInspection.js";
import { err, ok, type Result } from "../domain/result.js";
import { InternalInvariantError } from "../domain/internalInvariant.js";
import {
NativeCommandFailure,
NATIVE_COMMAND_OUTPUT_BUDGET_BYTES,
+2 -1
View File
@@ -18,6 +18,7 @@ import {
IPS_METADATA_SHAPE,
} from "./AppleIpsJsonShape.js";
import type { LosslessJson } from "./losslessJson.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
type Artifact = AppleIps["artifact"];
@@ -30,7 +31,7 @@ const IPS_LIMITATIONS = [
"Symbol names printed inside the report are recorded claims and are not verified.",
];
const utf8 = new TextDecoder("utf-8", { fatal: true });
const utf8 = strictUtf8Decoder("strip-signature");
const canonicalUuid = (
raw: string | null,
+2 -1
View File
@@ -12,6 +12,7 @@ import {
type KernelBuildConfigEntry,
} from "../../domain/native/kernelBuildConfig.js";
import { kernelCompressionCodecAt } from "./KernelCompressionCodec.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
type Report = KernelBuildConfig;
type Entry = KernelBuildConfigEntry;
@@ -234,7 +235,7 @@ const inflateGzipMember = async (
return { kind: "member", output, end: trailer + 8 };
};
const decoder = new TextDecoder("utf-8", { fatal: true });
const decoder = strictUtf8Decoder("strip-signature");
/** C0 controls other than tab, LF and CR, plus DEL. */
const CONTROL = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/u;
+2 -3
View File
@@ -44,6 +44,7 @@ import {
decodeXmlPlistText,
XmlPlistTextDecodeError,
} from "../../domain/propertyListXmlText.js";
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
/**
* Input ceiling matches native command output; projection is bounded separately.
@@ -202,9 +203,7 @@ export const readPlistJsonSource = (
bytes: Buffer,
): PlistJsonParseAttempt | undefined => {
try {
return readPlistJson(
new TextDecoder("utf-8", { fatal: true }).decode(bytes),
);
return readPlistJson(decodeStrictUtf8(bytes, "strip-signature"));
} catch {
return undefined;
}