refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy (#10728)

* refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy

* fix(domain): keep the leading-BOM behaviour every migrated UTF-8 decoder had

* refactor(domain): read the domain layer's strict UTF-8 through the BOM-policy owner (#10803)

Twelve domain decoders built their own fatal TextDecoder with ignoreBOM: true. They now use strictUtf8Decoder("preserve"), which keeps that behaviour: each reads bytes where a leading U+FEFF is content (binary-format names and fields, JSON and XML read byte-for-byte, recorded bodies, registry seeds whose kept bytes are re-emitted). The counted baseline drops from 74 to 62.

* refactor(artifacts): read artifact readers' strict UTF-8 through the BOM-policy owner (#10871)

Fourteen artifact reader modules (17 sites) built their own fatal TextDecoder. They now take it from strictUtf8Decoder with the policy each had: every one kept a leading BOM (ignoreBOM: true), so they use "preserve"; JavaScriptArtifactFiles keeps it for JavaScript and HTML entries and strips it otherwise, as before. The strict-utf8-decoder baseline drops those 17 sites.

* refactor(domain): drop TextDecoder imports left by the strict UTF-8 migration

* refactor(application): read MCP registration metadata through the strict UTF-8 owner

* refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner (#10903)

* refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner

* refactor(formats): drop the model readers' unused TextDecoder imports

* refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner (#10935)

* refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner

* refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner (#10936)

* refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner

* refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner (#10937)

* refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner

* refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner (#10938)

* refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner

* refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner (#10939)

* refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner

* refactor(evm): read the hex bytecode carrier's strict UTF-8 through the owner and refuse new local decoders (#10940)

* refactor(evm): read the hex bytecode carrier's strict UTF-8 through the BOM-policy owner

* chore(architecture): match local strict UTF-8 decoders in any option order
This commit is contained in:
morluto authored and GitHub committed 2026-11-15 19:54:39 +08:00
1 parent abd36dd250
commit cabd56ea71
103 files changed
+322 -251

No files matched your search

+2 -6
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import { jsonPointerTokens } from "../domain/jsonPointer.js";
import { visit, type JSONPath } from "jsonc-parser";
import { isExactJsonNumber } from "../domain/exactJsonNumber.js";
@@ -68,12 +69,7 @@ export const readResponseValues = (
return err(issue("the response is too large to save values from."));
let text: string;
try {
text =
typeof body === "string"
? body
: new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
body,
);
text = typeof body === "string" ? body : decodeStrictUtf8(body, "preserve");
} catch {
return err(
issue(
+2 -4
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import {
scenarioSerializationAllowance,
scenarioQueryLength,
@@ -590,10 +591,7 @@ export class BrowserScenarioSecrets {
).length === 0
)
return body;
const text = new TextDecoder("utf-8", {
fatal: true,
ignoreBOM: true,
}).decode(body);
const text = decodeStrictUtf8(body, "preserve");
const redacted = redactResponseSource(
text,
[],
+2 -1
View File
@@ -31,6 +31,7 @@ import {
DEFAULT_ASAR_HEADER_ADMISSION,
type AsarHeaderAdmission,
} from "../config/asarHeaderAdmission.js";
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import { isRecord } from "../domain/isRecord.js";
type ElectronFileOperation =
@@ -318,7 +319,7 @@ const asarFiles = async (
if (!sameRegularFileState(identity, afterRead))
throw new Error("ASAR identity changed during header read");
const parsed: unknown = JSON.parse(
new TextDecoder("utf-8", { fatal: true }).decode(json),
decodeStrictUtf8(json, "strip-signature"),
);
if (!isRecord(parsed) || !isRecord(parsed.files))
throw new Error("ASAR header has no file tree");
+2 -1
View File
@@ -25,6 +25,7 @@ import {
webBundleWorkerMessageSchema,
type WebBundleWorkerMessage,
} from "./WebBundleAnalysisWorkerProtocol.js";
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
export interface WebBundleScriptAnalyzer {
(
@@ -120,7 +121,7 @@ const analyzeJob = async (
let script: IncludedScript;
try {
const decoded: unknown = JSON.parse(
new TextDecoder("utf-8", { fatal: true }).decode(source.bytes),
decodeStrictUtf8(source.bytes, "strip-signature"),
);
script = includedWebScriptSchema.parse(decoded);
} catch (cause) {
+2 -5
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import * as t from "@babel/types";
import { createHash } from "node:crypto";
import { decodeValidatedSourceMapLeaves } from "../javascript/sourceMaps/DecodedSourceMap.js";
@@ -910,11 +911,7 @@ const readBoundedText = async (
};
try {
return {
text: ok(
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
joined,
),
),
text: ok(decodeStrictUtf8(joined, "preserve")),
responseIdentity,
};
} catch (cause: unknown) {
+2 -3
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
import type { WebArtifactAcquisition as Acquisition } from "../../domain/webArtifactAcquisition.js";
import { z } from "zod";
import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js";
@@ -50,9 +51,7 @@ const WEB_ARTIFACT_FORMAT = "captured-web-artifact";
/** Decode exact UTF-8 content, preserving a BOM for digest verification. */
export const decodeWebArtifact = (bytes: Buffer): string => {
try {
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes,
);
return decodeStrictUtf8(bytes, "preserve");
} catch (cause: unknown) {
throw malformedInput(
WEB_ARTIFACT_FORMAT,
+3 -2
View File
@@ -3,7 +3,7 @@ import { captureBudgets } from "./CaptureBudgets.js";
import { visit } from "jsonc-parser";
import { LosslessNumber } from "lossless-json";
import { CaptureFormatError } from "./CaptureFormatError.js";
import { TextDecoder } from "node:util";
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
type Frame =
| {
@@ -150,7 +150,8 @@ export const decodeCaptureText = (
format: "HAR" | "qlog",
): string => {
try {
return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
// Exported HAR and qlog files may carry a BOM signature (RFC 8259 8.1).
return decodeStrictUtf8(bytes, "strip-signature");
} catch {
throw new CaptureFormatError(
"malformed",
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../../../domain/strictUtf8.js";
import { fixedAllowanceOptions } from "../../../domain/fixedAllowanceRemediation.js";
import { getHeapStatistics } from "node:v8";
import { LosslessNumber, isSafeNumber } from "lossless-json";
@@ -41,9 +42,7 @@ const readRecord = (bytes: Buffer): unknown => {
);
let value: string;
try {
value = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes,
);
value = decodeStrictUtf8(bytes, "preserve");
} catch (cause) {
throw new CaptureFormatError(
"malformed",
@@ -1,3 +1,4 @@
import { strictUtf8Decoder } from "../../../domain/strictUtf8.js";
import { err, ok, type Result } from "../../../domain/result.js";
import type { ProtobufBudget } from "./ProtobufWire.js";
@@ -20,7 +21,7 @@ export const parseGrpcWebTrailers = (
bytes: Uint8Array,
budget: ProtobufBudget,
): Result<TrailerFacts, TrailerFailure> => {
let decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
let decoder = strictUtf8Decoder("preserve");
const trailers: TrailerFacts["trailers"] = [];
let examinedBytes = 0;
const facts = (): TrailerFacts => ({
@@ -103,7 +104,7 @@ export const parseGrpcWebTrailers = (
value = decoder.decode(bytes.subarray(valueStart, valueEnd));
} catch (cause) {
if (!(cause instanceof TypeError)) throw cause;
decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
decoder = strictUtf8Decoder("preserve");
opaqueValueFailure ??= {
...facts(),
kind: "invalid-utf8-value",
+2 -1
View File
@@ -1,3 +1,4 @@
import { strictUtf8Decoder } from "../../../domain/strictUtf8.js";
import { captureBudgets } from "../CaptureBudgets.js";
import { readLeb128, ULEB64 } from "../../../domain/leb128.js";
import { err, ok, type Result } from "../../../domain/result.js";
@@ -40,7 +41,7 @@ export const readProtobufVarint = (
return "failure" in read ? undefined : { value: read.value, end: read.next };
};
const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const utf8 = strictUtf8Decoder("preserve");
const textOf = (bytes: Uint8Array): string | undefined => {
try {
return utf8.decode(bytes);
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
import {
isJson,
mediaTypeEssence,
@@ -235,11 +236,7 @@ export const captureNetworkBody = async (
if (
jsonBody &&
!retained.equals(sourceBody) &&
!safeParseJson(
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
retained,
),
).ok
!safeParseJson(decodeStrictUtf8(retained, "preserve")).ok
)
return {
state: "unavailable",