mirror of
https://github.com/morluto/rea.git
synced 2026-11-15 19:23:24 +00:00
refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy (#10728)
* refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy * fix(domain): keep the leading-BOM behaviour every migrated UTF-8 decoder had * refactor(domain): read the domain layer's strict UTF-8 through the BOM-policy owner (#10803) Twelve domain decoders built their own fatal TextDecoder with ignoreBOM: true. They now use strictUtf8Decoder("preserve"), which keeps that behaviour: each reads bytes where a leading U+FEFF is content (binary-format names and fields, JSON and XML read byte-for-byte, recorded bodies, registry seeds whose kept bytes are re-emitted). The counted baseline drops from 74 to 62. * refactor(artifacts): read artifact readers' strict UTF-8 through the BOM-policy owner (#10871) Fourteen artifact reader modules (17 sites) built their own fatal TextDecoder. They now take it from strictUtf8Decoder with the policy each had: every one kept a leading BOM (ignoreBOM: true), so they use "preserve"; JavaScriptArtifactFiles keeps it for JavaScript and HTML entries and strips it otherwise, as before. The strict-utf8-decoder baseline drops those 17 sites. * refactor(domain): drop TextDecoder imports left by the strict UTF-8 migration * refactor(application): read MCP registration metadata through the strict UTF-8 owner * refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner (#10903) * refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner * refactor(formats): drop the model readers' unused TextDecoder imports * refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner (#10935) * refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner * refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner (#10936) * refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner * refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner (#10937) * refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner * refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner (#10938) * refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner * refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner (#10939) * refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner * refactor(evm): read the hex bytecode carrier's strict UTF-8 through the owner and refuse new local decoders (#10940) * refactor(evm): read the hex bytecode carrier's strict UTF-8 through the BOM-policy owner * chore(architecture): match local strict UTF-8 decoders in any option order
This commit is contained in:
1 parent
abd36dd250
commit
cabd56ea71
103 files changed
+322
-251
No files matched your search
@@ -1,3 +1,4 @@
|
||||
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
|
||||
import { jsonPointerTokens } from "../domain/jsonPointer.js";
|
||||
import { visit, type JSONPath } from "jsonc-parser";
|
||||
import { isExactJsonNumber } from "../domain/exactJsonNumber.js";
|
||||
@@ -68,12 +69,7 @@ export const readResponseValues = (
|
||||
return err(issue("the response is too large to save values from."));
|
||||
let text: string;
|
||||
try {
|
||||
text =
|
||||
typeof body === "string"
|
||||
? body
|
||||
: new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
|
||||
body,
|
||||
);
|
||||
text = typeof body === "string" ? body : decodeStrictUtf8(body, "preserve");
|
||||
} catch {
|
||||
return err(
|
||||
issue(
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
|
||||
import {
|
||||
scenarioSerializationAllowance,
|
||||
scenarioQueryLength,
|
||||
@@ -590,10 +591,7 @@ export class BrowserScenarioSecrets {
|
||||
).length === 0
|
||||
)
|
||||
return body;
|
||||
const text = new TextDecoder("utf-8", {
|
||||
fatal: true,
|
||||
ignoreBOM: true,
|
||||
}).decode(body);
|
||||
const text = decodeStrictUtf8(body, "preserve");
|
||||
const redacted = redactResponseSource(
|
||||
text,
|
||||
[],
|
||||
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
DEFAULT_ASAR_HEADER_ADMISSION,
|
||||
type AsarHeaderAdmission,
|
||||
} from "../config/asarHeaderAdmission.js";
|
||||
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
|
||||
import { isRecord } from "../domain/isRecord.js";
|
||||
|
||||
type ElectronFileOperation =
|
||||
@@ -318,7 +319,7 @@ const asarFiles = async (
|
||||
if (!sameRegularFileState(identity, afterRead))
|
||||
throw new Error("ASAR identity changed during header read");
|
||||
const parsed: unknown = JSON.parse(
|
||||
new TextDecoder("utf-8", { fatal: true }).decode(json),
|
||||
decodeStrictUtf8(json, "strip-signature"),
|
||||
);
|
||||
if (!isRecord(parsed) || !isRecord(parsed.files))
|
||||
throw new Error("ASAR header has no file tree");
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
webBundleWorkerMessageSchema,
|
||||
type WebBundleWorkerMessage,
|
||||
} from "./WebBundleAnalysisWorkerProtocol.js";
|
||||
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
|
||||
|
||||
export interface WebBundleScriptAnalyzer {
|
||||
(
|
||||
@@ -120,7 +121,7 @@ const analyzeJob = async (
|
||||
let script: IncludedScript;
|
||||
try {
|
||||
const decoded: unknown = JSON.parse(
|
||||
new TextDecoder("utf-8", { fatal: true }).decode(source.bytes),
|
||||
decodeStrictUtf8(source.bytes, "strip-signature"),
|
||||
);
|
||||
script = includedWebScriptSchema.parse(decoded);
|
||||
} catch (cause) {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
|
||||
import * as t from "@babel/types";
|
||||
import { createHash } from "node:crypto";
|
||||
import { decodeValidatedSourceMapLeaves } from "../javascript/sourceMaps/DecodedSourceMap.js";
|
||||
@@ -910,11 +911,7 @@ const readBoundedText = async (
|
||||
};
|
||||
try {
|
||||
return {
|
||||
text: ok(
|
||||
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
|
||||
joined,
|
||||
),
|
||||
),
|
||||
text: ok(decodeStrictUtf8(joined, "preserve")),
|
||||
responseIdentity,
|
||||
};
|
||||
} catch (cause: unknown) {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
|
||||
import type { WebArtifactAcquisition as Acquisition } from "../../domain/webArtifactAcquisition.js";
|
||||
import { z } from "zod";
|
||||
import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js";
|
||||
@@ -50,9 +51,7 @@ const WEB_ARTIFACT_FORMAT = "captured-web-artifact";
|
||||
/** Decode exact UTF-8 content, preserving a BOM for digest verification. */
|
||||
export const decodeWebArtifact = (bytes: Buffer): string => {
|
||||
try {
|
||||
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
|
||||
bytes,
|
||||
);
|
||||
return decodeStrictUtf8(bytes, "preserve");
|
||||
} catch (cause: unknown) {
|
||||
throw malformedInput(
|
||||
WEB_ARTIFACT_FORMAT,
|
||||
|
||||
@@ -3,7 +3,7 @@ import { captureBudgets } from "./CaptureBudgets.js";
|
||||
import { visit } from "jsonc-parser";
|
||||
import { LosslessNumber } from "lossless-json";
|
||||
import { CaptureFormatError } from "./CaptureFormatError.js";
|
||||
import { TextDecoder } from "node:util";
|
||||
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
|
||||
|
||||
type Frame =
|
||||
| {
|
||||
@@ -150,7 +150,8 @@ export const decodeCaptureText = (
|
||||
format: "HAR" | "qlog",
|
||||
): string => {
|
||||
try {
|
||||
return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
||||
// Exported HAR and qlog files may carry a BOM signature (RFC 8259 8.1).
|
||||
return decodeStrictUtf8(bytes, "strip-signature");
|
||||
} catch {
|
||||
throw new CaptureFormatError(
|
||||
"malformed",
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { decodeStrictUtf8 } from "../../../domain/strictUtf8.js";
|
||||
import { fixedAllowanceOptions } from "../../../domain/fixedAllowanceRemediation.js";
|
||||
import { getHeapStatistics } from "node:v8";
|
||||
import { LosslessNumber, isSafeNumber } from "lossless-json";
|
||||
@@ -41,9 +42,7 @@ const readRecord = (bytes: Buffer): unknown => {
|
||||
);
|
||||
let value: string;
|
||||
try {
|
||||
value = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
|
||||
bytes,
|
||||
);
|
||||
value = decodeStrictUtf8(bytes, "preserve");
|
||||
} catch (cause) {
|
||||
throw new CaptureFormatError(
|
||||
"malformed",
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { strictUtf8Decoder } from "../../../domain/strictUtf8.js";
|
||||
import { err, ok, type Result } from "../../../domain/result.js";
|
||||
import type { ProtobufBudget } from "./ProtobufWire.js";
|
||||
|
||||
@@ -20,7 +21,7 @@ export const parseGrpcWebTrailers = (
|
||||
bytes: Uint8Array,
|
||||
budget: ProtobufBudget,
|
||||
): Result<TrailerFacts, TrailerFailure> => {
|
||||
let decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
||||
let decoder = strictUtf8Decoder("preserve");
|
||||
const trailers: TrailerFacts["trailers"] = [];
|
||||
let examinedBytes = 0;
|
||||
const facts = (): TrailerFacts => ({
|
||||
@@ -103,7 +104,7 @@ export const parseGrpcWebTrailers = (
|
||||
value = decoder.decode(bytes.subarray(valueStart, valueEnd));
|
||||
} catch (cause) {
|
||||
if (!(cause instanceof TypeError)) throw cause;
|
||||
decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
||||
decoder = strictUtf8Decoder("preserve");
|
||||
opaqueValueFailure ??= {
|
||||
...facts(),
|
||||
kind: "invalid-utf8-value",
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { strictUtf8Decoder } from "../../../domain/strictUtf8.js";
|
||||
import { captureBudgets } from "../CaptureBudgets.js";
|
||||
import { readLeb128, ULEB64 } from "../../../domain/leb128.js";
|
||||
import { err, ok, type Result } from "../../../domain/result.js";
|
||||
@@ -40,7 +41,7 @@ export const readProtobufVarint = (
|
||||
return "failure" in read ? undefined : { value: read.value, end: read.next };
|
||||
};
|
||||
|
||||
const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
||||
const utf8 = strictUtf8Decoder("preserve");
|
||||
const textOf = (bytes: Uint8Array): string | undefined => {
|
||||
try {
|
||||
return utf8.decode(bytes);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
|
||||
import {
|
||||
isJson,
|
||||
mediaTypeEssence,
|
||||
@@ -235,11 +236,7 @@ export const captureNetworkBody = async (
|
||||
if (
|
||||
jsonBody &&
|
||||
!retained.equals(sourceBody) &&
|
||||
!safeParseJson(
|
||||
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
|
||||
retained,
|
||||
),
|
||||
).ok
|
||||
!safeParseJson(decodeStrictUtf8(retained, "preserve")).ok
|
||||
)
|
||||
return {
|
||||
state: "unavailable",
|
||||
|
||||
Reference in new issue
Block a user