mirror of
https://github.com/morluto/rea.git
synced 2026-11-15 19:23:24 +00:00
refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy (#10728)
* refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy * fix(domain): keep the leading-BOM behaviour every migrated UTF-8 decoder had * refactor(domain): read the domain layer's strict UTF-8 through the BOM-policy owner (#10803) Twelve domain decoders built their own fatal TextDecoder with ignoreBOM: true. They now use strictUtf8Decoder("preserve"), which keeps that behaviour: each reads bytes where a leading U+FEFF is content (binary-format names and fields, JSON and XML read byte-for-byte, recorded bodies, registry seeds whose kept bytes are re-emitted). The counted baseline drops from 74 to 62. * refactor(artifacts): read artifact readers' strict UTF-8 through the BOM-policy owner (#10871) Fourteen artifact reader modules (17 sites) built their own fatal TextDecoder. They now take it from strictUtf8Decoder with the policy each had: every one kept a leading BOM (ignoreBOM: true), so they use "preserve"; JavaScriptArtifactFiles keeps it for JavaScript and HTML entries and strips it otherwise, as before. The strict-utf8-decoder baseline drops those 17 sites. * refactor(domain): drop TextDecoder imports left by the strict UTF-8 migration * refactor(application): read MCP registration metadata through the strict UTF-8 owner * refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner (#10903) * refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner * refactor(formats): drop the model readers' unused TextDecoder imports * refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner (#10935) * refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner * refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner (#10936) * refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner * refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner (#10937) * refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner * refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner (#10938) * refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner * refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner (#10939) * refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner * refactor(evm): read the hex bytecode carrier's strict UTF-8 through the owner and refuse new local decoders (#10940) * refactor(evm): read the hex bytecode carrier's strict UTF-8 through the BOM-policy owner * chore(architecture): match local strict UTF-8 decoders in any option order
This commit is contained in:
1 parent
abd36dd250
commit
cabd56ea71
103 files changed
+322
-251
No files matched your search
@@ -5,6 +5,7 @@ import {
|
||||
absoluteCallerPath,
|
||||
appendCallerPath,
|
||||
} from "../filesystem/callerPath.js";
|
||||
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
|
||||
|
||||
/** An unresolved Hermes profile must not be configured as the default home. */
|
||||
export class HermesClientPathError extends Error {
|
||||
@@ -178,9 +179,10 @@ export const resolveHermesClientHome = (
|
||||
let name: string;
|
||||
try {
|
||||
// Python uses utf-8-sig and ignores unreadable / invalid-UTF8 active_profile files.
|
||||
name = new TextDecoder("utf-8", { fatal: true })
|
||||
.decode(readFileSync(appendCallerPath(stickyRoot, "active_profile")))
|
||||
.trim();
|
||||
name = decodeStrictUtf8(
|
||||
readFileSync(appendCallerPath(stickyRoot, "active_profile")),
|
||||
"strip-signature",
|
||||
).trim();
|
||||
} catch {
|
||||
return root;
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
|
||||
import { constants } from "node:buffer";
|
||||
import { createHash } from "node:crypto";
|
||||
import type { Stats } from "node:fs";
|
||||
@@ -717,7 +718,7 @@ async function* decodedChunks(
|
||||
observe?: (bytes: Uint8Array) => void,
|
||||
physicalRead: JsonInputPhysicalReadObservation = {},
|
||||
): AsyncGenerator<string> {
|
||||
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
||||
const decoder = strictUtf8Decoder("preserve");
|
||||
let leading = true;
|
||||
const checked = (text: string): string => {
|
||||
if (leading && text.length > 0) {
|
||||
|
||||
@@ -54,6 +54,7 @@ import {
|
||||
type LinuxPackageFamily,
|
||||
} from "./HopperReleaseFeed.js";
|
||||
import { withRegularFile } from "./RegularFileRead.js";
|
||||
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
|
||||
|
||||
/** Metadata/key inspection budget, independent of the compressed package budget. */
|
||||
const RELEASE_METADATA_BYTES = 1024 * 1024;
|
||||
@@ -1082,7 +1083,7 @@ export const retryLinuxHopperPublisherCleanup = async (): Promise<void> => {
|
||||
};
|
||||
|
||||
const decodeUtf8 = (bytes: Uint8Array): string =>
|
||||
new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
||||
decodeStrictUtf8(bytes, "strip-signature");
|
||||
const releaseFailure = (
|
||||
cause: unknown,
|
||||
signal: AbortSignal | undefined,
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
} from "../domain/analysisErrorCore.js";
|
||||
import { analysisErrorWithCancellation } from "../domain/analysisErrorCancellation.js";
|
||||
import { ConfigurationError } from "../domain/configurationErrors.js";
|
||||
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
|
||||
import { absoluteCallerPath } from "../filesystem/callerPath.js";
|
||||
import {
|
||||
ExecFileOutputCancelledError,
|
||||
@@ -65,7 +66,7 @@ const gitOutput = async (
|
||||
throw gitFailure(cause);
|
||||
}
|
||||
try {
|
||||
return new TextDecoder("utf-8", { fatal: true }).decode(observation.stdout);
|
||||
return decodeStrictUtf8(observation.stdout, "strip-signature");
|
||||
} catch {
|
||||
// The failure keeps the exact observation it could not decode.
|
||||
throw gitFailure(
|
||||
|
||||
@@ -26,6 +26,7 @@ import type {
|
||||
} from "../reference/ReferenceSourceReaderTypes.js";
|
||||
import { typescriptEmittedSourceCandidates } from "./TypeScriptEmittedSourceCandidates.js";
|
||||
import { appendAll } from "../domain/appendAll.js";
|
||||
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
|
||||
|
||||
export interface ParsedReferenceSourceEntries {
|
||||
readonly entries: HistoricalSourceGraphInput["entries"];
|
||||
@@ -152,7 +153,8 @@ const readPackageMetadata = (
|
||||
}
|
||||
try {
|
||||
const manifest: unknown = JSON.parse(
|
||||
new TextDecoder("utf-8", { fatal: true }).decode(entry.bytes),
|
||||
// npm and Node both accept a BOM-signed package.json.
|
||||
decodeStrictUtf8(entry.bytes, "strip-signature"),
|
||||
);
|
||||
if (
|
||||
typeof manifest !== "object" ||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
|
||||
import type { Stats } from "node:fs";
|
||||
import type { FileHandle } from "node:fs/promises";
|
||||
|
||||
@@ -343,9 +344,7 @@ export const readRegularFileTextSnapshot = (
|
||||
|
||||
const decodeFileText = (path: string, bytes: Buffer): string => {
|
||||
try {
|
||||
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
|
||||
bytes,
|
||||
);
|
||||
return decodeStrictUtf8(bytes, "preserve");
|
||||
} catch (cause: unknown) {
|
||||
if (!(cause instanceof TypeError)) throw cause;
|
||||
throw new InvalidUtf8FileError(path, cause);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
|
||||
import { constants } from "node:buffer";
|
||||
|
||||
import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js";
|
||||
@@ -45,7 +46,7 @@ export const parseUtf8Json = (
|
||||
);
|
||||
let text: string;
|
||||
try {
|
||||
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
||||
const decoder = strictUtf8Decoder("preserve");
|
||||
if (bytes.byteLength <= constants.MAX_STRING_LENGTH) {
|
||||
text = decoder.decode(bytes);
|
||||
} else {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import { z } from "zod";
|
||||
@@ -396,10 +397,7 @@ const parseCapture = (
|
||||
closeFailure?: { readonly cause: unknown },
|
||||
): Result<ReturnType<typeof selectScriptCapture>, AnalysisError> => {
|
||||
try {
|
||||
const text = new TextDecoder("utf-8", {
|
||||
fatal: true,
|
||||
ignoreBOM: true,
|
||||
}).decode(bytes);
|
||||
const text = decodeStrictUtf8(bytes, "preserve");
|
||||
const json = safeParseJson(text);
|
||||
if (!json.ok) throw new TypeError(json.error);
|
||||
return ok(selectScriptCapture(json.value));
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
|
||||
import { getHeapStatistics } from "node:v8";
|
||||
import { version } from "graphql";
|
||||
import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
|
||||
@@ -86,9 +87,7 @@ const decode = (
|
||||
),
|
||||
);
|
||||
try {
|
||||
return ok(
|
||||
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes),
|
||||
);
|
||||
return ok(decodeStrictUtf8(bytes, "preserve"));
|
||||
} catch (cause) {
|
||||
if (!(cause instanceof TypeError)) throw cause;
|
||||
return err(
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { constants } from "node:fs";
|
||||
import {
|
||||
@@ -737,10 +738,7 @@ const parseJournal = (
|
||||
const committedBytes = bytes.lastIndexOf(0x0a) + 1;
|
||||
let committed: string;
|
||||
try {
|
||||
committed = new TextDecoder("utf-8", {
|
||||
fatal: true,
|
||||
ignoreBOM: true,
|
||||
}).decode(bytes.subarray(0, committedBytes));
|
||||
committed = decodeStrictUtf8(bytes.subarray(0, committedBytes), "preserve");
|
||||
} catch {
|
||||
return err(incompatibleJournal());
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user