mirror of
https://github.com/morluto/rea.git
synced 2026-11-16 19:53:50 +00:00
refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy (#10728)
* refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy * fix(domain): keep the leading-BOM behaviour every migrated UTF-8 decoder had * refactor(domain): read the domain layer's strict UTF-8 through the BOM-policy owner (#10803) Twelve domain decoders built their own fatal TextDecoder with ignoreBOM: true. They now use strictUtf8Decoder("preserve"), which keeps that behaviour: each reads bytes where a leading U+FEFF is content (binary-format names and fields, JSON and XML read byte-for-byte, recorded bodies, registry seeds whose kept bytes are re-emitted). The counted baseline drops from 74 to 62. * refactor(artifacts): read artifact readers' strict UTF-8 through the BOM-policy owner (#10871) Fourteen artifact reader modules (17 sites) built their own fatal TextDecoder. They now take it from strictUtf8Decoder with the policy each had: every one kept a leading BOM (ignoreBOM: true), so they use "preserve"; JavaScriptArtifactFiles keeps it for JavaScript and HTML entries and strips it otherwise, as before. The strict-utf8-decoder baseline drops those 17 sites. * refactor(domain): drop TextDecoder imports left by the strict UTF-8 migration * refactor(application): read MCP registration metadata through the strict UTF-8 owner * refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner (#10903) * refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner * refactor(formats): drop the model readers' unused TextDecoder imports * refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner (#10935) * refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner * refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner (#10936) * refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner * refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner (#10937) * refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner * refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner (#10938) * refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner * refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner (#10939) * refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner * refactor(evm): read the hex bytecode carrier's strict UTF-8 through the owner and refuse new local decoders (#10940) * refactor(evm): read the hex bytecode carrier's strict UTF-8 through the BOM-policy owner * chore(architecture): match local strict UTF-8 decoders in any option order
This commit is contained in:
1 parent
abd36dd250
commit
cabd56ea71
103 files changed
+322
-251
No files matched your search
@@ -1,3 +1,4 @@
|
||||
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
|
||||
import {
|
||||
malformedInput,
|
||||
parseFailureFromCause,
|
||||
@@ -165,9 +166,7 @@ export function* decodeAppleDispatchMetadataSteps(
|
||||
);
|
||||
offset(address, end - start + 1);
|
||||
// A metadata C string owns every code point, including an initial U+FEFF.
|
||||
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
|
||||
bytes.subarray(start, end),
|
||||
);
|
||||
return decodeStrictUtf8(bytes.subarray(start, end), "preserve");
|
||||
};
|
||||
const result = objcSwiftMetadataSchema.parse({
|
||||
db_save_result: null,
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
|
||||
/** A dyld byte name projected losslessly as UTF-8, or retained exactly. */
|
||||
export type MachoName = string | { readonly bytes_base64: string };
|
||||
|
||||
/** NUL terminators and source bounds are validated by the owning format reader. */
|
||||
export const decodeMachoName = (bytes: Buffer): MachoName => {
|
||||
try {
|
||||
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
|
||||
bytes,
|
||||
);
|
||||
return decodeStrictUtf8(bytes, "preserve");
|
||||
} catch {
|
||||
return { bytes_base64: bytes.toString("base64") };
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
|
||||
import { bindArtifactFileCleanup } from "../artifacts/ArtifactFileOperation.js";
|
||||
import { HOST_PLATFORM } from "../platform/HostPlatform.js";
|
||||
import { invariantViolation } from "../domain/internalInvariant.js";
|
||||
@@ -1198,9 +1199,7 @@ const readTracerOutput = async (
|
||||
if (!bytes.value.ok) return bytes.value;
|
||||
let text: string;
|
||||
try {
|
||||
text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
|
||||
bytes.value.value,
|
||||
);
|
||||
text = decodeStrictUtf8(bytes.value.value, "preserve");
|
||||
} catch (cause: unknown) {
|
||||
return err(cause);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
|
||||
import {
|
||||
NATIVE_CALL_CAPTURE_DEFAULTS,
|
||||
type NativeCallCaptureLimits,
|
||||
@@ -108,7 +109,7 @@ const parseLldbObservationJournal = (bytes: Uint8Array): ParsedLldbJournal => {
|
||||
const events: ParsedLldbJournal["events"] = [];
|
||||
const otherStops: string[] = [];
|
||||
const limitations: string[] = [];
|
||||
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
||||
const decoder = strictUtf8Decoder("preserve");
|
||||
let malformedCount = 0;
|
||||
let firstMalformedRow: number | undefined;
|
||||
let start = 0;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
|
||||
import { legacyMachoPlatform } from "../domain/apple/machoPlatform.js";
|
||||
import {
|
||||
LC,
|
||||
@@ -88,9 +89,7 @@ const bytesText = (input: Uint8Array, fixed: boolean) => {
|
||||
const bytes = end < 0 ? input : input.subarray(0, end);
|
||||
try {
|
||||
return {
|
||||
text: new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
|
||||
bytes,
|
||||
),
|
||||
text: decodeStrictUtf8(bytes, "preserve"),
|
||||
bytesHex: null,
|
||||
};
|
||||
} catch {
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import { MAX_NODE_TIMER_DELAY_MS } from "../domain/analysisResourceLimits.js";
|
||||
import { withArtifactTargetIdentity } from "../artifacts/withArtifactTargetIdentity.js";
|
||||
import { invariantViolation } from "../domain/internalInvariant.js";
|
||||
import {
|
||||
invariantViolation,
|
||||
InternalInvariantError,
|
||||
} from "../domain/internalInvariant.js";
|
||||
import { translateNativeCommandFailure } from "./NativeCommandAnalysisError.js";
|
||||
import {
|
||||
plistSourceFormat,
|
||||
@@ -114,7 +117,6 @@ import {
|
||||
type NativeCommandInvocation,
|
||||
} from "../domain/native/nativeInspection.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import { InternalInvariantError } from "../domain/internalInvariant.js";
|
||||
import {
|
||||
NativeCommandFailure,
|
||||
NATIVE_COMMAND_OUTPUT_BUDGET_BYTES,
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
IPS_METADATA_SHAPE,
|
||||
} from "./AppleIpsJsonShape.js";
|
||||
import type { LosslessJson } from "./losslessJson.js";
|
||||
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
|
||||
|
||||
type Artifact = AppleIps["artifact"];
|
||||
|
||||
@@ -30,7 +31,7 @@ const IPS_LIMITATIONS = [
|
||||
"Symbol names printed inside the report are recorded claims and are not verified.",
|
||||
];
|
||||
|
||||
const utf8 = new TextDecoder("utf-8", { fatal: true });
|
||||
const utf8 = strictUtf8Decoder("strip-signature");
|
||||
|
||||
const canonicalUuid = (
|
||||
raw: string | null,
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
type KernelBuildConfigEntry,
|
||||
} from "../../domain/native/kernelBuildConfig.js";
|
||||
import { kernelCompressionCodecAt } from "./KernelCompressionCodec.js";
|
||||
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
|
||||
|
||||
type Report = KernelBuildConfig;
|
||||
type Entry = KernelBuildConfigEntry;
|
||||
@@ -234,7 +235,7 @@ const inflateGzipMember = async (
|
||||
return { kind: "member", output, end: trailer + 8 };
|
||||
};
|
||||
|
||||
const decoder = new TextDecoder("utf-8", { fatal: true });
|
||||
const decoder = strictUtf8Decoder("strip-signature");
|
||||
/** C0 controls other than tab, LF and CR, plus DEL. */
|
||||
const CONTROL = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/u;
|
||||
|
||||
|
||||
@@ -44,6 +44,7 @@ import {
|
||||
decodeXmlPlistText,
|
||||
XmlPlistTextDecodeError,
|
||||
} from "../../domain/propertyListXmlText.js";
|
||||
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
|
||||
|
||||
/**
|
||||
* Input ceiling matches native command output; projection is bounded separately.
|
||||
@@ -202,9 +203,7 @@ export const readPlistJsonSource = (
|
||||
bytes: Buffer,
|
||||
): PlistJsonParseAttempt | undefined => {
|
||||
try {
|
||||
return readPlistJson(
|
||||
new TextDecoder("utf-8", { fatal: true }).decode(bytes),
|
||||
);
|
||||
return readPlistJson(decodeStrictUtf8(bytes, "strip-signature"));
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user