refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy (#10728)

* refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy

* fix(domain): keep the leading-BOM behaviour every migrated UTF-8 decoder had

* refactor(domain): read the domain layer's strict UTF-8 through the BOM-policy owner (#10803)

Twelve domain decoders built their own fatal TextDecoder with ignoreBOM: true. They now use strictUtf8Decoder("preserve"), which keeps that behaviour: each reads bytes where a leading U+FEFF is content (binary-format names and fields, JSON and XML read byte-for-byte, recorded bodies, registry seeds whose kept bytes are re-emitted). The counted baseline drops from 74 to 62.

* refactor(artifacts): read artifact readers' strict UTF-8 through the BOM-policy owner (#10871)

Fourteen artifact reader modules (17 sites) built their own fatal TextDecoder. They now take it from strictUtf8Decoder with the policy each had: every one kept a leading BOM (ignoreBOM: true), so they use "preserve"; JavaScriptArtifactFiles keeps it for JavaScript and HTML entries and strips it otherwise, as before. The strict-utf8-decoder baseline drops those 17 sites.

* refactor(domain): drop TextDecoder imports left by the strict UTF-8 migration

* refactor(application): read MCP registration metadata through the strict UTF-8 owner

* refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner (#10903)

* refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner

* refactor(formats): drop the model readers' unused TextDecoder imports

* refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner (#10935)

* refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner

* refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner (#10936)

* refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner

* refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner (#10937)

* refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner

* refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner (#10938)

* refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner

* refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner (#10939)

* refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner

* refactor(evm): read the hex bytecode carrier's strict UTF-8 through the owner and refuse new local decoders (#10940)

* refactor(evm): read the hex bytecode carrier's strict UTF-8 through the BOM-policy owner

* chore(architecture): match local strict UTF-8 decoders in any option order
This commit is contained in:
morluto authored and GitHub committed 2026-11-15 19:54:39 +08:00
1 parent abd36dd250
commit cabd56ea71
103 files changed
+322 -251

No files matched your search

@@ -301,6 +301,9 @@
"src/sqlite/SqliteDatabaseWorkerExecution.ts": 2,
"src/wasm/WabtArtifactProvider.ts": 1
},
"strict-utf8-decoder": {
"src/browser/PlaywrightScenarioRequest.ts": 1
},
"path-search-split": {
"src/process/capture/ProcessScenarioRuntimeValidation.ts": 1
},
+13
View File
@@ -330,6 +330,19 @@ checkCounted(
["src/pcap/TsharkJsonPackets.ts"],
);
// Strict UTF-8 decoding has one owner, src/domain/strictUtf8.ts, which makes
// each site state its byte-order-mark policy. A local fatal decoder silently
// strips a leading BOM unless it remembers ignoreBOM. The pattern matches the
// options in any order and an omitted label. The remaining copy runs inside a
// browser page through page.evaluate, which cannot import the owner.
checkCounted(
"local strict UTF-8 TextDecoder (use strictUtf8Decoder, decodeStrictUtf8 or isStrictUtf8 from src/domain/strictUtf8)",
"strict-utf8-decoder",
files,
/new TextDecoder\(\s*(?:(?:["'`]utf-?8["'`]|undefined)\s*,\s*)?\{[^}]*\bfatal:\s*true/giu,
["src/domain/strictUtf8.ts"],
);
// Turning a completed result into a cancellation after cleanup is the
// finalization precedence that runOwned owns (src/providers/runOwned.ts).
checkCounted(
+3 -1
View File
@@ -28,6 +28,7 @@ import { ProviderCleanupError } from "../domain/providerCleanupError.js";
import { err, ok, type Result } from "../domain/result.js";
import { isSelectedRegularFileError } from "../filesystem/RegularFile.js";
import { jadxOperationTimeoutGuidance } from "./JadxOperationTimeout.js";
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
export interface AndroidMappingReadDeadline {
readonly timeout: AbortSignal;
@@ -192,7 +193,8 @@ export const readAndroidMapping = async (
);
let text: string;
try {
text = new TextDecoder("utf-8", { fatal: true }).decode(snapshot.bytes);
// A mapping file saved by an editor may start with a BOM signature.
text = decodeStrictUtf8(snapshot.bytes, "strip-signature");
} catch {
return malformed(`Mapping file is not valid UTF-8: ${path}`);
}
+2 -1
View File
@@ -27,6 +27,7 @@ import {
} from "../filesystem/RegularFile.js";
import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js";
import type { ApktoolCommandCleanup } from "./ApktoolCommandCleanup.js";
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
/** One decode's descriptors and root; removal follows confirmed child cleanup. */
export class ApktoolWorkspace {
@@ -203,7 +204,7 @@ export class ApktoolWorkspace {
const sourceBytes = Math.min(length, budget);
try {
return {
text: new TextDecoder("utf-8", { fatal: true }).decode(
text: strictUtf8Decoder("strip-signature").decode(
bytes.subarray(0, sourceBytes),
{ stream: truncated },
),
+5 -3
View File
@@ -5,6 +5,7 @@ import {
absoluteCallerPath,
appendCallerPath,
} from "../filesystem/callerPath.js";
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
/** An unresolved Hermes profile must not be configured as the default home. */
export class HermesClientPathError extends Error {
@@ -178,9 +179,10 @@ export const resolveHermesClientHome = (
let name: string;
try {
// Python uses utf-8-sig and ignores unreadable / invalid-UTF8 active_profile files.
name = new TextDecoder("utf-8", { fatal: true })
.decode(readFileSync(appendCallerPath(stickyRoot, "active_profile")))
.trim();
name = decodeStrictUtf8(
readFileSync(appendCallerPath(stickyRoot, "active_profile")),
"strip-signature",
).trim();
} catch {
return root;
}
+2 -1
View File
@@ -1,3 +1,4 @@
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
import { constants } from "node:buffer";
import { createHash } from "node:crypto";
import type { Stats } from "node:fs";
@@ -717,7 +718,7 @@ async function* decodedChunks(
observe?: (bytes: Uint8Array) => void,
physicalRead: JsonInputPhysicalReadObservation = {},
): AsyncGenerator<string> {
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const decoder = strictUtf8Decoder("preserve");
let leading = true;
const checked = (text: string): string => {
if (leading && text.length > 0) {
+2 -1
View File
@@ -54,6 +54,7 @@ import {
type LinuxPackageFamily,
} from "./HopperReleaseFeed.js";
import { withRegularFile } from "./RegularFileRead.js";
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
/** Metadata/key inspection budget, independent of the compressed package budget. */
const RELEASE_METADATA_BYTES = 1024 * 1024;
@@ -1082,7 +1083,7 @@ export const retryLinuxHopperPublisherCleanup = async (): Promise<void> => {
};
const decodeUtf8 = (bytes: Uint8Array): string =>
new TextDecoder("utf-8", { fatal: true }).decode(bytes);
decodeStrictUtf8(bytes, "strip-signature");
const releaseFailure = (
cause: unknown,
signal: AbortSignal | undefined,
+2 -1
View File
@@ -8,6 +8,7 @@ import {
} from "../domain/analysisErrorCore.js";
import { analysisErrorWithCancellation } from "../domain/analysisErrorCancellation.js";
import { ConfigurationError } from "../domain/configurationErrors.js";
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import { absoluteCallerPath } from "../filesystem/callerPath.js";
import {
ExecFileOutputCancelledError,
@@ -65,7 +66,7 @@ const gitOutput = async (
throw gitFailure(cause);
}
try {
return new TextDecoder("utf-8", { fatal: true }).decode(observation.stdout);
return decodeStrictUtf8(observation.stdout, "strip-signature");
} catch {
// The failure keeps the exact observation it could not decode.
throw gitFailure(
@@ -26,6 +26,7 @@ import type {
} from "../reference/ReferenceSourceReaderTypes.js";
import { typescriptEmittedSourceCandidates } from "./TypeScriptEmittedSourceCandidates.js";
import { appendAll } from "../domain/appendAll.js";
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
export interface ParsedReferenceSourceEntries {
readonly entries: HistoricalSourceGraphInput["entries"];
@@ -152,7 +153,8 @@ const readPackageMetadata = (
}
try {
const manifest: unknown = JSON.parse(
new TextDecoder("utf-8", { fatal: true }).decode(entry.bytes),
// npm and Node both accept a BOM-signed package.json.
decodeStrictUtf8(entry.bytes, "strip-signature"),
);
if (
typeof manifest !== "object" ||
+2 -3
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import type { Stats } from "node:fs";
import type { FileHandle } from "node:fs/promises";
@@ -343,9 +344,7 @@ export const readRegularFileTextSnapshot = (
const decodeFileText = (path: string, bytes: Buffer): string => {
try {
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes,
);
return decodeStrictUtf8(bytes, "preserve");
} catch (cause: unknown) {
if (!(cause instanceof TypeError)) throw cause;
throw new InvalidUtf8FileError(path, cause);
+2 -1
View File
@@ -1,3 +1,4 @@
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
import { constants } from "node:buffer";
import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js";
@@ -45,7 +46,7 @@ export const parseUtf8Json = (
);
let text: string;
try {
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const decoder = strictUtf8Decoder("preserve");
if (bytes.byteLength <= constants.MAX_STRING_LENGTH) {
text = decoder.decode(bytes);
} else {
+2 -4
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import { createHash } from "node:crypto";
import { z } from "zod";
@@ -396,10 +397,7 @@ const parseCapture = (
closeFailure?: { readonly cause: unknown },
): Result<ReturnType<typeof selectScriptCapture>, AnalysisError> => {
try {
const text = new TextDecoder("utf-8", {
fatal: true,
ignoreBOM: true,
}).decode(bytes);
const text = decodeStrictUtf8(bytes, "preserve");
const json = safeParseJson(text);
if (!json.ok) throw new TypeError(json.error);
return ok(selectScriptCapture(json.value));
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
import { getHeapStatistics } from "node:v8";
import { version } from "graphql";
import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
@@ -86,9 +87,7 @@ const decode = (
),
);
try {
return ok(
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes),
);
return ok(decodeStrictUtf8(bytes, "preserve"));
} catch (cause) {
if (!(cause instanceof TypeError)) throw cause;
return err(
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
import { createHash, randomUUID } from "node:crypto";
import { constants } from "node:fs";
import {
@@ -737,10 +738,7 @@ const parseJournal = (
const committedBytes = bytes.lastIndexOf(0x0a) + 1;
let committed: string;
try {
committed = new TextDecoder("utf-8", {
fatal: true,
ignoreBOM: true,
}).decode(bytes.subarray(0, committedBytes));
committed = decodeStrictUtf8(bytes.subarray(0, committedBytes), "preserve");
} catch {
return err(incompatibleJournal());
}
+2 -3
View File
@@ -17,6 +17,7 @@ import {
MAXIMUM_ASAR_MAX_HEADER_BYTES,
} from "../domain/analysisResourceLimits.js";
import { parseAsarListingAdmission } from "../config/asarListingAdmission.js";
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
/**
* Resolve the admission from a composition root's selected environment; an
@@ -435,9 +436,7 @@ export const readAsarHeader = async (
throw new ArtifactReaderFailure("format", "Truncated ASAR JSON header");
let json: string;
try {
json = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes,
);
json = strictUtf8Decoder("preserve").decode(bytes);
} catch (cause: unknown) {
throw new ArtifactReaderFailure(
"format",
+2 -1
View File
@@ -23,13 +23,14 @@ import {
type PositionalArtifactEntries,
type PositionalArtifactEntry,
} from "./PositionalArtifactReader.js";
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
const PACK_DIR_ENCRYPTED = 1;
const PACK_REL_FILEBASE = 2;
const PACK_SPARSE_BUNDLE = 4;
const PACK_FILE_ENCRYPTED = 1;
const DIRECTORY_CHUNK_BYTES = 64 * 1024;
const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const UTF8 = strictUtf8Decoder("preserve");
/** One descriptor, one validated directory, and bounded positional member reads. */
export class GodotPckArtifactReader extends PositionalArtifactReader {
+2 -1
View File
@@ -21,6 +21,7 @@ import {
UNITYFS_SIGNATURE,
unityFsLayoutFlags,
} from "../domain/unityfsFormat.js";
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
/** Per-decode resource limits, not format limits. Plain blocks are streamed. */
const MAX_BLOCK_UNCOMPRESSED_BYTES = 64 * 1024 * 1024;
@@ -49,7 +50,7 @@ class UnityFsDirectoryBudget {
}
/** Signature, version, two bounded strings and the fixed tail; reads are bounded. */
const HEADER_PREFIX_MAX = 8 + 4 + UNITYFS_HEADER_STRING_MAX * 2 + 20;
const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const UTF8 = strictUtf8Decoder("preserve");
export interface UnityFsHeader {
readonly formatVersion: number;
+2 -1
View File
@@ -15,6 +15,7 @@ import {
type WxapkgIndexLayout,
type WxapkgIndexLayoutFailure,
} from "../domain/wxapkgFormat.js";
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
/** The member table is a flat name/offset/size row list. */
const MAX_TABLE_BYTES = 16 * 1024 * 1024;
@@ -25,7 +26,7 @@ const MAX_MEMBER_NAME_BYTES = 64 * 1024;
/** Detecting prefix covers the fixed header, table head and encrypted magic. */
const HEADER_PREFIX_BYTES = 64;
const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const UTF8 = strictUtf8Decoder("preserve");
export const wxapkgFormatFailure = (detail: string): ArtifactReaderFailure =>
new ArtifactReaderFailure("format", `wxapkg is malformed: ${detail}`);
+2 -4
View File
@@ -52,12 +52,10 @@ import { artifactReaderLimitFailure } from "./ArtifactReaderLimits.js";
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { readFileHandleRange } from "../filesystem/readFileHandleRange.js";
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
import { invariantViolation } from "../domain/internalInvariant.js";
const ZIP_UTF8_DECODER = new TextDecoder("utf-8", {
fatal: true,
ignoreBOM: true,
});
const ZIP_UTF8_DECODER = strictUtf8Decoder("preserve");
/** Fixed method-93 decoder window: 2^27 bytes, matching the ELF/GPU decoders. */
const ZIP_ZSTANDARD_WINDOW_LOG_MAX = 27;
@@ -3,7 +3,6 @@ import {
isHostLimitFailure,
ParseFailureError,
} from "../../domain/parseFailure.js";
import { TextDecoder } from "node:util";
import { InternalInvariantError } from "../../domain/internalInvariant.js";
import { decodeBinaryPropertyList } from "./BinaryPlistMaterializer.js";
@@ -72,6 +71,7 @@ import {
hasBinaryPlistSignature,
hasNibArchiveSignature,
} from "../../domain/apple/appleArchiveFormat.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
const MAX_AGGREGATE_INPUT_BYTES = 32 * 1024 * 1024;
const MAX_AGGREGATE_DECODE_BYTES = 256 * 1024 * 1024;
@@ -730,7 +730,7 @@ const projectNibArchive = (
try {
// NSString's NS.bytes representation is UTF-8. A failed conversion
// does not establish a replacement-character class, title or selector.
value = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
value = strictUtf8Decoder("preserve").decode(
Buffer.from(encoded, "base64"),
);
} catch {
@@ -45,6 +45,7 @@ import {
type FatSliceDeclaration,
type FatSliceRangeIssue,
} from "../../domain/apple/machoContainer.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
/** Read `length` bytes at `offset`; shorter results mean end of file. */
export type ReadAt = (offset: number, length: number) => Promise<Uint8Array>;
@@ -1043,9 +1044,7 @@ const payloadString = (
chargeLazyString(work, end - offset + 1);
let text: string;
try {
text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
payload.subarray(offset, end),
);
text = strictUtf8Decoder("preserve").decode(payload.subarray(offset, end));
} catch {
throw machoIssue(
"malformed",
@@ -1116,9 +1115,7 @@ const throwCommandStringIssue = (
const decodeCommandString = (bytes: Uint8Array, index: number): string => {
try {
// lc_str contains name/path data, so a leading U+FEFF is part of its identity.
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes,
);
return strictUtf8Decoder("preserve").decode(bytes);
} catch {
throw machoIssue(
"malformed",
+2 -4
View File
@@ -1,7 +1,6 @@
import type { JsonValue } from "../../domain/jsonValue.js";
import { hasNibArchiveSignature } from "../../domain/apple/appleArchiveFormat.js";
import { isInterfaceBuilderConnectionClass } from "../../domain/apple/interfaceBuilderKeyedArchive.js";
import { TextDecoder } from "node:util";
import {
malformedInput,
truncatedInput,
@@ -11,6 +10,7 @@ import {
InterfaceBuilderDecodeBudget,
InterfaceBuilderDecodeBudgetExceeded,
} from "./InterfaceBuilderDecodeBudget.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
const DEFAULT_DECODE_BUDGET_BYTES = 128 * 1024 * 1024;
@@ -629,9 +629,7 @@ const decodeUtf8 = (
): string => {
try {
// These are length-prefixed names, not standalone text documents.
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes.subarray(start, end),
);
return strictUtf8Decoder("preserve").decode(bytes.subarray(start, end));
} catch (cause: unknown) {
throw malformedInput(
"nib-archive",
+3 -7
View File
@@ -1,10 +1,9 @@
import { TextDecoder } from "node:util";
import { fixedAllowanceRemediation } from "../../domain/fixedAllowanceRemediation.js";
import { jsonValueSchema, type JsonValue } from "../../domain/jsonValue.js";
import { abortIfNeeded } from "../ArtifactHash.js";
import { ArtifactReaderFailure } from "../ArtifactReader.js";
import { walkDenoVirtualFs, type DenoVfsWalk } from "./DenoVirtualFs.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
/** Opening and closing marker of a Deno compile data section. */
export const DENO_SECTION_MAGIC = Buffer.from("d3n0l4nd", "latin1");
@@ -20,11 +19,8 @@ const FLAG_TRANSPILED = 1 << 0;
const FLAG_SOURCE_MAP = 1 << 1;
const FLAG_CJS = 1 << 2;
const FLAG_UTF8 = 1 << 3;
const specifierDecoder = new TextDecoder("utf-8", {
fatal: true,
ignoreBOM: true,
});
const jsonDecoder = new TextDecoder("utf-8", { fatal: true });
const specifierDecoder = strictUtf8Decoder("preserve");
const jsonDecoder = strictUtf8Decoder("strip-signature");
const MEDIA_TYPE_NAMES = [
"JavaScript",
+2 -4
View File
@@ -15,6 +15,7 @@ import {
ArtifactReaderFailure,
artifactFixedLimitFailure,
} from "../ArtifactReader.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
/** SHA-256 marker embedded by Microsoft's apphost; the preceding UInt64 is the header offset. */
export const DOTNET_BUNDLE_SIGNATURE = Buffer.from(
@@ -285,10 +286,7 @@ class BundleCursor {
// decoder boundary so I/O and cancellation retain their original cause.
const bytes = await this.take(length);
try {
return new TextDecoder("utf-8", {
fatal: true,
ignoreBOM: true,
}).decode(bytes);
return strictUtf8Decoder("preserve").decode(bytes);
} catch (cause) {
throw new ArtifactReaderFailure(
"format",
+2 -1
View File
@@ -21,6 +21,7 @@ import {
} from "../../firmware/devicetree/DevicetreeParser.js";
import { BinaryFormatFailure } from "../../native/metadata/BinaryContainer.js";
import { ArtifactReaderFailure } from "../ArtifactReader.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
// Fixed resource bounds; configurable ceilings are tracked in #6176.
const FIT_MAX_IMAGES = 1024;
@@ -115,7 +116,7 @@ const valueOf = (fdt: Buffer, property: FitProperty): Buffer =>
fdt.subarray(property.offset, property.offset + property.bytes);
// A property value is data: a leading U+FEFF is not a file encoding signature.
const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const utf8 = strictUtf8Decoder("preserve");
const nulCount = (raw: Buffer): number => {
let count = 0;
for (const byte of raw) if (byte === 0) count += 1;
@@ -30,6 +30,7 @@ import type {
JavaScriptUnanalyzedArtifactLookup,
JavaScriptArtifactFileSet,
} from "../../domain/javascript/javascriptArtifactFiles.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
interface ExpectedFile {
readonly path: string;
@@ -388,12 +389,11 @@ const readText = async (
try {
return {
included: true,
value: new TextDecoder("utf-8", {
fatal: true,
ignoreBOM:
input.expected.kind === "javascript" ||
input.expected.kind === "html",
}).decode(bytes),
value: strictUtf8Decoder(
input.expected.kind === "javascript" || input.expected.kind === "html"
? "preserve"
: "strip-signature",
).decode(bytes),
};
} catch (cause: unknown) {
// Non-UTF8 bytes are counted; the fixed reason preserves the schema.
+2 -3
View File
@@ -7,6 +7,7 @@ import {
type NodeSeaReaderLimits,
} from "../../domain/artifactReaderLimits.js";
import { artifactReaderLimitFailure } from "../ArtifactReaderLimits.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
/** Node `SeaBlob::kMagic` from `src/node_sea.h` (v24). */
const NODE_SEA_MAGIC = 0x143da20;
@@ -231,9 +232,7 @@ class BlobCursor {
const raw = this.#demand(length);
let text: string;
try {
text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
raw,
);
text = strictUtf8Decoder("preserve").decode(raw);
} catch (cause: unknown) {
if (cause instanceof RangeError)
throw new ArtifactReaderFailure(
+3 -6
View File
@@ -6,6 +6,7 @@ import {
pyInstallerFailure,
type PyInstallerFailure,
} from "./PyInstallerFailure.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
/** Magic at the start of a CArchive cookie. */
const PYINSTALLER_COOKIE_MAGIC = Buffer.from("MEI\x0c\x0b\x0a\x0b\x0e");
@@ -299,9 +300,7 @@ const libraryName = (
);
if (text.length === 0) return ok(null);
try {
return ok(
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(text),
);
return ok(strictUtf8Decoder("preserve").decode(text));
} catch {
return err(
pyInstallerFailure(
@@ -318,9 +317,7 @@ const tocName = (bytes: Buffer): Result<string, PyInstallerFailure> => {
if (text.length === 0)
return err(pyInstallerFailure("format", "CArchive entry name is empty"));
try {
return ok(
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(text),
);
return ok(strictUtf8Decoder("preserve").decode(text));
} catch {
return err(
pyInstallerFailure("format", "CArchive entry name is not valid UTF-8"),
+2 -6
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import { jsonPointerTokens } from "../domain/jsonPointer.js";
import { visit, type JSONPath } from "jsonc-parser";
import { isExactJsonNumber } from "../domain/exactJsonNumber.js";
@@ -68,12 +69,7 @@ export const readResponseValues = (
return err(issue("the response is too large to save values from."));
let text: string;
try {
text =
typeof body === "string"
? body
: new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
body,
);
text = typeof body === "string" ? body : decodeStrictUtf8(body, "preserve");
} catch {
return err(
issue(
+2 -4
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import {
scenarioSerializationAllowance,
scenarioQueryLength,
@@ -590,10 +591,7 @@ export class BrowserScenarioSecrets {
).length === 0
)
return body;
const text = new TextDecoder("utf-8", {
fatal: true,
ignoreBOM: true,
}).decode(body);
const text = decodeStrictUtf8(body, "preserve");
const redacted = redactResponseSource(
text,
[],
+2 -1
View File
@@ -31,6 +31,7 @@ import {
DEFAULT_ASAR_HEADER_ADMISSION,
type AsarHeaderAdmission,
} from "../config/asarHeaderAdmission.js";
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import { isRecord } from "../domain/isRecord.js";
type ElectronFileOperation =
@@ -318,7 +319,7 @@ const asarFiles = async (
if (!sameRegularFileState(identity, afterRead))
throw new Error("ASAR identity changed during header read");
const parsed: unknown = JSON.parse(
new TextDecoder("utf-8", { fatal: true }).decode(json),
decodeStrictUtf8(json, "strip-signature"),
);
if (!isRecord(parsed) || !isRecord(parsed.files))
throw new Error("ASAR header has no file tree");
+2 -1
View File
@@ -25,6 +25,7 @@ import {
webBundleWorkerMessageSchema,
type WebBundleWorkerMessage,
} from "./WebBundleAnalysisWorkerProtocol.js";
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
export interface WebBundleScriptAnalyzer {
(
@@ -120,7 +121,7 @@ const analyzeJob = async (
let script: IncludedScript;
try {
const decoded: unknown = JSON.parse(
new TextDecoder("utf-8", { fatal: true }).decode(source.bytes),
decodeStrictUtf8(source.bytes, "strip-signature"),
);
script = includedWebScriptSchema.parse(decoded);
} catch (cause) {
+2 -5
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import * as t from "@babel/types";
import { createHash } from "node:crypto";
import { decodeValidatedSourceMapLeaves } from "../javascript/sourceMaps/DecodedSourceMap.js";
@@ -910,11 +911,7 @@ const readBoundedText = async (
};
try {
return {
text: ok(
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
joined,
),
),
text: ok(decodeStrictUtf8(joined, "preserve")),
responseIdentity,
};
} catch (cause: unknown) {
+2 -3
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
import type { WebArtifactAcquisition as Acquisition } from "../../domain/webArtifactAcquisition.js";
import { z } from "zod";
import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js";
@@ -50,9 +51,7 @@ const WEB_ARTIFACT_FORMAT = "captured-web-artifact";
/** Decode exact UTF-8 content, preserving a BOM for digest verification. */
export const decodeWebArtifact = (bytes: Buffer): string => {
try {
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes,
);
return decodeStrictUtf8(bytes, "preserve");
} catch (cause: unknown) {
throw malformedInput(
WEB_ARTIFACT_FORMAT,
+3 -2
View File
@@ -3,7 +3,7 @@ import { captureBudgets } from "./CaptureBudgets.js";
import { visit } from "jsonc-parser";
import { LosslessNumber } from "lossless-json";
import { CaptureFormatError } from "./CaptureFormatError.js";
import { TextDecoder } from "node:util";
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
type Frame =
| {
@@ -150,7 +150,8 @@ export const decodeCaptureText = (
format: "HAR" | "qlog",
): string => {
try {
return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
// Exported HAR and qlog files may carry a BOM signature (RFC 8259 8.1).
return decodeStrictUtf8(bytes, "strip-signature");
} catch {
throw new CaptureFormatError(
"malformed",
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../../../domain/strictUtf8.js";
import { fixedAllowanceOptions } from "../../../domain/fixedAllowanceRemediation.js";
import { getHeapStatistics } from "node:v8";
import { LosslessNumber, isSafeNumber } from "lossless-json";
@@ -41,9 +42,7 @@ const readRecord = (bytes: Buffer): unknown => {
);
let value: string;
try {
value = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes,
);
value = decodeStrictUtf8(bytes, "preserve");
} catch (cause) {
throw new CaptureFormatError(
"malformed",
@@ -1,3 +1,4 @@
import { strictUtf8Decoder } from "../../../domain/strictUtf8.js";
import { err, ok, type Result } from "../../../domain/result.js";
import type { ProtobufBudget } from "./ProtobufWire.js";
@@ -20,7 +21,7 @@ export const parseGrpcWebTrailers = (
bytes: Uint8Array,
budget: ProtobufBudget,
): Result<TrailerFacts, TrailerFailure> => {
let decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
let decoder = strictUtf8Decoder("preserve");
const trailers: TrailerFacts["trailers"] = [];
let examinedBytes = 0;
const facts = (): TrailerFacts => ({
@@ -103,7 +104,7 @@ export const parseGrpcWebTrailers = (
value = decoder.decode(bytes.subarray(valueStart, valueEnd));
} catch (cause) {
if (!(cause instanceof TypeError)) throw cause;
decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
decoder = strictUtf8Decoder("preserve");
opaqueValueFailure ??= {
...facts(),
kind: "invalid-utf8-value",
+2 -1
View File
@@ -1,3 +1,4 @@
import { strictUtf8Decoder } from "../../../domain/strictUtf8.js";
import { captureBudgets } from "../CaptureBudgets.js";
import { readLeb128, ULEB64 } from "../../../domain/leb128.js";
import { err, ok, type Result } from "../../../domain/result.js";
@@ -40,7 +41,7 @@ export const readProtobufVarint = (
return "failure" in read ? undefined : { value: read.value, end: read.next };
};
const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const utf8 = strictUtf8Decoder("preserve");
const textOf = (bytes: Uint8Array): string | undefined => {
try {
return utf8.decode(bytes);
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
import {
isJson,
mediaTypeEssence,
@@ -235,11 +236,7 @@ export const captureNetworkBody = async (
if (
jsonBody &&
!retained.equals(sourceBody) &&
!safeParseJson(
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
retained,
),
).ok
!safeParseJson(decodeStrictUtf8(retained, "preserve")).ok
)
return {
state: "unavailable",
+6
View File
@@ -54,6 +54,12 @@ describe("build secret files", () => {
const directory = await folder({
master: { text: `${VALUE}\n`, mode: 0o600 },
crlf: { text: `${VALUE}\r\n`, mode: 0o400 },
// An editor's UTF-8 signature is not part of the secret it saved.
bom: { text: `${VALUE}\n`, mode: 0o600 },
});
expect(await readBuildSecretFile(directory, "bom")).toEqual({
ok: true,
value: VALUE,
});
expect(await readBuildSecretFile(directory, "master")).toEqual({
ok: true,
+2 -1
View File
@@ -36,6 +36,7 @@ import {
} from "../filesystem/readStableFileBytes.js";
import { RegularFileChangedError } from "../filesystem/RegularFile.js";
import { xdgBase } from "./XdgBase.js";
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
/** The most bytes a secret file of at most `characters` may hold: those in UTF-8 and a line break. */
const maxFileBytes = (characters: number) => characters * 4 + 2;
@@ -179,7 +180,7 @@ export const readBuildSecretFile = async (
);
let text: string;
try {
text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
text = decodeStrictUtf8(bytes, "strip-signature");
} catch {
result = err(`${path} is not UTF-8 text.`);
return;
@@ -1,5 +1,6 @@
import { z } from "zod";
import type { WebNetworkCaptureRecord } from "../webNetworkCapture.js";
import { strictUtf8Decoder } from "../strictUtf8.js";
/** One record's retained response text, as replay and state extraction both read it. */
export interface RecordedResponseBody {
@@ -53,8 +54,7 @@ export const recordedResponseStatus = (
};
/** Strict, and keeping a leading byte-order mark, so callers see the recorded text unchanged. */
const strictUtf8 = (): TextDecoder =>
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const strictUtf8 = (): TextDecoder => strictUtf8Decoder("preserve");
/** Base64 characters that decode to just over `limitBytes`, so a cut prefix still fills it. */
export const base64PrefixChars = (limitBytes: number): number =>
@@ -20,6 +20,7 @@ import { analysisViewResultInputError } from "./analysisViewParent.js";
import type { AnalysisViewParent } from "./analysisViewParent.js";
import type { WebNetworkProjectionAccumulator } from "./WebNetworkProjectionAccumulator.js";
import { ENTRY_BYTES, textBytes } from "./webNetworkRouteTemplates.js";
import { strictUtf8Decoder } from "../strictUtf8.js";
type RequestEvent = Extract<BrowserScenarioEvent, { kind: "request" }>;
type ContentEvent = Extract<BrowserScenarioEvent, { kind: "network-content" }>;
@@ -79,7 +80,7 @@ const retainedBody = (
/** Strictly decoded text, keeping a byte-order mark; undefined for bytes that are not UTF-8. */
const utf8Text = (body: CapturedBody): string | undefined => {
try {
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
return strictUtf8Decoder("preserve").decode(
Buffer.from(body.content, "base64"),
);
} catch {
+2 -1
View File
@@ -1,3 +1,4 @@
import { strictUtf8Decoder } from "./strictUtf8.js";
/**
* Pure string predicates for artifact path syntax. The local artifact
* resolver, the Electron native specifier check, and (by reference) the
@@ -49,7 +50,7 @@ export const htmlUrlText = (value: string): string => {
return value.slice(start, end).replace(/[\t\n\r]/gu, "");
};
const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const UTF8 = strictUtf8Decoder("preserve");
const UTF8_ENCODER = new TextEncoder();
/**
+4
View File
@@ -93,6 +93,10 @@ describe("build secrets", () => {
expect(found(text)).toContain("licence");
expect(found(gzipSync(Buffer.from(text)))).toContain("licence");
}
// A UTF-8 signature must not hide the JSON strings from the scan.
const signed = Buffer.from(`{"password":"${encoded}"}`);
expect(found(signed)).toContain("licence");
expect(found(gzipSync(signed))).toContain("licence");
const utf16 = Buffer.from(`{"password":"${encoded}"}`, "utf16le");
for (const bytes of [
utf16,
@@ -50,3 +50,18 @@ describe("withoutRecoveryList", () => {
).toBeUndefined();
});
});
it("keeps a leading byte order mark among the bytes it ships", () => {
const bom = Buffer.from([0xef, 0xbb, 0xbf]);
const source = Buffer.concat([
bom,
Buffer.from(
registry(
`${KEPT}\n<item oor:path="/org.openoffice.Office.Recovery/RecoveryList">${RECOVERY}</item>`,
),
),
]);
const shippedBytes = withoutRecoveryList(REGISTRY, source) ?? source;
expect(shippedBytes.subarray(0, 3).equals(bom)).toBe(true);
expect(shippedBytes.toString("utf8")).not.toContain("recovery_item_1");
});
+2 -3
View File
@@ -1,4 +1,5 @@
import { xmlElementRanges } from "../xmlDocument.js";
import { strictUtf8Decoder } from "../strictUtf8.js";
/** LibreOffice's profile registry, where it lists the documents it has open while it runs. */
const REGISTRY = "registrymodifications.xcu";
@@ -20,9 +21,7 @@ export const withoutRecoveryList = (
if (home.split("/").at(-1) !== REGISTRY) return undefined;
let text: string;
try {
text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes,
);
text = strictUtf8Decoder("preserve").decode(bytes);
} catch {
return undefined;
}
+2 -4
View File
@@ -16,6 +16,7 @@ import {
isTextualFormat,
textSyntaxFormat,
} from "./stateFormats.js";
import { strictUtf8Decoder } from "../strictUtf8.js";
import { isRecord } from "../isRecord.js";
/** What the export ships for one seed. */
@@ -376,10 +377,7 @@ const seedText = (bytes: Buffer, { home, format }: Seed) => {
: { ...utf16, format: textSyntaxFormat(home, utf16.text) };
}
try {
const text = new TextDecoder("utf-8", {
fatal: true,
ignoreBOM: true,
}).decode(bytes);
const text = strictUtf8Decoder("preserve").decode(bytes);
return {
// The byte-order mark is kept, as Python's utf-8 codec keeps it.
text,
+2 -1
View File
@@ -16,6 +16,7 @@ import {
parseXmlDocument,
} from "../xmlDocument.js";
import { expatXmlName } from "./expatXmlNames.js";
import { decodeStrictUtf8 } from "../strictUtf8.js";
/** Formats the state view reads into JSON. */
export type DesktopStateFormat =
@@ -92,7 +93,7 @@ const isKeyfile = (text: string): boolean => {
export const decodeStateText = (bytes: Uint8Array): string | undefined => {
if (bytes.subarray(0, 8192).includes(0)) return undefined;
try {
return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
return decodeStrictUtf8(bytes, "strip-signature");
} catch {
return undefined;
}
+2 -1
View File
@@ -5,6 +5,7 @@ import {
ZIP_MAX_COMMENT_BYTES,
zipEndOfCentralDirectoryCandidates,
} from "../zipEndOfCentralDirectory.js";
import { strictUtf8Decoder } from "../strictUtf8.js";
const CENTRAL_HEADER = 0x02014b50;
const CENTRAL_HEADER_BYTES = 46;
@@ -17,7 +18,7 @@ const UNICODE_PATH = 0x7075;
const UNIX_TYPE = 0o170000;
const UNIX_DIRECTORY = 0o040000;
const MSDOS_DIRECTORY = 0x10;
const STRICT_UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const STRICT_UTF8 = strictUtf8Decoder("preserve");
/** An archive's central directory read a record at a time, bounded to the records it keeps. */
export interface ZipDirectory {
+2 -1
View File
@@ -5,6 +5,7 @@ import type {
EspImageMetadata,
EspImageSignatureBlockMetadata,
} from "../artifactFormatMetadata.js";
import { strictUtf8Decoder } from "../strictUtf8.js";
/*
* Fixed-layout structures inside an ESP image, from ESP-IDF v6.1
@@ -54,7 +55,7 @@ export const firstUnerased = (
const allZero = (bytes: Uint8Array): boolean =>
bytes.every((byte) => byte === 0);
const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const UTF8 = strictUtf8Decoder("preserve");
/** The bytes of a NUL-terminated C string in a fixed array. */
export const cStringBytes = (bytes: Uint8Array): Uint8Array => {
+2 -3
View File
@@ -5,6 +5,7 @@ import { safeParseJson } from "./safeJson.js";
import { jsonStringByteLength } from "./jsonStringByteLength.js";
import { jsonValueByteLength } from "./jsonValueByteLength.js";
import { err, ok, type Result } from "./result.js";
import { strictUtf8Decoder } from "./strictUtf8.js";
import { isRecord } from "./isRecord.js";
const jsonValueTypeSchema = z.enum([
@@ -86,9 +87,7 @@ export const inferJsonShape = (
text =
typeof source === "string"
? source
: new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
source,
);
: strictUtf8Decoder("preserve").decode(source);
} catch {
return err({ kind: "invalid-json" });
}
+3 -1
View File
@@ -1,5 +1,6 @@
import { ParseFailureError } from "./parseFailure.js";
import { byteOrderMarkEncoding } from "./textByteOrderMark.js";
import { decodeStrictUtf8 } from "./strictUtf8.js";
const XML_PLIST_FORMAT = "xml-plist";
@@ -85,8 +86,9 @@ const asciiXmlDeclaration = (bytes: Uint8Array): string | undefined => {
for (let index = start + 5; index < bytes.length; index += 1) {
if (bytes[index]! > 0x7f) return undefined;
if (bytes[index] === 0x3f && bytes[index + 1] === 0x3e)
return new TextDecoder("utf-8", { fatal: true }).decode(
return decodeStrictUtf8(
bytes.subarray(start, index + 2),
"strip-signature",
);
}
return undefined;
+2 -1
View File
@@ -1,7 +1,8 @@
import { err, ok, type Result } from "../result.js";
import { strictUtf8Decoder } from "../strictUtf8.js";
import { invariantViolation } from "../internalInvariant.js";
const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const UTF8 = strictUtf8Decoder("preserve");
export interface DecodedMarshalUtf8 {
readonly text: string;
+2 -1
View File
@@ -26,6 +26,7 @@ import {
isTSImportEqualsDeclaration,
isTSModuleDeclaration,
} from "@babel/types";
import { decodeStrictUtf8 } from "./strictUtf8.js";
type ReferenceSourceImportKind =
| "imports"
@@ -291,7 +292,7 @@ export const parseReferenceSourceImports = (
let source: string;
try {
source = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
source = decodeStrictUtf8(bytes, "strip-signature");
} catch {
return {
relationships: [],
+2 -3
View File
@@ -1,3 +1,4 @@
import { strictUtf8Decoder } from "./strictUtf8.js";
/** JSON parsing without throws at an input boundary. */
export type SafeJsonParseResult =
| { readonly ok: true; readonly value: unknown }
@@ -7,9 +8,7 @@ export type SafeJsonParseResult =
export const safeParseUtf8Json = (bytes: Uint8Array): SafeJsonParseResult => {
let text: string;
try {
text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes,
);
text = strictUtf8Decoder("preserve").decode(bytes);
} catch (cause: unknown) {
return {
ok: false,
+25
View File
@@ -0,0 +1,25 @@
import { expect, it } from "vitest";
import {
decodeStrictUtf8,
isStrictUtf8,
strictUtf8Decoder,
} from "./strictUtf8.js";
const bomPrefixed = Uint8Array.of(0xef, 0xbb, 0xbf, 0x7b, 0x7d);
it("keeps a leading BOM unless the caller states it is a signature", () => {
expect(decodeStrictUtf8(bomPrefixed, "preserve")).toBe("{}");
expect(decodeStrictUtf8(bomPrefixed, "strip-signature")).toBe("{}");
expect(strictUtf8Decoder("preserve").decode(bomPrefixed)).toBe("{}");
});
it("refuses malformed UTF-8 for every policy", () => {
const malformed = Uint8Array.of(0x61, 0xc3);
expect(() => decodeStrictUtf8(malformed, "preserve")).toThrow(TypeError);
expect(() => decodeStrictUtf8(malformed, "strip-signature")).toThrow(
TypeError,
);
expect(isStrictUtf8(malformed)).toBe(false);
expect(isStrictUtf8(bomPrefixed)).toBe(true);
});
+35
View File
@@ -0,0 +1,35 @@
import { isUtf8 } from "node:buffer";
/**
* What a strict UTF-8 decode does with one leading byte order mark.
*
* - `strip-signature` drops it as an encoding signature. This is the WHATWG
* decoder default, so it is what a plain `{ fatal: true }` decoder did.
* - `preserve` keeps it as U+FEFF, so the text matches the caller's bytes and
* offsets (`ignoreBOM: true`). Use it where a leading U+FEFF is content.
*
* Changing a site's policy changes what its consumers see (for example an
* exact-value secret match or a JSON parser that rejects U+FEFF), so a change
* needs a regression test at that consumer.
*/
export type Utf8BomPolicy = "preserve" | "strip-signature";
/**
* A fatal UTF-8 decoder with an explicit BOM policy. It throws a TypeError on
* malformed bytes. Callers that pass `{ stream: true }` need their own
* instance, so this returns a new decoder each call.
*/
export const strictUtf8Decoder = (bom: Utf8BomPolicy): TextDecoder =>
new TextDecoder("utf-8", { fatal: true, ignoreBOM: bom === "preserve" });
const preserving = strictUtf8Decoder("preserve");
const stripping = strictUtf8Decoder("strip-signature");
/** Decode complete UTF-8 bytes; malformed input throws a TypeError. */
export const decodeStrictUtf8 = (
bytes: Uint8Array,
bom: Utf8BomPolicy,
): string => (bom === "preserve" ? preserving : stripping).decode(bytes);
/** Whether the bytes are well-formed UTF-8, without building the string. */
export const isStrictUtf8 = (bytes: Uint8Array): boolean => isUtf8(bytes);
+2 -1
View File
@@ -1,6 +1,7 @@
import { crc32 } from "node:zlib";
import { err, ok, type Result } from "./result.js";
import { strictUtf8Decoder } from "./strictUtf8.js";
/**
* U-Boot legacy uImage header (U-Boot `include/image.h`, all fields
@@ -191,7 +192,7 @@ const malformed = (message: string): UImageParseError => ({
_tag: "UImageFormatFailure",
message,
});
const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const UTF8 = strictUtf8Decoder("preserve");
/** True when the first four bytes are the U-Boot legacy magic. */
export const hasUImageMagic = (prefix: Buffer): boolean =>
+2 -1
View File
@@ -1,3 +1,4 @@
import { strictUtf8Decoder } from "./strictUtf8.js";
/**
* UnityFS (AssetBundle) container structure: a fixed signature header followed
* by a compressed blocks-and-directory table and a block-compressed data area.
@@ -119,7 +120,7 @@ export interface UnityFsHeaderFacts {
readonly afterFixed: number;
}
const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const utf8 = strictUtf8Decoder("preserve");
/**
* Decode declared fields after the caller admits a prefix covering the
+2 -4
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import type { ManagedParseIssue } from "../domain/managed/managedArtifact.js";
import { createHash } from "node:crypto";
import { err, ok, type Result } from "../domain/result.js";
@@ -159,10 +160,7 @@ export const readMetadataString = (
);
managedDecodeBudget(layout).reserveString(end - start, start);
try {
const value = new TextDecoder("utf-8", {
fatal: true,
ignoreBOM: true,
}).decode(bytes.subarray(start, end));
const value = decodeStrictUtf8(bytes.subarray(start, end), "preserve");
managedDecodeBudget(layout).cacheString(index, value, end - start);
return value;
} catch (cause: unknown) {
+2 -3
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import { readMemberReferenceRowFacts } from "./ManagedMemberReferenceRows.js";
import { readTypeDefinitionRowFacts } from "./ManagedTypeDefinitionRows.js";
import { managedDecodeBudget } from "./ManagedDecodeBudget.js";
@@ -434,9 +435,7 @@ const decodeFixedString = (
layout.blob?.offset ?? layout.rootOffset,
);
try {
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
blob.subarray(start, start + length),
);
return decodeStrictUtf8(blob.subarray(start, start + length), "preserve");
} catch (cause: unknown) {
// Invalid UTF-8 blobs decode as null.
void cause;
+2 -4
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import { assertArtifactReadActive } from "../artifacts/ArtifactFileOperation.js";
import type { BigIntStats } from "node:fs";
import { lstat, readlink } from "node:fs/promises";
@@ -215,10 +216,7 @@ const resolveInside = async (
try {
// A replacement character can alias a different, valid sibling name.
// Preserve a literal UTF-8 BOM too: it is part of a filesystem name.
target = new TextDecoder("utf-8", {
fatal: true,
ignoreBOM: true,
}).decode(rawTarget);
target = decodeStrictUtf8(rawTarget, "preserve");
} catch {
throw new ElectronFusePathFailure(
"invalid_value",
+2 -1
View File
@@ -34,7 +34,8 @@ it("rejects a long interior separator run without pathological backtracking", as
`;
await promisify(execFile)(
process.execPath,
["--input-type=module", "-e", source],
// tsx resolves the module's own `.js` imports to their TypeScript sources.
["--import", "tsx", "--input-type=module", "-e", source],
{
timeout: 15_000,
killSignal: "SIGKILL",
+2 -3
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
/** Expected carrier encoding failure, independent of the analysis engine. */
export class EvmCarrierFailure extends Error {}
@@ -13,9 +14,7 @@ export const decodeEvmBytecodeCarrier = (
if (encoding === "raw") return Uint8Array.from(bytes);
let text: string;
try {
text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes,
);
text = decodeStrictUtf8(bytes, "preserve");
} catch (cause: unknown) {
throw new EvmCarrierFailure("Hex carrier must be valid UTF-8.", { cause });
}
+2 -1
View File
@@ -1,3 +1,4 @@
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
import { EvmAdmissionError } from "./EvmAdmissionError.js";
import { decode, Tokenizer, Type, type Token } from "cborg";
import type {
@@ -7,7 +8,7 @@ import type {
const PROFILE = "terminal-cbor-scalars-v1";
const MAX_DEPTH = 64;
const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const utf8 = strictUtf8Decoder("preserve");
const decodeOptions = {
useMaps: true,
allowBigInt: true,
+3 -1
View File
@@ -16,6 +16,7 @@ import {
} from "./GhidraFileArtifacts.js";
import { NonRegularFileReadError } from "../filesystem/RegularFile.js";
import { err, ok, type Result } from "../domain/result.js";
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
/** Seed file format admitted by this adapter. */
export const GHIDRA_SEED_FORMAT = "rea-ghidra-seeds-v1";
@@ -170,7 +171,8 @@ const readSeeds = (
(bytes) => {
let text: string;
try {
text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
// A user-authored seed file may start with a BOM signature.
text = decodeStrictUtf8(bytes, "strip-signature");
} catch {
throw seedLineError(0, "file is not valid UTF-8");
}
+2 -4
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import { GhidraAdmissionError } from "./GhidraAdmissionError.js";
import { randomUUID } from "node:crypto";
import { join } from "node:path";
@@ -147,10 +148,7 @@ const readArgumentFile = (
throw new GhidraAdmissionError("argument_files_exceed_bound", {
budget_bytes: OPTIONS_BYTES,
});
const text = new TextDecoder("utf-8", {
fatal: true,
ignoreBOM: true,
}).decode(data);
const text = decodeStrictUtf8(data, "preserve");
const tokens = argumentFileTokens(text);
if (tokens.some(fileReference))
throw new GhidraAdmissionError("nested_argument_file");
+2 -3
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import { readLeb128, ULEB64 } from "../domain/leb128.js";
import type {
FileMapping,
@@ -294,9 +295,7 @@ const inlineDataEnd = (
const observedText = (bytes: Buffer): string | null => {
try {
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes,
);
return decodeStrictUtf8(bytes, "preserve");
} catch (cause) {
if (cause instanceof TypeError) return null;
throw cause;
+2 -1
View File
@@ -1,6 +1,7 @@
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
import type { GoModule, GoModuleMetadata } from "../domain/go/goBinary.js";
const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const utf8 = strictUtf8Decoder("preserve");
const escapes = new Map([
["a", 7],
["b", 8],
+2 -1
View File
@@ -1,3 +1,4 @@
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
import { decode } from "@msgpack/msgpack";
import {
BinaryResourceFailure,
@@ -12,7 +13,7 @@ const AMD_METADATA_CONTAINER_ENTRIES = 65536;
const AMD_METADATA_VALUE_BYTES = 65536;
/** Validate allocation and nesting costs before the standard decoder allocates containers. */
const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const utf8 = strictUtf8Decoder("preserve");
const preflight = (bytes: Buffer): Set<string> => {
const strings = new Set<string>();
let at = 0,
+3 -1
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import { scheduler } from "node:timers/promises";
import type { GpuObject } from "../domain/gpu/gpuCodeObjects.js";
import {
@@ -69,8 +70,9 @@ export const readClangBundleDirectory = async (
);
let name: string;
try {
name = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
name = decodeStrictUtf8(
await source.read(position + 24, nameSize),
"preserve",
);
} catch (cause) {
if (cause instanceof TypeError)
+2 -4
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import {
CCOB_MAGIC,
CLANG_BUNDLE_MAGIC,
@@ -366,10 +367,7 @@ export class GpuInventory {
const content = bytes.subarray(0, length);
let text: string;
try {
text = new TextDecoder("utf-8", {
fatal: true,
ignoreBOM: true,
}).decode(content);
text = decodeStrictUtf8(content, "preserve");
} catch {
return rejectGpu("PTX is not valid UTF-8 text");
}
+2 -3
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import {
redactIdaJsonDiagnostic,
redactIdaUnclassifiedDiagnostic,
@@ -42,9 +43,7 @@ const basicPayloads = (payload: string): readonly string[] => {
if (octets.some((value) => value < 0x20 || value === 0x7f)) return [];
const candidates = [octets.toString("latin1")];
try {
candidates.push(
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(octets),
);
candidates.push(decodeStrictUtf8(octets, "preserve"));
} catch {
// Invalid UTF-8 has no decoded Unicode candidate. Exact Latin-1 preserves
// each octet; do not invent a replacement-character credential.
+2 -1
View File
@@ -1,3 +1,4 @@
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
import {
checkRange,
checkedSectionCount,
@@ -355,7 +356,7 @@ const createStringReader = (
) => {
const stringEnd = strings.offset + strings.size;
const pool = bytes.subarray(strings.offset, stringEnd);
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const decoder = strictUtf8Decoder("preserve");
const cache = new Map<number, string>();
return (index: number, label: string) => {
const cached = cache.get(index);
+2 -1
View File
@@ -1,3 +1,4 @@
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
import type { Il2cppMetadata } from "../domain/il2cpp/il2cppMetadata.js";
import {
checkRange,
@@ -141,7 +142,7 @@ function* readStringLiterals(
const { bytes, section, indexing, members } = tables;
const literals = section("stringLiterals"),
data = section("stringLiteralData");
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const decoder = strictUtf8Decoder("preserve");
// Repeated spans reuse their decoded text for this immutable file snapshot.
const decoded = new Map<string, Members["string_literals"][number]>();
const rows: Members["string_literals"] = [];
+2 -1
View File
@@ -1,3 +1,4 @@
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
import { IL2CPP_METADATA_SECTION_NAMES } from "../domain/il2cpp/il2cppMetadata.js";
import { observeIl2cppMetadataHeaderPrefix } from "../domain/il2cpp/il2cppMetadataHeaderPrefix.js";
import { Il2cppMetadataFormatFailure } from "./Il2cppMetadataFormat.js";
@@ -23,7 +24,7 @@ const HEADER_BYTES = 8 + SECTION_NAMES.length * 12;
const SAMPLE_STRINGS = 16;
const SAMPLE_BYTES = 128;
const SCAN_BATCH_BYTES = 64 * 1024;
const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const utf8 = strictUtf8Decoder("preserve");
/** A bounded, header-only observation; never a decoded metadata report. */
export const inspectIl2cppMetadataV108 = async (
+3 -6
View File
@@ -18,7 +18,6 @@ import {
writeFile,
} from "node:fs/promises";
import { basename, dirname, isAbsolute, join, relative, sep } from "node:path";
import { TextDecoder } from "node:util";
import { normalizeArtifactPath } from "../../artifacts/ArtifactPaths.js";
import {
AnalysisAccessDeniedError,
@@ -36,6 +35,7 @@ import {
sameRegularFileState,
} from "../../filesystem/RegularFile.js";
import { RECOVERY_LIMITS, wakaruFixedLimitRecovery } from "./WakaruRelease.js";
import { isStrictUtf8 } from "../../domain/strictUtf8.js";
const OPERATION = "recover_javascript_sources";
@@ -222,10 +222,8 @@ const snapshotRecoveryInput = async (
cause,
);
}
try {
new TextDecoder("utf-8", { fatal: true }).decode(bytes);
} catch (cause: unknown) {
throw new AnalysisInputError(OPERATION, { cause }, [
if (!isStrictUtf8(bytes))
throw new AnalysisInputError(OPERATION, {}, [
{
path: ["path"],
reason: "invalid_format",
@@ -233,7 +231,6 @@ const snapshotRecoveryInput = async (
message: `JavaScript input is not valid UTF-8: ${path}`,
},
]);
}
try {
await mkdir(dirname(snapshotPath), { recursive: true, mode: 0o700 });
await writeFile(snapshotPath, bytes, { flag: "wx", mode: 0o600 });
@@ -16,6 +16,7 @@ import {
sourceMapCodecReplySchema,
} from "./SourceMapCodecProtocol.js";
import { z } from "zod";
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
/** Dedicated process entrypoint invoked only for an explicitly selected map. */
const execute = async (): Promise<string> => {
@@ -28,7 +29,8 @@ const execute = async (): Promise<string> => {
);
const limits = selectedSourceMapLimits(selected);
const data = await readStableArtifact(path, limits.outputBytes);
const text = new TextDecoder("utf-8", { fatal: true }).decode(data.bytes);
// Source maps on disk may carry a BOM signature (RFC 8259 8.1).
const text = decodeStrictUtf8(data.bytes, "strip-signature");
const value: unknown = JSON.parse(text);
const input = sourceMapCodecInputSchema.parse(value);
if (Buffer.byteLength(input.text) > limits.mapBytes)
+2 -1
View File
@@ -16,6 +16,7 @@ import {
import { err, ok, type Result } from "../domain/result.js";
import { KunaDocumentError } from "./KunaDocumentError.js";
import { kunaAddressFromJsonNumber } from "./KunaJson.js";
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
/**
* stream-json 3's synchronous tokenizer behind `parser`, which its
@@ -136,7 +137,7 @@ const detached = (text: string): string =>
*/
export abstract class KunaJsonDocumentReader<Parsed> extends Writable {
readonly #hash = createHash("sha256");
readonly #decoder = new TextDecoder("utf-8", { fatal: true });
readonly #decoder = strictUtf8Decoder("strip-signature");
readonly #tokens = jsonParser({
packKeys: false,
packNumbers: false,
+2 -2
View File
@@ -1,5 +1,5 @@
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
import { isUtf8 } from "node:buffer";
import { TextDecoder } from "node:util";
import { setImmediate } from "node:timers/promises";
import type { JsonValue } from "../domain/jsonValue.js";
import type {
@@ -82,7 +82,7 @@ export class GgufHeader {
rejectModel("GGUF string is not valid UTF-8");
return "";
}
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const decoder = strictUtf8Decoder("preserve");
let result = "";
try {
for (let left = length; left > 0;) {
+2 -6
View File
@@ -1,7 +1,7 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import { escapeJsonPointerToken } from "../domain/jsonPointer.js";
import { setImmediate } from "node:timers/promises";
import { assertArtifactReadActive } from "../artifacts/ArtifactFileOperation.js";
import { TextDecoder } from "node:util";
import type {
ModelMetadata,
ModelTensor,
@@ -235,11 +235,7 @@ export const readSafetensorsHeader = async (
budget.charge(2 * size);
let text: string;
try {
text =
"{" +
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
await reader.take(size - 1),
);
text = "{" + decodeStrictUtf8(await reader.take(size - 1), "preserve");
} catch (cause) {
if (cause instanceof TypeError)
rejectModel("Safetensors header is not valid UTF-8");
+2 -3
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import {
malformedInput,
parseFailureFromCause,
@@ -165,9 +166,7 @@ export function* decodeAppleDispatchMetadataSteps(
);
offset(address, end - start + 1);
// A metadata C string owns every code point, including an initial U+FEFF.
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes.subarray(start, end),
);
return decodeStrictUtf8(bytes.subarray(start, end), "preserve");
};
const result = objcSwiftMetadataSchema.parse({
db_save_result: null,
+2 -3
View File
@@ -1,12 +1,11 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
/** A dyld byte name projected losslessly as UTF-8, or retained exactly. */
export type MachoName = string | { readonly bytes_base64: string };
/** NUL terminators and source bounds are validated by the owning format reader. */
export const decodeMachoName = (bytes: Buffer): MachoName => {
try {
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes,
);
return decodeStrictUtf8(bytes, "preserve");
} catch {
return { bytes_base64: bytes.toString("base64") };
}
+2 -3
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import { bindArtifactFileCleanup } from "../artifacts/ArtifactFileOperation.js";
import { HOST_PLATFORM } from "../platform/HostPlatform.js";
import { invariantViolation } from "../domain/internalInvariant.js";
@@ -1198,9 +1199,7 @@ const readTracerOutput = async (
if (!bytes.value.ok) return bytes.value;
let text: string;
try {
text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes.value.value,
);
text = decodeStrictUtf8(bytes.value.value, "preserve");
} catch (cause: unknown) {
return err(cause);
}
+2 -1
View File
@@ -1,3 +1,4 @@
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
import {
NATIVE_CALL_CAPTURE_DEFAULTS,
type NativeCallCaptureLimits,
@@ -108,7 +109,7 @@ const parseLldbObservationJournal = (bytes: Uint8Array): ParsedLldbJournal => {
const events: ParsedLldbJournal["events"] = [];
const otherStops: string[] = [];
const limitations: string[] = [];
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const decoder = strictUtf8Decoder("preserve");
let malformedCount = 0;
let firstMalformedRow: number | undefined;
let start = 0;
+2 -3
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import { legacyMachoPlatform } from "../domain/apple/machoPlatform.js";
import {
LC,
@@ -88,9 +89,7 @@ const bytesText = (input: Uint8Array, fixed: boolean) => {
const bytes = end < 0 ? input : input.subarray(0, end);
try {
return {
text: new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
bytes,
),
text: decodeStrictUtf8(bytes, "preserve"),
bytesHex: null,
};
} catch {
+4 -2
View File
@@ -1,6 +1,9 @@
import { MAX_NODE_TIMER_DELAY_MS } from "../domain/analysisResourceLimits.js";
import { withArtifactTargetIdentity } from "../artifacts/withArtifactTargetIdentity.js";
import { invariantViolation } from "../domain/internalInvariant.js";
import {
invariantViolation,
InternalInvariantError,
} from "../domain/internalInvariant.js";
import { translateNativeCommandFailure } from "./NativeCommandAnalysisError.js";
import {
plistSourceFormat,
@@ -114,7 +117,6 @@ import {
type NativeCommandInvocation,
} from "../domain/native/nativeInspection.js";
import { err, ok, type Result } from "../domain/result.js";
import { InternalInvariantError } from "../domain/internalInvariant.js";
import {
NativeCommandFailure,
NATIVE_COMMAND_OUTPUT_BUDGET_BYTES,
+2 -1
View File
@@ -18,6 +18,7 @@ import {
IPS_METADATA_SHAPE,
} from "./AppleIpsJsonShape.js";
import type { LosslessJson } from "./losslessJson.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
type Artifact = AppleIps["artifact"];
@@ -30,7 +31,7 @@ const IPS_LIMITATIONS = [
"Symbol names printed inside the report are recorded claims and are not verified.",
];
const utf8 = new TextDecoder("utf-8", { fatal: true });
const utf8 = strictUtf8Decoder("strip-signature");
const canonicalUuid = (
raw: string | null,
+2 -1
View File
@@ -12,6 +12,7 @@ import {
type KernelBuildConfigEntry,
} from "../../domain/native/kernelBuildConfig.js";
import { kernelCompressionCodecAt } from "./KernelCompressionCodec.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
type Report = KernelBuildConfig;
type Entry = KernelBuildConfigEntry;
@@ -234,7 +235,7 @@ const inflateGzipMember = async (
return { kind: "member", output, end: trailer + 8 };
};
const decoder = new TextDecoder("utf-8", { fatal: true });
const decoder = strictUtf8Decoder("strip-signature");
/** C0 controls other than tab, LF and CR, plus DEL. */
const CONTROL = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/u;
+2 -3
View File
@@ -44,6 +44,7 @@ import {
decodeXmlPlistText,
XmlPlistTextDecodeError,
} from "../../domain/propertyListXmlText.js";
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
/**
* Input ceiling matches native command output; projection is bounded separately.
@@ -202,9 +203,7 @@ export const readPlistJsonSource = (
bytes: Buffer,
): PlistJsonParseAttempt | undefined => {
try {
return readPlistJson(
new TextDecoder("utf-8", { fatal: true }).decode(bytes),
);
return readPlistJson(decodeStrictUtf8(bytes, "strip-signature"));
} catch {
return undefined;
}
+6 -5
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import { OwnerCancellation } from "../domain/analysisErrorCore.js";
import {
deserializeMessage,
@@ -386,11 +387,11 @@ export class OwnedMcpStdioTransport implements Transport {
);
this.#appendFrame(fragment, frameBytes);
if (newline === -1) return;
const line = new TextDecoder("utf-8", {
fatal: true,
// Preserve a leading BOM so the JSON-RPC parser can reject it.
ignoreBOM: true,
}).decode(this.#frameBuffer.subarray(0, this.#frameLength));
// Preserve a leading BOM so the JSON-RPC parser can reject it.
const line = decodeStrictUtf8(
this.#frameBuffer.subarray(0, this.#frameLength),
"preserve",
);
this.#frameBuffer = Buffer.alloc(0);
this.#frameLength = 0;
this.onmessage?.(deserializeMessage(line));
+2 -6
View File
@@ -37,6 +37,7 @@ import {
type ManagedRuntime,
type RuntimeExhaustion,
} from "./RuntimeExhaustion.js";
import { isStrictUtf8 } from "../domain/strictUtf8.js";
const DEFAULT_TERMINATION_GRACE_MS = 250;
const DEFAULT_KILL_GRACE_MS = 1_000;
@@ -1143,12 +1144,7 @@ class ProcessOutputCapture {
observedBytes: number,
): ProviderProcessSnapshot["stdout"] {
const text = buffer.toString("utf8");
let invalidUtf8 = false;
try {
new TextDecoder("utf-8", { fatal: true }).decode(buffer);
} catch {
invalidUtf8 = true;
}
const invalidUtf8 = !isStrictUtf8(buffer);
return {
text,
bytes: buffer.byteLength,
+4 -2
View File
@@ -32,6 +32,7 @@ import {
regexWorkerHeapRequestRecovery,
type RegexWorkerHeapRequest,
} from "../config/regexWorkerHeapRequest.js";
import { decodeStrictUtf8, strictUtf8Decoder } from "../domain/strictUtf8.js";
/**
* Node's Worker `resourceLimits.maxOldGenerationSizeMb` request.
* `REA_REGEX_WORKER_HEAP_REQUEST_MIB` is that request (default 128 MiB). A positive
@@ -232,8 +233,9 @@ const nodeOptionsFromEnviron = (
entry.subarray(0, nodeOptionsKey.length).equals(nodeOptionsKey)
) {
try {
return new TextDecoder("utf-8", { fatal: true }).decode(
return decodeStrictUtf8(
entry.subarray(nodeOptionsKey.length),
"strip-signature",
);
} catch {
return "invalid";
@@ -288,7 +290,7 @@ const commandLineTokens = (raw: Buffer): readonly string[] | "invalid" => {
if (raw.length === 0) return "invalid";
const end = raw[raw.length - 1] === 0 ? raw.length - 1 : raw.length;
const tokens: string[] = [];
const decoder = new TextDecoder("utf-8", { fatal: true });
const decoder = strictUtf8Decoder("strip-signature");
let offset = 0;
while (offset < end) {
const next = raw.indexOf(0, offset);
+2 -4
View File
@@ -1,3 +1,4 @@
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
import {
malformedInput,
type ParseFailureError,
@@ -12,10 +13,7 @@ export class Utf8NdjsonLineBuffer {
#buffer: Buffer = Buffer.alloc(0);
#fragments: Buffer[] = [];
#nextNewline = -1;
readonly #decoder = new TextDecoder("utf-8", {
fatal: true,
ignoreBOM: true,
});
readonly #decoder = strictUtf8Decoder("preserve");
/**
* Yield each complete trimmed nonblank line; preserve unconsumed lines on
+2 -1
View File
@@ -1,3 +1,4 @@
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
import { AnalysisProtocolError } from "../domain/analysisErrorCore.js";
/** Authored byte-validation failure, distinct from arbitrary upstream protocol diagnostics. */
@@ -14,7 +15,7 @@ export const validateMcpHttpResponse = (
): Response => {
// Failed HTTP responses belong to each adapter's existing diagnostic policy.
if (!response.ok || response.body === null) return response;
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const decoder = strictUtf8Decoder("preserve");
const validate = (bytes?: Uint8Array): void => {
try {
decoder.decode(bytes, { stream: bytes !== undefined });
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import type { BigIntStats, Dir, Dirent } from "node:fs";
import { opendir, readlink, realpath, stat } from "node:fs/promises";
import { basename, dirname, join, sep } from "node:path";
@@ -478,10 +479,7 @@ const readSymlinkTarget = async (
}
let target: string;
try {
target = new TextDecoder("utf-8", {
fatal: true,
ignoreBOM: true,
}).decode(rawTarget);
target = decodeStrictUtf8(rawTarget, "preserve");
} catch {
return entryFailure(
path,
+2 -1
View File
@@ -10,6 +10,7 @@ import {
import { jsonValueByteLength } from "../domain/jsonValueByteLength.js";
import { malformedInput, ParseFailureError } from "../domain/parseFailure.js";
import { findDuplicateJsonMember } from "../domain/jsonDuplicateMember.js";
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
/** Compressed zlib stream limit. */
const RUST_SECTION_BYTES = 1024 * 1024;
@@ -225,7 +226,7 @@ const parseDocument = (
): z.infer<typeof documentSchema> => {
let text: string;
try {
text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
text = decodeStrictUtf8(bytes, "strip-signature");
} catch (cause) {
throw malformedSection("Dependency JSON is not valid UTF-8", cause);
}
+3 -1
View File
@@ -19,6 +19,7 @@ import {
RegularFileCleanupFailure,
withOwnedRegularFile,
} from "../../filesystem/OwnedRegularFile.js";
import { decodeStrictUtf8 } from "../../domain/strictUtf8.js";
import { isRecord } from "../../domain/isRecord.js";
/** Why a named file or download could not be read, or a call was cancelled: a category naming neither the uri nor the content. */
@@ -133,7 +134,8 @@ const MAX_INLINE_BASE64_CHARACTERS =
const parseDocumentBytes = (bytes: Uint8Array): Result<unknown, RpcError> => {
let text: string;
try {
text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
// A user-saved JSON document may carry a BOM signature (RFC 8259 8.1).
text = decodeStrictUtf8(bytes, "strip-signature");
} catch {
return err(invalidParams("the file is not UTF-8 text"));
}
+2 -4
View File
@@ -1,3 +1,4 @@
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
import { WabtAdmissionError } from "./WabtAdmissionError.js";
import { HOST_PLATFORM } from "../platform/HostPlatform.js";
import { randomUUID, createHash } from "node:crypto";
@@ -494,10 +495,7 @@ export class WabtArtifactProvider implements WasmArtifactPort {
};
try {
// Keep source characters intact so locations match the hashed artifact.
const source = new TextDecoder("utf-8", {
fatal: true,
ignoreBOM: true,
}).decode(bytes.bytes);
const source = decodeStrictUtf8(bytes.bytes, "preserve");
glue[glueIndex] = {
artifact: glueArtifact,
...associateWasmGlue(
Loaded 100 of 103 files, more files were not shown because too many files have changed in this diff. Show more