diff --git a/scripts/verify/architecture-guard-baselines.json b/scripts/verify/architecture-guard-baselines.json index 5f1eb772af..3174a697f6 100644 --- a/scripts/verify/architecture-guard-baselines.json +++ b/scripts/verify/architecture-guard-baselines.json @@ -301,6 +301,9 @@ "src/sqlite/SqliteDatabaseWorkerExecution.ts": 2, "src/wasm/WabtArtifactProvider.ts": 1 }, + "strict-utf8-decoder": { + "src/browser/PlaywrightScenarioRequest.ts": 1 + }, "path-search-split": { "src/process/capture/ProcessScenarioRuntimeValidation.ts": 1 }, diff --git a/scripts/verify/architecture-guards.mjs b/scripts/verify/architecture-guards.mjs index b532bac84c..0a48e32802 100644 --- a/scripts/verify/architecture-guards.mjs +++ b/scripts/verify/architecture-guards.mjs @@ -330,6 +330,19 @@ checkCounted( ["src/pcap/TsharkJsonPackets.ts"], ); +// Strict UTF-8 decoding has one owner, src/domain/strictUtf8.ts, which makes +// each site state its byte-order-mark policy. A local fatal decoder silently +// strips a leading BOM unless it remembers ignoreBOM. The pattern matches the +// options in any order and an omitted label. The remaining copy runs inside a +// browser page through page.evaluate, which cannot import the owner. +checkCounted( + "local strict UTF-8 TextDecoder (use strictUtf8Decoder, decodeStrictUtf8 or isStrictUtf8 from src/domain/strictUtf8)", + "strict-utf8-decoder", + files, + /new TextDecoder\(\s*(?:(?:["'`]utf-?8["'`]|undefined)\s*,\s*)?\{[^}]*\bfatal:\s*true/giu, + ["src/domain/strictUtf8.ts"], +); + // Turning a completed result into a cancellation after cleanup is the // finalization precedence that runOwned owns (src/providers/runOwned.ts). checkCounted( diff --git a/src/android/AndroidMappingFile.ts b/src/android/AndroidMappingFile.ts index 200061ab0b..e8dcd71d10 100644 --- a/src/android/AndroidMappingFile.ts +++ b/src/android/AndroidMappingFile.ts @@ -28,6 +28,7 @@ import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { err, ok, type Result } from "../domain/result.js"; import { isSelectedRegularFileError } from "../filesystem/RegularFile.js"; import { jadxOperationTimeoutGuidance } from "./JadxOperationTimeout.js"; +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; export interface AndroidMappingReadDeadline { readonly timeout: AbortSignal; @@ -192,7 +193,8 @@ export const readAndroidMapping = async ( ); let text: string; try { - text = new TextDecoder("utf-8", { fatal: true }).decode(snapshot.bytes); + // A mapping file saved by an editor may start with a BOM signature. + text = decodeStrictUtf8(snapshot.bytes, "strip-signature"); } catch { return malformed(`Mapping file is not valid UTF-8: ${path}`); } diff --git a/src/apktool/ApktoolWorkspace.ts b/src/apktool/ApktoolWorkspace.ts index 176a7c5147..ce12e8ac46 100644 --- a/src/apktool/ApktoolWorkspace.ts +++ b/src/apktool/ApktoolWorkspace.ts @@ -27,6 +27,7 @@ import { } from "../filesystem/RegularFile.js"; import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js"; import type { ApktoolCommandCleanup } from "./ApktoolCommandCleanup.js"; +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; /** One decode's descriptors and root; removal follows confirmed child cleanup. */ export class ApktoolWorkspace { @@ -203,7 +204,7 @@ export class ApktoolWorkspace { const sourceBytes = Math.min(length, budget); try { return { - text: new TextDecoder("utf-8", { fatal: true }).decode( + text: strictUtf8Decoder("strip-signature").decode( bytes.subarray(0, sourceBytes), { stream: truncated }, ), diff --git a/src/application/HermesClientHome.ts b/src/application/HermesClientHome.ts index 91ec5f0a2a..e4ec2d7d93 100644 --- a/src/application/HermesClientHome.ts +++ b/src/application/HermesClientHome.ts @@ -5,6 +5,7 @@ import { absoluteCallerPath, appendCallerPath, } from "../filesystem/callerPath.js"; +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; /** An unresolved Hermes profile must not be configured as the default home. */ export class HermesClientPathError extends Error { @@ -178,9 +179,10 @@ export const resolveHermesClientHome = ( let name: string; try { // Python uses utf-8-sig and ignores unreadable / invalid-UTF8 active_profile files. - name = new TextDecoder("utf-8", { fatal: true }) - .decode(readFileSync(appendCallerPath(stickyRoot, "active_profile"))) - .trim(); + name = decodeStrictUtf8( + readFileSync(appendCallerPath(stickyRoot, "active_profile")), + "strip-signature", + ).trim(); } catch { return root; } diff --git a/src/application/JsonInputFile.ts b/src/application/JsonInputFile.ts index 3a25d42313..9853de559e 100644 --- a/src/application/JsonInputFile.ts +++ b/src/application/JsonInputFile.ts @@ -1,3 +1,4 @@ +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; import { constants } from "node:buffer"; import { createHash } from "node:crypto"; import type { Stats } from "node:fs"; @@ -717,7 +718,7 @@ async function* decodedChunks( observe?: (bytes: Uint8Array) => void, physicalRead: JsonInputPhysicalReadObservation = {}, ): AsyncGenerator { - const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + const decoder = strictUtf8Decoder("preserve"); let leading = true; const checked = (text: string): string => { if (leading && text.length > 0) { diff --git a/src/application/LinuxHopperPublisher.ts b/src/application/LinuxHopperPublisher.ts index a204fec15b..36e84b7d1a 100644 --- a/src/application/LinuxHopperPublisher.ts +++ b/src/application/LinuxHopperPublisher.ts @@ -54,6 +54,7 @@ import { type LinuxPackageFamily, } from "./HopperReleaseFeed.js"; import { withRegularFile } from "./RegularFileRead.js"; +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; /** Metadata/key inspection budget, independent of the compressed package budget. */ const RELEASE_METADATA_BYTES = 1024 * 1024; @@ -1082,7 +1083,7 @@ export const retryLinuxHopperPublisherCleanup = async (): Promise => { }; const decodeUtf8 = (bytes: Uint8Array): string => - new TextDecoder("utf-8", { fatal: true }).decode(bytes); + decodeStrictUtf8(bytes, "strip-signature"); const releaseFailure = ( cause: unknown, signal: AbortSignal | undefined, diff --git a/src/application/McpRegistrationGit.ts b/src/application/McpRegistrationGit.ts index 1f42dfacb6..3e444b27c6 100644 --- a/src/application/McpRegistrationGit.ts +++ b/src/application/McpRegistrationGit.ts @@ -8,6 +8,7 @@ import { } from "../domain/analysisErrorCore.js"; import { analysisErrorWithCancellation } from "../domain/analysisErrorCancellation.js"; import { ConfigurationError } from "../domain/configurationErrors.js"; +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { absoluteCallerPath } from "../filesystem/callerPath.js"; import { ExecFileOutputCancelledError, @@ -65,7 +66,7 @@ const gitOutput = async ( throw gitFailure(cause); } try { - return new TextDecoder("utf-8", { fatal: true }).decode(observation.stdout); + return decodeStrictUtf8(observation.stdout, "strip-signature"); } catch { // The failure keeps the exact observation it could not decode. throw gitFailure( diff --git a/src/application/ReferenceSourceImportEntries.ts b/src/application/ReferenceSourceImportEntries.ts index 2862a660f3..991c4aaa62 100644 --- a/src/application/ReferenceSourceImportEntries.ts +++ b/src/application/ReferenceSourceImportEntries.ts @@ -26,6 +26,7 @@ import type { } from "../reference/ReferenceSourceReaderTypes.js"; import { typescriptEmittedSourceCandidates } from "./TypeScriptEmittedSourceCandidates.js"; import { appendAll } from "../domain/appendAll.js"; +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; export interface ParsedReferenceSourceEntries { readonly entries: HistoricalSourceGraphInput["entries"]; @@ -152,7 +153,8 @@ const readPackageMetadata = ( } try { const manifest: unknown = JSON.parse( - new TextDecoder("utf-8", { fatal: true }).decode(entry.bytes), + // npm and Node both accept a BOM-signed package.json. + decodeStrictUtf8(entry.bytes, "strip-signature"), ); if ( typeof manifest !== "object" || diff --git a/src/application/RegularFileRead.ts b/src/application/RegularFileRead.ts index a6f0339e98..3449b587ce 100644 --- a/src/application/RegularFileRead.ts +++ b/src/application/RegularFileRead.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import type { Stats } from "node:fs"; import type { FileHandle } from "node:fs/promises"; @@ -343,9 +344,7 @@ export const readRegularFileTextSnapshot = ( const decodeFileText = (path: string, bytes: Buffer): string => { try { - return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - bytes, - ); + return decodeStrictUtf8(bytes, "preserve"); } catch (cause: unknown) { if (!(cause instanceof TypeError)) throw cause; throw new InvalidUtf8FileError(path, cause); diff --git a/src/application/Utf8JsonInput.ts b/src/application/Utf8JsonInput.ts index 6981a3146e..cb54859ceb 100644 --- a/src/application/Utf8JsonInput.ts +++ b/src/application/Utf8JsonInput.ts @@ -1,3 +1,4 @@ +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; import { constants } from "node:buffer"; import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; @@ -45,7 +46,7 @@ export const parseUtf8Json = ( ); let text: string; try { - const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + const decoder = strictUtf8Decoder("preserve"); if (bytes.byteLength <= constants.MAX_STRING_LENGTH) { text = decoder.decode(bytes); } else { diff --git a/src/application/WebScriptExportService.ts b/src/application/WebScriptExportService.ts index 8300c12f1d..d40458d264 100644 --- a/src/application/WebScriptExportService.ts +++ b/src/application/WebScriptExportService.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { createHash } from "node:crypto"; import { z } from "zod"; @@ -396,10 +397,7 @@ const parseCapture = ( closeFailure?: { readonly cause: unknown }, ): Result, AnalysisError> => { try { - const text = new TextDecoder("utf-8", { - fatal: true, - ignoreBOM: true, - }).decode(bytes); + const text = decodeStrictUtf8(bytes, "preserve"); const json = safeParseJson(text); if (!json.ok) throw new TypeError(json.error); return ok(selectScriptCapture(json.value)); diff --git a/src/application/apiContract/GraphqlContractImport.ts b/src/application/apiContract/GraphqlContractImport.ts index 245b210f2a..fa823b9f19 100644 --- a/src/application/apiContract/GraphqlContractImport.ts +++ b/src/application/apiContract/GraphqlContractImport.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../../domain/strictUtf8.js"; import { getHeapStatistics } from "node:v8"; import { version } from "graphql"; import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; @@ -86,9 +87,7 @@ const decode = ( ), ); try { - return ok( - new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes), - ); + return ok(decodeStrictUtf8(bytes, "preserve")); } catch (cause) { if (!(cause instanceof TypeError)) throw cause; return err( diff --git a/src/application/decompileDirectory/exportTree.ts b/src/application/decompileDirectory/exportTree.ts index 93e6a4548c..7a8eef0279 100644 --- a/src/application/decompileDirectory/exportTree.ts +++ b/src/application/decompileDirectory/exportTree.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../../domain/strictUtf8.js"; import { createHash, randomUUID } from "node:crypto"; import { constants } from "node:fs"; import { @@ -737,10 +738,7 @@ const parseJournal = ( const committedBytes = bytes.lastIndexOf(0x0a) + 1; let committed: string; try { - committed = new TextDecoder("utf-8", { - fatal: true, - ignoreBOM: true, - }).decode(bytes.subarray(0, committedBytes)); + committed = decodeStrictUtf8(bytes.subarray(0, committedBytes), "preserve"); } catch { return err(incompatibleJournal()); } diff --git a/src/artifacts/AsarHeader.ts b/src/artifacts/AsarHeader.ts index d52cf64abc..421e3ac7ef 100644 --- a/src/artifacts/AsarHeader.ts +++ b/src/artifacts/AsarHeader.ts @@ -17,6 +17,7 @@ import { MAXIMUM_ASAR_MAX_HEADER_BYTES, } from "../domain/analysisResourceLimits.js"; import { parseAsarListingAdmission } from "../config/asarListingAdmission.js"; +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; /** * Resolve the admission from a composition root's selected environment; an @@ -435,9 +436,7 @@ export const readAsarHeader = async ( throw new ArtifactReaderFailure("format", "Truncated ASAR JSON header"); let json: string; try { - json = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - bytes, - ); + json = strictUtf8Decoder("preserve").decode(bytes); } catch (cause: unknown) { throw new ArtifactReaderFailure( "format", diff --git a/src/artifacts/GodotPckArtifactReader.ts b/src/artifacts/GodotPckArtifactReader.ts index e0519749c3..abef031c0a 100644 --- a/src/artifacts/GodotPckArtifactReader.ts +++ b/src/artifacts/GodotPckArtifactReader.ts @@ -23,13 +23,14 @@ import { type PositionalArtifactEntries, type PositionalArtifactEntry, } from "./PositionalArtifactReader.js"; +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; const PACK_DIR_ENCRYPTED = 1; const PACK_REL_FILEBASE = 2; const PACK_SPARSE_BUNDLE = 4; const PACK_FILE_ENCRYPTED = 1; const DIRECTORY_CHUNK_BYTES = 64 * 1024; -const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const UTF8 = strictUtf8Decoder("preserve"); /** One descriptor, one validated directory, and bounded positional member reads. */ export class GodotPckArtifactReader extends PositionalArtifactReader { diff --git a/src/artifacts/UnityFsStructures.ts b/src/artifacts/UnityFsStructures.ts index ed5abf8e1c..6a8591ba69 100644 --- a/src/artifacts/UnityFsStructures.ts +++ b/src/artifacts/UnityFsStructures.ts @@ -21,6 +21,7 @@ import { UNITYFS_SIGNATURE, unityFsLayoutFlags, } from "../domain/unityfsFormat.js"; +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; /** Per-decode resource limits, not format limits. Plain blocks are streamed. */ const MAX_BLOCK_UNCOMPRESSED_BYTES = 64 * 1024 * 1024; @@ -49,7 +50,7 @@ class UnityFsDirectoryBudget { } /** Signature, version, two bounded strings and the fixed tail; reads are bounded. */ const HEADER_PREFIX_MAX = 8 + 4 + UNITYFS_HEADER_STRING_MAX * 2 + 20; -const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const UTF8 = strictUtf8Decoder("preserve"); export interface UnityFsHeader { readonly formatVersion: number; diff --git a/src/artifacts/WxapkgStructures.ts b/src/artifacts/WxapkgStructures.ts index 3437ead7a7..17787cdbb0 100644 --- a/src/artifacts/WxapkgStructures.ts +++ b/src/artifacts/WxapkgStructures.ts @@ -15,6 +15,7 @@ import { type WxapkgIndexLayout, type WxapkgIndexLayoutFailure, } from "../domain/wxapkgFormat.js"; +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; /** The member table is a flat name/offset/size row list. */ const MAX_TABLE_BYTES = 16 * 1024 * 1024; @@ -25,7 +26,7 @@ const MAX_MEMBER_NAME_BYTES = 64 * 1024; /** Detecting prefix covers the fixed header, table head and encrypted magic. */ const HEADER_PREFIX_BYTES = 64; -const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const UTF8 = strictUtf8Decoder("preserve"); export const wxapkgFormatFailure = (detail: string): ArtifactReaderFailure => new ArtifactReaderFailure("format", `wxapkg is malformed: ${detail}`); diff --git a/src/artifacts/ZipArtifactReader.ts b/src/artifacts/ZipArtifactReader.ts index 4efd6a7e28..21d53890ed 100644 --- a/src/artifacts/ZipArtifactReader.ts +++ b/src/artifacts/ZipArtifactReader.ts @@ -52,12 +52,10 @@ import { artifactReaderLimitFailure } from "./ArtifactReaderLimits.js"; import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js"; import { ArtifactFileSource } from "./ArtifactFileSource.js"; import { readFileHandleRange } from "../filesystem/readFileHandleRange.js"; +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; import { invariantViolation } from "../domain/internalInvariant.js"; -const ZIP_UTF8_DECODER = new TextDecoder("utf-8", { - fatal: true, - ignoreBOM: true, -}); +const ZIP_UTF8_DECODER = strictUtf8Decoder("preserve"); /** Fixed method-93 decoder window: 2^27 bytes, matching the ELF/GPU decoders. */ const ZIP_ZSTANDARD_WINDOW_LOG_MAX = 27; diff --git a/src/artifacts/apple/InterfaceBuilderAnalysis.ts b/src/artifacts/apple/InterfaceBuilderAnalysis.ts index c50a229eaa..369fa1c67b 100644 --- a/src/artifacts/apple/InterfaceBuilderAnalysis.ts +++ b/src/artifacts/apple/InterfaceBuilderAnalysis.ts @@ -3,7 +3,6 @@ import { isHostLimitFailure, ParseFailureError, } from "../../domain/parseFailure.js"; -import { TextDecoder } from "node:util"; import { InternalInvariantError } from "../../domain/internalInvariant.js"; import { decodeBinaryPropertyList } from "./BinaryPlistMaterializer.js"; @@ -72,6 +71,7 @@ import { hasBinaryPlistSignature, hasNibArchiveSignature, } from "../../domain/apple/appleArchiveFormat.js"; +import { strictUtf8Decoder } from "../../domain/strictUtf8.js"; const MAX_AGGREGATE_INPUT_BYTES = 32 * 1024 * 1024; const MAX_AGGREGATE_DECODE_BYTES = 256 * 1024 * 1024; @@ -730,7 +730,7 @@ const projectNibArchive = ( try { // NSString's NS.bytes representation is UTF-8. A failed conversion // does not establish a replacement-character class, title or selector. - value = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( + value = strictUtf8Decoder("preserve").decode( Buffer.from(encoded, "base64"), ); } catch { diff --git a/src/artifacts/apple/MachoLoadCommandReader.ts b/src/artifacts/apple/MachoLoadCommandReader.ts index 1566b8b58a..79ffa1929e 100644 --- a/src/artifacts/apple/MachoLoadCommandReader.ts +++ b/src/artifacts/apple/MachoLoadCommandReader.ts @@ -45,6 +45,7 @@ import { type FatSliceDeclaration, type FatSliceRangeIssue, } from "../../domain/apple/machoContainer.js"; +import { strictUtf8Decoder } from "../../domain/strictUtf8.js"; /** Read `length` bytes at `offset`; shorter results mean end of file. */ export type ReadAt = (offset: number, length: number) => Promise; @@ -1043,9 +1044,7 @@ const payloadString = ( chargeLazyString(work, end - offset + 1); let text: string; try { - text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - payload.subarray(offset, end), - ); + text = strictUtf8Decoder("preserve").decode(payload.subarray(offset, end)); } catch { throw machoIssue( "malformed", @@ -1116,9 +1115,7 @@ const throwCommandStringIssue = ( const decodeCommandString = (bytes: Uint8Array, index: number): string => { try { // lc_str contains name/path data, so a leading U+FEFF is part of its identity. - return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - bytes, - ); + return strictUtf8Decoder("preserve").decode(bytes); } catch { throw machoIssue( "malformed", diff --git a/src/artifacts/apple/NibArchive.ts b/src/artifacts/apple/NibArchive.ts index f7a266b66e..90fbe476a7 100644 --- a/src/artifacts/apple/NibArchive.ts +++ b/src/artifacts/apple/NibArchive.ts @@ -1,7 +1,6 @@ import type { JsonValue } from "../../domain/jsonValue.js"; import { hasNibArchiveSignature } from "../../domain/apple/appleArchiveFormat.js"; import { isInterfaceBuilderConnectionClass } from "../../domain/apple/interfaceBuilderKeyedArchive.js"; -import { TextDecoder } from "node:util"; import { malformedInput, truncatedInput, @@ -11,6 +10,7 @@ import { InterfaceBuilderDecodeBudget, InterfaceBuilderDecodeBudgetExceeded, } from "./InterfaceBuilderDecodeBudget.js"; +import { strictUtf8Decoder } from "../../domain/strictUtf8.js"; const DEFAULT_DECODE_BUDGET_BYTES = 128 * 1024 * 1024; @@ -629,9 +629,7 @@ const decodeUtf8 = ( ): string => { try { // These are length-prefixed names, not standalone text documents. - return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - bytes.subarray(start, end), - ); + return strictUtf8Decoder("preserve").decode(bytes.subarray(start, end)); } catch (cause: unknown) { throw malformedInput( "nib-archive", diff --git a/src/artifacts/deno/DenoSectionLayout.ts b/src/artifacts/deno/DenoSectionLayout.ts index 6bd659091c..3a2926761c 100644 --- a/src/artifacts/deno/DenoSectionLayout.ts +++ b/src/artifacts/deno/DenoSectionLayout.ts @@ -1,10 +1,9 @@ -import { TextDecoder } from "node:util"; - import { fixedAllowanceRemediation } from "../../domain/fixedAllowanceRemediation.js"; import { jsonValueSchema, type JsonValue } from "../../domain/jsonValue.js"; import { abortIfNeeded } from "../ArtifactHash.js"; import { ArtifactReaderFailure } from "../ArtifactReader.js"; import { walkDenoVirtualFs, type DenoVfsWalk } from "./DenoVirtualFs.js"; +import { strictUtf8Decoder } from "../../domain/strictUtf8.js"; /** Opening and closing marker of a Deno compile data section. */ export const DENO_SECTION_MAGIC = Buffer.from("d3n0l4nd", "latin1"); @@ -20,11 +19,8 @@ const FLAG_TRANSPILED = 1 << 0; const FLAG_SOURCE_MAP = 1 << 1; const FLAG_CJS = 1 << 2; const FLAG_UTF8 = 1 << 3; -const specifierDecoder = new TextDecoder("utf-8", { - fatal: true, - ignoreBOM: true, -}); -const jsonDecoder = new TextDecoder("utf-8", { fatal: true }); +const specifierDecoder = strictUtf8Decoder("preserve"); +const jsonDecoder = strictUtf8Decoder("strip-signature"); const MEDIA_TYPE_NAMES = [ "JavaScript", diff --git a/src/artifacts/dotnet/DotnetBundle.ts b/src/artifacts/dotnet/DotnetBundle.ts index 775116dfd4..0d92096bb4 100644 --- a/src/artifacts/dotnet/DotnetBundle.ts +++ b/src/artifacts/dotnet/DotnetBundle.ts @@ -15,6 +15,7 @@ import { ArtifactReaderFailure, artifactFixedLimitFailure, } from "../ArtifactReader.js"; +import { strictUtf8Decoder } from "../../domain/strictUtf8.js"; /** SHA-256 marker embedded by Microsoft's apphost; the preceding UInt64 is the header offset. */ export const DOTNET_BUNDLE_SIGNATURE = Buffer.from( @@ -285,10 +286,7 @@ class BundleCursor { // decoder boundary so I/O and cancellation retain their original cause. const bytes = await this.take(length); try { - return new TextDecoder("utf-8", { - fatal: true, - ignoreBOM: true, - }).decode(bytes); + return strictUtf8Decoder("preserve").decode(bytes); } catch (cause) { throw new ArtifactReaderFailure( "format", diff --git a/src/artifacts/fit/FitStructure.ts b/src/artifacts/fit/FitStructure.ts index 64f77cdba6..355436cb44 100644 --- a/src/artifacts/fit/FitStructure.ts +++ b/src/artifacts/fit/FitStructure.ts @@ -21,6 +21,7 @@ import { } from "../../firmware/devicetree/DevicetreeParser.js"; import { BinaryFormatFailure } from "../../native/metadata/BinaryContainer.js"; import { ArtifactReaderFailure } from "../ArtifactReader.js"; +import { strictUtf8Decoder } from "../../domain/strictUtf8.js"; // Fixed resource bounds; configurable ceilings are tracked in #6176. const FIT_MAX_IMAGES = 1024; @@ -115,7 +116,7 @@ const valueOf = (fdt: Buffer, property: FitProperty): Buffer => fdt.subarray(property.offset, property.offset + property.bytes); // A property value is data: a leading U+FEFF is not a file encoding signature. -const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const utf8 = strictUtf8Decoder("preserve"); const nulCount = (raw: Buffer): number => { let count = 0; for (const byte of raw) if (byte === 0) count += 1; diff --git a/src/artifacts/javascript/JavaScriptArtifactFiles.ts b/src/artifacts/javascript/JavaScriptArtifactFiles.ts index 5f901250eb..1b6bd8dc4e 100644 --- a/src/artifacts/javascript/JavaScriptArtifactFiles.ts +++ b/src/artifacts/javascript/JavaScriptArtifactFiles.ts @@ -30,6 +30,7 @@ import type { JavaScriptUnanalyzedArtifactLookup, JavaScriptArtifactFileSet, } from "../../domain/javascript/javascriptArtifactFiles.js"; +import { strictUtf8Decoder } from "../../domain/strictUtf8.js"; interface ExpectedFile { readonly path: string; @@ -388,12 +389,11 @@ const readText = async ( try { return { included: true, - value: new TextDecoder("utf-8", { - fatal: true, - ignoreBOM: - input.expected.kind === "javascript" || - input.expected.kind === "html", - }).decode(bytes), + value: strictUtf8Decoder( + input.expected.kind === "javascript" || input.expected.kind === "html" + ? "preserve" + : "strip-signature", + ).decode(bytes), }; } catch (cause: unknown) { // Non-UTF8 bytes are counted; the fixed reason preserves the schema. diff --git a/src/artifacts/nodeSea/NodeSeaBlob.ts b/src/artifacts/nodeSea/NodeSeaBlob.ts index 27127fe105..2513564797 100644 --- a/src/artifacts/nodeSea/NodeSeaBlob.ts +++ b/src/artifacts/nodeSea/NodeSeaBlob.ts @@ -7,6 +7,7 @@ import { type NodeSeaReaderLimits, } from "../../domain/artifactReaderLimits.js"; import { artifactReaderLimitFailure } from "../ArtifactReaderLimits.js"; +import { strictUtf8Decoder } from "../../domain/strictUtf8.js"; /** Node `SeaBlob::kMagic` from `src/node_sea.h` (v24). */ const NODE_SEA_MAGIC = 0x143da20; @@ -231,9 +232,7 @@ class BlobCursor { const raw = this.#demand(length); let text: string; try { - text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - raw, - ); + text = strictUtf8Decoder("preserve").decode(raw); } catch (cause: unknown) { if (cause instanceof RangeError) throw new ArtifactReaderFailure( diff --git a/src/artifacts/pyinstaller/CArchiveLayout.ts b/src/artifacts/pyinstaller/CArchiveLayout.ts index 0ae82cbc22..ffb892b6ed 100644 --- a/src/artifacts/pyinstaller/CArchiveLayout.ts +++ b/src/artifacts/pyinstaller/CArchiveLayout.ts @@ -6,6 +6,7 @@ import { pyInstallerFailure, type PyInstallerFailure, } from "./PyInstallerFailure.js"; +import { strictUtf8Decoder } from "../../domain/strictUtf8.js"; /** Magic at the start of a CArchive cookie. */ const PYINSTALLER_COOKIE_MAGIC = Buffer.from("MEI\x0c\x0b\x0a\x0b\x0e"); @@ -299,9 +300,7 @@ const libraryName = ( ); if (text.length === 0) return ok(null); try { - return ok( - new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(text), - ); + return ok(strictUtf8Decoder("preserve").decode(text)); } catch { return err( pyInstallerFailure( @@ -318,9 +317,7 @@ const tocName = (bytes: Buffer): Result => { if (text.length === 0) return err(pyInstallerFailure("format", "CArchive entry name is empty")); try { - return ok( - new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(text), - ); + return ok(strictUtf8Decoder("preserve").decode(text)); } catch { return err( pyInstallerFailure("format", "CArchive entry name is not valid UTF-8"), diff --git a/src/browser/BrowserScenarioResponseValues.ts b/src/browser/BrowserScenarioResponseValues.ts index 7ad607e838..60b92e308a 100644 --- a/src/browser/BrowserScenarioResponseValues.ts +++ b/src/browser/BrowserScenarioResponseValues.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { jsonPointerTokens } from "../domain/jsonPointer.js"; import { visit, type JSONPath } from "jsonc-parser"; import { isExactJsonNumber } from "../domain/exactJsonNumber.js"; @@ -68,12 +69,7 @@ export const readResponseValues = ( return err(issue("the response is too large to save values from.")); let text: string; try { - text = - typeof body === "string" - ? body - : new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - body, - ); + text = typeof body === "string" ? body : decodeStrictUtf8(body, "preserve"); } catch { return err( issue( diff --git a/src/browser/BrowserScenarioSecrets.ts b/src/browser/BrowserScenarioSecrets.ts index 33a53eebb4..320ef7a497 100644 --- a/src/browser/BrowserScenarioSecrets.ts +++ b/src/browser/BrowserScenarioSecrets.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { scenarioSerializationAllowance, scenarioQueryLength, @@ -590,10 +591,7 @@ export class BrowserScenarioSecrets { ).length === 0 ) return body; - const text = new TextDecoder("utf-8", { - fatal: true, - ignoreBOM: true, - }).decode(body); + const text = decodeStrictUtf8(body, "preserve"); const redacted = redactResponseSource( text, [], diff --git a/src/browser/ElectronFileScope.ts b/src/browser/ElectronFileScope.ts index d2463e4238..0dad0b2ae8 100644 --- a/src/browser/ElectronFileScope.ts +++ b/src/browser/ElectronFileScope.ts @@ -31,6 +31,7 @@ import { DEFAULT_ASAR_HEADER_ADMISSION, type AsarHeaderAdmission, } from "../config/asarHeaderAdmission.js"; +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { isRecord } from "../domain/isRecord.js"; type ElectronFileOperation = @@ -318,7 +319,7 @@ const asarFiles = async ( if (!sameRegularFileState(identity, afterRead)) throw new Error("ASAR identity changed during header read"); const parsed: unknown = JSON.parse( - new TextDecoder("utf-8", { fatal: true }).decode(json), + decodeStrictUtf8(json, "strip-signature"), ); if (!isRecord(parsed) || !isRecord(parsed.files)) throw new Error("ASAR header has no file tree"); diff --git a/src/browser/WebBundleAnalysisWorkerServer.ts b/src/browser/WebBundleAnalysisWorkerServer.ts index 130817fdcd..cc6ce5c895 100644 --- a/src/browser/WebBundleAnalysisWorkerServer.ts +++ b/src/browser/WebBundleAnalysisWorkerServer.ts @@ -25,6 +25,7 @@ import { webBundleWorkerMessageSchema, type WebBundleWorkerMessage, } from "./WebBundleAnalysisWorkerProtocol.js"; +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; export interface WebBundleScriptAnalyzer { ( @@ -120,7 +121,7 @@ const analyzeJob = async ( let script: IncludedScript; try { const decoded: unknown = JSON.parse( - new TextDecoder("utf-8", { fatal: true }).decode(source.bytes), + decodeStrictUtf8(source.bytes, "strip-signature"), ); script = includedWebScriptSchema.parse(decoded); } catch (cause) { diff --git a/src/browser/WebSourceMapFetcher.ts b/src/browser/WebSourceMapFetcher.ts index a88a58fabf..eef18c5a6d 100644 --- a/src/browser/WebSourceMapFetcher.ts +++ b/src/browser/WebSourceMapFetcher.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import * as t from "@babel/types"; import { createHash } from "node:crypto"; import { decodeValidatedSourceMapLeaves } from "../javascript/sourceMaps/DecodedSourceMap.js"; @@ -910,11 +911,7 @@ const readBoundedText = async ( }; try { return { - text: ok( - new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - joined, - ), - ), + text: ok(decodeStrictUtf8(joined, "preserve")), responseIdentity, }; } catch (cause: unknown) { diff --git a/src/browser/assets/WebArtifactReads.ts b/src/browser/assets/WebArtifactReads.ts index 496009a40b..ddb3f6d76e 100644 --- a/src/browser/assets/WebArtifactReads.ts +++ b/src/browser/assets/WebArtifactReads.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../../domain/strictUtf8.js"; import type { WebArtifactAcquisition as Acquisition } from "../../domain/webArtifactAcquisition.js"; import { z } from "zod"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; @@ -50,9 +51,7 @@ const WEB_ARTIFACT_FORMAT = "captured-web-artifact"; /** Decode exact UTF-8 content, preserving a BOM for digest verification. */ export const decodeWebArtifact = (bytes: Buffer): string => { try { - return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - bytes, - ); + return decodeStrictUtf8(bytes, "preserve"); } catch (cause: unknown) { throw malformedInput( WEB_ARTIFACT_FORMAT, diff --git a/src/browser/history/CaptureJson.ts b/src/browser/history/CaptureJson.ts index 22c0ad46c2..df58469be5 100644 --- a/src/browser/history/CaptureJson.ts +++ b/src/browser/history/CaptureJson.ts @@ -3,7 +3,7 @@ import { captureBudgets } from "./CaptureBudgets.js"; import { visit } from "jsonc-parser"; import { LosslessNumber } from "lossless-json"; import { CaptureFormatError } from "./CaptureFormatError.js"; -import { TextDecoder } from "node:util"; +import { decodeStrictUtf8 } from "../../domain/strictUtf8.js"; type Frame = | { @@ -150,7 +150,8 @@ export const decodeCaptureText = ( format: "HAR" | "qlog", ): string => { try { - return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + // Exported HAR and qlog files may carry a BOM signature (RFC 8259 8.1). + return decodeStrictUtf8(bytes, "strip-signature"); } catch { throw new CaptureFormatError( "malformed", diff --git a/src/browser/history/playwrightTrace/TraceStream.ts b/src/browser/history/playwrightTrace/TraceStream.ts index b90cf57c21..329fa9d286 100644 --- a/src/browser/history/playwrightTrace/TraceStream.ts +++ b/src/browser/history/playwrightTrace/TraceStream.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../../../domain/strictUtf8.js"; import { fixedAllowanceOptions } from "../../../domain/fixedAllowanceRemediation.js"; import { getHeapStatistics } from "node:v8"; import { LosslessNumber, isSafeNumber } from "lossless-json"; @@ -41,9 +42,7 @@ const readRecord = (bytes: Buffer): unknown => { ); let value: string; try { - value = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - bytes, - ); + value = decodeStrictUtf8(bytes, "preserve"); } catch (cause) { throw new CaptureFormatError( "malformed", diff --git a/src/browser/history/protobuf/ProtobufTrailers.ts b/src/browser/history/protobuf/ProtobufTrailers.ts index e0f81ea98f..fcf767b294 100644 --- a/src/browser/history/protobuf/ProtobufTrailers.ts +++ b/src/browser/history/protobuf/ProtobufTrailers.ts @@ -1,3 +1,4 @@ +import { strictUtf8Decoder } from "../../../domain/strictUtf8.js"; import { err, ok, type Result } from "../../../domain/result.js"; import type { ProtobufBudget } from "./ProtobufWire.js"; @@ -20,7 +21,7 @@ export const parseGrpcWebTrailers = ( bytes: Uint8Array, budget: ProtobufBudget, ): Result => { - let decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + let decoder = strictUtf8Decoder("preserve"); const trailers: TrailerFacts["trailers"] = []; let examinedBytes = 0; const facts = (): TrailerFacts => ({ @@ -103,7 +104,7 @@ export const parseGrpcWebTrailers = ( value = decoder.decode(bytes.subarray(valueStart, valueEnd)); } catch (cause) { if (!(cause instanceof TypeError)) throw cause; - decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + decoder = strictUtf8Decoder("preserve"); opaqueValueFailure ??= { ...facts(), kind: "invalid-utf8-value", diff --git a/src/browser/history/protobuf/ProtobufWire.ts b/src/browser/history/protobuf/ProtobufWire.ts index 0314cce729..142f4ed3a7 100644 --- a/src/browser/history/protobuf/ProtobufWire.ts +++ b/src/browser/history/protobuf/ProtobufWire.ts @@ -1,3 +1,4 @@ +import { strictUtf8Decoder } from "../../../domain/strictUtf8.js"; import { captureBudgets } from "../CaptureBudgets.js"; import { readLeb128, ULEB64 } from "../../../domain/leb128.js"; import { err, ok, type Result } from "../../../domain/result.js"; @@ -40,7 +41,7 @@ export const readProtobufVarint = ( return "failure" in read ? undefined : { value: read.value, end: read.next }; }; -const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const utf8 = strictUtf8Decoder("preserve"); const textOf = (bytes: Uint8Array): string | undefined => { try { return utf8.decode(bytes); diff --git a/src/browser/network/PlaywrightNetworkContent.ts b/src/browser/network/PlaywrightNetworkContent.ts index 216c251fd3..0127bad51e 100644 --- a/src/browser/network/PlaywrightNetworkContent.ts +++ b/src/browser/network/PlaywrightNetworkContent.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../../domain/strictUtf8.js"; import { isJson, mediaTypeEssence, @@ -235,11 +236,7 @@ export const captureNetworkBody = async ( if ( jsonBody && !retained.equals(sourceBody) && - !safeParseJson( - new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - retained, - ), - ).ok + !safeParseJson(decodeStrictUtf8(retained, "preserve")).ok ) return { state: "unavailable", diff --git a/src/desktop/BuildSecretFiles.test.ts b/src/desktop/BuildSecretFiles.test.ts index c9b20e5dee..5e1925cdc1 100644 --- a/src/desktop/BuildSecretFiles.test.ts +++ b/src/desktop/BuildSecretFiles.test.ts @@ -54,6 +54,12 @@ describe("build secret files", () => { const directory = await folder({ master: { text: `${VALUE}\n`, mode: 0o600 }, crlf: { text: `${VALUE}\r\n`, mode: 0o400 }, + // An editor's UTF-8 signature is not part of the secret it saved. + bom: { text: `${VALUE}\n`, mode: 0o600 }, + }); + expect(await readBuildSecretFile(directory, "bom")).toEqual({ + ok: true, + value: VALUE, }); expect(await readBuildSecretFile(directory, "master")).toEqual({ ok: true, diff --git a/src/desktop/BuildSecretFiles.ts b/src/desktop/BuildSecretFiles.ts index 7f4762d937..8beca6f60e 100644 --- a/src/desktop/BuildSecretFiles.ts +++ b/src/desktop/BuildSecretFiles.ts @@ -36,6 +36,7 @@ import { } from "../filesystem/readStableFileBytes.js"; import { RegularFileChangedError } from "../filesystem/RegularFile.js"; import { xdgBase } from "./XdgBase.js"; +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; /** The most bytes a secret file of at most `characters` may hold: those in UTF-8 and a line break. */ const maxFileBytes = (characters: number) => characters * 4 + 2; @@ -179,7 +180,7 @@ export const readBuildSecretFile = async ( ); let text: string; try { - text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + text = decodeStrictUtf8(bytes, "strip-signature"); } catch { result = err(`${path} is not UTF-8 text.`); return; diff --git a/src/domain/analysisView/recordedResponseBody.ts b/src/domain/analysisView/recordedResponseBody.ts index 981eb703e9..9d811d177c 100644 --- a/src/domain/analysisView/recordedResponseBody.ts +++ b/src/domain/analysisView/recordedResponseBody.ts @@ -1,5 +1,6 @@ import { z } from "zod"; import type { WebNetworkCaptureRecord } from "../webNetworkCapture.js"; +import { strictUtf8Decoder } from "../strictUtf8.js"; /** One record's retained response text, as replay and state extraction both read it. */ export interface RecordedResponseBody { @@ -53,8 +54,7 @@ export const recordedResponseStatus = ( }; /** Strict, and keeping a leading byte-order mark, so callers see the recorded text unchanged. */ -const strictUtf8 = (): TextDecoder => - new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const strictUtf8 = (): TextDecoder => strictUtf8Decoder("preserve"); /** Base64 characters that decode to just over `limitBytes`, so a cut prefix still fills it. */ export const base64PrefixChars = (limitBytes: number): number => diff --git a/src/domain/analysisView/scenarioCaptureRecords.ts b/src/domain/analysisView/scenarioCaptureRecords.ts index 243231b708..8cf1388a04 100644 --- a/src/domain/analysisView/scenarioCaptureRecords.ts +++ b/src/domain/analysisView/scenarioCaptureRecords.ts @@ -20,6 +20,7 @@ import { analysisViewResultInputError } from "./analysisViewParent.js"; import type { AnalysisViewParent } from "./analysisViewParent.js"; import type { WebNetworkProjectionAccumulator } from "./WebNetworkProjectionAccumulator.js"; import { ENTRY_BYTES, textBytes } from "./webNetworkRouteTemplates.js"; +import { strictUtf8Decoder } from "../strictUtf8.js"; type RequestEvent = Extract; type ContentEvent = Extract; @@ -79,7 +80,7 @@ const retainedBody = ( /** Strictly decoded text, keeping a byte-order mark; undefined for bytes that are not UTF-8. */ const utf8Text = (body: CapturedBody): string | undefined => { try { - return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( + return strictUtf8Decoder("preserve").decode( Buffer.from(body.content, "base64"), ); } catch { diff --git a/src/domain/artifactPathSyntax.ts b/src/domain/artifactPathSyntax.ts index 4884bd646c..7db6b4161d 100644 --- a/src/domain/artifactPathSyntax.ts +++ b/src/domain/artifactPathSyntax.ts @@ -1,3 +1,4 @@ +import { strictUtf8Decoder } from "./strictUtf8.js"; /** * Pure string predicates for artifact path syntax. The local artifact * resolver, the Electron native specifier check, and (by reference) the @@ -49,7 +50,7 @@ export const htmlUrlText = (value: string): string => { return value.slice(start, end).replace(/[\t\n\r]/gu, ""); }; -const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const UTF8 = strictUtf8Decoder("preserve"); const UTF8_ENCODER = new TextEncoder(); /** diff --git a/src/domain/desktop/buildSecrets.test.ts b/src/domain/desktop/buildSecrets.test.ts index aa3aca73ad..3e21fe1195 100644 --- a/src/domain/desktop/buildSecrets.test.ts +++ b/src/domain/desktop/buildSecrets.test.ts @@ -93,6 +93,10 @@ describe("build secrets", () => { expect(found(text)).toContain("licence"); expect(found(gzipSync(Buffer.from(text)))).toContain("licence"); } + // A UTF-8 signature must not hide the JSON strings from the scan. + const signed = Buffer.from(`{"password":"${encoded}"}`); + expect(found(signed)).toContain("licence"); + expect(found(gzipSync(signed))).toContain("licence"); const utf16 = Buffer.from(`{"password":"${encoded}"}`, "utf16le"); for (const bytes of [ utf16, diff --git a/src/domain/desktop/seedRecoveryList.test.ts b/src/domain/desktop/seedRecoveryList.test.ts index bfeb7b9991..1053a8e61b 100644 --- a/src/domain/desktop/seedRecoveryList.test.ts +++ b/src/domain/desktop/seedRecoveryList.test.ts @@ -50,3 +50,18 @@ describe("withoutRecoveryList", () => { ).toBeUndefined(); }); }); + +it("keeps a leading byte order mark among the bytes it ships", () => { + const bom = Buffer.from([0xef, 0xbb, 0xbf]); + const source = Buffer.concat([ + bom, + Buffer.from( + registry( + `${KEPT}\n${RECOVERY}`, + ), + ), + ]); + const shippedBytes = withoutRecoveryList(REGISTRY, source) ?? source; + expect(shippedBytes.subarray(0, 3).equals(bom)).toBe(true); + expect(shippedBytes.toString("utf8")).not.toContain("recovery_item_1"); +}); diff --git a/src/domain/desktop/seedRecoveryList.ts b/src/domain/desktop/seedRecoveryList.ts index 6b9939e028..65b20c14b0 100644 --- a/src/domain/desktop/seedRecoveryList.ts +++ b/src/domain/desktop/seedRecoveryList.ts @@ -1,4 +1,5 @@ import { xmlElementRanges } from "../xmlDocument.js"; +import { strictUtf8Decoder } from "../strictUtf8.js"; /** LibreOffice's profile registry, where it lists the documents it has open while it runs. */ const REGISTRY = "registrymodifications.xcu"; @@ -20,9 +21,7 @@ export const withoutRecoveryList = ( if (home.split("/").at(-1) !== REGISTRY) return undefined; let text: string; try { - text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - bytes, - ); + text = strictUtf8Decoder("preserve").decode(bytes); } catch { return undefined; } diff --git a/src/domain/desktop/seedScrub.ts b/src/domain/desktop/seedScrub.ts index 246ac3cce1..bcaaaf0c6f 100644 --- a/src/domain/desktop/seedScrub.ts +++ b/src/domain/desktop/seedScrub.ts @@ -16,6 +16,7 @@ import { isTextualFormat, textSyntaxFormat, } from "./stateFormats.js"; +import { strictUtf8Decoder } from "../strictUtf8.js"; import { isRecord } from "../isRecord.js"; /** What the export ships for one seed. */ @@ -376,10 +377,7 @@ const seedText = (bytes: Buffer, { home, format }: Seed) => { : { ...utf16, format: textSyntaxFormat(home, utf16.text) }; } try { - const text = new TextDecoder("utf-8", { - fatal: true, - ignoreBOM: true, - }).decode(bytes); + const text = strictUtf8Decoder("preserve").decode(bytes); return { // The byte-order mark is kept, as Python's utf-8 codec keeps it. text, diff --git a/src/domain/desktop/stateFormats.ts b/src/domain/desktop/stateFormats.ts index 19ff4af6da..c36156f667 100644 --- a/src/domain/desktop/stateFormats.ts +++ b/src/domain/desktop/stateFormats.ts @@ -16,6 +16,7 @@ import { parseXmlDocument, } from "../xmlDocument.js"; import { expatXmlName } from "./expatXmlNames.js"; +import { decodeStrictUtf8 } from "../strictUtf8.js"; /** Formats the state view reads into JSON. */ export type DesktopStateFormat = @@ -92,7 +93,7 @@ const isKeyfile = (text: string): boolean => { export const decodeStateText = (bytes: Uint8Array): string | undefined => { if (bytes.subarray(0, 8192).includes(0)) return undefined; try { - return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + return decodeStrictUtf8(bytes, "strip-signature"); } catch { return undefined; } diff --git a/src/domain/desktop/zipDirectory.ts b/src/domain/desktop/zipDirectory.ts index 05e1474ae7..fba7a595de 100644 --- a/src/domain/desktop/zipDirectory.ts +++ b/src/domain/desktop/zipDirectory.ts @@ -5,6 +5,7 @@ import { ZIP_MAX_COMMENT_BYTES, zipEndOfCentralDirectoryCandidates, } from "../zipEndOfCentralDirectory.js"; +import { strictUtf8Decoder } from "../strictUtf8.js"; const CENTRAL_HEADER = 0x02014b50; const CENTRAL_HEADER_BYTES = 46; @@ -17,7 +18,7 @@ const UNICODE_PATH = 0x7075; const UNIX_TYPE = 0o170000; const UNIX_DIRECTORY = 0o040000; const MSDOS_DIRECTORY = 0x10; -const STRICT_UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const STRICT_UTF8 = strictUtf8Decoder("preserve"); /** An archive's central directory read a record at a time, bounded to the records it keeps. */ export interface ZipDirectory { diff --git a/src/domain/firmware/espImageStructures.ts b/src/domain/firmware/espImageStructures.ts index 7711ec8220..b04d79d7aa 100644 --- a/src/domain/firmware/espImageStructures.ts +++ b/src/domain/firmware/espImageStructures.ts @@ -5,6 +5,7 @@ import type { EspImageMetadata, EspImageSignatureBlockMetadata, } from "../artifactFormatMetadata.js"; +import { strictUtf8Decoder } from "../strictUtf8.js"; /* * Fixed-layout structures inside an ESP image, from ESP-IDF v6.1 @@ -54,7 +55,7 @@ export const firstUnerased = ( const allZero = (bytes: Uint8Array): boolean => bytes.every((byte) => byte === 0); -const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const UTF8 = strictUtf8Decoder("preserve"); /** The bytes of a NUL-terminated C string in a fixed array. */ export const cStringBytes = (bytes: Uint8Array): Uint8Array => { diff --git a/src/domain/jsonShape.ts b/src/domain/jsonShape.ts index 0e0f6c5376..b16af3cd74 100644 --- a/src/domain/jsonShape.ts +++ b/src/domain/jsonShape.ts @@ -5,6 +5,7 @@ import { safeParseJson } from "./safeJson.js"; import { jsonStringByteLength } from "./jsonStringByteLength.js"; import { jsonValueByteLength } from "./jsonValueByteLength.js"; import { err, ok, type Result } from "./result.js"; +import { strictUtf8Decoder } from "./strictUtf8.js"; import { isRecord } from "./isRecord.js"; const jsonValueTypeSchema = z.enum([ @@ -86,9 +87,7 @@ export const inferJsonShape = ( text = typeof source === "string" ? source - : new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - source, - ); + : strictUtf8Decoder("preserve").decode(source); } catch { return err({ kind: "invalid-json" }); } diff --git a/src/domain/propertyListXmlText.ts b/src/domain/propertyListXmlText.ts index a291166295..74c2657b5a 100644 --- a/src/domain/propertyListXmlText.ts +++ b/src/domain/propertyListXmlText.ts @@ -1,5 +1,6 @@ import { ParseFailureError } from "./parseFailure.js"; import { byteOrderMarkEncoding } from "./textByteOrderMark.js"; +import { decodeStrictUtf8 } from "./strictUtf8.js"; const XML_PLIST_FORMAT = "xml-plist"; @@ -85,8 +86,9 @@ const asciiXmlDeclaration = (bytes: Uint8Array): string | undefined => { for (let index = start + 5; index < bytes.length; index += 1) { if (bytes[index]! > 0x7f) return undefined; if (bytes[index] === 0x3f && bytes[index + 1] === 0x3e) - return new TextDecoder("utf-8", { fatal: true }).decode( + return decodeStrictUtf8( bytes.subarray(start, index + 2), + "strip-signature", ); } return undefined; diff --git a/src/domain/python/marshalUtf8.ts b/src/domain/python/marshalUtf8.ts index 2a6e7d3a43..7ed89f7861 100644 --- a/src/domain/python/marshalUtf8.ts +++ b/src/domain/python/marshalUtf8.ts @@ -1,7 +1,8 @@ import { err, ok, type Result } from "../result.js"; +import { strictUtf8Decoder } from "../strictUtf8.js"; import { invariantViolation } from "../internalInvariant.js"; -const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const UTF8 = strictUtf8Decoder("preserve"); export interface DecodedMarshalUtf8 { readonly text: string; diff --git a/src/domain/referenceSourceImportParsing.ts b/src/domain/referenceSourceImportParsing.ts index 84a16c6117..d5c17a68a6 100644 --- a/src/domain/referenceSourceImportParsing.ts +++ b/src/domain/referenceSourceImportParsing.ts @@ -26,6 +26,7 @@ import { isTSImportEqualsDeclaration, isTSModuleDeclaration, } from "@babel/types"; +import { decodeStrictUtf8 } from "./strictUtf8.js"; type ReferenceSourceImportKind = | "imports" @@ -291,7 +292,7 @@ export const parseReferenceSourceImports = ( let source: string; try { - source = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + source = decodeStrictUtf8(bytes, "strip-signature"); } catch { return { relationships: [], diff --git a/src/domain/safeJson.ts b/src/domain/safeJson.ts index 96439db075..097150ed19 100644 --- a/src/domain/safeJson.ts +++ b/src/domain/safeJson.ts @@ -1,3 +1,4 @@ +import { strictUtf8Decoder } from "./strictUtf8.js"; /** JSON parsing without throws at an input boundary. */ export type SafeJsonParseResult = | { readonly ok: true; readonly value: unknown } @@ -7,9 +8,7 @@ export type SafeJsonParseResult = export const safeParseUtf8Json = (bytes: Uint8Array): SafeJsonParseResult => { let text: string; try { - text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - bytes, - ); + text = strictUtf8Decoder("preserve").decode(bytes); } catch (cause: unknown) { return { ok: false, diff --git a/src/domain/strictUtf8.test.ts b/src/domain/strictUtf8.test.ts new file mode 100644 index 0000000000..324f4332da --- /dev/null +++ b/src/domain/strictUtf8.test.ts @@ -0,0 +1,25 @@ +import { expect, it } from "vitest"; + +import { + decodeStrictUtf8, + isStrictUtf8, + strictUtf8Decoder, +} from "./strictUtf8.js"; + +const bomPrefixed = Uint8Array.of(0xef, 0xbb, 0xbf, 0x7b, 0x7d); + +it("keeps a leading BOM unless the caller states it is a signature", () => { + expect(decodeStrictUtf8(bomPrefixed, "preserve")).toBe("{}"); + expect(decodeStrictUtf8(bomPrefixed, "strip-signature")).toBe("{}"); + expect(strictUtf8Decoder("preserve").decode(bomPrefixed)).toBe("{}"); +}); + +it("refuses malformed UTF-8 for every policy", () => { + const malformed = Uint8Array.of(0x61, 0xc3); + expect(() => decodeStrictUtf8(malformed, "preserve")).toThrow(TypeError); + expect(() => decodeStrictUtf8(malformed, "strip-signature")).toThrow( + TypeError, + ); + expect(isStrictUtf8(malformed)).toBe(false); + expect(isStrictUtf8(bomPrefixed)).toBe(true); +}); diff --git a/src/domain/strictUtf8.ts b/src/domain/strictUtf8.ts new file mode 100644 index 0000000000..ec57d15714 --- /dev/null +++ b/src/domain/strictUtf8.ts @@ -0,0 +1,35 @@ +import { isUtf8 } from "node:buffer"; + +/** + * What a strict UTF-8 decode does with one leading byte order mark. + * + * - `strip-signature` drops it as an encoding signature. This is the WHATWG + * decoder default, so it is what a plain `{ fatal: true }` decoder did. + * - `preserve` keeps it as U+FEFF, so the text matches the caller's bytes and + * offsets (`ignoreBOM: true`). Use it where a leading U+FEFF is content. + * + * Changing a site's policy changes what its consumers see (for example an + * exact-value secret match or a JSON parser that rejects U+FEFF), so a change + * needs a regression test at that consumer. + */ +export type Utf8BomPolicy = "preserve" | "strip-signature"; + +/** + * A fatal UTF-8 decoder with an explicit BOM policy. It throws a TypeError on + * malformed bytes. Callers that pass `{ stream: true }` need their own + * instance, so this returns a new decoder each call. + */ +export const strictUtf8Decoder = (bom: Utf8BomPolicy): TextDecoder => + new TextDecoder("utf-8", { fatal: true, ignoreBOM: bom === "preserve" }); + +const preserving = strictUtf8Decoder("preserve"); +const stripping = strictUtf8Decoder("strip-signature"); + +/** Decode complete UTF-8 bytes; malformed input throws a TypeError. */ +export const decodeStrictUtf8 = ( + bytes: Uint8Array, + bom: Utf8BomPolicy, +): string => (bom === "preserve" ? preserving : stripping).decode(bytes); + +/** Whether the bytes are well-formed UTF-8, without building the string. */ +export const isStrictUtf8 = (bytes: Uint8Array): boolean => isUtf8(bytes); diff --git a/src/domain/uimageFormat.ts b/src/domain/uimageFormat.ts index 54e0e2d193..015d4fe6c6 100644 --- a/src/domain/uimageFormat.ts +++ b/src/domain/uimageFormat.ts @@ -1,6 +1,7 @@ import { crc32 } from "node:zlib"; import { err, ok, type Result } from "./result.js"; +import { strictUtf8Decoder } from "./strictUtf8.js"; /** * U-Boot legacy uImage header (U-Boot `include/image.h`, all fields @@ -191,7 +192,7 @@ const malformed = (message: string): UImageParseError => ({ _tag: "UImageFormatFailure", message, }); -const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const UTF8 = strictUtf8Decoder("preserve"); /** True when the first four bytes are the U-Boot legacy magic. */ export const hasUImageMagic = (prefix: Buffer): boolean => diff --git a/src/domain/unityfsFormat.ts b/src/domain/unityfsFormat.ts index 67d777cb16..4c17a4e41d 100644 --- a/src/domain/unityfsFormat.ts +++ b/src/domain/unityfsFormat.ts @@ -1,3 +1,4 @@ +import { strictUtf8Decoder } from "./strictUtf8.js"; /** * UnityFS (AssetBundle) container structure: a fixed signature header followed * by a compressed blocks-and-directory table and a block-compressed data area. @@ -119,7 +120,7 @@ export interface UnityFsHeaderFacts { readonly afterFixed: number; } -const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const utf8 = strictUtf8Decoder("preserve"); /** * Decode declared fields after the caller admits a prefix covering the diff --git a/src/dotnet/ManagedMetadataHeaps.ts b/src/dotnet/ManagedMetadataHeaps.ts index c512b8dbaa..eb3f292854 100644 --- a/src/dotnet/ManagedMetadataHeaps.ts +++ b/src/dotnet/ManagedMetadataHeaps.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import type { ManagedParseIssue } from "../domain/managed/managedArtifact.js"; import { createHash } from "node:crypto"; import { err, ok, type Result } from "../domain/result.js"; @@ -159,10 +160,7 @@ export const readMetadataString = ( ); managedDecodeBudget(layout).reserveString(end - start, start); try { - const value = new TextDecoder("utf-8", { - fatal: true, - ignoreBOM: true, - }).decode(bytes.subarray(start, end)); + const value = decodeStrictUtf8(bytes.subarray(start, end), "preserve"); managedDecodeBudget(layout).cacheString(index, value, end - start); return value; } catch (cause: unknown) { diff --git a/src/dotnet/ManagedMetadataInventoryRows.ts b/src/dotnet/ManagedMetadataInventoryRows.ts index 96b479d9f4..c75ab71927 100644 --- a/src/dotnet/ManagedMetadataInventoryRows.ts +++ b/src/dotnet/ManagedMetadataInventoryRows.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { readMemberReferenceRowFacts } from "./ManagedMemberReferenceRows.js"; import { readTypeDefinitionRowFacts } from "./ManagedTypeDefinitionRows.js"; import { managedDecodeBudget } from "./ManagedDecodeBudget.js"; @@ -434,9 +435,7 @@ const decodeFixedString = ( layout.blob?.offset ?? layout.rootOffset, ); try { - return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - blob.subarray(start, start + length), - ); + return decodeStrictUtf8(blob.subarray(start, start + length), "preserve"); } catch (cause: unknown) { // Invalid UTF-8 blobs decode as null. void cause; diff --git a/src/electron/ElectronFusePath.ts b/src/electron/ElectronFusePath.ts index 0a03bff64f..ec9c18458d 100644 --- a/src/electron/ElectronFusePath.ts +++ b/src/electron/ElectronFusePath.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { assertArtifactReadActive } from "../artifacts/ArtifactFileOperation.js"; import type { BigIntStats } from "node:fs"; import { lstat, readlink } from "node:fs/promises"; @@ -215,10 +216,7 @@ const resolveInside = async ( try { // A replacement character can alias a different, valid sibling name. // Preserve a literal UTF-8 BOM too: it is part of a filesystem name. - target = new TextDecoder("utf-8", { - fatal: true, - ignoreBOM: true, - }).decode(rawTarget); + target = decodeStrictUtf8(rawTarget, "preserve"); } catch { throw new ElectronFusePathFailure( "invalid_value", diff --git a/src/evm/EvmBytecodeCarrier.test.ts b/src/evm/EvmBytecodeCarrier.test.ts index bd291cbf6e..332e5f850c 100644 --- a/src/evm/EvmBytecodeCarrier.test.ts +++ b/src/evm/EvmBytecodeCarrier.test.ts @@ -34,7 +34,8 @@ it("rejects a long interior separator run without pathological backtracking", as `; await promisify(execFile)( process.execPath, - ["--input-type=module", "-e", source], + // tsx resolves the module's own `.js` imports to their TypeScript sources. + ["--import", "tsx", "--input-type=module", "-e", source], { timeout: 15_000, killSignal: "SIGKILL", diff --git a/src/evm/EvmBytecodeCarrier.ts b/src/evm/EvmBytecodeCarrier.ts index 9af031ec0c..93c7a77639 100644 --- a/src/evm/EvmBytecodeCarrier.ts +++ b/src/evm/EvmBytecodeCarrier.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; /** Expected carrier encoding failure, independent of the analysis engine. */ export class EvmCarrierFailure extends Error {} @@ -13,9 +14,7 @@ export const decodeEvmBytecodeCarrier = ( if (encoding === "raw") return Uint8Array.from(bytes); let text: string; try { - text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - bytes, - ); + text = decodeStrictUtf8(bytes, "preserve"); } catch (cause: unknown) { throw new EvmCarrierFailure("Hex carrier must be valid UTF-8.", { cause }); } diff --git a/src/evm/SolidityMetadata.ts b/src/evm/SolidityMetadata.ts index e43c06c63d..b21a7c5bf4 100644 --- a/src/evm/SolidityMetadata.ts +++ b/src/evm/SolidityMetadata.ts @@ -1,3 +1,4 @@ +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; import { EvmAdmissionError } from "./EvmAdmissionError.js"; import { decode, Tokenizer, Type, type Token } from "cborg"; import type { @@ -7,7 +8,7 @@ import type { const PROFILE = "terminal-cbor-scalars-v1"; const MAX_DEPTH = 64; -const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const utf8 = strictUtf8Decoder("preserve"); const decodeOptions = { useMaps: true, allowBigInt: true, diff --git a/src/ghidra/GhidraAnalysisSeeds.ts b/src/ghidra/GhidraAnalysisSeeds.ts index 78291b40ab..c9d6dcfb3e 100644 --- a/src/ghidra/GhidraAnalysisSeeds.ts +++ b/src/ghidra/GhidraAnalysisSeeds.ts @@ -16,6 +16,7 @@ import { } from "./GhidraFileArtifacts.js"; import { NonRegularFileReadError } from "../filesystem/RegularFile.js"; import { err, ok, type Result } from "../domain/result.js"; +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; /** Seed file format admitted by this adapter. */ export const GHIDRA_SEED_FORMAT = "rea-ghidra-seeds-v1"; @@ -170,7 +171,8 @@ const readSeeds = ( (bytes) => { let text: string; try { - text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + // A user-authored seed file may start with a BOM signature. + text = decodeStrictUtf8(bytes, "strip-signature"); } catch { throw seedLineError(0, "file is not valid UTF-8"); } diff --git a/src/ghidra/GhidraJavaOptions.ts b/src/ghidra/GhidraJavaOptions.ts index e53ffbe3df..1e407f8c09 100644 --- a/src/ghidra/GhidraJavaOptions.ts +++ b/src/ghidra/GhidraJavaOptions.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { GhidraAdmissionError } from "./GhidraAdmissionError.js"; import { randomUUID } from "node:crypto"; import { join } from "node:path"; @@ -147,10 +148,7 @@ const readArgumentFile = ( throw new GhidraAdmissionError("argument_files_exceed_bound", { budget_bytes: OPTIONS_BYTES, }); - const text = new TextDecoder("utf-8", { - fatal: true, - ignoreBOM: true, - }).decode(data); + const text = decodeStrictUtf8(data, "preserve"); const tokens = argumentFileTokens(text); if (tokens.some(fileReference)) throw new GhidraAdmissionError("nested_argument_file"); diff --git a/src/go/GoBinaryImage.ts b/src/go/GoBinaryImage.ts index e3fa11fc36..f69da8d172 100644 --- a/src/go/GoBinaryImage.ts +++ b/src/go/GoBinaryImage.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { readLeb128, ULEB64 } from "../domain/leb128.js"; import type { FileMapping, @@ -294,9 +295,7 @@ const inlineDataEnd = ( const observedText = (bytes: Buffer): string | null => { try { - return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - bytes, - ); + return decodeStrictUtf8(bytes, "preserve"); } catch (cause) { if (cause instanceof TypeError) return null; throw cause; diff --git a/src/go/GoModuleText.ts b/src/go/GoModuleText.ts index b3444d2af0..0a3e950216 100644 --- a/src/go/GoModuleText.ts +++ b/src/go/GoModuleText.ts @@ -1,6 +1,7 @@ +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; import type { GoModule, GoModuleMetadata } from "../domain/go/goBinary.js"; -const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const utf8 = strictUtf8Decoder("preserve"); const escapes = new Map([ ["a", 7], ["b", 8], diff --git a/src/gpu/AmdGpuMetadata.ts b/src/gpu/AmdGpuMetadata.ts index ace4bd5451..a506ecbbf1 100644 --- a/src/gpu/AmdGpuMetadata.ts +++ b/src/gpu/AmdGpuMetadata.ts @@ -1,3 +1,4 @@ +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; import { decode } from "@msgpack/msgpack"; import { BinaryResourceFailure, @@ -12,7 +13,7 @@ const AMD_METADATA_CONTAINER_ENTRIES = 65536; const AMD_METADATA_VALUE_BYTES = 65536; /** Validate allocation and nesting costs before the standard decoder allocates containers. */ -const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const utf8 = strictUtf8Decoder("preserve"); const preflight = (bytes: Buffer): Set => { const strings = new Set(); let at = 0, diff --git a/src/gpu/ClangOffloadBundle.ts b/src/gpu/ClangOffloadBundle.ts index 05777d7c14..8ad96a071e 100644 --- a/src/gpu/ClangOffloadBundle.ts +++ b/src/gpu/ClangOffloadBundle.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { scheduler } from "node:timers/promises"; import type { GpuObject } from "../domain/gpu/gpuCodeObjects.js"; import { @@ -69,8 +70,9 @@ export const readClangBundleDirectory = async ( ); let name: string; try { - name = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( + name = decodeStrictUtf8( await source.read(position + 24, nameSize), + "preserve", ); } catch (cause) { if (cause instanceof TypeError) diff --git a/src/gpu/GpuInventory.ts b/src/gpu/GpuInventory.ts index 2c48691fe3..4574a179c1 100644 --- a/src/gpu/GpuInventory.ts +++ b/src/gpu/GpuInventory.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { CCOB_MAGIC, CLANG_BUNDLE_MAGIC, @@ -366,10 +367,7 @@ export class GpuInventory { const content = bytes.subarray(0, length); let text: string; try { - text = new TextDecoder("utf-8", { - fatal: true, - ignoreBOM: true, - }).decode(content); + text = decodeStrictUtf8(content, "preserve"); } catch { return rejectGpu("PTX is not valid UTF-8 text"); } diff --git a/src/ida/IdaDiagnosticRedactor.ts b/src/ida/IdaDiagnosticRedactor.ts index a2954b2fb9..6c232d6b64 100644 --- a/src/ida/IdaDiagnosticRedactor.ts +++ b/src/ida/IdaDiagnosticRedactor.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { redactIdaJsonDiagnostic, redactIdaUnclassifiedDiagnostic, @@ -42,9 +43,7 @@ const basicPayloads = (payload: string): readonly string[] => { if (octets.some((value) => value < 0x20 || value === 0x7f)) return []; const candidates = [octets.toString("latin1")]; try { - candidates.push( - new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(octets), - ); + candidates.push(decodeStrictUtf8(octets, "preserve")); } catch { // Invalid UTF-8 has no decoded Unicode candidate. Exact Latin-1 preserves // each octet; do not invent a replacement-character credential. diff --git a/src/il2cpp/Il2cppMetadataFile.ts b/src/il2cpp/Il2cppMetadataFile.ts index c831036660..060dc1cb31 100644 --- a/src/il2cpp/Il2cppMetadataFile.ts +++ b/src/il2cpp/Il2cppMetadataFile.ts @@ -1,3 +1,4 @@ +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; import { checkRange, checkedSectionCount, @@ -355,7 +356,7 @@ const createStringReader = ( ) => { const stringEnd = strings.offset + strings.size; const pool = bytes.subarray(strings.offset, stringEnd); - const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + const decoder = strictUtf8Decoder("preserve"); const cache = new Map(); return (index: number, label: string) => { const cached = cache.get(index); diff --git a/src/il2cpp/Il2cppMetadataMembers.ts b/src/il2cpp/Il2cppMetadataMembers.ts index 74d4c231af..5dee6bacb6 100644 --- a/src/il2cpp/Il2cppMetadataMembers.ts +++ b/src/il2cpp/Il2cppMetadataMembers.ts @@ -1,3 +1,4 @@ +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; import type { Il2cppMetadata } from "../domain/il2cpp/il2cppMetadata.js"; import { checkRange, @@ -141,7 +142,7 @@ function* readStringLiterals( const { bytes, section, indexing, members } = tables; const literals = section("stringLiterals"), data = section("stringLiteralData"); - const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + const decoder = strictUtf8Decoder("preserve"); // Repeated spans reuse their decoded text for this immutable file snapshot. const decoded = new Map(); const rows: Members["string_literals"] = []; diff --git a/src/il2cpp/Il2cppMetadataV108.ts b/src/il2cpp/Il2cppMetadataV108.ts index e510d81637..7e8d5eeaaf 100644 --- a/src/il2cpp/Il2cppMetadataV108.ts +++ b/src/il2cpp/Il2cppMetadataV108.ts @@ -1,3 +1,4 @@ +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; import { IL2CPP_METADATA_SECTION_NAMES } from "../domain/il2cpp/il2cppMetadata.js"; import { observeIl2cppMetadataHeaderPrefix } from "../domain/il2cpp/il2cppMetadataHeaderPrefix.js"; import { Il2cppMetadataFormatFailure } from "./Il2cppMetadataFormat.js"; @@ -23,7 +24,7 @@ const HEADER_BYTES = 8 + SECTION_NAMES.length * 12; const SAMPLE_STRINGS = 16; const SAMPLE_BYTES = 128; const SCAN_BATCH_BYTES = 64 * 1024; -const utf8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); +const utf8 = strictUtf8Decoder("preserve"); /** A bounded, header-only observation; never a decoded metadata report. */ export const inspectIl2cppMetadataV108 = async ( diff --git a/src/javascript/recovery/RecoveryFiles.ts b/src/javascript/recovery/RecoveryFiles.ts index 253635c309..d03a74af5a 100644 --- a/src/javascript/recovery/RecoveryFiles.ts +++ b/src/javascript/recovery/RecoveryFiles.ts @@ -18,7 +18,6 @@ import { writeFile, } from "node:fs/promises"; import { basename, dirname, isAbsolute, join, relative, sep } from "node:path"; -import { TextDecoder } from "node:util"; import { normalizeArtifactPath } from "../../artifacts/ArtifactPaths.js"; import { AnalysisAccessDeniedError, @@ -36,6 +35,7 @@ import { sameRegularFileState, } from "../../filesystem/RegularFile.js"; import { RECOVERY_LIMITS, wakaruFixedLimitRecovery } from "./WakaruRelease.js"; +import { isStrictUtf8 } from "../../domain/strictUtf8.js"; const OPERATION = "recover_javascript_sources"; @@ -222,10 +222,8 @@ const snapshotRecoveryInput = async ( cause, ); } - try { - new TextDecoder("utf-8", { fatal: true }).decode(bytes); - } catch (cause: unknown) { - throw new AnalysisInputError(OPERATION, { cause }, [ + if (!isStrictUtf8(bytes)) + throw new AnalysisInputError(OPERATION, {}, [ { path: ["path"], reason: "invalid_format", @@ -233,7 +231,6 @@ const snapshotRecoveryInput = async ( message: `JavaScript input is not valid UTF-8: ${path}`, }, ]); - } try { await mkdir(dirname(snapshotPath), { recursive: true, mode: 0o700 }); await writeFile(snapshotPath, bytes, { flag: "wx", mode: 0o600 }); diff --git a/src/javascript/sourceMaps/SourceMapCodecProcess.ts b/src/javascript/sourceMaps/SourceMapCodecProcess.ts index 51dfd12330..382a7bc2e1 100644 --- a/src/javascript/sourceMaps/SourceMapCodecProcess.ts +++ b/src/javascript/sourceMaps/SourceMapCodecProcess.ts @@ -16,6 +16,7 @@ import { sourceMapCodecReplySchema, } from "./SourceMapCodecProtocol.js"; import { z } from "zod"; +import { decodeStrictUtf8 } from "../../domain/strictUtf8.js"; /** Dedicated process entrypoint invoked only for an explicitly selected map. */ const execute = async (): Promise => { @@ -28,7 +29,8 @@ const execute = async (): Promise => { ); const limits = selectedSourceMapLimits(selected); const data = await readStableArtifact(path, limits.outputBytes); - const text = new TextDecoder("utf-8", { fatal: true }).decode(data.bytes); + // Source maps on disk may carry a BOM signature (RFC 8259 8.1). + const text = decodeStrictUtf8(data.bytes, "strip-signature"); const value: unknown = JSON.parse(text); const input = sourceMapCodecInputSchema.parse(value); if (Buffer.byteLength(input.text) > limits.mapBytes) diff --git a/src/kuna/KunaJsonStream.ts b/src/kuna/KunaJsonStream.ts index b113ac725f..28e01d740c 100644 --- a/src/kuna/KunaJsonStream.ts +++ b/src/kuna/KunaJsonStream.ts @@ -16,6 +16,7 @@ import { import { err, ok, type Result } from "../domain/result.js"; import { KunaDocumentError } from "./KunaDocumentError.js"; import { kunaAddressFromJsonNumber } from "./KunaJson.js"; +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; /** * stream-json 3's synchronous tokenizer behind `parser`, which its @@ -136,7 +137,7 @@ const detached = (text: string): string => */ export abstract class KunaJsonDocumentReader extends Writable { readonly #hash = createHash("sha256"); - readonly #decoder = new TextDecoder("utf-8", { fatal: true }); + readonly #decoder = strictUtf8Decoder("strip-signature"); readonly #tokens = jsonParser({ packKeys: false, packNumbers: false, diff --git a/src/model/GgufHeader.ts b/src/model/GgufHeader.ts index 55ed527b23..9e6078827e 100644 --- a/src/model/GgufHeader.ts +++ b/src/model/GgufHeader.ts @@ -1,5 +1,5 @@ +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; import { isUtf8 } from "node:buffer"; -import { TextDecoder } from "node:util"; import { setImmediate } from "node:timers/promises"; import type { JsonValue } from "../domain/jsonValue.js"; import type { @@ -82,7 +82,7 @@ export class GgufHeader { rejectModel("GGUF string is not valid UTF-8"); return ""; } - const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + const decoder = strictUtf8Decoder("preserve"); let result = ""; try { for (let left = length; left > 0;) { diff --git a/src/model/SafetensorsHeader.ts b/src/model/SafetensorsHeader.ts index f9073fd5c7..34da2c0097 100644 --- a/src/model/SafetensorsHeader.ts +++ b/src/model/SafetensorsHeader.ts @@ -1,7 +1,7 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { escapeJsonPointerToken } from "../domain/jsonPointer.js"; import { setImmediate } from "node:timers/promises"; import { assertArtifactReadActive } from "../artifacts/ArtifactFileOperation.js"; -import { TextDecoder } from "node:util"; import type { ModelMetadata, ModelTensor, @@ -235,11 +235,7 @@ export const readSafetensorsHeader = async ( budget.charge(2 * size); let text: string; try { - text = - "{" + - new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - await reader.take(size - 1), - ); + text = "{" + decodeStrictUtf8(await reader.take(size - 1), "preserve"); } catch (cause) { if (cause instanceof TypeError) rejectModel("Safetensors header is not valid UTF-8"); diff --git a/src/native/AppleDispatchMetadata.ts b/src/native/AppleDispatchMetadata.ts index bede7ec1dd..92f78a31f2 100644 --- a/src/native/AppleDispatchMetadata.ts +++ b/src/native/AppleDispatchMetadata.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { malformedInput, parseFailureFromCause, @@ -165,9 +166,7 @@ export function* decodeAppleDispatchMetadataSteps( ); offset(address, end - start + 1); // A metadata C string owns every code point, including an initial U+FEFF. - return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - bytes.subarray(start, end), - ); + return decodeStrictUtf8(bytes.subarray(start, end), "preserve"); }; const result = objcSwiftMetadataSchema.parse({ db_save_result: null, diff --git a/src/native/AppleMachoNames.ts b/src/native/AppleMachoNames.ts index 6fd9e3a6ff..8ee17eaf20 100644 --- a/src/native/AppleMachoNames.ts +++ b/src/native/AppleMachoNames.ts @@ -1,12 +1,11 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; /** A dyld byte name projected losslessly as UTF-8, or retained exactly. */ export type MachoName = string | { readonly bytes_base64: string }; /** NUL terminators and source bounds are validated by the owning format reader. */ export const decodeMachoName = (bytes: Buffer): MachoName => { try { - return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - bytes, - ); + return decodeStrictUtf8(bytes, "preserve"); } catch { return { bytes_base64: bytes.toString("base64") }; } diff --git a/src/native/LldbCallTracer.ts b/src/native/LldbCallTracer.ts index 3e04bde148..8853fe7b03 100644 --- a/src/native/LldbCallTracer.ts +++ b/src/native/LldbCallTracer.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { bindArtifactFileCleanup } from "../artifacts/ArtifactFileOperation.js"; import { HOST_PLATFORM } from "../platform/HostPlatform.js"; import { invariantViolation } from "../domain/internalInvariant.js"; @@ -1198,9 +1199,7 @@ const readTracerOutput = async ( if (!bytes.value.ok) return bytes.value; let text: string; try { - text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - bytes.value.value, - ); + text = decodeStrictUtf8(bytes.value.value, "preserve"); } catch (cause: unknown) { return err(cause); } diff --git a/src/native/LldbRetainedObservations.ts b/src/native/LldbRetainedObservations.ts index 1081aec41d..489ef6b594 100644 --- a/src/native/LldbRetainedObservations.ts +++ b/src/native/LldbRetainedObservations.ts @@ -1,3 +1,4 @@ +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; import { NATIVE_CALL_CAPTURE_DEFAULTS, type NativeCallCaptureLimits, @@ -108,7 +109,7 @@ const parseLldbObservationJournal = (bytes: Uint8Array): ParsedLldbJournal => { const events: ParsedLldbJournal["events"] = []; const otherStops: string[] = []; const limitations: string[] = []; - const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + const decoder = strictUtf8Decoder("preserve"); let malformedCount = 0; let firstMalformedRow: number | undefined; let start = 0; diff --git a/src/native/MachoBinaryProjection.ts b/src/native/MachoBinaryProjection.ts index 00c387e80e..f356379f16 100644 --- a/src/native/MachoBinaryProjection.ts +++ b/src/native/MachoBinaryProjection.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { legacyMachoPlatform } from "../domain/apple/machoPlatform.js"; import { LC, @@ -88,9 +89,7 @@ const bytesText = (input: Uint8Array, fixed: boolean) => { const bytes = end < 0 ? input : input.subarray(0, end); try { return { - text: new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - bytes, - ), + text: decodeStrictUtf8(bytes, "preserve"), bytesHex: null, }; } catch { diff --git a/src/native/NativeMacOSProvider.ts b/src/native/NativeMacOSProvider.ts index 9b1d5356a2..6eafe092b1 100644 --- a/src/native/NativeMacOSProvider.ts +++ b/src/native/NativeMacOSProvider.ts @@ -1,6 +1,9 @@ import { MAX_NODE_TIMER_DELAY_MS } from "../domain/analysisResourceLimits.js"; import { withArtifactTargetIdentity } from "../artifacts/withArtifactTargetIdentity.js"; -import { invariantViolation } from "../domain/internalInvariant.js"; +import { + invariantViolation, + InternalInvariantError, +} from "../domain/internalInvariant.js"; import { translateNativeCommandFailure } from "./NativeCommandAnalysisError.js"; import { plistSourceFormat, @@ -114,7 +117,6 @@ import { type NativeCommandInvocation, } from "../domain/native/nativeInspection.js"; import { err, ok, type Result } from "../domain/result.js"; -import { InternalInvariantError } from "../domain/internalInvariant.js"; import { NativeCommandFailure, NATIVE_COMMAND_OUTPUT_BUDGET_BYTES, diff --git a/src/native/apple/AppleIpsParser.ts b/src/native/apple/AppleIpsParser.ts index fca3a774d8..7341a093ef 100644 --- a/src/native/apple/AppleIpsParser.ts +++ b/src/native/apple/AppleIpsParser.ts @@ -18,6 +18,7 @@ import { IPS_METADATA_SHAPE, } from "./AppleIpsJsonShape.js"; import type { LosslessJson } from "./losslessJson.js"; +import { strictUtf8Decoder } from "../../domain/strictUtf8.js"; type Artifact = AppleIps["artifact"]; @@ -30,7 +31,7 @@ const IPS_LIMITATIONS = [ "Symbol names printed inside the report are recorded claims and are not verified.", ]; -const utf8 = new TextDecoder("utf-8", { fatal: true }); +const utf8 = strictUtf8Decoder("strip-signature"); const canonicalUuid = ( raw: string | null, diff --git a/src/native/elf/KernelBuildConfig.ts b/src/native/elf/KernelBuildConfig.ts index cd735b964f..4ed233876f 100644 --- a/src/native/elf/KernelBuildConfig.ts +++ b/src/native/elf/KernelBuildConfig.ts @@ -12,6 +12,7 @@ import { type KernelBuildConfigEntry, } from "../../domain/native/kernelBuildConfig.js"; import { kernelCompressionCodecAt } from "./KernelCompressionCodec.js"; +import { strictUtf8Decoder } from "../../domain/strictUtf8.js"; type Report = KernelBuildConfig; type Entry = KernelBuildConfigEntry; @@ -234,7 +235,7 @@ const inflateGzipMember = async ( return { kind: "member", output, end: trailer + 8 }; }; -const decoder = new TextDecoder("utf-8", { fatal: true }); +const decoder = strictUtf8Decoder("strip-signature"); /** C0 controls other than tab, LF and CR, plus DEL. */ const CONTROL = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/u; diff --git a/src/native/parsers/plist.ts b/src/native/parsers/plist.ts index 331cc633f5..5898ee5318 100644 --- a/src/native/parsers/plist.ts +++ b/src/native/parsers/plist.ts @@ -44,6 +44,7 @@ import { decodeXmlPlistText, XmlPlistTextDecodeError, } from "../../domain/propertyListXmlText.js"; +import { decodeStrictUtf8 } from "../../domain/strictUtf8.js"; /** * Input ceiling matches native command output; projection is bounded separately. @@ -202,9 +203,7 @@ export const readPlistJsonSource = ( bytes: Buffer, ): PlistJsonParseAttempt | undefined => { try { - return readPlistJson( - new TextDecoder("utf-8", { fatal: true }).decode(bytes), - ); + return readPlistJson(decodeStrictUtf8(bytes, "strip-signature")); } catch { return undefined; } diff --git a/src/process/OwnedMcpStdioTransport.ts b/src/process/OwnedMcpStdioTransport.ts index baaa4fa134..0c7320341b 100644 --- a/src/process/OwnedMcpStdioTransport.ts +++ b/src/process/OwnedMcpStdioTransport.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { OwnerCancellation } from "../domain/analysisErrorCore.js"; import { deserializeMessage, @@ -386,11 +387,11 @@ export class OwnedMcpStdioTransport implements Transport { ); this.#appendFrame(fragment, frameBytes); if (newline === -1) return; - const line = new TextDecoder("utf-8", { - fatal: true, - // Preserve a leading BOM so the JSON-RPC parser can reject it. - ignoreBOM: true, - }).decode(this.#frameBuffer.subarray(0, this.#frameLength)); + // Preserve a leading BOM so the JSON-RPC parser can reject it. + const line = decodeStrictUtf8( + this.#frameBuffer.subarray(0, this.#frameLength), + "preserve", + ); this.#frameBuffer = Buffer.alloc(0); this.#frameLength = 0; this.onmessage?.(deserializeMessage(line)); diff --git a/src/process/ProviderProcess.ts b/src/process/ProviderProcess.ts index ddf6b4c298..d0819a4f48 100644 --- a/src/process/ProviderProcess.ts +++ b/src/process/ProviderProcess.ts @@ -37,6 +37,7 @@ import { type ManagedRuntime, type RuntimeExhaustion, } from "./RuntimeExhaustion.js"; +import { isStrictUtf8 } from "../domain/strictUtf8.js"; const DEFAULT_TERMINATION_GRACE_MS = 250; const DEFAULT_KILL_GRACE_MS = 1_000; @@ -1143,12 +1144,7 @@ class ProcessOutputCapture { observedBytes: number, ): ProviderProcessSnapshot["stdout"] { const text = buffer.toString("utf8"); - let invalidUtf8 = false; - try { - new TextDecoder("utf-8", { fatal: true }).decode(buffer); - } catch { - invalidUtf8 = true; - } + const invalidUtf8 = !isStrictUtf8(buffer); return { text, bytes: buffer.byteLength, diff --git a/src/process/RegexMatcher.ts b/src/process/RegexMatcher.ts index 70772b05f9..dc74691b3b 100644 --- a/src/process/RegexMatcher.ts +++ b/src/process/RegexMatcher.ts @@ -32,6 +32,7 @@ import { regexWorkerHeapRequestRecovery, type RegexWorkerHeapRequest, } from "../config/regexWorkerHeapRequest.js"; +import { decodeStrictUtf8, strictUtf8Decoder } from "../domain/strictUtf8.js"; /** * Node's Worker `resourceLimits.maxOldGenerationSizeMb` request. * `REA_REGEX_WORKER_HEAP_REQUEST_MIB` is that request (default 128 MiB). A positive @@ -232,8 +233,9 @@ const nodeOptionsFromEnviron = ( entry.subarray(0, nodeOptionsKey.length).equals(nodeOptionsKey) ) { try { - return new TextDecoder("utf-8", { fatal: true }).decode( + return decodeStrictUtf8( entry.subarray(nodeOptionsKey.length), + "strip-signature", ); } catch { return "invalid"; @@ -288,7 +290,7 @@ const commandLineTokens = (raw: Buffer): readonly string[] | "invalid" => { if (raw.length === 0) return "invalid"; const end = raw[raw.length - 1] === 0 ? raw.length - 1 : raw.length; const tokens: string[] = []; - const decoder = new TextDecoder("utf-8", { fatal: true }); + const decoder = strictUtf8Decoder("strip-signature"); let offset = 0; while (offset < end) { const next = raw.indexOf(0, offset); diff --git a/src/process/Utf8NdjsonLineBuffer.ts b/src/process/Utf8NdjsonLineBuffer.ts index db46d5e4f4..60f1970b90 100644 --- a/src/process/Utf8NdjsonLineBuffer.ts +++ b/src/process/Utf8NdjsonLineBuffer.ts @@ -1,3 +1,4 @@ +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; import { malformedInput, type ParseFailureError, @@ -12,10 +13,7 @@ export class Utf8NdjsonLineBuffer { #buffer: Buffer = Buffer.alloc(0); #fragments: Buffer[] = []; #nextNewline = -1; - readonly #decoder = new TextDecoder("utf-8", { - fatal: true, - ignoreBOM: true, - }); + readonly #decoder = strictUtf8Decoder("preserve"); /** * Yield each complete trimmed nonblank line; preserve unconsumed lines on diff --git a/src/providers/Utf8McpResponse.ts b/src/providers/Utf8McpResponse.ts index a106ede5e4..287e5713c9 100644 --- a/src/providers/Utf8McpResponse.ts +++ b/src/providers/Utf8McpResponse.ts @@ -1,3 +1,4 @@ +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; import { AnalysisProtocolError } from "../domain/analysisErrorCore.js"; /** Authored byte-validation failure, distinct from arbitrary upstream protocol diagnostics. */ @@ -14,7 +15,7 @@ export const validateMcpHttpResponse = ( ): Response => { // Failed HTTP responses belong to each adapter's existing diagnostic policy. if (!response.ok || response.body === null) return response; - const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + const decoder = strictUtf8Decoder("preserve"); const validate = (bytes?: Uint8Array): void => { try { decoder.decode(bytes, { stream: bytes !== undefined }); diff --git a/src/reference/ReferenceSourceReaderEntries.ts b/src/reference/ReferenceSourceReaderEntries.ts index adac16d650..cdefe2c03c 100644 --- a/src/reference/ReferenceSourceReaderEntries.ts +++ b/src/reference/ReferenceSourceReaderEntries.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import type { BigIntStats, Dir, Dirent } from "node:fs"; import { opendir, readlink, realpath, stat } from "node:fs/promises"; import { basename, dirname, join, sep } from "node:path"; @@ -478,10 +479,7 @@ const readSymlinkTarget = async ( } let target: string; try { - target = new TextDecoder("utf-8", { - fatal: true, - ignoreBOM: true, - }).decode(rawTarget); + target = decodeStrictUtf8(rawTarget, "preserve"); } catch { return entryFailure( path, diff --git a/src/rust/RustAuditableReader.ts b/src/rust/RustAuditableReader.ts index c1d5565e5d..772a704934 100644 --- a/src/rust/RustAuditableReader.ts +++ b/src/rust/RustAuditableReader.ts @@ -10,6 +10,7 @@ import { import { jsonValueByteLength } from "../domain/jsonValueByteLength.js"; import { malformedInput, ParseFailureError } from "../domain/parseFailure.js"; import { findDuplicateJsonMember } from "../domain/jsonDuplicateMember.js"; +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; /** Compressed zlib stream limit. */ const RUST_SECTION_BYTES = 1024 * 1024; @@ -225,7 +226,7 @@ const parseDocument = ( ): z.infer => { let text: string; try { - text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + text = decodeStrictUtf8(bytes, "strip-signature"); } catch (cause) { throw malformedSection("Dependency JSON is not valid UTF-8", cause); } diff --git a/src/server/environment/stateIntake.ts b/src/server/environment/stateIntake.ts index cf21e75a65..e0ea1cf944 100644 --- a/src/server/environment/stateIntake.ts +++ b/src/server/environment/stateIntake.ts @@ -19,6 +19,7 @@ import { RegularFileCleanupFailure, withOwnedRegularFile, } from "../../filesystem/OwnedRegularFile.js"; +import { decodeStrictUtf8 } from "../../domain/strictUtf8.js"; import { isRecord } from "../../domain/isRecord.js"; /** Why a named file or download could not be read, or a call was cancelled: a category naming neither the uri nor the content. */ @@ -133,7 +134,8 @@ const MAX_INLINE_BASE64_CHARACTERS = const parseDocumentBytes = (bytes: Uint8Array): Result => { let text: string; try { - text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + // A user-saved JSON document may carry a BOM signature (RFC 8259 8.1). + text = decodeStrictUtf8(bytes, "strip-signature"); } catch { return err(invalidParams("the file is not UTF-8 text")); } diff --git a/src/wasm/WabtArtifactProvider.ts b/src/wasm/WabtArtifactProvider.ts index 5afff5e0a5..dadbbee1e0 100644 --- a/src/wasm/WabtArtifactProvider.ts +++ b/src/wasm/WabtArtifactProvider.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { WabtAdmissionError } from "./WabtAdmissionError.js"; import { HOST_PLATFORM } from "../platform/HostPlatform.js"; import { randomUUID, createHash } from "node:crypto"; @@ -494,10 +495,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { }; try { // Keep source characters intact so locations match the hashed artifact. - const source = new TextDecoder("utf-8", { - fatal: true, - ignoreBOM: true, - }).decode(bytes.bytes); + const source = decodeStrictUtf8(bytes.bytes, "preserve"); glue[glueIndex] = { artifact: glueArtifact, ...associateWasmGlue( diff --git a/src/wasm/WabtInterface.ts b/src/wasm/WabtInterface.ts index a2a17fd3c7..b4eb9c2117 100644 --- a/src/wasm/WabtInterface.ts +++ b/src/wasm/WabtInterface.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { WabtAdmissionError } from "./WabtAdmissionError.js"; import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; import { wasmInterfaceEntryRepresentationBytes } from "../domain/wasm/wasmInterface.js"; @@ -57,9 +58,7 @@ const splitParts = (form: string, maximumBytes?: number): string[] => { const name = (text: string | undefined): string => { if (text?.startsWith('"') !== true) throw new WabtAdmissionError("name_missing"); - return new TextDecoder("utf8", { fatal: true, ignoreBOM: true }).decode( - watString(text, 1).bytes, - ); + return decodeStrictUtf8(watString(text, 1).bytes, "preserve"); }; const integer = (text: string): string | null => { if (!/^(?:\d+|0x[\da-f]+)$/iu.test(text)) return null; diff --git a/src/wasm/WabtOutput.ts b/src/wasm/WabtOutput.ts index bc55593403..4a63f437ff 100644 --- a/src/wasm/WabtOutput.ts +++ b/src/wasm/WabtOutput.ts @@ -1,5 +1,6 @@ import { WabtAdmissionError } from "./WabtAdmissionError.js"; import { readLeb128, ULEB32 } from "../domain/leb128.js"; +import { isStrictUtf8 } from "../domain/strictUtf8.js"; /** WABT prints names with %.*s, so an embedded NUL ends their display. */ const wabtDisplayedName = (name: string | undefined): string | undefined => @@ -104,12 +105,5 @@ const invalidCustomNameEnd = ( if ("failure" in read) return undefined; const nameEnd = read.next + Number(read.value); if (nameEnd > end) return undefined; - try { - new TextDecoder("utf-8", { fatal: true }).decode( - bytes.subarray(read.next, nameEnd), - ); - return undefined; - } catch { - return nameEnd; - } + return isStrictUtf8(bytes.subarray(read.next, nameEnd)) ? undefined : nameEnd; }; diff --git a/src/wasm/WabtWat.ts b/src/wasm/WabtWat.ts index f9c909eec6..051a3fba3e 100644 --- a/src/wasm/WabtWat.ts +++ b/src/wasm/WabtWat.ts @@ -1,3 +1,4 @@ +import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { WabtAdmissionError } from "./WabtAdmissionError.js"; import { parseWabtInterface } from "./WabtInterface.js"; import { watString, watStringEnd } from "./WabtString.js"; @@ -45,9 +46,7 @@ const moduleNames = ( if (nameEnd > end) throw new WabtAdmissionError("incomplete_module_name"); names.push( - new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - bytes.subarray(offset, nameEnd), - ), + decodeStrictUtf8(bytes.subarray(offset, nameEnd), "preserve"), ); // WABT calls OnModuleName before rejecting trailing subsection bytes. if (nameEnd !== end) @@ -121,10 +120,7 @@ export const parseWabtWat = (text: string, maximumInterfaceBytes?: number) => { throw new WabtAdmissionError("unsupported_custom_name"); const nameString = watString(form, prefix[0].length); // A WASM name is length-delimited UTF-8; leading U+FEFF is name data. - const name = new TextDecoder("utf-8", { - fatal: true, - ignoreBOM: true, - }).decode(nameString.bytes); + const name = decodeStrictUtf8(nameString.bytes, "preserve"); customSectionNames.push(name); if (name === "name") { const payloadPrefix = /^\s+"/u.exec(form.slice(nameString.end));