From 81fc9975827325758bc8d43afe986e5da3ceb2a3 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Sun, 1 Nov 2026 14:51:49 -0800 Subject: [PATCH] fix(release-exposure): decode wide text members before scanning A member's kind bypassed the NUL guard, so a UTF-16 plist was decoded as UTF-8, matched nothing and still counted as scanned. A directory holding only that member reported text.credential_markers as not_observed. Decode by byte order mark at one boundary instead. UTF-16 is scanned in either byte order, and inline maps read the same text. UTF-32 and NULs no mark explains count as skipped, whatever the member's kind. A binary plist keeps its raw scan. Unnamed members with a mark now reach the scanner, and the plist decoder shares the same mark detection. --- src/artifacts/ReleaseExposureReader.test.ts | 90 +++++++++++++++++++++ src/artifacts/ReleaseExposureReader.ts | 55 ++++++++++--- src/domain/propertyListXmlText.ts | 18 ++--- src/domain/textByteOrderMark.ts | 20 +++++ 4 files changed, 157 insertions(+), 26 deletions(-) create mode 100644 src/domain/textByteOrderMark.ts diff --git a/src/artifacts/ReleaseExposureReader.test.ts b/src/artifacts/ReleaseExposureReader.test.ts index 10b8d2af14..080ed72a02 100644 --- a/src/artifacts/ReleaseExposureReader.test.ts +++ b/src/artifacts/ReleaseExposureReader.test.ts @@ -1254,6 +1254,96 @@ const bundled = async ( return root; }; +describe("ReleaseExposureReader text encodings", () => { + const marker = "-----BEGIN PRIVATE KEY-----\n"; + + const utf16 = (text: string, order: "le" | "be"): Buffer => { + const little = Buffer.from(`\ufeff${text}`, "utf16le"); + return order === "le" ? little : Buffer.from(little).swap16(); + }; + + const visited = async (path: string) => { + const reader = new ReleaseExposureReader(); + try { + const texts: string[] = []; + const result = await reader.inspect({ path }, undefined, (text) => { + texts.push(text); + }); + if (!result.ok) throw result.error; + return { texts, members: result.value.text_members }; + } finally { + await reader.close(); + } + }; + + it.each(["le", "be"] as const)( + "decodes UTF-16 %s members instead of scanning NUL-interleaved bytes", + async (order) => { + const root = await createTestTempDirectory("rea-release-exposure-text-"); + const plist = `${marker}`; + await writeFile(join(root, "Info.plist"), utf16(plist, order)); + await writeFile(join(root, "credentials"), utf16(marker, order)); + const { texts, members } = await visited(root); + expect(texts.sort()).toEqual([marker, plist].sort()); + expect(members).toEqual({ scanned: 2, skipped: 0 }); + }, + ); + + it("counts wide text it cannot decode as skipped, whatever the member declares", async () => { + const root = await createTestTempDirectory("rea-release-exposure-text-"); + const wide = Buffer.from(marker, "utf16le"); + await writeFile(join(root, "bomless.plist"), wide); + await writeFile(join(root, "bomless.js"), wide); + await writeFile( + join(root, "wide.plist"), + Buffer.concat([ + Buffer.from([0xff, 0xfe, 0, 0]), + Buffer.from( + [...marker].flatMap((char) => [char.charCodeAt(0), 0, 0, 0]), + ), + ]), + ); + const { texts, members } = await visited(root); + expect(texts).toEqual([]); + expect(members).toEqual({ scanned: 0, skipped: 3 }); + }); + + it("keeps a binary plist's ASCII strings in the scan", async () => { + const root = await createTestTempDirectory("rea-release-exposure-text-"); + const bytes = Buffer.concat([ + Buffer.from("bplist00"), + Buffer.from([0]), + Buffer.from(marker), + ]); + await writeFile(join(root, "Info.plist"), bytes); + const { texts, members } = await visited(root); + expect(texts.join("")).toContain(marker); + expect(members).toEqual({ scanned: 1, skipped: 0 }); + }); + + it("reads an inline map from a UTF-16 script", async () => { + const root = await createTestTempDirectory("rea-release-exposure-text-"); + const map = Buffer.from( + JSON.stringify({ + version: 3, + sources: ["src/app.ts"], + sourcesContent: ["export {};"], + mappings: "", + }), + ).toString("base64"); + await writeFile( + join(root, "app.js"), + utf16( + `console.log(1);\n//# sourceMappingURL=data:application/json;base64,${map}\n`, + "le", + ), + ); + const scanned = await inspect(root); + expect(scanned.text_members).toEqual({ scanned: 1, skipped: 0 }); + expect(scanned.source_maps).toMatchObject({ embedded: 1, skipped: 0 }); + }); +}); + describe("ReleaseExposureReader text recognition", () => { const marker = "-----BEGIN PRIVATE KEY-----\n"; diff --git a/src/artifacts/ReleaseExposureReader.ts b/src/artifacts/ReleaseExposureReader.ts index 7e6f122fe7..76ad0dca1a 100644 --- a/src/artifacts/ReleaseExposureReader.ts +++ b/src/artifacts/ReleaseExposureReader.ts @@ -53,9 +53,11 @@ import { readMachoHeaderPrefixFacts, } from "../domain/apple/machoContainer.js"; import { inspectPeCodeViewFromBuffer } from "../domain/binaryTarget.js"; +import { hasBinaryPlistSignature } from "../domain/apple/appleArchiveFormat.js"; import { logicalPathNfc } from "../domain/artifactIdentity.js"; import { observeMzWindowsHeader } from "../domain/dosMz.js"; import { evidenceSubjectFormatSchema } from "../domain/evidenceSubject.js"; +import { byteOrderMarkEncoding } from "../domain/textByteOrderMark.js"; import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { RELEASE_EXPOSURE_PROVIDER } from "../domain/investigationProviders.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; @@ -868,7 +870,13 @@ const examineMapMember = async ( ? member : { ...member, logical: `${member.logical}.js.map` }; const loaded = await examineMap(state, declared); - if (loaded?.kind === "bytes") scanText(state, member, loaded.bytes); + if (loaded?.kind === "bytes") + scanText( + state, + member, + loaded.bytes.length, + decodeMemberText(loaded.bytes), + ); else state.skippedText += 1; return loaded?.kind !== "unread"; }; @@ -1038,9 +1046,12 @@ const examineTextMember = async ( Math.min(MAX_RELEASE_EXPOSURE_TEXT_MEMBER_BYTES, remaining), ); if (loaded.kind === "bytes") { - scanText(state, member, loaded.bytes); - if (isJavaScriptMember(member)) - examineInlineMaps(state, member, loaded.bytes); + const text = decodeMemberText(loaded.bytes); + scanText(state, member, loaded.bytes.length, text); + if (isJavaScriptMember(member)) { + if (text === null) state.skippedMaps += 1; + else examineInlineMaps(state, member, text); + } } else { state.skippedText += 1; if (isJavaScriptMember(member)) state.skippedMaps += 1; @@ -1065,16 +1076,18 @@ const examineProbedText = async ( ? probe.prefix : null; if (prefix === null) state.skippedText += 1; - else if (prefix.length > 0 && !prefix.includes(0)) + else if ( + prefix.length > 0 && + (!prefix.includes(0) || byteOrderMarkEncoding(prefix) !== null) + ) await examineTextMember(state, member); }; const examineInlineMaps = ( state: ScanState, member: MemberView, - bytes: Buffer, + source: string, ): void => { - const source = bytes.toString("utf8"); const parsed = parseJavaScriptSource(source, member.logical); if (parsed === null) { state.skippedMaps += 1; @@ -1160,22 +1173,38 @@ const noteUnreachedMapTargets = (state: ScanState): void => { if (!state.examinedMaps.has(target)) state.skippedMaps += 1; }; +/** + * The text a member's bytes encode, or null when they cannot be read as text. + * A member's kind says nothing about its encoding: NULs that no byte order mark + * explains are not UTF-8, so a scan of them would match nothing and still count + * as complete. A binary plist is the exception, because its ASCII strings are + * stored in the raw bytes. + */ +const decodeMemberText = (bytes: Buffer): string | null => { + const encoding = byteOrderMarkEncoding(bytes); + if (encoding === "utf-32") return null; + if (encoding !== null) return new TextDecoder(encoding).decode(bytes); + if (hasBinaryNul(bytes) && !hasBinaryPlistSignature(bytes)) return null; + return bytes.toString("utf8"); +}; + const scanText = ( state: ScanState, member: MemberView, - bytes: Buffer, + byteLength: number, + text: string | null, ): void => { if ( - bytes.length > MAX_RELEASE_EXPOSURE_TEXT_MEMBER_BYTES || - state.textBytes + bytes.length > MAX_RELEASE_EXPOSURE_TEXT_BYTES || - (hasBinaryNul(bytes) && !isDeclaredText(member.kind)) + byteLength > MAX_RELEASE_EXPOSURE_TEXT_MEMBER_BYTES || + state.textBytes + byteLength > MAX_RELEASE_EXPOSURE_TEXT_BYTES || + text === null ) { state.skippedText += 1; return; } - state.textBytes += bytes.length; + state.textBytes += byteLength; state.scannedText += 1; - state.visitText(bytes.toString("utf8"), member.location); + state.visitText(text, member.location); }; /** diff --git a/src/domain/propertyListXmlText.ts b/src/domain/propertyListXmlText.ts index 0cbccfede4..3cae8c3b69 100644 --- a/src/domain/propertyListXmlText.ts +++ b/src/domain/propertyListXmlText.ts @@ -1,3 +1,5 @@ +import { byteOrderMarkEncoding } from "./textByteOrderMark.js"; + export class XmlPlistTextDecodeError extends TypeError { constructor( readonly reason: @@ -15,24 +17,14 @@ export class XmlPlistTextDecodeError extends TypeError { /** Decode BOM-aware XML plist text and verify any declaration against its bytes. */ export const decodeXmlPlistText = (bytes: Uint8Array): string => { - if ( - (bytes[0] === 0xff && - bytes[1] === 0xfe && - bytes[2] === 0 && - bytes[3] === 0) || - (bytes[0] === 0 && bytes[1] === 0 && bytes[2] === 0xfe && bytes[3] === 0xff) - ) + const marked = byteOrderMarkEncoding(bytes); + if (marked === "utf-32") throw new XmlPlistTextDecodeError( "unsupported-encoding", "Unsupported XML encoding: UTF-32. Supported encodings are UTF-8 and BOM-marked UTF-16 in either byte order.", "UTF-32", ); - const encoding = - bytes[0] === 0xff && bytes[1] === 0xfe - ? "utf-16le" - : bytes[0] === 0xfe && bytes[1] === 0xff - ? "utf-16be" - : "utf-8"; + const encoding = marked ?? "utf-8"; // An ASCII declaration can identify an unsupported legacy encoding before // its body is incorrectly subjected to UTF-8 validation. const prolog = encoding === "utf-8" ? asciiXmlDeclaration(bytes) : undefined; diff --git a/src/domain/textByteOrderMark.ts b/src/domain/textByteOrderMark.ts new file mode 100644 index 0000000000..5ba3ca497f --- /dev/null +++ b/src/domain/textByteOrderMark.ts @@ -0,0 +1,20 @@ +/** + * The wide Unicode encoding a byte order mark declares, or null when there is + * none. UTF-32 is reported apart because no consumer decodes it, and it is + * checked first because its little-endian mark begins with UTF-16's. + */ +export const byteOrderMarkEncoding = ( + bytes: Uint8Array, +): "utf-16le" | "utf-16be" | "utf-32" | null => { + if ( + (bytes[0] === 0xff && + bytes[1] === 0xfe && + bytes[2] === 0 && + bytes[3] === 0) || + (bytes[0] === 0 && bytes[1] === 0 && bytes[2] === 0xfe && bytes[3] === 0xff) + ) + return "utf-32"; + if (bytes[0] === 0xff && bytes[1] === 0xfe) return "utf-16le"; + if (bytes[0] === 0xfe && bytes[1] === 0xff) return "utf-16be"; + return null; +};