fix(release-exposure): decode wide text members before scanning

A member's kind bypassed the NUL guard, so a UTF-16 plist was decoded as
UTF-8, matched nothing and still counted as scanned. A directory holding
only that member reported text.credential_markers as not_observed.

Decode by byte order mark at one boundary instead. UTF-16 is scanned in
either byte order, and inline maps read the same text. UTF-32 and NULs no
mark explains count as skipped, whatever the member's kind. A binary
plist keeps its raw scan. Unnamed members with a mark now reach the
scanner, and the plist decoder shares the same mark detection.

src/artifacts/ReleaseExposureReader.test.ts

@@ -1254,6 +1254,96 @@ const bundled = async (
   return root;
 };
 
+describe("ReleaseExposureReader text encodings", () => {
+  const marker = "-----BEGIN PRIVATE KEY-----\n";
+
+  const utf16 = (text: string, order: "le" | "be"): Buffer => {
+    const little = Buffer.from(`\ufeff${text}`, "utf16le");
+    return order === "le" ? little : Buffer.from(little).swap16();
+  };
+
+  const visited = async (path: string) => {
+    const reader = new ReleaseExposureReader();
+    try {
+      const texts: string[] = [];
+      const result = await reader.inspect({ path }, undefined, (text) => {
+        texts.push(text);
+      });
+      if (!result.ok) throw result.error;
+      return { texts, members: result.value.text_members };
+    } finally {
+      await reader.close();
+    }
+  };
+
+  it.each(["le", "be"] as const)(
+    "decodes UTF-16 %s members instead of scanning NUL-interleaved bytes",
+    async (order) => {
+      const root = await createTestTempDirectory("rea-release-exposure-text-");
+      const plist = `<?xml version="1.0" encoding="UTF-16"?><plist><string>${marker}</string></plist>`;
+      await writeFile(join(root, "Info.plist"), utf16(plist, order));
+      await writeFile(join(root, "credentials"), utf16(marker, order));
+      const { texts, members } = await visited(root);
+      expect(texts.sort()).toEqual([marker, plist].sort());
+      expect(members).toEqual({ scanned: 2, skipped: 0 });
+    },
+  );
+
+  it("counts wide text it cannot decode as skipped, whatever the member declares", async () => {
+    const root = await createTestTempDirectory("rea-release-exposure-text-");
+    const wide = Buffer.from(marker, "utf16le");
+    await writeFile(join(root, "bomless.plist"), wide);
+    await writeFile(join(root, "bomless.js"), wide);
+    await writeFile(
+      join(root, "wide.plist"),
+      Buffer.concat([
+        Buffer.from([0xff, 0xfe, 0, 0]),
+        Buffer.from(
+          [...marker].flatMap((char) => [char.charCodeAt(0), 0, 0, 0]),
+        ),
+      ]),
+    );
+    const { texts, members } = await visited(root);
+    expect(texts).toEqual([]);
+    expect(members).toEqual({ scanned: 0, skipped: 3 });
+  });
+
+  it("keeps a binary plist's ASCII strings in the scan", async () => {
+    const root = await createTestTempDirectory("rea-release-exposure-text-");
+    const bytes = Buffer.concat([
+      Buffer.from("bplist00"),
+      Buffer.from([0]),
+      Buffer.from(marker),
+    ]);
+    await writeFile(join(root, "Info.plist"), bytes);
+    const { texts, members } = await visited(root);
+    expect(texts.join("")).toContain(marker);
+    expect(members).toEqual({ scanned: 1, skipped: 0 });
+  });
+
+  it("reads an inline map from a UTF-16 script", async () => {
+    const root = await createTestTempDirectory("rea-release-exposure-text-");
+    const map = Buffer.from(
+      JSON.stringify({
+        version: 3,
+        sources: ["src/app.ts"],
+        sourcesContent: ["export {};"],
+        mappings: "",
+      }),
+    ).toString("base64");
+    await writeFile(
+      join(root, "app.js"),
+      utf16(
+        `console.log(1);\n//# sourceMappingURL=data:application/json;base64,${map}\n`,
+        "le",
+      ),
+    );
+    const scanned = await inspect(root);
+    expect(scanned.text_members).toEqual({ scanned: 1, skipped: 0 });
+    expect(scanned.source_maps).toMatchObject({ embedded: 1, skipped: 0 });
+  });
+});
+
 describe("ReleaseExposureReader text recognition", () => {
   const marker = "-----BEGIN PRIVATE KEY-----\n";
 

src/artifacts/ReleaseExposureReader.ts

@@ -53,9 +53,11 @@ import {
   readMachoHeaderPrefixFacts,
 } from "../domain/apple/machoContainer.js";
 import { inspectPeCodeViewFromBuffer } from "../domain/binaryTarget.js";
+import { hasBinaryPlistSignature } from "../domain/apple/appleArchiveFormat.js";
 import { logicalPathNfc } from "../domain/artifactIdentity.js";
 import { observeMzWindowsHeader } from "../domain/dosMz.js";
 import { evidenceSubjectFormatSchema } from "../domain/evidenceSubject.js";
+import { byteOrderMarkEncoding } from "../domain/textByteOrderMark.js";
 import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js";
 import { RELEASE_EXPOSURE_PROVIDER } from "../domain/investigationProviders.js";
 import { ProviderAdapterError } from "../domain/providerAdapterError.js";
@@ -868,7 +870,13 @@ const examineMapMember = async (
     ? member
     : { ...member, logical: `${member.logical}.js.map` };
   const loaded = await examineMap(state, declared);
-  if (loaded?.kind === "bytes") scanText(state, member, loaded.bytes);
+  if (loaded?.kind === "bytes")
+    scanText(
+      state,
+      member,
+      loaded.bytes.length,
+      decodeMemberText(loaded.bytes),
+    );
   else state.skippedText += 1;
   return loaded?.kind !== "unread";
 };
@@ -1038,9 +1046,12 @@ const examineTextMember = async (
     Math.min(MAX_RELEASE_EXPOSURE_TEXT_MEMBER_BYTES, remaining),
   );
   if (loaded.kind === "bytes") {
-    scanText(state, member, loaded.bytes);
-    if (isJavaScriptMember(member))
-      examineInlineMaps(state, member, loaded.bytes);
+    const text = decodeMemberText(loaded.bytes);
+    scanText(state, member, loaded.bytes.length, text);
+    if (isJavaScriptMember(member)) {
+      if (text === null) state.skippedMaps += 1;
+      else examineInlineMaps(state, member, text);
+    }
   } else {
     state.skippedText += 1;
     if (isJavaScriptMember(member)) state.skippedMaps += 1;
@@ -1065,16 +1076,18 @@ const examineProbedText = async (
         ? probe.prefix
         : null;
   if (prefix === null) state.skippedText += 1;
-  else if (prefix.length > 0 && !prefix.includes(0))
+  else if (
+    prefix.length > 0 &&
+    (!prefix.includes(0) || byteOrderMarkEncoding(prefix) !== null)
+  )
     await examineTextMember(state, member);
 };
 
 const examineInlineMaps = (
   state: ScanState,
   member: MemberView,
-  bytes: Buffer,
+  source: string,
 ): void => {
-  const source = bytes.toString("utf8");
   const parsed = parseJavaScriptSource(source, member.logical);
   if (parsed === null) {
     state.skippedMaps += 1;
@@ -1160,22 +1173,38 @@ const noteUnreachedMapTargets = (state: ScanState): void => {
     if (!state.examinedMaps.has(target)) state.skippedMaps += 1;
 };
 
+/**
+ * The text a member's bytes encode, or null when they cannot be read as text.
+ * A member's kind says nothing about its encoding: NULs that no byte order mark
+ * explains are not UTF-8, so a scan of them would match nothing and still count
+ * as complete. A binary plist is the exception, because its ASCII strings are
+ * stored in the raw bytes.
+ */
+const decodeMemberText = (bytes: Buffer): string | null => {
+  const encoding = byteOrderMarkEncoding(bytes);
+  if (encoding === "utf-32") return null;
+  if (encoding !== null) return new TextDecoder(encoding).decode(bytes);
+  if (hasBinaryNul(bytes) && !hasBinaryPlistSignature(bytes)) return null;
+  return bytes.toString("utf8");
+};
+
 const scanText = (
   state: ScanState,
   member: MemberView,
-  bytes: Buffer,
+  byteLength: number,
+  text: string | null,
 ): void => {
   if (
-    bytes.length > MAX_RELEASE_EXPOSURE_TEXT_MEMBER_BYTES ||
-    state.textBytes + bytes.length > MAX_RELEASE_EXPOSURE_TEXT_BYTES ||
-    (hasBinaryNul(bytes) && !isDeclaredText(member.kind))
+    byteLength > MAX_RELEASE_EXPOSURE_TEXT_MEMBER_BYTES ||
+    state.textBytes + byteLength > MAX_RELEASE_EXPOSURE_TEXT_BYTES ||
+    text === null
   ) {
     state.skippedText += 1;
     return;
   }
-  state.textBytes += bytes.length;
+  state.textBytes += byteLength;
   state.scannedText += 1;
-  state.visitText(bytes.toString("utf8"), member.location);
+  state.visitText(text, member.location);
 };
 
 /**

src/domain/propertyListXmlText.ts

@@ -1,3 +1,5 @@
+import { byteOrderMarkEncoding } from "./textByteOrderMark.js";
+
 export class XmlPlistTextDecodeError extends TypeError {
   constructor(
     readonly reason:
@@ -15,24 +17,14 @@ export class XmlPlistTextDecodeError extends TypeError {
 
 /** Decode BOM-aware XML plist text and verify any declaration against its bytes. */
 export const decodeXmlPlistText = (bytes: Uint8Array): string => {
-  if (
-    (bytes[0] === 0xff &&
-      bytes[1] === 0xfe &&
-      bytes[2] === 0 &&
-      bytes[3] === 0) ||
-    (bytes[0] === 0 && bytes[1] === 0 && bytes[2] === 0xfe && bytes[3] === 0xff)
-  )
+  const marked = byteOrderMarkEncoding(bytes);
+  if (marked === "utf-32")
     throw new XmlPlistTextDecodeError(
       "unsupported-encoding",
       "Unsupported XML encoding: UTF-32. Supported encodings are UTF-8 and BOM-marked UTF-16 in either byte order.",
       "UTF-32",
     );
-  const encoding =
-    bytes[0] === 0xff && bytes[1] === 0xfe
-      ? "utf-16le"
-      : bytes[0] === 0xfe && bytes[1] === 0xff
-        ? "utf-16be"
-        : "utf-8";
+  const encoding = marked ?? "utf-8";
   // An ASCII declaration can identify an unsupported legacy encoding before
   // its body is incorrectly subjected to UTF-8 validation.
   const prolog = encoding === "utf-8" ? asciiXmlDeclaration(bytes) : undefined;

src/domain/textByteOrderMark.ts

@@ -0,0 +1,20 @@
+/**
+ * The wide Unicode encoding a byte order mark declares, or null when there is
+ * none. UTF-32 is reported apart because no consumer decodes it, and it is
+ * checked first because its little-endian mark begins with UTF-16's.
+ */
+export const byteOrderMarkEncoding = (
+  bytes: Uint8Array,
+): "utf-16le" | "utf-16be" | "utf-32" | null => {
+  if (
+    (bytes[0] === 0xff &&
+      bytes[1] === 0xfe &&
+      bytes[2] === 0 &&
+      bytes[3] === 0) ||
+    (bytes[0] === 0 && bytes[1] === 0 && bytes[2] === 0xfe && bytes[3] === 0xff)
+  )
+    return "utf-32";
+  if (bytes[0] === 0xff && bytes[1] === 0xfe) return "utf-16le";
+  if (bytes[0] === 0xfe && bytes[1] === 0xff) return "utf-16be";
+  return null;
+};