fix(desktop): read BOM-marked UTF-16 in state and task views (#12052)

Co-authored-by: ShuaixinHuang <197200901+ShuaixinHuang@users.noreply.github.com>
This commit is contained in:
ShuaixinHuangandShuaixinHuang authored and GitHub committed 2026-11-18 09:41:35 -05:00
1 parent 1868d04ef9
commit c47442a2eb
8 files changed
+134 -60

No files matched your search

+6 -2
View File
@@ -146,8 +146,12 @@ whether the file was present, the SHA-256 of its retained bytes, and a view:
| `json` | A `.json` name or a leading `{` or `[` that parses | The parsed value, from files up to 4 MiB nested up to 500 levels |
| `xml` | An XML name, a declaration or a leading element that parses | An element tree (`name`, `attributes`, `children`); text kept exactly, DTDs never read |
| `keyfile` | Only `[group]`, `key=value` and comment lines, with at least one group | Groups of `key` → string value, from files up to 4 MiB |
| `text` | Other UTF-8 text | Lines, up to 5,000 |
| `binary` | NUL bytes or invalid UTF-8 | None |
| `text` | Other decoded text | Lines, up to 5,000 |
| `binary` | Content that cannot be read as text | None |
Text views decode UTF-8 or strictly valid UTF-16 with a leading byte-order
mark. UTF-16 containing a NUL character stays binary. These rules also apply
to documents inside gzip and ZIP files and to the exported task verifier.
A gzip file is read as the document inside it; see
[compressed documents](#compressed-documents).
@@ -18,7 +18,7 @@ type StatePath = DesktopApplicationState["paths"][number];
/**
* The owned files present at the seed with their digests, and those read as
* uncompressed text, which boot respells as UTF-8, but for those `isLeftOut`
* uncompressed text, which boot respells in its original encoding, excluding `isLeftOut`
* names: lock files the app holds while it runs, since a copied one makes it
* find its profile or document in use, and its logs, which record facts about
* the capture host.
+23 -14
View File
@@ -99,16 +99,20 @@ def _respelled(text: str, home: Path) -> str:
return re.sub(re.escape(SEED_HOME) + r"(?![\w.@+~-])", lambda _: str(home), text)
UTF16_CODECS = {b"\xff\xfe": "utf-16-le", b"\xfe\xff": "utf-16-be"}
def _respelled_utf16(data: bytes, home: Path) -> bytes:
"""A UTF-16 text file with a byte-order mark respelled like a text seed, its mark kept; any other bytes as given."""
for mark, codec in ((b"\xff\xfe", "utf-16-le"), (b"\xfe\xff", "utf-16-be")):
if data.startswith(mark):
try:
text = data[len(mark):].decode(codec)
except UnicodeDecodeError:
return data
return mark + _respelled(text, home).encode(codec) if SEED_HOME in text else data
return data
mark = data[:2]
codec = UTF16_CODECS.get(mark)
if codec is None:
return data
try:
text = data[2:].decode(codec)
except UnicodeDecodeError:
return data
return mark + _respelled(text, home).encode(codec) if SEED_HOME in text else data
#: What marks a window as not yet the app's own: skipping the taskbar, as splash screens and dialogs do, or a splash type.
@@ -141,10 +145,10 @@ def _seed_home(home: Path) -> None:
target = home / relative
target.parent.mkdir(parents=True, exist_ok=True)
data = source.read_bytes()
if relative in TEXT_SEEDS:
data = _respelled(data.decode("utf-8"), home).encode("utf-8")
else:
if data[:2] in UTF16_CODECS:
data = _respelled_utf16(data, home)
elif relative in TEXT_SEEDS:
data = _respelled(data.decode("utf-8"), home).encode("utf-8")
target.write_bytes(data)
mode = SEED_MODES.get(relative)
if mode is not None:
@@ -400,10 +404,14 @@ def _text_view(name: str, text: str, size: int) -> dict:
def _bytes_view(name: str, data: bytes) -> dict:
"""The view of a document that is no database or archive: its text's, or none when it is binary."""
try:
text = data.decode("utf-8-sig") if b"\x00" not in data[:8192] else None
codec = UTF16_CODECS.get(bytes(data[:2]))
if codec is not None:
text = data[2:].decode(codec)
else:
text = data.decode("utf-8-sig") if b"\x00" not in data[:8192] else None
except UnicodeDecodeError:
text = None
if text is None:
if text is None or (codec is not None and "\x00" in text):
return {"format": "binary", "view": None, "truncated": False}
return _text_view(name, text, len(data))
@@ -417,7 +425,8 @@ def _view(path: Path) -> dict:
data = file.read(min(8192, MAX_VIEW_BYTES + 1))
if len(data) > MAX_VIEW_BYTES:
return too_large
whole = data.startswith(GZIP_HEADER) or (not data.startswith(SQLITE_HEADER) and b"\x00" not in data)
whole = (data.startswith(GZIP_HEADER) or data[:2] in UTF16_CODECS
or (not data.startswith(SQLITE_HEADER) and b"\x00" not in data))
remaining = MAX_VIEW_BYTES + 1 - len(data)
if whole and remaining > 0:
data += file.read(remaining)
+13 -32
View File
@@ -16,6 +16,7 @@ import {
isTextualFormat,
textSyntaxFormat,
} from "./stateFormats.js";
import { decodeStateUtf16 } from "./stateTextEncoding.js";
import { strictUtf8Decoder } from "../strictUtf8.js";
import { isRecord } from "../../kernel/isRecord.js";
@@ -330,33 +331,6 @@ const plainScrub = (text: string, identity: HostIdentity): TextScrub => {
return { text: scrubbed, decoded: scrubbed };
};
/** UTF-16 text after a byte-order mark, decoded and encoded as the computer's boot respelling does; undefined otherwise. */
const utf16Text = (bytes: Buffer) => {
const order = [
{ mark: [0xff, 0xfe], swap: false },
{ mark: [0xfe, 0xff], swap: true },
].find(({ mark }) => mark.every((byte, index) => bytes[index] === byte));
if (order === undefined) return undefined;
const little = Buffer.from(bytes.subarray(2));
if (order.swap && little.length % 2 === 0) little.swap16();
try {
return {
// The signature is already removed; a leading U+FEFF here is content.
text: new TextDecoder("utf-16le", {
fatal: true,
ignoreBOM: true,
}).decode(little),
encode: (scrubbed: string) => {
const encoded = Buffer.from(scrubbed, "utf16le");
if (order.swap) encoded.swap16();
return Buffer.concat([Buffer.from(order.mark), encoded]);
},
};
} catch {
return undefined;
}
};
/** A seed by its home path and the format the state view read it as. */
interface Seed {
readonly home: string;
@@ -370,12 +344,19 @@ interface Seed {
*/
const seedText = (bytes: Buffer, { home, format }: Seed) => {
if (bytes.length > MAX_SCRUBBED_BYTES) return undefined;
if (!isTextualFormat(format)) {
const utf16 = utf16Text(bytes);
return utf16 === undefined
? undefined
: { ...utf16, format: textSyntaxFormat(home, utf16.text) };
const utf16 = decodeStateUtf16(bytes);
if (utf16 !== undefined) {
return {
text: utf16.text,
format: textSyntaxFormat(home, utf16.text),
encode: (scrubbed: string) => {
const encoded = Buffer.from(scrubbed, "utf16le");
if (utf16.order === "be") encoded.swap16();
return Buffer.concat([bytes.subarray(0, 2), encoded]);
},
};
}
if (!isTextualFormat(format)) return undefined;
try {
const text = strictUtf8Decoder("preserve").decode(bytes);
return {
+6 -2
View File
@@ -16,6 +16,7 @@ import {
parseXmlDocument,
} from "../xmlDocument.js";
import { expatXmlName } from "./expatXmlNames.js";
import { decodeStateUtf16 } from "./stateTextEncoding.js";
import { decodeStrictUtf8 } from "../strictUtf8.js";
/** Formats the state view reads into JSON. */
@@ -89,8 +90,11 @@ const isKeyfile = (text: string): boolean => {
return group;
};
/** Decode bytes as UTF-8 text, or undefined for binary content. */
/** Decode UTF-8 and BOM-marked UTF-16, or return undefined for binary content. */
export const decodeStateText = (bytes: Uint8Array): string | undefined => {
const utf16 = decodeStateUtf16(bytes);
if (utf16 !== undefined)
return utf16.text.includes("\0") ? undefined : utf16.text;
if (bytes.subarray(0, 8192).includes(0)) return undefined;
try {
return decodeStrictUtf8(bytes, "strip-signature");
@@ -566,7 +570,7 @@ export const xmlView = (
/** Formats read from a file's bytes alone, without a tool or worker. */
export type TextualStateFormat = "text" | "json" | "keyfile" | "xml";
/** Whether a format is one of the text formats, read as UTF-8. */
/** Whether a format is one of the text formats. */
export const isTextualFormat = (
format: string | null,
): format is TextualStateFormat =>
+20
View File
@@ -0,0 +1,20 @@
const UTF16 = {
le: new TextDecoder("utf-16le", { fatal: true, ignoreBOM: true }),
be: new TextDecoder("utf-16be", { fatal: true, ignoreBOM: true }),
};
/** Decode UTF-16 only with a byte-order mark. Strip exactly that first mark. */
export const decodeStateUtf16 = (bytes: Uint8Array) => {
const order =
bytes[0] === 0xff && bytes[1] === 0xfe
? "le"
: bytes[0] === 0xfe && bytes[1] === 0xff
? "be"
: undefined;
if (order === undefined) return undefined;
try {
return { order, text: UTF16[order].decode(bytes.subarray(2)) };
} catch {
return undefined;
}
};
@@ -214,31 +214,29 @@ it("seeds each file the bundle lists in SEED_MODES with that mode, as a lock who
"[s]\n",
);
await writeFile(join(root, "seed", "run.sh"), "#!/bin/sh\n");
const work = "work.txt";
const home = join(root, "home");
await writeFile(join(root, "seed", work), utf16("/home/ada/project\n", "be"));
const imports = PYTHON_IMPORTS.split("\n")
.filter((line) => !line.includes("agentenv_protocol"))
.join("\n");
const program = `${imports}
SEED_HOME = "/home/ada"
TEXT_SEEDS = frozenset()
TEXT_SEEDS = frozenset(["work.txt"])
SEED_MODES = {".config/notes/secret.ini": 0o600, "run.sh": 0o755}
HERE = Path(sys.argv[1])
${PYTHON_HELPERS}
_seed_home(Path(sys.argv[2]))
`;
execFileSync("python3", [
"-I",
"-B",
"-c",
program,
root,
join(root, "home"),
]);
execFileSync("python3", ["-I", "-B", "-c", program, root, home]);
const mode = async (path: string) =>
(await stat(join(root, "home", path))).mode & 0o777;
expect(await mode(".config/notes/secret.ini")).toBe(0o600);
expect(await mode("run.sh")).toBe(0o755);
const restored = await readFile(join(home, work));
expect(restored.equals(utf16(`${home}/project\n`, "be"))).toBe(true);
});
it.each([true, false])(
@@ -40,6 +40,64 @@ const sqlite = Buffer.concat([
Buffer.alloc(84),
]);
it("reads and checks BOM-signed UTF-16 project files in either byte order", async () => {
const name = "settings.xml";
const text = `<?xml version="1.0" encoding="UTF-16"?>
<project title="Café 🎨">${" ".repeat(4200)}</project>`;
const little = Buffer.from(`\ufeff${text}`, "utf16le");
const big = Buffer.from(little).swap16();
const files = [
{ name, bytes: little },
{ name, bytes: big },
{ name, bytes: gzipSync(big, { level: 0 }) },
{
name: "project.zip",
bytes: zipArchive([{ name, data: little, method: 0 }]),
},
{ name, bytes: Buffer.from([0xff, 0xfe, 0x00, 0xd8]) },
{ name, bytes: Buffer.from(text, "utf16le") },
{ name, bytes: Buffer.from("\ufeffa\0b", "utf16le") },
{
name: "settings.json",
bytes: Buffer.from('\ufeff{"zoom":1.5}', "utf16le"),
},
];
await expectSameFileViews(files);
const views = await Promise.all(files.map(typescriptFileView));
expect(views.map(({ format }) => format)).toEqual([
"xml",
"xml",
"xml",
"zip",
"binary",
"binary",
"binary",
"json",
]);
expect(views[3]?.view).toMatchObject({
entries: [{ name, format: "xml", view: views[0]?.view }],
});
expect(views[7]?.view).toEqual({ zoom: 1.5 });
const outcomes = await expectSameCheckResults(
name,
[
{
id: "project-title",
kind: "value_equals",
file: name,
pointer: "/root/attributes/title",
value: "Café 🎨",
},
],
files.slice(0, 3),
);
expect(outcomes.map(([outcome]) => outcome?.passed)).toEqual([
true,
true,
true,
]);
});
describe("gzip files in the exported module", () => {
it("reads a compressed drawing as the view of the drawing inside, and grades it alike", async () => {
const files = await posters();