diff --git a/docs/desktop-application-state.md b/docs/desktop-application-state.md index 530a150905..a3e9644c5c 100644 --- a/docs/desktop-application-state.md +++ b/docs/desktop-application-state.md @@ -146,8 +146,12 @@ whether the file was present, the SHA-256 of its retained bytes, and a view: | `json` | A `.json` name or a leading `{` or `[` that parses | The parsed value, from files up to 4 MiB nested up to 500 levels | | `xml` | An XML name, a declaration or a leading element that parses | An element tree (`name`, `attributes`, `children`); text kept exactly, DTDs never read | | `keyfile` | Only `[group]`, `key=value` and comment lines, with at least one group | Groups of `key` → string value, from files up to 4 MiB | -| `text` | Other UTF-8 text | Lines, up to 5,000 | -| `binary` | NUL bytes or invalid UTF-8 | None | +| `text` | Other decoded text | Lines, up to 5,000 | +| `binary` | Content that cannot be read as text | None | + +Text views decode UTF-8 or strictly valid UTF-16 with a leading byte-order +mark. UTF-16 containing a NUL character stays binary. These rules also apply +to documents inside gzip and ZIP files and to the exported task verifier. A gzip file is read as the document inside it; see [compressed documents](#compressed-documents). diff --git a/src/application/desktop/DesktopExportLaunch.ts b/src/application/desktop/DesktopExportLaunch.ts index 8a3d1457b4..d422059047 100644 --- a/src/application/desktop/DesktopExportLaunch.ts +++ b/src/application/desktop/DesktopExportLaunch.ts @@ -18,7 +18,7 @@ type StatePath = DesktopApplicationState["paths"][number]; /** * The owned files present at the seed with their digests, and those read as - * uncompressed text, which boot respells as UTF-8, but for those `isLeftOut` + * uncompressed text, which boot respells in its original encoding, excluding `isLeftOut` * names: lock files the app holds while it runs, since a copied one makes it * find its profile or document in use, and its logs, which record facts about * the capture host. diff --git a/src/domain/desktop/appExportPython.ts b/src/domain/desktop/appExportPython.ts index 004baa4542..2d6a7d75da 100644 --- a/src/domain/desktop/appExportPython.ts +++ b/src/domain/desktop/appExportPython.ts @@ -99,16 +99,20 @@ def _respelled(text: str, home: Path) -> str: return re.sub(re.escape(SEED_HOME) + r"(?![\w.@+~-])", lambda _: str(home), text) +UTF16_CODECS = {b"\xff\xfe": "utf-16-le", b"\xfe\xff": "utf-16-be"} + + def _respelled_utf16(data: bytes, home: Path) -> bytes: """A UTF-16 text file with a byte-order mark respelled like a text seed, its mark kept; any other bytes as given.""" - for mark, codec in ((b"\xff\xfe", "utf-16-le"), (b"\xfe\xff", "utf-16-be")): - if data.startswith(mark): - try: - text = data[len(mark):].decode(codec) - except UnicodeDecodeError: - return data - return mark + _respelled(text, home).encode(codec) if SEED_HOME in text else data - return data + mark = data[:2] + codec = UTF16_CODECS.get(mark) + if codec is None: + return data + try: + text = data[2:].decode(codec) + except UnicodeDecodeError: + return data + return mark + _respelled(text, home).encode(codec) if SEED_HOME in text else data #: What marks a window as not yet the app's own: skipping the taskbar, as splash screens and dialogs do, or a splash type. @@ -141,10 +145,10 @@ def _seed_home(home: Path) -> None: target = home / relative target.parent.mkdir(parents=True, exist_ok=True) data = source.read_bytes() - if relative in TEXT_SEEDS: - data = _respelled(data.decode("utf-8"), home).encode("utf-8") - else: + if data[:2] in UTF16_CODECS: data = _respelled_utf16(data, home) + elif relative in TEXT_SEEDS: + data = _respelled(data.decode("utf-8"), home).encode("utf-8") target.write_bytes(data) mode = SEED_MODES.get(relative) if mode is not None: @@ -400,10 +404,14 @@ def _text_view(name: str, text: str, size: int) -> dict: def _bytes_view(name: str, data: bytes) -> dict: """The view of a document that is no database or archive: its text's, or none when it is binary.""" try: - text = data.decode("utf-8-sig") if b"\x00" not in data[:8192] else None + codec = UTF16_CODECS.get(bytes(data[:2])) + if codec is not None: + text = data[2:].decode(codec) + else: + text = data.decode("utf-8-sig") if b"\x00" not in data[:8192] else None except UnicodeDecodeError: text = None - if text is None: + if text is None or (codec is not None and "\x00" in text): return {"format": "binary", "view": None, "truncated": False} return _text_view(name, text, len(data)) @@ -417,7 +425,8 @@ def _view(path: Path) -> dict: data = file.read(min(8192, MAX_VIEW_BYTES + 1)) if len(data) > MAX_VIEW_BYTES: return too_large - whole = data.startswith(GZIP_HEADER) or (not data.startswith(SQLITE_HEADER) and b"\x00" not in data) + whole = (data.startswith(GZIP_HEADER) or data[:2] in UTF16_CODECS + or (not data.startswith(SQLITE_HEADER) and b"\x00" not in data)) remaining = MAX_VIEW_BYTES + 1 - len(data) if whole and remaining > 0: data += file.read(remaining) diff --git a/src/domain/desktop/seedScrub.ts b/src/domain/desktop/seedScrub.ts index 48f7ccdae3..be4e1e6803 100644 --- a/src/domain/desktop/seedScrub.ts +++ b/src/domain/desktop/seedScrub.ts @@ -16,6 +16,7 @@ import { isTextualFormat, textSyntaxFormat, } from "./stateFormats.js"; +import { decodeStateUtf16 } from "./stateTextEncoding.js"; import { strictUtf8Decoder } from "../strictUtf8.js"; import { isRecord } from "../../kernel/isRecord.js"; @@ -330,33 +331,6 @@ const plainScrub = (text: string, identity: HostIdentity): TextScrub => { return { text: scrubbed, decoded: scrubbed }; }; -/** UTF-16 text after a byte-order mark, decoded and encoded as the computer's boot respelling does; undefined otherwise. */ -const utf16Text = (bytes: Buffer) => { - const order = [ - { mark: [0xff, 0xfe], swap: false }, - { mark: [0xfe, 0xff], swap: true }, - ].find(({ mark }) => mark.every((byte, index) => bytes[index] === byte)); - if (order === undefined) return undefined; - const little = Buffer.from(bytes.subarray(2)); - if (order.swap && little.length % 2 === 0) little.swap16(); - try { - return { - // The signature is already removed; a leading U+FEFF here is content. - text: new TextDecoder("utf-16le", { - fatal: true, - ignoreBOM: true, - }).decode(little), - encode: (scrubbed: string) => { - const encoded = Buffer.from(scrubbed, "utf16le"); - if (order.swap) encoded.swap16(); - return Buffer.concat([Buffer.from(order.mark), encoded]); - }, - }; - } catch { - return undefined; - } -}; - /** A seed by its home path and the format the state view read it as. */ interface Seed { readonly home: string; @@ -370,12 +344,19 @@ interface Seed { */ const seedText = (bytes: Buffer, { home, format }: Seed) => { if (bytes.length > MAX_SCRUBBED_BYTES) return undefined; - if (!isTextualFormat(format)) { - const utf16 = utf16Text(bytes); - return utf16 === undefined - ? undefined - : { ...utf16, format: textSyntaxFormat(home, utf16.text) }; + const utf16 = decodeStateUtf16(bytes); + if (utf16 !== undefined) { + return { + text: utf16.text, + format: textSyntaxFormat(home, utf16.text), + encode: (scrubbed: string) => { + const encoded = Buffer.from(scrubbed, "utf16le"); + if (utf16.order === "be") encoded.swap16(); + return Buffer.concat([bytes.subarray(0, 2), encoded]); + }, + }; } + if (!isTextualFormat(format)) return undefined; try { const text = strictUtf8Decoder("preserve").decode(bytes); return { diff --git a/src/domain/desktop/stateFormats.ts b/src/domain/desktop/stateFormats.ts index c36156f667..8f5b127325 100644 --- a/src/domain/desktop/stateFormats.ts +++ b/src/domain/desktop/stateFormats.ts @@ -16,6 +16,7 @@ import { parseXmlDocument, } from "../xmlDocument.js"; import { expatXmlName } from "./expatXmlNames.js"; +import { decodeStateUtf16 } from "./stateTextEncoding.js"; import { decodeStrictUtf8 } from "../strictUtf8.js"; /** Formats the state view reads into JSON. */ @@ -89,8 +90,11 @@ const isKeyfile = (text: string): boolean => { return group; }; -/** Decode bytes as UTF-8 text, or undefined for binary content. */ +/** Decode UTF-8 and BOM-marked UTF-16, or return undefined for binary content. */ export const decodeStateText = (bytes: Uint8Array): string | undefined => { + const utf16 = decodeStateUtf16(bytes); + if (utf16 !== undefined) + return utf16.text.includes("\0") ? undefined : utf16.text; if (bytes.subarray(0, 8192).includes(0)) return undefined; try { return decodeStrictUtf8(bytes, "strip-signature"); @@ -566,7 +570,7 @@ export const xmlView = ( /** Formats read from a file's bytes alone, without a tool or worker. */ export type TextualStateFormat = "text" | "json" | "keyfile" | "xml"; -/** Whether a format is one of the text formats, read as UTF-8. */ +/** Whether a format is one of the text formats. */ export const isTextualFormat = ( format: string | null, ): format is TextualStateFormat => diff --git a/src/domain/desktop/stateTextEncoding.ts b/src/domain/desktop/stateTextEncoding.ts new file mode 100644 index 0000000000..77b3cac29b --- /dev/null +++ b/src/domain/desktop/stateTextEncoding.ts @@ -0,0 +1,20 @@ +const UTF16 = { + le: new TextDecoder("utf-16le", { fatal: true, ignoreBOM: true }), + be: new TextDecoder("utf-16be", { fatal: true, ignoreBOM: true }), +}; + +/** Decode UTF-16 only with a byte-order mark. Strip exactly that first mark. */ +export const decodeStateUtf16 = (bytes: Uint8Array) => { + const order = + bytes[0] === 0xff && bytes[1] === 0xfe + ? "le" + : bytes[0] === 0xfe && bytes[1] === 0xff + ? "be" + : undefined; + if (order === undefined) return undefined; + try { + return { order, text: UTF16[order].decode(bytes.subarray(2)) }; + } catch { + return undefined; + } +}; diff --git a/tests/boundary/providers/desktopExportPython.test.ts b/tests/boundary/providers/desktopExportPython.test.ts index 70cbf4ee27..941d841326 100644 --- a/tests/boundary/providers/desktopExportPython.test.ts +++ b/tests/boundary/providers/desktopExportPython.test.ts @@ -214,31 +214,29 @@ it("seeds each file the bundle lists in SEED_MODES with that mode, as a lock who "[s]\n", ); await writeFile(join(root, "seed", "run.sh"), "#!/bin/sh\n"); + const work = "work.txt"; + const home = join(root, "home"); + await writeFile(join(root, "seed", work), utf16("/home/ada/project\n", "be")); const imports = PYTHON_IMPORTS.split("\n") .filter((line) => !line.includes("agentenv_protocol")) .join("\n"); const program = `${imports} SEED_HOME = "/home/ada" -TEXT_SEEDS = frozenset() +TEXT_SEEDS = frozenset(["work.txt"]) SEED_MODES = {".config/notes/secret.ini": 0o600, "run.sh": 0o755} HERE = Path(sys.argv[1]) ${PYTHON_HELPERS} _seed_home(Path(sys.argv[2])) `; - execFileSync("python3", [ - "-I", - "-B", - "-c", - program, - root, - join(root, "home"), - ]); + execFileSync("python3", ["-I", "-B", "-c", program, root, home]); const mode = async (path: string) => (await stat(join(root, "home", path))).mode & 0o777; expect(await mode(".config/notes/secret.ini")).toBe(0o600); expect(await mode("run.sh")).toBe(0o755); + const restored = await readFile(join(home, work)); + expect(restored.equals(utf16(`${home}/project\n`, "be"))).toBe(true); }); it.each([true, false])( diff --git a/tests/boundary/providers/desktopGzipPython.test.ts b/tests/boundary/providers/desktopGzipPython.test.ts index 5da8c9b932..59518c93ad 100644 --- a/tests/boundary/providers/desktopGzipPython.test.ts +++ b/tests/boundary/providers/desktopGzipPython.test.ts @@ -40,6 +40,64 @@ const sqlite = Buffer.concat([ Buffer.alloc(84), ]); +it("reads and checks BOM-signed UTF-16 project files in either byte order", async () => { + const name = "settings.xml"; + const text = ` +${" ".repeat(4200)}`; + const little = Buffer.from(`\ufeff${text}`, "utf16le"); + const big = Buffer.from(little).swap16(); + const files = [ + { name, bytes: little }, + { name, bytes: big }, + { name, bytes: gzipSync(big, { level: 0 }) }, + { + name: "project.zip", + bytes: zipArchive([{ name, data: little, method: 0 }]), + }, + { name, bytes: Buffer.from([0xff, 0xfe, 0x00, 0xd8]) }, + { name, bytes: Buffer.from(text, "utf16le") }, + { name, bytes: Buffer.from("\ufeffa\0b", "utf16le") }, + { + name: "settings.json", + bytes: Buffer.from('\ufeff{"zoom":1.5}', "utf16le"), + }, + ]; + await expectSameFileViews(files); + const views = await Promise.all(files.map(typescriptFileView)); + expect(views.map(({ format }) => format)).toEqual([ + "xml", + "xml", + "xml", + "zip", + "binary", + "binary", + "binary", + "json", + ]); + expect(views[3]?.view).toMatchObject({ + entries: [{ name, format: "xml", view: views[0]?.view }], + }); + expect(views[7]?.view).toEqual({ zoom: 1.5 }); + const outcomes = await expectSameCheckResults( + name, + [ + { + id: "project-title", + kind: "value_equals", + file: name, + pointer: "/root/attributes/title", + value: "Café 🎨", + }, + ], + files.slice(0, 3), + ); + expect(outcomes.map(([outcome]) => outcome?.passed)).toEqual([ + true, + true, + true, + ]); +}); + describe("gzip files in the exported module", () => { it("reads a compressed drawing as the view of the drawing inside, and grades it alike", async () => { const files = await posters();