mirror of
https://github.com/morluto/rea.git
synced 2026-11-18 20:53:00 +00:00
fix(desktop): read BOM-marked UTF-16 in state and task views (#12052)
Co-authored-by: ShuaixinHuang <197200901+ShuaixinHuang@users.noreply.github.com>
This commit is contained in:
1 parent
1868d04ef9
commit
c47442a2eb
8 files changed
+134
-60
No files matched your search
@@ -146,8 +146,12 @@ whether the file was present, the SHA-256 of its retained bytes, and a view:
|
||||
| `json` | A `.json` name or a leading `{` or `[` that parses | The parsed value, from files up to 4 MiB nested up to 500 levels |
|
||||
| `xml` | An XML name, a declaration or a leading element that parses | An element tree (`name`, `attributes`, `children`); text kept exactly, DTDs never read |
|
||||
| `keyfile` | Only `[group]`, `key=value` and comment lines, with at least one group | Groups of `key` → string value, from files up to 4 MiB |
|
||||
| `text` | Other UTF-8 text | Lines, up to 5,000 |
|
||||
| `binary` | NUL bytes or invalid UTF-8 | None |
|
||||
| `text` | Other decoded text | Lines, up to 5,000 |
|
||||
| `binary` | Content that cannot be read as text | None |
|
||||
|
||||
Text views decode UTF-8 or strictly valid UTF-16 with a leading byte-order
|
||||
mark. UTF-16 containing a NUL character stays binary. These rules also apply
|
||||
to documents inside gzip and ZIP files and to the exported task verifier.
|
||||
|
||||
A gzip file is read as the document inside it; see
|
||||
[compressed documents](#compressed-documents).
|
||||
|
||||
@@ -18,7 +18,7 @@ type StatePath = DesktopApplicationState["paths"][number];
|
||||
|
||||
/**
|
||||
* The owned files present at the seed with their digests, and those read as
|
||||
* uncompressed text, which boot respells as UTF-8, but for those `isLeftOut`
|
||||
* uncompressed text, which boot respells in its original encoding, excluding `isLeftOut`
|
||||
* names: lock files the app holds while it runs, since a copied one makes it
|
||||
* find its profile or document in use, and its logs, which record facts about
|
||||
* the capture host.
|
||||
|
||||
@@ -99,16 +99,20 @@ def _respelled(text: str, home: Path) -> str:
|
||||
return re.sub(re.escape(SEED_HOME) + r"(?![\w.@+~-])", lambda _: str(home), text)
|
||||
|
||||
|
||||
UTF16_CODECS = {b"\xff\xfe": "utf-16-le", b"\xfe\xff": "utf-16-be"}
|
||||
|
||||
|
||||
def _respelled_utf16(data: bytes, home: Path) -> bytes:
|
||||
"""A UTF-16 text file with a byte-order mark respelled like a text seed, its mark kept; any other bytes as given."""
|
||||
for mark, codec in ((b"\xff\xfe", "utf-16-le"), (b"\xfe\xff", "utf-16-be")):
|
||||
if data.startswith(mark):
|
||||
try:
|
||||
text = data[len(mark):].decode(codec)
|
||||
except UnicodeDecodeError:
|
||||
return data
|
||||
return mark + _respelled(text, home).encode(codec) if SEED_HOME in text else data
|
||||
return data
|
||||
mark = data[:2]
|
||||
codec = UTF16_CODECS.get(mark)
|
||||
if codec is None:
|
||||
return data
|
||||
try:
|
||||
text = data[2:].decode(codec)
|
||||
except UnicodeDecodeError:
|
||||
return data
|
||||
return mark + _respelled(text, home).encode(codec) if SEED_HOME in text else data
|
||||
|
||||
|
||||
#: What marks a window as not yet the app's own: skipping the taskbar, as splash screens and dialogs do, or a splash type.
|
||||
@@ -141,10 +145,10 @@ def _seed_home(home: Path) -> None:
|
||||
target = home / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = source.read_bytes()
|
||||
if relative in TEXT_SEEDS:
|
||||
data = _respelled(data.decode("utf-8"), home).encode("utf-8")
|
||||
else:
|
||||
if data[:2] in UTF16_CODECS:
|
||||
data = _respelled_utf16(data, home)
|
||||
elif relative in TEXT_SEEDS:
|
||||
data = _respelled(data.decode("utf-8"), home).encode("utf-8")
|
||||
target.write_bytes(data)
|
||||
mode = SEED_MODES.get(relative)
|
||||
if mode is not None:
|
||||
@@ -400,10 +404,14 @@ def _text_view(name: str, text: str, size: int) -> dict:
|
||||
def _bytes_view(name: str, data: bytes) -> dict:
|
||||
"""The view of a document that is no database or archive: its text's, or none when it is binary."""
|
||||
try:
|
||||
text = data.decode("utf-8-sig") if b"\x00" not in data[:8192] else None
|
||||
codec = UTF16_CODECS.get(bytes(data[:2]))
|
||||
if codec is not None:
|
||||
text = data[2:].decode(codec)
|
||||
else:
|
||||
text = data.decode("utf-8-sig") if b"\x00" not in data[:8192] else None
|
||||
except UnicodeDecodeError:
|
||||
text = None
|
||||
if text is None:
|
||||
if text is None or (codec is not None and "\x00" in text):
|
||||
return {"format": "binary", "view": None, "truncated": False}
|
||||
return _text_view(name, text, len(data))
|
||||
|
||||
@@ -417,7 +425,8 @@ def _view(path: Path) -> dict:
|
||||
data = file.read(min(8192, MAX_VIEW_BYTES + 1))
|
||||
if len(data) > MAX_VIEW_BYTES:
|
||||
return too_large
|
||||
whole = data.startswith(GZIP_HEADER) or (not data.startswith(SQLITE_HEADER) and b"\x00" not in data)
|
||||
whole = (data.startswith(GZIP_HEADER) or data[:2] in UTF16_CODECS
|
||||
or (not data.startswith(SQLITE_HEADER) and b"\x00" not in data))
|
||||
remaining = MAX_VIEW_BYTES + 1 - len(data)
|
||||
if whole and remaining > 0:
|
||||
data += file.read(remaining)
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
isTextualFormat,
|
||||
textSyntaxFormat,
|
||||
} from "./stateFormats.js";
|
||||
import { decodeStateUtf16 } from "./stateTextEncoding.js";
|
||||
import { strictUtf8Decoder } from "../strictUtf8.js";
|
||||
import { isRecord } from "../../kernel/isRecord.js";
|
||||
|
||||
@@ -330,33 +331,6 @@ const plainScrub = (text: string, identity: HostIdentity): TextScrub => {
|
||||
return { text: scrubbed, decoded: scrubbed };
|
||||
};
|
||||
|
||||
/** UTF-16 text after a byte-order mark, decoded and encoded as the computer's boot respelling does; undefined otherwise. */
|
||||
const utf16Text = (bytes: Buffer) => {
|
||||
const order = [
|
||||
{ mark: [0xff, 0xfe], swap: false },
|
||||
{ mark: [0xfe, 0xff], swap: true },
|
||||
].find(({ mark }) => mark.every((byte, index) => bytes[index] === byte));
|
||||
if (order === undefined) return undefined;
|
||||
const little = Buffer.from(bytes.subarray(2));
|
||||
if (order.swap && little.length % 2 === 0) little.swap16();
|
||||
try {
|
||||
return {
|
||||
// The signature is already removed; a leading U+FEFF here is content.
|
||||
text: new TextDecoder("utf-16le", {
|
||||
fatal: true,
|
||||
ignoreBOM: true,
|
||||
}).decode(little),
|
||||
encode: (scrubbed: string) => {
|
||||
const encoded = Buffer.from(scrubbed, "utf16le");
|
||||
if (order.swap) encoded.swap16();
|
||||
return Buffer.concat([Buffer.from(order.mark), encoded]);
|
||||
},
|
||||
};
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
|
||||
/** A seed by its home path and the format the state view read it as. */
|
||||
interface Seed {
|
||||
readonly home: string;
|
||||
@@ -370,12 +344,19 @@ interface Seed {
|
||||
*/
|
||||
const seedText = (bytes: Buffer, { home, format }: Seed) => {
|
||||
if (bytes.length > MAX_SCRUBBED_BYTES) return undefined;
|
||||
if (!isTextualFormat(format)) {
|
||||
const utf16 = utf16Text(bytes);
|
||||
return utf16 === undefined
|
||||
? undefined
|
||||
: { ...utf16, format: textSyntaxFormat(home, utf16.text) };
|
||||
const utf16 = decodeStateUtf16(bytes);
|
||||
if (utf16 !== undefined) {
|
||||
return {
|
||||
text: utf16.text,
|
||||
format: textSyntaxFormat(home, utf16.text),
|
||||
encode: (scrubbed: string) => {
|
||||
const encoded = Buffer.from(scrubbed, "utf16le");
|
||||
if (utf16.order === "be") encoded.swap16();
|
||||
return Buffer.concat([bytes.subarray(0, 2), encoded]);
|
||||
},
|
||||
};
|
||||
}
|
||||
if (!isTextualFormat(format)) return undefined;
|
||||
try {
|
||||
const text = strictUtf8Decoder("preserve").decode(bytes);
|
||||
return {
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
parseXmlDocument,
|
||||
} from "../xmlDocument.js";
|
||||
import { expatXmlName } from "./expatXmlNames.js";
|
||||
import { decodeStateUtf16 } from "./stateTextEncoding.js";
|
||||
import { decodeStrictUtf8 } from "../strictUtf8.js";
|
||||
|
||||
/** Formats the state view reads into JSON. */
|
||||
@@ -89,8 +90,11 @@ const isKeyfile = (text: string): boolean => {
|
||||
return group;
|
||||
};
|
||||
|
||||
/** Decode bytes as UTF-8 text, or undefined for binary content. */
|
||||
/** Decode UTF-8 and BOM-marked UTF-16, or return undefined for binary content. */
|
||||
export const decodeStateText = (bytes: Uint8Array): string | undefined => {
|
||||
const utf16 = decodeStateUtf16(bytes);
|
||||
if (utf16 !== undefined)
|
||||
return utf16.text.includes("\0") ? undefined : utf16.text;
|
||||
if (bytes.subarray(0, 8192).includes(0)) return undefined;
|
||||
try {
|
||||
return decodeStrictUtf8(bytes, "strip-signature");
|
||||
@@ -566,7 +570,7 @@ export const xmlView = (
|
||||
/** Formats read from a file's bytes alone, without a tool or worker. */
|
||||
export type TextualStateFormat = "text" | "json" | "keyfile" | "xml";
|
||||
|
||||
/** Whether a format is one of the text formats, read as UTF-8. */
|
||||
/** Whether a format is one of the text formats. */
|
||||
export const isTextualFormat = (
|
||||
format: string | null,
|
||||
): format is TextualStateFormat =>
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
const UTF16 = {
|
||||
le: new TextDecoder("utf-16le", { fatal: true, ignoreBOM: true }),
|
||||
be: new TextDecoder("utf-16be", { fatal: true, ignoreBOM: true }),
|
||||
};
|
||||
|
||||
/** Decode UTF-16 only with a byte-order mark. Strip exactly that first mark. */
|
||||
export const decodeStateUtf16 = (bytes: Uint8Array) => {
|
||||
const order =
|
||||
bytes[0] === 0xff && bytes[1] === 0xfe
|
||||
? "le"
|
||||
: bytes[0] === 0xfe && bytes[1] === 0xff
|
||||
? "be"
|
||||
: undefined;
|
||||
if (order === undefined) return undefined;
|
||||
try {
|
||||
return { order, text: UTF16[order].decode(bytes.subarray(2)) };
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
@@ -214,31 +214,29 @@ it("seeds each file the bundle lists in SEED_MODES with that mode, as a lock who
|
||||
"[s]\n",
|
||||
);
|
||||
await writeFile(join(root, "seed", "run.sh"), "#!/bin/sh\n");
|
||||
const work = "work.txt";
|
||||
const home = join(root, "home");
|
||||
await writeFile(join(root, "seed", work), utf16("/home/ada/project\n", "be"));
|
||||
const imports = PYTHON_IMPORTS.split("\n")
|
||||
.filter((line) => !line.includes("agentenv_protocol"))
|
||||
.join("\n");
|
||||
const program = `${imports}
|
||||
SEED_HOME = "/home/ada"
|
||||
TEXT_SEEDS = frozenset()
|
||||
TEXT_SEEDS = frozenset(["work.txt"])
|
||||
SEED_MODES = {".config/notes/secret.ini": 0o600, "run.sh": 0o755}
|
||||
HERE = Path(sys.argv[1])
|
||||
${PYTHON_HELPERS}
|
||||
_seed_home(Path(sys.argv[2]))
|
||||
`;
|
||||
|
||||
execFileSync("python3", [
|
||||
"-I",
|
||||
"-B",
|
||||
"-c",
|
||||
program,
|
||||
root,
|
||||
join(root, "home"),
|
||||
]);
|
||||
execFileSync("python3", ["-I", "-B", "-c", program, root, home]);
|
||||
|
||||
const mode = async (path: string) =>
|
||||
(await stat(join(root, "home", path))).mode & 0o777;
|
||||
expect(await mode(".config/notes/secret.ini")).toBe(0o600);
|
||||
expect(await mode("run.sh")).toBe(0o755);
|
||||
const restored = await readFile(join(home, work));
|
||||
expect(restored.equals(utf16(`${home}/project\n`, "be"))).toBe(true);
|
||||
});
|
||||
|
||||
it.each([true, false])(
|
||||
|
||||
@@ -40,6 +40,64 @@ const sqlite = Buffer.concat([
|
||||
Buffer.alloc(84),
|
||||
]);
|
||||
|
||||
it("reads and checks BOM-signed UTF-16 project files in either byte order", async () => {
|
||||
const name = "settings.xml";
|
||||
const text = `<?xml version="1.0" encoding="UTF-16"?>
|
||||
<project title="Café 🎨">${" ".repeat(4200)}</project>`;
|
||||
const little = Buffer.from(`\ufeff${text}`, "utf16le");
|
||||
const big = Buffer.from(little).swap16();
|
||||
const files = [
|
||||
{ name, bytes: little },
|
||||
{ name, bytes: big },
|
||||
{ name, bytes: gzipSync(big, { level: 0 }) },
|
||||
{
|
||||
name: "project.zip",
|
||||
bytes: zipArchive([{ name, data: little, method: 0 }]),
|
||||
},
|
||||
{ name, bytes: Buffer.from([0xff, 0xfe, 0x00, 0xd8]) },
|
||||
{ name, bytes: Buffer.from(text, "utf16le") },
|
||||
{ name, bytes: Buffer.from("\ufeffa\0b", "utf16le") },
|
||||
{
|
||||
name: "settings.json",
|
||||
bytes: Buffer.from('\ufeff{"zoom":1.5}', "utf16le"),
|
||||
},
|
||||
];
|
||||
await expectSameFileViews(files);
|
||||
const views = await Promise.all(files.map(typescriptFileView));
|
||||
expect(views.map(({ format }) => format)).toEqual([
|
||||
"xml",
|
||||
"xml",
|
||||
"xml",
|
||||
"zip",
|
||||
"binary",
|
||||
"binary",
|
||||
"binary",
|
||||
"json",
|
||||
]);
|
||||
expect(views[3]?.view).toMatchObject({
|
||||
entries: [{ name, format: "xml", view: views[0]?.view }],
|
||||
});
|
||||
expect(views[7]?.view).toEqual({ zoom: 1.5 });
|
||||
const outcomes = await expectSameCheckResults(
|
||||
name,
|
||||
[
|
||||
{
|
||||
id: "project-title",
|
||||
kind: "value_equals",
|
||||
file: name,
|
||||
pointer: "/root/attributes/title",
|
||||
value: "Café 🎨",
|
||||
},
|
||||
],
|
||||
files.slice(0, 3),
|
||||
);
|
||||
expect(outcomes.map(([outcome]) => outcome?.passed)).toEqual([
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
]);
|
||||
});
|
||||
|
||||
describe("gzip files in the exported module", () => {
|
||||
it("reads a compressed drawing as the view of the drawing inside, and grades it alike", async () => {
|
||||
const files = await posters();
|
||||
|
||||
Reference in new issue
Block a user