mirror of
https://github.com/morluto/rea.git
synced 2026-11-19 13:12:26 +00:00
fix(desktop): recognize empty ZIP state views
This commit is contained in:
1 parent
2ac8578827
commit
263b25bd34
9 files changed
+193
-26
No files matched your search
@@ -64,16 +64,16 @@ Views hold the captured bytes as they are, the capture host's name and user
|
||||
included; the export removes those from the bundle. A version records
|
||||
whether the file was present, the SHA-256 of its retained bytes, and a view:
|
||||
|
||||
| Format | Detected by | View |
|
||||
| --------- | ---------------------------------------------------------------------- | -------------------------------------------------------------------------------------- |
|
||||
| `sqlite` | The `SQLite format 3` header | Every ordinary table's columns, primary key and up to 500 rows |
|
||||
| `dconf` | The `GVariant` header | Groups whose path names the application, plus the names of the other groups |
|
||||
| `zip` | A zip local file header, `PK\x03\x04`, whatever the name | Its file entries in archive order, each with its own format and view |
|
||||
| `json` | A `.json` name or a leading `{` or `[` that parses | The parsed value, from files up to 4 MiB nested up to 500 levels |
|
||||
| `xml` | An XML name or declaration that parses | An element tree (`name`, `attributes`, `children`); text kept exactly, DTDs never read |
|
||||
| `keyfile` | Only `[group]`, `key=value` and comment lines, with at least one group | Groups of `key` → string value, from files up to 4 MiB |
|
||||
| `text` | Other UTF-8 text | Lines, up to 5,000 |
|
||||
| `binary` | NUL bytes or invalid UTF-8 | None |
|
||||
| Format | Detected by | View |
|
||||
| --------- | -------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- |
|
||||
| `sqlite` | The `SQLite format 3` header | Every ordinary table's columns, primary key and up to 500 rows |
|
||||
| `dconf` | The `GVariant` header | Groups whose path names the application, plus the names of the other groups |
|
||||
| `zip` | A zip local file header or an empty classic/Zip64 archive signature, whatever the name | Its file entries in archive order, each with its own format and view |
|
||||
| `json` | A `.json` name or a leading `{` or `[` that parses | The parsed value, from files up to 4 MiB nested up to 500 levels |
|
||||
| `xml` | An XML name or declaration that parses | An element tree (`name`, `attributes`, `children`); text kept exactly, DTDs never read |
|
||||
| `keyfile` | Only `[group]`, `key=value` and comment lines, with at least one group | Groups of `key` → string value, from files up to 4 MiB |
|
||||
| `text` | Other UTF-8 text | Lines, up to 5,000 |
|
||||
| `binary` | NUL bytes or invalid UTF-8 | None |
|
||||
|
||||
A JSON value holding a number JavaScript would round, such as an integer above
|
||||
2^53, is not read rather than shown rounded.
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# Plan
|
||||
|
||||
1. Format and reader: `zip` detected by its local file header; the text, JSON,
|
||||
1. Format and reader: `zip` detected by its local file header or an empty
|
||||
classic/Zip64 archive signature; the text, JSON,
|
||||
key file and XML readers shared as `textualView`, so a file and an entry
|
||||
read alike; a domain module for the entry limits, the entry view and the
|
||||
archive view, and an application reader that walks the verified bytes with
|
||||
|
||||
@@ -13,8 +13,9 @@ evaluators, REA's and the exported verifier's, agree.
|
||||
|
||||
## Requirements
|
||||
|
||||
- Format: a file starting with a zip local file header (`PK\x03\x04`) is
|
||||
`zip`, whatever its name. Its view is
|
||||
- Format: a file starting with a zip local file header (`PK\x03\x04`), or
|
||||
a classic/Zip64 empty-archive signature with at least 22 bytes, is `zip`,
|
||||
whatever its name. A signature is not proof of directory validity. Its view is
|
||||
`{entries, entry_count, truncated}`: the file entries in archive order,
|
||||
directories left out, each `{name, size, sha256, format, view, view_error}`.
|
||||
- Reading: from the retained bytes already checked against their digest, in
|
||||
|
||||
@@ -33,8 +33,8 @@ generation reads the kinds of change.
|
||||
with exact numbers; XML through the shared strict parser, a DOCTYPE read
|
||||
only when it has no internal subset and its DTD never read
|
||||
([XML text checks](../xml-text-checks/spec.md)); key files up to 4 MiB
|
||||
whose every line parses; zip archives by
|
||||
their local file header, as their entries with each entry's own format and
|
||||
whose every line parses; zip archives by their local file header or a
|
||||
classic/Zip64 empty-archive signature, as entries with each entry's own format and
|
||||
view ([document views](../document-views/spec.md)); text; binary.
|
||||
- Changes between versions are created, deleted, or lines, values, keys or
|
||||
rows with added, removed and changed lists (100 each, with counts); content
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { PYTHON_JSON_READER } from "./appExportPythonJson.js";
|
||||
import { PYTHON_XML_READER } from "./appExportPythonXml.js";
|
||||
import { PYTHON_ZIP_HELPERS } from "./appExportPythonZip.js";
|
||||
import { ZIP_SIGNATURES } from "../zipPackageFormat.js";
|
||||
import { ZIP_EOCD_BYTES } from "../zipEndOfCentralDirectory.js";
|
||||
|
||||
/** The generated module's imports. */
|
||||
/** The class method that holds readiness until the app's own window shows, not a splash screen with its class. */
|
||||
@@ -46,6 +48,8 @@ LINE_BREAK = re.compile(r"\r?\n")
|
||||
SQLITE_HEADER = b"SQLite format 3\x00"
|
||||
GVDB_HEADER = b"GVariant"
|
||||
ZIP_HEADER = b"PK\x03\x04"
|
||||
ZIP_SIGNATURES = (${ZIP_SIGNATURES.map((signature) => `bytes(${JSON.stringify(signature)})`).join(", ")})
|
||||
ZIP_EOCD_BYTES = ${String(ZIP_EOCD_BYTES)}
|
||||
XML_SPACE = "xml:space"
|
||||
MAX_VIEW_BYTES = 64 * 1024 * 1024
|
||||
MAX_SAFE_INTEGER = 2**53 - 1
|
||||
@@ -272,6 +276,11 @@ def _text_view(name: str, text: str, size: int) -> dict:
|
||||
return _text(text)
|
||||
|
||||
|
||||
def _is_zip(data: bytes) -> bool:
|
||||
"""Known ZIP storage signatures, including empty archives; the reader owns directory validity."""
|
||||
return data.startswith(ZIP_HEADER) or (len(data) >= ZIP_EOCD_BYTES and data.startswith(ZIP_SIGNATURES))
|
||||
|
||||
|
||||
def _view(path: Path) -> dict:
|
||||
"""A typed JSON view of one file in the shapes of REA's state view; truncated says the view is partial."""
|
||||
too_large = {"format": None, "view": None, "error": "larger than 64 MiB, the most the view reads"}
|
||||
@@ -283,7 +292,7 @@ def _view(path: Path) -> dict:
|
||||
data += file.read(MAX_VIEW_BYTES + 1 - len(data)) if whole else b""
|
||||
if data.startswith(SQLITE_HEADER):
|
||||
return _sqlite(path)
|
||||
if data.startswith(ZIP_HEADER):
|
||||
if _is_zip(data):
|
||||
return _zip(path)
|
||||
if len(data) > MAX_VIEW_BYTES:
|
||||
return too_large
|
||||
|
||||
@@ -14,8 +14,7 @@ ZIP_CENTRAL = b"PK\x01\x02"
|
||||
ZIP64_LOCATOR = b"PK\x06\x07"
|
||||
ZIP64_END = b"PK\x06\x06"
|
||||
#: Formats an archive entry may have whose reader needs a tool, a worker or another archive opened.
|
||||
NOT_READ_INSIDE = ((SQLITE_HEADER, "sqlite", "a SQLite database"), (GVDB_HEADER, "dconf", "a dconf database"),
|
||||
(ZIP_HEADER, "zip", "an archive"))
|
||||
NOT_READ_INSIDE = ((SQLITE_HEADER, "sqlite", "a SQLite database"), (GVDB_HEADER, "dconf", "a dconf database"))
|
||||
|
||||
|
||||
def _entry_view(name: str, data: bytes) -> dict:
|
||||
@@ -24,6 +23,8 @@ def _entry_view(name: str, data: bytes) -> dict:
|
||||
for header, form, inside in NOT_READ_INSIDE:
|
||||
if data.startswith(header):
|
||||
return {"format": form, "view": None, "error": f"{inside} inside an archive is not read"}
|
||||
if _is_zip(data):
|
||||
return {"format": "zip", "view": None, "error": "an archive inside an archive is not read"}
|
||||
try:
|
||||
text = data.decode("utf-8-sig") if b"\x00" not in data[:8192] else None
|
||||
except UnicodeDecodeError:
|
||||
|
||||
@@ -4,6 +4,8 @@ import { isExactJsonNumber } from "../exactJsonNumber.js";
|
||||
import { MAX_JSON_DEPTH, type JsonValue } from "../jsonValue.js";
|
||||
import { err, ok, type Result } from "../result.js";
|
||||
import { safeParseJson } from "../safeJson.js";
|
||||
import { hasZipSignature } from "../zipPackageFormat.js";
|
||||
import { ZIP_EOCD_BYTES } from "../zipEndOfCentralDirectory.js";
|
||||
import {
|
||||
declaredDocumentType,
|
||||
isWellFormedXml,
|
||||
@@ -246,7 +248,11 @@ export const detectStateFormat = (
|
||||
const header = new TextDecoder("latin1").decode(bytes.subarray(0, 16));
|
||||
if (header === SQLITE_HEADER) return "sqlite";
|
||||
if (header.startsWith(GVDB_HEADER)) return "dconf";
|
||||
if (header.startsWith(ZIP_HEADER)) return "zip";
|
||||
if (
|
||||
header.startsWith(ZIP_HEADER) ||
|
||||
(bytes.length >= ZIP_EOCD_BYTES && hasZipSignature(bytes))
|
||||
)
|
||||
return "zip";
|
||||
const text = decodeStateText(bytes);
|
||||
if (text === undefined) return "binary";
|
||||
const trimmed = text.trimStart();
|
||||
|
||||
@@ -23,15 +23,20 @@ export const isZipPackageFormat = (
|
||||
): format is ZipPackageFormat =>
|
||||
ZIP_PACKAGE_FORMATS.some((candidate) => candidate === format);
|
||||
|
||||
/** Recognize local-header, classic/Zip64 empty-archive, or split ZIP signatures, not validity. */
|
||||
/** Local-header, classic/Zip64 empty-archive, and split ZIP signatures. */
|
||||
export const ZIP_SIGNATURES = [
|
||||
[0x50, 0x4b, 0x03, 0x04],
|
||||
[0x50, 0x4b, 0x05, 0x06],
|
||||
[0x50, 0x4b, 0x06, 0x06],
|
||||
[0x50, 0x4b, 0x07, 0x08],
|
||||
] as const;
|
||||
|
||||
/** Recognize a ZIP signature, not directory validity or supported storage. */
|
||||
export const hasZipSignature = (bytes: Uint8Array): boolean =>
|
||||
bytes.length >= 4 &&
|
||||
bytes[0] === 0x50 &&
|
||||
bytes[1] === 0x4b &&
|
||||
((bytes[2] === 0x03 && bytes[3] === 0x04) ||
|
||||
(bytes[2] === 0x05 && bytes[3] === 0x06) ||
|
||||
(bytes[2] === 0x06 && bytes[3] === 0x06) ||
|
||||
(bytes[2] === 0x07 && bytes[3] === 0x08));
|
||||
ZIP_SIGNATURES.some((signature) =>
|
||||
signature.every((byte, index) => bytes[index] === byte),
|
||||
);
|
||||
|
||||
const CENTRAL_DIRECTORY_HEADER = 0x02014b50;
|
||||
const LOCAL_FILE_HEADER = 0x04034b50;
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { readZipView } from "../../../src/application/desktop/DesktopZipView.js";
|
||||
import {
|
||||
PYTHON_HELPERS,
|
||||
PYTHON_IMPORTS,
|
||||
} from "../../../src/domain/desktop/appExportPython.js";
|
||||
import { PYTHON_TASK_EVALUATOR } from "../../../src/domain/desktop/appTaskPython.js";
|
||||
import { evaluateTaskChecks } from "../../../src/domain/desktop/appTaskChecks.js";
|
||||
import { detectStateFormat } from "../../../src/domain/desktop/stateFormats.js";
|
||||
import {
|
||||
withZip64End,
|
||||
zipArchive,
|
||||
} from "../../../src/domain/desktop/zipArchive.fixture.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
|
||||
const empty = zipArchive([]);
|
||||
const commented = Buffer.concat([empty, Buffer.from("archive comment")]);
|
||||
commented.writeUInt16LE(commented.length - 22, 20);
|
||||
const archives = [
|
||||
["classic", empty],
|
||||
["commented", commented],
|
||||
["Zip64 sentinels", withZip64End(empty)],
|
||||
["Zip64 ordinary counts", withZip64End(empty, false)],
|
||||
] as const;
|
||||
|
||||
const pythonViews = async (files: readonly Buffer[]) => {
|
||||
const root = await createTestTempDirectory("desktop-empty-zip-");
|
||||
const paths = await Promise.all(
|
||||
files.map(async (bytes, index) => {
|
||||
const path = join(root, `${String(index)}.dat`);
|
||||
await writeFile(path, bytes);
|
||||
return path;
|
||||
}),
|
||||
);
|
||||
const imports = PYTHON_IMPORTS.split("\n")
|
||||
.filter((line) => !line.includes("agentenv_protocol"))
|
||||
.join("\n");
|
||||
const script = join(root, "read.py");
|
||||
await writeFile(
|
||||
script,
|
||||
`${imports}${PYTHON_HELPERS}${PYTHON_TASK_EVALUATOR}
|
||||
check = {"id": "empty", "kind": "value_equals", "file": "archive.zip", "pointer": "/entry_count", "value": 0}
|
||||
for path in json.load(sys.stdin):
|
||||
view = _view(Path(path))
|
||||
print(json.dumps({"file": view, "passed": evaluate([check], {"archive.zip": view})[0]["result"]}))
|
||||
`,
|
||||
);
|
||||
return execFileSync(
|
||||
process.platform === "win32" ? "python" : "python3",
|
||||
["-I", "-B", "-X", "utf8", script],
|
||||
{ input: JSON.stringify(paths), encoding: "utf8" },
|
||||
)
|
||||
.trim()
|
||||
.split(/\r?\n/u)
|
||||
.map(
|
||||
(line) =>
|
||||
JSON.parse(line) as {
|
||||
file: { format: string; view: unknown; error?: string };
|
||||
passed: boolean;
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
it.each(archives)(
|
||||
"reads an empty %s archive without relying on its name",
|
||||
async (_name, bytes) => {
|
||||
const expected = { entries: [], entry_count: 0, truncated: false };
|
||||
expect(detectStateFormat("document.dat", bytes)).toBe("zip");
|
||||
expect(await readZipView(bytes)).toEqual({ ok: true, value: expected });
|
||||
expect((await pythonViews([bytes]))[0]).toEqual({
|
||||
file: { format: "zip", view: expected, truncated: false },
|
||||
passed: true,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps empty nested archives viewless instead of recursing or calling them binary", async () => {
|
||||
const outer = zipArchive(
|
||||
archives.map(([name, data]) => ({ name: `${name}.zip`, data })),
|
||||
);
|
||||
const native = await readZipView(outer);
|
||||
if (!native.ok) throw new Error(native.error);
|
||||
const python = (await pythonViews([outer]))[0]?.file;
|
||||
expect(python?.format).toBe("zip");
|
||||
expect(python?.view).toEqual(native.value);
|
||||
expect(native.value).toMatchObject({
|
||||
entries: archives.map(() => ({
|
||||
format: "zip",
|
||||
view: null,
|
||||
view_error: "an archive inside an archive is not read",
|
||||
})),
|
||||
entry_count: archives.length,
|
||||
truncated: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("grades removing every entry as a change from a nonempty archive", async () => {
|
||||
const files = [zipArchive([{ name: "old.txt", data: "old" }]), empty];
|
||||
const check = {
|
||||
id: "empty",
|
||||
kind: "value_equals" as const,
|
||||
file: "archive.zip",
|
||||
pointer: "/entry_count",
|
||||
value: 0,
|
||||
};
|
||||
const verdicts = await Promise.all(
|
||||
files.map(async (bytes) => {
|
||||
const read = await readZipView(bytes);
|
||||
if (!read.ok) throw new Error(read.error);
|
||||
return evaluateTaskChecks(
|
||||
[check],
|
||||
new Map([
|
||||
[
|
||||
check.file,
|
||||
{ format: detectStateFormat(check.file, bytes), view: read.value },
|
||||
],
|
||||
]),
|
||||
)[0]?.passed;
|
||||
}),
|
||||
);
|
||||
expect(verdicts).toEqual([false, true]);
|
||||
expect((await pythonViews(files)).map((row) => row.passed)).toEqual(verdicts);
|
||||
});
|
||||
|
||||
it("does not turn incomplete or damaged end records into empty views", async () => {
|
||||
const incomplete = empty.subarray(0, 6);
|
||||
expect(detectStateFormat("archive.zip", incomplete)).toBe("binary");
|
||||
const damaged = Buffer.from(empty);
|
||||
damaged.writeUInt16LE(1, 20);
|
||||
const zip64 = withZip64End(empty).subarray(0, 56);
|
||||
for (const bytes of [damaged, zip64]) {
|
||||
expect(detectStateFormat("archive.dat", bytes)).toBe("zip");
|
||||
expect((await readZipView(bytes)).ok).toBe(false);
|
||||
const python = (await pythonViews([bytes]))[0];
|
||||
expect(python?.file).toMatchObject({ format: "zip", view: null });
|
||||
expect(python?.file.error).toBeTypeOf("string");
|
||||
expect(python?.passed).toBe(false);
|
||||
}
|
||||
});
|
||||
Reference in new issue
Block a user