fix(desktop): recognize empty ZIP state views

This commit is contained in:
wunianze666-netizen committed 2026-11-08 14:45:11 +08:00
1 parent 2ac8578827
commit 263b25bd34
9 files changed
+193 -26

No files matched your search

+10 -10
View File
@@ -64,16 +64,16 @@ Views hold the captured bytes as they are, the capture host's name and user
included; the export removes those from the bundle. A version records
whether the file was present, the SHA-256 of its retained bytes, and a view:
| Format | Detected by | View |
| --------- | ---------------------------------------------------------------------- | -------------------------------------------------------------------------------------- |
| `sqlite` | The `SQLite format 3` header | Every ordinary table's columns, primary key and up to 500 rows |
| `dconf` | The `GVariant` header | Groups whose path names the application, plus the names of the other groups |
| `zip` | A zip local file header, `PK\x03\x04`, whatever the name | Its file entries in archive order, each with its own format and view |
| `json` | A `.json` name or a leading `{` or `[` that parses | The parsed value, from files up to 4 MiB nested up to 500 levels |
| `xml` | An XML name or declaration that parses | An element tree (`name`, `attributes`, `children`); text kept exactly, DTDs never read |
| `keyfile` | Only `[group]`, `key=value` and comment lines, with at least one group | Groups of `key` → string value, from files up to 4 MiB |
| `text` | Other UTF-8 text | Lines, up to 5,000 |
| `binary` | NUL bytes or invalid UTF-8 | None |
| Format | Detected by | View |
| --------- | -------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- |
| `sqlite` | The `SQLite format 3` header | Every ordinary table's columns, primary key and up to 500 rows |
| `dconf` | The `GVariant` header | Groups whose path names the application, plus the names of the other groups |
| `zip` | A zip local file header or an empty classic/Zip64 archive signature, whatever the name | Its file entries in archive order, each with its own format and view |
| `json` | A `.json` name or a leading `{` or `[` that parses | The parsed value, from files up to 4 MiB nested up to 500 levels |
| `xml` | An XML name or declaration that parses | An element tree (`name`, `attributes`, `children`); text kept exactly, DTDs never read |
| `keyfile` | Only `[group]`, `key=value` and comment lines, with at least one group | Groups of `key` → string value, from files up to 4 MiB |
| `text` | Other UTF-8 text | Lines, up to 5,000 |
| `binary` | NUL bytes or invalid UTF-8 | None |
A JSON value holding a number JavaScript would round, such as an integer above
2^53, is not read rather than shown rounded.
@@ -1,6 +1,7 @@
# Plan
1. Format and reader: `zip` detected by its local file header; the text, JSON,
1. Format and reader: `zip` detected by its local file header or an empty
classic/Zip64 archive signature; the text, JSON,
key file and XML readers shared as `textualView`, so a file and an entry
read alike; a domain module for the entry limits, the entry view and the
archive view, and an application reader that walks the verified bytes with
@@ -13,8 +13,9 @@ evaluators, REA's and the exported verifier's, agree.
## Requirements
- Format: a file starting with a zip local file header (`PK\x03\x04`) is
`zip`, whatever its name. Its view is
- Format: a file starting with a zip local file header (`PK\x03\x04`), or
a classic/Zip64 empty-archive signature with at least 22 bytes, is `zip`,
whatever its name. A signature is not proof of directory validity. Its view is
`{entries, entry_count, truncated}`: the file entries in archive order,
directories left out, each `{name, size, sha256, format, view, view_error}`.
- Reading: from the retained bytes already checked against their digest, in
@@ -33,8 +33,8 @@ generation reads the kinds of change.
with exact numbers; XML through the shared strict parser, a DOCTYPE read
only when it has no internal subset and its DTD never read
([XML text checks](../xml-text-checks/spec.md)); key files up to 4 MiB
whose every line parses; zip archives by
their local file header, as their entries with each entry's own format and
whose every line parses; zip archives by their local file header or a
classic/Zip64 empty-archive signature, as entries with each entry's own format and
view ([document views](../document-views/spec.md)); text; binary.
- Changes between versions are created, deleted, or lines, values, keys or
rows with added, removed and changed lists (100 each, with counts); content
+10 -1
View File
@@ -1,6 +1,8 @@
import { PYTHON_JSON_READER } from "./appExportPythonJson.js";
import { PYTHON_XML_READER } from "./appExportPythonXml.js";
import { PYTHON_ZIP_HELPERS } from "./appExportPythonZip.js";
import { ZIP_SIGNATURES } from "../zipPackageFormat.js";
import { ZIP_EOCD_BYTES } from "../zipEndOfCentralDirectory.js";
/** The generated module's imports. */
/** The class method that holds readiness until the app's own window shows, not a splash screen with its class. */
@@ -46,6 +48,8 @@ LINE_BREAK = re.compile(r"\r?\n")
SQLITE_HEADER = b"SQLite format 3\x00"
GVDB_HEADER = b"GVariant"
ZIP_HEADER = b"PK\x03\x04"
ZIP_SIGNATURES = (${ZIP_SIGNATURES.map((signature) => `bytes(${JSON.stringify(signature)})`).join(", ")})
ZIP_EOCD_BYTES = ${String(ZIP_EOCD_BYTES)}
XML_SPACE = "xml:space"
MAX_VIEW_BYTES = 64 * 1024 * 1024
MAX_SAFE_INTEGER = 2**53 - 1
@@ -272,6 +276,11 @@ def _text_view(name: str, text: str, size: int) -> dict:
return _text(text)
def _is_zip(data: bytes) -> bool:
"""Known ZIP storage signatures, including empty archives; the reader owns directory validity."""
return data.startswith(ZIP_HEADER) or (len(data) >= ZIP_EOCD_BYTES and data.startswith(ZIP_SIGNATURES))
def _view(path: Path) -> dict:
"""A typed JSON view of one file in the shapes of REA's state view; truncated says the view is partial."""
too_large = {"format": None, "view": None, "error": "larger than 64 MiB, the most the view reads"}
@@ -283,7 +292,7 @@ def _view(path: Path) -> dict:
data += file.read(MAX_VIEW_BYTES + 1 - len(data)) if whole else b""
if data.startswith(SQLITE_HEADER):
return _sqlite(path)
if data.startswith(ZIP_HEADER):
if _is_zip(data):
return _zip(path)
if len(data) > MAX_VIEW_BYTES:
return too_large
+3 -2
View File
@@ -14,8 +14,7 @@ ZIP_CENTRAL = b"PK\x01\x02"
ZIP64_LOCATOR = b"PK\x06\x07"
ZIP64_END = b"PK\x06\x06"
#: Formats an archive entry may have whose reader needs a tool, a worker or another archive opened.
NOT_READ_INSIDE = ((SQLITE_HEADER, "sqlite", "a SQLite database"), (GVDB_HEADER, "dconf", "a dconf database"),
(ZIP_HEADER, "zip", "an archive"))
NOT_READ_INSIDE = ((SQLITE_HEADER, "sqlite", "a SQLite database"), (GVDB_HEADER, "dconf", "a dconf database"))
def _entry_view(name: str, data: bytes) -> dict:
@@ -24,6 +23,8 @@ def _entry_view(name: str, data: bytes) -> dict:
for header, form, inside in NOT_READ_INSIDE:
if data.startswith(header):
return {"format": form, "view": None, "error": f"{inside} inside an archive is not read"}
if _is_zip(data):
return {"format": "zip", "view": None, "error": "an archive inside an archive is not read"}
try:
text = data.decode("utf-8-sig") if b"\x00" not in data[:8192] else None
except UnicodeDecodeError:
+7 -1
View File
@@ -4,6 +4,8 @@ import { isExactJsonNumber } from "../exactJsonNumber.js";
import { MAX_JSON_DEPTH, type JsonValue } from "../jsonValue.js";
import { err, ok, type Result } from "../result.js";
import { safeParseJson } from "../safeJson.js";
import { hasZipSignature } from "../zipPackageFormat.js";
import { ZIP_EOCD_BYTES } from "../zipEndOfCentralDirectory.js";
import {
declaredDocumentType,
isWellFormedXml,
@@ -246,7 +248,11 @@ export const detectStateFormat = (
const header = new TextDecoder("latin1").decode(bytes.subarray(0, 16));
if (header === SQLITE_HEADER) return "sqlite";
if (header.startsWith(GVDB_HEADER)) return "dconf";
if (header.startsWith(ZIP_HEADER)) return "zip";
if (
header.startsWith(ZIP_HEADER) ||
(bytes.length >= ZIP_EOCD_BYTES && hasZipSignature(bytes))
)
return "zip";
const text = decodeStateText(bytes);
if (text === undefined) return "binary";
const trimmed = text.trimStart();
+12 -7
View File
@@ -23,15 +23,20 @@ export const isZipPackageFormat = (
): format is ZipPackageFormat =>
ZIP_PACKAGE_FORMATS.some((candidate) => candidate === format);
/** Recognize local-header, classic/Zip64 empty-archive, or split ZIP signatures, not validity. */
/** Local-header, classic/Zip64 empty-archive, and split ZIP signatures. */
export const ZIP_SIGNATURES = [
[0x50, 0x4b, 0x03, 0x04],
[0x50, 0x4b, 0x05, 0x06],
[0x50, 0x4b, 0x06, 0x06],
[0x50, 0x4b, 0x07, 0x08],
] as const;
/** Recognize a ZIP signature, not directory validity or supported storage. */
export const hasZipSignature = (bytes: Uint8Array): boolean =>
bytes.length >= 4 &&
bytes[0] === 0x50 &&
bytes[1] === 0x4b &&
((bytes[2] === 0x03 && bytes[3] === 0x04) ||
(bytes[2] === 0x05 && bytes[3] === 0x06) ||
(bytes[2] === 0x06 && bytes[3] === 0x06) ||
(bytes[2] === 0x07 && bytes[3] === 0x08));
ZIP_SIGNATURES.some((signature) =>
signature.every((byte, index) => bytes[index] === byte),
);
const CENTRAL_DIRECTORY_HEADER = 0x02014b50;
const LOCAL_FILE_HEADER = 0x04034b50;
@@ -0,0 +1,144 @@
import { execFileSync } from "node:child_process";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect, it } from "vitest";
import { readZipView } from "../../../src/application/desktop/DesktopZipView.js";
import {
PYTHON_HELPERS,
PYTHON_IMPORTS,
} from "../../../src/domain/desktop/appExportPython.js";
import { PYTHON_TASK_EVALUATOR } from "../../../src/domain/desktop/appTaskPython.js";
import { evaluateTaskChecks } from "../../../src/domain/desktop/appTaskChecks.js";
import { detectStateFormat } from "../../../src/domain/desktop/stateFormats.js";
import {
withZip64End,
zipArchive,
} from "../../../src/domain/desktop/zipArchive.fixture.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
const empty = zipArchive([]);
const commented = Buffer.concat([empty, Buffer.from("archive comment")]);
commented.writeUInt16LE(commented.length - 22, 20);
const archives = [
["classic", empty],
["commented", commented],
["Zip64 sentinels", withZip64End(empty)],
["Zip64 ordinary counts", withZip64End(empty, false)],
] as const;
const pythonViews = async (files: readonly Buffer[]) => {
const root = await createTestTempDirectory("desktop-empty-zip-");
const paths = await Promise.all(
files.map(async (bytes, index) => {
const path = join(root, `${String(index)}.dat`);
await writeFile(path, bytes);
return path;
}),
);
const imports = PYTHON_IMPORTS.split("\n")
.filter((line) => !line.includes("agentenv_protocol"))
.join("\n");
const script = join(root, "read.py");
await writeFile(
script,
`${imports}${PYTHON_HELPERS}${PYTHON_TASK_EVALUATOR}
check = {"id": "empty", "kind": "value_equals", "file": "archive.zip", "pointer": "/entry_count", "value": 0}
for path in json.load(sys.stdin):
view = _view(Path(path))
print(json.dumps({"file": view, "passed": evaluate([check], {"archive.zip": view})[0]["result"]}))
`,
);
return execFileSync(
process.platform === "win32" ? "python" : "python3",
["-I", "-B", "-X", "utf8", script],
{ input: JSON.stringify(paths), encoding: "utf8" },
)
.trim()
.split(/\r?\n/u)
.map(
(line) =>
JSON.parse(line) as {
file: { format: string; view: unknown; error?: string };
passed: boolean;
},
);
};
it.each(archives)(
"reads an empty %s archive without relying on its name",
async (_name, bytes) => {
const expected = { entries: [], entry_count: 0, truncated: false };
expect(detectStateFormat("document.dat", bytes)).toBe("zip");
expect(await readZipView(bytes)).toEqual({ ok: true, value: expected });
expect((await pythonViews([bytes]))[0]).toEqual({
file: { format: "zip", view: expected, truncated: false },
passed: true,
});
},
);
it("keeps empty nested archives viewless instead of recursing or calling them binary", async () => {
const outer = zipArchive(
archives.map(([name, data]) => ({ name: `${name}.zip`, data })),
);
const native = await readZipView(outer);
if (!native.ok) throw new Error(native.error);
const python = (await pythonViews([outer]))[0]?.file;
expect(python?.format).toBe("zip");
expect(python?.view).toEqual(native.value);
expect(native.value).toMatchObject({
entries: archives.map(() => ({
format: "zip",
view: null,
view_error: "an archive inside an archive is not read",
})),
entry_count: archives.length,
truncated: false,
});
});
it("grades removing every entry as a change from a nonempty archive", async () => {
const files = [zipArchive([{ name: "old.txt", data: "old" }]), empty];
const check = {
id: "empty",
kind: "value_equals" as const,
file: "archive.zip",
pointer: "/entry_count",
value: 0,
};
const verdicts = await Promise.all(
files.map(async (bytes) => {
const read = await readZipView(bytes);
if (!read.ok) throw new Error(read.error);
return evaluateTaskChecks(
[check],
new Map([
[
check.file,
{ format: detectStateFormat(check.file, bytes), view: read.value },
],
]),
)[0]?.passed;
}),
);
expect(verdicts).toEqual([false, true]);
expect((await pythonViews(files)).map((row) => row.passed)).toEqual(verdicts);
});
it("does not turn incomplete or damaged end records into empty views", async () => {
const incomplete = empty.subarray(0, 6);
expect(detectStateFormat("archive.zip", incomplete)).toBe("binary");
const damaged = Buffer.from(empty);
damaged.writeUInt16LE(1, 20);
const zip64 = withZip64End(empty).subarray(0, 56);
for (const bytes of [damaged, zip64]) {
expect(detectStateFormat("archive.dat", bytes)).toBe("zip");
expect((await readZipView(bytes)).ok).toBe(false);
const python = (await pythonViews([bytes]))[0];
expect(python?.file).toMatchObject({ format: "zip", view: null });
expect(python?.file.error).toBeTypeOf("string");
expect(python?.passed).toBe(false);
}
});