From 263b25bd34bc644600a8957f5c1a01b8d0ba49fc Mon Sep 17 00:00:00 2001 From: wunianze666-netizen Date: Sun, 8 Nov 2026 14:45:11 +0800 Subject: [PATCH] fix(desktop): recognize empty ZIP state views --- docs/desktop-application-state.md | 20 +-- .../document-views/plan.md | 3 +- .../document-views/spec.md | 5 +- .../state-view/spec.md | 4 +- src/domain/desktop/appExportPython.ts | 11 +- src/domain/desktop/appExportPythonZip.ts | 5 +- src/domain/desktop/stateFormats.ts | 8 +- src/domain/zipPackageFormat.ts | 19 ++- .../providers/desktopEmptyZip.test.ts | 144 ++++++++++++++++++ 9 files changed, 193 insertions(+), 26 deletions(-) create mode 100644 tests/boundary/providers/desktopEmptyZip.test.ts diff --git a/docs/desktop-application-state.md b/docs/desktop-application-state.md index 5d306979f0..fbe193af9e 100644 --- a/docs/desktop-application-state.md +++ b/docs/desktop-application-state.md @@ -64,16 +64,16 @@ Views hold the captured bytes as they are, the capture host's name and user included; the export removes those from the bundle. A version records whether the file was present, the SHA-256 of its retained bytes, and a view: -| Format | Detected by | View | -| --------- | ---------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | -| `sqlite` | The `SQLite format 3` header | Every ordinary table's columns, primary key and up to 500 rows | -| `dconf` | The `GVariant` header | Groups whose path names the application, plus the names of the other groups | -| `zip` | A zip local file header, `PK\x03\x04`, whatever the name | Its file entries in archive order, each with its own format and view | -| `json` | A `.json` name or a leading `{` or `[` that parses | The parsed value, from files up to 4 MiB nested up to 500 levels | -| `xml` | An XML name or declaration that parses | An element tree (`name`, `attributes`, `children`); text kept exactly, DTDs never read | -| `keyfile` | Only `[group]`, `key=value` and comment lines, with at least one group | Groups of `key` → string value, from files up to 4 MiB | -| `text` | Other UTF-8 text | Lines, up to 5,000 | -| `binary` | NUL bytes or invalid UTF-8 | None | +| Format | Detected by | View | +| --------- | -------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | +| `sqlite` | The `SQLite format 3` header | Every ordinary table's columns, primary key and up to 500 rows | +| `dconf` | The `GVariant` header | Groups whose path names the application, plus the names of the other groups | +| `zip` | A zip local file header or an empty classic/Zip64 archive signature, whatever the name | Its file entries in archive order, each with its own format and view | +| `json` | A `.json` name or a leading `{` or `[` that parses | The parsed value, from files up to 4 MiB nested up to 500 levels | +| `xml` | An XML name or declaration that parses | An element tree (`name`, `attributes`, `children`); text kept exactly, DTDs never read | +| `keyfile` | Only `[group]`, `key=value` and comment lines, with at least one group | Groups of `key` → string value, from files up to 4 MiB | +| `text` | Other UTF-8 text | Lines, up to 5,000 | +| `binary` | NUL bytes or invalid UTF-8 | None | A JSON value holding a number JavaScript would round, such as an integer above 2^53, is not read rather than shown rounded. diff --git a/docs/specs/desktop-app-environments/document-views/plan.md b/docs/specs/desktop-app-environments/document-views/plan.md index 73e31d6bf9..45d4cee6fc 100644 --- a/docs/specs/desktop-app-environments/document-views/plan.md +++ b/docs/specs/desktop-app-environments/document-views/plan.md @@ -1,6 +1,7 @@ # Plan -1. Format and reader: `zip` detected by its local file header; the text, JSON, +1. Format and reader: `zip` detected by its local file header or an empty + classic/Zip64 archive signature; the text, JSON, key file and XML readers shared as `textualView`, so a file and an entry read alike; a domain module for the entry limits, the entry view and the archive view, and an application reader that walks the verified bytes with diff --git a/docs/specs/desktop-app-environments/document-views/spec.md b/docs/specs/desktop-app-environments/document-views/spec.md index 9b06107c1d..784051f185 100644 --- a/docs/specs/desktop-app-environments/document-views/spec.md +++ b/docs/specs/desktop-app-environments/document-views/spec.md @@ -13,8 +13,9 @@ evaluators, REA's and the exported verifier's, agree. ## Requirements -- Format: a file starting with a zip local file header (`PK\x03\x04`) is - `zip`, whatever its name. Its view is +- Format: a file starting with a zip local file header (`PK\x03\x04`), or + a classic/Zip64 empty-archive signature with at least 22 bytes, is `zip`, + whatever its name. A signature is not proof of directory validity. Its view is `{entries, entry_count, truncated}`: the file entries in archive order, directories left out, each `{name, size, sha256, format, view, view_error}`. - Reading: from the retained bytes already checked against their digest, in diff --git a/docs/specs/desktop-app-environments/state-view/spec.md b/docs/specs/desktop-app-environments/state-view/spec.md index c7b934fcd6..82a65bcf95 100644 --- a/docs/specs/desktop-app-environments/state-view/spec.md +++ b/docs/specs/desktop-app-environments/state-view/spec.md @@ -33,8 +33,8 @@ generation reads the kinds of change. with exact numbers; XML through the shared strict parser, a DOCTYPE read only when it has no internal subset and its DTD never read ([XML text checks](../xml-text-checks/spec.md)); key files up to 4 MiB - whose every line parses; zip archives by - their local file header, as their entries with each entry's own format and + whose every line parses; zip archives by their local file header or a + classic/Zip64 empty-archive signature, as entries with each entry's own format and view ([document views](../document-views/spec.md)); text; binary. - Changes between versions are created, deleted, or lines, values, keys or rows with added, removed and changed lists (100 each, with counts); content diff --git a/src/domain/desktop/appExportPython.ts b/src/domain/desktop/appExportPython.ts index 1237265b55..9db016015e 100644 --- a/src/domain/desktop/appExportPython.ts +++ b/src/domain/desktop/appExportPython.ts @@ -1,6 +1,8 @@ import { PYTHON_JSON_READER } from "./appExportPythonJson.js"; import { PYTHON_XML_READER } from "./appExportPythonXml.js"; import { PYTHON_ZIP_HELPERS } from "./appExportPythonZip.js"; +import { ZIP_SIGNATURES } from "../zipPackageFormat.js"; +import { ZIP_EOCD_BYTES } from "../zipEndOfCentralDirectory.js"; /** The generated module's imports. */ /** The class method that holds readiness until the app's own window shows, not a splash screen with its class. */ @@ -46,6 +48,8 @@ LINE_BREAK = re.compile(r"\r?\n") SQLITE_HEADER = b"SQLite format 3\x00" GVDB_HEADER = b"GVariant" ZIP_HEADER = b"PK\x03\x04" +ZIP_SIGNATURES = (${ZIP_SIGNATURES.map((signature) => `bytes(${JSON.stringify(signature)})`).join(", ")}) +ZIP_EOCD_BYTES = ${String(ZIP_EOCD_BYTES)} XML_SPACE = "xml:space" MAX_VIEW_BYTES = 64 * 1024 * 1024 MAX_SAFE_INTEGER = 2**53 - 1 @@ -272,6 +276,11 @@ def _text_view(name: str, text: str, size: int) -> dict: return _text(text) +def _is_zip(data: bytes) -> bool: + """Known ZIP storage signatures, including empty archives; the reader owns directory validity.""" + return data.startswith(ZIP_HEADER) or (len(data) >= ZIP_EOCD_BYTES and data.startswith(ZIP_SIGNATURES)) + + def _view(path: Path) -> dict: """A typed JSON view of one file in the shapes of REA's state view; truncated says the view is partial.""" too_large = {"format": None, "view": None, "error": "larger than 64 MiB, the most the view reads"} @@ -283,7 +292,7 @@ def _view(path: Path) -> dict: data += file.read(MAX_VIEW_BYTES + 1 - len(data)) if whole else b"" if data.startswith(SQLITE_HEADER): return _sqlite(path) - if data.startswith(ZIP_HEADER): + if _is_zip(data): return _zip(path) if len(data) > MAX_VIEW_BYTES: return too_large diff --git a/src/domain/desktop/appExportPythonZip.ts b/src/domain/desktop/appExportPythonZip.ts index c20d197dd1..cf0755c007 100644 --- a/src/domain/desktop/appExportPythonZip.ts +++ b/src/domain/desktop/appExportPythonZip.ts @@ -14,8 +14,7 @@ ZIP_CENTRAL = b"PK\x01\x02" ZIP64_LOCATOR = b"PK\x06\x07" ZIP64_END = b"PK\x06\x06" #: Formats an archive entry may have whose reader needs a tool, a worker or another archive opened. -NOT_READ_INSIDE = ((SQLITE_HEADER, "sqlite", "a SQLite database"), (GVDB_HEADER, "dconf", "a dconf database"), - (ZIP_HEADER, "zip", "an archive")) +NOT_READ_INSIDE = ((SQLITE_HEADER, "sqlite", "a SQLite database"), (GVDB_HEADER, "dconf", "a dconf database")) def _entry_view(name: str, data: bytes) -> dict: @@ -24,6 +23,8 @@ def _entry_view(name: str, data: bytes) -> dict: for header, form, inside in NOT_READ_INSIDE: if data.startswith(header): return {"format": form, "view": None, "error": f"{inside} inside an archive is not read"} + if _is_zip(data): + return {"format": "zip", "view": None, "error": "an archive inside an archive is not read"} try: text = data.decode("utf-8-sig") if b"\x00" not in data[:8192] else None except UnicodeDecodeError: diff --git a/src/domain/desktop/stateFormats.ts b/src/domain/desktop/stateFormats.ts index 9294e6b52a..b9378cca62 100644 --- a/src/domain/desktop/stateFormats.ts +++ b/src/domain/desktop/stateFormats.ts @@ -4,6 +4,8 @@ import { isExactJsonNumber } from "../exactJsonNumber.js"; import { MAX_JSON_DEPTH, type JsonValue } from "../jsonValue.js"; import { err, ok, type Result } from "../result.js"; import { safeParseJson } from "../safeJson.js"; +import { hasZipSignature } from "../zipPackageFormat.js"; +import { ZIP_EOCD_BYTES } from "../zipEndOfCentralDirectory.js"; import { declaredDocumentType, isWellFormedXml, @@ -246,7 +248,11 @@ export const detectStateFormat = ( const header = new TextDecoder("latin1").decode(bytes.subarray(0, 16)); if (header === SQLITE_HEADER) return "sqlite"; if (header.startsWith(GVDB_HEADER)) return "dconf"; - if (header.startsWith(ZIP_HEADER)) return "zip"; + if ( + header.startsWith(ZIP_HEADER) || + (bytes.length >= ZIP_EOCD_BYTES && hasZipSignature(bytes)) + ) + return "zip"; const text = decodeStateText(bytes); if (text === undefined) return "binary"; const trimmed = text.trimStart(); diff --git a/src/domain/zipPackageFormat.ts b/src/domain/zipPackageFormat.ts index 04ddd31620..035adec025 100644 --- a/src/domain/zipPackageFormat.ts +++ b/src/domain/zipPackageFormat.ts @@ -23,15 +23,20 @@ export const isZipPackageFormat = ( ): format is ZipPackageFormat => ZIP_PACKAGE_FORMATS.some((candidate) => candidate === format); -/** Recognize local-header, classic/Zip64 empty-archive, or split ZIP signatures, not validity. */ +/** Local-header, classic/Zip64 empty-archive, and split ZIP signatures. */ +export const ZIP_SIGNATURES = [ + [0x50, 0x4b, 0x03, 0x04], + [0x50, 0x4b, 0x05, 0x06], + [0x50, 0x4b, 0x06, 0x06], + [0x50, 0x4b, 0x07, 0x08], +] as const; + +/** Recognize a ZIP signature, not directory validity or supported storage. */ export const hasZipSignature = (bytes: Uint8Array): boolean => bytes.length >= 4 && - bytes[0] === 0x50 && - bytes[1] === 0x4b && - ((bytes[2] === 0x03 && bytes[3] === 0x04) || - (bytes[2] === 0x05 && bytes[3] === 0x06) || - (bytes[2] === 0x06 && bytes[3] === 0x06) || - (bytes[2] === 0x07 && bytes[3] === 0x08)); + ZIP_SIGNATURES.some((signature) => + signature.every((byte, index) => bytes[index] === byte), + ); const CENTRAL_DIRECTORY_HEADER = 0x02014b50; const LOCAL_FILE_HEADER = 0x04034b50; diff --git a/tests/boundary/providers/desktopEmptyZip.test.ts b/tests/boundary/providers/desktopEmptyZip.test.ts new file mode 100644 index 0000000000..3a7e2fd29c --- /dev/null +++ b/tests/boundary/providers/desktopEmptyZip.test.ts @@ -0,0 +1,144 @@ +import { execFileSync } from "node:child_process"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; + +import { readZipView } from "../../../src/application/desktop/DesktopZipView.js"; +import { + PYTHON_HELPERS, + PYTHON_IMPORTS, +} from "../../../src/domain/desktop/appExportPython.js"; +import { PYTHON_TASK_EVALUATOR } from "../../../src/domain/desktop/appTaskPython.js"; +import { evaluateTaskChecks } from "../../../src/domain/desktop/appTaskChecks.js"; +import { detectStateFormat } from "../../../src/domain/desktop/stateFormats.js"; +import { + withZip64End, + zipArchive, +} from "../../../src/domain/desktop/zipArchive.fixture.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const empty = zipArchive([]); +const commented = Buffer.concat([empty, Buffer.from("archive comment")]); +commented.writeUInt16LE(commented.length - 22, 20); +const archives = [ + ["classic", empty], + ["commented", commented], + ["Zip64 sentinels", withZip64End(empty)], + ["Zip64 ordinary counts", withZip64End(empty, false)], +] as const; + +const pythonViews = async (files: readonly Buffer[]) => { + const root = await createTestTempDirectory("desktop-empty-zip-"); + const paths = await Promise.all( + files.map(async (bytes, index) => { + const path = join(root, `${String(index)}.dat`); + await writeFile(path, bytes); + return path; + }), + ); + const imports = PYTHON_IMPORTS.split("\n") + .filter((line) => !line.includes("agentenv_protocol")) + .join("\n"); + const script = join(root, "read.py"); + await writeFile( + script, + `${imports}${PYTHON_HELPERS}${PYTHON_TASK_EVALUATOR} +check = {"id": "empty", "kind": "value_equals", "file": "archive.zip", "pointer": "/entry_count", "value": 0} +for path in json.load(sys.stdin): + view = _view(Path(path)) + print(json.dumps({"file": view, "passed": evaluate([check], {"archive.zip": view})[0]["result"]})) +`, + ); + return execFileSync( + process.platform === "win32" ? "python" : "python3", + ["-I", "-B", "-X", "utf8", script], + { input: JSON.stringify(paths), encoding: "utf8" }, + ) + .trim() + .split(/\r?\n/u) + .map( + (line) => + JSON.parse(line) as { + file: { format: string; view: unknown; error?: string }; + passed: boolean; + }, + ); +}; + +it.each(archives)( + "reads an empty %s archive without relying on its name", + async (_name, bytes) => { + const expected = { entries: [], entry_count: 0, truncated: false }; + expect(detectStateFormat("document.dat", bytes)).toBe("zip"); + expect(await readZipView(bytes)).toEqual({ ok: true, value: expected }); + expect((await pythonViews([bytes]))[0]).toEqual({ + file: { format: "zip", view: expected, truncated: false }, + passed: true, + }); + }, +); + +it("keeps empty nested archives viewless instead of recursing or calling them binary", async () => { + const outer = zipArchive( + archives.map(([name, data]) => ({ name: `${name}.zip`, data })), + ); + const native = await readZipView(outer); + if (!native.ok) throw new Error(native.error); + const python = (await pythonViews([outer]))[0]?.file; + expect(python?.format).toBe("zip"); + expect(python?.view).toEqual(native.value); + expect(native.value).toMatchObject({ + entries: archives.map(() => ({ + format: "zip", + view: null, + view_error: "an archive inside an archive is not read", + })), + entry_count: archives.length, + truncated: false, + }); +}); + +it("grades removing every entry as a change from a nonempty archive", async () => { + const files = [zipArchive([{ name: "old.txt", data: "old" }]), empty]; + const check = { + id: "empty", + kind: "value_equals" as const, + file: "archive.zip", + pointer: "/entry_count", + value: 0, + }; + const verdicts = await Promise.all( + files.map(async (bytes) => { + const read = await readZipView(bytes); + if (!read.ok) throw new Error(read.error); + return evaluateTaskChecks( + [check], + new Map([ + [ + check.file, + { format: detectStateFormat(check.file, bytes), view: read.value }, + ], + ]), + )[0]?.passed; + }), + ); + expect(verdicts).toEqual([false, true]); + expect((await pythonViews(files)).map((row) => row.passed)).toEqual(verdicts); +}); + +it("does not turn incomplete or damaged end records into empty views", async () => { + const incomplete = empty.subarray(0, 6); + expect(detectStateFormat("archive.zip", incomplete)).toBe("binary"); + const damaged = Buffer.from(empty); + damaged.writeUInt16LE(1, 20); + const zip64 = withZip64End(empty).subarray(0, 56); + for (const bytes of [damaged, zip64]) { + expect(detectStateFormat("archive.dat", bytes)).toBe("zip"); + expect((await readZipView(bytes)).ok).toBe(false); + const python = (await pythonViews([bytes]))[0]; + expect(python?.file).toMatchObject({ format: "zip", view: null }); + expect(python?.file.error).toBeTypeOf("string"); + expect(python?.passed).toBe(false); + } +});