import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { crc32 } from "node:zlib"; import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; import { describe, expect, it } from "vitest"; import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; import type { ArtifactReaderTraversal } from "../resources/ArtifactReader.js"; import { scanReader } from "./inventory/scanReader.js"; import { ZipArtifactReader } from "./ZipArtifactReader.js"; import { accountZipMemberRegion } from "../domain/zip/zipMemberRegion.js"; interface Member { readonly name: string; readonly data: Buffer; /** Whether the central directory lists this local entry. */ readonly listed: boolean; } const member = (name: string, listed = true): Member => ({ name, data: Buffer.from(`${name} bytes`), listed, }); const localEntry = ({ name, data }: Member): Buffer => { const header = Buffer.alloc(30); header.writeUInt32LE(0x04034b50, 0); header.writeUInt16LE(20, 4); header.writeUInt32LE(crc32(data), 14); header.writeUInt32LE(data.length, 18); header.writeUInt32LE(data.length, 22); header.writeUInt16LE(Buffer.byteLength(name), 26); return Buffer.concat([header, Buffer.from(name), data]); }; const centralRecord = ({ name, data }: Member, offset: number): Buffer => { const record = Buffer.alloc(46); record.writeUInt32LE(0x02014b50, 0); record.writeUInt16LE(20, 4); record.writeUInt16LE(20, 6); record.writeUInt32LE(crc32(data), 16); record.writeUInt32LE(data.length, 20); record.writeUInt32LE(data.length, 24); record.writeUInt16LE(Buffer.byteLength(name), 28); record.writeUInt32LE(offset, 42); return Buffer.concat([record, Buffer.from(name)]); }; /** * A stored ZIP. Unlisted members keep their local entry but no directory * record, unless `trailingRecords` keeps their record past the EOCD count. */ const zip = ( members: readonly Member[], options: { prefix?: Buffer; gap?: Buffer; trailingRecords?: boolean } = {}, ): Buffer => { const prefix = options.prefix ?? Buffer.alloc(0); const locals: Buffer[] = [prefix]; const listed: Buffer[] = []; const trailing: Buffer[] = []; let offset = prefix.length; for (const item of members) { const local = localEntry(item); if (item.listed) listed.push(centralRecord(item, offset)); else if (options.trailingRecords === true) trailing.push(centralRecord(item, offset)); locals.push(local); offset += local.length; } if (options.gap !== undefined) { locals.push(options.gap); offset += options.gap.length; } const directory = Buffer.concat([...listed, ...trailing]); const end = Buffer.alloc(22); end.writeUInt32LE(0x06054b50, 0); end.writeUInt16LE(listed.length, 8); end.writeUInt16LE(listed.length, 10); end.writeUInt32LE(directory.length, 12); end.writeUInt32LE(offset, 16); return Buffer.concat([...locals, directory, end]); }; /** An APK Signature Scheme v2 block shape: size, pairs, size, magic. */ const apkSigningBlock = (magic = "APK Sig Block 42"): Buffer => { const pairs = Buffer.alloc(12); pairs.writeBigUInt64LE(4n, 0); const size = BigInt(pairs.length + 8 + 16); const leading = Buffer.alloc(8); leading.writeBigUInt64LE(size); return Buffer.concat([leading, pairs, leading, Buffer.from(magic, "latin1")]); }; const list = async ( bytes: Buffer | Uint8Array, ): Promise<{ paths: string[]; coverage: ArtifactReaderTraversal[] }> => { const directory = await createTestTempDirectory("rea-zip-member-region-"); const path = join(directory, "fixture.zip"); await writeFile(path, bytes); const reader = new ZipArtifactReader(path, "zip"); const paths: string[] = []; const coverage: ArtifactReaderTraversal[] = []; try { for await (const entry of reader.entries(undefined, { onCoverage: (value) => coverage.push(value), })) paths.push(entry.path); } finally { await reader.close(); } return { paths, coverage }; }; const complete = { status: "complete", limitations: [] }; const unaccounted = (bytes: number, ranges: number, first: number) => ({ status: "unknown", limitations: [ `ZIP central directory does not account for ${String(bytes)} byte(s) in ${String(ranges)} range(s) before it, first at offset ${String(first)}; local entries it does not list may exist there.`, ], }); /** Bytes of `localEntry(member("hidden.txt"))`. */ const HIDDEN_ENTRY_BYTES = 30 + 10 + 16; /** Bytes of `localEntry(member("a.txt"))`, the offset after it. */ const A_ENTRY_BYTES = 30 + 5 + 11; describe("ZIP listing coverage against the member region", () => { it("counts a missing header's uncovered region only once", async () => { const valid = localEntry(member("a.txt")); const bytes = Buffer.concat([Buffer.alloc(200), valid]); const records = [ { offset: 100, compressedSize: 0, uncompressedSize: 0, crc32: 0, zip64: false, dataDescriptor: false, }, { offset: 200, compressedSize: member("a.txt").data.length, uncompressedSize: member("a.txt").data.length, crc32: crc32(member("a.txt").data), zip64: false, dataDescriptor: false, }, ]; expect( await accountZipMemberRegion(records, bytes.length, async (at, length) => bytes.subarray(at, at + length), ), ).toMatchObject({ unaccountedBytes: 200, unaccountedRanges: 1, firstUnaccountedOffset: 0, unlocatedHeaders: 1, }); }); it("accounts for a validated data descriptor before an unlisted entry", async () => { const first = member("a.txt"); const local = localEntry(first); const descriptor = Buffer.alloc(16); descriptor.writeUInt32LE(0x08074b50, 0); descriptor.writeUInt32LE(crc32(first.data), 4); descriptor.writeUInt32LE(first.data.length, 8); descriptor.writeUInt32LE(first.data.length, 12); const hidden = localEntry(member("hidden.txt", false)); const bytes = Buffer.concat([local, descriptor, hidden]); const records = [ { offset: 0, compressedSize: first.data.length, uncompressedSize: first.data.length, crc32: crc32(first.data), zip64: false, dataDescriptor: true, }, ]; expect( await accountZipMemberRegion(records, bytes.length, async (at, length) => bytes.subarray(at, at + length), ), ).toMatchObject({ unaccountedBytes: hidden.length, unaccountedRanges: 1, firstUnaccountedOffset: local.length + descriptor.length, }); }); it("accounts for an unsigned data descriptor whose CRC equals the descriptor signature", async () => { const first = member("a.txt"); const local = localEntry(first); // CRC 0x08074b50 makes the unsigned descriptor start with the signature bytes. const descriptor = Buffer.alloc(12); descriptor.writeUInt32LE(0x08074b50, 0); descriptor.writeUInt32LE(first.data.length, 4); descriptor.writeUInt32LE(first.data.length, 8); const bytes = Buffer.concat([local, descriptor]); const records = [ { offset: 0, compressedSize: first.data.length, uncompressedSize: first.data.length, crc32: 0x08074b50, zip64: false, dataDescriptor: true, }, ]; expect( await accountZipMemberRegion(records, bytes.length, async (at, length) => bytes.subarray(at, at + length), ), ).toMatchObject({ unaccountedBytes: 0, unaccountedRanges: 0 }); // With room for a signed layout, the unsigned layout still matches. const padded = Buffer.concat([bytes, Buffer.alloc(4)]); expect( await accountZipMemberRegion(records, padded.length, async (at, length) => padded.subarray(at, at + length), ), ).toMatchObject({ unaccountedBytes: 4, unaccountedRanges: 1, firstUnaccountedOffset: bytes.length, }); }); it("accounts for an AE-2 data descriptor whose directory reports no CRC", async () => { const first = member("a.txt"); const local = localEntry(first); const descriptor = Buffer.alloc(16); descriptor.writeUInt32LE(0x08074b50, 0); descriptor.writeUInt32LE(first.data.length, 8); descriptor.writeUInt32LE(first.data.length, 12); const bytes = Buffer.concat([local, descriptor]); const record = { offset: 0, compressedSize: first.data.length, uncompressedSize: first.data.length, crc32: undefined, zip64: false, dataDescriptor: true, }; const read = async (at: number, length: number) => bytes.subarray(at, at + length); expect( await accountZipMemberRegion([record], bytes.length, read), ).toMatchObject({ unaccountedBytes: 0, unaccountedRanges: 0 }); // Sizes still have to match. expect( await accountZipMemberRegion( [{ ...record, compressedSize: first.data.length + 1 }], bytes.length, read, ), ).toMatchObject({ unaccountedRanges: 1 }); }); }); describe("ZIP listing coverage from the central directory", () => { it("declares complete coverage when directory records cover every byte", async () => { expect(await list(zip([member("a.txt"), member("b.txt")]))).toEqual({ paths: ["a.txt", "b.txt"], coverage: [complete], }); }); it("declares complete coverage for writer-produced data descriptors", async () => { const writer = new ZipWriter(new Uint8ArrayWriter(), { dataDescriptor: true, dataDescriptorSignature: true, }); await writer.add("a.txt", new TextReader("a".repeat(500))); await writer.add("b.txt", new TextReader("b")); expect((await list(await writer.close())).coverage).toEqual([complete]); }); it.each([ [ "between listed members", [member("a.txt"), member("hidden.txt", false), member("b.txt")], A_ENTRY_BYTES, ], [ "before the first listed member", [member("hidden.txt", false), member("a.txt")], 0, ], [ "after the last listed member", [member("a.txt"), member("hidden.txt", false)], A_ENTRY_BYTES, ], ] as const)( "reports a local entry the central directory omits %s as unknown", async (_placement, members, first) => { const { paths, coverage } = await list(zip(members)); expect(paths).not.toContain("hidden.txt"); expect(coverage).toEqual([unaccounted(HIDDEN_ENTRY_BYTES, 1, first)]); }, ); it("reports a directory record past the end record's declared count as unknown", async () => { const { paths, coverage } = await list( zip([member("a.txt"), member("hidden.txt", false)], { trailingRecords: true, }), ); expect(paths).toEqual(["a.txt"]); expect(coverage).toEqual([ unaccounted(HIDDEN_ENTRY_BYTES, 1, A_ENTRY_BYTES), ]); }); it("reports bytes prepended before the first member as unknown", async () => { const prefix = Buffer.from("#!/bin/sh\nexit 0\n"); expect((await list(zip([member("a.txt")], { prefix }))).coverage).toEqual([ unaccounted(prefix.length, 1, 0), ]); }); it("accounts for a well-formed APK signing block before the directory", async () => { expect( (await list(zip([member("a.txt")], { gap: apkSigningBlock() }))).coverage, ).toEqual([complete]); }); it("does not account for a block whose footer magic differs", async () => { const gap = apkSigningBlock("APK Sig Block 41"); expect((await list(zip([member("a.txt")], { gap }))).coverage).toEqual([ unaccounted(gap.length, 1, A_ENTRY_BYTES), ]); }); it("does not hide an unlisted local entry inside malformed signing-block framing", async () => { const hidden = localEntry(member("hidden.txt", false)); const size = BigInt(hidden.length + 24); const framing = Buffer.alloc(8); framing.writeBigUInt64LE(size); const block = Buffer.concat([ framing, hidden, framing, Buffer.from("APK Sig Block 42"), ]); expect( (await list(zip([member("a.txt")], { gap: block }))).coverage, ).toEqual([unaccounted(block.length, 1, A_ENTRY_BYTES)]); }); it("carries the unaccounted region into the artifact inventory", async () => { const directory = await createTestTempDirectory("rea-zip-member-region-"); const path = join(directory, "fixture.zip"); await writeFile( path, zip([member("a.txt"), member("hidden.txt", false), member("b.txt")]), ); const reader = new ZipArtifactReader(path, "zip"); try { const result = await scanReader(reader, [reader]); expect(result.occurrences.map((item) => item.logical_path)).toEqual([ "a.txt", "b.txt", ]); expect(result.traversal).toEqual( unaccounted(HIDDEN_ENTRY_BYTES, 1, A_ENTRY_BYTES), ); } finally { await reader.close(); } }); });