mirror of
https://github.com/morluto/rea.git
synced 2026-11-16 11:43:20 +00:00
375 lines
13 KiB
TypeScript
375 lines
13 KiB
TypeScript
import { writeFile } from "node:fs/promises";
|
|
import { join } from "node:path";
|
|
import { crc32 } from "node:zlib";
|
|
|
|
import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js";
|
|
import { describe, expect, it } from "vitest";
|
|
|
|
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
|
|
import type { ArtifactReaderTraversal } from "../resources/ArtifactReader.js";
|
|
import { scanReader } from "./inventory/scanReader.js";
|
|
import { ZipArtifactReader } from "./ZipArtifactReader.js";
|
|
import { accountZipMemberRegion } from "../domain/zip/zipMemberRegion.js";
|
|
|
|
interface Member {
|
|
readonly name: string;
|
|
readonly data: Buffer;
|
|
/** Whether the central directory lists this local entry. */
|
|
readonly listed: boolean;
|
|
}
|
|
|
|
const member = (name: string, listed = true): Member => ({
|
|
name,
|
|
data: Buffer.from(`${name} bytes`),
|
|
listed,
|
|
});
|
|
|
|
const localEntry = ({ name, data }: Member): Buffer => {
|
|
const header = Buffer.alloc(30);
|
|
header.writeUInt32LE(0x04034b50, 0);
|
|
header.writeUInt16LE(20, 4);
|
|
header.writeUInt32LE(crc32(data), 14);
|
|
header.writeUInt32LE(data.length, 18);
|
|
header.writeUInt32LE(data.length, 22);
|
|
header.writeUInt16LE(Buffer.byteLength(name), 26);
|
|
return Buffer.concat([header, Buffer.from(name), data]);
|
|
};
|
|
|
|
const centralRecord = ({ name, data }: Member, offset: number): Buffer => {
|
|
const record = Buffer.alloc(46);
|
|
record.writeUInt32LE(0x02014b50, 0);
|
|
record.writeUInt16LE(20, 4);
|
|
record.writeUInt16LE(20, 6);
|
|
record.writeUInt32LE(crc32(data), 16);
|
|
record.writeUInt32LE(data.length, 20);
|
|
record.writeUInt32LE(data.length, 24);
|
|
record.writeUInt16LE(Buffer.byteLength(name), 28);
|
|
record.writeUInt32LE(offset, 42);
|
|
return Buffer.concat([record, Buffer.from(name)]);
|
|
};
|
|
|
|
/**
|
|
* A stored ZIP. Unlisted members keep their local entry but no directory
|
|
* record, unless `trailingRecords` keeps their record past the EOCD count.
|
|
*/
|
|
const zip = (
|
|
members: readonly Member[],
|
|
options: { prefix?: Buffer; gap?: Buffer; trailingRecords?: boolean } = {},
|
|
): Buffer => {
|
|
const prefix = options.prefix ?? Buffer.alloc(0);
|
|
const locals: Buffer[] = [prefix];
|
|
const listed: Buffer[] = [];
|
|
const trailing: Buffer[] = [];
|
|
let offset = prefix.length;
|
|
for (const item of members) {
|
|
const local = localEntry(item);
|
|
if (item.listed) listed.push(centralRecord(item, offset));
|
|
else if (options.trailingRecords === true)
|
|
trailing.push(centralRecord(item, offset));
|
|
locals.push(local);
|
|
offset += local.length;
|
|
}
|
|
if (options.gap !== undefined) {
|
|
locals.push(options.gap);
|
|
offset += options.gap.length;
|
|
}
|
|
const directory = Buffer.concat([...listed, ...trailing]);
|
|
const end = Buffer.alloc(22);
|
|
end.writeUInt32LE(0x06054b50, 0);
|
|
end.writeUInt16LE(listed.length, 8);
|
|
end.writeUInt16LE(listed.length, 10);
|
|
end.writeUInt32LE(directory.length, 12);
|
|
end.writeUInt32LE(offset, 16);
|
|
return Buffer.concat([...locals, directory, end]);
|
|
};
|
|
|
|
/** An APK Signature Scheme v2 block shape: size, pairs, size, magic. */
|
|
const apkSigningBlock = (magic = "APK Sig Block 42"): Buffer => {
|
|
const pairs = Buffer.alloc(12);
|
|
pairs.writeBigUInt64LE(4n, 0);
|
|
const size = BigInt(pairs.length + 8 + 16);
|
|
const leading = Buffer.alloc(8);
|
|
leading.writeBigUInt64LE(size);
|
|
return Buffer.concat([leading, pairs, leading, Buffer.from(magic, "latin1")]);
|
|
};
|
|
|
|
const list = async (
|
|
bytes: Buffer | Uint8Array,
|
|
): Promise<{ paths: string[]; coverage: ArtifactReaderTraversal[] }> => {
|
|
const directory = await createTestTempDirectory("rea-zip-member-region-");
|
|
const path = join(directory, "fixture.zip");
|
|
await writeFile(path, bytes);
|
|
const reader = new ZipArtifactReader(path, "zip");
|
|
const paths: string[] = [];
|
|
const coverage: ArtifactReaderTraversal[] = [];
|
|
try {
|
|
for await (const entry of reader.entries(undefined, {
|
|
onCoverage: (value) => coverage.push(value),
|
|
}))
|
|
paths.push(entry.path);
|
|
} finally {
|
|
await reader.close();
|
|
}
|
|
return { paths, coverage };
|
|
};
|
|
|
|
const complete = { status: "complete", limitations: [] };
|
|
const unaccounted = (bytes: number, ranges: number, first: number) => ({
|
|
status: "unknown",
|
|
limitations: [
|
|
`ZIP central directory does not account for ${String(bytes)} byte(s) in ${String(ranges)} range(s) before it, first at offset ${String(first)}; local entries it does not list may exist there.`,
|
|
],
|
|
});
|
|
/** Bytes of `localEntry(member("hidden.txt"))`. */
|
|
const HIDDEN_ENTRY_BYTES = 30 + 10 + 16;
|
|
/** Bytes of `localEntry(member("a.txt"))`, the offset after it. */
|
|
const A_ENTRY_BYTES = 30 + 5 + 11;
|
|
|
|
describe("ZIP listing coverage against the member region", () => {
|
|
it("counts a missing header's uncovered region only once", async () => {
|
|
const valid = localEntry(member("a.txt"));
|
|
const bytes = Buffer.concat([Buffer.alloc(200), valid]);
|
|
const records = [
|
|
{
|
|
offset: 100,
|
|
compressedSize: 0,
|
|
uncompressedSize: 0,
|
|
crc32: 0,
|
|
zip64: false,
|
|
dataDescriptor: false,
|
|
},
|
|
{
|
|
offset: 200,
|
|
compressedSize: member("a.txt").data.length,
|
|
uncompressedSize: member("a.txt").data.length,
|
|
crc32: crc32(member("a.txt").data),
|
|
zip64: false,
|
|
dataDescriptor: false,
|
|
},
|
|
];
|
|
expect(
|
|
await accountZipMemberRegion(records, bytes.length, async (at, length) =>
|
|
bytes.subarray(at, at + length),
|
|
),
|
|
).toMatchObject({
|
|
unaccountedBytes: 200,
|
|
unaccountedRanges: 1,
|
|
firstUnaccountedOffset: 0,
|
|
unlocatedHeaders: 1,
|
|
});
|
|
});
|
|
|
|
it("accounts for a validated data descriptor before an unlisted entry", async () => {
|
|
const first = member("a.txt");
|
|
const local = localEntry(first);
|
|
const descriptor = Buffer.alloc(16);
|
|
descriptor.writeUInt32LE(0x08074b50, 0);
|
|
descriptor.writeUInt32LE(crc32(first.data), 4);
|
|
descriptor.writeUInt32LE(first.data.length, 8);
|
|
descriptor.writeUInt32LE(first.data.length, 12);
|
|
const hidden = localEntry(member("hidden.txt", false));
|
|
const bytes = Buffer.concat([local, descriptor, hidden]);
|
|
const records = [
|
|
{
|
|
offset: 0,
|
|
compressedSize: first.data.length,
|
|
uncompressedSize: first.data.length,
|
|
crc32: crc32(first.data),
|
|
zip64: false,
|
|
dataDescriptor: true,
|
|
},
|
|
];
|
|
expect(
|
|
await accountZipMemberRegion(records, bytes.length, async (at, length) =>
|
|
bytes.subarray(at, at + length),
|
|
),
|
|
).toMatchObject({
|
|
unaccountedBytes: hidden.length,
|
|
unaccountedRanges: 1,
|
|
firstUnaccountedOffset: local.length + descriptor.length,
|
|
});
|
|
});
|
|
|
|
it("accounts for an unsigned data descriptor whose CRC equals the descriptor signature", async () => {
|
|
const first = member("a.txt");
|
|
const local = localEntry(first);
|
|
// CRC 0x08074b50 makes the unsigned descriptor start with the signature bytes.
|
|
const descriptor = Buffer.alloc(12);
|
|
descriptor.writeUInt32LE(0x08074b50, 0);
|
|
descriptor.writeUInt32LE(first.data.length, 4);
|
|
descriptor.writeUInt32LE(first.data.length, 8);
|
|
const bytes = Buffer.concat([local, descriptor]);
|
|
const records = [
|
|
{
|
|
offset: 0,
|
|
compressedSize: first.data.length,
|
|
uncompressedSize: first.data.length,
|
|
crc32: 0x08074b50,
|
|
zip64: false,
|
|
dataDescriptor: true,
|
|
},
|
|
];
|
|
expect(
|
|
await accountZipMemberRegion(records, bytes.length, async (at, length) =>
|
|
bytes.subarray(at, at + length),
|
|
),
|
|
).toMatchObject({ unaccountedBytes: 0, unaccountedRanges: 0 });
|
|
// With room for a signed layout, the unsigned layout still matches.
|
|
const padded = Buffer.concat([bytes, Buffer.alloc(4)]);
|
|
expect(
|
|
await accountZipMemberRegion(records, padded.length, async (at, length) =>
|
|
padded.subarray(at, at + length),
|
|
),
|
|
).toMatchObject({
|
|
unaccountedBytes: 4,
|
|
unaccountedRanges: 1,
|
|
firstUnaccountedOffset: bytes.length,
|
|
});
|
|
});
|
|
|
|
it("accounts for an AE-2 data descriptor whose directory reports no CRC", async () => {
|
|
const first = member("a.txt");
|
|
const local = localEntry(first);
|
|
const descriptor = Buffer.alloc(16);
|
|
descriptor.writeUInt32LE(0x08074b50, 0);
|
|
descriptor.writeUInt32LE(first.data.length, 8);
|
|
descriptor.writeUInt32LE(first.data.length, 12);
|
|
const bytes = Buffer.concat([local, descriptor]);
|
|
const record = {
|
|
offset: 0,
|
|
compressedSize: first.data.length,
|
|
uncompressedSize: first.data.length,
|
|
crc32: undefined,
|
|
zip64: false,
|
|
dataDescriptor: true,
|
|
};
|
|
const read = async (at: number, length: number) =>
|
|
bytes.subarray(at, at + length);
|
|
expect(
|
|
await accountZipMemberRegion([record], bytes.length, read),
|
|
).toMatchObject({ unaccountedBytes: 0, unaccountedRanges: 0 });
|
|
// Sizes still have to match.
|
|
expect(
|
|
await accountZipMemberRegion(
|
|
[{ ...record, compressedSize: first.data.length + 1 }],
|
|
bytes.length,
|
|
read,
|
|
),
|
|
).toMatchObject({ unaccountedRanges: 1 });
|
|
});
|
|
});
|
|
|
|
describe("ZIP listing coverage from the central directory", () => {
|
|
it("declares complete coverage when directory records cover every byte", async () => {
|
|
expect(await list(zip([member("a.txt"), member("b.txt")]))).toEqual({
|
|
paths: ["a.txt", "b.txt"],
|
|
coverage: [complete],
|
|
});
|
|
});
|
|
|
|
it("declares complete coverage for writer-produced data descriptors", async () => {
|
|
const writer = new ZipWriter(new Uint8ArrayWriter(), {
|
|
dataDescriptor: true,
|
|
dataDescriptorSignature: true,
|
|
});
|
|
await writer.add("a.txt", new TextReader("a".repeat(500)));
|
|
await writer.add("b.txt", new TextReader("b"));
|
|
expect((await list(await writer.close())).coverage).toEqual([complete]);
|
|
});
|
|
|
|
it.each([
|
|
[
|
|
"between listed members",
|
|
[member("a.txt"), member("hidden.txt", false), member("b.txt")],
|
|
A_ENTRY_BYTES,
|
|
],
|
|
[
|
|
"before the first listed member",
|
|
[member("hidden.txt", false), member("a.txt")],
|
|
0,
|
|
],
|
|
[
|
|
"after the last listed member",
|
|
[member("a.txt"), member("hidden.txt", false)],
|
|
A_ENTRY_BYTES,
|
|
],
|
|
] as const)(
|
|
"reports a local entry the central directory omits %s as unknown",
|
|
async (_placement, members, first) => {
|
|
const { paths, coverage } = await list(zip(members));
|
|
expect(paths).not.toContain("hidden.txt");
|
|
expect(coverage).toEqual([unaccounted(HIDDEN_ENTRY_BYTES, 1, first)]);
|
|
},
|
|
);
|
|
|
|
it("reports a directory record past the end record's declared count as unknown", async () => {
|
|
const { paths, coverage } = await list(
|
|
zip([member("a.txt"), member("hidden.txt", false)], {
|
|
trailingRecords: true,
|
|
}),
|
|
);
|
|
expect(paths).toEqual(["a.txt"]);
|
|
expect(coverage).toEqual([
|
|
unaccounted(HIDDEN_ENTRY_BYTES, 1, A_ENTRY_BYTES),
|
|
]);
|
|
});
|
|
|
|
it("reports bytes prepended before the first member as unknown", async () => {
|
|
const prefix = Buffer.from("#!/bin/sh\nexit 0\n");
|
|
expect((await list(zip([member("a.txt")], { prefix }))).coverage).toEqual([
|
|
unaccounted(prefix.length, 1, 0),
|
|
]);
|
|
});
|
|
|
|
it("accounts for a well-formed APK signing block before the directory", async () => {
|
|
expect(
|
|
(await list(zip([member("a.txt")], { gap: apkSigningBlock() }))).coverage,
|
|
).toEqual([complete]);
|
|
});
|
|
|
|
it("does not account for a block whose footer magic differs", async () => {
|
|
const gap = apkSigningBlock("APK Sig Block 41");
|
|
expect((await list(zip([member("a.txt")], { gap }))).coverage).toEqual([
|
|
unaccounted(gap.length, 1, A_ENTRY_BYTES),
|
|
]);
|
|
});
|
|
|
|
it("does not hide an unlisted local entry inside malformed signing-block framing", async () => {
|
|
const hidden = localEntry(member("hidden.txt", false));
|
|
const size = BigInt(hidden.length + 24);
|
|
const framing = Buffer.alloc(8);
|
|
framing.writeBigUInt64LE(size);
|
|
const block = Buffer.concat([
|
|
framing,
|
|
hidden,
|
|
framing,
|
|
Buffer.from("APK Sig Block 42"),
|
|
]);
|
|
expect(
|
|
(await list(zip([member("a.txt")], { gap: block }))).coverage,
|
|
).toEqual([unaccounted(block.length, 1, A_ENTRY_BYTES)]);
|
|
});
|
|
|
|
it("carries the unaccounted region into the artifact inventory", async () => {
|
|
const directory = await createTestTempDirectory("rea-zip-member-region-");
|
|
const path = join(directory, "fixture.zip");
|
|
await writeFile(
|
|
path,
|
|
zip([member("a.txt"), member("hidden.txt", false), member("b.txt")]),
|
|
);
|
|
const reader = new ZipArtifactReader(path, "zip");
|
|
try {
|
|
const result = await scanReader(reader, [reader]);
|
|
expect(result.occurrences.map((item) => item.logical_path)).toEqual([
|
|
"a.txt",
|
|
"b.txt",
|
|
]);
|
|
expect(result.traversal).toEqual(
|
|
unaccounted(HIDDEN_ENTRY_BYTES, 1, A_ENTRY_BYTES),
|
|
);
|
|
} finally {
|
|
await reader.close();
|
|
}
|
|
});
|
|
});
|