diff --git a/scripts/verify/baselines/raw-buffer-read.json b/scripts/verify/baselines/raw-buffer-read.json index 752cc295e3..3e69a47574 100644 --- a/scripts/verify/baselines/raw-buffer-read.json +++ b/scripts/verify/baselines/raw-buffer-read.json @@ -11,7 +11,7 @@ "src/artifacts/pyinstaller/CArchiveLayout.ts": 8, "src/domain/apkSignature/apkSignatureSchemes.ts": 1, "src/domain/apkSignature/apkSigningBlock.ts": 2, - "src/domain/apkSignature/apkV1Signature.ts": 15, + "src/domain/apkSignature/apkV1Signature.ts": 5, "src/domain/apple/appleMachoFixups.ts": 22, "src/domain/apple/appleMachoSelection.ts": 2, "src/domain/apple/diskImageFormat.ts": 4, diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.ts index a09f19c906..74232b742e 100644 --- a/src/application/BinaryTargetResolver.ts +++ b/src/application/BinaryTargetResolver.ts @@ -97,7 +97,7 @@ import { } from "../domain/firmware/espFlashLocation.js"; import { espFlashSelectionError } from "../domain/firmware/espFlashSelection.js"; import { - locateTrailingZipArchive, + hasTrailingZipArchive, zipRootPackageFormat, } from "../domain/zipPackageFormat.js"; @@ -528,9 +528,9 @@ const readOpenedBinaryTarget = async ( // executable with an appended archive keeps its executable route. if ( !(await claimsLeadingExecutableHeader(handle, metadata.size, signal)) && - (await locateTrailingZipArchive(metadata.size, (position, length) => + (await hasTrailingZipArchive(metadata.size, (position, length) => readHandleRange(handle, position, length, signal), - )) !== undefined + )) ) return ok({ ...(await artifactIdentity()), diff --git a/src/artifacts/inventory/classify.ts b/src/artifacts/inventory/classify.ts index 3e5a65658c..bf2a9f1808 100644 --- a/src/artifacts/inventory/classify.ts +++ b/src/artifacts/inventory/classify.ts @@ -53,7 +53,7 @@ import { createArtifactClassificationProbe } from "../ArtifactClassificationProb import type { ArtifactOccurrence } from "../../domain/artifactGraph.js"; import { hasZipSignature, - locateTrailingZipArchive, + hasTrailingZipArchive, zipRootPackageFormat, } from "../../domain/zipPackageFormat.js"; import { artifactFormatDescriptor } from "../artifactFormats.js"; @@ -436,7 +436,7 @@ const classifyRootFormat = async ( if ( (format === "file" || format === "unknown") && !claimsExecutableHeader(evidence.prefix, archiveBytes) && - (await locateTrailingZipArchive(archiveBytes, read)) !== undefined + (await hasTrailingZipArchive(archiveBytes, read)) ) return zipRootPackageFormat(lower); if (["elf", "pe", "mach-o", "mach-o-universal"].includes(format)) { diff --git a/src/domain/apkSignature/apkV1Signature.ts b/src/domain/apkSignature/apkV1Signature.ts index 9b5cca9eee..05dd964d48 100644 --- a/src/domain/apkSignature/apkV1Signature.ts +++ b/src/domain/apkSignature/apkV1Signature.ts @@ -17,7 +17,12 @@ * names decoded. */ import { boundedList } from "../boundedList.js"; +import { BinaryReader } from "../binaryContainer.js"; import { type Result, err, ok } from "../result.js"; +import { + ZIP_CENTRAL_HEADER, + readZipCentralExtent, +} from "../zip/zipCentralHeader.js"; import { canonicalNameHash, compareCanonicalNames, @@ -36,9 +41,6 @@ import { v1SignatureFileNameForBlock, } from "./apkV1MetadataName.js"; -const CD_SIGNATURE = 0x02014b50; -const CD_FIXED_SIZE = 46; - export type ApkCentralDirectoryError = | { readonly kind: "malformed-central-directory"; @@ -263,7 +265,6 @@ export const centralDirectoryV1SignatureEntries = ( totalEntries: number, apksigRecords: number = totalEntries, ): Result => { - const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); const end = centralDirectoryOffset + centralDirectorySize; if ( !Number.isSafeInteger(end) || @@ -276,39 +277,43 @@ export const centralDirectoryV1SignatureEntries = ( offset: centralDirectoryOffset, detail: "declared central-directory extent is outside the file", }); + const reader = new BinaryReader( + Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength), + true, + "zip", + ); // A record takes at least its fixed bytes, so the walk fails before a // declared count past this capacity is reached. const capacity = Math.min( totalEntries, - Math.max(0, Math.floor((end - centralDirectoryOffset) / CD_FIXED_SIZE)), + Math.max( + 0, + Math.floor((end - centralDirectoryOffset) / ZIP_CENTRAL_HEADER.bytes), + ), ); const starts = new Uint32Array(capacity); const lengths = new Uint16Array(capacity); const entries: ApkV1Entry[] = []; let at = centralDirectoryOffset; for (let i = 0; i < totalEntries; i += 1) { - if (at + CD_FIXED_SIZE > end) + const extent = readZipCentralExtent(reader, at, end); + if (!extent.ok) { + const { kind, offset } = extent.error; + const detail = { + "fixed-fields": `central directory ends after ${String(i)} of ${String(totalEntries)} records`, + signature: "central-directory record signature mismatch", + name: "central-directory file name runs past the directory", + "extra-or-comment": + "central-directory extra field or comment runs past the directory", + }[kind]; return err({ kind: "malformed-central-directory", - offset: at, - detail: `central directory ends after ${String(i)} of ${String(totalEntries)} records`, - }); - if (dv.getUint32(at, true) !== CD_SIGNATURE) - return err({ - kind: "malformed-central-directory", - offset: at, - detail: "central-directory record signature mismatch", - }); - const nameLength = dv.getUint16(at + 28, true); - const extraLength = dv.getUint16(at + 30, true); - const commentLength = dv.getUint16(at + 32, true); - const nameStart = at + CD_FIXED_SIZE; - if (nameStart + nameLength > end) - return err({ - kind: "malformed-central-directory", - offset: nameStart, - detail: "central-directory file name runs past the directory", + offset, + detail, }); + } + const { name: nameStart, extra, next } = extent.value; + const nameLength = extra - nameStart; starts[i] = nameStart; lengths[i] = nameLength; if (startsWithV1MetaInfPrefix(bytes, nameStart, nameLength)) { @@ -324,16 +329,18 @@ export const centralDirectoryV1SignatureEntries = ( entries.push({ name, rawName, - flags: dv.getUint16(at + 8, true), - method: dv.getUint16(at + 10, true), - crc32: dv.getUint32(at + 16, true), - compressedSize: dv.getUint32(at + 20, true), - uncompressedSize: dv.getUint32(at + 24, true), - localHeaderOffset: dv.getUint32(at + 42, true), + flags: reader.u16(at + 8), + method: reader.u16(at + 10), + crc32: reader.u32(at + 16), + compressedSize: reader.u32(at + 20), + uncompressedSize: reader.u32(at + 24), + localHeaderOffset: reader.u32( + at + ZIP_CENTRAL_HEADER.localHeaderOffset, + ), }); } } - at = nameStart + nameLength + extraLength + commentLength; + at = next; } // An under-declared record count would silently hide entries (and so v1 // signing): the walk must consume exactly the declared directory. diff --git a/src/domain/desktop/appExportPythonZip.ts b/src/domain/desktop/appExportPythonZip.ts index d1422373dd..4688451b18 100644 --- a/src/domain/desktop/appExportPythonZip.ts +++ b/src/domain/desktop/appExportPythonZip.ts @@ -5,6 +5,7 @@ import { ZIP64_END, ZIP64_LOCATOR, } from "../zipEndOfCentralDirectory.js"; +import { ZIP_CENTRAL_HEADER } from "../zip/zipCentralHeader.js"; import { ZIP_EXTRA_FIELD } from "../zip/zipExtraFields.js"; /** @@ -18,7 +19,8 @@ ZIP_READ_CHUNK = 64 * 1024 #: Small, since the end of each gzip member copies what is left of the piece it was read from. GZIP_READ_CHUNK = 4 * 1024 ZIP_END = b"PK\x05\x06" -ZIP_CENTRAL = b"PK\x01\x02" +ZIP_CENTRAL_LAYOUT = ${JSON.stringify(ZIP_CENTRAL_HEADER)} +ZIP_CENTRAL = ZIP_CENTRAL_LAYOUT["signature"].to_bytes(4, "little") ZIP64_LOCATOR = b"PK\x06\x07" ZIP64_END = b"PK\x06\x06" ZIP64_END_LAYOUT = ${JSON.stringify(ZIP64_END)} @@ -196,7 +198,7 @@ def _zip_kept_record(fixed: bytes, raw: bytes, extra, comment: bytes): occurrence, retaining an earlier valid name or rejecting invalid UTF-8 that zip.js already superseded. All other fields, comments and the original artifact bytes remain unchanged.""" header = bytearray(fixed) - struct.pack_into(" end or len(fixed) < 46 or fixed[:4] != ZIP_CENTRAL: + if at + fixed_bytes > end: + return None + fixed = file.read(fixed_bytes) + if len(fixed) < fixed_bytes or fixed[:4] != ZIP_CENTRAL: return None made_by, flags = struct.unpack_from(" end: + variable_bytes = name_length + extra_length + comment_length + at += fixed_bytes + variable_bytes + if at > end: + return None + rest = file.read(variable_bytes) + if len(rest) < variable_bytes: return None seen += 1 extra_end = name_length + extra_length diff --git a/src/domain/desktop/zipDirectory.ts b/src/domain/desktop/zipDirectory.ts index 1e6e7f5d0e..6b7370c5dd 100644 --- a/src/domain/desktop/zipDirectory.ts +++ b/src/domain/desktop/zipDirectory.ts @@ -14,10 +14,9 @@ import { zipEndUsesSingleVolume, } from "../zipEndOfCentralDirectory.js"; import { strictUtf8Decoder } from "../strictUtf8.js"; +import { readZipCentralExtent } from "../zip/zipCentralHeader.js"; import { ZIP_EXTRA_FIELD, lastZipExtraField } from "../zip/zipExtraFields.js"; -const CENTRAL_HEADER = 0x02014b50; -const CENTRAL_HEADER_BYTES = 46; const UTF8_NAME = 0x800; const UNIX_TYPE = 0o170000; const UNIX_DIRECTORY = 0o040000; @@ -172,16 +171,9 @@ export const scanZipDirectory = ( let count = 0; let seen = 0; for (let at = start; at < located.end;) { - if ( - at + CENTRAL_HEADER_BYTES > located.end || - reader.u32(at) !== CENTRAL_HEADER - ) - return undefined; - const name = at + CENTRAL_HEADER_BYTES; - const extra = name + reader.u16(at + 28); - const comment = extra + reader.u16(at + 30); - const next = comment + reader.u16(at + 32); - if (next > located.end) return undefined; + const extent = readZipCentralExtent(reader, at, located.end); + if (!extent.ok) return undefined; + const { name, extra, comment, next } = extent.value; const slash = endsInSlash(reader, { name, extra, diff --git a/src/domain/zip/zipCentralHeader.ts b/src/domain/zip/zipCentralHeader.ts new file mode 100644 index 0000000000..bd79e200c2 --- /dev/null +++ b/src/domain/zip/zipCentralHeader.ts @@ -0,0 +1,47 @@ +import type { BinaryReader } from "../binaryContainer.js"; +import { type Result, err, ok } from "../result.js"; + +/** ZIP APPNOTE ยง4.3.12. The three variable fields follow the fixed header in order. */ +export const ZIP_CENTRAL_HEADER = { + signature: 0x02014b50, + bytes: 46, + nameBytesOffset: 28, + extraBytesOffset: 30, + commentBytesOffset: 32, + localHeaderOffset: 42, +} as const; + +type ZipCentralExtent = { + readonly name: number; + readonly extra: number; + readonly comment: number; + readonly next: number; +}; + +type ZipCentralExtentError = { + readonly kind: "fixed-fields" | "signature" | "name" | "extra-or-comment"; + readonly offset: number; +}; + +/** + * Admit the entire record against the containing directory. A positional + * probe can supply just the fixed bytes, which are all this calculation reads. + */ +export const readZipCentralExtent = ( + reader: BinaryReader, + at: number, + end: number, +): Result => { + const layout = ZIP_CENTRAL_HEADER; + const name = at + layout.bytes; + if (at < 0 || name > end || name > reader.bytes.length) + return err({ kind: "fixed-fields", offset: at }); + if (reader.u32(at) !== layout.signature) + return err({ kind: "signature", offset: at }); + const extra = name + reader.u16(at + layout.nameBytesOffset); + if (extra > end) return err({ kind: "name", offset: name }); + const comment = extra + reader.u16(at + layout.extraBytesOffset); + const next = comment + reader.u16(at + layout.commentBytesOffset); + if (next > end) return err({ kind: "extra-or-comment", offset: extra }); + return ok({ name, extra, comment, next }); +}; diff --git a/src/domain/zipPackageFormat.test.ts b/src/domain/zipPackageFormat.test.ts index 88576c6fbe..caa1047664 100644 --- a/src/domain/zipPackageFormat.test.ts +++ b/src/domain/zipPackageFormat.test.ts @@ -1,17 +1,18 @@ import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; import { expect, it } from "vitest"; -import { locateTrailingZipArchive } from "./zipPackageFormat.js"; +import { hasTrailingZipArchive } from "./zipPackageFormat.js"; -it("requires the first local ZIP header and its variable fields to fit before the directory", async () => { +it("requires the first trailing ZIP directory record and its local header to fit their regions", async () => { const writer = new ZipWriter(new Uint8ArrayWriter()); await writer.add("a.txt", new TextReader("hello")); const zip = Buffer.from(await writer.close()); const prefix = Buffer.from("#!/bin/sh\nexit 0\n"); + const directoryBytes = zip.readUInt32LE(zip.length - 22 + 12); const directory = zip.readUInt32LE(zip.length - 22 + 16); - const locate = async (archive: Buffer) => { + const probe = async (archive: Buffer) => { const file = Buffer.concat([prefix, archive]); - return locateTrailingZipArchive(file.length, async (start, length) => + return hasTrailingZipArchive(file.length, async (start, length) => file.subarray(start, start + length), ); }; @@ -21,8 +22,11 @@ it("requires the first local ZIP header and its variable fields to fit before th overlappingFixedHeader.writeUInt32LE(directory - 4, directory + 42); const overlappingExtraField = Buffer.from(zip); overlappingExtraField.writeUInt16LE(directory, 28); + const overlappingComment = Buffer.from(zip); + overlappingComment.writeUInt16LE(directoryBytes, directory + 32); - expect(await locate(zip)).toEqual({ leadingBytes: prefix.length }); - expect(await locate(overlappingFixedHeader)).toBeUndefined(); - expect(await locate(overlappingExtraField)).toBeUndefined(); + expect(await probe(zip)).toBe(true); + expect(await probe(overlappingFixedHeader)).toBe(false); + expect(await probe(overlappingExtraField)).toBe(false); + expect(await probe(overlappingComment)).toBe(false); }); diff --git a/src/domain/zipPackageFormat.ts b/src/domain/zipPackageFormat.ts index 3b7a519fa5..2a7a03a8a6 100644 --- a/src/domain/zipPackageFormat.ts +++ b/src/domain/zipPackageFormat.ts @@ -1,4 +1,8 @@ import { BinaryReader } from "./binaryContainer.js"; +import { + ZIP_CENTRAL_HEADER, + readZipCentralExtent, +} from "./zip/zipCentralHeader.js"; import { ZIP_LOCAL_HEADER_BYTES, zipLocalDataOffset, @@ -53,9 +57,6 @@ export const hasZipSignature = (bytes: Uint8Array): boolean => signature.every((byte, index) => bytes[index] === byte), ); -const CENTRAL_DIRECTORY_HEADER = 0x02014b50; -const CENTRAL_HEADER_BYTES = 46; - /** Positional read of at most `length` bytes; a short result means end of file. */ export type PositionalRead = ( position: number, @@ -63,20 +64,20 @@ export type PositionalRead = ( ) => Promise; /** - * Locate a single-disk ZIP whose end-of-central-directory record ends the + * Recognize a single-disk ZIP whose end-of-central-directory record ends the * file after leading non-ZIP bytes, such as a self-extractor or script stub. - * Its central directory and first local header must resolve, whether member - * offsets include the leading bytes or were written before them. Zip64, - * split and empty archives are not located. Returns the leading byte count. + * The first directory entry and the local header it refers to must fit their + * respective regions. The probe handles adjusted and unadjusted member offsets + * for nonempty classic ZIPs on a single volume. */ -export const locateTrailingZipArchive = async ( +export const hasTrailingZipArchive = async ( size: number, read: PositionalRead, -): Promise<{ readonly leadingBytes: number } | undefined> => { - if (!Number.isSafeInteger(size) || size < ZIP_EOCD_BYTES) return undefined; +): Promise => { + if (!Number.isSafeInteger(size) || size < ZIP_EOCD_BYTES) return false; const tailStart = Math.max(0, size - ZIP_EOCD_BYTES - ZIP_MAX_COMMENT_BYTES); const tail = Buffer.from(await read(tailStart, size - tailStart)); - if (tail.length !== size - tailStart) return undefined; + if (tail.length !== size - tailStart) return false; // A comment may contain EOCD-like bytes; validate each candidate's directory. for (const candidate of zipEndOfCentralDirectoryCandidates(tail)) { const { @@ -90,6 +91,7 @@ export const locateTrailingZipArchive = async ( entriesOnDisk === 0 || entriesOnDisk === 0xffff || entriesOnDisk !== candidate.totalEntries || + directoryBytes < ZIP_CENTRAL_HEADER.bytes || directoryBytes === 0xffffffff || declaredDirectory === 0xffffffff ) @@ -98,25 +100,21 @@ export const locateTrailingZipArchive = async ( // Offsets written before leading bytes were prepended are shifted by them. const shift = directory - declaredDirectory; if (directory < 0 || shift < 0) continue; - const header = Buffer.from(await read(directory, CENTRAL_HEADER_BYTES)); + const header = Buffer.from(await read(directory, ZIP_CENTRAL_HEADER.bytes)); const headerReader = new BinaryReader(header, true, "zip"); - if ( - header.length !== CENTRAL_HEADER_BYTES || - headerReader.u32(0) !== CENTRAL_DIRECTORY_HEADER - ) + if (!readZipCentralExtent(headerReader, 0, directoryBytes).ok) continue; + const localOffset = + headerReader.u32(ZIP_CENTRAL_HEADER.localHeaderOffset) + shift; + if (localOffset === 0 || localOffset > directory - ZIP_LOCAL_HEADER_BYTES) continue; - const firstLocal = headerReader.u32(42) + shift; - if (firstLocal === 0 || firstLocal > directory - ZIP_LOCAL_HEADER_BYTES) - continue; - const local = Buffer.from(await read(firstLocal, ZIP_LOCAL_HEADER_BYTES)); + const local = Buffer.from(await read(localOffset, ZIP_LOCAL_HEADER_BYTES)); const dataOffset = zipLocalDataOffset( new BinaryReader(local, true, "zip"), - firstLocal, + localOffset, ); - if (dataOffset !== undefined && dataOffset <= directory) - return { leadingBytes: firstLocal }; + if (dataOffset !== undefined && dataOffset <= directory) return true; } - return undefined; + return false; }; /**