diff --git a/src/application/AppBundleExecutable.ts b/src/application/AppBundleExecutable.ts index 7df3ca03aa..0ae1c073e4 100644 --- a/src/application/AppBundleExecutable.ts +++ b/src/application/AppBundleExecutable.ts @@ -1,5 +1,5 @@ import { execFile } from "node:child_process"; -import { lstat, readdir, readFile, realpath, stat } from "node:fs/promises"; +import { lstat, opendir, readFile, realpath, stat } from "node:fs/promises"; import { join } from "node:path"; import { promisify } from "node:util"; @@ -156,13 +156,14 @@ const wrappedBundle = async (bundle: string): Promise => { if (isAbsence(cause)) return undefined; throw cause; } - const apps = (await readdir(wrapper, { withFileTypes: true })).filter( - (entry) => entry.isDirectory() && entry.name.toLowerCase().endsWith(".app"), - ); - const [app] = apps; - return apps.length === 1 && app !== undefined - ? join(wrapper, app.name) - : undefined; + let app: string | undefined; + for await (const entry of await opendir(wrapper)) { + if (!entry.isDirectory() || !entry.name.toLowerCase().endsWith(".app")) + continue; + if (app !== undefined) return undefined; + app = join(wrapper, entry.name); + } + return app; }; const resolveLayoutExecutable = async ( diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.ts index 3163d17b39..8b532b0d54 100644 --- a/src/application/BinaryTargetResolver.ts +++ b/src/application/BinaryTargetResolver.ts @@ -1,4 +1,4 @@ -import { constants } from "node:fs"; +import { constants, type Stats } from "node:fs"; import { createHash } from "node:crypto"; import { access, @@ -248,7 +248,7 @@ const isArchiveFormat = ( const sha256Handle = async ( handle: FileHandle, path: string, - initial: Awaited>, + initial: Stats, signal?: AbortSignal, ): Promise => { const hash = createHash("sha256"); @@ -256,11 +256,18 @@ const sha256Handle = async ( let position = 0; while (true) { throwIfTargetResolutionCancelled(signal); - const observed = await handle.read(chunk, 0, chunk.length, position); + const observed = await handle.read( + chunk, + 0, + Math.min(chunk.length, initial.size - position + 1), + position, + ); throwIfTargetResolutionCancelled(signal); if (observed.bytesRead === 0) break; hash.update(chunk.subarray(0, observed.bytesRead)); position += observed.bytesRead; + if (position > initial.size) + throw new BinaryTargetError(path, "target grew while being read"); } throwIfTargetResolutionCancelled(signal); const [opened, currentPath] = await Promise.all([handle.stat(), lstat(path)]); diff --git a/src/application/JsonFiles.ts b/src/application/JsonFiles.ts index 0ccca760ab..916d4fd61b 100644 --- a/src/application/JsonFiles.ts +++ b/src/application/JsonFiles.ts @@ -42,7 +42,7 @@ export const readJsonFile = async ( if (!decoded.ok) { return err( new EvidenceFileError("read", "invalid-json", { - cause: decoded.cause, + cause: decoded.cause ?? new SyntaxError(decoded.error), path: requestedPath, }), ); diff --git a/src/artifacts/AsarArtifactReader.ts b/src/artifacts/AsarArtifactReader.ts index bedd9a5f20..80e9fec0f2 100644 --- a/src/artifacts/AsarArtifactReader.ts +++ b/src/artifacts/AsarArtifactReader.ts @@ -27,6 +27,7 @@ import { type ArtifactReader, } from "./ArtifactReader.js"; import { closeAsarHandle, readValidatedAsarEntry } from "./AsarEntryStream.js"; +import { admitAsarHeader } from "./AsarHeader.js"; /** * Official Electron ASAR adapter with range-streamed member reads. @@ -57,6 +58,17 @@ export class AsarArtifactReader implements ArtifactReader { const snapshotPath = this.#snapshotPath(); this.#resetArchiveState(); this.#entries.clear(); + const headerHandle = await open( + snapshotPath, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + try { + const admitted = await admitAsarHeader(headerHandle); + this.#archiveSize = admitted.archiveBytes; + } finally { + await headerHandle.close(); + } + abortIfNeeded(signal); paths = listPackage(snapshotPath, { isPack: false }).sort((left, right) => left.localeCompare(right, "en"), ); diff --git a/src/artifacts/AsarHeader.ts b/src/artifacts/AsarHeader.ts new file mode 100644 index 0000000000..7c17f1cc76 --- /dev/null +++ b/src/artifacts/AsarHeader.ts @@ -0,0 +1,48 @@ +import type { FileHandle } from "node:fs/promises"; + +import { ArtifactReaderFailure } from "./ArtifactReader.js"; + +/** Shared resource budget for the eagerly decoded ASAR header, not member bytes. */ +export const MAX_ASAR_HEADER_BYTES = 16 * 1024 * 1024; + +/** Admit both Chromium pickle extents before an upstream decoder allocates them. */ +export const admitAsarHeader = async ( + handle: FileHandle, +): Promise<{ + readonly archiveBytes: number; + readonly headerBytes: number; + readonly jsonBytes: number; +}> => { + const observed = await handle.stat(); + if (!observed.isFile()) + throw new ArtifactReaderFailure( + "format", + "ASAR container is not a regular file", + ); + const prefix = Buffer.alloc(16); + const read = await handle.read(prefix, 0, prefix.length, 0); + if (read.bytesRead !== prefix.length || prefix.readUInt32LE(0) !== 4) + throw new ArtifactReaderFailure( + "format", + "Truncated or invalid ASAR size pickle", + ); + const headerBytes = prefix.readUInt32LE(4); + const payloadBytes = prefix.readUInt32LE(8); + const jsonBytes = prefix.readUInt32LE(12); + if ( + headerBytes < 8 || + headerBytes > observed.size - 8 || + payloadBytes !== headerBytes - 4 || + jsonBytes > payloadBytes - 4 + ) + throw new ArtifactReaderFailure( + "format", + "ASAR header leaves its container or has invalid pickle extents", + ); + if (headerBytes > MAX_ASAR_HEADER_BYTES) + throw new ArtifactReaderFailure( + "limit", + `ASAR header exceeds the ${String(MAX_ASAR_HEADER_BYTES)} byte decode budget`, + ); + return { archiveBytes: observed.size, headerBytes, jsonBytes }; +}; diff --git a/src/artifacts/DirectoryArtifactReader.ts b/src/artifacts/DirectoryArtifactReader.ts index a9e339be52..123a1cb092 100644 --- a/src/artifacts/DirectoryArtifactReader.ts +++ b/src/artifacts/DirectoryArtifactReader.ts @@ -3,15 +3,13 @@ import { lstat, open, opendir, realpath } from "node:fs/promises"; import { isAbsolute, join, relative, sep } from "node:path"; import type { Readable } from "node:stream"; -import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; - import { ArtifactReaderFailure, type ArtifactEntry, type ArtifactReader, } from "./ArtifactReader.js"; -/** Deterministic, non-symlink-following directory reader. */ +/** Non-symlink-following directory reader. */ export class DirectoryArtifactReader implements ArtifactReader { readonly format = "directory" as const; readonly #rootPromise: Promise; @@ -38,13 +36,52 @@ export class DirectoryArtifactReader implements ArtifactReader { await withDirectoryIoContext("inspect directory", directory, () => assertContainedDirectory(root, directory), ); - const children: string[] = []; const handle = await opendir(directory).catch((cause: unknown) => { throw directoryIoFailure("open directory", directory, cause); }); + const directories: string[] = []; try { - for await (const child of handle) children.push(child.name); + for await (const child of handle) { + abortIfNeeded(signal); + const absolute = join(directory, child.name); + const metadata = await lstat(absolute).catch((cause: unknown) => { + throw directoryIoFailure("inspect entry", absolute, cause); + }); + const path = relative(root, absolute).split(sep).join("/"); + const kind = metadata.isSymbolicLink() + ? "symlink" + : metadata.isDirectory() + ? "directory" + : metadata.isFile() + ? "file" + : undefined; + if (kind === undefined) continue; + yield { + path, + kind, + declaredSize: kind === "file" ? metadata.size : null, + compressedSize: null, + executable: (metadata.mode & 0o111) !== 0, + encrypted: false, + byteOffset: null, + declaredSha256: null, + unpacked: false, + limitations: + kind === "symlink" + ? ["Symlink target was not followed or disclosed."] + : [], + adapterKey: absolute, + ...(kind === "file" + ? { + sourceIdentity: { device: metadata.dev, inode: metadata.ino }, + } + : {}), + }; + if (kind === "directory" && (descendInto?.(path) ?? true)) + directories.push(absolute); + } } catch (cause: unknown) { + if (cause instanceof ArtifactReaderFailure) throw cause; throw directoryIoFailure("read directory", directory, cause); } finally { // best-effort cleanup: directory-handle close must not mask traversal. @@ -53,45 +90,6 @@ export class DirectoryArtifactReader implements ArtifactReader { await withDirectoryIoContext("inspect directory", directory, () => assertContainedDirectory(root, directory), ); - children.sort(compareUnicodeCodePoints); - const directories: string[] = []; - for (const name of children) { - abortIfNeeded(signal); - const absolute = join(directory, name); - const metadata = await lstat(absolute).catch((cause: unknown) => { - throw directoryIoFailure("inspect entry", absolute, cause); - }); - const path = relative(root, absolute).split(sep).join("/"); - const kind = metadata.isSymbolicLink() - ? "symlink" - : metadata.isDirectory() - ? "directory" - : metadata.isFile() - ? "file" - : undefined; - if (kind === undefined) continue; - if (kind === "directory" && (descendInto?.(path) ?? true)) - directories.push(absolute); - yield { - path, - kind, - declaredSize: kind === "file" ? metadata.size : null, - compressedSize: null, - executable: (metadata.mode & 0o111) !== 0, - encrypted: false, - byteOffset: null, - declaredSha256: null, - unpacked: false, - limitations: - kind === "symlink" - ? ["Symlink target was not followed or disclosed."] - : [], - adapterKey: absolute, - ...(kind === "file" - ? { sourceIdentity: { device: metadata.dev, inode: metadata.ino } } - : {}), - }; - } directories.reverse(); pending.push(...directories); } diff --git a/src/artifacts/SafeOutputTree.ts b/src/artifacts/SafeOutputTree.ts index 05ce0ac680..d9610192eb 100644 --- a/src/artifacts/SafeOutputTree.ts +++ b/src/artifacts/SafeOutputTree.ts @@ -105,7 +105,26 @@ export class SafeOutputTree { } return new SafeOutputTree(canonicalOutput, platform); } catch (cause: unknown) { - await rm(canonicalOutput, { recursive: true, force: true }); + let removalFailure: unknown; + try { + await rm(canonicalOutput, { recursive: true, force: true }); + } catch (cleanupCause: unknown) { + removalFailure = cleanupCause; + } + let absent = false; + try { + absent = await isAbsent(canonicalOutput); + } catch (cleanupCause: unknown) { + removalFailure ??= cleanupCause; + } + if (!absent) + throw ArtifactReaderFailure.withCleanup(cause, { + reason: + removalFailure instanceof Error + ? removalFailure.message + : "Extraction output root remains after setup failure", + resources: [canonicalOutput], + }); throw cause; } } @@ -122,56 +141,84 @@ export class SafeOutputTree { async write( relativePath: string, source: Readable, - expectedSha256: string, + expected: { readonly sha256: string; readonly bytes: number }, signal?: AbortSignal, ): Promise { - this.#assertWritable(); - const path = normalizeArtifactPath(relativePath); - this.#registry.add(path, "file"); - const destination = await this.#prepareParent(path); - const handle = await open( - destination, - constants.O_CREAT | - constants.O_EXCL | - constants.O_WRONLY | - constants.O_NOFOLLOW, - 0o600, - ).catch(async (cause: unknown) => { - await throwIfDestinationCaseCollision(dirname(destination), path, cause); - throw new ArtifactReaderFailure( - "path", - `Could not exclusively create extraction path: ${path}`, - { cause }, - ); - }); - const hash = createHash("sha256"); - let bytes = 0; try { - for await (const raw of source) { - abortIfNeeded(signal); - const chunk = streamChunkToBuffer(raw); - bytes += chunk.length; - hash.update(chunk); - await writeAll(handle, chunk); + this.#assertWritable(); + if (!Number.isSafeInteger(expected.bytes) || expected.bytes < 0) + throw new ArtifactReaderFailure( + "format", + `Invalid expected extraction size for ${relativePath}`, + ); + const path = normalizeArtifactPath(relativePath); + this.#registry.add(path, "file"); + const destination = await this.#prepareParent(path); + const handle = await open( + destination, + constants.O_CREAT | + constants.O_EXCL | + constants.O_WRONLY | + constants.O_NOFOLLOW, + 0o600, + ).catch(async (cause: unknown) => { + await throwIfDestinationCaseCollision( + dirname(destination), + path, + cause, + ); + throw new ArtifactReaderFailure( + "path", + `Could not exclusively create extraction path: ${path}`, + { cause }, + ); + }); + const hash = createHash("sha256"); + let bytes = 0; + try { + for await (const raw of source) { + abortIfNeeded(signal); + const chunk = streamChunkToBuffer(raw); + if (chunk.length > expected.bytes - bytes) + throw new ArtifactReaderFailure( + "integrity", + `Extracted content exceeds the inventoried size: ${path}`, + ); + bytes += chunk.length; + hash.update(chunk); + await writeAll(handle, chunk); + } + if (bytes !== expected.bytes) + throw new ArtifactReaderFailure( + "integrity", + `Extracted content size disagrees with inventory: ${path}`, + ); + const sha256 = hash.digest("hex"); + if (sha256 !== expected.sha256) + throw new ArtifactReaderFailure( + "integrity", + `Extracted content disagrees with inventory: ${path}`, + ); + await handle.sync(); + await handle.close(); + const readback = await hashFile(destination, bytes, signal); + if (readback.sha256 !== sha256 || readback.bytes !== bytes) + throw new ArtifactReaderFailure( + "integrity", + `Durable readback verification failed: ${path}`, + ); + return { relativePath: path, sha256, bytesWritten: bytes }; + } catch (cause: unknown) { + // best-effort cleanup: file-handle close must not mask the write failure. + await handle.close().catch(() => undefined); + throw cause; } - const sha256 = hash.digest("hex"); - if (sha256 !== expectedSha256) - throw new ArtifactReaderFailure( - "integrity", - `Extracted content disagrees with inventory: ${path}`, - ); - await handle.sync(); - await handle.close(); - const readback = await hashFile(destination, bytes, signal); - if (readback.sha256 !== sha256 || readback.bytes !== bytes) - throw new ArtifactReaderFailure( - "integrity", - `Durable readback verification failed: ${path}`, - ); - return { relativePath: path, sha256, bytesWritten: bytes }; } catch (cause: unknown) { - // best-effort cleanup: file-handle close must not mask the write failure. - await handle.close().catch(() => undefined); + try { + source.destroy(); + } catch { + // Preserve the write refusal or failure as the caller-visible error. + } throw cause; } } diff --git a/src/artifacts/ZipArtifactReader.ts b/src/artifacts/ZipArtifactReader.ts index 3c7a92ea1b..17637ca45a 100644 --- a/src/artifacts/ZipArtifactReader.ts +++ b/src/artifacts/ZipArtifactReader.ts @@ -1,4 +1,4 @@ -import { open, type FileHandle } from "node:fs/promises"; +import type { FileHandle } from "node:fs/promises"; import { PassThrough, type Readable } from "node:stream"; import { once } from "node:events"; @@ -10,20 +10,57 @@ import { type ArtifactReader, } from "./ArtifactReader.js"; import type { ZipPackageFormat } from "../domain/zipPackageFormat.js"; +import { + NonRegularFileReadError, + openRegularFile, + sameRegularFileState, + type StableRegularFileDescriptor, +} from "../filesystem/RegularFile.js"; class NodeFileReader extends Reader { #handle: FileHandle | undefined; + #initPromise: Promise | undefined; + readonly #ownsHandle: boolean; constructor( private readonly path: string, private readonly maximumReadBytes?: number, + private readonly admitted?: StableRegularFileDescriptor, ) { super(path); + this.#ownsHandle = admitted === undefined; } - override async init(): Promise { - this.#handle = await open(this.path, "r"); - this.size = (await this.#handle.stat()).size; + override init(): Promise { + this.#initPromise ??= this.#initialize(); + return this.#initPromise; + } + + async #initialize(): Promise { + try { + this.#handle = + this.admitted?.handle ?? + (await openRegularFile(this.path, { symlinks: "reject" })); + } catch (cause: unknown) { + if (cause instanceof NonRegularFileReadError) + throw new ArtifactReaderFailure( + "format", + `ZIP source is not a regular file: ${this.path}`, + { cause }, + ); + throw cause; + } + const metadata = await this.#handle.stat(); + if ( + !metadata.isFile() || + (this.admitted !== undefined && + !sameRegularFileState(this.admitted.initial, metadata)) + ) + throw new ArtifactReaderFailure( + "integrity", + `ZIP source identity changed before reading: ${this.path}`, + ); + this.size = metadata.size; } override async readUint8Array( @@ -47,7 +84,10 @@ class NodeFileReader extends Reader { } async closeHandle(): Promise { - await this.#handle?.close(); + if (!this.#ownsHandle) return; + await this.#initPromise?.catch(() => undefined); + if (this.#handle === undefined) return; + await this.#handle.close(); this.#handle = undefined; } } @@ -64,9 +104,10 @@ export class ZipArtifactReader implements ArtifactReader { path: string, format: ZipPackageFormat, maximumMetadataReadBytes?: number, + admitted?: StableRegularFileDescriptor, ) { this.format = format; - this.#source = new NodeFileReader(path, maximumMetadataReadBytes); + this.#source = new NodeFileReader(path, maximumMetadataReadBytes, admitted); this.#reader = new ZipReader(this.#source, { checkSignature: true, checkOverlappingEntry: true, diff --git a/src/artifacts/apple/InterfaceBuilderAnalysis.ts b/src/artifacts/apple/InterfaceBuilderAnalysis.ts index f47c5c56c9..e15a81b445 100644 --- a/src/artifacts/apple/InterfaceBuilderAnalysis.ts +++ b/src/artifacts/apple/InterfaceBuilderAnalysis.ts @@ -25,6 +25,7 @@ import { type NibHierarchyNode, } from "./NibViewHierarchy.js"; import { jsonValueSchema, type JsonValue } from "../../domain/jsonValue.js"; +import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; import { decodeXmlPlistText } from "../../domain/propertyListXmlText.js"; @@ -49,7 +50,6 @@ export const analyzeInterfaceBuilderBundle = async (input: { const prototypeKeyOmissions = new Map(); let omitted = 0; let attempted = 0; - let candidates = 0; let aggregateInputBytes = 0; const decodeBudget = new InterfaceBuilderDecodeBudget( MAX_AGGREGATE_DECODE_BYTES, @@ -60,15 +60,16 @@ export const analyzeInterfaceBuilderBundle = async (input: { else unreportedOmittedArchives += 1; }; try { - for await (const entry of reader.entries(input.signal)) { - if (entry.kind !== "file" || !isInterfaceBuilderArchive(entry.path)) - continue; - candidates += 1; - if (candidates > limits.max_documents) { - omitted += 1; - recordOmittedArchive(entry.path, "max_documents"); - continue; - } + const selection = await selectInterfaceBuilderCandidates( + reader, + input.signal, + limits.max_documents, + ); + omitted = selection.omitted; + for (const entry of selection.omittedSamples) + recordOmittedArchive(entry.path, "max_documents"); + unreportedOmittedArchives += selection.unreportedOmitted; + for (const entry of selection.selected) { try { const remainingInputBytes = MAX_AGGREGATE_INPUT_BYTES - aggregateInputBytes; @@ -167,6 +168,60 @@ export const analyzeInterfaceBuilderBundle = async (input: { }); }; +/** Select lexically earliest document paths independent of host traversal order. */ +const selectInterfaceBuilderCandidates = async ( + reader: DirectoryArtifactReader, + signal: AbortSignal | undefined, + maximumDocuments: number, +): Promise<{ + readonly selected: readonly ArtifactEntry[]; + readonly omittedSamples: readonly ArtifactEntry[]; + readonly omitted: number; + readonly unreportedOmitted: number; +}> => { + const capacity = maximumDocuments + MAX_REPORTED_OMITTED_ARCHIVES; + const candidates: ArtifactEntry[] = []; + let total = 0; + for await (const entry of reader.entries(signal)) { + if (entry.kind !== "file" || !isInterfaceBuilderArchive(entry.path)) + continue; + total += 1; + retainCandidate(candidates, entry, capacity); + } + const omittedSamples = candidates.slice(maximumDocuments, capacity); + return { + selected: candidates.slice(0, maximumDocuments), + omittedSamples, + omitted: Math.max(0, total - maximumDocuments), + unreportedOmitted: Math.max( + 0, + total - maximumDocuments - omittedSamples.length, + ), + }; +}; + +/** Keep the earliest bounded path set; directory iteration order is host-defined. */ +const retainCandidate = ( + candidates: ArtifactEntry[], + entry: ArtifactEntry, + maximum: number, +): void => { + let low = 0; + let high = candidates.length; + while (low < high) { + const middle = Math.floor((low + high) / 2); + const existing = candidates[middle]; + if ( + existing !== undefined && + compareUnicodeCodePoints(existing.path, entry.path) < 0 + ) + low = middle + 1; + else high = middle; + } + candidates.splice(low, 0, entry); + if (candidates.length > maximum) candidates.pop(); +}; + const finalizeHierarchyCoverage = ( result: ReturnType, counts: { diff --git a/src/artifacts/apple/KeyedArchiveReader.ts b/src/artifacts/apple/KeyedArchiveReader.ts index ec50ab4cf8..3a24f09622 100644 --- a/src/artifacts/apple/KeyedArchiveReader.ts +++ b/src/artifacts/apple/KeyedArchiveReader.ts @@ -147,29 +147,28 @@ export const inspectBundleKeyedArchive = async (input: { try { // Match inventory spelling while retaining the raw path for I/O and evidence. const wanted = selected.path.normalize("NFC"); - const matches: ArtifactEntry[] = []; - for await (const entry of reader.entries(input.signal, (directory) => + let entry: ArtifactEntry | undefined; + for await (const candidate of reader.entries(input.signal, (directory) => wanted.startsWith(`${directory.normalize("NFC")}/`), )) { - if (entry.path.normalize("NFC") !== wanted) continue; - if (entry.kind !== "file") + if (candidate.path.normalize("NFC") !== wanted) continue; + if (candidate.kind !== "file") throw archivePathError( "invalid_value", - `Archive path selects a ${entry.kind}, not a regular file: ${selected.path}`, + `Archive path selects a ${candidate.kind}, not a regular file: ${selected.path}`, ); - if ((entry.declaredSize ?? 0) > MAX_BYTES) + if ((candidate.declaredSize ?? 0) > MAX_BYTES) throw new ArtifactReaderFailure( "limit", "Keyed archive exceeds 64 MiB", ); - matches.push(entry); + if (entry !== undefined) + throw archivePathError( + "invalid_value", + `Archive path matches multiple Unicode-equivalent bundle entries; selection is ambiguous: ${selected.path}`, + ); + entry = candidate; } - if (matches.length > 1) - throw archivePathError( - "invalid_value", - `Archive path matches ${String(matches.length)} Unicode-equivalent bundle entries; selection is ambiguous: ${selected.path}`, - ); - const entry = matches[0]; if (entry === undefined) throw archivePathError( "invalid_value", diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts index 5bf6e94cf7..5fcfba7205 100644 --- a/src/artifacts/extraction/ArtifactExtraction.ts +++ b/src/artifacts/extraction/ArtifactExtraction.ts @@ -215,7 +215,10 @@ const materializeSelection = async ({ const written = await output.write( path, stream, - selectedItem.node.sha256, + { + sha256: selectedItem.node.sha256, + bytes: selectedItem.node.size, + }, signal, ); extracted.push({ diff --git a/src/artifacts/inventory/classify.ts b/src/artifacts/inventory/classify.ts index fe29c60574..6e69c64c84 100644 --- a/src/artifacts/inventory/classify.ts +++ b/src/artifacts/inventory/classify.ts @@ -5,6 +5,7 @@ import { } from "../../filesystem/RegularFile.js"; import type { FileHandle } from "node:fs/promises"; import type { Stats } from "node:fs"; +import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js"; import { classifyArtifactContent } from "./ArtifactGraphConstruction.js"; import { ARTIFACT_CLASSIFICATION_PREFIX_BYTES } from "../ArtifactHash.js"; @@ -17,18 +18,46 @@ import { ArtifactReaderFailure } from "../ArtifactReader.js"; import type { HashResult } from "../ArtifactHash.js"; import { hashStableRootArtifactHandle } from "./hashStableRootArtifact.js"; +interface RootClassification { + readonly format: ArtifactOccurrence["artifact_format"]; + readonly digest: HashResult | null; +} + +type RootInventoryClassification = RootClassification & { + readonly zipSource: StableRegularFileDescriptor | undefined; +}; + /** Classify and hash one file root through the same stable open descriptor. */ export const classifyAndHashRoot = async ( path: string, directory: boolean, expectedMetadata: Stats, signal?: AbortSignal, -): Promise<{ - readonly format: ArtifactOccurrence["artifact_format"]; - readonly digest: HashResult | null; -}> => { - if (directory) return { format: "directory", digest: null }; +): Promise => { + const result = await classifyAndHashRootForInventory( + path, + directory, + expectedMetadata, + signal, + ); + try { + return { format: result.format, digest: result.digest }; + } finally { + await result.zipSource?.handle.close(); + } +}; + +/** Classify a root and retain its admitted ZIP descriptor for child inventory. */ +export const classifyAndHashRootForInventory = async ( + path: string, + directory: boolean, + expectedMetadata: Stats, + signal?: AbortSignal, +): Promise => { + if (directory) + return { format: "directory", digest: null, zipSource: undefined }; const handle = await openRootFile(path, signal); + let transferHandle = false; try { const initial = await handle.stat(); if (!sameRegularFileState(expectedMetadata, initial)) @@ -43,12 +72,25 @@ export const classifyAndHashRoot = async ( initial, signal, ); - return { format, digest }; + const zipSource = isZipFormat(format); + if (zipSource) transferHandle = true; + return { + format, + digest, + zipSource: zipSource ? { handle, initial } : undefined, + }; } finally { - await handle.close(); + if (!transferHandle) await handle.close(); } }; +const isZipFormat = (format: ArtifactOccurrence["artifact_format"]): boolean => + format === "zip" || + format === "ipa" || + format === "apk" || + format === "msix" || + format === "appx"; + const openRootFile = async ( path: string, signal?: AbortSignal, diff --git a/src/artifacts/inventory/hashStableRootArtifact.ts b/src/artifacts/inventory/hashStableRootArtifact.ts index e441062d3b..cd3cc302c9 100644 --- a/src/artifacts/inventory/hashStableRootArtifact.ts +++ b/src/artifacts/inventory/hashStableRootArtifact.ts @@ -1,4 +1,5 @@ import { lstat } from "node:fs/promises"; +import type { Stats } from "node:fs"; import type { FileHandle } from "node:fs/promises"; import { ArtifactReaderFailure } from "../ArtifactReader.js"; @@ -45,11 +46,13 @@ export const hashStableRootArtifact = async ( export const hashStableRootArtifactHandle = async ( path: string, handle: FileHandle, - initial: Awaited>, + initial: Stats, signal?: AbortSignal, ): Promise => { const digest = await hashReadable( - handle.createReadStream({ autoClose: false }), + // Read at most the admitted extent plus one byte to detect growth without + // following an actively appended file indefinitely. + handle.createReadStream({ autoClose: false, start: 0, end: initial.size }), signal, ); abortIfNeeded(signal); diff --git a/src/artifacts/inventory/reader.ts b/src/artifacts/inventory/reader.ts index 692431a00e..21ad5adeac 100644 --- a/src/artifacts/inventory/reader.ts +++ b/src/artifacts/inventory/reader.ts @@ -6,12 +6,14 @@ import { NativeDmgArtifactReader } from "../NativeDmgArtifactReader.js"; import { ZipArtifactReader } from "../ZipArtifactReader.js"; import type { ArtifactOccurrence } from "../../domain/artifactGraph.js"; import { ZIP_NON_ENTRY_TAIL_LIMITATION } from "../../domain/zipPackageFormat.js"; +import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js"; export const createReader = async ( path: string, format: ArtifactOccurrence["artifact_format"], environment: Readonly, signal?: AbortSignal, + zipSource?: StableRegularFileDescriptor, ): Promise => { switch (format) { case "directory": @@ -21,7 +23,7 @@ export const createReader = async ( case "apk": case "msix": case "appx": - return new ZipArtifactReader(path, format); + return new ZipArtifactReader(path, format, undefined, zipSource); case "asar": return new AsarArtifactReader(path); case "mach-o-universal": diff --git a/src/artifacts/inventory/scanCanonical.ts b/src/artifacts/inventory/scanCanonical.ts index 0719e58f4c..9ae0db083c 100644 --- a/src/artifacts/inventory/scanCanonical.ts +++ b/src/artifacts/inventory/scanCanonical.ts @@ -5,6 +5,7 @@ import { AsarArtifactReader } from "../AsarArtifactReader.js"; import { lstat } from "node:fs/promises"; import type { Stats } from "node:fs"; +import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js"; import { ArtifactReaderFailure, @@ -31,9 +32,12 @@ import { artifactGraphDigest, artifactManifestId, } from "../../domain/artifactIdentity.js"; -import { classifyAndHashRoot } from "./classify.js"; +import { classifyAndHashRootForInventory } from "./classify.js"; import { type HashResult } from "../ArtifactHash.js"; -import { hashStableRootArtifact } from "./hashStableRootArtifact.js"; +import { + hashStableRootArtifact, + hashStableRootArtifactHandle, +} from "./hashStableRootArtifact.js"; import { createReader, inventoryLimitations } from "./reader.js"; import { scanReader, @@ -51,18 +55,29 @@ export const scanCanonicalArtifactInventory = async ( ): Promise => { const integrity = options.integrity ?? STRICT_INTEGRITY_POLICY; const metadata = await lstat(path); - const { format: rootFormat, digest: rootDigest } = await classifyAndHashRoot( + const { + format: rootFormat, + digest: rootDigest, + zipSource, + } = await classifyAndHashRootForInventory( path, metadata.isDirectory(), metadata, options.signal, ); - const reader = await readerFactory( - path, - rootFormat, - options.environment ?? {}, - options.signal, - ); + let reader: ArtifactReader | undefined; + let readerCreationFailure: { readonly cause: unknown } | undefined; + try { + reader = await readerFactory( + path, + rootFormat, + options.environment ?? {}, + options.signal, + zipSource, + ); + } catch (cause: unknown) { + readerCreationFailure = { cause }; + } const ownedReaders: ArtifactReader[] = reader === undefined ? [] : [reader]; let outcome: | { @@ -71,6 +86,7 @@ export const scanCanonicalArtifactInventory = async ( } | { readonly kind: "failed"; readonly cause: unknown }; try { + if (readerCreationFailure !== undefined) throw readerCreationFailure.cause; if (reader instanceof AsarArtifactReader) await reader.prepareContainer(rootDigest?.sha256, options.signal); const { @@ -86,6 +102,7 @@ export const scanCanonicalArtifactInventory = async ( metadata, rootFormat, rootDigest, + zipSource, reader, signal: options.signal, nodes, @@ -116,6 +133,24 @@ export const scanCanonicalArtifactInventory = async ( ); } } + if (zipSource !== undefined) { + try { + await zipSource.handle.close(); + } catch (cleanupCause: unknown) { + const cleanup = ArtifactReaderFailure.cleanupObservation( + cleanupCause, + `root ZIP descriptor for ${path}`, + ); + cleanupFailure = ArtifactReaderFailure.withCleanup( + cleanupFailure ?? + (outcome.kind === "failed" ? outcome.cause : cleanupCause), + cleanup, + outcome.kind === "completed" + ? { kind: "artifact-inventory", inventory: outcome.snapshot } + : undefined, + ); + } + } if (cleanupFailure !== undefined) throw cleanupFailure; if (outcome.kind === "failed") throw outcome.cause; return outcome.snapshot; @@ -126,6 +161,7 @@ interface SnapshotBuildInput { readonly metadata: Stats; readonly rootFormat: ArtifactOccurrence["artifact_format"]; readonly rootDigest: HashResult | null; + readonly zipSource: StableRegularFileDescriptor | undefined; readonly reader: ArtifactReader | undefined; readonly signal: AbortSignal | undefined; readonly nodes: Map; @@ -137,7 +173,8 @@ interface SnapshotBuildInput { const buildInventorySnapshot = async ( input: SnapshotBuildInput, ): Promise => { - const { path, metadata, rootFormat, rootDigest, reader, signal } = input; + const { path, metadata, rootFormat, rootDigest, zipSource, reader, signal } = + input; materializeDirectoryNodes(input.occurrences, input.nodes); const rootNode = createRootNode({ digest: rootDigest, @@ -181,7 +218,7 @@ const buildInventorySnapshot = async ( ordinal, })); - await verifyRootDigest(path, rootDigest, signal); + await verifyRootDigest(path, rootDigest, zipSource, signal); const graphSha256 = artifactGraphDigest({ nodes: orderedNodes, @@ -256,10 +293,19 @@ const buildIntegrityContradictions = ( const verifyRootDigest = async ( path: string, rootDigest: HashResult | null, + zipSource: StableRegularFileDescriptor | undefined, signal: AbortSignal | undefined, ): Promise => { if (rootDigest === null) return; - const verified = await hashStableRootArtifact(path, signal); + const verified = + zipSource === undefined + ? await hashStableRootArtifact(path, signal) + : await hashStableRootArtifactHandle( + path, + zipSource.handle, + zipSource.initial, + signal, + ); if ( verified.sha256 !== rootDigest.sha256 || verified.bytes !== rootDigest.bytes diff --git a/src/browser/ElectronFileScope.ts b/src/browser/ElectronFileScope.ts index 507edb2334..74062fd8d3 100644 --- a/src/browser/ElectronFileScope.ts +++ b/src/browser/ElectronFileScope.ts @@ -3,6 +3,11 @@ import { open, realpath, stat } from "node:fs/promises"; import { isAbsolute, join, sep } from "node:path"; import { fileURLToPath } from "node:url"; +import { + admitAsarHeader, + MAX_ASAR_HEADER_BYTES, +} from "../artifacts/AsarHeader.js"; + /** Resolve one local file URL while rejecting remote hosts and encoded separators. */ export const authorizedElectronFile = async ( value: string, @@ -77,7 +82,6 @@ const authorizedAsarMember = async ( type ArchiveIdentity = Pick; // This is a retained-memory budget for JSON headers, shared across all archives. -const MAX_HEADER_BYTES = 16 * 1024 * 1024; const MAX_CACHED_HEADERS = 8; const headerCache = new Map< string, @@ -105,23 +109,8 @@ const asarFiles = async ( const opened = await handle.stat(); if (!opened.isFile() || archiveIdentity(opened) !== identity) throw new Error("ASAR identity changed before header read"); - const prefix = Buffer.alloc(16); - const read = await handle.read(prefix, 0, prefix.length, 0); - if (read.bytesRead !== prefix.length || prefix.readUInt32LE(0) !== 4) - throw new Error("Truncated or invalid ASAR size pickle"); - const headerBytes = prefix.readUInt32LE(4); - const payloadBytes = prefix.readUInt32LE(8); - jsonBytes = prefix.readUInt32LE(12); - if ( - headerBytes < 8 || - headerBytes > opened.size - 8 || - payloadBytes !== headerBytes - 4 || - jsonBytes > payloadBytes - 4 || - jsonBytes > MAX_HEADER_BYTES - ) - throw new Error( - "ASAR JSON header leaves the file or exceeds its memory budget", - ); + const admitted = await admitAsarHeader(handle); + jsonBytes = admitted.jsonBytes; const json = Buffer.alloc(jsonBytes); let offset = 0; while (offset < json.length) { @@ -151,7 +140,7 @@ const asarFiles = async ( [...headerCache.values()].reduce((sum, entry) => sum + entry.bytes, 0); while ( headerCache.size > MAX_CACHED_HEADERS || - retainedBytes() > MAX_HEADER_BYTES + retainedBytes() > MAX_ASAR_HEADER_BYTES ) headerCache.delete(headerCache.keys().next().value ?? archive); return files; diff --git a/src/browser/assets/PublishWebScripts.ts b/src/browser/assets/PublishWebScripts.ts index f3af5b8b66..52439494e4 100644 --- a/src/browser/assets/PublishWebScripts.ts +++ b/src/browser/assets/PublishWebScripts.ts @@ -76,7 +76,12 @@ export const publishWebScripts = async ( bytes: bytes.length, }, }); - await tree.write("manifest.json", Readable.from([bytes]), digest, signal); + await tree.write( + "manifest.json", + Readable.from([bytes]), + { sha256: digest, bytes: bytes.byteLength }, + signal, + ); signal?.throwIfAborted(); await tree.commit(); return result; @@ -109,7 +114,7 @@ const publishPlannedScript = async ( tree.write( `files/${relativePath}`, Readable.from([content.bytes]), - content.sha256, + { sha256: content.sha256, bytes: content.bytes.byteLength }, signal, ); try { diff --git a/src/filesystem/RegularFile.ts b/src/filesystem/RegularFile.ts index 13982a8966..b9a0914c36 100644 --- a/src/filesystem/RegularFile.ts +++ b/src/filesystem/RegularFile.ts @@ -16,6 +16,12 @@ export class NonRegularFileReadError extends Error { } } +/** One admitted file descriptor and the metadata that identified its contents. */ +export interface StableRegularFileDescriptor { + readonly handle: FileHandle; + readonly initial: Stats; +} + /** Open and verify the selected file descriptor without blocking on pipes. */ export const openRegularFile = async ( path: string, diff --git a/src/firmware/FirmwarePublication.ts b/src/firmware/FirmwarePublication.ts index 56b011145f..47f9c3a5b5 100644 --- a/src/firmware/FirmwarePublication.ts +++ b/src/firmware/FirmwarePublication.ts @@ -60,7 +60,7 @@ export const publishFirmwareExtraction = async (context: { await tree.write( entry.relativePath, createReadStream(entry.path), - sha256, + { sha256, bytes: entry.size }, context.signal, ); files.push({ diff --git a/src/javascript/recovery/WakaruPublication.ts b/src/javascript/recovery/WakaruPublication.ts index 5b89ebe68c..d909f219f5 100644 --- a/src/javascript/recovery/WakaruPublication.ts +++ b/src/javascript/recovery/WakaruPublication.ts @@ -50,7 +50,12 @@ export const publishWakaruArtifacts = async ( const { tree, source, parsed, staging, signal } = context; const write = async (path: string, bytes: Buffer) => { const sha256 = recoveryDigest(bytes); - const file = await tree.write(path, Readable.from([bytes]), sha256, signal); + const file = await tree.write( + path, + Readable.from([bytes]), + { sha256, bytes: bytes.byteLength }, + signal, + ); return { path: join(tree.outputRoot, file.relativePath), sha256: file.sha256, diff --git a/tests/boundary/filesystem/safeOutputTree.test.ts b/tests/boundary/filesystem/safeOutputTree.test.ts index 32e3c3d325..87c0846406 100644 --- a/tests/boundary/filesystem/safeOutputTree.test.ts +++ b/tests/boundary/filesystem/safeOutputTree.test.ts @@ -43,11 +43,10 @@ describe("safe artifact output tree", () => { const output = join(parent, "published"); const tree = await SafeOutputTree.create(output); await expect( - tree.write( - "nested/file.txt", - Readable.from(Buffer.from("unexpected")), - "0".repeat(64), - ), + tree.write("nested/file.txt", Readable.from(Buffer.from("unexpected")), { + sha256: "0".repeat(64), + bytes: Buffer.byteLength("unexpected"), + }), ).rejects.toThrow(/disagrees/u); expect(await tree.rollback()).toMatchObject({ status: "complete", @@ -64,11 +63,10 @@ describe("safe artifact output tree", () => { const output = join(parent, "published"); const tree = await SafeOutputTree.create(output); const bytes = Buffer.from("retained until rollback can retry"); - await tree.write( - "nested/file.txt", - Readable.from(bytes), - createHash("sha256").update(bytes).digest("hex"), - ); + await tree.write("nested/file.txt", Readable.from(bytes), { + sha256: createHash("sha256").update(bytes).digest("hex"), + bytes: bytes.byteLength, + }); try { await chmod(output, 0o500); const failure = await tree.rollback().catch((cause: unknown) => cause); @@ -90,7 +88,10 @@ describe("safe artifact output tree", () => { const bytes = Buffer.from("visible before seal"); const digest = createHash("sha256").update(bytes).digest("hex"); - await tree.write("file.txt", Readable.from(bytes), digest); + await tree.write("file.txt", Readable.from(bytes), { + sha256: digest, + bytes: bytes.byteLength, + }); expect(await readFile(join(output, "file.txt"), "utf8")).toBe( "visible before seal", ); @@ -123,7 +124,10 @@ describe("safe artifact output tree", () => { const bytes = Buffer.from("windows output"); const digest = createHash("sha256").update(bytes).digest("hex"); - await tree.write("nested/file.txt", Readable.from(bytes), digest); + await tree.write("nested/file.txt", Readable.from(bytes), { + sha256: digest, + bytes: bytes.byteLength, + }); await tree.commit(); expect(await readFile(join(output, "nested", "file.txt"), "utf8")).toBe( "windows output",