import { ArtifactPreparation } from "./ArtifactPreparation.js"; import { artifactExpansionBudgetBytes } from "../domain/artifactExpansionBudget.js"; import type { FileHandle } from "node:fs/promises"; import { setImmediate } from "node:timers/promises"; import { Readable } from "node:stream"; import { ArtifactEntryContentUnavailable, ArtifactReaderFailure, copyArtifactEntry, assertProducedEntry, type ArtifactEntry, type ArtifactReader, type ArtifactTraversalOptions, } from "./ArtifactReader.js"; import { ArtifactResourceScope } from "./ArtifactResourceScope.js"; import { ownAsyncArtifactStream } from "./OwnedAsyncArtifactStream.js"; import { assertArtifactReadActive } from "./ArtifactFileOperation.js"; import { readArtifactFileBytes } from "./ArtifactFileRange.js"; import type { ArtifactFormatMetadata, UnityFsMetadata, } from "../domain/artifactFormatMetadata.js"; import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js"; import { ArtifactFileSource } from "./ArtifactFileSource.js"; import { UNITYFS_COMPRESSION_NONE } from "../domain/unityfsFormat.js"; import { decodeUnityFsLz4, readUnityFsBlocksInfo, readUnityFsHeader, unityFsFormatFailure, type UnityFsBlock, type UnityFsNode, } from "./UnityFsStructures.js"; const CHUNK_BYTES = 64 * 1024; // A decode can retain both encoded and 64 MiB decoded buffers. Bound active // decodes/paused block owners as well as the single completed cache entry. const MAX_MEMBER_STREAMS = 4; type UnityFsEntry = ArtifactEntry & { readonly declaredSize: number; readonly formatMetadata: Extract< ArtifactFormatMetadata, { kind: "unityfs-entry" } >; }; interface UnityFsEntryState { readonly entry: UnityFsEntry; readonly firstBlock: number; readonly lastBlock: number; readonly unavailableCompression: number | undefined; } /** One descriptor, one admitted directory and one completed decoded-block cache. */ export class UnityFsArtifactReader implements ArtifactReader { readonly format = "unityfs" as const; readonly #scope = new ArtifactResourceScope(); #entries = new Map(); readonly #streams = new Map(); readonly #source: ArtifactFileSource; readonly #preparation = new ArtifactPreparation(); #header: UnityFsMetadata | undefined; #blocks: readonly UnityFsBlock[] = []; #dataStart = 0; #remainingDecodeBytes = 0; #remainingReadOperations = 0; #cachedBlock: { readonly index: number; readonly bytes: Buffer } | undefined; #closing = false; constructor(path: string, admitted?: StableRegularFileDescriptor) { this.#source = new ArtifactFileSource(path, "UnityFS bundle", admitted); } formatMetadata(): UnityFsMetadata | undefined { return this.#header === undefined ? undefined : { ...this.#header }; } async prepareContainer( expectedSha256: string, signal?: AbortSignal, ): Promise { await this.#scope.run(async () => { this.#assertOpen(signal); await this.#source.bindSha256(expectedSha256, signal); }); } async *entries( signal?: AbortSignal, options?: ArtifactTraversalOptions, ): AsyncIterable { await this.#scope.run(async () => { this.#assertOpen(signal); await this.#preparation.run( signal, () => this.#prepare(signal), () => this.#assertOpen(signal), ); await this.#source.read(signal); }); this.#assertOpen(signal); if (this.#header?.directory_status !== "available") { options?.onCoverage?.({ status: "unknown", limitations: [ this.#header?.directory_status === "encrypted" ? "UnityFS bundle declares asset-bundle encryption; member identities and contents remain unknown." : this.#header?.directory_status === "unsupported-layout" ? "UnityFS producer revision does not establish alignment or flag semantics; member identities remain unknown." : "UnityFS blocks-and-directory table uses an unsupported compression; member identities remain unknown.", ], }); this.#assertOpen(signal); return; } let index = 0; for (const { entry } of this.#entries.values()) { if (index++ % 256 === 0) await setImmediate(); this.#assertOpen(signal); yield copyArtifactEntry(entry); } // A close while paused at yield must not turn a cleared map into completion. this.#assertOpen(signal); options?.onCoverage?.({ status: "complete", limitations: [] }); } async open(entry: ArtifactEntry, signal?: AbortSignal): Promise { return this.#scope.run(async () => { this.#assertOpen(signal); const state = this.#entries.get(entry.adapterKey); assertProducedEntry( entry, state?.entry, "UnityFS entry does not match its owning inventory", ); // Validate carrier freshness even when a cached block or unavailable fact suffices. const { handle } = await this.#source.read(signal); this.#assertOpen(signal); if (state.entry.kind !== "file") throw new ArtifactReaderFailure( "unavailable", "UnityFS directory entries have no file contents", ); if (state.unavailableCompression !== undefined) { const compression = String(state.unavailableCompression); const limitation = `UnityFS member compression ${compression} is unsupported; no content hash or child artifact was produced.`; throw new ArtifactEntryContentUnavailable( `UnityFS directory entry ${String(state.entry.formatMetadata.index)} requires unsupported compression ${compression}`, limitation, { traversal: { status: "unknown", limitations: [limitation] } }, ); } if (this.#streams.size >= MAX_MEMBER_STREAMS) throw new ArtifactReaderFailure( "limit", "UnityFS reader already owns four active member streams", { boundary: "capacity" }, ); const controller = new AbortController(); const output = ownAsyncArtifactStream( this.#memberChunks(handle, state, controller.signal), controller, { scope: this.#scope, streams: this.#streams, signal, label: "UnityFS", highWaterMark: CHUNK_BYTES, }, ); return output; }); } async close(): Promise { this.#closing = true; for (const [stream, controller] of this.#streams) { const cause = new ArtifactReaderFailure( "unavailable", "UnityFS reader closed", ); controller.abort(cause); stream.destroy(cause); } await this.#scope.close(); await this.#source.close(); this.#entries.clear(); this.#blocks = []; this.#cachedBlock = undefined; } provenance(): readonly [] { return []; } #assertOpen(signal?: AbortSignal): void { assertArtifactReadActive(signal); if (this.#closing) throw new ArtifactReaderFailure( "unavailable", "UnityFS reader is closed", ); } async #prepare(signal?: AbortSignal): Promise { const { handle, initial } = await this.#source.read(signal); const header = await readUnityFsHeader(handle, initial.size, signal); if (header === undefined) throw unityFsFormatFailure("no complete UnityFS header"); const status = header.layout.status; const metadata: UnityFsMetadata = { kind: "unityfs", signature: "UnityFS", format_version: header.formatVersion, player_version: header.playerVersion, engine_version: header.engineVersion, declared_size: header.declaredSize, flags: header.flags, blocks_info_compressed_size: header.compressedBlocksInfoSize, blocks_info_uncompressed_size: header.uncompressedBlocksInfoSize, blocks_info_at_end: header.blocksInfoAtEnd, data_start: null, block_count: null, data_hash: null, directory_status: status, }; if (status !== "available") { await this.#source.read(signal); this.#assertOpen(signal); this.#header = metadata; return; } const { dataHash, blocks, nodes, dataStart } = await readUnityFsBlocksInfo( handle, header, signal, ); const budget = artifactExpansionBudgetBytes(header.declaredSize); let streamBytes = 0n; for (const block of blocks) { streamBytes += BigInt(block.uncompressedSize); if (streamBytes > BigInt(budget)) throw expansionLimit("declared virtual stream", streamBytes, budget); } const preparedMetadata: UnityFsMetadata = { ...metadata, data_hash: dataHash, block_count: blocks.length, data_start: dataStart, }; // Keep the source block count above, but do not revisit zero-output rows // for every member. Parsed stream/compressed offsets remain authoritative. const readableBlocks = blocks.filter((block) => block.uncompressedSize > 0); const registered = await this.#registerNodes( nodes, readableBlocks, budget, signal, ); await this.#source.read(signal); this.#assertOpen(signal); this.#header = preparedMetadata; this.#dataStart = dataStart; this.#blocks = readableBlocks; this.#remainingDecodeBytes = budget; this.#entries = registered.entries; this.#remainingReadOperations = registered.remainingReadOperations; } async #registerNodes( nodes: readonly UnityFsNode[], blocks: readonly UnityFsBlock[], budget: number, signal?: AbortSignal, ): Promise<{ entries: Map; remainingReadOperations: number; }> { const entries = new Map(); let readableBytes = 0n; for (let index = 0; index < nodes.length; index++) { if (index % 256 === 0) await setImmediate(); this.#assertOpen(signal); const node = nodes[index]!; const firstBlock = node.size === 0 ? -1 : blockEndingAfter(blocks, node.offset); const lastBlock = node.size === 0 ? -1 : blockEndingAfter(blocks, node.offset + node.size - 1); if (node.size !== 0 && (firstBlock < 0 || lastBlock < firstBlock)) throw unityFsFormatFailure("member has no block range"); const unavailable = firstUnavailableBlock(blocks, firstBlock, lastBlock); const kind = (node.flags & 1) !== 0 ? ("directory" as const) : ("file" as const); if (kind === "file" && unavailable === undefined) { readableBytes += BigInt(node.size); if (readableBytes > BigInt(budget)) throw expansionLimit( "readable members including aliases", readableBytes, budget, ); } const adapterKey = `unityfs-entry:${String(index)}`; const entry = { path: node.path, sourcePath: node.path, kind, declaredSize: node.size, compressedSize: null, executable: false, encrypted: false, // These offsets address a decoded virtual stream, not the carrier. byteOffset: null, declaredIntegrity: null, unpacked: false, limitations: [], adapterKey, formatMetadata: { kind: "unityfs-entry", index, source_path: node.path, declared_offset: node.offset, flags: node.flags, }, } satisfies UnityFsEntry; entries.set(adapterKey, { entry, firstBlock, lastBlock, unavailableCompression: unavailable?.compression, }); } // Four ordinary passes over admitted blocks, nodes and plaintext chunks, // with a small-reader floor. Byte quotas alone cannot bound tiny-block I/O. const remainingReadOperations = 4 * Math.max( 1024, blocks.length + nodes.length + Math.ceil(Number(readableBytes) / CHUNK_BYTES), ); return { entries, remainingReadOperations }; } async *#memberChunks( handle: FileHandle, state: UnityFsEntryState, signal: AbortSignal, ): AsyncGenerator { const start = state.entry.formatMetadata.declared_offset; const end = start + state.entry.declaredSize; let delivered = 0, chunks = 0; for ( let index = state.firstBlock; state.entry.declaredSize > 0 && index <= state.lastBlock; index++ ) { this.#assertOpen(signal); const block = this.#blocks[index]!; const first = Math.max(start - block.streamStart, 0); const last = Math.min(end - block.streamStart, block.uncompressedSize); if (last <= first) continue; const plain = block.compression === UNITYFS_COMPRESSION_NONE ? undefined : await this.#decodedBlock(handle, index, signal); for (let offset = first; offset < last; offset += CHUNK_BYTES) { if (chunks++ % 64 === 0) await setImmediate(); this.#assertOpen(signal); const size = Math.min(CHUNK_BYTES, last - offset); if (plain === undefined) this.#reserveMemberRead(); const chunk = plain === undefined ? await readArtifactFileBytes( handle, this.#dataStart + block.compressedStart + offset, size, this.#dataStart + block.compressedStart + block.compressedSize, signal, ) : // Cached buffers remain reader-owned; consumers receive independent chunks. Buffer.from(plain.subarray(offset, offset + size)); delivered += chunk.length; yield chunk; } } if (delivered !== state.entry.declaredSize) throw unityFsFormatFailure("member ended before its declared size"); await this.#source.read(signal); } async #decodedBlock( handle: FileHandle, index: number, signal: AbortSignal, ): Promise { if (this.#cachedBlock?.index === index) return this.#cachedBlock.bytes; const block = this.#blocks[index]!; if (block.uncompressedSize > this.#remainingDecodeBytes) throw expansionLimit( "cumulative block decoding", BigInt(block.uncompressedSize), this.#remainingDecodeBytes, ); // Charge attempts too, before I/O/allocation; failed/cancelled work is not free. this.#reserveMemberRead(); this.#remainingDecodeBytes -= block.uncompressedSize; const encoded = await readArtifactFileBytes( handle, this.#dataStart + block.compressedStart, block.compressedSize, this.#dataStart + block.compressedStart + block.compressedSize, signal, ); const bytes = await decodeUnityFsLz4( encoded, block.uncompressedSize, signal, ); this.#assertOpen(signal); this.#cachedBlock = { index, bytes }; return bytes; } #reserveMemberRead(): void { if (this.#remainingReadOperations <= 0) throw new ArtifactReaderFailure( "limit", "UnityFS exceeds its directory-and-chunk I/O work budget", { boundary: "capacity" }, ); this.#remainingReadOperations--; } } const expansionLimit = ( stage: string, bytes: bigint, budget: number, ): ArtifactReaderFailure => new ArtifactReaderFailure( "limit", `UnityFS ${stage} requires ${String(bytes)} bytes, exceeding the ${String(budget)}-byte plaintext-work budget`, ); /** First nonempty block containing the selected virtual byte. */ const blockEndingAfter = ( blocks: readonly UnityFsBlock[], offset: number, ): number => { let low = 0, high = blocks.length; while (low < high) { const middle = low + Math.floor((high - low) / 2), block = blocks[middle]!; if (block.streamStart + block.uncompressedSize <= offset) low = middle + 1; else high = middle; } return low < blocks.length ? low : -1; }; /** Prefix counts avoid scanning every block for every small or overlapping node. */ const firstUnavailableBlock = ( blocks: readonly UnityFsBlock[], first: number, last: number, ): UnityFsBlock | undefined => { if (first < 0) return undefined; const before = first === 0 ? 0 : blocks[first - 1]!.unavailableCount; if (blocks[last]!.unavailableCount === before) return undefined; let low = first, high = last; while (low < high) { const middle = low + Math.floor((high - low) / 2); if (blocks[middle]!.unavailableCount > before) high = middle; else low = middle + 1; } return blocks[low]; };