diff --git a/docs/reference/mcp-contracts.md b/docs/reference/mcp-contracts.md index 892c51824a..2e64593562 100644 --- a/docs/reference/mcp-contracts.md +++ b/docs/reference/mcp-contracts.md @@ -152,6 +152,10 @@ Exhausted traversals with deliberately omitted filesystem entries retain socket and device entries are not opened. Case-distinct archive members remain separate observations. Their case-insensitive materialization diagnostic does not reduce traversal coverage when both spellings were read. +Each container reader declares its own traversal coverage, including complete +coverage. A traversal that ends without that declaration, or that stops before +its reader finishes, is `unknown` with a limitation naming the reader format; +a complete traversal of one container never covers another. PyInstaller inventory also preserves distinct table rows that share a logical path, including decoded children beneath a colliding file/directory path. Inventory coverage describes observed archive facts; extraction still refuses diff --git a/src/artifacts/ArtifactReader.ts b/src/artifacts/ArtifactReader.ts index 30af6d1af4..7bd969629e 100644 --- a/src/artifacts/ArtifactReader.ts +++ b/src/artifacts/ArtifactReader.ts @@ -186,12 +186,20 @@ export interface ArtifactReaderTraversal { readonly examinedEntries?: number; } -/** Per-iterator observation sink; adapters report intentional omissions without shared state. */ +/** + * Per-iterator observation sink; adapters report intentional omissions without shared state. + * A reader declares its coverage through `onCoverage` before its iterator ends, + * including complete coverage; consumers that need a completeness claim + * iterate through `declaredArtifactEntries`, which treats silence as unknown. + */ export interface ArtifactTraversalOptions { readonly onOmission?: (omission: ArtifactTraversalOmission) => void; readonly onCoverage?: (coverage: ArtifactReaderTraversal) => void; } +/** The sink of a traversal whose coverage feeds a completeness claim. */ +export type ArtifactTraversalSink = Required; + /** Read-only adapter over one directory, archive, or virtual container. */ export interface ArtifactReader { readonly format: diff --git a/src/artifacts/ArtifactTraversalObservation.test.ts b/src/artifacts/ArtifactTraversalObservation.test.ts new file mode 100644 index 0000000000..2f02638d52 --- /dev/null +++ b/src/artifacts/ArtifactTraversalObservation.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it } from "vitest"; +import { Readable } from "node:stream"; +import type { + ArtifactEntry, + ArtifactReader, + ArtifactReaderTraversal, +} from "./ArtifactReader.js"; +import { ArtifactTraversalObservation } from "./ArtifactTraversalObservation.js"; + +const entry = (path: string): ArtifactEntry => ({ + path, + kind: "file", + declaredSize: 1, + compressedSize: null, + executable: false, + encrypted: false, + byteOffset: null, + declaredIntegrity: null, + unpacked: false, + limitations: [], + adapterKey: path, +}); + +/** A reader listing two members that declares `coverage`, or nothing. */ +const reader = ( + format: ArtifactReader["format"], + coverage: ArtifactReaderTraversal | undefined, +): ArtifactReader => ({ + format, + async *entries(_signal, options) { + yield entry("a"); + yield entry("b"); + if (coverage !== undefined) options?.onCoverage?.(coverage); + }, + open: async () => Readable.from([]), + provenance: () => [], + async close() {}, +}); + +const complete = { status: "complete", limitations: [] } as const; + +const drain = async (iterable: AsyncIterable) => { + const paths: string[] = []; + for await (const item of iterable) paths.push(item.path); + return paths; +}; + +describe("ArtifactTraversalObservation", () => { + it("is complete only when the reader declares complete coverage and finishes", async () => { + const traversal = new ArtifactTraversalObservation(); + expect( + await drain(traversal.entries(reader("zip", complete), undefined)), + ).toEqual(["a", "b"]); + expect(traversal.snapshot()).toEqual(complete); + }); + + it("reports a reader that ends without declaring coverage as unknown", async () => { + const traversal = new ArtifactTraversalObservation(); + await drain(traversal.entries(reader("ines", undefined), undefined)); + expect(traversal.snapshot()).toEqual({ + status: "unknown", + limitations: [ + "The ines reader ended its traversal without declaring coverage; members it did not list are unknown.", + ], + }); + }); + + it("does not let one declared-complete traversal cover a silent one", async () => { + const traversal = new ArtifactTraversalObservation(); + await drain(traversal.entries(reader("directory", complete), undefined)); + await drain(traversal.entries(reader("asar", undefined), undefined)); + await drain(traversal.entries(reader("zip", complete), undefined)); + expect(traversal.snapshot()).toEqual({ + status: "unknown", + limitations: [ + "The asar reader ended its traversal without declaring coverage; members it did not list are unknown.", + ], + }); + }); + + it("reports a traversal its consumer stopped as unknown even after a declaration", async () => { + const traversal = new ArtifactTraversalObservation(); + const early: ArtifactReader = { + ...reader("zip", undefined), + async *entries(_signal, options) { + options?.onCoverage?.(complete); + yield entry("a"); + yield entry("b"); + }, + }; + for await (const item of traversal.entries(early, undefined)) + if (item.path === "a") break; + expect(traversal.snapshot()).toEqual({ + status: "unknown", + limitations: [ + "The zip traversal stopped before the reader finished; members after the last listed entry are unknown.", + ], + }); + }); + + it("leaves a failed traversal's result to its failure", async () => { + const traversal = new ArtifactTraversalObservation(); + const failure = new Error("damaged"); + const failing: ArtifactReader = { + ...reader("zip", undefined), + async *entries() { + throw failure; + }, + }; + await expect(drain(traversal.entries(failing, undefined))).rejects.toBe( + failure, + ); + expect(traversal.snapshot()).toEqual(complete); + }); + + it("keeps a declared unknown coverage and its limitations", async () => { + const traversal = new ArtifactTraversalObservation(); + await drain( + traversal.entries( + reader("wxapkg", { status: "unknown", limitations: ["encrypted"] }), + undefined, + ), + ); + expect(traversal.snapshot()).toEqual({ + status: "unknown", + limitations: ["encrypted"], + }); + }); +}); diff --git a/src/artifacts/ArtifactTraversalObservation.ts b/src/artifacts/ArtifactTraversalObservation.ts index c3369de295..25dd1446b5 100644 --- a/src/artifacts/ArtifactTraversalObservation.ts +++ b/src/artifacts/ArtifactTraversalObservation.ts @@ -1,23 +1,103 @@ import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; import { normalizeArtifactPath } from "./ArtifactPaths.js"; import type { + ArtifactReader, ArtifactReaderTraversal, ArtifactTraversalOmission, - ArtifactTraversalOptions, + ArtifactTraversalSink, } from "./ArtifactReader.js"; -/** Retain producer coverage and omissions across one or more exhausted traversals. */ -export class ArtifactTraversalObservation implements ArtifactTraversalOptions { +/** A reader whose traversal reports through a sink; its format names it in limitations. */ +interface TraversableArtifactReader { + readonly format: ArtifactReader["format"]; + entries( + signal: AbortSignal | undefined, + options: Options, + ): AsyncIterable; +} + +/** + * Iterate one reader traversal whose completeness the reader must declare. + * A traversal that ends without a coverage declaration, or that its consumer + * stops before the reader finished, reports unknown coverage naming the + * reader. A failed traversal reports nothing here; its failure is the result. + * The returned iterator forwards `next` and `return` unchanged, so callers + * keep unwinding the reader's iterator exactly as they would without it. + */ +export const declaredArtifactEntries = < + Entry, + Options extends ArtifactTraversalSink, +>( + reader: TraversableArtifactReader>, + signal: AbortSignal | undefined, + options: Options, +): AsyncIterableIterator => { + let declared = false; + let end: "exhausted" | "failed" | "stopped" | undefined; + const iterator = reader + .entries(signal, { + ...options, + onCoverage: (coverage: ArtifactReaderTraversal) => { + declared = true; + options.onCoverage(coverage); + }, + }) + [Symbol.asyncIterator](); + const unknown = (limitation: string) => + options.onCoverage({ status: "unknown", limitations: [limitation] }); + return { + [Symbol.asyncIterator]() { + return this; + }, + async next() { + let next: IteratorResult; + try { + next = await iterator.next(); + } catch (cause: unknown) { + end ??= "failed"; + throw cause; + } + if (next.done !== true) return next; + if (end === undefined) { + end = "exhausted"; + if (!declared) + unknown( + `The ${reader.format} reader ended its traversal without declaring coverage; members it did not list are unknown.`, + ); + } + return { done: true, value: undefined }; + }, + async return() { + if (end === undefined) { + end = "stopped"; + unknown( + `The ${reader.format} traversal stopped before the reader finished; members after the last listed entry are unknown.`, + ); + } + await iterator.return?.(); + return { done: true, value: undefined }; + }, + }; +}; + +/** + * Retain coverage and omissions across one or more reader traversals. Each + * traversal must declare its own coverage; one declared-complete traversal + * never covers another that stayed silent. + */ +export class ArtifactTraversalObservation { readonly #limitations = new Set(); #unknown = false; - readonly onCoverage = (coverage: ArtifactReaderTraversal): void => { + /** Record coverage a reader declared or a consumer established itself. */ + readonly record = (coverage: ArtifactReaderTraversal): void => { this.#unknown ||= coverage.status === "unknown"; for (const limitation of coverage.limitations) this.#limitations.add(limitation); }; - readonly onOmission = (omission: ArtifactTraversalOmission): void => { + /** Record an entry or subtree a traversal deliberately excluded. */ + readonly omit = (omission: ArtifactTraversalOmission): void => { const path = normalizeArtifactPath(omission.path); this.#unknown = true; this.#limitations.add( @@ -25,6 +105,17 @@ export class ArtifactTraversalObservation implements ArtifactTraversalOptions { ); }; + /** Iterate one reader traversal into this observation. */ + entries( + reader: TraversableArtifactReader, + signal: AbortSignal | undefined, + ): AsyncIterableIterator { + return declaredArtifactEntries(reader, signal, { + onCoverage: this.record, + onOmission: this.omit, + }); + } + snapshot(): ArtifactReaderTraversal { return { status: this.#unknown ? "unknown" : "complete", diff --git a/src/artifacts/AsarArtifactReader.ts b/src/artifacts/AsarArtifactReader.ts index fed845c12e..7a2c6d469f 100644 --- a/src/artifacts/AsarArtifactReader.ts +++ b/src/artifacts/AsarArtifactReader.ts @@ -24,6 +24,7 @@ import { sameArtifactEntry, type ArtifactEntry, type ArtifactReader, + type ArtifactTraversalOptions, } from "./ArtifactReader.js"; import { closeAsarHandle, readValidatedAsarEntry } from "./AsarEntryStream.js"; import { @@ -64,7 +65,10 @@ export class AsarArtifactReader implements ArtifactReader { private readonly options: AsarArtifactReaderOptions = {}, ) {} - async *entries(signal?: AbortSignal): AsyncIterable { + async *entries( + signal?: AbortSignal, + options?: ArtifactTraversalOptions, + ): AsyncIterable { const generation = this.#snapshotGeneration; let paths: readonly AsarListedEntry[]; try { @@ -133,6 +137,8 @@ export class AsarArtifactReader implements ArtifactReader { yield entry; } this.#assertInventoryActive(generation, signal); + // The listing names every member record the header declares. + options?.onCoverage?.({ status: "complete", limitations: [] }); } #assertInventoryActive(generation: number, signal?: AbortSignal): void { diff --git a/src/artifacts/BunStandaloneArtifactReader.ts b/src/artifacts/BunStandaloneArtifactReader.ts index 776c856f4c..971b482306 100644 --- a/src/artifacts/BunStandaloneArtifactReader.ts +++ b/src/artifacts/BunStandaloneArtifactReader.ts @@ -97,7 +97,7 @@ export class BunStandaloneArtifactReader implements ArtifactReader { async *entries( signal?: AbortSignal, - _options?: ArtifactTraversalOptions, + options?: ArtifactTraversalOptions, ): AsyncIterable { await this.#scope.run(async () => { this.#assertOpen(signal); @@ -116,6 +116,8 @@ export class BunStandaloneArtifactReader implements ArtifactReader { yield copyArtifactEntry(entry); } this.#assertOpen(signal); + // Every present module region is listed; absent regions are not members. + options?.onCoverage?.({ status: "complete", limitations: [] }); } async open(entry: ArtifactEntry, signal?: AbortSignal): Promise { diff --git a/src/artifacts/DecodedRegionArtifactReader.ts b/src/artifacts/DecodedRegionArtifactReader.ts index f3406908c5..f82af9701c 100644 --- a/src/artifacts/DecodedRegionArtifactReader.ts +++ b/src/artifacts/DecodedRegionArtifactReader.ts @@ -37,8 +37,8 @@ export interface DecodedRegion { export interface DecodedRegions { readonly metadata: ArtifactFormatMetadata; readonly regions: readonly DecodedRegion[]; - /** Omitted when every byte was examined and no limitation applies. */ - readonly coverage?: ArtifactReaderTraversal; + /** What the decoder examined, including complete coverage. */ + readonly coverage: ArtifactReaderTraversal; readonly containerIntegrity?: ArtifactContainerIntegrityPair; } diff --git a/src/artifacts/DenoStandaloneArtifactReader.ts b/src/artifacts/DenoStandaloneArtifactReader.ts index 8f7a79b5b5..ee49216887 100644 --- a/src/artifacts/DenoStandaloneArtifactReader.ts +++ b/src/artifacts/DenoStandaloneArtifactReader.ts @@ -120,11 +120,11 @@ export class DenoStandaloneArtifactReader implements ArtifactReader { ); await this.#source.readSelectedPath(signal); }); - if (this.#header?.vfs_listing === "unsupported") - options?.onCoverage?.({ - status: "unknown", - limitations: [UNLISTED_VFS_LIMITATION], - }); + options?.onCoverage?.( + this.#header?.vfs_listing === "unsupported" + ? { status: "unknown", limitations: [UNLISTED_VFS_LIMITATION] } + : { status: "complete", limitations: [] }, + ); let index = 0; for (const entry of this.#entries.values()) { if (index++ % 256 === 0) await setImmediate(); diff --git a/src/artifacts/DesktopApplicationReader.ts b/src/artifacts/DesktopApplicationReader.ts index 7bf2efc7f9..616d9642be 100644 --- a/src/artifacts/DesktopApplicationReader.ts +++ b/src/artifacts/DesktopApplicationReader.ts @@ -13,6 +13,7 @@ import { type ArtifactResourceOwner, } from "./ArtifactResourceScope.js"; import { abortIfNeeded } from "./ArtifactHash.js"; +import { declaredArtifactEntries } from "./ArtifactTraversalObservation.js"; import { translateArtifactFailure } from "./ArtifactOperationFailure.js"; import { inspectDesktopFile, @@ -270,7 +271,7 @@ const scanDirectory = async ( "Unity reference decoding stops after a witness in each Data/Managed directory; other assemblies' references are unexamined. Header facts remain per-file.", ]; try { - for await (const entry of reader.entries(signal, { + for await (const entry of declaredArtifactEntries(reader, signal, { maximumEntries: input.max_entries, onOmission: (omission) => { if (omission.kind !== "directory-descendants") { diff --git a/src/artifacts/DfuSeArtifactReader.ts b/src/artifacts/DfuSeArtifactReader.ts index 7982db76fe..52f9f54ced 100644 --- a/src/artifacts/DfuSeArtifactReader.ts +++ b/src/artifacts/DfuSeArtifactReader.ts @@ -36,9 +36,7 @@ const dfuSeRegions: RegionDecoding = { bytes: element.bytes, }; }), - ...(image.limitations.length === 0 - ? {} - : { coverage: { status: "complete", limitations: image.limitations } }), + coverage: { status: "complete", limitations: image.limitations }, containerIntegrity: { algorithm: "dfu-crc32", declared_crc32: crcText(image.metadata.suffix_crc), diff --git a/src/artifacts/DirectoryArtifactReader.ts b/src/artifacts/DirectoryArtifactReader.ts index 00e14ce33d..f5dd622fea 100644 --- a/src/artifacts/DirectoryArtifactReader.ts +++ b/src/artifacts/DirectoryArtifactReader.ts @@ -170,14 +170,18 @@ export class DirectoryArtifactReader implements ArtifactReader { ); } abortIfNeeded(signal); - if (unrepresentableNameCount > 0) - options?.onCoverage?.({ - status: "unknown", - examinedEntries: examined, - limitations: [ - `Directory contains ${String(unrepresentableNameCount)} entry name(s) that are not valid UTF-8; no filesystem operations were performed for them. First raw basename bytes: ${firstUnrepresentableNameHex ?? "unknown"}.`, - ], - }); + options?.onCoverage?.( + unrepresentableNameCount > 0 + ? { + status: "unknown", + examinedEntries: examined, + limitations: [ + `Directory contains ${String(unrepresentableNameCount)} entry name(s) that are not valid UTF-8; no filesystem operations were performed for them. First raw basename bytes: ${firstUnrepresentableNameHex ?? "unknown"}.`, + ], + } + : // Unreadable directories declared their own omission above. + { status: "complete", limitations: [] }, + ); } /** Admission facts, shared with traversal; the selected root may be a symlink. */ diff --git a/src/artifacts/DotnetBundleArtifactReader.ts b/src/artifacts/DotnetBundleArtifactReader.ts index 3cb88f68d6..bac09ef193 100644 --- a/src/artifacts/DotnetBundleArtifactReader.ts +++ b/src/artifacts/DotnetBundleArtifactReader.ts @@ -18,6 +18,7 @@ import { sameArtifactEntry, type ArtifactEntry, type ArtifactReader, + type ArtifactTraversalOptions, } from "./ArtifactReader.js"; import { findDotnetBundleHeader, @@ -66,7 +67,10 @@ export class DotnetBundleArtifactReader implements ArtifactReader { }); } - async *entries(signal?: AbortSignal): AsyncIterable { + async *entries( + signal?: AbortSignal, + options?: ArtifactTraversalOptions, + ): AsyncIterable { const bundle = await this.#scope.run(async () => { this.#assertOpen(signal); const source = await this.#source.readSelectedPath(signal); @@ -132,6 +136,8 @@ export class DotnetBundleArtifactReader implements ArtifactReader { yield entry; } await this.#source.readSelectedPath(signal); + // Every manifest file is listed; manifest damage fails the traversal. + options?.onCoverage?.({ status: "complete", limitations: [] }); } async open(entry: ArtifactEntry, signal?: AbortSignal): Promise { diff --git a/src/artifacts/EspFlashImageArtifactReader.ts b/src/artifacts/EspFlashImageArtifactReader.ts index 72e94974cd..92f794dc5f 100644 --- a/src/artifacts/EspFlashImageArtifactReader.ts +++ b/src/artifacts/EspFlashImageArtifactReader.ts @@ -119,10 +119,7 @@ export class EspFlashImageArtifactReader extends PositionalArtifactReader { entries: plan.regions.map((region) => regionEntry(region, written.get(region), fileSize), ), - coverage: - facts.coverage === "unknown" || limitations.length > 0 - ? { status: facts.coverage, limitations } - : undefined, + coverage: { status: facts.coverage, limitations }, }; } diff --git a/src/artifacts/EspImageArtifactReader.ts b/src/artifacts/EspImageArtifactReader.ts index 7e61aeafbb..54108b5773 100644 --- a/src/artifacts/EspImageArtifactReader.ts +++ b/src/artifacts/EspImageArtifactReader.ts @@ -113,10 +113,7 @@ export class EspImageArtifactReader extends PositionalArtifactReader { ]; return { entries, - coverage: - image.coverage === "unknown" || limitations.length > 0 - ? { status: image.coverage, limitations } - : undefined, + coverage: { status: image.coverage, limitations }, }; } diff --git a/src/artifacts/FitArtifactReader.ts b/src/artifacts/FitArtifactReader.ts index 37cd34b15f..0825834e75 100644 --- a/src/artifacts/FitArtifactReader.ts +++ b/src/artifacts/FitArtifactReader.ts @@ -208,11 +208,10 @@ export class FitArtifactReader implements ArtifactReader { this.#assertOpen(signal); this.#entries = entries; this.#metadata = structure.metadata; - if (structure.limitations.length > 0) - this.#coverage = { - status: "complete", - limitations: structure.limitations, - }; + this.#coverage = { + status: "complete", + limitations: structure.limitations, + }; } /** One streamed pass per image feeds every supported digest. */ diff --git a/src/artifacts/GodotPckArtifactReader.ts b/src/artifacts/GodotPckArtifactReader.ts index 05d081adcd..e0519749c3 100644 --- a/src/artifacts/GodotPckArtifactReader.ts +++ b/src/artifacts/GodotPckArtifactReader.ts @@ -124,7 +124,7 @@ export class GodotPckArtifactReader extends PositionalArtifactReader { entries.push(metadata); } await this.verifyCarrier(signal); - return { entries, coverage: undefined }; + return { entries, coverage: { status: "complete", limitations: [] } }; } async #readEntry( cursor: PckCursor, diff --git a/src/artifacts/HexRecordArtifactReader.ts b/src/artifacts/HexRecordArtifactReader.ts index 3f5628c2a8..8ed97bdf82 100644 --- a/src/artifacts/HexRecordArtifactReader.ts +++ b/src/artifacts/HexRecordArtifactReader.ts @@ -33,14 +33,10 @@ const hexRecordRegions = ( }, bytes: region.bytes, })), - ...(image.terminated && image.limitations.length === 0 - ? {} - : { - coverage: { - status: image.terminated ? "complete" : "unknown", - limitations: image.limitations, - }, - }), + coverage: { + status: image.terminated ? "complete" : "unknown", + limitations: image.limitations, + }, }), }); diff --git a/src/artifacts/InesArtifactReader.ts b/src/artifacts/InesArtifactReader.ts index 3ff48f8562..21e2870d76 100644 --- a/src/artifacts/InesArtifactReader.ts +++ b/src/artifacts/InesArtifactReader.ts @@ -20,6 +20,7 @@ import { sameArtifactEntry, type ArtifactEntry, type ArtifactReader, + type ArtifactTraversalOptions, } from "./ArtifactReader.js"; import { abortIfNeeded } from "./ArtifactHash.js"; import { streamChunkToBuffer } from "./StreamBytes.js"; @@ -43,7 +44,10 @@ export class InesArtifactReader implements ArtifactReader { private readonly ownedSource?: OwnedFileHandle, ) {} - async *entries(signal?: AbortSignal): AsyncIterable { + async *entries( + signal?: AbortSignal, + options?: ArtifactTraversalOptions, + ): AsyncIterable { abortIfNeeded(signal); await this.#verifySource(signal); const prefix = await readFileHandlePrefix(this.source.handle, 16, () => @@ -75,6 +79,8 @@ export class InesArtifactReader implements ArtifactReader { yield entry; } this.#assertOpen(signal); + // The regions tile the file: header, trainer, units and any trailing bytes. + options?.onCoverage?.({ status: "complete", limitations: [] }); } async open(entry: ArtifactEntry, signal?: AbortSignal): Promise { diff --git a/src/artifacts/MachOSliceArtifactReader.ts b/src/artifacts/MachOSliceArtifactReader.ts index 30d6904faa..7a1f53c1bc 100644 --- a/src/artifacts/MachOSliceArtifactReader.ts +++ b/src/artifacts/MachOSliceArtifactReader.ts @@ -24,6 +24,7 @@ import { sameArtifactEntry, type ArtifactEntry, type ArtifactReader, + type ArtifactTraversalOptions, } from "./ArtifactReader.js"; import { streamChunkToBuffer } from "./StreamBytes.js"; import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js"; @@ -57,12 +58,17 @@ export class MachOSliceArtifactReader implements ArtifactReader { this.#source = new ArtifactFileSource(path, "Universal Mach-O", source); } - async *entries(signal?: AbortSignal): AsyncIterable { + async *entries( + signal?: AbortSignal, + options?: ArtifactTraversalOptions, + ): AsyncIterable { const enumeration = this.#readEntries(signal); this.#enumerations.add(enumeration); try { yield* enumeration; this.#assertOpen(signal); + // Every FAT slice is listed; a disagreement with lipo fails the traversal. + options?.onCoverage?.({ status: "complete", limitations: [] }); } finally { this.#enumerations.delete(enumeration); } diff --git a/src/artifacts/NativeDmgArtifactReader.ts b/src/artifacts/NativeDmgArtifactReader.ts index 235b5a795b..9eb954a9f2 100644 --- a/src/artifacts/NativeDmgArtifactReader.ts +++ b/src/artifacts/NativeDmgArtifactReader.ts @@ -12,6 +12,7 @@ import { type ArtifactTraversalOptions, } from "./ArtifactReader.js"; import { DirectoryArtifactReader } from "./DirectoryArtifactReader.js"; +import { declaredArtifactEntries } from "./ArtifactTraversalObservation.js"; import { NonRegularFileReadError, RegularFileAdmissionFailure, @@ -201,16 +202,15 @@ export class NativeDmgArtifactReader implements ArtifactReader { limitations: [DMG_CHECKSUM_ABSENT_LIMITATION], }); const prefix = basename(this.path); - const entries = directory - .entries(signal, { - onCoverage: (coverage) => options?.onCoverage?.(coverage), - onOmission: (omission) => - options?.onOmission?.({ - ...omission, - path: `${prefix}/${omission.path}`, - }), - }) - [Symbol.asyncIterator](); + // The attached directory declares its own coverage; silence stays unknown. + const entries = declaredArtifactEntries(directory, signal, { + onCoverage: (coverage) => options?.onCoverage?.(coverage), + onOmission: (omission) => + options?.onOmission?.({ + ...omission, + path: `${prefix}/${omission.path}`, + }), + })[Symbol.asyncIterator](); try { for (;;) { if (this.#closed) diff --git a/src/artifacts/NativeDmgProtocol.test.ts b/src/artifacts/NativeDmgProtocol.test.ts index 7e9106a734..4db82029a7 100644 --- a/src/artifacts/NativeDmgProtocol.test.ts +++ b/src/artifacts/NativeDmgProtocol.test.ts @@ -35,9 +35,16 @@ it.skipIf(process.platform === "win32").each([false, true])( signal, options, ) { - yield* entries.call(this, signal, options); - if (unknownCoverage && typeof options !== "function") - options?.onCoverage?.(producerCoverage); + yield* entries.call( + this, + signal, + unknownCoverage && typeof options !== "function" + ? { + ...options, + onCoverage: () => options?.onCoverage?.(producerCoverage), + } + : options, + ); }); const host: NativeDmgHost = { run: async (args) => { @@ -80,7 +87,11 @@ it.skipIf(process.platform === "win32").each([false, true])( })) paths.push(entry.path); expect(paths).toContain("image.dmg/Volume/readable.txt"); - expect(coverage).toEqual(unknownCoverage ? [producerCoverage] : []); + expect(coverage).toEqual([ + unknownCoverage + ? producerCoverage + : { status: "complete", limitations: [] }, + ]); const paused = reader.entries()[Symbol.asyncIterator](); expect((await paused.next()).done).toBe(false); await reader.close(); diff --git a/src/artifacts/NodeSeaArtifactReader.ts b/src/artifacts/NodeSeaArtifactReader.ts index ef2f0b07ba..aa4e815ed7 100644 --- a/src/artifacts/NodeSeaArtifactReader.ts +++ b/src/artifacts/NodeSeaArtifactReader.ts @@ -175,10 +175,11 @@ export class NodeSeaArtifactReader extends PositionalArtifactReader { this.#admission = undefined; return { entries, - coverage: - fuse.fuseContradiction === null - ? undefined - : { status: "complete", limitations: [fuse.fuseContradiction] }, + coverage: { + status: "complete", + limitations: + fuse.fuseContradiction === null ? [] : [fuse.fuseContradiction], + }, }; } } diff --git a/src/artifacts/PositionalArtifactReader.ts b/src/artifacts/PositionalArtifactReader.ts index 2e01b8ab29..ba9d39382e 100644 --- a/src/artifacts/PositionalArtifactReader.ts +++ b/src/artifacts/PositionalArtifactReader.ts @@ -27,8 +27,8 @@ export type PositionalArtifactEntry = ArtifactEntry & { /** The entries one preparation established and the coverage to report. */ export interface PositionalArtifactEntries { readonly entries: readonly PositionalArtifactEntry[]; - /** Undefined when coverage is complete without limitations. */ - readonly coverage: ArtifactReaderTraversal | undefined; + /** The coverage the preparation established, including complete coverage. */ + readonly coverage: ArtifactReaderTraversal; } /** @@ -86,6 +86,7 @@ export abstract class PositionalArtifactReader implements ArtifactReader { await this.#source.read(signal); }); this.assertOpen(signal); + // An unprepared reader declares nothing, which consumers read as unknown. if (this.#coverage !== undefined) options?.onCoverage?.({ ...this.#coverage, diff --git a/src/artifacts/ReleaseExposureReader.ts b/src/artifacts/ReleaseExposureReader.ts index cb9d323f9b..f5a9c01dc6 100644 --- a/src/artifacts/ReleaseExposureReader.ts +++ b/src/artifacts/ReleaseExposureReader.ts @@ -2980,7 +2980,7 @@ const containerEntries = async ( const traversal = new ArtifactTraversalObservation(); let count = 0; let truncated = false; - for await (const entry of container.reader.entries(signal, traversal)) { + for await (const entry of traversal.entries(container.reader, signal)) { count += 1; if (count > CONTAINER_LIST_LIMIT) { if (!truncated) { diff --git a/src/artifacts/Uf2ArtifactReader.ts b/src/artifacts/Uf2ArtifactReader.ts index df0a93c091..17ed67f80f 100644 --- a/src/artifacts/Uf2ArtifactReader.ts +++ b/src/artifacts/Uf2ArtifactReader.ts @@ -34,14 +34,10 @@ const uf2Regions: RegionDecoding = { }, bytes: region.bytes, })), - ...(image.complete && image.limitations.length === 0 - ? {} - : { - coverage: { - status: image.complete ? "complete" : "unknown", - limitations: image.limitations, - }, - }), + coverage: { + status: image.complete ? "complete" : "unknown", + limitations: image.limitations, + }, }), }; diff --git a/src/artifacts/UnityFsArtifactReader.ts b/src/artifacts/UnityFsArtifactReader.ts index 5d6ef1c023..f4f50a745c 100644 --- a/src/artifacts/UnityFsArtifactReader.ts +++ b/src/artifacts/UnityFsArtifactReader.ts @@ -120,6 +120,7 @@ export class UnityFsArtifactReader implements ArtifactReader { } // A close while paused at yield must not turn a cleared map into completion. this.#assertOpen(signal); + options?.onCoverage?.({ status: "complete", limitations: [] }); } async open(entry: ArtifactEntry, signal?: AbortSignal): Promise { diff --git a/src/artifacts/WxapkgArtifactReader.ts b/src/artifacts/WxapkgArtifactReader.ts index 7aeb15c8d5..3ecf787ea5 100644 --- a/src/artifacts/WxapkgArtifactReader.ts +++ b/src/artifacts/WxapkgArtifactReader.ts @@ -109,6 +109,7 @@ export class WxapkgArtifactReader implements ArtifactReader { } // A close while paused at yield must not turn a cleared map into completion. this.#assertOpen(signal); + options?.onCoverage?.({ status: "complete", limitations: [] }); } async open(entry: ArtifactEntry, signal?: AbortSignal): Promise { diff --git a/src/artifacts/ZipArtifactReader.ts b/src/artifacts/ZipArtifactReader.ts index b786adbb94..ac1b3ef69e 100644 --- a/src/artifacts/ZipArtifactReader.ts +++ b/src/artifacts/ZipArtifactReader.ts @@ -37,6 +37,7 @@ import { sameArtifactEntry, type ArtifactEntry, type ArtifactReader, + type ArtifactTraversalOptions, } from "./ArtifactReader.js"; import type { ZipPackageFormat } from "../domain/zipPackageFormat.js"; import { artifactExpansionBudgetBytes } from "../domain/artifactExpansionBudget.js"; @@ -193,7 +194,10 @@ export class ZipArtifactReader implements ArtifactReader { this.#reader = new ZipReader(this.#source, ZIP_READER_OPTIONS); } - async *entries(signal?: AbortSignal): AsyncIterable { + async *entries( + signal?: AbortSignal, + options?: ArtifactTraversalOptions, + ): AsyncIterable { this.#assertOpen(signal); let index = 0; // A traversal owns its accounting, including overlapping or repeated passes. @@ -249,6 +253,7 @@ export class ZipArtifactReader implements ArtifactReader { this.#assertOpen(signal); await this.#source.verify(); this.#assertOpen(signal); + options?.onCoverage?.({ status: "complete", limitations: [] }); } finally { this.#enumerations.delete(enumeration); } diff --git a/src/artifacts/apple/AppleAssetCatalogAnalysis.ts b/src/artifacts/apple/AppleAssetCatalogAnalysis.ts index c400ed5452..e486df4378 100644 --- a/src/artifacts/apple/AppleAssetCatalogAnalysis.ts +++ b/src/artifacts/apple/AppleAssetCatalogAnalysis.ts @@ -16,7 +16,10 @@ import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js"; import { ArtifactResourceScope } from "../ArtifactResourceScope.js"; import { translateArtifactFailure } from "../ArtifactOperationFailure.js"; import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorCleanup.js"; -import { ArtifactTraversalObservation } from "../ArtifactTraversalObservation.js"; +import { + ArtifactTraversalObservation, + declaredArtifactEntries, +} from "../ArtifactTraversalObservation.js"; import { ArtifactReaderFailure, sameArtifactSourceIdentity, @@ -206,10 +209,10 @@ const collectAssetCatalogs = async (input: { kind: "completed", }; try { - for await (const entry of reader.entries(input.signal, { - onCoverage: traversal.onCoverage, + for await (const entry of declaredArtifactEntries(reader, input.signal, { + onCoverage: traversal.record, onOmission: (omission) => { - traversal.onOmission(omission); + traversal.omit(omission); const { path, kind } = omission; if (path.split("/").at(-1) === "Assets.car") uninspected.push(`${path} (${kind})`); @@ -269,7 +272,7 @@ const collectAssetCatalogs = async (input: { } if (outcome.kind === "failed") throw outcome.cause; if (symlinks > 0) - traversal.onCoverage({ + traversal.record({ // Source entries are represented even when their link targets were not // inspected. Keep that inventory scope separate from catalog discovery. status: "complete", diff --git a/src/artifacts/apple/DylibResolutionReader.ts b/src/artifacts/apple/DylibResolutionReader.ts index 0f13c0faf9..acf8bb6a62 100644 --- a/src/artifacts/apple/DylibResolutionReader.ts +++ b/src/artifacts/apple/DylibResolutionReader.ts @@ -258,7 +258,7 @@ const executableRoots = async ( }; try { await consumeWithOrderedReads( - reader.entries(signal, traversal), + traversal.entries(reader, signal), MAGIC_READ_CONCURRENCY, (entry) => { // The ordered-read owner skips consume() when read() returns undefined. @@ -298,7 +298,7 @@ const executableRoots = async ( if (outcome.kind === "failed") throw outcome.cause; assertArtifactReadActive(signal); if (symlinks > 0) - traversal.onCoverage({ + traversal.record({ status: "unknown", limitations: [ `Executable-root discovery did not follow ${symlinks} symlink(s); their root roles and possible descendants are unknown. First path: ${JSON.stringify(firstSymlink)}.`, diff --git a/src/artifacts/apple/InterfaceBuilderAnalysis.ts b/src/artifacts/apple/InterfaceBuilderAnalysis.ts index 67ddf7ae0a..bfcc13dcb2 100644 --- a/src/artifacts/apple/InterfaceBuilderAnalysis.ts +++ b/src/artifacts/apple/InterfaceBuilderAnalysis.ts @@ -369,7 +369,7 @@ const selectInterfaceBuilderCandidates = async ( let unclassifiedSymlinks = 0; let firstUnclassifiedSymlink: string | undefined; let total = 0; - for await (const entry of reader.entries(signal, traversal)) { + for await (const entry of traversal.entries(reader, signal)) { if (entry.kind === "symlink") { if ( isInterfaceBuilderArchive(entry.path) || @@ -389,7 +389,7 @@ const selectInterfaceBuilderCandidates = async ( retainCandidate(candidates, entry, capacity); } if (unclassifiedSymlinks > 0) - traversal.onCoverage({ + traversal.record({ status: "unknown", limitations: [ `Interface Builder discovery did not follow ${unclassifiedSymlinks} symlink(s); whether they contain archives is unknown. First path: ${JSON.stringify(firstUnclassifiedSymlink)}.`, diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts index c84a6a1b6f..1e51508de8 100644 --- a/src/artifacts/extraction/ArtifactExtraction.ts +++ b/src/artifacts/extraction/ArtifactExtraction.ts @@ -373,7 +373,7 @@ const materializeSelection = async ({ throw cause; } const traversal = new ArtifactTraversalObservation(); - traversal.onCoverage({ + traversal.record({ status: inventory.coverage.status, limitations: inventory.limitations, }); @@ -478,7 +478,7 @@ const writeSelectedEntries = async ({ else queue.occurrences.push(occurrence); } const registry = new ArtifactPathRegistry(); - for await (const entry of reader.entries(signal, traversal)) { + for await (const entry of traversal.entries(reader, signal)) { const path = normalizeArtifactPath(entry.path); // Logical declarations may share a query path with a real file. They do // not consume that file's selection or participate in materialization. diff --git a/src/artifacts/inventory/scanReader.test.ts b/src/artifacts/inventory/scanReader.test.ts index 59af6373bd..49e55a6402 100644 --- a/src/artifacts/inventory/scanReader.test.ts +++ b/src/artifacts/inventory/scanReader.test.ts @@ -87,8 +87,9 @@ const entry: ArtifactEntry = { }; const readerFailingWith = (failure: ArtifactReaderFailure): ArtifactReader => ({ format: "asar", - async *entries() { + async *entries(_signal, options) { yield entry; + options?.onCoverage?.({ status: "complete", limitations: [] }); }, async open() { throw failure; @@ -133,6 +134,28 @@ it("keeps known member enumeration complete when only its content is unavailable ]); }); +it("reports unknown coverage naming a reader that never declared its coverage", async () => { + const reader: ArtifactReader = { + format: "asar", + async *entries() { + yield entry; + }, + open: async () => Readable.from([Buffer.from("x")]), + provenance: () => [], + async close() {}, + }; + const result = await scanReader(reader, [reader]); + expect(result.occurrences).toMatchObject([ + { logical_path: "member", hash_status: "verified" }, + ]); + expect(result.traversal).toEqual({ + status: "unknown", + limitations: [ + "The asar reader ended its traversal without declaring coverage; members it did not list are unknown.", + ], + }); +}); + it.each(["closed reader", "content failure with incomplete cleanup"])( "propagates a %s rather than treating it as a missing unpacked companion", async (kind) => { diff --git a/src/artifacts/inventory/scanReader.ts b/src/artifacts/inventory/scanReader.ts index 53c0ad680d..337149407b 100644 --- a/src/artifacts/inventory/scanReader.ts +++ b/src/artifacts/inventory/scanReader.ts @@ -21,7 +21,10 @@ import { type ArtifactReader, type ArtifactReaderTraversal, } from "../ArtifactReader.js"; -import { ArtifactTraversalObservation } from "../ArtifactTraversalObservation.js"; +import { + ArtifactTraversalObservation, + declaredArtifactEntries, +} from "../ArtifactTraversalObservation.js"; import { DirectoryArtifactEntryAccessDenied, DirectoryArtifactReader, @@ -201,25 +204,22 @@ const inventoryEntries = ( reader: ArtifactReader, prefix: string, ): AsyncIterator => - reader - .entries(context.signal, { - onCoverage: context.traversal.onCoverage, - ...(reader instanceof DirectoryArtifactReader && - context.acquisitionLimits !== undefined - ? { - maximumEntries: - context.acquisitionLimits.maximumEntries - - context.entriesExamined, - } - : {}), - onOmission: (omission) => - context.traversal.onOmission({ - ...omission, - path: - prefix.length === 0 ? omission.path : `${prefix}/${omission.path}`, - }), - }) - [Symbol.asyncIterator](); + declaredArtifactEntries(reader, context.signal, { + onCoverage: context.traversal.record, + ...(reader instanceof DirectoryArtifactReader && + context.acquisitionLimits !== undefined + ? { + maximumEntries: + context.acquisitionLimits.maximumEntries - context.entriesExamined, + } + : {}), + onOmission: (omission) => + context.traversal.omit({ + ...omission, + path: + prefix.length === 0 ? omission.path : `${prefix}/${omission.path}`, + }), + })[Symbol.asyncIterator](); const visitArtifactEntries = async ( context: ScanContext, @@ -314,7 +314,7 @@ const visitArtifactEntries = async ( cause instanceof ArtifactEntryContentUnavailable ? cause.traversal : { status: "unknown" as const, limitations: [limitation] }; - if (traversal !== undefined) context.traversal.onCoverage(traversal); + if (traversal !== undefined) context.traversal.record(traversal); occurrence.hash_status = "unavailable"; occurrence.limitations.push(limitation); } @@ -369,7 +369,7 @@ const visitArtifactEntries = async ( cause instanceof InventoryAcquisitionLimit || cause instanceof DirectoryArtifactTraversalLimit ) - context.traversal.onCoverage({ + context.traversal.record({ status: "unknown", limitations: [cause.message], }); diff --git a/tests/boundary/filesystem/desktopApplication.test.ts b/tests/boundary/filesystem/desktopApplication.test.ts index 7287015569..65793f80ee 100644 --- a/tests/boundary/filesystem/desktopApplication.test.ts +++ b/tests/boundary/filesystem/desktopApplication.test.ts @@ -942,7 +942,7 @@ const inspectSimulatedRows = async ( const service = new DesktopApplicationService( new DesktopApplicationReader((selected) => { const reader = new DirectoryArtifactReader(selected); - reader.entries = async function* () { + reader.entries = async function* (_signal, options) { for (let index = 0; index < count; index++) { yielded++; yield { @@ -959,6 +959,8 @@ const inspectSimulatedRows = async ( adapterKey: `${root}/${path(index)}`, }; } + if (typeof options !== "function") + options?.onCoverage?.({ status: "complete", limitations: [] }); }; return reader; }), diff --git a/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts b/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts index 604f465c40..5df143ea40 100644 --- a/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts +++ b/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts @@ -1615,6 +1615,8 @@ describe("native DMG checksum-free images", () => { ), ], }, + // The mounted directory declares its own complete traversal. + { status: "complete", limitations: [] }, ]); expect( reader