mirror of
https://github.com/morluto/rea.git
synced 2026-11-15 11:13:21 +00:00
fix(artifacts): require each reader traversal to declare its coverage (#10572)
* fix(artifacts): require each reader traversal to declare its coverage * docs(artifacts): state that silent traversals report unknown coverage
This commit is contained in:
1 parent
e53212655b
commit
2f4c588584
36 files changed
+413
-117
No files matched your search
@@ -186,12 +186,20 @@ export interface ArtifactReaderTraversal {
|
||||
readonly examinedEntries?: number;
|
||||
}
|
||||
|
||||
/** Per-iterator observation sink; adapters report intentional omissions without shared state. */
|
||||
/**
|
||||
* Per-iterator observation sink; adapters report intentional omissions without shared state.
|
||||
* A reader declares its coverage through `onCoverage` before its iterator ends,
|
||||
* including complete coverage; consumers that need a completeness claim
|
||||
* iterate through `declaredArtifactEntries`, which treats silence as unknown.
|
||||
*/
|
||||
export interface ArtifactTraversalOptions {
|
||||
readonly onOmission?: (omission: ArtifactTraversalOmission) => void;
|
||||
readonly onCoverage?: (coverage: ArtifactReaderTraversal) => void;
|
||||
}
|
||||
|
||||
/** The sink of a traversal whose coverage feeds a completeness claim. */
|
||||
export type ArtifactTraversalSink = Required<ArtifactTraversalOptions>;
|
||||
|
||||
/** Read-only adapter over one directory, archive, or virtual container. */
|
||||
export interface ArtifactReader {
|
||||
readonly format:
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { Readable } from "node:stream";
|
||||
import type {
|
||||
ArtifactEntry,
|
||||
ArtifactReader,
|
||||
ArtifactReaderTraversal,
|
||||
} from "./ArtifactReader.js";
|
||||
import { ArtifactTraversalObservation } from "./ArtifactTraversalObservation.js";
|
||||
|
||||
const entry = (path: string): ArtifactEntry => ({
|
||||
path,
|
||||
kind: "file",
|
||||
declaredSize: 1,
|
||||
compressedSize: null,
|
||||
executable: false,
|
||||
encrypted: false,
|
||||
byteOffset: null,
|
||||
declaredIntegrity: null,
|
||||
unpacked: false,
|
||||
limitations: [],
|
||||
adapterKey: path,
|
||||
});
|
||||
|
||||
/** A reader listing two members that declares `coverage`, or nothing. */
|
||||
const reader = (
|
||||
format: ArtifactReader["format"],
|
||||
coverage: ArtifactReaderTraversal | undefined,
|
||||
): ArtifactReader => ({
|
||||
format,
|
||||
async *entries(_signal, options) {
|
||||
yield entry("a");
|
||||
yield entry("b");
|
||||
if (coverage !== undefined) options?.onCoverage?.(coverage);
|
||||
},
|
||||
open: async () => Readable.from([]),
|
||||
provenance: () => [],
|
||||
async close() {},
|
||||
});
|
||||
|
||||
const complete = { status: "complete", limitations: [] } as const;
|
||||
|
||||
const drain = async (iterable: AsyncIterable<ArtifactEntry>) => {
|
||||
const paths: string[] = [];
|
||||
for await (const item of iterable) paths.push(item.path);
|
||||
return paths;
|
||||
};
|
||||
|
||||
describe("ArtifactTraversalObservation", () => {
|
||||
it("is complete only when the reader declares complete coverage and finishes", async () => {
|
||||
const traversal = new ArtifactTraversalObservation();
|
||||
expect(
|
||||
await drain(traversal.entries(reader("zip", complete), undefined)),
|
||||
).toEqual(["a", "b"]);
|
||||
expect(traversal.snapshot()).toEqual(complete);
|
||||
});
|
||||
|
||||
it("reports a reader that ends without declaring coverage as unknown", async () => {
|
||||
const traversal = new ArtifactTraversalObservation();
|
||||
await drain(traversal.entries(reader("ines", undefined), undefined));
|
||||
expect(traversal.snapshot()).toEqual({
|
||||
status: "unknown",
|
||||
limitations: [
|
||||
"The ines reader ended its traversal without declaring coverage; members it did not list are unknown.",
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("does not let one declared-complete traversal cover a silent one", async () => {
|
||||
const traversal = new ArtifactTraversalObservation();
|
||||
await drain(traversal.entries(reader("directory", complete), undefined));
|
||||
await drain(traversal.entries(reader("asar", undefined), undefined));
|
||||
await drain(traversal.entries(reader("zip", complete), undefined));
|
||||
expect(traversal.snapshot()).toEqual({
|
||||
status: "unknown",
|
||||
limitations: [
|
||||
"The asar reader ended its traversal without declaring coverage; members it did not list are unknown.",
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("reports a traversal its consumer stopped as unknown even after a declaration", async () => {
|
||||
const traversal = new ArtifactTraversalObservation();
|
||||
const early: ArtifactReader = {
|
||||
...reader("zip", undefined),
|
||||
async *entries(_signal, options) {
|
||||
options?.onCoverage?.(complete);
|
||||
yield entry("a");
|
||||
yield entry("b");
|
||||
},
|
||||
};
|
||||
for await (const item of traversal.entries(early, undefined))
|
||||
if (item.path === "a") break;
|
||||
expect(traversal.snapshot()).toEqual({
|
||||
status: "unknown",
|
||||
limitations: [
|
||||
"The zip traversal stopped before the reader finished; members after the last listed entry are unknown.",
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves a failed traversal's result to its failure", async () => {
|
||||
const traversal = new ArtifactTraversalObservation();
|
||||
const failure = new Error("damaged");
|
||||
const failing: ArtifactReader = {
|
||||
...reader("zip", undefined),
|
||||
async *entries() {
|
||||
throw failure;
|
||||
},
|
||||
};
|
||||
await expect(drain(traversal.entries(failing, undefined))).rejects.toBe(
|
||||
failure,
|
||||
);
|
||||
expect(traversal.snapshot()).toEqual(complete);
|
||||
});
|
||||
|
||||
it("keeps a declared unknown coverage and its limitations", async () => {
|
||||
const traversal = new ArtifactTraversalObservation();
|
||||
await drain(
|
||||
traversal.entries(
|
||||
reader("wxapkg", { status: "unknown", limitations: ["encrypted"] }),
|
||||
undefined,
|
||||
),
|
||||
);
|
||||
expect(traversal.snapshot()).toEqual({
|
||||
status: "unknown",
|
||||
limitations: ["encrypted"],
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,23 +1,103 @@
|
||||
import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
|
||||
import { normalizeArtifactPath } from "./ArtifactPaths.js";
|
||||
import type {
|
||||
ArtifactReader,
|
||||
ArtifactReaderTraversal,
|
||||
ArtifactTraversalOmission,
|
||||
ArtifactTraversalOptions,
|
||||
ArtifactTraversalSink,
|
||||
} from "./ArtifactReader.js";
|
||||
|
||||
/** Retain producer coverage and omissions across one or more exhausted traversals. */
|
||||
export class ArtifactTraversalObservation implements ArtifactTraversalOptions {
|
||||
/** A reader whose traversal reports through a sink; its format names it in limitations. */
|
||||
interface TraversableArtifactReader<Entry, Options> {
|
||||
readonly format: ArtifactReader["format"];
|
||||
entries(
|
||||
signal: AbortSignal | undefined,
|
||||
options: Options,
|
||||
): AsyncIterable<Entry>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterate one reader traversal whose completeness the reader must declare.
|
||||
* A traversal that ends without a coverage declaration, or that its consumer
|
||||
* stops before the reader finished, reports unknown coverage naming the
|
||||
* reader. A failed traversal reports nothing here; its failure is the result.
|
||||
* The returned iterator forwards `next` and `return` unchanged, so callers
|
||||
* keep unwinding the reader's iterator exactly as they would without it.
|
||||
*/
|
||||
export const declaredArtifactEntries = <
|
||||
Entry,
|
||||
Options extends ArtifactTraversalSink,
|
||||
>(
|
||||
reader: TraversableArtifactReader<Entry, NoInfer<Options>>,
|
||||
signal: AbortSignal | undefined,
|
||||
options: Options,
|
||||
): AsyncIterableIterator<Entry, undefined> => {
|
||||
let declared = false;
|
||||
let end: "exhausted" | "failed" | "stopped" | undefined;
|
||||
const iterator = reader
|
||||
.entries(signal, {
|
||||
...options,
|
||||
onCoverage: (coverage: ArtifactReaderTraversal) => {
|
||||
declared = true;
|
||||
options.onCoverage(coverage);
|
||||
},
|
||||
})
|
||||
[Symbol.asyncIterator]();
|
||||
const unknown = (limitation: string) =>
|
||||
options.onCoverage({ status: "unknown", limitations: [limitation] });
|
||||
return {
|
||||
[Symbol.asyncIterator]() {
|
||||
return this;
|
||||
},
|
||||
async next() {
|
||||
let next: IteratorResult<Entry>;
|
||||
try {
|
||||
next = await iterator.next();
|
||||
} catch (cause: unknown) {
|
||||
end ??= "failed";
|
||||
throw cause;
|
||||
}
|
||||
if (next.done !== true) return next;
|
||||
if (end === undefined) {
|
||||
end = "exhausted";
|
||||
if (!declared)
|
||||
unknown(
|
||||
`The ${reader.format} reader ended its traversal without declaring coverage; members it did not list are unknown.`,
|
||||
);
|
||||
}
|
||||
return { done: true, value: undefined };
|
||||
},
|
||||
async return() {
|
||||
if (end === undefined) {
|
||||
end = "stopped";
|
||||
unknown(
|
||||
`The ${reader.format} traversal stopped before the reader finished; members after the last listed entry are unknown.`,
|
||||
);
|
||||
}
|
||||
await iterator.return?.();
|
||||
return { done: true, value: undefined };
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Retain coverage and omissions across one or more reader traversals. Each
|
||||
* traversal must declare its own coverage; one declared-complete traversal
|
||||
* never covers another that stayed silent.
|
||||
*/
|
||||
export class ArtifactTraversalObservation {
|
||||
readonly #limitations = new Set<string>();
|
||||
#unknown = false;
|
||||
|
||||
readonly onCoverage = (coverage: ArtifactReaderTraversal): void => {
|
||||
/** Record coverage a reader declared or a consumer established itself. */
|
||||
readonly record = (coverage: ArtifactReaderTraversal): void => {
|
||||
this.#unknown ||= coverage.status === "unknown";
|
||||
for (const limitation of coverage.limitations)
|
||||
this.#limitations.add(limitation);
|
||||
};
|
||||
|
||||
readonly onOmission = (omission: ArtifactTraversalOmission): void => {
|
||||
/** Record an entry or subtree a traversal deliberately excluded. */
|
||||
readonly omit = (omission: ArtifactTraversalOmission): void => {
|
||||
const path = normalizeArtifactPath(omission.path);
|
||||
this.#unknown = true;
|
||||
this.#limitations.add(
|
||||
@@ -25,6 +105,17 @@ export class ArtifactTraversalObservation implements ArtifactTraversalOptions {
|
||||
);
|
||||
};
|
||||
|
||||
/** Iterate one reader traversal into this observation. */
|
||||
entries<Entry>(
|
||||
reader: TraversableArtifactReader<Entry, ArtifactTraversalSink>,
|
||||
signal: AbortSignal | undefined,
|
||||
): AsyncIterableIterator<Entry, undefined> {
|
||||
return declaredArtifactEntries(reader, signal, {
|
||||
onCoverage: this.record,
|
||||
onOmission: this.omit,
|
||||
});
|
||||
}
|
||||
|
||||
snapshot(): ArtifactReaderTraversal {
|
||||
return {
|
||||
status: this.#unknown ? "unknown" : "complete",
|
||||
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
sameArtifactEntry,
|
||||
type ArtifactEntry,
|
||||
type ArtifactReader,
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { closeAsarHandle, readValidatedAsarEntry } from "./AsarEntryStream.js";
|
||||
import {
|
||||
@@ -64,7 +65,10 @@ export class AsarArtifactReader implements ArtifactReader {
|
||||
private readonly options: AsarArtifactReaderOptions = {},
|
||||
) {}
|
||||
|
||||
async *entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry> {
|
||||
async *entries(
|
||||
signal?: AbortSignal,
|
||||
options?: ArtifactTraversalOptions,
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
const generation = this.#snapshotGeneration;
|
||||
let paths: readonly AsarListedEntry[];
|
||||
try {
|
||||
@@ -133,6 +137,8 @@ export class AsarArtifactReader implements ArtifactReader {
|
||||
yield entry;
|
||||
}
|
||||
this.#assertInventoryActive(generation, signal);
|
||||
// The listing names every member record the header declares.
|
||||
options?.onCoverage?.({ status: "complete", limitations: [] });
|
||||
}
|
||||
|
||||
#assertInventoryActive(generation: number, signal?: AbortSignal): void {
|
||||
|
||||
@@ -97,7 +97,7 @@ export class BunStandaloneArtifactReader implements ArtifactReader {
|
||||
|
||||
async *entries(
|
||||
signal?: AbortSignal,
|
||||
_options?: ArtifactTraversalOptions,
|
||||
options?: ArtifactTraversalOptions,
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
@@ -116,6 +116,8 @@ export class BunStandaloneArtifactReader implements ArtifactReader {
|
||||
yield copyArtifactEntry(entry);
|
||||
}
|
||||
this.#assertOpen(signal);
|
||||
// Every present module region is listed; absent regions are not members.
|
||||
options?.onCoverage?.({ status: "complete", limitations: [] });
|
||||
}
|
||||
|
||||
async open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable> {
|
||||
|
||||
@@ -37,8 +37,8 @@ export interface DecodedRegion {
|
||||
export interface DecodedRegions {
|
||||
readonly metadata: ArtifactFormatMetadata;
|
||||
readonly regions: readonly DecodedRegion[];
|
||||
/** Omitted when every byte was examined and no limitation applies. */
|
||||
readonly coverage?: ArtifactReaderTraversal;
|
||||
/** What the decoder examined, including complete coverage. */
|
||||
readonly coverage: ArtifactReaderTraversal;
|
||||
readonly containerIntegrity?: ArtifactContainerIntegrityPair;
|
||||
}
|
||||
|
||||
|
||||
@@ -120,11 +120,11 @@ export class DenoStandaloneArtifactReader implements ArtifactReader {
|
||||
);
|
||||
await this.#source.readSelectedPath(signal);
|
||||
});
|
||||
if (this.#header?.vfs_listing === "unsupported")
|
||||
options?.onCoverage?.({
|
||||
status: "unknown",
|
||||
limitations: [UNLISTED_VFS_LIMITATION],
|
||||
});
|
||||
options?.onCoverage?.(
|
||||
this.#header?.vfs_listing === "unsupported"
|
||||
? { status: "unknown", limitations: [UNLISTED_VFS_LIMITATION] }
|
||||
: { status: "complete", limitations: [] },
|
||||
);
|
||||
let index = 0;
|
||||
for (const entry of this.#entries.values()) {
|
||||
if (index++ % 256 === 0) await setImmediate();
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
type ArtifactResourceOwner,
|
||||
} from "./ArtifactResourceScope.js";
|
||||
import { abortIfNeeded } from "./ArtifactHash.js";
|
||||
import { declaredArtifactEntries } from "./ArtifactTraversalObservation.js";
|
||||
import { translateArtifactFailure } from "./ArtifactOperationFailure.js";
|
||||
import {
|
||||
inspectDesktopFile,
|
||||
@@ -270,7 +271,7 @@ const scanDirectory = async (
|
||||
"Unity reference decoding stops after a witness in each Data/Managed directory; other assemblies' references are unexamined. Header facts remain per-file.",
|
||||
];
|
||||
try {
|
||||
for await (const entry of reader.entries(signal, {
|
||||
for await (const entry of declaredArtifactEntries(reader, signal, {
|
||||
maximumEntries: input.max_entries,
|
||||
onOmission: (omission) => {
|
||||
if (omission.kind !== "directory-descendants") {
|
||||
|
||||
@@ -36,9 +36,7 @@ const dfuSeRegions: RegionDecoding<DfuSeImage> = {
|
||||
bytes: element.bytes,
|
||||
};
|
||||
}),
|
||||
...(image.limitations.length === 0
|
||||
? {}
|
||||
: { coverage: { status: "complete", limitations: image.limitations } }),
|
||||
coverage: { status: "complete", limitations: image.limitations },
|
||||
containerIntegrity: {
|
||||
algorithm: "dfu-crc32",
|
||||
declared_crc32: crcText(image.metadata.suffix_crc),
|
||||
|
||||
@@ -170,14 +170,18 @@ export class DirectoryArtifactReader implements ArtifactReader {
|
||||
);
|
||||
}
|
||||
abortIfNeeded(signal);
|
||||
if (unrepresentableNameCount > 0)
|
||||
options?.onCoverage?.({
|
||||
status: "unknown",
|
||||
examinedEntries: examined,
|
||||
limitations: [
|
||||
`Directory contains ${String(unrepresentableNameCount)} entry name(s) that are not valid UTF-8; no filesystem operations were performed for them. First raw basename bytes: ${firstUnrepresentableNameHex ?? "unknown"}.`,
|
||||
],
|
||||
});
|
||||
options?.onCoverage?.(
|
||||
unrepresentableNameCount > 0
|
||||
? {
|
||||
status: "unknown",
|
||||
examinedEntries: examined,
|
||||
limitations: [
|
||||
`Directory contains ${String(unrepresentableNameCount)} entry name(s) that are not valid UTF-8; no filesystem operations were performed for them. First raw basename bytes: ${firstUnrepresentableNameHex ?? "unknown"}.`,
|
||||
],
|
||||
}
|
||||
: // Unreadable directories declared their own omission above.
|
||||
{ status: "complete", limitations: [] },
|
||||
);
|
||||
}
|
||||
|
||||
/** Admission facts, shared with traversal; the selected root may be a symlink. */
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
sameArtifactEntry,
|
||||
type ArtifactEntry,
|
||||
type ArtifactReader,
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import {
|
||||
findDotnetBundleHeader,
|
||||
@@ -66,7 +67,10 @@ export class DotnetBundleArtifactReader implements ArtifactReader {
|
||||
});
|
||||
}
|
||||
|
||||
async *entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry> {
|
||||
async *entries(
|
||||
signal?: AbortSignal,
|
||||
options?: ArtifactTraversalOptions,
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
const bundle = await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
const source = await this.#source.readSelectedPath(signal);
|
||||
@@ -132,6 +136,8 @@ export class DotnetBundleArtifactReader implements ArtifactReader {
|
||||
yield entry;
|
||||
}
|
||||
await this.#source.readSelectedPath(signal);
|
||||
// Every manifest file is listed; manifest damage fails the traversal.
|
||||
options?.onCoverage?.({ status: "complete", limitations: [] });
|
||||
}
|
||||
|
||||
async open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable> {
|
||||
|
||||
@@ -119,10 +119,7 @@ export class EspFlashImageArtifactReader extends PositionalArtifactReader {
|
||||
entries: plan.regions.map((region) =>
|
||||
regionEntry(region, written.get(region), fileSize),
|
||||
),
|
||||
coverage:
|
||||
facts.coverage === "unknown" || limitations.length > 0
|
||||
? { status: facts.coverage, limitations }
|
||||
: undefined,
|
||||
coverage: { status: facts.coverage, limitations },
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -113,10 +113,7 @@ export class EspImageArtifactReader extends PositionalArtifactReader {
|
||||
];
|
||||
return {
|
||||
entries,
|
||||
coverage:
|
||||
image.coverage === "unknown" || limitations.length > 0
|
||||
? { status: image.coverage, limitations }
|
||||
: undefined,
|
||||
coverage: { status: image.coverage, limitations },
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -208,11 +208,10 @@ export class FitArtifactReader implements ArtifactReader {
|
||||
this.#assertOpen(signal);
|
||||
this.#entries = entries;
|
||||
this.#metadata = structure.metadata;
|
||||
if (structure.limitations.length > 0)
|
||||
this.#coverage = {
|
||||
status: "complete",
|
||||
limitations: structure.limitations,
|
||||
};
|
||||
this.#coverage = {
|
||||
status: "complete",
|
||||
limitations: structure.limitations,
|
||||
};
|
||||
}
|
||||
|
||||
/** One streamed pass per image feeds every supported digest. */
|
||||
|
||||
@@ -124,7 +124,7 @@ export class GodotPckArtifactReader extends PositionalArtifactReader {
|
||||
entries.push(metadata);
|
||||
}
|
||||
await this.verifyCarrier(signal);
|
||||
return { entries, coverage: undefined };
|
||||
return { entries, coverage: { status: "complete", limitations: [] } };
|
||||
}
|
||||
async #readEntry(
|
||||
cursor: PckCursor,
|
||||
|
||||
@@ -33,14 +33,10 @@ const hexRecordRegions = (
|
||||
},
|
||||
bytes: region.bytes,
|
||||
})),
|
||||
...(image.terminated && image.limitations.length === 0
|
||||
? {}
|
||||
: {
|
||||
coverage: {
|
||||
status: image.terminated ? "complete" : "unknown",
|
||||
limitations: image.limitations,
|
||||
},
|
||||
}),
|
||||
coverage: {
|
||||
status: image.terminated ? "complete" : "unknown",
|
||||
limitations: image.limitations,
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
sameArtifactEntry,
|
||||
type ArtifactEntry,
|
||||
type ArtifactReader,
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { abortIfNeeded } from "./ArtifactHash.js";
|
||||
import { streamChunkToBuffer } from "./StreamBytes.js";
|
||||
@@ -43,7 +44,10 @@ export class InesArtifactReader implements ArtifactReader {
|
||||
private readonly ownedSource?: OwnedFileHandle,
|
||||
) {}
|
||||
|
||||
async *entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry> {
|
||||
async *entries(
|
||||
signal?: AbortSignal,
|
||||
options?: ArtifactTraversalOptions,
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
abortIfNeeded(signal);
|
||||
await this.#verifySource(signal);
|
||||
const prefix = await readFileHandlePrefix(this.source.handle, 16, () =>
|
||||
@@ -75,6 +79,8 @@ export class InesArtifactReader implements ArtifactReader {
|
||||
yield entry;
|
||||
}
|
||||
this.#assertOpen(signal);
|
||||
// The regions tile the file: header, trainer, units and any trailing bytes.
|
||||
options?.onCoverage?.({ status: "complete", limitations: [] });
|
||||
}
|
||||
|
||||
async open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable> {
|
||||
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
sameArtifactEntry,
|
||||
type ArtifactEntry,
|
||||
type ArtifactReader,
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { streamChunkToBuffer } from "./StreamBytes.js";
|
||||
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
|
||||
@@ -57,12 +58,17 @@ export class MachOSliceArtifactReader implements ArtifactReader {
|
||||
this.#source = new ArtifactFileSource(path, "Universal Mach-O", source);
|
||||
}
|
||||
|
||||
async *entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry> {
|
||||
async *entries(
|
||||
signal?: AbortSignal,
|
||||
options?: ArtifactTraversalOptions,
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
const enumeration = this.#readEntries(signal);
|
||||
this.#enumerations.add(enumeration);
|
||||
try {
|
||||
yield* enumeration;
|
||||
this.#assertOpen(signal);
|
||||
// Every FAT slice is listed; a disagreement with lipo fails the traversal.
|
||||
options?.onCoverage?.({ status: "complete", limitations: [] });
|
||||
} finally {
|
||||
this.#enumerations.delete(enumeration);
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { DirectoryArtifactReader } from "./DirectoryArtifactReader.js";
|
||||
import { declaredArtifactEntries } from "./ArtifactTraversalObservation.js";
|
||||
import {
|
||||
NonRegularFileReadError,
|
||||
RegularFileAdmissionFailure,
|
||||
@@ -201,16 +202,15 @@ export class NativeDmgArtifactReader implements ArtifactReader {
|
||||
limitations: [DMG_CHECKSUM_ABSENT_LIMITATION],
|
||||
});
|
||||
const prefix = basename(this.path);
|
||||
const entries = directory
|
||||
.entries(signal, {
|
||||
onCoverage: (coverage) => options?.onCoverage?.(coverage),
|
||||
onOmission: (omission) =>
|
||||
options?.onOmission?.({
|
||||
...omission,
|
||||
path: `${prefix}/${omission.path}`,
|
||||
}),
|
||||
})
|
||||
[Symbol.asyncIterator]();
|
||||
// The attached directory declares its own coverage; silence stays unknown.
|
||||
const entries = declaredArtifactEntries(directory, signal, {
|
||||
onCoverage: (coverage) => options?.onCoverage?.(coverage),
|
||||
onOmission: (omission) =>
|
||||
options?.onOmission?.({
|
||||
...omission,
|
||||
path: `${prefix}/${omission.path}`,
|
||||
}),
|
||||
})[Symbol.asyncIterator]();
|
||||
try {
|
||||
for (;;) {
|
||||
if (this.#closed)
|
||||
|
||||
@@ -35,9 +35,16 @@ it.skipIf(process.platform === "win32").each([false, true])(
|
||||
signal,
|
||||
options,
|
||||
) {
|
||||
yield* entries.call(this, signal, options);
|
||||
if (unknownCoverage && typeof options !== "function")
|
||||
options?.onCoverage?.(producerCoverage);
|
||||
yield* entries.call(
|
||||
this,
|
||||
signal,
|
||||
unknownCoverage && typeof options !== "function"
|
||||
? {
|
||||
...options,
|
||||
onCoverage: () => options?.onCoverage?.(producerCoverage),
|
||||
}
|
||||
: options,
|
||||
);
|
||||
});
|
||||
const host: NativeDmgHost = {
|
||||
run: async (args) => {
|
||||
@@ -80,7 +87,11 @@ it.skipIf(process.platform === "win32").each([false, true])(
|
||||
}))
|
||||
paths.push(entry.path);
|
||||
expect(paths).toContain("image.dmg/Volume/readable.txt");
|
||||
expect(coverage).toEqual(unknownCoverage ? [producerCoverage] : []);
|
||||
expect(coverage).toEqual([
|
||||
unknownCoverage
|
||||
? producerCoverage
|
||||
: { status: "complete", limitations: [] },
|
||||
]);
|
||||
const paused = reader.entries()[Symbol.asyncIterator]();
|
||||
expect((await paused.next()).done).toBe(false);
|
||||
await reader.close();
|
||||
|
||||
@@ -175,10 +175,11 @@ export class NodeSeaArtifactReader extends PositionalArtifactReader {
|
||||
this.#admission = undefined;
|
||||
return {
|
||||
entries,
|
||||
coverage:
|
||||
fuse.fuseContradiction === null
|
||||
? undefined
|
||||
: { status: "complete", limitations: [fuse.fuseContradiction] },
|
||||
coverage: {
|
||||
status: "complete",
|
||||
limitations:
|
||||
fuse.fuseContradiction === null ? [] : [fuse.fuseContradiction],
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,8 +27,8 @@ export type PositionalArtifactEntry = ArtifactEntry & {
|
||||
/** The entries one preparation established and the coverage to report. */
|
||||
export interface PositionalArtifactEntries {
|
||||
readonly entries: readonly PositionalArtifactEntry[];
|
||||
/** Undefined when coverage is complete without limitations. */
|
||||
readonly coverage: ArtifactReaderTraversal | undefined;
|
||||
/** The coverage the preparation established, including complete coverage. */
|
||||
readonly coverage: ArtifactReaderTraversal;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -86,6 +86,7 @@ export abstract class PositionalArtifactReader implements ArtifactReader {
|
||||
await this.#source.read(signal);
|
||||
});
|
||||
this.assertOpen(signal);
|
||||
// An unprepared reader declares nothing, which consumers read as unknown.
|
||||
if (this.#coverage !== undefined)
|
||||
options?.onCoverage?.({
|
||||
...this.#coverage,
|
||||
|
||||
@@ -2980,7 +2980,7 @@ const containerEntries = async (
|
||||
const traversal = new ArtifactTraversalObservation();
|
||||
let count = 0;
|
||||
let truncated = false;
|
||||
for await (const entry of container.reader.entries(signal, traversal)) {
|
||||
for await (const entry of traversal.entries(container.reader, signal)) {
|
||||
count += 1;
|
||||
if (count > CONTAINER_LIST_LIMIT) {
|
||||
if (!truncated) {
|
||||
|
||||
@@ -34,14 +34,10 @@ const uf2Regions: RegionDecoding<Uf2Image> = {
|
||||
},
|
||||
bytes: region.bytes,
|
||||
})),
|
||||
...(image.complete && image.limitations.length === 0
|
||||
? {}
|
||||
: {
|
||||
coverage: {
|
||||
status: image.complete ? "complete" : "unknown",
|
||||
limitations: image.limitations,
|
||||
},
|
||||
}),
|
||||
coverage: {
|
||||
status: image.complete ? "complete" : "unknown",
|
||||
limitations: image.limitations,
|
||||
},
|
||||
}),
|
||||
};
|
||||
|
||||
|
||||
@@ -120,6 +120,7 @@ export class UnityFsArtifactReader implements ArtifactReader {
|
||||
}
|
||||
// A close while paused at yield must not turn a cleared map into completion.
|
||||
this.#assertOpen(signal);
|
||||
options?.onCoverage?.({ status: "complete", limitations: [] });
|
||||
}
|
||||
|
||||
async open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable> {
|
||||
|
||||
@@ -109,6 +109,7 @@ export class WxapkgArtifactReader implements ArtifactReader {
|
||||
}
|
||||
// A close while paused at yield must not turn a cleared map into completion.
|
||||
this.#assertOpen(signal);
|
||||
options?.onCoverage?.({ status: "complete", limitations: [] });
|
||||
}
|
||||
|
||||
async open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable> {
|
||||
|
||||
@@ -37,6 +37,7 @@ import {
|
||||
sameArtifactEntry,
|
||||
type ArtifactEntry,
|
||||
type ArtifactReader,
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import type { ZipPackageFormat } from "../domain/zipPackageFormat.js";
|
||||
import { artifactExpansionBudgetBytes } from "../domain/artifactExpansionBudget.js";
|
||||
@@ -193,7 +194,10 @@ export class ZipArtifactReader implements ArtifactReader {
|
||||
this.#reader = new ZipReader(this.#source, ZIP_READER_OPTIONS);
|
||||
}
|
||||
|
||||
async *entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry> {
|
||||
async *entries(
|
||||
signal?: AbortSignal,
|
||||
options?: ArtifactTraversalOptions,
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
this.#assertOpen(signal);
|
||||
let index = 0;
|
||||
// A traversal owns its accounting, including overlapping or repeated passes.
|
||||
@@ -249,6 +253,7 @@ export class ZipArtifactReader implements ArtifactReader {
|
||||
this.#assertOpen(signal);
|
||||
await this.#source.verify();
|
||||
this.#assertOpen(signal);
|
||||
options?.onCoverage?.({ status: "complete", limitations: [] });
|
||||
} finally {
|
||||
this.#enumerations.delete(enumeration);
|
||||
}
|
||||
|
||||
@@ -16,7 +16,10 @@ import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "../ArtifactResourceScope.js";
|
||||
import { translateArtifactFailure } from "../ArtifactOperationFailure.js";
|
||||
import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorCleanup.js";
|
||||
import { ArtifactTraversalObservation } from "../ArtifactTraversalObservation.js";
|
||||
import {
|
||||
ArtifactTraversalObservation,
|
||||
declaredArtifactEntries,
|
||||
} from "../ArtifactTraversalObservation.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
sameArtifactSourceIdentity,
|
||||
@@ -206,10 +209,10 @@ const collectAssetCatalogs = async (input: {
|
||||
kind: "completed",
|
||||
};
|
||||
try {
|
||||
for await (const entry of reader.entries(input.signal, {
|
||||
onCoverage: traversal.onCoverage,
|
||||
for await (const entry of declaredArtifactEntries(reader, input.signal, {
|
||||
onCoverage: traversal.record,
|
||||
onOmission: (omission) => {
|
||||
traversal.onOmission(omission);
|
||||
traversal.omit(omission);
|
||||
const { path, kind } = omission;
|
||||
if (path.split("/").at(-1) === "Assets.car")
|
||||
uninspected.push(`${path} (${kind})`);
|
||||
@@ -269,7 +272,7 @@ const collectAssetCatalogs = async (input: {
|
||||
}
|
||||
if (outcome.kind === "failed") throw outcome.cause;
|
||||
if (symlinks > 0)
|
||||
traversal.onCoverage({
|
||||
traversal.record({
|
||||
// Source entries are represented even when their link targets were not
|
||||
// inspected. Keep that inventory scope separate from catalog discovery.
|
||||
status: "complete",
|
||||
|
||||
@@ -258,7 +258,7 @@ const executableRoots = async (
|
||||
};
|
||||
try {
|
||||
await consumeWithOrderedReads(
|
||||
reader.entries(signal, traversal),
|
||||
traversal.entries(reader, signal),
|
||||
MAGIC_READ_CONCURRENCY,
|
||||
(entry) => {
|
||||
// The ordered-read owner skips consume() when read() returns undefined.
|
||||
@@ -298,7 +298,7 @@ const executableRoots = async (
|
||||
if (outcome.kind === "failed") throw outcome.cause;
|
||||
assertArtifactReadActive(signal);
|
||||
if (symlinks > 0)
|
||||
traversal.onCoverage({
|
||||
traversal.record({
|
||||
status: "unknown",
|
||||
limitations: [
|
||||
`Executable-root discovery did not follow ${symlinks} symlink(s); their root roles and possible descendants are unknown. First path: ${JSON.stringify(firstSymlink)}.`,
|
||||
|
||||
@@ -369,7 +369,7 @@ const selectInterfaceBuilderCandidates = async (
|
||||
let unclassifiedSymlinks = 0;
|
||||
let firstUnclassifiedSymlink: string | undefined;
|
||||
let total = 0;
|
||||
for await (const entry of reader.entries(signal, traversal)) {
|
||||
for await (const entry of traversal.entries(reader, signal)) {
|
||||
if (entry.kind === "symlink") {
|
||||
if (
|
||||
isInterfaceBuilderArchive(entry.path) ||
|
||||
@@ -389,7 +389,7 @@ const selectInterfaceBuilderCandidates = async (
|
||||
retainCandidate(candidates, entry, capacity);
|
||||
}
|
||||
if (unclassifiedSymlinks > 0)
|
||||
traversal.onCoverage({
|
||||
traversal.record({
|
||||
status: "unknown",
|
||||
limitations: [
|
||||
`Interface Builder discovery did not follow ${unclassifiedSymlinks} symlink(s); whether they contain archives is unknown. First path: ${JSON.stringify(firstUnclassifiedSymlink)}.`,
|
||||
|
||||
@@ -373,7 +373,7 @@ const materializeSelection = async ({
|
||||
throw cause;
|
||||
}
|
||||
const traversal = new ArtifactTraversalObservation();
|
||||
traversal.onCoverage({
|
||||
traversal.record({
|
||||
status: inventory.coverage.status,
|
||||
limitations: inventory.limitations,
|
||||
});
|
||||
@@ -478,7 +478,7 @@ const writeSelectedEntries = async ({
|
||||
else queue.occurrences.push(occurrence);
|
||||
}
|
||||
const registry = new ArtifactPathRegistry();
|
||||
for await (const entry of reader.entries(signal, traversal)) {
|
||||
for await (const entry of traversal.entries(reader, signal)) {
|
||||
const path = normalizeArtifactPath(entry.path);
|
||||
// Logical declarations may share a query path with a real file. They do
|
||||
// not consume that file's selection or participate in materialization.
|
||||
|
||||
@@ -87,8 +87,9 @@ const entry: ArtifactEntry = {
|
||||
};
|
||||
const readerFailingWith = (failure: ArtifactReaderFailure): ArtifactReader => ({
|
||||
format: "asar",
|
||||
async *entries() {
|
||||
async *entries(_signal, options) {
|
||||
yield entry;
|
||||
options?.onCoverage?.({ status: "complete", limitations: [] });
|
||||
},
|
||||
async open() {
|
||||
throw failure;
|
||||
@@ -133,6 +134,28 @@ it("keeps known member enumeration complete when only its content is unavailable
|
||||
]);
|
||||
});
|
||||
|
||||
it("reports unknown coverage naming a reader that never declared its coverage", async () => {
|
||||
const reader: ArtifactReader = {
|
||||
format: "asar",
|
||||
async *entries() {
|
||||
yield entry;
|
||||
},
|
||||
open: async () => Readable.from([Buffer.from("x")]),
|
||||
provenance: () => [],
|
||||
async close() {},
|
||||
};
|
||||
const result = await scanReader(reader, [reader]);
|
||||
expect(result.occurrences).toMatchObject([
|
||||
{ logical_path: "member", hash_status: "verified" },
|
||||
]);
|
||||
expect(result.traversal).toEqual({
|
||||
status: "unknown",
|
||||
limitations: [
|
||||
"The asar reader ended its traversal without declaring coverage; members it did not list are unknown.",
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["closed reader", "content failure with incomplete cleanup"])(
|
||||
"propagates a %s rather than treating it as a missing unpacked companion",
|
||||
async (kind) => {
|
||||
|
||||
@@ -21,7 +21,10 @@ import {
|
||||
type ArtifactReader,
|
||||
type ArtifactReaderTraversal,
|
||||
} from "../ArtifactReader.js";
|
||||
import { ArtifactTraversalObservation } from "../ArtifactTraversalObservation.js";
|
||||
import {
|
||||
ArtifactTraversalObservation,
|
||||
declaredArtifactEntries,
|
||||
} from "../ArtifactTraversalObservation.js";
|
||||
import {
|
||||
DirectoryArtifactEntryAccessDenied,
|
||||
DirectoryArtifactReader,
|
||||
@@ -201,25 +204,22 @@ const inventoryEntries = (
|
||||
reader: ArtifactReader,
|
||||
prefix: string,
|
||||
): AsyncIterator<ArtifactEntry> =>
|
||||
reader
|
||||
.entries(context.signal, {
|
||||
onCoverage: context.traversal.onCoverage,
|
||||
...(reader instanceof DirectoryArtifactReader &&
|
||||
context.acquisitionLimits !== undefined
|
||||
? {
|
||||
maximumEntries:
|
||||
context.acquisitionLimits.maximumEntries -
|
||||
context.entriesExamined,
|
||||
}
|
||||
: {}),
|
||||
onOmission: (omission) =>
|
||||
context.traversal.onOmission({
|
||||
...omission,
|
||||
path:
|
||||
prefix.length === 0 ? omission.path : `${prefix}/${omission.path}`,
|
||||
}),
|
||||
})
|
||||
[Symbol.asyncIterator]();
|
||||
declaredArtifactEntries(reader, context.signal, {
|
||||
onCoverage: context.traversal.record,
|
||||
...(reader instanceof DirectoryArtifactReader &&
|
||||
context.acquisitionLimits !== undefined
|
||||
? {
|
||||
maximumEntries:
|
||||
context.acquisitionLimits.maximumEntries - context.entriesExamined,
|
||||
}
|
||||
: {}),
|
||||
onOmission: (omission) =>
|
||||
context.traversal.omit({
|
||||
...omission,
|
||||
path:
|
||||
prefix.length === 0 ? omission.path : `${prefix}/${omission.path}`,
|
||||
}),
|
||||
})[Symbol.asyncIterator]();
|
||||
|
||||
const visitArtifactEntries = async (
|
||||
context: ScanContext,
|
||||
@@ -314,7 +314,7 @@ const visitArtifactEntries = async (
|
||||
cause instanceof ArtifactEntryContentUnavailable
|
||||
? cause.traversal
|
||||
: { status: "unknown" as const, limitations: [limitation] };
|
||||
if (traversal !== undefined) context.traversal.onCoverage(traversal);
|
||||
if (traversal !== undefined) context.traversal.record(traversal);
|
||||
occurrence.hash_status = "unavailable";
|
||||
occurrence.limitations.push(limitation);
|
||||
}
|
||||
@@ -369,7 +369,7 @@ const visitArtifactEntries = async (
|
||||
cause instanceof InventoryAcquisitionLimit ||
|
||||
cause instanceof DirectoryArtifactTraversalLimit
|
||||
)
|
||||
context.traversal.onCoverage({
|
||||
context.traversal.record({
|
||||
status: "unknown",
|
||||
limitations: [cause.message],
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user