diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index efe7af72cc..2bfd55b6c8 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -1012,6 +1012,12 @@ materialized; extraction fails as `unavailable` with that logical path. Extraction materializes regular files exposed by the active archive reader. When that reader exposes a nested archive as a file, extraction copies the archive file; inspect and extract that file separately to unpack its contents. +Extraction retains inventory and materialization traversal limitations in its +result and outer Evidence. If either traversal has unknown coverage, the +observed regular files can still be copied, but the result does not establish +a complete source copy. For example, a POSIX filename that is not valid UTF-8 +remains omitted with its raw-name explanation; it is not replacement-decoded +into an output filename. Packaging tools commonly sign or strip `.asar.unpacked` native binaries after writing the archive header, so their declared hashes no longer match. Electron diff --git a/docs/native-investigation.md b/docs/native-investigation.md index 96f41b1093..0b3ce232cd 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -433,6 +433,9 @@ complete observation. its outcome. Paths outside the analyzed root, including shared-cache system libraries, stay undetermined. See [Apple application analysis](apple-application-analysis.md#dylib-load-resolution). + Bundle root enumeration retains directory traversal uncertainty, including + undecodable filenames; known executable roots remain available with partial + coverage instead of establishing that no other roots exist. - `inspect_asset_catalog` / `rea inspect-asset-catalog ` reads compiled `Assets.car` metadata through macOS `assetutil --info`. Catalog digests, raw rendition fields, pagination and exact UI resource-name matches are returned. @@ -780,6 +783,15 @@ unresolved runtime/value semantics remain visible in results. ## Interface Builder hierarchy coverage +Bundle archive selection preserves directory traversal uncertainty, including +filenames that cannot be decoded as UTF-8. These names are not guessed or +counted as known archive omissions. Recognized symlinked `.nib` entries are +not followed; they retain their paths and reader limitations as counted archive +omissions. Either gap makes `archive_decode` partial and the graph truncated +while preserving decoded documents. Its `examined` count remains actual decode +attempts, and asset-catalog UI joins retain their existing incomplete-graph +qualification. + A document's `hierarchy_complete` flag follows its own hierarchy coverage. If graph limits omit a hierarchy source or destination, or prevent a hierarchy edge or traversal from being retained, that document reports partial hierarchy diff --git a/src/artifacts/ArtifactTraversalObservation.ts b/src/artifacts/ArtifactTraversalObservation.ts new file mode 100644 index 0000000000..c3369de295 --- /dev/null +++ b/src/artifacts/ArtifactTraversalObservation.ts @@ -0,0 +1,34 @@ +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; +import { normalizeArtifactPath } from "./ArtifactPaths.js"; +import type { + ArtifactReaderTraversal, + ArtifactTraversalOmission, + ArtifactTraversalOptions, +} from "./ArtifactReader.js"; + +/** Retain producer coverage and omissions across one or more exhausted traversals. */ +export class ArtifactTraversalObservation implements ArtifactTraversalOptions { + readonly #limitations = new Set(); + #unknown = false; + + readonly onCoverage = (coverage: ArtifactReaderTraversal): void => { + this.#unknown ||= coverage.status === "unknown"; + for (const limitation of coverage.limitations) + this.#limitations.add(limitation); + }; + + readonly onOmission = (omission: ArtifactTraversalOmission): void => { + const path = normalizeArtifactPath(omission.path); + this.#unknown = true; + this.#limitations.add( + `Artifact traversal omitted ${omission.kind} at ${JSON.stringify(path)}.`, + ); + }; + + snapshot(): ArtifactReaderTraversal { + return { + status: this.#unknown ? "unknown" : "complete", + limitations: [...this.#limitations].sort(compareUnicodeCodePoints), + }; + } +} diff --git a/src/artifacts/apple/AppleAssetCatalogAnalysis.test.ts b/src/artifacts/apple/AppleAssetCatalogAnalysis.test.ts index b3f446ac80..112f62665b 100644 --- a/src/artifacts/apple/AppleAssetCatalogAnalysis.test.ts +++ b/src/artifacts/apple/AppleAssetCatalogAnalysis.test.ts @@ -147,6 +147,34 @@ describe("Apple asset catalog application workflow", () => { }); }); +it.skipIf(process.platform === "win32")( + "qualifies UI resource joins when a symlinked nib was not decoded", + async () => { + const bundle = await createTestTempDirectory("rea-asset-ui-symlink-"); + const resources = join(bundle, "Contents", "Resources"); + await mkdir(resources, { recursive: true }); + await writeFile( + join(resources, "Panel.data"), + encodeNibArchiveFixture({ + classes: ["NSView"], + objects: [{ classIndex: 0, values: { title: "Panel" } }], + }), + ); + await symlink("Panel.data", join(resources, "Panel.nib")); + const result = await analyzeAppleAssetCatalogs({ + bundlePath: bundle, + environment: {}, + targetSha256: "a".repeat(64), + runAssetUtil: () => { + throw new Error("No catalog is present"); + }, + }); + expect(result.limitations).toContain( + "Interface Builder resource-key joins may be incomplete because the decoded UI graph was truncated.", + ); + }, +); + it.each([0, 1, 5])( "keeps an empty assetutil inventory complete at offset %i", async (offset) => { diff --git a/src/artifacts/apple/DylibResolutionReader.ts b/src/artifacts/apple/DylibResolutionReader.ts index c9d74c930a..5af99e759d 100644 --- a/src/artifacts/apple/DylibResolutionReader.ts +++ b/src/artifacts/apple/DylibResolutionReader.ts @@ -15,7 +15,11 @@ import { type MachoImageFacts, } from "../../domain/apple/dylibResolution.js"; import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; -import { ArtifactReaderFailure } from "../ArtifactReader.js"; +import { + ArtifactReaderFailure, + type ArtifactReaderTraversal, +} from "../ArtifactReader.js"; +import { ArtifactTraversalObservation } from "../ArtifactTraversalObservation.js"; import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js"; import { consumeWithOrderedReads } from "../OrderedReads.js"; import { @@ -136,12 +140,15 @@ export const traceDylibResolution = async (options: { files, options.signal, ); - const { roots: requested, unclassified } = - parsed.data.roots !== undefined - ? { roots: parsed.data.roots, unclassified: [] } - : options.enumerateRoots - ? await executableRoots(root, view, files, options.signal) - : { roots: [target], unclassified: [] }; + const { + roots: requested, + unclassified, + traversal, + } = parsed.data.roots !== undefined + ? { roots: parsed.data.roots, unclassified: [], traversal: undefined } + : options.enumerateRoots + ? await executableRoots(root, view, files, options.signal) + : { roots: [target], unclassified: [], traversal: undefined }; const roots = await requireMachoRoots( view, requested, @@ -150,6 +157,7 @@ export const traceDylibResolution = async (options: { const trace = await traceDylibLoading(view, { roots, unclassified, + ...(traversal === undefined ? {} : { rootEnumeration: traversal }), ...(parsed.data.architecture === undefined ? {} : { architecture: parsed.data.architecture }), @@ -196,13 +204,18 @@ const executableRoots = async ( view: FilesystemTreeView, files: DylibFileAccess, signal?: AbortSignal, -): Promise<{ readonly roots: string[]; readonly unclassified: string[] }> => { +): Promise<{ + readonly roots: string[]; + readonly unclassified: string[]; + readonly traversal: ArtifactReaderTraversal; +}> => { const roots: string[] = []; const unclassified: string[] = []; + const traversal = new ArtifactTraversalObservation(); // Bundles hold tens of thousands of files and few Mach-O images, so the // magic reads overlap the walk while results are classified in walk order. await consumeWithOrderedReads( - new DirectoryArtifactReader(root).entries(signal), + new DirectoryArtifactReader(root).entries(signal, traversal), MAGIC_READ_CONCURRENCY, (entry) => entry.kind === "file" ? files.startsWithMachoMagic(entry) : undefined, @@ -223,6 +236,7 @@ const executableRoots = async ( return { roots: roots.sort(compare), unclassified: unclassified.sort(compare), + traversal: traversal.snapshot(), }; }; diff --git a/src/artifacts/apple/InterfaceBuilderAnalysis.ts b/src/artifacts/apple/InterfaceBuilderAnalysis.ts index 4afc97a8e7..08b3f2876a 100644 --- a/src/artifacts/apple/InterfaceBuilderAnalysis.ts +++ b/src/artifacts/apple/InterfaceBuilderAnalysis.ts @@ -9,6 +9,7 @@ import { isUIKitRuntimeConnectionClass, } from "../../domain/apple/interfaceBuilderKeyedArchive.js"; import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js"; +import { ArtifactTraversalObservation } from "../ArtifactTraversalObservation.js"; import { projectPlistValue, binaryDatePrecisionOverlapLimitation, @@ -28,7 +29,10 @@ import { InterfaceBuilderDecodeBudget, InterfaceBuilderDecodeBudgetExceeded, } from "./InterfaceBuilderDecodeBudget.js"; -import type { ArtifactEntry } from "../ArtifactReader.js"; +import type { + ArtifactEntry, + ArtifactReaderTraversal, +} from "../ArtifactReader.js"; import { buildInterfaceBuilderAnalysis, interfaceBuilderLimitsSchema, @@ -58,6 +62,13 @@ import { const MAX_AGGREGATE_INPUT_BYTES = 32 * 1024 * 1024; const MAX_AGGREGATE_DECODE_BYTES = 256 * 1024 * 1024; const MAX_REPORTED_OMITTED_ARCHIVES = 64; + +interface OmittedArchive { + readonly path: string; + readonly reason: string; + readonly limitations: readonly string[]; +} + /** Decode compiled Interface Builder archives from a local app bundle. */ export const analyzeInterfaceBuilderBundle = async (input: { readonly bundlePath: string; @@ -70,7 +81,7 @@ export const analyzeInterfaceBuilderBundle = async (input: { const reader = new DirectoryArtifactReader(input.bundlePath); const documents: InterfaceBuilderDocumentInput[] = []; const invalid: string[] = []; - const omittedArchives: Array<{ path: string; reason: string }> = []; + const omittedArchives: OmittedArchive[] = []; let unreportedOmittedArchives = 0; const incompleteHierarchies = new Map(); const unresolvedConnections = new Map(); @@ -83,12 +94,17 @@ export const analyzeInterfaceBuilderBundle = async (input: { let omitted = 0; let attempted = 0; let aggregateInputBytes = 0; + const traversal = new ArtifactTraversalObservation(); const decodeBudget = new InterfaceBuilderDecodeBudget( MAX_AGGREGATE_DECODE_BYTES, ); - const recordOmittedArchive = (path: string, reason: string): void => { + const recordOmittedArchive = ( + path: string, + reason: string, + limitations: readonly string[] = [], + ): void => { if (omittedArchives.length < MAX_REPORTED_OMITTED_ARCHIVES) - omittedArchives.push({ path, reason }); + omittedArchives.push({ path, reason, limitations }); else unreportedOmittedArchives += 1; }; try { @@ -96,10 +112,17 @@ export const analyzeInterfaceBuilderBundle = async (input: { reader, input.signal, limits.max_documents, + traversal, ); omitted = selection.omitted; for (const entry of selection.omittedSamples) recordOmittedArchive(entry.path, "max_documents"); + for (const entry of selection.symlinkSamples) + recordOmittedArchive( + entry.path, + "symlink_not_followed", + entry.limitations, + ); unreportedOmittedArchives += selection.unreportedOmitted; for (const entry of selection.selected) { // Plist representation reserved for this document before decoding. @@ -264,6 +287,7 @@ export const analyzeInterfaceBuilderBundle = async (input: { unreportedOmittedArchives, attempted, omitted, + traversal: traversal.snapshot(), }); }; @@ -272,18 +296,27 @@ const selectInterfaceBuilderCandidates = async ( reader: DirectoryArtifactReader, signal: AbortSignal | undefined, maximumDocuments: number, + traversal: ArtifactTraversalObservation, ): Promise<{ readonly selected: readonly ArtifactEntry[]; readonly omittedSamples: readonly ArtifactEntry[]; + readonly symlinkSamples: readonly ArtifactEntry[]; readonly omitted: number; readonly unreportedOmitted: number; }> => { const capacity = maximumDocuments + MAX_REPORTED_OMITTED_ARCHIVES; const candidates: ArtifactEntry[] = []; + const symlinkSamples: ArtifactEntry[] = []; + let symlinks = 0; let total = 0; - for await (const entry of reader.entries(signal)) { - if (entry.kind !== "file" || !isInterfaceBuilderArchive(entry.path)) + for await (const entry of reader.entries(signal, traversal)) { + if (!isInterfaceBuilderArchive(entry.path)) continue; + if (entry.kind === "symlink") { + symlinks += 1; + retainCandidate(symlinkSamples, entry, MAX_REPORTED_OMITTED_ARCHIVES); continue; + } + if (entry.kind !== "file") continue; total += 1; retainCandidate(candidates, entry, capacity); } @@ -291,11 +324,12 @@ const selectInterfaceBuilderCandidates = async ( return { selected: candidates.slice(0, maximumDocuments), omittedSamples, - omitted: Math.max(0, total - maximumDocuments), - unreportedOmitted: Math.max( - 0, - total - maximumDocuments - omittedSamples.length, - ), + symlinkSamples, + omitted: Math.max(0, total - maximumDocuments) + symlinks, + unreportedOmitted: + Math.max(0, total - maximumDocuments - omittedSamples.length) + + symlinks - + symlinkSamples.length, }; }; @@ -333,10 +367,11 @@ const finalizeHierarchyCoverage = ( nonFiniteReals: ReadonlyMap; negativeZeroReals: ReadonlyMap; invalid: readonly string[]; - omittedArchives: readonly { path: string; reason: string }[]; + omittedArchives: readonly OmittedArchive[]; unreportedOmittedArchives: number; attempted: number; omitted: number; + traversal: ArtifactReaderTraversal; }, ) => { const { @@ -355,7 +390,8 @@ const finalizeHierarchyCoverage = ( const archiveDecodePartial = invalid.length > 0 || prototypeKeyOmissions.size > 0 || - omittedArchives.length > 0; + counts.omitted > 0 || + counts.traversal.status === "unknown"; return { ...result, documents: result.documents.map((document) => ({ @@ -395,13 +431,19 @@ const finalizeHierarchyCoverage = ( ? "aggregate_decode_budget_exhausted" : omittedArchives.some(({ reason }) => reason === "max_documents") ? "document_limit_reached" - : omittedArchives.length > 0 - ? "archive_decode_budget_exhausted" - : invalid.length > 0 - ? "one_or_more_archives_invalid" - : prototypeKeyOmissions.size > 0 - ? "dictionary_entries_omitted" - : null, + : omittedArchives.some( + ({ reason }) => reason === "symlink_not_followed", + ) + ? "archive_symlink_not_followed" + : omittedArchives.length > 0 + ? "archive_decode_budget_exhausted" + : invalid.length > 0 + ? "one_or_more_archives_invalid" + : prototypeKeyOmissions.size > 0 + ? "dictionary_entries_omitted" + : counts.traversal.status === "unknown" + ? "source_traversal_unknown" + : null, examined: counts.attempted, omitted: counts.omitted, }, @@ -414,6 +456,7 @@ const finalizeHierarchyCoverage = ( }, limitations: [ ...result.limitations, + ...counts.traversal.limitations, "Compiled Interface Builder archives are private serialized object graphs. The decoder reports only recognized keyed-archive objects and connections; unrecognized object classes and fields remain unknown.", ...(incompleteHierarchies.size === 0 ? [] @@ -448,10 +491,10 @@ const finalizeHierarchyCoverage = ( ...[...negativeZeroReals].map( ([path, count]) => `${path}: ${negativeZeroRealLimitation(count)}`, ), - ...omittedArchives.map( - ({ path, reason }) => - `${path}: omitted because ${reason.replaceAll("_", " ")}.`, - ), + ...omittedArchives.flatMap(({ path, reason, limitations }) => [ + `${path}: omitted because ${reason.replaceAll("_", " ")}.`, + ...limitations.map((limitation) => `${path}: ${limitation}`), + ]), ...(unreportedOmittedArchives === 0 ? [] : [ diff --git a/src/artifacts/apple/MachoLoadCommandReader.test.ts b/src/artifacts/apple/MachoLoadCommandReader.test.ts index 2d8fe1272f..87942cf51e 100644 --- a/src/artifacts/apple/MachoLoadCommandReader.test.ts +++ b/src/artifacts/apple/MachoLoadCommandReader.test.ts @@ -2,7 +2,7 @@ import { createHash } from "node:crypto"; import { mkdir, realpath, rename, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { CPU, @@ -31,12 +31,75 @@ import { readMachoImage, } from "./MachoLoadCommandReader.js"; import { traceDylibResolution } from "./DylibResolutionReader.js"; +import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js"; import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; import { javaClassFile } from "../../../tests/fixtures/javaClassFile.js"; const read = (bytes: Uint8Array) => readMachoImage(readerOf(bytes), bytes.length); +it("retains unknown executable-root traversal after a complete callback and keeps known image facts", async () => { + const root = await createTestTempDirectory("rea-dylib-traversal-callback-"); + const bytes = machoImage({ commands: [buildVersionCommand(1)] }); + const path = join(root, "main"); + await writeFile(path, bytes); + const digest = createHash("sha256").update(bytes).digest("hex"); + const reason = "Controlled producer executable-root uncertainty"; + const entries = DirectoryArtifactReader.prototype.entries; + const spy = vi + .spyOn(DirectoryArtifactReader.prototype, "entries") + .mockImplementation(async function* ( + this: DirectoryArtifactReader, + signal, + options, + ) { + yield* entries.call(this, signal, options); + if (typeof options === "function") return; + options?.onCoverage?.({ + status: "unknown", + examinedEntries: 1, + limitations: [reason], + }); + options?.onCoverage?.({ + status: "complete", + examinedEntries: 1, + limitations: [], + }); + }); + try { + const trace = await traceDylibResolution({ + rootPath: root, + targetPath: path, + targetSha256: digest, + enumerateRoots: true, + parameters: {}, + }); + expect(trace.coverage).toMatchObject({ + status: "partial", + unparsed_images: [], + roots_without_architecture: [], + }); + expect(trace.roots).toEqual([{ image: "main", architecture: "arm64" }]); + expect(trace.images).toContainEqual( + expect.objectContaining({ + path: "main", + sha256: digest, + parse_status: "parsed", + slices: [ + expect.objectContaining({ + architecture: "arm64", + file_type: "execute", + platform: 1, + }), + ], + }), + ); + expect(trace.limitations).toContain(reason); + } finally { + spy.mockRestore(); + } +}); + // Legacy `LC_VERSION_MIN_*` commands carry no simulator flag: per dyld // rules Intel images are simulator, ARM images are device (tvOS simulator // is x86_64 only). A legacy simulator executable must not fail resolution diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts index d27e36b07a..149a2bb946 100644 --- a/src/artifacts/extraction/ArtifactExtraction.ts +++ b/src/artifacts/extraction/ArtifactExtraction.ts @@ -16,7 +16,9 @@ import { ArtifactReaderFailure, type ArtifactEntry, type ArtifactReader, + type ArtifactReaderTraversal, } from "../ArtifactReader.js"; +import { ArtifactTraversalObservation } from "../ArtifactTraversalObservation.js"; import { abortIfNeeded } from "../ArtifactHash.js"; import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js"; import { SafeOutputTree } from "../SafeOutputTree.js"; @@ -153,6 +155,8 @@ const materializeArtifactInventoryInScope = async ( if (neededNodes.has(node.artifact_id)) nodes.set(node.artifact_id, node); const inventory: ExtractionInventory = { manifest: snapshot.manifest, + coverage: snapshot.coverage, + limitations: snapshot.limitations, integrityContradictions: snapshot.integrity_contradictions.filter( ({ occurrence_id: id }) => selectedIds.has(id), ), @@ -269,7 +273,19 @@ const materializeSelection = async ({ if (cause instanceof SafeOutputTreeCreationFailure) output = cause.tree; throw cause; } - await writeSelectedEntries({ reader, output, byPath, extracted, signal }); + const traversal = new ArtifactTraversalObservation(); + traversal.onCoverage({ + status: inventory.coverage.status, + limitations: inventory.limitations, + }); + await writeSelectedEntries({ + reader, + output, + byPath, + extracted, + traversal, + signal, + }); localReaderOwner = undefined; const closeAttempt = await input.resourceScope.release(readerOwner); if (closeAttempt.kind === "failed") @@ -279,7 +295,13 @@ const materializeSelection = async ({ extracted.sort((left, right) => compareUnicodeCodePoints(left.relative_path, right.relative_path), ); - completed = createExtractionResult(input, inventory, selected, extracted); + completed = createExtractionResult( + input, + inventory, + selected, + extracted, + traversal.snapshot(), + ); await output.commit(signal); return completed; } catch (cause: unknown) { @@ -309,16 +331,18 @@ const writeSelectedEntries = async ({ output, byPath, extracted, + traversal, signal, }: { readonly reader: ArtifactReader; readonly output: SafeOutputTree; readonly byPath: Map; readonly extracted: ExtractedOccurrence[]; + readonly traversal: ArtifactTraversalObservation; readonly signal: AbortSignal | undefined; }): Promise => { const registry = new ArtifactPathRegistry(); - for await (const entry of reader.entries(signal)) { + for await (const entry of reader.entries(signal, traversal)) { const path = normalizeArtifactPath(entry.path); registry.add(path, entry.kind); // Logical declarations may share a query path with a real file. They do @@ -389,6 +413,7 @@ const createExtractionResult = ( inventory: ExtractionInventory, selected: readonly SelectedOccurrence[], extracted: readonly ExtractedOccurrence[], + traversal: ArtifactReaderTraversal, ): ArtifactExtractionResult => { const extractionSemantic = { source_manifest_id: inventory.manifest.manifest_id, @@ -411,7 +436,10 @@ const createExtractionResult = ( provenance: [], integrity_contradictions: inventory.integrityContradictions, limitations: [ - "All regular files in the active artifact were materialized; nested archive contents remain represented by their containing file.", + ...traversal.limitations, + traversal.status === "complete" + ? "All regular files in the active artifact were materialized; nested archive contents remain represented by their containing file." + : "All inventoried regular files exposed by the active reader were materialized; source traversal coverage is unknown, so the output is not established as a complete source copy. Nested archive contents remain represented by their containing file.", ...(inventory.integrityContradictions.length === 0 ? [] : [ @@ -423,6 +451,8 @@ const createExtractionResult = ( interface ExtractionInventory { readonly manifest: ArtifactGraphManifest; + readonly coverage: ArtifactInventorySnapshot["coverage"]; + readonly limitations: readonly string[]; readonly integrityContradictions: readonly IntegrityContradiction[]; } diff --git a/src/artifacts/inventory/scanReader.ts b/src/artifacts/inventory/scanReader.ts index 9102f092ef..6f00661a3f 100644 --- a/src/artifacts/inventory/scanReader.ts +++ b/src/artifacts/inventory/scanReader.ts @@ -16,11 +16,10 @@ import { type ArtifactEntry, type ArtifactReader, type ArtifactReaderTraversal, - type ArtifactTraversalOmission, } from "../ArtifactReader.js"; +import { ArtifactTraversalObservation } from "../ArtifactTraversalObservation.js"; import { AsarArtifactReader } from "../AsarArtifactReader.js"; import type { ArtifactNode } from "../../domain/artifactGraph.js"; -import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; import { classifyArtifactContent, classifyArtifactBytes, @@ -65,8 +64,7 @@ interface ScanContext { readonly registry: ArtifactPathRegistry; readonly expandedContainerIds: Set; readonly ownedReaders: ArtifactReader[]; - readonly omissions: ArtifactTraversalOmission[]; - readonly coverage: ArtifactReaderTraversal[]; + readonly traversal: ArtifactTraversalObservation; } export const scanReader = async ( @@ -105,8 +103,7 @@ export const scanReader = async ( registry: new ArtifactPathRegistry(), expandedContainerIds: new Set(), ownedReaders, - omissions: [], - coverage: [], + traversal: new ArtifactTraversalObservation(), }; await visitArtifactEntries(context, reader, ""); // Archive directories may appear after their children. Resolve containment @@ -118,30 +115,17 @@ export const scanReader = async ( context.expandedContainerIds, ); const structuralLimitations = [...parentResolution.limitations]; + context.traversal.onCoverage({ + status: structuralLimitations.length === 0 ? "complete" : "unknown", + limitations: structuralLimitations, + }); return { nodes, occurrences, pendingContradictions, caseCollisionLimitation: annotateEnglishUnicodeCaseCollisions(occurrences), unknownParentOccurrences: parentResolution.unknownParentOccurrences, - traversal: { - status: - context.omissions.length === 0 && - structuralLimitations.length === 0 && - context.coverage.every((value) => value.status === "complete") - ? "complete" - : "unknown", - limitations: [ - ...new Set([ - ...context.omissions.map( - ({ path, kind }) => - `Artifact traversal omitted ${kind} at ${JSON.stringify(path)}.`, - ), - ...structuralLimitations, - ...context.coverage.flatMap((value) => value.limitations), - ]), - ].sort(compareUnicodeCodePoints), - }, + traversal: context.traversal.snapshot(), }; }; @@ -152,13 +136,12 @@ const inventoryEntries = ( ): AsyncIterator => reader .entries(context.signal, { - onCoverage: (coverage) => context.coverage.push(coverage), + onCoverage: context.traversal.onCoverage, onOmission: (omission) => - context.omissions.push({ + context.traversal.onOmission({ ...omission, - path: normalizeArtifactPath( + path: prefix.length === 0 ? omission.path : `${prefix}/${omission.path}`, - ), }), }) [Symbol.asyncIterator](); diff --git a/src/domain/apple/dylibResolution.ts b/src/domain/apple/dylibResolution.ts index 25963be9ae..006953c24d 100644 --- a/src/domain/apple/dylibResolution.ts +++ b/src/domain/apple/dylibResolution.ts @@ -648,6 +648,12 @@ const traceProcess = async ( return edges; }; +/** Observed completeness of deriving executable roots from a directory walk. */ +export interface DylibRootEnumeration { + readonly status: "complete" | "unknown"; + readonly limitations: readonly string[]; +} + const projectTraceResult = (input: { readonly images: ReadonlyMap; readonly roots: DylibTrace["roots"]; @@ -657,6 +663,7 @@ const projectTraceResult = (input: { readonly environmentRoots: ReadonlySet; readonly unknownPlatformCandidates: ReadonlySet; readonly implicitMacOSPlatformCandidates: ReadonlySet; + readonly rootEnumeration: DylibRootEnumeration | undefined; }): DylibTrace => { const reported = [...input.images.entries()] .filter(([, facts]) => facts.status !== "not-mach-o") @@ -686,7 +693,8 @@ const projectTraceResult = (input: { unparsed.length === 0 && input.withoutArchitecture.length === 0 && input.environmentRoots.size === 0 && - input.unknownPlatformCandidates.size === 0 + input.unknownPlatformCandidates.size === 0 && + input.rootEnumeration?.status !== "unknown" ? "complete" : "partial", unparsed_images: unparsed, @@ -694,6 +702,7 @@ const projectTraceResult = (input: { }, limitations: [ ...DYLIB_RESOLUTION_LIMITATIONS, + ...(input.rootEnumeration?.limitations ?? []), ...(input.environmentRoots.size === 0 ? [] : [ @@ -720,6 +729,7 @@ export const traceDylibLoading = async ( readonly roots: readonly string[]; /** Mach-O files whose root role could not be classified because they did not parse. */ readonly unclassified?: readonly string[]; + readonly rootEnumeration?: DylibRootEnumeration; readonly architecture?: string; readonly signal?: AbortSignal; }, @@ -796,6 +806,7 @@ export const traceDylibLoading = async ( environmentRoots, unknownPlatformCandidates, implicitMacOSPlatformCandidates, + rootEnumeration: request.rootEnumeration, }); }; diff --git a/tests/acceptance/applications/appleBundleTraversalCoverage.test.ts b/tests/acceptance/applications/appleBundleTraversalCoverage.test.ts new file mode 100644 index 0000000000..d368d5c479 --- /dev/null +++ b/tests/acceptance/applications/appleBundleTraversalCoverage.test.ts @@ -0,0 +1,179 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile, rename, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect } from "vitest"; + +import { encodeNibArchiveFixture } from "../../../src/artifacts/apple/NibArchive.fixture.js"; +import { machoImage } from "../../../src/artifacts/apple/MachoImage.fixture.js"; +import { interfaceBuilderAnalysisSchema } from "../../../src/domain/apple/interfaceBuilderGraph.js"; +import { dylibResolutionResultSchema } from "../../../src/domain/apple/dylibResolution.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest, type TestCli } from "../../support/cli/cliFixture.js"; + +const bundle = async () => { + const root = await createTestTempDirectory("rea-apple-traversal-"); + const app = join(root, "Fixture.app"); + const resources = join(app, "Contents", "Resources"); + const executables = join(app, "Contents", "MacOS"); + await mkdir(resources, { recursive: true }); + await mkdir(executables); + await writeFile( + join(app, "Contents", "Info.plist"), + "CFBundleExecutablemain", + ); + const executable = machoImage({}); + await writeFile(join(executables, "main"), executable); + const nib = encodeNibArchiveFixture({ + classes: ["NSView"], + objects: [{ classIndex: 0, values: { title: "Healthy view" } }], + }); + await writeFile(join(resources, "Healthy.nib"), nib); + return { app, resources, executables, executable, nib }; +}; + +const interfaceBuilderReports = async (cli: TestCli, app: string) => { + const output = await cli.run({ + arguments: ["decode-interface-builder", app, "--json"], + }); + expect(output.exitCode, output.stdout + output.stderr).toBe(0); + const call = await cli.sharedMcpCall?.(); + if (call === undefined) throw new Error("Expected stdio MCP session"); + const opened = await call("open_bundle_resources", { path: app }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const reply = await call("decode_interface_builder", {}); + expect(reply.isError, JSON.stringify(reply)).not.toBe(true); + return [ + parseEvidence(output.json), + parseEvidence(reply.structuredContent), + ].map((evidence) => ({ + evidence, + report: interfaceBuilderAnalysisSchema.parse(evidence.normalized_result), + })); +}; + +cliTest.skipIf(process.platform === "win32")( + "reports symlinked nibs as unexamined while preserving healthy UI documents through CLI and MCP", + async ({ cli }) => { + const fixture = await bundle(); + const link = join(fixture.resources, "Panel.nib"); + await writeFile(join(fixture.resources, "Panel.data"), fixture.nib); + await symlink("Panel.data", link); + for (const { evidence, report } of await interfaceBuilderReports( + cli, + fixture.app, + )) { + expect(report.documents).toHaveLength(1); + expect(report.documents[0]).toMatchObject({ + relative_path: "Contents/Resources/Healthy.nib", + archive_sha256: createHash("sha256").update(fixture.nib).digest("hex"), + }); + expect(report.graph.coverage).toContainEqual( + expect.objectContaining({ + facet: "archive_decode", + status: "partial", + reason: "archive_symlink_not_followed", + examined: 1, + omitted: 1, + }), + ); + expect(report.graph.truncated).toBe(true); + for (const limitations of [report.limitations, evidence.limitations]) + expect(limitations).toContain( + "Contents/Resources/Panel.nib: Symlink target was not followed or disclosed.", + ); + } + expect(await readFile(link)).toEqual(fixture.nib); + }, +); + +cliTest.skipIf(process.platform !== "linux")( + "retains unknown Apple resource and executable filename coverage and same-byte rename controls through CLI and MCP", + async ({ cli }) => { + const fixture = await bundle(); + const rawNib = Buffer.concat([ + Buffer.from(join(fixture.resources, "Panel")), + Buffer.from([0xff]), + Buffer.from(".nib"), + ]); + const rawExecutable = Buffer.concat([ + Buffer.from(join(fixture.executables, "helper")), + Buffer.from([0xff]), + ]); + await writeFile(rawNib, fixture.nib); + await writeFile(rawExecutable, fixture.executable); + for (const { evidence, report } of await interfaceBuilderReports( + cli, + fixture.app, + )) { + expect(report.documents).toHaveLength(1); + expect(report.graph.coverage).toContainEqual( + expect.objectContaining({ + facet: "archive_decode", + status: "partial", + reason: "source_traversal_unknown", + examined: 1, + omitted: 0, + }), + ); + expect(report.graph.truncated).toBe(true); + for (const limitations of [report.limitations, evidence.limitations]) + expect(limitations).toContainEqual( + expect.stringContaining("not valid UTF-8"), + ); + } + const call = await cli.sharedMcpCall?.(); + if (call === undefined) throw new Error("Expected stdio MCP session"); + const opened = await call("open_binary", { path: fixture.app }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const cliTrace = await cli.run({ + arguments: ["trace-dylib-resolution", fixture.app, "--json"], + }); + expect(cliTrace.exitCode, cliTrace.stdout + cliTrace.stderr).toBe(0); + const mcpTrace = await call("trace_dylib_resolution", {}); + expect(mcpTrace.isError, JSON.stringify(mcpTrace)).not.toBe(true); + for (const evidence of [ + parseEvidence(cliTrace.json), + parseEvidence(mcpTrace.structuredContent), + ]) { + const trace = dylibResolutionResultSchema.parse( + evidence.normalized_result, + ); + expect(trace.roots).toEqual([ + { image: "Contents/MacOS/main", architecture: "arm64" }, + ]); + expect(trace.coverage.status).toBe("partial"); + for (const limitations of [trace.limitations, evidence.limitations]) + expect(limitations).toContainEqual( + expect.stringContaining("not valid UTF-8"), + ); + } + await rename(rawNib, join(fixture.resources, "Panel.nib")); + await rename(rawExecutable, join(fixture.executables, "helper")); + for (const { report } of await interfaceBuilderReports(cli, fixture.app)) { + expect(report.documents).toHaveLength(2); + expect(report.graph.coverage).toContainEqual( + expect.objectContaining({ + facet: "archive_decode", + status: "complete", + examined: 2, + omitted: 0, + }), + ); + expect(report.limitations.join(" ")).not.toContain("not valid UTF-8"); + } + const control = await cli.run({ + arguments: ["trace-dylib-resolution", fixture.app, "--json"], + }); + expect(control.exitCode, control.stdout + control.stderr).toBe(0); + const trace = dylibResolutionResultSchema.parse( + parseEvidence(control.json).normalized_result, + ); + expect(trace.roots.map(({ image }) => image)).toEqual([ + "Contents/MacOS/helper", + "Contents/MacOS/main", + ]); + expect(trace.coverage.status).toBe("complete"); + }, +); diff --git a/tests/acceptance/artifactExtractionCoverage.test.ts b/tests/acceptance/artifactExtractionCoverage.test.ts new file mode 100644 index 0000000000..f22ba370f4 --- /dev/null +++ b/tests/acceptance/artifactExtractionCoverage.test.ts @@ -0,0 +1,189 @@ +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { join } from "node:path"; + +import { expect, onTestFinished } from "vitest"; + +import { artifactExtractionResultSchema } from "../../src/domain/artifactGraph.js"; +import { parseEvidence } from "../../src/domain/evidence.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; +import { + cArchiveExecutable, + pyzArchive, + PYTHON_39_MAGIC, +} from "../../src/artifacts/pyinstaller/CArchive.fixture.js"; + +// APFS rejects the raw FF basename with EILSEQ; match the filename-byte lane. +cliTest.skipIf(process.platform !== "linux").for([false, true])( + "preserves extraction traversal uncertainty through CLI and stdio MCP (opaque name: %s)", + async (withOpaqueName, { cli }) => { + const root = await createTestTempDirectory("rea-extraction-coverage-"); + const source = join(root, "Fixture.app"); + await mkdir(join(source, "Contents", "MacOS"), { recursive: true }); + const macho = Buffer.alloc(32); + macho.writeUInt32LE(0xfeedfacf, 0); + macho.writeUInt32LE(0x0100000c, 4); + macho.writeUInt32LE(2, 12); + const files = new Map([ + [ + "Contents/Info.plist", + Buffer.from( + "CFBundlePackageTypeAPPLCFBundleExecutablefixture", + ), + ], + ["Contents/MacOS/fixture", macho], + ["Contents/good.txt", Buffer.from("retained bytes\n")], + ]); + for (const [path, bytes] of files) + await writeFile(join(source, path), bytes); + const opaque = Buffer.concat([ + Buffer.from(join(source, "Contents", "opaque-")), + Buffer.from([0xff]), + Buffer.from(".txt"), + ]); + if (withOpaqueName) await writeFile(opaque, "omitted bytes\n"); + + const cliResult = await cli.run({ + arguments: ["extract-artifact", source, "--json"], + }); + expect(cliResult.exitCode, cliResult.stdout + cliResult.stderr).toBe(0); + const cliEvidence = parseEvidence(cliResult.json); + const cliExtraction = artifactExtractionResultSchema.parse( + cliEvidence.normalized_result, + ); + onTestFinished(() => + rm(cliExtraction.output_root, { recursive: true, force: true }), + ); + const call = await cli.sharedMcpCall?.(); + if (call === undefined) throw new Error("Expected a stdio MCP session"); + const opened = await call("open_binary", { path: source }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const reply = await call("extract_artifact", {}); + expect(reply.isError, JSON.stringify(reply)).not.toBe(true); + const mcpEvidence = parseEvidence(reply.structuredContent); + const mcpExtraction = artifactExtractionResultSchema.parse( + mcpEvidence.normalized_result, + ); + onTestFinished(() => + rm(mcpExtraction.output_root, { recursive: true, force: true }), + ); + for (const { evidence, result } of [ + { evidence: cliEvidence, result: cliExtraction }, + { evidence: mcpEvidence, result: mcpExtraction }, + ]) { + expect( + result.artifacts.map(({ relative_path }) => relative_path), + ).toEqual([...files.keys()]); + for (const [path, bytes] of files) + expect(await readFile(join(result.output_root, path))).toEqual(bytes); + for (const limitations of [result.limitations, evidence.limitations]) { + expect( + limitations.some((value) => value.includes("not valid UTF-8")), + ).toBe(withOpaqueName); + expect( + limitations.some((value) => + value.includes("source traversal coverage is unknown"), + ), + ).toBe(withOpaqueName); + expect( + limitations.some((value) => value.startsWith("All regular files")), + ).toBe(!withOpaqueName); + } + } + if (withOpaqueName) + expect(await readFile(opaque, "utf8")).toBe("omitted bytes\n"); + }, +); + +cliTest.for([false, true])( + "retains PYZ traversal coverage through CLI and stdio MCP extraction (opaque: %s)", + async (opaque, { cli }) => { + const root = await createTestTempDirectory("rea-extraction-pyz-coverage-"); + const source = join(root, "fixture.exe"); + const pyz = pyzArchive({ + magic: opaque ? Buffer.alloc(4) : PYTHON_39_MAGIC, + modules: [], + }); + const healthy = Buffer.from("retained sibling bytes\n"); + await writeFile( + source, + cArchiveExecutable({ + carrier: "pe", + layout: "2.1", + pythonVersion: 309, + members: [ + { name: "PYZ.pyz", typeCode: "z", payload: pyz, compress: true }, + { + name: "good.txt", + typeCode: "x", + payload: healthy, + compress: false, + }, + ], + }), + ); + const cliResult = await cli.run({ + arguments: ["extract-artifact", source, "--json"], + }); + expect(cliResult.exitCode, cliResult.stdout + cliResult.stderr).toBe(0); + const cliEvidence = parseEvidence(cliResult.json); + const cliExtraction = artifactExtractionResultSchema.parse( + cliEvidence.normalized_result, + ); + onTestFinished(() => + rm(cliExtraction.output_root, { recursive: true, force: true }), + ); + const call = await cli.sharedMcpCall?.(); + if (call === undefined) throw new Error("Expected a stdio MCP session"); + const opened = await call("open_binary", { path: source }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const reply = await call("extract_artifact", {}); + expect(reply.isError, JSON.stringify(reply)).not.toBe(true); + const mcpEvidence = parseEvidence(reply.structuredContent); + const mcpExtraction = artifactExtractionResultSchema.parse( + mcpEvidence.normalized_result, + ); + onTestFinished(() => + rm(mcpExtraction.output_root, { recursive: true, force: true }), + ); + const files = opaque + ? ([ + ["PYZ.pyz", pyz], + ["good.txt", healthy], + ] as const) + : ([["good.txt", healthy]] as const); + for (const { evidence, result } of [ + { evidence: cliEvidence, result: cliExtraction }, + { evidence: mcpEvidence, result: mcpExtraction }, + ]) { + expect( + result.artifacts.map(({ relative_path }) => relative_path), + ).toEqual(files.map(([path]) => path)); + for (const [path, bytes] of files) { + expect(await readFile(join(result.output_root, path))).toEqual(bytes); + expect( + result.artifacts.find((file) => file.relative_path === path), + ).toMatchObject({ + sha256: createHash("sha256").update(bytes).digest("hex"), + bytes_written: bytes.length, + }); + } + for (const limitations of [result.limitations, evidence.limitations]) { + expect( + limitations.some((value) => + value.includes("PYZ archives remain opaque"), + ), + ).toBe(opaque); + expect( + limitations.some((value) => + value.includes("source traversal coverage is unknown"), + ), + ).toBe(opaque); + expect( + limitations.some((value) => value.startsWith("All regular files")), + ).toBe(!opaque); + } + } + }, +); diff --git a/tests/boundary/filesystem/interfaceBuilderBundle.test.ts b/tests/boundary/filesystem/interfaceBuilderBundle.test.ts index 0277f627f0..67a56c1f96 100644 --- a/tests/boundary/filesystem/interfaceBuilderBundle.test.ts +++ b/tests/boundary/filesystem/interfaceBuilderBundle.test.ts @@ -1,5 +1,5 @@ import { execFile } from "node:child_process"; -import { mkdir, writeFile } from "node:fs/promises"; +import { mkdir, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { promisify } from "node:util"; @@ -7,6 +7,7 @@ import { buildBinary } from "plist"; import { describe, expect, it, vi } from "vitest"; import { analyzeInterfaceBuilderBundle } from "../../../src/artifacts/apple/InterfaceBuilderAnalysis.js"; +import { DirectoryArtifactReader } from "../../../src/artifacts/DirectoryArtifactReader.js"; import { InterfaceBuilderDecodeBudget } from "../../../src/artifacts/apple/InterfaceBuilderDecodeBudget.js"; import { encodeNibArchiveFixture } from "../../../src/artifacts/apple/NibArchive.fixture.js"; import { thinMach } from "../../../src/domain/binaryTarget.fixture.js"; @@ -188,6 +189,132 @@ const encodeNibArchiveWithSharedDataValue = (input: { return Buffer.concat([header, objects, keys, values, classes]); }; +describe("compiled Interface Builder traversal coverage", () => { + it.skipIf(process.platform === "win32")( + "shares the 64-path omission budget between regular-document capacity and symlink source notices", + async () => { + const bundle = await createTestTempDirectory( + "rea-ib-mixed-omission-bound-", + ); + const bytes = encodeNibArchiveFixture({ + classes: ["NSView"], + objects: [{ classIndex: 0, values: {} }], + }); + await writeFile(join(bundle, "payload.data"), bytes); + // One regular document is decoded; the other 61 regular documents and + // all 65 symlinks are omitted. Exactly three symlink samples still fit. + for (let index = 0; index < 62; index += 1) + await writeFile( + join(bundle, `${String(index).padStart(3, "0")}.nib`), + bytes, + ); + for (let index = 0; index < 65; index += 1) + await symlink( + "payload.data", + join(bundle, `link-${String(index).padStart(3, "0")}.nib`), + ); + const result = await analyzeInterfaceBuilderBundle({ + bundlePath: bundle, + targetSha256: "a".repeat(64), + limits: { max_documents: 1 }, + }); + expect(result.documents).toHaveLength(1); + expect(result.graph.coverage).toContainEqual( + expect.objectContaining({ + facet: "archive_decode", + status: "partial", + reason: "document_limit_reached", + examined: 1, + omitted: 126, + }), + ); + const reasons = result.limitations.filter((value) => + value.includes(": omitted because "), + ); + const sourceNotices = result.limitations.filter((value) => + value.endsWith(": Symlink target was not followed or disclosed."), + ); + expect(reasons).toHaveLength(64); + expect(sourceNotices).toEqual( + [0, 1, 2].map( + (index) => + `link-${String(index).padStart(3, "0")}.nib: Symlink target was not followed or disclosed.`, + ), + ); + const reportedPaths = new Set( + [...reasons, ...sourceNotices].map((value) => + value.slice(0, value.indexOf(":")), + ), + ); + expect(reportedPaths.size).toBe(64); + expect( + result.limitations.some((value) => value.startsWith("link-003.nib:")), + ).toBe(false); + expect(result.limitations).toContain( + "62 additional omitted archive path(s) and their reasons were not retained; reporting is bounded to 64 paths.", + ); + }, + ); + + it("preserves unknown traversal after a later complete callback without inventing archive omissions", async () => { + const bundle = await createTestTempDirectory("rea-ib-traversal-callback-"); + const bytes = encodeNibArchiveFixture({ + classes: ["NSView"], + objects: [{ classIndex: 0, values: { title: "Retained view" } }], + }); + await writeFile(join(bundle, "Main.nib"), bytes); + const reason = "Controlled producer traversal uncertainty"; + const entries = DirectoryArtifactReader.prototype.entries; + const spy = vi + .spyOn(DirectoryArtifactReader.prototype, "entries") + .mockImplementation(async function* ( + this: DirectoryArtifactReader, + signal, + options, + ) { + yield* entries.call(this, signal, options); + if (typeof options === "function") return; + options?.onCoverage?.({ + status: "unknown", + examinedEntries: 1, + limitations: [reason], + }); + options?.onCoverage?.({ + status: "complete", + examinedEntries: 1, + limitations: [], + }); + }); + try { + const result = await analyzeInterfaceBuilderBundle({ + bundlePath: bundle, + targetSha256: "a".repeat(64), + }); + expect(result.documents).toHaveLength(1); + expect(result.documents[0]).toMatchObject({ relative_path: "Main.nib" }); + expect(result.graph.nodes).toContainEqual( + expect.objectContaining({ + kind: "view", + attributes: expect.objectContaining({ class_name: "NSView" }), + }), + ); + expect(result.graph.coverage).toContainEqual( + expect.objectContaining({ + facet: "archive_decode", + status: "partial", + reason: "source_traversal_unknown", + examined: 1, + omitted: 0, + }), + ); + expect(result.graph.truncated).toBe(true); + expect(result.limitations).toContain(reason); + } finally { + spy.mockRestore(); + } + }); +}); + describe("compiled Interface Builder bundle reader", () => { it("reads nib plist archives and reports provenance", async () => { const root = await createTestTempDirectory("rea-ib-test-"); diff --git a/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts b/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts index 53ffcd7d38..7f32f57c61 100644 --- a/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts +++ b/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts @@ -62,6 +62,150 @@ it("still refuses a regular file added after inventory and rolls back", async () await expect(access(output)).rejects.toThrow(); }); +it + .skipIf(process.platform !== "linux") + .each(["inventory", "materialization"] as const)( + "retains unknown filename coverage established during %s", + async (phase) => { + const root = await createTestTempDirectory( + "rea-extract-unknown-traversal-", + ); + const source = join(root, "source"); + await mkdir(source); + await writeFile(join(source, "known.txt"), "retained bytes\n"); + const opaque = Buffer.concat([ + Buffer.from(join(source, "opaque-")), + Buffer.from([0xff]), + ]); + if (phase === "inventory") await writeFile(opaque, "opaque bytes\n"); + const output = join(root, "output"); + const materialize = await inventoriedExtraction(source, output); + if (phase === "inventory") await rm(opaque); + else await writeFile(opaque, "opaque bytes\n"); + + const result = await materialize(); + expect(result.artifacts.map(({ relative_path }) => relative_path)).toEqual([ + "known.txt", + ]); + expect(await readFile(join(output, "known.txt"), "utf8")).toBe( + "retained bytes\n", + ); + expect(result.limitations).toContainEqual( + expect.stringContaining("not valid UTF-8"), + ); + expect(result.limitations).toContainEqual( + expect.stringContaining("source traversal coverage is unknown"), + ); + expect( + result.limitations.some((value) => value.startsWith("All regular files")), + ).toBe(false); + }, +); + +it.each(["inventory", "materialization"] as const)( + "retains typed unknown coverage from only the %s producer stage", + async (phase) => { + const root = await createTestTempDirectory("rea-extract-stage-coverage-"); + const source = join(root, "source"); + await mkdir(source); + await writeFile(join(source, "known.txt"), "retained bytes\n"); + const output = join(root, "output"); + let stage: "inventory" | "materialization" = "inventory"; + const limitation = `Producer traversal coverage is unknown during ${phase}.`; + const entries = DirectoryArtifactReader.prototype.entries; + const spy = vi + .spyOn(DirectoryArtifactReader.prototype, "entries") + .mockImplementation(async function* ( + this: DirectoryArtifactReader, + signal, + selection, + ) { + yield* entries.call(this, signal, selection); + if (typeof selection === "function") return; + if (stage === phase) + selection?.onCoverage?.({ + status: "unknown", + limitations: [limitation], + }); + // A later complete observation must not erase established uncertainty. + selection?.onCoverage?.({ status: "complete", limitations: [] }); + }); + try { + const materialize = await inventoriedExtraction(source, output); + stage = "materialization"; + const result = await materialize(); + expect(result.limitations).toContain(limitation); + expect(result.limitations).toContainEqual( + expect.stringContaining("source traversal coverage is unknown"), + ); + expect( + result.limitations.some((value) => + value.startsWith("All regular files"), + ), + ).toBe(false); + expect( + result.artifacts.map(({ relative_path }) => relative_path), + ).toEqual(["known.txt"]); + expect(await readFile(join(output, "known.txt"), "utf8")).toBe( + "retained bytes\n", + ); + } finally { + spy.mockRestore(); + } + }, +); + +it("normalizes an NFD omission once across inventory and materialization", async () => { + const root = await createTestTempDirectory("rea-extract-omission-nfc-"); + const source = join(root, "source"); + await mkdir(source); + await writeFile(join(source, "known.txt"), "retained bytes\n"); + const output = join(root, "output"); + const entries = DirectoryArtifactReader.prototype.entries; + const spy = vi + .spyOn(DirectoryArtifactReader.prototype, "entries") + .mockImplementation(async function* ( + this: DirectoryArtifactReader, + signal, + selection, + ) { + yield* entries.call(this, signal, selection); + if (typeof selection === "function") return; + selection?.onOmission?.({ path: "cafe\u0301.pipe", kind: "fifo" }); + selection?.onCoverage?.({ status: "complete", limitations: [] }); + }); + try { + const snapshot = await scanArtifactInventory(source); + expect(snapshot.coverage.status).toBe("unknown"); + const result = await materializeArtifactInventory( + { + inputPath: source, + inputFormat: "asar", + outputRoot: output, + environment: process.env, + integrityPolicy: "fail", + }, + source, + snapshot, + ); + const omission = 'Artifact traversal omitted fifo at "caf\u00e9.pipe".'; + for (const limitations of [snapshot.limitations, result.limitations]) + expect( + limitations.filter((value) => + value.startsWith("Artifact traversal omitted"), + ), + ).toEqual([omission]); + expect(result.limitations).toContainEqual( + expect.stringContaining("source traversal coverage is unknown"), + ); + expect(await readFile(join(output, "known.txt"), "utf8")).toBe( + "retained bytes\n", + ); + } finally { + spy.mockRestore(); + } +}); + it("rolls back when cancellation arrives while closing the materialization reader", async () => { const root = await createTestTempDirectory("rea-extract-cancel-close-"); const source = join(root, "source");