refactor(resources): own the stable-root digest with classification as an observer (#11445)

* refactor(resources): own the stable-root digest with classification as an observer

* refactor(resources): move the artifact file source into src/resources (#11484)
This commit is contained in:
morluto authored and GitHub committed 2026-11-16 05:25:40 +08:00
1 parent 8c9cceceb4
commit c24f0085ae
58 files changed
+263 -183

No files matched your search

+2 -2
View File
@@ -22,7 +22,7 @@
"apktool -> artifacts": 2,
"artifacts -> dotnet": 3,
"artifacts -> native": 1,
"browser -> artifacts": 6,
"browser -> artifacts": 4,
"captures -> pcap": 2,
"desktop -> browser": 1,
"desktop -> electron": 1,
@@ -31,7 +31,7 @@
"il2cpp -> ghidra": 1,
"inspector -> browser": 2,
"model -> artifacts": 1,
"native -> artifacts": 5
"native -> artifacts": 2
},
"entry": {
"application -> artifacts": 7,
+1 -1
View File
@@ -1,7 +1,7 @@
import type { FileHandle } from "node:fs/promises";
import { createHash } from "node:crypto";
import { readFileHandleRange } from "../filesystem/readFileHandleRange.js";
import { abortIfNeeded } from "./ArtifactHash.js";
import { abortIfNeeded } from "../resources/index.js";
interface ArtifactRangeCarrier {
readonly handle: FileHandle;
@@ -20,7 +20,6 @@ import {
} from "../filesystem/RegularFile.js";
import { findDotnetBundleHeader } from "./dotnet/DotnetBundle.js";
import { ArtifactReaderFailure } from "../resources/index.js";
import { abortIfNeeded } from "./ArtifactHash.js";
import { locateGodotPck, type GodotPckLocation } from "./GodotPckLocation.js";
import {
locateNodeSea,
@@ -31,6 +30,7 @@ import {
type PyInstallerCookie,
type PyInstallerCookieObservation,
} from "./pyinstaller/CArchiveLayout.js";
import { abortIfNeeded } from "../resources/index.js";
/** Completed observations and explicit recognition limits; null alone is examined absence. */
export interface ArtifactEmbeddedFormatProbeFacts {
+37 -36
View File
@@ -1,30 +1,21 @@
import type { AsarHeaderAdmission } from "../config/asarHeaderAdmission.js";
import { createHash } from "node:crypto";
import type { FileHandle } from "node:fs/promises";
import type { Stats } from "node:fs";
import type { Readable } from "node:stream";
import { ArtifactReaderFailure } from "../resources/index.js";
import { streamChunkToBuffer } from "../resources/index.js";
import {
abortIfNeeded,
digestReadableToCompletion,
hashStableRootArtifactHandle,
type StreamDigest,
} from "../resources/index.js";
import {
createArtifactClassificationProbe,
type ArtifactClassificationEvidence,
} from "./ArtifactClassificationProbe.js";
/** Full stream digest and byte count with bounded classification evidence. */
export type HashResult = ArtifactClassificationEvidence & {
readonly sha256: string;
readonly bytes: number;
readonly md5?: string;
};
/** Preserve the inventory cancellation reason at a stream boundary. */
export const abortIfNeeded = (signal?: AbortSignal): void => {
if (signal?.aborted === true)
throw new ArtifactReaderFailure(
"cancelled",
"Artifact inventory cancelled",
{ cause: signal.reason },
);
};
export type HashResult = ArtifactClassificationEvidence & StreamDigest;
/** Hash every stream byte, optionally feeding the same bytes to a scan aggregate. */
export const hashReadable = async (
@@ -55,23 +46,33 @@ export const hashReadableToCompletion = async (
asarHeaderAdmission?: AsarHeaderAdmission,
observeChunk?: (chunk: Buffer) => void,
): Promise<HashResult> => {
const hash = createHash("sha256");
const md5 = integrityAlgorithm === "md5" ? createHash("md5") : undefined;
const probe = createArtifactClassificationProbe(asarHeaderAdmission);
let bytes = 0;
for await (const raw of stream) {
abortIfNeeded(signal);
const chunk = streamChunkToBuffer(raw);
bytes += chunk.length;
hash.update(chunk);
observeChunk?.(chunk);
md5?.update(chunk);
probe.write(chunk);
}
return {
sha256: hash.digest("hex"),
...(md5 === undefined ? {} : { md5: md5.digest("hex") }),
bytes,
...probe.finish(bytes),
};
const digest = await digestReadableToCompletion(
stream,
signal,
integrityAlgorithm,
(chunk) => {
observeChunk?.(chunk);
probe.write(chunk);
},
);
return { ...digest, ...probe.finish(digest.bytes) };
};
/** Hash and revalidate an open root descriptor, with the root's classification evidence. */
export const classifyStableRootArtifactHandle = async (
path: string,
handle: FileHandle,
initial: Stats,
signal?: AbortSignal,
): Promise<HashResult> => {
const probe = createArtifactClassificationProbe(undefined);
const digest = await hashStableRootArtifactHandle(
path,
handle,
initial,
signal,
(chunk) => probe.write(chunk),
);
return { ...digest, ...probe.finish(digest.bytes) };
};
+1 -1
View File
@@ -30,7 +30,7 @@ import {
} from "./apple/KeyedArchiveReader.js";
import { traceDylibResolution } from "./apple/DylibResolutionReader.js";
import { dirname } from "node:path";
import { withArtifactTargetIdentity } from "./withArtifactTargetIdentity.js";
import { withArtifactTargetIdentity } from "../resources/index.js";
import { inventoryArtifact } from "./inventory/ArtifactInventory.js";
import { extractArtifact } from "./extraction/ArtifactExtraction.js";
import { analyzeInterfaceBuilderBundle } from "./apple/InterfaceBuilderAnalysis.js";
@@ -4,13 +4,13 @@ import { expect, it } from "vitest";
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
import { openRegularFile } from "../filesystem/RegularFile.js";
import { assertArtifactFileSelectedSource } from "./ArtifactFileSource.js";
import { assertArtifactSelectedSource } from "./ArtifactSelectedSource.js";
import { readStableArtifact } from "../resources/readStableArtifact.js";
import { DirectoryArtifactReader } from "./DirectoryArtifactReader.js";
import { createDesktopApplicationService } from "../composition/desktop.js";
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
import { desktopApplicationSchema } from "../domain/desktopApplication.js";
import { assertArtifactFileSelectedSource } from "../resources/index.js";
it.skipIf(process.platform === "win32")(
"admits a caller-selected directory symlink without changing its target identity",
+1 -1
View File
@@ -26,9 +26,9 @@ import {
ArtifactResourceScope,
normalizeArtifactPath,
ArtifactPathRegistry,
ArtifactFileSource,
artifactReaderLimitFailure,
} from "../resources/index.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { ownArtifactStream } from "./OwnedArtifactStream.js";
import {
bunFormatFailure,
+1 -1
View File
@@ -19,8 +19,8 @@ import {
type ArtifactTraversalOptions,
ArtifactResourceScope,
assertArtifactReadActive,
ArtifactFileSource,
} from "../resources/index.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
@@ -21,11 +21,9 @@ import {
type ArtifactReader,
type ArtifactTraversalOptions,
ArtifactResourceScope,
} from "../resources/index.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import {
ArtifactPathRegistry,
normalizeArtifactPath,
ArtifactFileSource,
} from "../resources/index.js";
import { ownArtifactStream } from "./OwnedArtifactStream.js";
import type { DenoStandaloneAdmission } from "./deno/DenoStandaloneAdmission.js";
+1 -1
View File
@@ -12,7 +12,6 @@ import {
ArtifactResourceScope,
type ArtifactResourceOwner,
} from "../resources/index.js";
import { abortIfNeeded } from "./ArtifactHash.js";
import { declaredArtifactEntries } from "./ArtifactTraversalObservation.js";
import { translateArtifactFailure } from "./ArtifactOperationFailure.js";
import {
@@ -45,6 +44,7 @@ import type {
AnalysisPartialObservation,
} from "../domain/analysisErrorBase.js";
import { BinaryTargetError } from "../domain/configurationErrors.js";
import { abortIfNeeded } from "../resources/index.js";
const RETAINED_METADATA_BYTES = 32 * 1024 * 1024;
/**
+1 -1
View File
@@ -22,12 +22,12 @@ import { readManagedCliHeaderPrefix } from "../domain/managed/managedCliHeaderPr
import { readAssemblyReferenceName } from "../dotnet/ManagedMetadataInventoryRows.js";
import { ManagedReaderFailure } from "../dotnet/ManagedReaderFailure.js";
import { ArtifactReaderFailure } from "../resources/index.js";
import { abortIfNeeded } from "./ArtifactHash.js";
import { hasMachoMagic } from "../domain/apple/machoContainer.js";
import {
hasUnityFsHeader,
readUnityFsHeaderFacts,
} from "../domain/unityfsFormat.js";
import { abortIfNeeded } from "../resources/index.js";
export interface DesktopReadBudget {
bytes: number;
+3 -3
View File
@@ -7,7 +7,6 @@ import {
} from "../domain/artifactReaderLimits.js";
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import {
ArtifactResourceScope,
ArtifactReaderFailure,
@@ -16,10 +15,11 @@ import {
type ArtifactEntry,
type ArtifactReader,
type ArtifactTraversalOptions,
readArtifactFileChunks,
abortIfNeeded,
ArtifactFileSource,
} from "../resources/index.js";
import type { DotnetBundleAdmission } from "./dotnet/DotnetBundleAdmission.js";
import { readArtifactFileChunks } from "../resources/index.js";
import { abortIfNeeded } from "./ArtifactHash.js";
import {
findDotnetBundleHeader,
readDotnetBundle,
+1 -1
View File
@@ -7,7 +7,6 @@ import { expect, it, vi } from "vitest";
import { FitArtifactReader } from "./FitArtifactReader.js";
import { ArtifactResourceScope } from "../resources/ArtifactResourceScope.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { ArtifactReaderFailure } from "../resources/ArtifactReader.js";
import { extractArtifact } from "./extraction/ArtifactExtraction.js";
import * as fitStructure from "./fit/FitStructure.js";
@@ -26,6 +25,7 @@ import {
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
import { phaseGate } from "../../tests/support/lifecycleHarness.js";
import { scanCanonicalArtifactInventory } from "../../tests/fixtures/artifactInventory.js";
import { ArtifactFileSource } from "../resources/index.js";
const MIB = 1024 * 1024;
+1 -1
View File
@@ -14,9 +14,9 @@ import {
type ArtifactTraversalOptions,
ArtifactResourceScope,
assertArtifactReadActive,
ArtifactFileSource,
} from "../resources/index.js";
import { ownArtifactStream } from "./OwnedArtifactStream.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import {
createFitDigest,
readFitStructure,
@@ -1,5 +1,4 @@
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { openRegularFile } from "../filesystem/RegularFile.js";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
@@ -19,6 +18,7 @@ import {
} from "../domain/firmware/hexRecords.fixture.js";
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
import { scanCanonicalArtifactInventory } from "../../tests/fixtures/artifactInventory.js";
import { ArtifactFileSource } from "../resources/index.js";
const writeImage = async (name: string, text: string): Promise<string> => {
const directory = await createTestTempDirectory("rea-hex-records-");
+1 -1
View File
@@ -25,7 +25,7 @@ import {
streamChunkToBuffer,
} from "../resources/index.js";
import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
import { abortIfNeeded } from "./ArtifactHash.js";
import { abortIfNeeded } from "../resources/index.js";
export const INES_LIMITATION =
"Classic iNES file regions are observed; mapper execution, CPU bank mappings, CHR-RAM contents, 6502 analysis and emulator behavior are not observed. Trailing bytes are retained without interpretation. NES 2.0 and UNIF are unsupported.";
+1 -1
View File
@@ -9,6 +9,7 @@ import {
type ArtifactTraversalOptions,
streamChunkToBuffer,
readArtifactFileChunks,
ArtifactFileSource,
} from "../resources/index.js";
import type { FileHandle } from "node:fs/promises";
import { Readable } from "node:stream";
@@ -29,7 +30,6 @@ import {
type ReadAt,
} from "../domain/apple/machoLoadCommandReader.js";
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { readFileHandleRange } from "../filesystem/readFileHandleRange.js";
import type { ArtifactIntegrityPolicyName } from "../domain/artifactIntegrityPolicy.js";
import {
+1 -1
View File
@@ -3,7 +3,6 @@ import { join } from "node:path";
import { expect, it, vi } from "vitest";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { assertArtifactReadActive } from "../resources/ArtifactFileOperation.js";
import { NtfsMftArtifactReader } from "./NtfsMftArtifactReader.js";
@@ -15,6 +14,7 @@ import { phaseGate } from "../../tests/support/lifecycleHarness.js";
const MFT_DEFAULT_RECORD_BYTES = 1024;
import { linkedMftFixture } from "../../tests/fixtures/ntfsMftExtension.js";
import { scanCanonicalArtifactInventory } from "../../tests/fixtures/artifactInventory.js";
import { ArtifactFileSource } from "../resources/index.js";
/** A 1 KiB FILE record whose update sequence array covers both sectors. */
const recordBytes = (options: {
+1 -1
View File
@@ -18,8 +18,8 @@ import {
type ArtifactTraversalOptions,
ArtifactResourceScope,
assertArtifactReadActive,
ArtifactFileSource,
} from "../resources/index.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
+1 -1
View File
@@ -12,9 +12,9 @@ import {
type ArtifactTraversalOptions,
ArtifactResourceScope,
assertArtifactReadActive,
ArtifactFileSource,
} from "../resources/index.js";
import { ownArtifactStream } from "./OwnedArtifactStream.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
+13 -13
View File
@@ -4,7 +4,19 @@ import { finished } from "node:stream/promises";
import { setImmediate } from "node:timers/promises";
import { jsonValueByteLength } from "../domain/jsonValueByteLength.js";
import type { JsonValue } from "../domain/jsonValue.js";
import { normalizeArtifactPath } from "../resources/index.js";
import {
normalizeArtifactPath,
ArtifactReaderFailure,
assertProducedEntry,
type ArtifactEntry,
type ArtifactReader,
type ArtifactReaderTraversal,
type ArtifactTraversalOptions,
artifactReadCancelledBy,
ArtifactResourceScope,
abortIfNeeded,
ArtifactFileSource,
} from "../resources/index.js";
import type { FileHandle } from "node:fs/promises";
import type { PyInstallerMetadata } from "../domain/artifactFormatMetadata.js";
@@ -14,18 +26,6 @@ import {
} from "./pyinstaller/PyInstallerMetadata.js";
import { artifactExpansionBudgetBytes } from "../domain/artifactExpansionBudget.js";
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
import {
ArtifactReaderFailure,
assertProducedEntry,
type ArtifactEntry,
type ArtifactReader,
type ArtifactReaderTraversal,
type ArtifactTraversalOptions,
artifactReadCancelledBy,
ArtifactResourceScope,
} from "../resources/index.js";
import { abortIfNeeded } from "./ArtifactHash.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { parseMzHeaderEvidence } from "../domain/dosMz.js";
import {
readFatContainerHeader,
+1 -1
View File
@@ -55,7 +55,7 @@ import {
hashStableRootArtifactHandle,
openRootArtifact,
releaseRootArtifact,
} from "./inventory/hashStableRootArtifact.js";
} from "../resources/index.js";
import { createArtifactReader } from "./createArtifactReader.js";
import { AnalysisError } from "../domain/analysisErrorBase.js";
import {
+1 -1
View File
@@ -8,7 +8,6 @@ import type {
UImageEntryMetadata,
UImageMetadata,
} from "../domain/artifactFormatMetadata.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import {
assertArtifactReadActive,
ArtifactReaderFailure,
@@ -18,6 +17,7 @@ import {
type ArtifactReader,
type ArtifactTraversalOptions,
ArtifactResourceScope,
ArtifactFileSource,
} from "../resources/index.js";
import { ownArtifactStream } from "./OwnedArtifactStream.js";
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
@@ -1,5 +1,4 @@
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { truncate, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { buffer, text } from "node:stream/consumers";
@@ -12,6 +11,7 @@ import { buildUnityFs } from "../../tests/fixtures/unityfs.js";
import { openRegularFile } from "../filesystem/RegularFile.js";
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
import { UnityFsArtifactReader } from "./UnityFsArtifactReader.js";
import { ArtifactFileSource } from "../resources/index.js";
const fixture = async (bytes: Buffer): Promise<string> => {
const path = join(
+1 -1
View File
@@ -14,6 +14,7 @@ import {
type ArtifactTraversalOptions,
ArtifactResourceScope,
assertArtifactReadActive,
ArtifactFileSource,
} from "../resources/index.js";
import { ownAsyncArtifactStream } from "./OwnedAsyncArtifactStream.js";
import { readArtifactFileBytes } from "./ArtifactFileRange.js";
@@ -22,7 +23,6 @@ import type {
UnityFsMetadata,
} from "../domain/artifactFormatMetadata.js";
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { UNITYFS_COMPRESSION_NONE } from "../domain/unityfsFormat.js";
import {
decodeUnityFsLz4,
@@ -1,5 +1,4 @@
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { once } from "node:events";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
@@ -13,6 +12,7 @@ import { buildWxapkg } from "../../tests/fixtures/wxapkg.js";
import { openRegularFile } from "../filesystem/RegularFile.js";
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
import { WxapkgArtifactReader } from "./WxapkgArtifactReader.js";
import { ArtifactFileSource } from "../resources/index.js";
const fixture = async (bytes: Buffer): Promise<string> => {
const path = join(
+1 -1
View File
@@ -13,6 +13,7 @@ import {
type ArtifactTraversalOptions,
ArtifactResourceScope,
assertArtifactReadActive,
ArtifactFileSource,
} from "../resources/index.js";
import { ownAsyncArtifactStream } from "./OwnedAsyncArtifactStream.js";
import { readArtifactFileBytes } from "./ArtifactFileRange.js";
@@ -21,7 +22,6 @@ import type {
WxapkgMetadata,
} from "../domain/artifactFormatMetadata.js";
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import {
readWxapkgFileTable,
readWxapkgHeader,
+1 -1
View File
@@ -8,6 +8,7 @@ import {
type ArtifactReader,
type ArtifactReaderTraversal,
type ArtifactTraversalOptions,
ArtifactFileSource,
artifactReaderLimitFailure,
} from "../resources/index.js";
import type { FileHandle } from "node:fs/promises";
@@ -50,7 +51,6 @@ import {
import type { ZipPackageFormat } from "../domain/zipPackageFormat.js";
import { artifactExpansionBudgetBytes } from "../domain/artifactExpansionBudget.js";
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { readFileHandleRange } from "../filesystem/readFileHandleRange.js";
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
import { decodeZipCp437 } from "../domain/zip/zipEntryText.js";
+1 -1
View File
@@ -25,7 +25,7 @@ import {
} from "../../resources/index.js";
import { ArtifactTraversalObservation } from "../ArtifactTraversalObservation.js";
import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js";
import { targetDigestChangedFailure } from "../withArtifactTargetIdentity.js";
import { targetDigestChangedFailure } from "../../resources/index.js";
import { consumeWithOrderedReads } from "../OrderedReads.js";
import {
DylibFileAccess,
+1 -1
View File
@@ -1,5 +1,5 @@
import { HOST_PLATFORM } from "../../platform/HostPlatform.js";
import { targetDigestChangedFailure } from "../withArtifactTargetIdentity.js";
import { targetDigestChangedFailure } from "../../resources/index.js";
import { createHash } from "node:crypto";
import {
decodeLimit,
+1 -1
View File
@@ -14,9 +14,9 @@ import {
DEFAULT_BUN_READER_LIMITS,
type BunReaderLimits,
} from "../../domain/artifactReaderLimits.js";
import { abortIfNeeded } from "../ArtifactHash.js";
import {
ArtifactReaderFailure,
abortIfNeeded,
artifactReaderLimitFailure,
} from "../../resources/index.js";
+1 -1
View File
@@ -1,9 +1,9 @@
import { fixedAllowanceRemediation } from "../../domain/fixedAllowanceRemediation.js";
import { jsonValueSchema, type JsonValue } from "../../domain/jsonValue.js";
import { abortIfNeeded } from "../ArtifactHash.js";
import { ArtifactReaderFailure } from "../../resources/index.js";
import { walkDenoVirtualFs, type DenoVfsWalk } from "./DenoVirtualFs.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
import { abortIfNeeded } from "../../resources/index.js";
/** Opening and closing marker of a Deno compile data section. */
export const DENO_SECTION_MAGIC = Buffer.from("d3n0l4nd", "latin1");
+1 -2
View File
@@ -39,8 +39,7 @@ import {
readSliceCommandTable,
type ReadAt,
} from "../../domain/apple/machoLoadCommandReader.js";
import { abortIfNeeded } from "../ArtifactHash.js";
import { ArtifactReaderFailure } from "../../resources/index.js";
import { abortIfNeeded, ArtifactReaderFailure } from "../../resources/index.js";
import { locatePeResourcePayloads } from "../pe/PeResourceLayout.js";
import { DENO_SECTION_MAGIC } from "./DenoSectionLayout.js";
+1 -1
View File
@@ -1,7 +1,7 @@
import { fixedAllowanceRemediation } from "../../domain/fixedAllowanceRemediation.js";
import { abortIfNeeded } from "../ArtifactHash.js";
import { ArtifactReaderFailure } from "../../resources/index.js";
import { normalizeArtifactPath } from "../../resources/index.js";
import { abortIfNeeded } from "../../resources/index.js";
/** Directory, file, and symlink records admitted from one virtual filesystem. */
const MAX_DENO_VFS_ENTRIES = 100_000;
+1 -1
View File
@@ -11,8 +11,8 @@ import {
ArtifactReaderFailure,
artifactFixedLimitFailure,
artifactReaderLimitFailure,
abortIfNeeded,
} from "../../resources/index.js";
import { abortIfNeeded } from "../ArtifactHash.js";
import { strictUtf8Decoder } from "../../domain/strictUtf8.js";
/** SHA-256 marker embedded by Microsoft's apphost; the preceding UInt64 is the header offset. */
+16 -20
View File
@@ -1,7 +1,6 @@
import { selectArtifactExtractionOccurrences } from "./ArtifactExtractionSelection.js";
import type { ArtifactReaderLimits } from "../../domain/artifactReaderLimits.js";
import { assertAdmittedRootDigest } from "../inventory/classify.js";
import { assertArtifactFileSelectedSource } from "../ArtifactFileSource.js";
import {
assertArtifactSelectedSource,
resolveArtifactSourcePath,
@@ -10,7 +9,22 @@ import { InesArtifactReader } from "../InesArtifactReader.js";
import {
openRootArtifact,
hashStableRootArtifactHandle,
} from "../inventory/hashStableRootArtifact.js";
abortIfNeeded,
withArtifactRegularFile,
type ArtifactCleanupAttempt,
type ArtifactResourceOwner,
type ArtifactResourceScope,
ArtifactReaderFailure,
type ArtifactEntry,
type ArtifactReader,
type ArtifactReaderTraversal,
SafeOutputTreeDestinationRefusal,
ArtifactPathRegistry,
normalizeArtifactPath,
SafeOutputTree,
SafeOutputTreeCreationFailure,
assertArtifactFileSelectedSource,
} from "../../resources/index.js";
import { absoluteCallerPath } from "../../filesystem/callerPath.js";
import type { AsarHeaderAdmission } from "../../config/asarHeaderAdmission.js";
import type { EspFlashSelection } from "../../domain/firmware/espFlashLocation.js";
@@ -21,31 +35,13 @@ import {
ArtifactEmbeddedFormatProbes,
type ArtifactEmbeddedFormatProbeFacts,
} from "../ArtifactEmbeddedFormatProbes.js";
import {
withArtifactRegularFile,
type ArtifactCleanupAttempt,
type ArtifactResourceOwner,
type ArtifactResourceScope,
ArtifactReaderFailure,
type ArtifactEntry,
type ArtifactReader,
type ArtifactReaderTraversal,
} from "../../resources/index.js";
import { lstat, realpath } from "node:fs/promises";
import { dirname } from "node:path";
import { isPathWithinRoot } from "../../domain/localPath.js";
import {
SafeOutputTreeDestinationRefusal,
ArtifactPathRegistry,
normalizeArtifactPath,
SafeOutputTree,
SafeOutputTreeCreationFailure,
} from "../../resources/index.js";
import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js";
import { digestCanonicalValue } from "../../domain/canonicalDigest.js";
import { ArtifactTraversalObservation } from "../ArtifactTraversalObservation.js";
import { abortIfNeeded } from "../ArtifactHash.js";
import {
artifactExtractionResultSchema,
type ArtifactExtractionResult,
+1 -1
View File
@@ -5,7 +5,6 @@ import {
artifactInventoryResultSchema,
type ArtifactInventoryResult,
} from "../../domain/artifactGraph.js";
import { abortIfNeeded } from "../ArtifactHash.js";
import {
scanCanonicalArtifactInventoryInScope,
scanCanonicalArtifactInventoryForExtractionInScope,
@@ -14,6 +13,7 @@ import {
import type { ArtifactInventoryOptions } from "./types.js";
import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js";
import type { ArtifactEmbeddedFormatProbeFacts } from "../ArtifactEmbeddedFormatProbes.js";
import { abortIfNeeded } from "../../resources/index.js";
/** Inventory one local artifact and return every graph collection inline. */
export const inventoryArtifact = async (
+8 -6
View File
@@ -41,7 +41,6 @@ import type { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
import { decodeNodeSeaBlob } from "../nodeSea/NodeSeaBlob.js";
import { readNodeSeaBlob } from "../nodeSea/NodeSeaLocation.js";
import { classifyArtifactContent } from "./ArtifactGraphConstruction.js";
import { abortIfNeeded } from "../ArtifactHash.js";
import { claimsExecutableHeader } from "../../domain/binaryTarget.js";
import { createArtifactClassificationProbe } from "../ArtifactClassificationProbe.js";
import type { ArtifactOccurrence } from "../../domain/artifactGraph.js";
@@ -51,12 +50,15 @@ import {
zipRootPackageFormat,
} from "../../domain/zipPackageFormat.js";
import { artifactFormatDescriptor } from "../artifactFormats.js";
import type { HashResult } from "../ArtifactHash.js";
import {
hashStableRootArtifactHandle,
classifyStableRootArtifactHandle,
type HashResult,
} from "../ArtifactHash.js";
import {
openRootArtifact,
releaseRootArtifact,
} from "./hashStableRootArtifact.js";
abortIfNeeded,
} from "../../resources/index.js";
import type { ArtifactInventoryOptions } from "./types.js";
import { readMachoHeaderPrefixFacts } from "../../domain/apple/machoContainer.js";
import { hasDiskImageSignature } from "../../domain/apple/diskImageFormat.js";
@@ -160,7 +162,7 @@ export const classifyAndHashRootForInventory = async (
const admittedDigest =
options.expectedRootSha256 === undefined
? undefined
: await hashStableRootArtifactHandle(
: await classifyStableRootArtifactHandle(
path,
handle,
initial,
@@ -186,7 +188,7 @@ export const classifyAndHashRootForInventory = async (
typeof classified === "string" ? classified : classified.kind;
const digest =
admittedDigest ??
(await hashStableRootArtifactHandle(
(await classifyStableRootArtifactHandle(
path,
handle,
initial,
+1 -1
View File
@@ -48,7 +48,7 @@ import type { HashResult } from "../ArtifactHash.js";
import {
hashStableRootArtifact,
hashStableRootArtifactHandle,
} from "./hashStableRootArtifact.js";
} from "../../resources/index.js";
import { createReader, inventoryCoverage } from "./reader.js";
import {
scanReader,
+2 -1
View File
@@ -49,7 +49,8 @@ import {
type MutableOccurrence,
} from "./ArtifactGraphConstruction.js";
import type { ArtifactIntegrityPolicyName } from "../../domain/artifactIntegrityPolicy.js";
import { abortIfNeeded, hashReadable } from "../ArtifactHash.js";
import { hashReadable } from "../ArtifactHash.js";
import { abortIfNeeded } from "../../resources/index.js";
export interface PendingIntegrityContradiction {
readonly occurrence: MutableOccurrence;
+1 -1
View File
@@ -22,10 +22,10 @@ import {
import type { FileHandle } from "node:fs/promises";
import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js";
import { abortIfNeeded } from "../ArtifactHash.js";
import {
ArtifactReaderFailure,
artifactReaderLimitFailure,
abortIfNeeded,
} from "../../resources/index.js";
import {
DEFAULT_NODE_SEA_READER_LIMITS,
@@ -1,11 +1,11 @@
import type { FileHandle } from "node:fs/promises";
import { err, ok, type Result } from "../../domain/result.js";
import { ArtifactReaderFailure } from "../../resources/index.js";
import { abortIfNeeded } from "../ArtifactHash.js";
import {
pyInstallerFailure,
type PyInstallerFailure,
} from "./PyInstallerFailure.js";
import { abortIfNeeded } from "../../resources/index.js";
/** Fill one already-budgeted format range; only zero bytes establish EOF. */
export const readPyInstallerBytes = async (
@@ -2,7 +2,7 @@ import { Readable, Transform } from "node:stream";
import { pipeline } from "node:stream/promises";
import { createInflate } from "node:zlib";
import { ArtifactReaderFailure } from "../../resources/index.js";
import { abortIfNeeded } from "../ArtifactHash.js";
import { abortIfNeeded } from "../../resources/index.js";
interface MemberStreamOptions {
readonly sourceBytes: number;
@@ -15,9 +15,10 @@ import {
ArtifactReaderFailure,
ArtifactResourceScope,
type ArtifactResourceOwner,
readArtifactFileChunks,
abortIfNeeded,
artifactFileAdmissionFailure,
} from "../../resources/index.js";
import { abortIfNeeded } from "../ArtifactHash.js";
import { readArtifactFileChunks } from "../../resources/index.js";
import {
pyInstallerMemberStream,
readPyInstallerMemberPrefix,
@@ -27,7 +28,6 @@ import {
type CArchiveEntry,
} from "./CArchiveLayout.js";
import type { PyInstallerMember } from "./CArchiveMembers.js";
import { artifactFileAdmissionFailure } from "../ArtifactFileAdmission.js";
import {
reconstructedPycHeader,
isHeaderlessPythonCode,
@@ -7,10 +7,10 @@ import {
assertArtifactReadActive,
withArtifactRegularFile,
projectStableArtifactFailure,
hashStableRootArtifactHandle,
assertArtifactFileSelectedSource,
} from "../../../resources/index.js";
import { ZipArtifactReader } from "../../../artifacts/ZipArtifactReader.js";
import { assertArtifactFileSelectedSource } from "../../../artifacts/ArtifactFileSource.js";
import { hashStableRootArtifactHandle } from "../../../artifacts/inventory/hashStableRootArtifact.js";
import {
AnalysisResourceConstraintError,
AnalysisUnsupportedTargetError,
+1 -1
View File
@@ -10,9 +10,9 @@ import {
AnalysisCleanupLeaseGroup,
ArtifactResourceScope,
ArtifactReaderFailure,
withArtifactTargetIdentity,
} from "../resources/index.js";
import { MAX_NODE_TIMER_DELAY_MS } from "../domain/analysisResourceLimits.js";
import { withArtifactTargetIdentity } from "../artifacts/withArtifactTargetIdentity.js";
import {
invariantViolation,
InternalInvariantError,
+4 -2
View File
@@ -7,8 +7,10 @@ import {
ArtifactReaderFailure,
type ArtifactResourceScope,
} from "../resources/index.js";
import { hashStableRootArtifactHandle } from "../artifacts/inventory/hashStableRootArtifact.js";
import { targetDigestChangedFailure } from "../artifacts/withArtifactTargetIdentity.js";
import {
hashStableRootArtifactHandle,
targetDigestChangedFailure,
} from "../resources/index.js";
/** A positional reader for one slice of the descriptor bound to the target. */
export interface NativeMachoSource {
@@ -4,7 +4,7 @@ import {
RegularFileChangedError,
RegularFileOperationCancelled,
} from "../filesystem/RegularFile.js";
import { ArtifactReaderFailure } from "../resources/index.js";
import { ArtifactReaderFailure } from "./ArtifactReader.js";
/** Project physical admission failures without taking the caller's descriptor ownership. */
export const artifactFileAdmissionFailure = (
@@ -1,4 +1,4 @@
import { lstat } from "node:fs/promises";
import { lstat, type FileHandle } from "node:fs/promises";
import { artifactIntegrityDetails } from "../domain/artifactIntegrity.js";
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
@@ -12,11 +12,11 @@ import {
artifactSelectedPathErrno,
artifactReadCancelledBy,
assertArtifactReadActive,
ArtifactReaderFailure,
ArtifactResourceScope,
} from "../resources/index.js";
} from "./ArtifactFileOperation.js";
import { ArtifactReaderFailure } from "./ArtifactReader.js";
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
import { digestReadableToCompletion } from "./StreamDigest.js";
import { artifactFileAdmissionFailure } from "./ArtifactFileAdmission.js";
import { hashReadable } from "./ArtifactHash.js";
/** Verify an admitted descriptor against the regular file still named by its path. */
export const assertArtifactFileSelectedSource = async (
@@ -118,6 +118,30 @@ export class ArtifactFileSource {
return this.#scope.close();
}
/**
* Digest the bound descriptor's bytes; only the SHA-256 identity is used.
* A fully delivered digest is returned even when cancellation arrives
* after its last chunk, so the caller validates the source first.
*/
async #sha256(
handle: FileHandle,
signal: AbortSignal | undefined,
): Promise<string> {
const { sha256 } = await digestReadableToCompletion(
readFileHandleChunks(handle, {
signal,
cancellationFailure: (reason) =>
new ArtifactReaderFailure(
"cancelled",
`${this.diagnosticLabel} verification cancelled`,
{ cause: reason },
),
}),
signal,
);
return sha256;
}
async #bind(
expectedSha256: string,
signal: AbortSignal | undefined,
@@ -125,22 +149,7 @@ export class ArtifactFileSource {
verifyCompletion: () => Promise<StableRegularFileDescriptor>,
): Promise<void> {
const { handle } = await verify();
const observed =
this.#boundSha256 ??
(
await hashReadable(
readFileHandleChunks(handle, {
signal,
cancellationFailure: (reason) =>
new ArtifactReaderFailure(
"cancelled",
`${this.diagnosticLabel} verification cancelled`,
{ cause: reason },
),
}),
signal,
)
).sha256;
const observed = this.#boundSha256 ?? (await this.#sha256(handle, signal));
// A completed digest needs source validation before late cancellation can
// decide the outcome; unverified bytes must not establish a contradiction.
await verifyCompletion();
@@ -1,13 +1,11 @@
import {
artifactOperationCause,
ArtifactReaderFailure,
type ArtifactResourceScope,
} from "../../resources/index.js";
import { regularFileCancellation } from "../../filesystem/OwnedRegularFile.js";
import { artifactOperationCause } from "./ArtifactFileOperation.js";
import { ArtifactReaderFailure } from "./ArtifactReader.js";
import type { ArtifactResourceScope } from "./ArtifactResourceScope.js";
import { regularFileCancellation } from "../filesystem/OwnedRegularFile.js";
import type { Stats } from "node:fs";
import type { FileHandle } from "node:fs/promises";
import { readArtifactFileChunks } from "../../resources/index.js";
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
import { readArtifactFileChunks } from "./readArtifactFileChunks.js";
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
import {
assertStableRegularFileIdentity,
@@ -16,13 +14,14 @@ import {
RegularFileAdmissionFailure,
openRegularFile,
type StableRegularFileDescriptor,
} from "../../filesystem/RegularFile.js";
} from "../filesystem/RegularFile.js";
import {
abortIfNeeded,
hashReadable,
type HashResult,
} from "../ArtifactHash.js";
import { fileSnapshotPartialObservationSchema } from "../../domain/fileSnapshotObservation.js";
digestReadableToCompletion,
type StreamDigest,
type StreamDigestObserver,
} from "./StreamDigest.js";
import { fileSnapshotPartialObservationSchema } from "../domain/fileSnapshotObservation.js";
/** Finalize the admitted root while preserving its completed identity and exact owner. */
export const releaseRootArtifact = async (
@@ -30,7 +29,7 @@ export const releaseRootArtifact = async (
owner: OwnedFileHandle,
resourceScope: ArtifactResourceScope,
outcome:
| { readonly kind: "completed"; readonly digest: HashResult | null }
| { readonly kind: "completed"; readonly digest: StreamDigest | null }
| { readonly kind: "failed"; readonly cause: unknown },
signal?: AbortSignal,
): Promise<void> => {
@@ -114,14 +113,14 @@ export const hashStableRootArtifact = async (
path: string,
resourceScope: ArtifactResourceScope,
signal?: AbortSignal,
): Promise<HashResult> => {
): Promise<StreamDigest> => {
const { handle, initial, owner } = await openRootArtifact(
path,
resourceScope,
signal,
);
let outcome:
| { readonly kind: "completed"; readonly value: HashResult }
| { readonly kind: "completed"; readonly value: StreamDigest }
| { readonly kind: "failed"; readonly cause: unknown };
try {
outcome = {
@@ -144,18 +143,21 @@ export const hashStableRootArtifact = async (
return outcome.value;
};
/** Hash and revalidate the same open descriptor used for root classification. */
/** Hash and revalidate the same open descriptor, feeding `observe` the hashed bytes. */
export const hashStableRootArtifactHandle = async (
path: string,
handle: FileHandle,
initial: Stats,
signal?: AbortSignal,
): Promise<HashResult> => {
const digest = await hashReadable(
observe?: StreamDigestObserver,
): Promise<StreamDigest> => {
const digest = await digestReadableToCompletion(
// Read at most the admitted extent plus one byte to detect growth without
// following an actively appended file indefinitely.
readArtifactFileChunks(handle, { start: 0, end: initial.size, signal }),
signal,
undefined,
observe,
);
abortIfNeeded(signal);
try {
+55
View File
@@ -0,0 +1,55 @@
import { createHash } from "node:crypto";
import type { Readable } from "node:stream";
import { ArtifactReaderFailure } from "./ArtifactReader.js";
import { streamChunkToBuffer } from "./StreamBytes.js";
/** Full stream digest and byte count. */
export interface StreamDigest {
readonly sha256: string;
readonly bytes: number;
readonly md5?: string;
}
/**
* Receives every digested chunk, in order, after it is hashed. A format owner
* uses it to derive its own facts from the same bytes without a second read.
*/
export type StreamDigestObserver = (chunk: Buffer) => void;
/** Preserve the inventory cancellation reason at a stream boundary. */
export const abortIfNeeded = (signal?: AbortSignal): void => {
if (signal?.aborted === true)
throw new ArtifactReaderFailure(
"cancelled",
"Artifact inventory cancelled",
{ cause: signal.reason },
);
};
/** Keep a fully delivered digest so its owner can validate before late cancellation.
* Source stability remains the stream or caller's responsibility.
*/
export const digestReadableToCompletion = async (
stream: Readable,
signal?: AbortSignal,
integrityAlgorithm?: "sha256" | "md5",
observe?: StreamDigestObserver,
): Promise<StreamDigest> => {
const hash = createHash("sha256");
const md5 = integrityAlgorithm === "md5" ? createHash("md5") : undefined;
let bytes = 0;
for await (const raw of stream) {
abortIfNeeded(signal);
const chunk = streamChunkToBuffer(raw);
bytes += chunk.length;
hash.update(chunk);
md5?.update(chunk);
observe?.(chunk);
}
return {
sha256: hash.digest("hex"),
...(md5 === undefined ? {} : { md5: md5.digest("hex") }),
bytes,
};
};
+17
View File
@@ -8,6 +8,11 @@ export {
AnalysisCleanupLease,
AnalysisCleanupLeaseGroup,
} from "./AnalysisCleanupLease.js";
export { artifactFileAdmissionFailure } from "./ArtifactFileAdmission.js";
export {
ArtifactFileSource,
assertArtifactFileSelectedSource,
} from "./ArtifactFileSource.js";
export {
ArtifactFileCancelled,
ArtifactFileCleanupFailure,
@@ -96,3 +101,15 @@ export {
export { SafeOutputTreeCreationFailure } from "./SafeOutputTreeCreationFailure.js";
export { streamChunkToBuffer } from "./StreamBytes.js";
export { readArtifactFileChunks } from "./readArtifactFileChunks.js";
export { abortIfNeeded, digestReadableToCompletion } from "./StreamDigest.js";
export type { StreamDigest, StreamDigestObserver } from "./StreamDigest.js";
export {
hashStableRootArtifact,
hashStableRootArtifactHandle,
openRootArtifact,
releaseRootArtifact,
} from "./StableRootArtifact.js";
export {
targetDigestChangedFailure,
withArtifactTargetIdentity,
} from "./withArtifactTargetIdentity.js";
@@ -1,9 +1,7 @@
import type { BinaryTarget } from "../domain/binaryTargetTypes.js";
import {
ArtifactReaderFailure,
type ArtifactResourceScope,
} from "../resources/index.js";
import { hashStableRootArtifact } from "./inventory/hashStableRootArtifact.js";
import { ArtifactReaderFailure } from "./ArtifactReader.js";
import type { ArtifactResourceScope } from "./ArtifactResourceScope.js";
import { hashStableRootArtifact } from "./StableRootArtifact.js";
/**
* The `container-identity` failure for a selected target whose observed
@@ -1,6 +1,5 @@
import * as rootHash from "../../../src/artifacts/inventory/hashStableRootArtifact.js";
import * as rootHash from "../../../src/resources/StableRootArtifact.js";
import { readFileHandlePrefix } from "../../../src/filesystem/readFileHandlePrefix.js";
import { abortIfNeeded } from "../../../src/artifacts/ArtifactHash.js";
import { createHash } from "node:crypto";
import {
lstat,
@@ -29,6 +28,7 @@ import {
extractArtifact,
materializeArtifactInventoryInScope as materializeInScope,
} from "../../../src/artifacts/extraction/ArtifactExtraction.js";
import { abortIfNeeded } from "../../../src/resources/StreamDigest.js";
const AGED_FIXTURE_TIME = new Date("2000-01-01T00:00:00Z");
@@ -5,7 +5,7 @@ import { expect, it, vi } from "vitest";
import { parseBinaryTarget } from "../../../src/application/BinaryTargetResolver.js";
import { InesArtifactReader } from "../../../src/artifacts/InesArtifactReader.js";
import { scanReader } from "../../../src/artifacts/inventory/scanReader.js";
import { hashStableRootArtifactHandle } from "../../../src/artifacts/inventory/hashStableRootArtifact.js";
import { hashStableRootArtifactHandle } from "../../../src/resources/StableRootArtifact.js";
import { inventoryArtifact } from "../../fixtures/artifactInventory.js";
import { inesFixture } from "../../fixtures/ines.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
@@ -19,7 +19,7 @@ import {
withArtifactFileHandle,
} from "../../../src/resources/ArtifactFileOperation.js";
import { readFileHandlePrefix } from "../../../src/filesystem/readFileHandlePrefix.js";
import { abortIfNeeded } from "../../../src/artifacts/ArtifactHash.js";
import { abortIfNeeded } from "../../../src/resources/StreamDigest.js";
it.each(["close", "cancel"] as const)(
"does not complete an iNES inventory after %s while paused",
@@ -13,7 +13,7 @@ import { join } from "node:path";
import { expect, it, vi } from "vitest";
import * as rootHash from "../../../../src/artifacts/inventory/hashStableRootArtifact.js";
import * as rootHash from "../../../../src/resources/StableRootArtifact.js";
import * as pckLocation from "../../../../src/artifacts/GodotPckLocation.js";
import * as seaLocation from "../../../../src/artifacts/nodeSea/NodeSeaLocation.js";
import * as pyInstallerLocation from "../../../../src/artifacts/pyinstaller/CArchiveLayout.js";
@@ -13,8 +13,7 @@ import { setImmediate } from "node:timers/promises";
import { afterEach, expect, it, vi } from "vitest";
import { phaseGate } from "../support/lifecycleHarness.js";
import { ArtifactFileSource } from "../../src/artifacts/ArtifactFileSource.js";
import * as artifactHash from "../../src/artifacts/ArtifactHash.js";
import * as streamDigest from "../../src/resources/StreamDigest.js";
import {
ArtifactReaderFailure,
type ArtifactEntry,
@@ -28,6 +27,7 @@ import {
import * as regularFile from "../../src/filesystem/RegularFile.js";
import { NativeFixtureRunner } from "../fixtures/nativeCommands.js";
import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";
import { ArtifactFileSource } from "../../src/resources/index.js";
interface OpenControl {
path: string | undefined;
@@ -295,8 +295,8 @@ it.each([false, true])(
const fixture = await machoCarrier();
const hashCompletion = phaseGate();
const controller = new AbortController();
const originalHash = artifactHash.hashReadable;
vi.spyOn(artifactHash, "hashReadable").mockImplementation(
const originalHash = streamDigest.digestReadableToCompletion;
vi.spyOn(streamDigest, "digestReadableToCompletion").mockImplementation(
async (...args) => {
const hashed = await originalHash(...args);
expect(hashed.sha256).toBe(fixture.sha256);