diff --git a/scripts/lib/module-layers.baseline.json b/scripts/lib/module-layers.baseline.json index 6a34887167..7b79f742b3 100644 --- a/scripts/lib/module-layers.baseline.json +++ b/scripts/lib/module-layers.baseline.json @@ -16,7 +16,6 @@ "entry": { "application -> browser": 2, "application -> desktop": 41, - "application -> frida": 2, - "application -> reference": 8 + "application -> frida": 2 } } diff --git a/scripts/lib/module-layers.mjs b/scripts/lib/module-layers.mjs index ee74e5d995..1501afcf87 100644 --- a/scripts/lib/module-layers.mjs +++ b/scripts/lib/module-layers.mjs @@ -365,7 +365,16 @@ export const ENTRY_FILES = Object.freeze( "PcapProviderIdentity.ts", ], python: ["PythonPycProvider.ts", "PythonPycProviderIdentity.ts"], - reference: [], + reference: [ + // The reference-source import workflow (src/application/ReferenceSource*) + // runs the reader and consumes its result and failure types. It reports + // its own root and VCS reads with the reader's two failure builders, so + // both kinds of failure share one shape and wording. Traversal state, + // symlink reconciliation and the per-entry failure helpers stay private. + "ReferenceSourceFailureBuilders.ts", + "ReferenceSourceReader.ts", + "ReferenceSourceReaderTypes.ts", + ], rizin: [ "RizinCommand.ts", "RizinDebugSessionManager.ts", diff --git a/src/application/ReferenceSourceImportPolicy.ts b/src/application/ReferenceSourceImportPolicy.ts index e1a4ae1c97..3557f2148d 100644 --- a/src/application/ReferenceSourceImportPolicy.ts +++ b/src/application/ReferenceSourceImportPolicy.ts @@ -25,12 +25,12 @@ import { type ReferenceSourceImportError, type ReferenceSourceImportOptions, } from "./ReferenceSourceImportTypes.js"; -import { rootFilesystemFailure } from "../reference/ReferenceSourceReaderErrors.js"; import { RegularFileCleanupFailure, regularFileCancellation, } from "../filesystem/OwnedRegularFile.js"; import { errorMessage } from "../kernel/errorMessage.js"; +import { rootFilesystemFailure } from "../reference/ReferenceSourceFailureBuilders.js"; /** Validated inputs ready for filesystem traversal. */ export interface PreparedReferenceSourceImport { diff --git a/src/application/ReferenceSourceVcsAdapter.ts b/src/application/ReferenceSourceVcsAdapter.ts index f527492bfd..bc236807df 100644 --- a/src/application/ReferenceSourceVcsAdapter.ts +++ b/src/application/ReferenceSourceVcsAdapter.ts @@ -19,9 +19,9 @@ import { RegularFileReadFailure, retryRegularFileCleanup, } from "../process/RegularFileRead.js"; -import { referenceSourceResourceFailure } from "../reference/ReferenceSourceReaderErrors.js"; import type { ReferenceSourceReaderError } from "../reference/ReferenceSourceReaderTypes.js"; import { errorMessage } from "../kernel/errorMessage.js"; +import { referenceSourceResourceFailure } from "../reference/ReferenceSourceFailureBuilders.js"; export type ReferenceSourceVcsInfo = | { diff --git a/src/reference/ReferenceSourceFailureBuilders.ts b/src/reference/ReferenceSourceFailureBuilders.ts new file mode 100644 index 0000000000..e7e4e95d14 --- /dev/null +++ b/src/reference/ReferenceSourceFailureBuilders.ts @@ -0,0 +1,48 @@ +import type { ArtifactReaderFailure } from "../resources/index.js"; +import { FILESYSTEM_ERROR_CODES } from "./ReferenceSourceReaderErrors.js"; +import type { ReferenceSourceReaderError } from "./ReferenceSourceReaderTypes.js"; + +/** Project resource failures into the reference-source error contract. */ +export const referenceSourceResourceFailure = ( + cause: ArtifactReaderFailure, +): ReferenceSourceReaderError => ({ + tag: "reference-source-reader", + code: cause.reason === "cancelled" ? "cancelled" : "io", + message: cause.message, + ...(cause.cleanup === undefined ? {} : { cleanup: cause.cleanup }), + cause, +}); + +/** Preserve root I/O detail while separating bad selections from access failures. */ +export const rootFilesystemFailure = ( + cause: unknown, + operation: string, +): + | { + readonly code: "invalid-root" | "io"; + readonly message: string; + readonly remediationAction?: string; + } + | undefined => { + if (!(cause instanceof Error)) return undefined; + const code: unknown = Reflect.get(cause, "code"); + if (typeof code !== "string" || !FILESYSTEM_ERROR_CODES.has(code)) + return undefined; + return { + code: ["ENOENT", "ENOTDIR", "ELOOP", "ENAMETOOLONG"].includes(code) + ? "invalid-root" + : "io", + message: `${operation}: ${cause.message}`, + ...(code === "ELOOP" + ? { + remediationAction: + "Repair the symbolic-link chain or select another reference source directory.", + } + : code === "ENAMETOOLONG" + ? { + remediationAction: + "Select a reference source path within the operating system's path-length limits.", + } + : {}), + }; +}; diff --git a/src/reference/ReferenceSourceReader.test.ts b/src/reference/ReferenceSourceReader.test.ts index 27cf4c7a4b..fe8ffc96fd 100644 --- a/src/reference/ReferenceSourceReader.test.ts +++ b/src/reference/ReferenceSourceReader.test.ts @@ -1,10 +1,8 @@ import { describe, expect, it } from "vitest"; -import type { - ReferenceSourceEntry, - TraversalState, -} from "./ReferenceSourceReaderTypes.js"; -import { reconcileUnobservedSymlinkTargets } from "./ReferenceSourceReader.js"; +import type { ReferenceSourceEntry } from "./ReferenceSourceReaderTypes.js"; +import type { TraversalState } from "./ReferenceSourceReaderTraversal.js"; +import { reconcileUnobservedSymlinkTargets } from "./ReferenceSourceSymlinkTargets.js"; const failedDirectory = ( path: string, diff --git a/src/reference/ReferenceSourceReader.ts b/src/reference/ReferenceSourceReader.ts index c3ef086496..96bc879f52 100644 --- a/src/reference/ReferenceSourceReader.ts +++ b/src/reference/ReferenceSourceReader.ts @@ -1,7 +1,4 @@ -import { posix } from "node:path"; - import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; -import { createReferenceSourcePathLookup } from "../domain/referenceSourcePathIdentity.js"; import { err, ok } from "../domain/result.js"; import { ArtifactReaderFailure, @@ -10,21 +7,18 @@ import { import { traverseDirectory } from "./ReferenceSourceReaderEntries.js"; import type { ReferenceSourceReaderOptions, - ReferenceSourceEntry, - ReferenceSourceRootFailure, ReferenceSourceRead, ReferenceSourceResult, - TraversalState, } from "./ReferenceSourceReaderTypes.js"; import { isAborted, noFollowOpenSupported, prepareRoot, } from "./ReferenceSourceReaderValidate.js"; -import { - cancelled, - referenceSourceResourceFailure, -} from "./ReferenceSourceReaderErrors.js"; +import { cancelled } from "./ReferenceSourceReaderErrors.js"; +import type { TraversalState } from "./ReferenceSourceReaderTraversal.js"; +import { referenceSourceResourceFailure } from "./ReferenceSourceFailureBuilders.js"; +import { reconcileUnobservedSymlinkTargets } from "./ReferenceSourceSymlinkTargets.js"; const PATH_RACE_LIMITATION = "Path identity is revalidated around operations; Node lacks portable descriptor-relative openat traversal, so a syscall-boundary pathname race remains."; @@ -120,100 +114,6 @@ const readSource = async ( return traversal.ok ? ok(read) : err({ ...traversal.error, partial: read }); }; -/** - * Resolve internal link observations against the completed reader inventory. - * If the closest containing directory failed before enumeration completed, - * the target remains unknown; a broader failed ancestor cannot override a - * successfully enumerated directory or a recorded exclusion. - */ -export const reconcileUnobservedSymlinkTargets = ( - entries: ReferenceSourceEntry[], - exclusions: TraversalState["exclusions"] = [], - signal?: AbortSignal, - rootFailure?: ReferenceSourceRootFailure, -): void => { - const failedDirectories = new Map( - entries.flatMap((entry) => - entry.status === "failed" && - entry.kind === "directory" && - entry.code !== "cancelled" - ? [[entry.path, entry] as const] - : [], - ), - ); - if ( - (failedDirectories.size === 0 && rootFailure === undefined) || - isAborted(signal) - ) - return; - const directoryEntries = entries.filter( - (entry) => entry.kind === "directory", - ); - const directoryPaths = createReferenceSourcePathLookup( - directoryEntries.map(({ path }) => path), - ); - const observedPaths = createReferenceSourcePathLookup( - entries.map(({ path }) => path), - ); - const excludedPaths = createReferenceSourcePathLookup( - exclusions.map(({ path }) => path), - ); - for (const [index, entry] of entries.entries()) { - if (isAborted(signal)) return; - if ( - entry.status !== "read" || - entry.kind !== "symlink" || - entry.targetState !== "internal" || - observedPaths.resolve(entry.target) !== undefined || - excludedPaths.resolve(entry.target) !== undefined - ) - continue; - const ancestor = nearestFailedAncestor( - entry.target, - directoryPaths, - failedDirectories, - excludedPaths, - signal, - rootFailure, - ); - if (ancestor === undefined) continue; - entries[index] = { - status: "failed", - kind: "symlink", - path: entry.path, - code: ancestor.code, - message: `Symbolic link target ${JSON.stringify(entry.target)} was not inventoried because directory ${JSON.stringify(ancestor.path)} could not be enumerated: ${ancestor.message}`, - }; - } -}; - -const nearestFailedAncestor = ( - target: string, - directoryPaths: ReturnType, - failedDirectories: ReadonlyMap< - string, - Extract - >, - excludedPaths: ReturnType, - signal: AbortSignal | undefined, - rootFailure: ReferenceSourceRootFailure | undefined, -): (ReferenceSourceRootFailure & { readonly path: string }) | undefined => { - for ( - let candidate = posix.dirname(target); - candidate !== "." && candidate !== "/"; - candidate = posix.dirname(candidate) - ) { - if (isAborted(signal)) return undefined; - if (excludedPaths.resolve(candidate) !== undefined) return undefined; - const observedDirectory = directoryPaths.resolve(candidate); - if (observedDirectory !== undefined) - return failedDirectories.get(observedDirectory); - } - if (isAborted(signal)) return undefined; - if (excludedPaths.resolve(".") !== undefined) return undefined; - return rootFailure === undefined ? undefined : { path: ".", ...rootFailure }; -}; - const traverse = async ( state: TraversalState, ): Promise> => { diff --git a/src/reference/ReferenceSourceReaderEntries.ts b/src/reference/ReferenceSourceReaderEntries.ts index b5bb1e9555..4cd8ab3d36 100644 --- a/src/reference/ReferenceSourceReaderEntries.ts +++ b/src/reference/ReferenceSourceReaderEntries.ts @@ -32,13 +32,15 @@ import { bigLstat, } from "./ReferenceSourceReaderValidate.js"; import type { - PendingDirectory, ReferenceSourceEntry, ReferenceSourceExclusionKind, ReferenceSourceFailureCode, ReferenceSourceResult, - TraversalState, } from "./ReferenceSourceReaderTypes.js"; +import type { + PendingDirectory, + TraversalState, +} from "./ReferenceSourceReaderTraversal.js"; const recordDirectoryFailure = ( state: TraversalState, diff --git a/src/reference/ReferenceSourceReaderErrors.ts b/src/reference/ReferenceSourceReaderErrors.ts index d7a9e60798..14850123c6 100644 --- a/src/reference/ReferenceSourceReaderErrors.ts +++ b/src/reference/ReferenceSourceReaderErrors.ts @@ -1,4 +1,3 @@ -import type { ArtifactReaderFailure } from "../resources/index.js"; import type { ReferenceSourceEntry, ReferenceSourceFailureCode, @@ -21,23 +20,12 @@ export const cancelled = ( ...(signal?.aborted === true ? { cause: signal.reason } : {}), }); -/** Project resource failures into the reference-source error contract. */ -export const referenceSourceResourceFailure = ( - cause: ArtifactReaderFailure, -): ReferenceSourceReaderError => ({ - tag: "reference-source-reader", - code: cause.reason === "cancelled" ? "cancelled" : "io", - message: cause.message, - ...(cause.cleanup === undefined ? {} : { cleanup: cause.cleanup }), - cause, -}); - export const safeSize = (size: bigint): number | undefined => { const value = Number(size); return Number.isSafeInteger(value) && value >= 0 ? value : undefined; }; -const FILESYSTEM_ERROR_CODES = new Set([ +export const FILESYSTEM_ERROR_CODES = new Set([ "EACCES", "EBADF", "EEXIST", @@ -83,40 +71,6 @@ export const filesystemFailureDetail = ( }; }; -/** Preserve root I/O detail while separating bad selections from access failures. */ -export const rootFilesystemFailure = ( - cause: unknown, - operation: string, -): - | { - readonly code: "invalid-root" | "io"; - readonly message: string; - readonly remediationAction?: string; - } - | undefined => { - if (!(cause instanceof Error)) return undefined; - const code: unknown = Reflect.get(cause, "code"); - if (typeof code !== "string" || !FILESYSTEM_ERROR_CODES.has(code)) - return undefined; - return { - code: ["ENOENT", "ENOTDIR", "ELOOP", "ENAMETOOLONG"].includes(code) - ? "invalid-root" - : "io", - message: `${operation}: ${cause.message}`, - ...(code === "ELOOP" - ? { - remediationAction: - "Repair the symbolic-link chain or select another reference source directory.", - } - : code === "ENAMETOOLONG" - ? { - remediationAction: - "Select a reference source path within the operating system's path-length limits.", - } - : {}), - }; -}; - export const entryFailure = ( ...[path, kind, code, message, size, localDiagnostic]: readonly [ string, diff --git a/src/reference/ReferenceSourceReaderFile.ts b/src/reference/ReferenceSourceReaderFile.ts index f85931a032..09121b0c5a 100644 --- a/src/reference/ReferenceSourceReaderFile.ts +++ b/src/reference/ReferenceSourceReaderFile.ts @@ -22,10 +22,8 @@ import { sameFile, validateDirectory, } from "./ReferenceSourceReaderValidate.js"; -import type { - ReferenceSourceEntry, - StableFileRequest, -} from "./ReferenceSourceReaderTypes.js"; +import type { ReferenceSourceEntry } from "./ReferenceSourceReaderTypes.js"; +import type { StableFileRequest } from "./ReferenceSourceReaderTraversal.js"; const READ_CHUNK_BYTES = 64 * 1024; diff --git a/src/reference/ReferenceSourceReaderTraversal.ts b/src/reference/ReferenceSourceReaderTraversal.ts new file mode 100644 index 0000000000..2321ce1747 --- /dev/null +++ b/src/reference/ReferenceSourceReaderTraversal.ts @@ -0,0 +1,48 @@ +import type { BigIntStats } from "node:fs"; + +import type { ArtifactResourceScope } from "../resources/index.js"; +import type { + ReferenceSourceEntry, + ReferenceSourceExclusionKind, + ReferenceSourceResult, + ReferenceSourceRootFailure, +} from "./ReferenceSourceReaderTypes.js"; + +/** Traversal state private to the reader; not part of its contract. */ +export type PendingDirectory = { + readonly path: string; +}; + +export type TraversalState = { + readonly resources: ArtifactResourceScope; + readonly root: string; + readonly rootIdentity: BigIntStats; + readonly signal?: AbortSignal; + /** Prepare policy after identity admission and opening, before enumeration; refusal closes the owned handle. */ + readonly beforeDirectory?: ( + path: string, + ) => Promise>; + readonly shouldExclude?: ( + path: string, + kind: ReferenceSourceExclusionKind, + ) => boolean; + readonly entries: ReferenceSourceEntry[]; + readonly exclusions: Array<{ + readonly path: string; + readonly kind: ReferenceSourceExclusionKind; + }>; + readonly pending: PendingDirectory[]; + rootFailure?: ReferenceSourceRootFailure; + rootFailureCause?: { readonly value: unknown }; + bytesRead: number; +}; + +export type StableFileRequest = { + readonly resources: ArtifactResourceScope; + readonly root: string; + readonly rootIdentity: BigIntStats; + readonly absolute: string; + readonly path: string; + readonly expected: BigIntStats; + readonly signal?: AbortSignal; +}; diff --git a/src/reference/ReferenceSourceReaderTypes.ts b/src/reference/ReferenceSourceReaderTypes.ts index eff57b1cb9..4ada182f9a 100644 --- a/src/reference/ReferenceSourceReaderTypes.ts +++ b/src/reference/ReferenceSourceReaderTypes.ts @@ -1,8 +1,5 @@ -import type { BigIntStats } from "node:fs"; - import type { Result } from "../domain/result.js"; import type { AnalysisCleanupObservation } from "../domain/analysisErrorBase.js"; -import type { ArtifactResourceScope } from "../resources/index.js"; export type ReferenceSourceEntryKind = | "file" @@ -95,42 +92,4 @@ export interface ReferenceSourceReaderError { readonly cause?: unknown; } -export type PendingDirectory = { - readonly path: string; -}; - -export type TraversalState = { - readonly resources: ArtifactResourceScope; - readonly root: string; - readonly rootIdentity: BigIntStats; - readonly signal?: AbortSignal; - /** Prepare policy after identity admission and opening, before enumeration; refusal closes the owned handle. */ - readonly beforeDirectory?: ( - path: string, - ) => Promise>; - readonly shouldExclude?: ( - path: string, - kind: ReferenceSourceExclusionKind, - ) => boolean; - readonly entries: ReferenceSourceEntry[]; - readonly exclusions: Array<{ - readonly path: string; - readonly kind: ReferenceSourceExclusionKind; - }>; - readonly pending: PendingDirectory[]; - rootFailure?: ReferenceSourceRootFailure; - rootFailureCause?: { readonly value: unknown }; - bytesRead: number; -}; - -export type StableFileRequest = { - readonly resources: ArtifactResourceScope; - readonly root: string; - readonly rootIdentity: BigIntStats; - readonly absolute: string; - readonly path: string; - readonly expected: BigIntStats; - readonly signal?: AbortSignal; -}; - export type ReferenceSourceResult = Result; diff --git a/src/reference/ReferenceSourceReaderValidate.ts b/src/reference/ReferenceSourceReaderValidate.ts index f2ca0c5da1..b0b36ebe9d 100644 --- a/src/reference/ReferenceSourceReaderValidate.ts +++ b/src/reference/ReferenceSourceReaderValidate.ts @@ -7,13 +7,13 @@ import { cancelled, failure, filesystemFailureDetail, - rootFilesystemFailure, } from "./ReferenceSourceReaderErrors.js"; import { isPathWithinRoot } from "../domain/localPath.js"; import type { ReferenceSourceFailureCode, ReferenceSourceResult, } from "./ReferenceSourceReaderTypes.js"; +import { rootFilesystemFailure } from "./ReferenceSourceFailureBuilders.js"; export const bigLstat = (path: string): Promise => lstat(path, { bigint: true }); diff --git a/src/reference/ReferenceSourceSymlinkTargets.ts b/src/reference/ReferenceSourceSymlinkTargets.ts new file mode 100644 index 0000000000..a8c73ea215 --- /dev/null +++ b/src/reference/ReferenceSourceSymlinkTargets.ts @@ -0,0 +1,103 @@ +import { posix } from "node:path"; + +import { createReferenceSourcePathLookup } from "../domain/referenceSourcePathIdentity.js"; +import type { TraversalState } from "./ReferenceSourceReaderTraversal.js"; +import type { + ReferenceSourceEntry, + ReferenceSourceRootFailure, +} from "./ReferenceSourceReaderTypes.js"; +import { isAborted } from "./ReferenceSourceReaderValidate.js"; + +/** + * Resolve internal link observations against the completed reader inventory. + * If the closest containing directory failed before enumeration completed, + * the target remains unknown; a broader failed ancestor cannot override a + * successfully enumerated directory or a recorded exclusion. + */ +export const reconcileUnobservedSymlinkTargets = ( + entries: ReferenceSourceEntry[], + exclusions: TraversalState["exclusions"] = [], + signal?: AbortSignal, + rootFailure?: ReferenceSourceRootFailure, +): void => { + const failedDirectories = new Map( + entries.flatMap((entry) => + entry.status === "failed" && + entry.kind === "directory" && + entry.code !== "cancelled" + ? [[entry.path, entry] as const] + : [], + ), + ); + if ( + (failedDirectories.size === 0 && rootFailure === undefined) || + isAborted(signal) + ) + return; + const directoryEntries = entries.filter( + (entry) => entry.kind === "directory", + ); + const directoryPaths = createReferenceSourcePathLookup( + directoryEntries.map(({ path }) => path), + ); + const observedPaths = createReferenceSourcePathLookup( + entries.map(({ path }) => path), + ); + const excludedPaths = createReferenceSourcePathLookup( + exclusions.map(({ path }) => path), + ); + for (const [index, entry] of entries.entries()) { + if (isAborted(signal)) return; + if ( + entry.status !== "read" || + entry.kind !== "symlink" || + entry.targetState !== "internal" || + observedPaths.resolve(entry.target) !== undefined || + excludedPaths.resolve(entry.target) !== undefined + ) + continue; + const ancestor = nearestFailedAncestor( + entry.target, + directoryPaths, + failedDirectories, + excludedPaths, + signal, + rootFailure, + ); + if (ancestor === undefined) continue; + entries[index] = { + status: "failed", + kind: "symlink", + path: entry.path, + code: ancestor.code, + message: `Symbolic link target ${JSON.stringify(entry.target)} was not inventoried because directory ${JSON.stringify(ancestor.path)} could not be enumerated: ${ancestor.message}`, + }; + } +}; + +const nearestFailedAncestor = ( + target: string, + directoryPaths: ReturnType, + failedDirectories: ReadonlyMap< + string, + Extract + >, + excludedPaths: ReturnType, + signal: AbortSignal | undefined, + rootFailure: ReferenceSourceRootFailure | undefined, +): (ReferenceSourceRootFailure & { readonly path: string }) | undefined => { + for ( + let candidate = posix.dirname(target); + candidate !== "." && candidate !== "/"; + candidate = posix.dirname(candidate) + ) { + if (isAborted(signal)) return undefined; + if (excludedPaths.resolve(candidate) !== undefined) return undefined; + const observedDirectory = directoryPaths.resolve(candidate); + if (observedDirectory !== undefined) + return failedDirectories.get(observedDirectory); + } + if (isAborted(signal)) return undefined; + if (excludedPaths.resolve(".") !== undefined) return undefined; + return rootFailure === undefined ? undefined : { path: ".", ...rootFailure }; +};