diff --git a/src/application/ReferenceSourceImport.ts b/src/application/ReferenceSourceImport.ts index 769f8bb8cd..f917f1965e 100644 --- a/src/application/ReferenceSourceImport.ts +++ b/src/application/ReferenceSourceImport.ts @@ -12,7 +12,7 @@ import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.j import type { AnalysisCleanupObservation } from "../domain/analysisErrorBase.js"; import { readReferenceSource } from "../reference/ReferenceSourceReader.js"; import type { - ReferenceSourceEntryKind, + ReferenceSourceExclusionKind, ReferenceSourceRead, ReferenceSourceReaderError, } from "../reference/ReferenceSourceReaderTypes.js"; @@ -111,7 +111,8 @@ const deriveInventoryState = ( entry.limitations.length > 0 || (entry.kind === "file" && entry.content_state !== "hashed") || (entry.kind === "directory" && entry.tree_state !== "enumerated") || - (entry.kind === "symlink" && entry.target_state !== "internal"), + (entry.kind === "symlink" && entry.target_state !== "internal") || + entry.kind === "unknown", ); if (partialEntry) return "partial"; const partialRelationship = input.relationships.some( @@ -142,11 +143,24 @@ const createShouldExclude = exclusions: HistoricalSourceGraphInput["exclusions"], secrets: PreparedReferenceSourceImport["secrets"], ignored: PreparedReferenceSourceImport["ignored"], - ): ((path: string, kind: ReferenceSourceEntryKind) => boolean) => + ): ((path: string, kind: ReferenceSourceExclusionKind) => boolean) => (path, kind) => { - const patternPath = kind === "directory" ? `${path}/` : path; - const secretMatch = secrets.test(patternPath); - if (secretMatch.ignored) { + // An entry of unknown kind may be a directory, so directory-only rules + // must still exclude it; this fails closed for caller-marked secrets. + const patternPaths = + kind === "directory" + ? [`${path}/`] + : kind === "unknown" + ? [path, `${path}/`] + : [path]; + const firstMatch = (matcher: typeof secrets) => { + const results = patternPaths.map((patternPath) => + matcher.test(patternPath), + ); + return results.find(({ ignored }) => ignored) ?? results[0]; + }; + const secretMatch = firstMatch(secrets); + if (secretMatch?.ignored === true) { if (!secretMatch.rule) throw new Error(`Ignored secret path has no matching rule: ${path}`); exclusions.push({ @@ -156,8 +170,8 @@ const createShouldExclude = }); return true; } - const match = ignored.test(patternPath); - if (!match.ignored) return false; + const match = firstMatch(ignored); + if (match?.ignored !== true) return false; if (!match.rule) throw new Error(`Ignored path has no matching rule: ${path}`); const reason = match.rule.mark; diff --git a/src/application/ReferenceSourceImportEntries.ts b/src/application/ReferenceSourceImportEntries.ts index 49c6f82f54..29dbef3568 100644 --- a/src/application/ReferenceSourceImportEntries.ts +++ b/src/application/ReferenceSourceImportEntries.ts @@ -45,6 +45,8 @@ const entryRecoveryGuidance = ( return "This directory could not be read. Check its permissions, then try again."; if (entry.kind === "symlink") return "This symbolic link could not be read safely. Check the link and its permissions, then try again."; + if (entry.kind === "unknown") + return "This entry's type could not be determined. Check the parent directory and try again."; return "This file could not be read. Check its permissions, then try again."; }; @@ -73,6 +75,13 @@ const failedEntry = ( classifications, limitations: [limitation], }; + if (entry.kind === "unknown") + return { + path: entry.path, + kind: "unknown", + classifications: ["unknown"], + limitations: [limitation], + }; return { path: entry.path, kind: "file", diff --git a/src/domain/referenceSourceGraph.test.ts b/src/domain/referenceSourceGraph.test.ts index 1286cdde6b..8fe868832c 100644 --- a/src/domain/referenceSourceGraph.test.ts +++ b/src/domain/referenceSourceGraph.test.ts @@ -259,6 +259,30 @@ describe("historical source graph", () => { }); describe("partial historical source graphs", () => { + it("keeps unknown-entry diagnostics outside the root commitment", () => { + const base = graphInput(); + base.inventory_state = "partial"; + base.entries[2] = { + path: "src/package.json", + kind: "unknown", + classifications: ["unknown"], + limitations: ["Permission denied at /private/first"], + }; + const graph = createHistoricalSourceGraph(base); + const changedMessage = structuredClone(base); + changedMessage.entries[2] = { + path: "src/package.json", + kind: "unknown", + classifications: ["unknown"], + limitations: ["Permission denied at /private/second"], + }; + const revised = createHistoricalSourceGraph(changedMessage); + expect(revised.root_sha256).toBe(graph.root_sha256); + expect(computeHistoricalSourceGraphSha256(revised)).not.toBe( + computeHistoricalSourceGraphSha256(graph), + ); + }); + it("keeps an unreadable symlink target unknown and out of root identity", () => { const base = graphInput(); base.inventory_state = "partial"; diff --git a/src/domain/referenceSourceGraph.ts b/src/domain/referenceSourceGraph.ts index 197e473bfc..a830e0aa05 100644 --- a/src/domain/referenceSourceGraph.ts +++ b/src/domain/referenceSourceGraph.ts @@ -111,6 +111,10 @@ const sourceEntrySchema = z.union([ sourceFileSchema, sourceDirectorySchema, sourceSymlinkSchema, + z.strictObject({ + ...entryBaseShape, + kind: z.literal("unknown"), + }), ]); const sourceRelationshipSchema = z.strictObject({ @@ -361,7 +365,8 @@ const checkCompleteInventory = ( entry.limitations.length > 0 || (entry.kind === "file" && entry.content_state !== "hashed") || (entry.kind === "directory" && entry.tree_state !== "enumerated") || - (entry.kind === "symlink" && entry.target_state !== "internal"), + (entry.kind === "symlink" && entry.target_state !== "internal") || + entry.kind === "unknown", ); const partialRelationship = graph.relationships.some( ({ parse_state, resolution }) => @@ -478,6 +483,11 @@ const rootCommitment = ( target: entry.target, target_state: entry.target_state, }; + case "unknown": + return { + kind: entry.kind, + path: entry.path, + }; } }), exclusions, diff --git a/src/domain/referenceSourceSummary.test.ts b/src/domain/referenceSourceSummary.test.ts index a0d23b0e16..3f34a073f4 100644 --- a/src/domain/referenceSourceSummary.test.ts +++ b/src/domain/referenceSourceSummary.test.ts @@ -81,6 +81,26 @@ describe("committed historical-source summaries", () => { expect(graph).toEqual(original); }); + it("counts entries of unknown kind separately from symlinks", () => { + const graph = createHistoricalSourceGraph( + input([ + { + path: "deep", + kind: "unknown", + classifications: ["unknown"], + limitations: ["lstat: ENAMETOOLONG"], + }, + ]), + ); + expect(createHistoricalSourceSummary(graph, 1).counts).toMatchObject({ + entries: 1, + files: 0, + directories: 0, + symlinks: 0, + unknown: 1, + }); + }); + it("sums known bytes losslessly beyond the JSON safe integer range", () => { const graph = createHistoricalSourceGraph( input( diff --git a/src/domain/referenceSourceSummary.ts b/src/domain/referenceSourceSummary.ts index e7e99e3fa9..55d24025d2 100644 --- a/src/domain/referenceSourceSummary.ts +++ b/src/domain/referenceSourceSummary.ts @@ -29,6 +29,7 @@ export const createHistoricalSourceSummary = ( files: 0, directories: 0, symlinks: 0, + unknown: 0, relationships: graph.relationships.length, parse_failures: graph.parse_failures.length, exclusions: graph.exclusions.length, @@ -59,7 +60,8 @@ export const createHistoricalSourceSummary = ( } else if (entry.kind === "directory") { counts.directories += 1; treeStates[entry.tree_state] += 1; - } else counts.symlinks += 1; + } else if (entry.kind === "symlink") counts.symlinks += 1; + else counts.unknown += 1; // Canonical entries already use Unicode code-point order; filtering preserves it. if (entry.path.split("/").length <= outlineDepth) outline.push(outlineEntry(entry)); diff --git a/src/reference/ReferenceSourceReaderEntries.ts b/src/reference/ReferenceSourceReaderEntries.ts index e684a2b5cb..8ec5e9d803 100644 --- a/src/reference/ReferenceSourceReaderEntries.ts +++ b/src/reference/ReferenceSourceReaderEntries.ts @@ -25,7 +25,7 @@ import { import type { PendingDirectory, ReferenceSourceEntry, - ReferenceSourceEntryKind, + ReferenceSourceExclusionKind, ReferenceSourceResult, TraversalState, } from "./ReferenceSourceReaderTypes.js"; @@ -201,6 +201,13 @@ const processEntry = async ( const metadata = await readMetadata(absolute); if (isAborted(state.signal)) return { ok: false, error: cancelled() }; if (!metadata.ok) { + const excluded = applyExclusion(state.shouldExclude, path, "unknown"); + if (!excluded.ok) + return { + ok: false, + error: failure("io", "Reference source exclusion check failed"), + }; + if (excluded.value) return { ok: true, value: undefined }; state.entries.push(entryFailure(path, "unknown", "io", metadata.message)); return { ok: true, value: undefined }; } @@ -371,10 +378,10 @@ const readSymlinkTarget = async ( const applyExclusion = ( shouldExclude: - | ((path: string, kind: ReferenceSourceEntryKind) => boolean) + | ((path: string, kind: ReferenceSourceExclusionKind) => boolean) | undefined, path: string, - kind: ReferenceSourceEntryKind, + kind: ReferenceSourceExclusionKind, ): { readonly ok: true; readonly value: boolean } | { readonly ok: false } => { try { return { ok: true, value: shouldExclude?.(path, kind) === true }; diff --git a/src/reference/ReferenceSourceReaderErrors.ts b/src/reference/ReferenceSourceReaderErrors.ts index 6ddd02a3f2..2c826b4c96 100644 --- a/src/reference/ReferenceSourceReaderErrors.ts +++ b/src/reference/ReferenceSourceReaderErrors.ts @@ -29,6 +29,9 @@ const FILESYSTEM_ERROR_CODES = new Set([ "EISDIR", "ELOOP", "EMFILE", + // A traversed child can exceed the platform's absolute path limit even + // though the selected root and every component are valid. + "ENAMETOOLONG", "ENFILE", "ENOENT", "ENOTDIR", @@ -44,6 +47,8 @@ export const filesystemFailureDetail = ( ): string | undefined => { if (!(cause instanceof Error)) return undefined; const code: unknown = Reflect.get(cause, "code"); + if (code === "ENAMETOOLONG") + return `${operation}: ENAMETOOLONG (entry path exceeds the operating system path limit)`; return typeof code === "string" && FILESYSTEM_ERROR_CODES.has(code) ? `${operation}: ${cause.message}` : undefined; diff --git a/src/reference/ReferenceSourceReaderTypes.ts b/src/reference/ReferenceSourceReaderTypes.ts index 3b0a368b51..1fec383131 100644 --- a/src/reference/ReferenceSourceReaderTypes.ts +++ b/src/reference/ReferenceSourceReaderTypes.ts @@ -10,11 +10,17 @@ export type ReferenceSourceEntryKind = | "symlink" | "other"; +/** + * Kind offered to exclusion predicates. `unknown` marks an entry whose + * metadata could not be read, so its concrete kind cannot be claimed. + */ +export type ReferenceSourceExclusionKind = ReferenceSourceEntryKind | "unknown"; + export interface ReferenceSourceReaderOptions { readonly signal?: AbortSignal; readonly shouldExclude?: ( path: string, - kind: ReferenceSourceEntryKind, + kind: ReferenceSourceExclusionKind, ) => boolean; } @@ -84,7 +90,7 @@ export type TraversalState = { readonly signal?: AbortSignal; readonly shouldExclude?: ( path: string, - kind: ReferenceSourceEntryKind, + kind: ReferenceSourceExclusionKind, ) => boolean; readonly entries: ReferenceSourceEntry[]; readonly pending: PendingDirectory[]; diff --git a/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts b/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts index dd87ea92a2..82476d6db8 100644 --- a/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts +++ b/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts @@ -133,3 +133,88 @@ describe("reference import native entry failure details", () => { ).toMatchObject({ kind: "file", content_state: "hashed", limitations: [] }); }); }); + +describe("reference import paths beyond the platform limit", () => { + it.skipIf(process.platform === "win32")( + "reports an entry beyond the absolute path limit as an I/O failure", + async () => { + const root = await createTestTempDirectory("rea-import-deep-path-"); + await writeFile(join(root, "kept.txt"), "k"); + // Relative creation lets the tree exceed PATH_MAX (1024 on macOS, 4096 + // on Linux) while each 200-byte component stays below NAME_MAX. + const deepTree = (action: "create" | "remove") => + promisify(execFile)(process.execPath, [ + "-e", + `const fs = require("node:fs"); + const names = Array.from({ length: 24 }, (_, i) => "d" + String(i).padStart(2, "0") + "x".repeat(197)); + process.chdir(${JSON.stringify(root)}); + if (${JSON.stringify(action)} === "create") { + for (const name of names) { fs.mkdirSync(name); process.chdir(name); } + fs.writeFileSync("deep.txt", "d"); + } else { + for (const name of names) process.chdir(name); + fs.rmSync("deep.txt"); + for (const name of [...names].reverse()) { process.chdir(".."); fs.rmdirSync(name); } + }`, + ]); + await deepTree("create"); + try { + const read = await readReferenceSource(root); + if (!read.ok) throw new Error(read.error.message); + expect(read.value.entries).toContainEqual( + expect.objectContaining({ path: "kept.txt", status: "read" }), + ); + const failed = read.value.entries.find( + (entry) => entry.status === "failed", + ); + expect(failed).toMatchObject({ + status: "failed", + code: "io", + message: expect.stringContaining("ENAMETOOLONG"), + }); + const filteredRead = await readReferenceSource(root, { + shouldExclude: (path, kind) => + path === failed?.path && kind === "directory", + }); + if (!filteredRead.ok) throw new Error(filteredRead.error.message); + expect( + filteredRead.value.entries.find( + (entry) => entry.path === failed?.path, + ), + ).toMatchObject({ status: "failed", kind: "unknown", code: "io" }); + const imported = await importTree(root); + if (!imported.ok) throw new Error(imported.error.message); + expect(imported.value.entries).toContainEqual( + expect.objectContaining({ + path: "kept.txt", + content_state: "hashed", + }), + ); + const deep = imported.value.entries.find( + (entry) => entry.path === failed?.path, + ); + expect(deep?.limitations.join(" ")).toContain( + failed?.status === "failed" ? failed.message : "missing failure", + ); + // A directory-only secret rule still applies when the kind is unknown. + const secretPattern = `${failed?.path ?? "missing"}/`; + const secret = await importReferenceSource({ + root, + caller: "native-entry-failure-detail-test", + policy: { secretPatterns: [secretPattern] }, + }); + if (!secret.ok) throw new Error(secret.error.message); + expect( + secret.value.entries.some(({ path }) => path === failed?.path), + ).toBe(false); + expect(secret.value.exclusions).toContainEqual({ + path: failed?.path, + reason: "configured-secret", + pattern: secretPattern, + }); + } finally { + await deepTree("remove"); + } + }, + ); +});