diff --git a/docs/firmware-analysis.md b/docs/firmware-analysis.md index e5a7a8f4cc..ac5d109614 100644 --- a/docs/firmware-analysis.md +++ b/docs/firmware-analysis.md @@ -82,6 +82,11 @@ Published file `relative_path` preserves producer spelling for lineage; `path` uses the writer's actual normalized filename. An extracted filename that cannot be represented losslessly as UTF-8 fails publication explicitly with its raw name bytes in hexadecimal instead of silently omitting the entry. +Unpublished symlinks retain the extracted tree's target bytes in lowercase +`link_target_hex`; `link_target` is null when those bytes are not valid UTF-8. +Older retained Evidence may omit the hexadecimal field. Unblob may have rewritten +the target before REA observed it, so these bytes are not necessarily those stored +in the firmware image. ## CLI and MCP diff --git a/docs/native-investigation.md b/docs/native-investigation.md index aea5fb7f2d..5a951cdab3 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -638,6 +638,9 @@ complete observation. undecodable filenames and skipped symlinks, as partial coverage. Known executable roots remain available; unexamined names and symlink targets cannot establish an empty root set or the absence of other roots. + A dependency symlink whose target is not valid UTF-8 is undetermined, with + its stored target bytes in the limitation and partial coverage; a selected + root crossing such a symlink is refused with the same bytes in its diagnostic. - `inspect_asset_catalog` / `rea inspect-asset-catalog ` reads compiled `Assets.car` metadata through macOS `assetutil --info`. Catalog digests, raw rendition fields, pagination and exact UI resource-name matches are returned. diff --git a/src/artifacts/apple/DylibResolutionReader.ts b/src/artifacts/apple/DylibResolutionReader.ts index 3a6ccba6c8..0f13c0faf9 100644 --- a/src/artifacts/apple/DylibResolutionReader.ts +++ b/src/artifacts/apple/DylibResolutionReader.ts @@ -4,6 +4,7 @@ import { isAbsolute, join, relative, sep } from "node:path"; import { AnalysisInputError } from "../../domain/analysisErrorCore.js"; import { resolveTreePath } from "../../domain/apple/dyldPaths.js"; +import { observedSymlinkTarget } from "../../domain/symlinkTarget.js"; import { dylibResolutionInputSchema, dylibResolutionResultSchema, @@ -113,7 +114,12 @@ class FilesystemTreeView implements DylibTreeView { try { const metadata = await lstat(absolute); if (metadata.isSymbolicLink()) { - const target = await readlink(absolute); + const observed = observedSymlinkTarget( + await readlink(absolute, { encoding: "buffer" }), + ); + if (observed.text === null) + return { kind: "symlink", target: null, targetHex: observed.hex }; + const target = observed.text; // Windows link text separates components with `\`. return { kind: "symlink", @@ -333,6 +339,11 @@ const requireMachoRoots = async ( const lookup = await resolveTreePath(view, path); if (lookup.kind === "escapes") throw reject("path", `Root ${path} resolves outside the analyzed root`); + if (lookup.kind === "undetermined") + throw reject( + "path", + `Root ${path} crosses symlink ${lookup.path} with non-UTF-8 target bytes ${lookup.targetHex}`, + ); if (lookup.kind !== "file") throw reject( "path", diff --git a/src/artifacts/apple/MachoLoadCommandReader.test.ts b/src/artifacts/apple/MachoLoadCommandReader.test.ts index bb2fd7598a..bc98f425b4 100644 --- a/src/artifacts/apple/MachoLoadCommandReader.test.ts +++ b/src/artifacts/apple/MachoLoadCommandReader.test.ts @@ -38,6 +38,66 @@ import { javaClassFile } from "../../../tests/fixtures/javaClassFile.js"; const read = (bytes: Uint8Array) => readMachoImage(readerOf(bytes), bytes.length); +it.skipIf(process.platform === "win32")( + "keeps non-UTF-8 symlink targets distinct from a real replacement character", + async () => { + const root = await createTestTempDirectory("rea-dylib-raw-symlink-"); + const executable = machoImage({ + commands: [ + buildVersionCommand(1), + dylibCommand(LC.LOAD_DYLIB, "@loader_path/raw-link"), + dylibCommand(LC.LOAD_DYLIB, "@loader_path/unicode-link"), + ], + }); + await writeFile(join(root, "App"), executable); + await writeFile( + join(root, "lib\ufffd"), + machoImage({ + fileType: FILE_TYPE.dylib, + commands: [buildVersionCommand(1)], + }), + ); + await symlink( + Buffer.from([0x6c, 0x69, 0x62, 0xff]), + join(root, "raw-link"), + ); + await symlink("lib\ufffd", join(root, "unicode-link")); + const trace = await traceDylibResolution({ + rootPath: root, + targetPath: join(root, "App"), + targetSha256: createHash("sha256").update(executable).digest("hex"), + enumerateRoots: false, + parameters: { roots: ["App"] }, + }); + expect(trace.edges.map((edge) => edge.resolution)).toEqual([ + { status: "undetermined", image: null }, + { status: "resolved", image: "lib\ufffd" }, + ]); + expect(trace.coverage.status).toBe("partial"); + expect(trace.limitations).toEqual( + expect.arrayContaining([ + expect.stringContaining('"raw-link":"6c6962ff"'), + ]), + ); + await expect( + traceDylibResolution({ + rootPath: root, + targetPath: join(root, "App"), + targetSha256: createHash("sha256").update(executable).digest("hex"), + enumerateRoots: false, + parameters: { roots: ["raw-link"] }, + }), + ).rejects.toMatchObject({ + issues: [ + { + path: ["roots", 0], + message: expect.stringContaining("non-UTF-8 target bytes 6c6962ff"), + }, + ], + }); + }, +); + it("retains unknown executable-root traversal after a complete callback and keeps known image facts", async () => { const root = await createTestTempDirectory("rea-dylib-traversal-callback-"); const bytes = machoImage({ commands: [buildVersionCommand(1)] }); diff --git a/src/domain/apple/dyldPaths.ts b/src/domain/apple/dyldPaths.ts index 9209de02c1..63c7cd4a1b 100644 --- a/src/domain/apple/dyldPaths.ts +++ b/src/domain/apple/dyldPaths.ts @@ -12,6 +12,11 @@ const MAX_SYMLINK_HOPS = 32; export type PathLookup = | { readonly kind: "file"; readonly path: string } + | { + readonly kind: "undetermined"; + readonly path: string; + readonly targetHex: string; + } | { readonly kind: "absent" | "escapes" | "not-file" }; /** @@ -40,6 +45,8 @@ export const resolveTreePath = async ( const entry = await view.entry(path); if (entry === undefined) return { kind: "absent" }; if (entry.kind === "symlink") { + if (entry.target === null) + return { kind: "undetermined", path, targetHex: entry.targetHex }; hops += 1; if (hops > MAX_SYMLINK_HOPS) return { kind: "absent" }; if (entry.target.startsWith("/")) return { kind: "escapes" }; diff --git a/src/domain/apple/dylibResolution.ts b/src/domain/apple/dylibResolution.ts index 5850a4253e..3b15b485ed 100644 --- a/src/domain/apple/dylibResolution.ts +++ b/src/domain/apple/dylibResolution.ts @@ -132,7 +132,12 @@ export const machoImageSlices = ( /** One observed path below the analyzed root. Symlink targets are not followed by the view. */ export type DylibTreeEntry = | { readonly kind: "file" | "directory" } - | { readonly kind: "symlink"; readonly target: string }; + | { readonly kind: "symlink"; readonly target: string } + | { + readonly kind: "symlink"; + readonly target: null; + readonly targetHex: string; + }; /** Read-only, lazily probed view of the analyzed root. Paths are root-relative. */ export interface DylibTreeView { @@ -264,6 +269,7 @@ interface ProcessContext { readonly searchOverrides: readonly DyldSearchOverride[]; readonly environmentRoots: Set; readonly unknownPlatformCandidates: Set; + readonly undecodableSymlinkTargets: Map; readonly implicitMacOSPlatformCandidates: Set; readonly platforms: readonly number[]; readonly executable: string | null; @@ -359,6 +365,10 @@ const evaluateCandidate = async ( ); if (lookup.kind === "escapes") return { ...base, outcome: "escapes-target", resolved_path: null }; + if (lookup.kind === "undetermined") { + context.undecodableSymlinkTargets.set(lookup.path, lookup.targetHex); + return { ...base, outcome: "undetermined", resolved_path: null }; + } if (lookup.kind !== "file") return { ...base, outcome: "absent", resolved_path: null }; const facts = await imageFacts(context, lookup.path); @@ -663,6 +673,7 @@ const projectTraceResult = (input: { readonly withoutArchitecture: readonly string[]; readonly environmentRoots: ReadonlySet; readonly unknownPlatformCandidates: ReadonlySet; + readonly undecodableSymlinkTargets: ReadonlyMap; readonly implicitMacOSPlatformCandidates: ReadonlySet; readonly rootEnumeration: DylibRootEnumeration | undefined; }): DylibTrace => { @@ -695,6 +706,7 @@ const projectTraceResult = (input: { input.withoutArchitecture.length === 0 && input.environmentRoots.size === 0 && input.unknownPlatformCandidates.size === 0 && + input.undecodableSymlinkTargets.size === 0 && input.rootEnumeration?.status !== "unknown" ? "complete" : "partial", @@ -704,6 +716,11 @@ const projectTraceResult = (input: { limitations: [ ...DYLIB_RESOLUTION_LIMITATIONS, ...(input.rootEnumeration?.limitations ?? []), + ...(input.undecodableSymlinkTargets.size === 0 + ? [] + : [ + `Symlink targets cannot be represented as UTF-8; their paths and stored target bytes (hex) are ${JSON.stringify(Object.fromEntries(input.undecodableSymlinkTargets))}. Affected dependency resolution remains undetermined.`, + ]), ...(input.environmentRoots.size === 0 ? [] : [ @@ -744,6 +761,7 @@ export const traceDylibLoading = async ( const withoutArchitecture: string[] = []; const environmentRoots = new Set(); const unknownPlatformCandidates = new Set(); + const undecodableSymlinkTargets = new Map(); const implicitMacOSPlatformCandidates = new Set(); for (const root of request.roots) { const facts = await view.image(root); @@ -785,6 +803,7 @@ export const traceDylibLoading = async ( searchOverrides, environmentRoots, unknownPlatformCandidates, + undecodableSymlinkTargets, implicitMacOSPlatformCandidates, platforms: slice.platforms, executable, @@ -806,6 +825,7 @@ export const traceDylibLoading = async ( withoutArchitecture, environmentRoots, unknownPlatformCandidates, + undecodableSymlinkTargets, implicitMacOSPlatformCandidates, rootEnumeration: request.rootEnumeration, }); diff --git a/src/domain/firmware/firmwareAnalysis.ts b/src/domain/firmware/firmwareAnalysis.ts index 9a1434fda5..d7e3801d23 100644 --- a/src/domain/firmware/firmwareAnalysis.ts +++ b/src/domain/firmware/firmwareAnalysis.ts @@ -196,6 +196,11 @@ export const firmwareResultSchemas = { relative_path: z.string(), kind: z.enum(["symlink", "special"]), link_target: z.string().nullable(), + // Older retained Evidence may lack raw target bytes. + link_target_hex: z + .string() + .regex(/^(?:[0-9a-f]{2})*$/u) + .optional(), }), ), chunks: z.array(chunk), diff --git a/src/domain/process/processCapture.ts b/src/domain/process/processCapture.ts index 6ee34b42cd..784d4c1c4a 100644 --- a/src/domain/process/processCapture.ts +++ b/src/domain/process/processCapture.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import { observedSymlinkTarget } from "./symlinkTarget.js"; +import { observedSymlinkTarget } from "../symlinkTarget.js"; import { utcDateTimeSchema } from "../offsetDateTime.js"; import { jsonValueSchema } from "../jsonValue.js"; diff --git a/src/domain/process/symlinkTarget.ts b/src/domain/process/symlinkTarget.ts index 2f13596fea..9d93b1d184 100644 --- a/src/domain/process/symlinkTarget.ts +++ b/src/domain/process/symlinkTarget.ts @@ -1,16 +1,5 @@ import type { FileState } from "./processCapture.js"; -/** Preserve stored bytes; undecodable targets have no invented text spelling. */ -export const observedSymlinkTarget = ( - raw: Buffer, -): { text: string | null; hex: string } => { - const decoded = raw.toString("utf8"); - return { - text: Buffer.from(decoded, "utf8").equals(raw) ? decoded : null, - hex: raw.toString("hex"), - }; -}; - /** Old replacement characters cannot establish the target's original bytes. */ export const symlinkTargetIdentity = ( file: Extract, diff --git a/src/domain/symlinkTarget.ts b/src/domain/symlinkTarget.ts new file mode 100644 index 0000000000..70f6d064aa --- /dev/null +++ b/src/domain/symlinkTarget.ts @@ -0,0 +1,10 @@ +/** Preserve stored bytes; undecodable targets have no invented text spelling. */ +export const observedSymlinkTarget = ( + raw: Buffer, +): { text: string | null; hex: string } => { + const decoded = raw.toString("utf8"); + return { + text: Buffer.from(decoded, "utf8").equals(raw) ? decoded : null, + hex: raw.toString("hex"), + }; +}; diff --git a/src/firmware/FirmwareFiles.ts b/src/firmware/FirmwareFiles.ts index dde5cf30da..e0eeeb7532 100644 --- a/src/firmware/FirmwareFiles.ts +++ b/src/firmware/FirmwareFiles.ts @@ -35,6 +35,7 @@ import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; import { readArtifactFileChunks } from "../artifacts/readArtifactFileChunks.js"; import { decodeDirectoryEntryName } from "../filesystem/decodeDirectoryEntryName.js"; import { NonRegularFileReadError } from "../filesystem/RegularFile.js"; +import { observedSymlinkTarget } from "../domain/symlinkTarget.js"; /** One regular file or unpublished filesystem entry in a private extraction. */ export interface FirmwareEntry { @@ -43,6 +44,7 @@ export interface FirmwareEntry { readonly size: number; readonly kind: "file" | "symlink" | "special"; readonly linkTarget: string | null; + readonly linkTargetHex?: string; } const throwIfFirmwareInventoryCancelled = ( @@ -428,6 +430,9 @@ export const inventoryFirmwareOutput = async ( if (stat.isFile()) bytes += stat.size; if (bytes > maxBytes) throw firmwareStagingLimit(operation, "max_output_bytes", maxBytes); + const linkTarget = stat.isSymbolicLink() + ? observedSymlinkTarget(await readlink(path, { encoding: "buffer" })) + : undefined; entries.push({ path, relativePath: relative(root, path).split(sep).join("/"), @@ -437,7 +442,8 @@ export const inventoryFirmwareOutput = async ( : stat.isFile() ? "file" : "special", - linkTarget: stat.isSymbolicLink() ? await readlink(path) : null, + linkTarget: linkTarget?.text ?? null, + ...(linkTarget === undefined ? {} : { linkTargetHex: linkTarget.hex }), }); } } diff --git a/src/firmware/FirmwarePublication.ts b/src/firmware/FirmwarePublication.ts index 86e26e26d7..1f40772ce4 100644 --- a/src/firmware/FirmwarePublication.ts +++ b/src/firmware/FirmwarePublication.ts @@ -141,6 +141,9 @@ export const publishFirmwareExtraction = async (context: { relative_path: entry.relativePath, kind: entry.kind, link_target: entry.linkTarget, + ...(entry.linkTargetHex === undefined + ? {} + : { link_target_hex: entry.linkTargetHex }), }); continue; } diff --git a/src/process/capture/FilesystemSnapshot.ts b/src/process/capture/FilesystemSnapshot.ts index 1bc48beb84..06366fa376 100644 --- a/src/process/capture/FilesystemSnapshot.ts +++ b/src/process/capture/FilesystemSnapshot.ts @@ -1,4 +1,4 @@ -import { observedSymlinkTarget } from "../../domain/process/symlinkTarget.js"; +import { observedSymlinkTarget } from "../../domain/symlinkTarget.js"; import { createHash } from "node:crypto"; import { constants as fsConstants } from "node:fs"; import { lstat, open, opendir, readlink, realpath } from "node:fs/promises"; diff --git a/tests/boundary/firmware/firmwareIntegration.test.ts b/tests/boundary/firmware/firmwareIntegration.test.ts index daf7c9796d..0270baa030 100644 --- a/tests/boundary/firmware/firmwareIntegration.test.ts +++ b/tests/boundary/firmware/firmwareIntegration.test.ts @@ -720,6 +720,7 @@ it.each(["depth", "dependency", "link"])( relative_path: "escape", kind: "symlink", link_target: "/etc/passwd", + link_target_hex: Buffer.from("/etc/passwd", "utf8").toString("hex"), }, ]); expect(gapsWithContinuations(normalized)).toEqual([ diff --git a/tests/boundary/firmware/firmwarePublicationPaths.test.ts b/tests/boundary/firmware/firmwarePublicationPaths.test.ts index 6d21371da1..61b26a3c73 100644 --- a/tests/boundary/firmware/firmwarePublicationPaths.test.ts +++ b/tests/boundary/firmware/firmwarePublicationPaths.test.ts @@ -1,16 +1,81 @@ import { createHash } from "node:crypto"; -import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import { + access, + lstat, + mkdir, + readFile, + readlink, + symlink, + writeFile, +} from "node:fs/promises"; import { join } from "node:path"; import { expect, it, vi } from "vitest"; +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import type { AnySchema } from "ajv"; +import { FIRMWARE_TOOL_CONTRACTS } from "../../../src/contracts/firmware/firmwareToolContracts.js"; +import { toolOutputSchemaWithMetadata } from "../../../src/contracts/toolSchemaMetadata.js"; +import { createEvidence } from "../../../src/domain/evidence.js"; +import { RequestTrackingMcpServer } from "../../../src/server/RequestTrackingMcpServer.js"; import { FirmwareProvider } from "../../../src/firmware/FirmwareProvider.js"; import { FirmwareAnalysisService } from "../../../src/application/firmware/FirmwareAnalysisService.js"; -import { FirmwareFileAccess } from "../../../src/firmware/FirmwareFiles.js"; +import { + FirmwareFileAccess, + inventoryFirmwareOutput, +} from "../../../src/firmware/FirmwareFiles.js"; import { publishFirmwareExtraction } from "../../../src/firmware/FirmwarePublication.js"; import { normalizeUnblobReport } from "../../../src/firmware/FirmwareReports.js"; import { startFirmwareDeadline } from "../../../src/firmware/FirmwareDeadline.js"; import { SafeOutputTree } from "../../../src/artifacts/SafeOutputTree.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +const validateAdvertisedFirmwareOutput = async ( + result: Awaited>, +): Promise => { + const contract = FIRMWARE_TOOL_CONTRACTS.find( + (item) => item.name === "extract_firmware", + ); + if (contract === undefined) + throw new Error("Missing extract_firmware contract"); + const server = new RequestTrackingMcpServer({ + name: "schema", + version: "1", + }); + const client = new Client({ name: "schema-client", version: "1" }); + server.registerTool( + contract.name, + { outputSchema: toolOutputSchemaWithMetadata(contract) }, + () => ({ content: [] }), + ); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + const schema = (await client.listTools()).tools.find( + (tool) => tool.name === contract.name, + )?.outputSchema; + if (schema === undefined) throw new Error("Missing firmware output schema"); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + expect(ajv.validateSchema(schema as AnySchema)).toBe(true); + const validate = ajv.compile(schema as AnySchema); + const evidence = createEvidence( + undefined, + { id: "fixture", name: "fixture", version: null }, + { + operation: "extract_firmware", + parameters: {}, + result, + confidence: "observed", + }, + ); + expect(validate(evidence), JSON.stringify(validate.errors)).toBe(true); + } finally { + await client.close(); + await server.close(); + } +}; + it.each(["existing", "missing-parent", "relative"] as const)( "refuses %s destinations before selecting or launching a firmware engine", async (kind) => { @@ -70,6 +135,14 @@ it("publishes the writer's normalized path while retaining source spelling and u const bytes = Buffer.from("child"); const path = join(staging, name); await writeFile(path, bytes); + if (process.platform !== "win32") { + await symlink( + Buffer.from([0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0xff]), + join(staging, "raw-link"), + ); + await symlink("sample\ufffd", join(staging, "unicode-link")); + await symlink("sample", join(staging, "control-link")); + } const sha256 = createHash("sha256").update(bytes).digest("hex"); const inputPath = join(root, "input"); const inputHash = "a".repeat(64); @@ -111,15 +184,12 @@ it("publishes the writer's normalized path while retaining source spelling and u deadline: startFirmwareDeadline("publication", 120_000), analysisTimeoutMs: 120_000, outputDirectory: output, - entries: [ - { - path, - relativePath: name, - size: bytes.length, - kind: "file", - linkTarget: null, - }, - ], + entries: await inventoryFirmwareOutput( + staging, + 1024, + 10, + "extract_firmware", + ), normalized, selection: { offset: 7, length: 1, sha256: inputHash }, maxDepth: 3, @@ -144,18 +214,46 @@ it("publishes the writer's normalized path while retaining source spelling and u const published = result.files[0]; if (published === undefined) throw new Error("Expected published child"); expect(Buffer.compare(await readFile(published.path), bytes)).toBe(0); + if (process.platform !== "win32") { + expect(result.unpublished_entries).toEqual([ + { + relative_path: "control-link", + kind: "symlink", + link_target: "sample", + link_target_hex: "73616d706c65", + }, + { + relative_path: "raw-link", + kind: "symlink", + link_target: null, + link_target_hex: "73616d706c65ff", + }, + { + relative_path: "unicode-link", + kind: "symlink", + link_target: "sample\ufffd", + link_target_hex: "73616d706c65efbfbd", + }, + ]); + const rawLink = join(output, "raw-link"); + await expect(lstat(rawLink)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(readlink(rawLink)).rejects.toMatchObject({ + code: "ENOENT", + }); + await validateAdvertisedFirmwareOutput(result); + } expect(result).toMatchObject({ coverage: "partial", - extraction_gaps: [ - { + extraction_gaps: expect.arrayContaining([ + expect.objectContaining({ kind: "unclassified-range", input_path: "$input", provider_id: null, handler: null, range: { offset: 0, length: 1 }, root_file_range: { offset: 7, length: 1 }, - }, - ], + }), + ]), }); expect(retainCleanup).not.toHaveBeenCalled(); } finally { diff --git a/tests/boundary/mcp/dylibResolutionMcp.test.ts b/tests/boundary/mcp/dylibResolutionMcp.test.ts index f7b0990d65..e36fba4af0 100644 --- a/tests/boundary/mcp/dylibResolutionMcp.test.ts +++ b/tests/boundary/mcp/dylibResolutionMcp.test.ts @@ -373,6 +373,38 @@ it("follows relative symlinks only while they stay inside the root", async () => expect(JSON.stringify(trace)).not.toContain(outsideSha256); }); +it.skipIf(process.platform === "win32")( + "does not resolve a non-UTF-8 link target as a replacement-character path", + async () => { + const { trace } = await traceBesideOutside( + [dylibCommand(LC.LOAD_DYLIB, "@loader_path/raw-link")], + async (analyzed) => { + await symlink( + Buffer.from([0x6c, 0x69, 0x62, 0xff]), + join(analyzed, "raw-link"), + ); + await writeFile( + join(analyzed, "lib\ufffd"), + machoImage({ + fileType: FILE_TYPE.dylib, + commands: [buildVersionCommand(1)], + }), + ); + }, + ); + expect(trace.edges[0]?.resolution).toEqual({ + status: "undetermined", + image: null, + }); + expect(trace.coverage.status).toBe("partial"); + expect(trace.limitations).toEqual( + expect.arrayContaining([ + expect.stringContaining('"raw-link":"6c6962ff"'), + ]), + ); + }, +); + it.skipIf(process.platform === "win32")( "resolves a POSIX file name that contains backslashes", async () => {