fix(filesystem): preserve undecodable symlink targets (#7935)

* fix(firmware): preserve symlink target bytes

* fix(apple): preserve undecodable dylib link targets

* style(tests): fix CI formatting

* test(firmware): select the named contract directly

* test(firmware): record raw bytes for a decodable symlink target

* test(mcp): assert bundle path issues by field

compare_bundles reports each relative path as an invalid_format issue
instead of one joined diagnostic sentence.

* test(firmware): stat raw links without following them

access follows a symlink, so a published dangling raw-link still looked
absent. lstat and readlink inspect the link itself.

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
This commit is contained in:
morlutoandCursor Agent authored and GitHub committed 2026-10-29 20:07:30 +08:00
1 parent e124bdfc54
commit 34577eb1cc
16 files changed
+281 -31

No files matched your search

+5
View File
@@ -82,6 +82,11 @@ Published file `relative_path` preserves producer spelling for lineage; `path`
uses the writer's actual normalized filename. An extracted filename that cannot
be represented losslessly as UTF-8 fails publication explicitly with its raw
name bytes in hexadecimal instead of silently omitting the entry.
Unpublished symlinks retain the extracted tree's target bytes in lowercase
`link_target_hex`; `link_target` is null when those bytes are not valid UTF-8.
Older retained Evidence may omit the hexadecimal field. Unblob may have rewritten
the target before REA observed it, so these bytes are not necessarily those stored
in the firmware image.
## CLI and MCP
+3
View File
@@ -638,6 +638,9 @@ complete observation.
undecodable filenames and skipped symlinks, as partial coverage. Known
executable roots remain available; unexamined names and symlink targets
cannot establish an empty root set or the absence of other roots.
A dependency symlink whose target is not valid UTF-8 is undetermined, with
its stored target bytes in the limitation and partial coverage; a selected
root crossing such a symlink is refused with the same bytes in its diagnostic.
- `inspect_asset_catalog` / `rea inspect-asset-catalog <app>` reads compiled
`Assets.car` metadata through macOS `assetutil --info`. Catalog digests, raw
rendition fields, pagination and exact UI resource-name matches are returned.
+12 -1
View File
@@ -4,6 +4,7 @@ import { isAbsolute, join, relative, sep } from "node:path";
import { AnalysisInputError } from "../../domain/analysisErrorCore.js";
import { resolveTreePath } from "../../domain/apple/dyldPaths.js";
import { observedSymlinkTarget } from "../../domain/symlinkTarget.js";
import {
dylibResolutionInputSchema,
dylibResolutionResultSchema,
@@ -113,7 +114,12 @@ class FilesystemTreeView implements DylibTreeView {
try {
const metadata = await lstat(absolute);
if (metadata.isSymbolicLink()) {
const target = await readlink(absolute);
const observed = observedSymlinkTarget(
await readlink(absolute, { encoding: "buffer" }),
);
if (observed.text === null)
return { kind: "symlink", target: null, targetHex: observed.hex };
const target = observed.text;
// Windows link text separates components with `\`.
return {
kind: "symlink",
@@ -333,6 +339,11 @@ const requireMachoRoots = async (
const lookup = await resolveTreePath(view, path);
if (lookup.kind === "escapes")
throw reject("path", `Root ${path} resolves outside the analyzed root`);
if (lookup.kind === "undetermined")
throw reject(
"path",
`Root ${path} crosses symlink ${lookup.path} with non-UTF-8 target bytes ${lookup.targetHex}`,
);
if (lookup.kind !== "file")
throw reject(
"path",
@@ -38,6 +38,66 @@ import { javaClassFile } from "../../../tests/fixtures/javaClassFile.js";
const read = (bytes: Uint8Array) =>
readMachoImage(readerOf(bytes), bytes.length);
it.skipIf(process.platform === "win32")(
"keeps non-UTF-8 symlink targets distinct from a real replacement character",
async () => {
const root = await createTestTempDirectory("rea-dylib-raw-symlink-");
const executable = machoImage({
commands: [
buildVersionCommand(1),
dylibCommand(LC.LOAD_DYLIB, "@loader_path/raw-link"),
dylibCommand(LC.LOAD_DYLIB, "@loader_path/unicode-link"),
],
});
await writeFile(join(root, "App"), executable);
await writeFile(
join(root, "lib\ufffd"),
machoImage({
fileType: FILE_TYPE.dylib,
commands: [buildVersionCommand(1)],
}),
);
await symlink(
Buffer.from([0x6c, 0x69, 0x62, 0xff]),
join(root, "raw-link"),
);
await symlink("lib\ufffd", join(root, "unicode-link"));
const trace = await traceDylibResolution({
rootPath: root,
targetPath: join(root, "App"),
targetSha256: createHash("sha256").update(executable).digest("hex"),
enumerateRoots: false,
parameters: { roots: ["App"] },
});
expect(trace.edges.map((edge) => edge.resolution)).toEqual([
{ status: "undetermined", image: null },
{ status: "resolved", image: "lib\ufffd" },
]);
expect(trace.coverage.status).toBe("partial");
expect(trace.limitations).toEqual(
expect.arrayContaining([
expect.stringContaining('"raw-link":"6c6962ff"'),
]),
);
await expect(
traceDylibResolution({
rootPath: root,
targetPath: join(root, "App"),
targetSha256: createHash("sha256").update(executable).digest("hex"),
enumerateRoots: false,
parameters: { roots: ["raw-link"] },
}),
).rejects.toMatchObject({
issues: [
{
path: ["roots", 0],
message: expect.stringContaining("non-UTF-8 target bytes 6c6962ff"),
},
],
});
},
);
it("retains unknown executable-root traversal after a complete callback and keeps known image facts", async () => {
const root = await createTestTempDirectory("rea-dylib-traversal-callback-");
const bytes = machoImage({ commands: [buildVersionCommand(1)] });
+7
View File
@@ -12,6 +12,11 @@ const MAX_SYMLINK_HOPS = 32;
export type PathLookup =
| { readonly kind: "file"; readonly path: string }
| {
readonly kind: "undetermined";
readonly path: string;
readonly targetHex: string;
}
| { readonly kind: "absent" | "escapes" | "not-file" };
/**
@@ -40,6 +45,8 @@ export const resolveTreePath = async (
const entry = await view.entry(path);
if (entry === undefined) return { kind: "absent" };
if (entry.kind === "symlink") {
if (entry.target === null)
return { kind: "undetermined", path, targetHex: entry.targetHex };
hops += 1;
if (hops > MAX_SYMLINK_HOPS) return { kind: "absent" };
if (entry.target.startsWith("/")) return { kind: "escapes" };
+21 -1
View File
@@ -132,7 +132,12 @@ export const machoImageSlices = (
/** One observed path below the analyzed root. Symlink targets are not followed by the view. */
export type DylibTreeEntry =
| { readonly kind: "file" | "directory" }
| { readonly kind: "symlink"; readonly target: string };
| { readonly kind: "symlink"; readonly target: string }
| {
readonly kind: "symlink";
readonly target: null;
readonly targetHex: string;
};
/** Read-only, lazily probed view of the analyzed root. Paths are root-relative. */
export interface DylibTreeView {
@@ -264,6 +269,7 @@ interface ProcessContext {
readonly searchOverrides: readonly DyldSearchOverride[];
readonly environmentRoots: Set<string>;
readonly unknownPlatformCandidates: Set<string>;
readonly undecodableSymlinkTargets: Map<string, string>;
readonly implicitMacOSPlatformCandidates: Set<string>;
readonly platforms: readonly number[];
readonly executable: string | null;
@@ -359,6 +365,10 @@ const evaluateCandidate = async (
);
if (lookup.kind === "escapes")
return { ...base, outcome: "escapes-target", resolved_path: null };
if (lookup.kind === "undetermined") {
context.undecodableSymlinkTargets.set(lookup.path, lookup.targetHex);
return { ...base, outcome: "undetermined", resolved_path: null };
}
if (lookup.kind !== "file")
return { ...base, outcome: "absent", resolved_path: null };
const facts = await imageFacts(context, lookup.path);
@@ -663,6 +673,7 @@ const projectTraceResult = (input: {
readonly withoutArchitecture: readonly string[];
readonly environmentRoots: ReadonlySet<string>;
readonly unknownPlatformCandidates: ReadonlySet<string>;
readonly undecodableSymlinkTargets: ReadonlyMap<string, string>;
readonly implicitMacOSPlatformCandidates: ReadonlySet<string>;
readonly rootEnumeration: DylibRootEnumeration | undefined;
}): DylibTrace => {
@@ -695,6 +706,7 @@ const projectTraceResult = (input: {
input.withoutArchitecture.length === 0 &&
input.environmentRoots.size === 0 &&
input.unknownPlatformCandidates.size === 0 &&
input.undecodableSymlinkTargets.size === 0 &&
input.rootEnumeration?.status !== "unknown"
? "complete"
: "partial",
@@ -704,6 +716,11 @@ const projectTraceResult = (input: {
limitations: [
...DYLIB_RESOLUTION_LIMITATIONS,
...(input.rootEnumeration?.limitations ?? []),
...(input.undecodableSymlinkTargets.size === 0
? []
: [
`Symlink targets cannot be represented as UTF-8; their paths and stored target bytes (hex) are ${JSON.stringify(Object.fromEntries(input.undecodableSymlinkTargets))}. Affected dependency resolution remains undetermined.`,
]),
...(input.environmentRoots.size === 0
? []
: [
@@ -744,6 +761,7 @@ export const traceDylibLoading = async (
const withoutArchitecture: string[] = [];
const environmentRoots = new Set<string>();
const unknownPlatformCandidates = new Set<string>();
const undecodableSymlinkTargets = new Map<string, string>();
const implicitMacOSPlatformCandidates = new Set<string>();
for (const root of request.roots) {
const facts = await view.image(root);
@@ -785,6 +803,7 @@ export const traceDylibLoading = async (
searchOverrides,
environmentRoots,
unknownPlatformCandidates,
undecodableSymlinkTargets,
implicitMacOSPlatformCandidates,
platforms: slice.platforms,
executable,
@@ -806,6 +825,7 @@ export const traceDylibLoading = async (
withoutArchitecture,
environmentRoots,
unknownPlatformCandidates,
undecodableSymlinkTargets,
implicitMacOSPlatformCandidates,
rootEnumeration: request.rootEnumeration,
});
+5
View File
@@ -196,6 +196,11 @@ export const firmwareResultSchemas = {
relative_path: z.string(),
kind: z.enum(["symlink", "special"]),
link_target: z.string().nullable(),
// Older retained Evidence may lack raw target bytes.
link_target_hex: z
.string()
.regex(/^(?:[0-9a-f]{2})*$/u)
.optional(),
}),
),
chunks: z.array(chunk),
+1 -1
View File
@@ -1,5 +1,5 @@
import { z } from "zod";
import { observedSymlinkTarget } from "./symlinkTarget.js";
import { observedSymlinkTarget } from "../symlinkTarget.js";
import { utcDateTimeSchema } from "../offsetDateTime.js";
import { jsonValueSchema } from "../jsonValue.js";
-11
View File
@@ -1,16 +1,5 @@
import type { FileState } from "./processCapture.js";
/** Preserve stored bytes; undecodable targets have no invented text spelling. */
export const observedSymlinkTarget = (
raw: Buffer,
): { text: string | null; hex: string } => {
const decoded = raw.toString("utf8");
return {
text: Buffer.from(decoded, "utf8").equals(raw) ? decoded : null,
hex: raw.toString("hex"),
};
};
/** Old replacement characters cannot establish the target's original bytes. */
export const symlinkTargetIdentity = (
file: Extract<FileState, { type: "symlink" }>,
+10
View File
@@ -0,0 +1,10 @@
/** Preserve stored bytes; undecodable targets have no invented text spelling. */
export const observedSymlinkTarget = (
raw: Buffer,
): { text: string | null; hex: string } => {
const decoded = raw.toString("utf8");
return {
text: Buffer.from(decoded, "utf8").equals(raw) ? decoded : null,
hex: raw.toString("hex"),
};
};
+7 -1
View File
@@ -35,6 +35,7 @@ import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
import { readArtifactFileChunks } from "../artifacts/readArtifactFileChunks.js";
import { decodeDirectoryEntryName } from "../filesystem/decodeDirectoryEntryName.js";
import { NonRegularFileReadError } from "../filesystem/RegularFile.js";
import { observedSymlinkTarget } from "../domain/symlinkTarget.js";
/** One regular file or unpublished filesystem entry in a private extraction. */
export interface FirmwareEntry {
@@ -43,6 +44,7 @@ export interface FirmwareEntry {
readonly size: number;
readonly kind: "file" | "symlink" | "special";
readonly linkTarget: string | null;
readonly linkTargetHex?: string;
}
const throwIfFirmwareInventoryCancelled = (
@@ -428,6 +430,9 @@ export const inventoryFirmwareOutput = async (
if (stat.isFile()) bytes += stat.size;
if (bytes > maxBytes)
throw firmwareStagingLimit(operation, "max_output_bytes", maxBytes);
const linkTarget = stat.isSymbolicLink()
? observedSymlinkTarget(await readlink(path, { encoding: "buffer" }))
: undefined;
entries.push({
path,
relativePath: relative(root, path).split(sep).join("/"),
@@ -437,7 +442,8 @@ export const inventoryFirmwareOutput = async (
: stat.isFile()
? "file"
: "special",
linkTarget: stat.isSymbolicLink() ? await readlink(path) : null,
linkTarget: linkTarget?.text ?? null,
...(linkTarget === undefined ? {} : { linkTargetHex: linkTarget.hex }),
});
}
}
+3
View File
@@ -141,6 +141,9 @@ export const publishFirmwareExtraction = async (context: {
relative_path: entry.relativePath,
kind: entry.kind,
link_target: entry.linkTarget,
...(entry.linkTargetHex === undefined
? {}
: { link_target_hex: entry.linkTargetHex }),
});
continue;
}
+1 -1
View File
@@ -1,4 +1,4 @@
import { observedSymlinkTarget } from "../../domain/process/symlinkTarget.js";
import { observedSymlinkTarget } from "../../domain/symlinkTarget.js";
import { createHash } from "node:crypto";
import { constants as fsConstants } from "node:fs";
import { lstat, open, opendir, readlink, realpath } from "node:fs/promises";
@@ -720,6 +720,7 @@ it.each(["depth", "dependency", "link"])(
relative_path: "escape",
kind: "symlink",
link_target: "/etc/passwd",
link_target_hex: Buffer.from("/etc/passwd", "utf8").toString("hex"),
},
]);
expect(gapsWithContinuations(normalized)).toEqual([
@@ -1,16 +1,81 @@
import { createHash } from "node:crypto";
import { access, mkdir, readFile, writeFile } from "node:fs/promises";
import {
access,
lstat,
mkdir,
readFile,
readlink,
symlink,
writeFile,
} from "node:fs/promises";
import { join } from "node:path";
import { expect, it, vi } from "vitest";
import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
import { Ajv2020 } from "ajv/dist/2020.js";
import type { AnySchema } from "ajv";
import { FIRMWARE_TOOL_CONTRACTS } from "../../../src/contracts/firmware/firmwareToolContracts.js";
import { toolOutputSchemaWithMetadata } from "../../../src/contracts/toolSchemaMetadata.js";
import { createEvidence } from "../../../src/domain/evidence.js";
import { RequestTrackingMcpServer } from "../../../src/server/RequestTrackingMcpServer.js";
import { FirmwareProvider } from "../../../src/firmware/FirmwareProvider.js";
import { FirmwareAnalysisService } from "../../../src/application/firmware/FirmwareAnalysisService.js";
import { FirmwareFileAccess } from "../../../src/firmware/FirmwareFiles.js";
import {
FirmwareFileAccess,
inventoryFirmwareOutput,
} from "../../../src/firmware/FirmwareFiles.js";
import { publishFirmwareExtraction } from "../../../src/firmware/FirmwarePublication.js";
import { normalizeUnblobReport } from "../../../src/firmware/FirmwareReports.js";
import { startFirmwareDeadline } from "../../../src/firmware/FirmwareDeadline.js";
import { SafeOutputTree } from "../../../src/artifacts/SafeOutputTree.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
const validateAdvertisedFirmwareOutput = async (
result: Awaited<ReturnType<typeof publishFirmwareExtraction>>,
): Promise<void> => {
const contract = FIRMWARE_TOOL_CONTRACTS.find(
(item) => item.name === "extract_firmware",
);
if (contract === undefined)
throw new Error("Missing extract_firmware contract");
const server = new RequestTrackingMcpServer({
name: "schema",
version: "1",
});
const client = new Client({ name: "schema-client", version: "1" });
server.registerTool(
contract.name,
{ outputSchema: toolOutputSchemaWithMetadata(contract) },
() => ({ content: [] }),
);
const [clientTransport, serverTransport] =
InMemoryTransport.createLinkedPair();
try {
await server.connect(serverTransport);
await client.connect(clientTransport);
const schema = (await client.listTools()).tools.find(
(tool) => tool.name === contract.name,
)?.outputSchema;
if (schema === undefined) throw new Error("Missing firmware output schema");
const ajv = new Ajv2020({ strict: false, validateFormats: false });
expect(ajv.validateSchema(schema as AnySchema)).toBe(true);
const validate = ajv.compile(schema as AnySchema);
const evidence = createEvidence(
undefined,
{ id: "fixture", name: "fixture", version: null },
{
operation: "extract_firmware",
parameters: {},
result,
confidence: "observed",
},
);
expect(validate(evidence), JSON.stringify(validate.errors)).toBe(true);
} finally {
await client.close();
await server.close();
}
};
it.each(["existing", "missing-parent", "relative"] as const)(
"refuses %s destinations before selecting or launching a firmware engine",
async (kind) => {
@@ -70,6 +135,14 @@ it("publishes the writer's normalized path while retaining source spelling and u
const bytes = Buffer.from("child");
const path = join(staging, name);
await writeFile(path, bytes);
if (process.platform !== "win32") {
await symlink(
Buffer.from([0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0xff]),
join(staging, "raw-link"),
);
await symlink("sample\ufffd", join(staging, "unicode-link"));
await symlink("sample", join(staging, "control-link"));
}
const sha256 = createHash("sha256").update(bytes).digest("hex");
const inputPath = join(root, "input");
const inputHash = "a".repeat(64);
@@ -111,15 +184,12 @@ it("publishes the writer's normalized path while retaining source spelling and u
deadline: startFirmwareDeadline("publication", 120_000),
analysisTimeoutMs: 120_000,
outputDirectory: output,
entries: [
{
path,
relativePath: name,
size: bytes.length,
kind: "file",
linkTarget: null,
},
],
entries: await inventoryFirmwareOutput(
staging,
1024,
10,
"extract_firmware",
),
normalized,
selection: { offset: 7, length: 1, sha256: inputHash },
maxDepth: 3,
@@ -144,18 +214,46 @@ it("publishes the writer's normalized path while retaining source spelling and u
const published = result.files[0];
if (published === undefined) throw new Error("Expected published child");
expect(Buffer.compare(await readFile(published.path), bytes)).toBe(0);
if (process.platform !== "win32") {
expect(result.unpublished_entries).toEqual([
{
relative_path: "control-link",
kind: "symlink",
link_target: "sample",
link_target_hex: "73616d706c65",
},
{
relative_path: "raw-link",
kind: "symlink",
link_target: null,
link_target_hex: "73616d706c65ff",
},
{
relative_path: "unicode-link",
kind: "symlink",
link_target: "sample\ufffd",
link_target_hex: "73616d706c65efbfbd",
},
]);
const rawLink = join(output, "raw-link");
await expect(lstat(rawLink)).rejects.toMatchObject({ code: "ENOENT" });
await expect(readlink(rawLink)).rejects.toMatchObject({
code: "ENOENT",
});
await validateAdvertisedFirmwareOutput(result);
}
expect(result).toMatchObject({
coverage: "partial",
extraction_gaps: [
{
extraction_gaps: expect.arrayContaining([
expect.objectContaining({
kind: "unclassified-range",
input_path: "$input",
provider_id: null,
handler: null,
range: { offset: 0, length: 1 },
root_file_range: { offset: 7, length: 1 },
},
],
}),
]),
});
expect(retainCleanup).not.toHaveBeenCalled();
} finally {
@@ -373,6 +373,38 @@ it("follows relative symlinks only while they stay inside the root", async () =>
expect(JSON.stringify(trace)).not.toContain(outsideSha256);
});
it.skipIf(process.platform === "win32")(
"does not resolve a non-UTF-8 link target as a replacement-character path",
async () => {
const { trace } = await traceBesideOutside(
[dylibCommand(LC.LOAD_DYLIB, "@loader_path/raw-link")],
async (analyzed) => {
await symlink(
Buffer.from([0x6c, 0x69, 0x62, 0xff]),
join(analyzed, "raw-link"),
);
await writeFile(
join(analyzed, "lib\ufffd"),
machoImage({
fileType: FILE_TYPE.dylib,
commands: [buildVersionCommand(1)],
}),
);
},
);
expect(trace.edges[0]?.resolution).toEqual({
status: "undetermined",
image: null,
});
expect(trace.coverage.status).toBe("partial");
expect(trace.limitations).toEqual(
expect.arrayContaining([
expect.stringContaining('"raw-link":"6c6962ff"'),
]),
);
},
);
it.skipIf(process.platform === "win32")(
"resolves a POSIX file name that contains backslashes",
async () => {