fix(analysis): preserve extraction gaps and database failure reasons

Derive firmware completeness from examined coverage, retain linked continuation evidence, and preserve reported extraction limits. Keep Android package attribution and Go image fixtures coherent, and classify SQLite busy failures without discarding descriptor cleanup facts.

Reviewed fixes include firmware, ADB, SQLite, and recorded-transport bug reports and their focused PRs.

Validation: firmware report source tests passed 21 cases; prior focused artifact/provider boundaries passed. Real Android devices and external firmware tools were not rerun.
This commit is contained in:
morluto committed 2026-10-14 08:32:15 +08:00
1 parent 80de3145e4
commit 5b7fe94710
22 files changed
+2179 -100

No files matched your search

+23
View File
@@ -103,6 +103,23 @@ build's source revision, so that field remains unknown.
engine finished its configured extraction without those reported conditions;
default skip rules and format coverage still apply. It does not establish
complete firmware reconstruction.
- `extraction_gaps` makes those conditions recoverable: a row identifies an
unextracted region or a failed task/stage. A chunk retains its identity and
byte range, plus its original-file range when it belongs to the selected input.
A multi-file stage retains its reported ID and handler with unknown byte ranges;
a standalone task failure keeps unavailable stage identity null. Rows retain exact dependency names from
the producer's missing-dependency reports; an accepted report with no names
remains a failure with unknown dependencies; no executable identity or specific installation step is inferred. Each row's `continuation_id`
references one instruction in `extraction_continuations`, so many regions
share recovery prose without losing any region identity. Instructions use
already configured capabilities where one exists. Every extraction rerun
requires a fresh absent `output_directory`, preserving the prior evidence.
Nested unknown chunks use `inspect_firmware_regions` on the actual published
child rather than repeating the same extraction with unchanged handlers.
Inspection requires separately configured Binwalk (`REA_BINWALK_COMMAND`);
without it, the inspection step is unavailable and the region remains unexamined. A listed gap does not
mean the target is corrupt or that a retained region is empty; REA installs
nothing.
- The destination must be absent, absolute, with an existing parent. Publication
uses exclusive, symlink-resistant writes and durable digest readback. Existing
destinations are never overwritten. This is not an atomic directory rename:
@@ -167,6 +184,12 @@ records gzip offset/length and extracted file hashes. Real verification checks
CLI/MCP, full-file and selected-range extraction, unknown chunks, depth limits,
content parity and unchanged source bytes. No binary fixture is committed.
The optional missing-extractor lane asserts dependency names and the owning
handler in `extraction_gaps`. Those assertions must be run against the pinned
Unblob release to establish real-provider coverage; synthetic producer fixtures
alone do not verify Unblob's report nesting or stage identities. The gap ownership
claims remain unverified against real Unblob until that lane runs.
Additional lanes are explicit, so the base lane stays small:
```sh
@@ -0,0 +1,60 @@
import { mkdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { resolveAppBundleExecutable } from "./AppBundleExecutable.js";
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
describe("OpenStep Info.plist nesting during app target resolution", () => {
it.each([127, 128, 3000])(
"applies the shared bound to %s nested payload arrays",
async (depth) => {
const root = await createTestTempDirectory("rea-app-openstep-depth-");
const app = join(root, "Example.app");
const programs = join(app, "Contents", "MacOS");
await mkdir(programs, { recursive: true });
await writeFile(join(programs, "App"), "inert program-path fixture");
await writeFile(
join(app, "Contents", "Info.plist"),
`{ CFBundleExecutable = App; payload = ${"(".repeat(depth)}x${")".repeat(depth)}; }`,
);
const result = await resolveAppBundleExecutable(app);
if (depth === 127) {
expect(result).toMatchObject({
ok: true,
value: { executable: join(programs, "App") },
});
} else {
expect(result).toMatchObject({
ok: false,
error: {
path: join(app, "Contents", "Info.plist"),
cause: expect.objectContaining({
message: "Plist exceeds 128 nesting levels",
}),
},
});
}
},
);
it("resolves the first Info.plist root without decoding deeply nested trailing text", async () => {
const root = await createTestTempDirectory("rea-app-openstep-trailing-");
const app = join(root, "Example.app");
const programs = join(app, "Contents", "MacOS");
await mkdir(programs, { recursive: true });
await writeFile(join(programs, "App"), "inert program-path fixture");
await writeFile(
join(app, "Contents", "Info.plist"),
`{ CFBundleExecutable = App; }${"(".repeat(3000)}x${")".repeat(3000)}`,
);
expect(await resolveAppBundleExecutable(app)).toMatchObject({
ok: true,
value: { executable: join(programs, "App") },
});
});
});
+31
View File
@@ -93,6 +93,32 @@ const chunk = z.strictObject({
handler: z.string().nullable(),
encrypted: z.boolean().nullable(),
root_file_range: range.nullable(),
/** The producer reported extraction problems for this chunk. */
extraction_failed: z.boolean(),
});
/** One unextracted region or failing task/stage, with a supported continuation
* using already configured capabilities when one exists. */
export const extractionGap = z.strictObject({
kind: z.enum([
"missing-extractor-dependency",
"unknown-chunk-handler",
"encrypted-chunk",
"depth-limited",
"extractor-failure",
"unpublished-entry",
]),
/** Logical task path from the producer report; `$output/…` for unpublished entries. */
input_path: z.string(),
relative_path: z.string().nullable(),
handler: z.string().nullable(),
/** Reported chunk or multi-file stage identity; it does not imply a byte range. */
provider_id: z.string().nullable(),
range: range.nullable(),
root_file_range: range.nullable(),
/** Extractor executables named by the producer; empty when it named none. */
dependencies: z.array(z.string()),
/** Lookup key in the result's extraction_continuations table. */
continuation_id: z.string(),
});
/** Completed firmware result retained when private-runtime cleanup cannot be confirmed. */
@@ -158,6 +184,11 @@ export const firmwareResultSchemas = {
),
depth_limited_paths: z.array(z.string()),
diagnostics: z.array(jsonValueSchema),
extraction_gaps: z.array(extractionGap),
/** Each distinct recovery instruction is serialized once, shared by gap ID. */
extraction_continuations: z.array(
z.strictObject({ id: z.string(), instruction: z.string() }),
),
published_bytes: bytes,
}),
} as const;
+4 -24
View File
@@ -18,7 +18,8 @@ import {
projectionCoverage,
projectionDigest,
projectionEvidenceIdSchema,
projectionPathSchema,
unavailableComponentSchema,
unavailableComponents,
} from "../mobileApplicationGraphProjection.js";
const evidenceIdSchema = projectionEvidenceIdSchema;
@@ -46,16 +47,7 @@ export const harmonyApplicationProjectionResultSchema = z.strictObject({
javascript: z.array(componentSchema),
signing: z.array(componentSchema),
}),
unavailable_components: z.array(
z.strictObject({
path: projectionPathSchema,
artifact_id: z.null(),
sha256: z.null(),
format: z.string().min(1),
encrypted: z.boolean(),
limitations: z.array(z.string()),
}),
),
unavailable_components: z.array(unavailableComponentSchema),
app_pack_children: z.array(componentSchema),
runtime_families: z.array(
z.enum(["ark", "native", "javascript", "react-native", "flutter", "unity"]),
@@ -93,19 +85,7 @@ export const projectHarmonyApplication = (
);
const rootFormat = harmonyRootFormat(inventory.manifest.root_format);
const all = projectedComponents(inventory);
const unavailable = inventory.occurrences
.filter(
(occurrence) =>
occurrence.logical_path !== "." && occurrence.artifact_id === null,
)
.map((occurrence) => ({
path: occurrence.logical_path,
artifact_id: null,
sha256: null,
format: occurrence.artifact_format,
encrypted: occurrence.encrypted,
limitations: occurrence.limitations,
}));
const unavailable = unavailableComponents(inventory);
const classified = classify(all);
const bridgeProjection = bridgeCandidates(
classified.bytecode,
@@ -32,6 +32,16 @@ export const projectedComponentSchema = z.strictObject({
format: z.string().min(1),
});
/** Inventoried paths whose content identity is unavailable, retained as unknown. */
export const unavailableComponentSchema = z.strictObject({
path: projectionPathSchema,
artifact_id: z.null(),
sha256: z.null(),
format: z.string().min(1),
encrypted: z.boolean(),
limitations: z.array(z.string()),
});
export type ProjectedComponent = z.infer<typeof projectedComponentSchema>;
/** Path-pair hypotheses connecting managed content to a native library. */
@@ -90,6 +100,23 @@ export const projectedComponents = (
});
};
export const unavailableComponents = (
inventory: ParsedInventorySet["inventory"],
) =>
inventory.occurrences
.filter(
(occurrence) =>
occurrence.logical_path !== "." && occurrence.artifact_id === null,
)
.map((occurrence) => ({
path: occurrence.logical_path,
artifact_id: null,
sha256: null,
format: occurrence.artifact_format,
encrypted: occurrence.encrypted,
limitations: occurrence.limitations,
}));
export { bridgeCandidateCoverageSchema, projectCartesianCandidates };
/** Unicode code point ordering so projection arrays sort deterministically. */
+11
View File
@@ -0,0 +1,11 @@
import { expect, it } from "vitest";
import { parseRecordedTransportUrl } from "./recordedTransportUrl.js";
it("recognizes an empty userinfo marker at the authority start", () => {
expect(parseRecordedTransportUrl("https://@host")).toMatchObject({
origin: "https://host",
path: "/",
credentials: { start: 8, end: 9 },
});
});
+17 -6
View File
@@ -1,24 +1,35 @@
import { z } from "zod";
const diagnostics = z.array(z.string().min(1)).min(1);
const completeCoverage = z.strictObject({ status: z.literal("complete") });
const diagnostics = z.array(z.string().min(1)).min(1).exactOptional();
const completeCoverage = z.strictObject({
status: z.literal("complete"),
diagnostics,
});
const limitedDirections = z.union([
z.tuple([z.literal("callers")]).check(z.minLength(1), z.maxLength(1)),
z.tuple([z.literal("callees")]).check(z.minLength(1), z.maxLength(1)),
z
.tuple([z.literal("callers"), z.literal("callees")])
.check(z.minLength(2), z.maxLength(2)),
]);
const partialCoverage = z.strictObject({
status: z.literal("partial"),
limitations: z.array(z.string().min(1)).min(1),
limited_directions: limitedDirections.exactOptional(),
diagnostics,
});
const unknownCoverage = z.strictObject({
status: z.literal("unknown"),
reason: z.string().min(1),
limited_directions: limitedDirections.exactOptional(),
diagnostics,
});
/** Explicit coverage state for a provider's observed caller/callee set. */
export const relationshipCoverageSchema = z.union([
export const relationshipCoverageSchema = z.discriminatedUnion("status", [
completeCoverage,
completeCoverage.extend({ diagnostics }),
partialCoverage,
partialCoverage.extend({ diagnostics }),
unknownCoverage,
unknownCoverage.extend({ diagnostics }),
]);
export type RelationshipCoverage = z.infer<typeof relationshipCoverageSchema>;
+2
View File
@@ -97,6 +97,7 @@ const limitations = [
"Signature matches and reported sizes do not establish partition validity, encryption or runtime mappings. Binwalk may infer unknown lengths before emitting its report.",
"Only Linux is supported by this integration. One worker, inherited per-process address-space/file-size/CPU limits and a deadline constrain execution; staging byte and entry budgets are polled and may transiently overshoot. Limits are not aggregate process-memory or filesystem quotas.",
"Unblob's default sandbox and skip rules remain enabled. Missing external extractors, unknown chunks, encrypted chunks and depth limits yield partial coverage. Extracted code is never executed; only regular files are published, without executable permission bits.",
"extraction_gaps lists regions that were carved or left unextracted and names extractor dependencies the producer reported as missing. A listed gap does not mean the target is corrupt or that a retained region is empty.",
"The actual launcher executable is hashed. Interpreter, shared libraries and installed Python package dependencies are not independently attested; source revision remains unknown for caller-supplied builds.",
] as const;
@@ -381,6 +382,7 @@ export class FirmwareProvider implements FirmwareAnalysisPort {
entries,
normalized,
selection: target.selection,
maxDepth: request.input.max_depth,
engine,
signal,
retainCleanup: (tree) => {
+313
View File
@@ -1,3 +1,4 @@
import { createHash } from "node:crypto";
import { createReadStream } from "node:fs";
import { join } from "node:path";
import type { z } from "zod";
@@ -9,6 +10,74 @@ import { jsonValueSchema } from "../domain/jsonValue.js";
import { firmwareResultSchemas } from "../domain/firmware/firmwareAnalysis.js";
import { hashFirmwareFile, type FirmwareEntry } from "./FirmwareFiles.js";
import type { normalizeUnblobReport } from "./FirmwareReports.js";
import type { extractionGap } from "../domain/firmware/firmwareAnalysis.js";
type ExtractionGap = z.infer<typeof extractionGap>;
/** Static, truthful continuations that use already configured capabilities. */
const GAP_CONTINUATIONS: Record<ExtractionGap["kind"], string> = {
"missing-extractor-dependency":
"REA installs nothing. Add the named dependency to your own environment and rerun extract_firmware on the same immutable input with a fresh output_directory to re-evaluate the affected region; otherwise it stays unextracted.",
"unknown-chunk-handler":
"If Binwalk is configured with REA_BINWALK_COMMAND, run inspect_firmware_regions on the same input for signature hypotheses. Without configured Binwalk that inspection is unavailable. Alternatively, select a materially narrower range with extract_firmware and a fresh output_directory; the region is not decoded content.",
"encrypted-chunk":
"No static extraction continues past encryption; treat the region as unexamined rather than empty or corrupt.",
"extractor-failure":
"The producer retained this chunk's extraction problem in diagnostics. Review the reported failure, then rerun extract_firmware on the same immutable input with a fresh output_directory; the region is not decoded content.",
"depth-limited":
"Rerun extract_firmware with a higher max_depth (up to 10) and a fresh output_directory to descend into this path.",
"unpublished-entry":
"The entry is retained as metadata; only regular files are published, without executable permission bits.",
};
const UNNAMED_DEPENDENCY_CONTINUATION =
"The producer reported a missing extractor dependency without naming it; its identity is unavailable in this evidence. No specific installation or rerun can be prescribed from this report. Retain the gap as unextracted unless additional provider evidence establishes the dependency. REA installs nothing.";
import { FIRMWARE_LIMITS } from "./FirmwareRelease.js";
/** The depth ceiling cannot be raised past the input schema maximum. */
const MAX_EXTRACTION_DEPTH = 10;
const depthContinuation = (
maxDepth: number,
publishedChildPaths: readonly string[],
): string =>
maxDepth < MAX_EXTRACTION_DEPTH
? GAP_CONTINUATIONS["depth-limited"]
: publishedChildPaths.length > 0
? `This extraction already ran at the lane's maximum depth. To continue, rerun extract_firmware with path set to any eligible published child below this path and a fresh output directory; each child listed here restarts the depth budget: ${publishedChildPaths.map((path) => JSON.stringify(path)).join(", ")}.`
: `This extraction already ran at the lane's maximum depth, and no published child under this path is both nonempty and within the ${FIRMWARE_LIMITS.inputBytes}-byte firmware input limit, so the depth budget cannot be restarted from this output. Treat the region below this path as unexamined.`;
// Use a different analyzer on the actual published child; repeating the same
// whole-child extraction with unchanged handlers would reproduce this gap.
const nestedUnknownContinuation = (
publishedFile: { readonly path: string; readonly size: number } | undefined,
): string => {
const base =
"This unknown chunk was carved from a nested extraction, so its range is relative to the published child named by this gap's input_path, not to the original input.";
return publishedFile !== undefined &&
publishedFile.size > 0 &&
publishedFile.size <= FIRMWARE_LIMITS.inputBytes
? `${base} Run inspect_firmware_regions with path set to that published child for signature hypotheses only if Binwalk is configured with REA_BINWALK_COMMAND. This inspection step is unavailable without configured Binwalk; retain the region as unexamined. Repeating the same whole-child extraction with unchanged handlers would not advance analysis.`
: `${base} That published child is not a nonempty file within the ${FIRMWARE_LIMITS.inputBytes}-byte firmware input limit, so inspect_firmware_regions cannot accept it as a fresh input; treat the region as unexamined.`;
};
const emptyGap = {
relative_path: null,
handler: null,
provider_id: null,
range: null,
root_file_range: null,
dependencies: [],
};
const gapSortKey = (gap: ExtractionGap): string =>
[
gap.input_path,
gap.kind,
gap.provider_id ?? "",
gap.relative_path ?? "",
].join("\u0000");
/** Publish verified regular files, preserving links as metadata and rolling back failed writes. */
export const publishFirmwareExtraction = async (context: {
@@ -16,6 +85,8 @@ export const publishFirmwareExtraction = async (context: {
entries: readonly FirmwareEntry[];
normalized: ReturnType<typeof normalizeUnblobReport>;
selection: { offset: number; length: number; sha256: string };
/** Caller-selected extraction depth, for boundary-aware continuations. */
maxDepth: number;
engine: z.infer<typeof firmwareResultSchemas.extract_firmware>["engine"];
signal: AbortSignal;
retainCleanup: (tree: SafeOutputTree) => void;
@@ -93,6 +164,12 @@ export const publishFirmwareExtraction = async (context: {
derivations: context.normalized.derivations,
depth_limited_paths: context.normalized.depth_limited_paths,
diagnostics: context.normalized.diagnostics,
...extractionGaps(
unpublished,
files,
context.normalized,
context.maxDepth,
),
published_bytes: files.reduce((sum, file) => sum + file.size, 0),
});
context.signal.throwIfAborted();
@@ -135,3 +212,239 @@ export const publishFirmwareExtraction = async (context: {
throw cause;
}
};
/** Derive explicit recovery rows from retained report facts; no new claims. */
/** Gap rows for missing-dependency reports: chunk-nested rows keep their owning chunk. */
const dependencyExtractionGaps = (
normalized: ReturnType<typeof normalizeUnblobReport>,
continuation: (instruction: string) => string,
): ExtractionGap[] => {
// Chunk-nested dependency reports name the chunk that could not extract;
// task-wide reports (standalone or multi-file) attach to the task.
// One index over the chunk list keeps dependency-row resolution linear.
const chunksByKey = new Map(
normalized.chunks.map((chunk) => [
JSON.stringify([chunk.input_path, chunk.provider_id]),
chunk,
]),
);
return [
...normalized.chunk_missing_dependencies.flatMap(
({ input_path, provider_id, dependencies }): ExtractionGap[] => {
const affected = chunksByKey.get(
JSON.stringify([input_path, provider_id]),
);
return [
{
kind: "missing-extractor-dependency",
input_path,
...emptyGap,
...(affected === undefined
? {}
: {
handler: affected.handler,
provider_id,
range: affected.range,
root_file_range: affected.root_file_range,
}),
dependencies: [...dependencies],
continuation_id: continuation(
dependencies.length > 0
? GAP_CONTINUATIONS["missing-extractor-dependency"]
: UNNAMED_DEPENDENCY_CONTINUATION,
),
},
];
},
),
...normalized.failed_extraction_stages
.filter(({ kind }) => kind === "missing-extractor-dependency")
.map(
({
input_path,
provider_id,
handler,
dependencies,
}): ExtractionGap => ({
kind: "missing-extractor-dependency",
input_path,
...emptyGap,
handler,
provider_id,
dependencies: [...dependencies],
continuation_id: continuation(
dependencies.length > 0
? GAP_CONTINUATIONS["missing-extractor-dependency"]
: UNNAMED_DEPENDENCY_CONTINUATION,
),
}),
),
// Standalone reports name only the task; do not borrow another stage's id.
...normalized.missing_dependencies.map(
({ input_path, dependencies }): ExtractionGap => ({
kind: "missing-extractor-dependency",
input_path,
...emptyGap,
dependencies: [...dependencies],
continuation_id: continuation(
dependencies.length > 0
? GAP_CONTINUATIONS["missing-extractor-dependency"]
: UNNAMED_DEPENDENCY_CONTINUATION,
),
}),
),
];
};
const extractionGaps = (
unpublished: z.infer<
typeof firmwareResultSchemas.extract_firmware
>["unpublished_entries"],
files: readonly z.infer<
typeof firmwareResultSchemas.extract_firmware
>["files"][number][],
normalized: ReturnType<typeof normalizeUnblobReport>,
maxDepth: number,
) => {
// Intern before row construction: N gaps retain N identities, while K unique
// recovery instructions consume prose only K times in the public result.
const instructions = new Map<string, string>();
const continuation = (instruction: string): string => {
const known = instructions.get(instruction);
if (known !== undefined) return known;
const id = createHash("sha256").update(instruction).digest("hex");
instructions.set(instruction, id);
return id;
};
const dependencyGaps = dependencyExtractionGaps(normalized, continuation);
const publishedFiles = new Map(
files.map((file) => [`$output/${file.relative_path}`, file]),
);
const nestedContinuationId = (inputPath: string): string => {
return continuation(
nestedUnknownContinuation(publishedFiles.get(inputPath)),
);
};
const chunkGaps = (
kind: ExtractionGap["kind"],
matches: (chunk: (typeof normalized.chunks)[number]) => boolean,
): ExtractionGap[] =>
normalized.chunks.filter(matches).map((chunk) => ({
kind,
input_path: chunk.input_path,
...emptyGap,
handler: chunk.handler,
provider_id: chunk.provider_id,
range: chunk.range,
root_file_range: chunk.root_file_range,
dependencies: [],
continuation_id:
kind === "unknown-chunk-handler" &&
chunk.input_path.startsWith("$output/")
? nestedContinuationId(chunk.input_path)
: continuation(GAP_CONTINUATIONS[kind]),
}));
// Chunks whose failure a dependency row already covers stay out of the
// generic extractor-failure emission.
const dependencyChunkKeys = new Set(
normalized.chunk_missing_dependencies.map(({ input_path, provider_id }) =>
JSON.stringify([input_path, provider_id]),
),
);
// A depth ceiling can only be re-rooted through a published child that a
// fresh firmware input could actually accept; the index is only needed
// when a depth row will actually consult it.
const ceilingReached =
maxDepth >= MAX_EXTRACTION_DEPTH &&
normalized.depth_limited_paths.length > 0;
const reRootablePaths = new Map<string, Set<string>>();
if (ceilingReached)
for (const [filePath, file] of [...publishedFiles].sort(
([left], [right]) => (left < right ? -1 : left > right ? 1 : 0),
)) {
const size = file.size;
if (size <= 0 || size > FIRMWARE_LIMITS.inputBytes) continue;
// The published file itself can re-root a fresh run; so can every
// directory containing it.
const segments = filePath.split("/");
while (segments.length > 1) {
const ancestor = segments.join("/");
const childPaths = reRootablePaths.get(ancestor) ?? new Set<string>();
childPaths.add(file.path);
reRootablePaths.set(ancestor, childPaths);
segments.pop();
}
}
const depthContinuationIds = new Map<string, string>();
const depthContinuationId = (inputPath: string): string => {
const existing = depthContinuationIds.get(inputPath);
if (existing !== undefined) return existing;
const id = continuation(
depthContinuation(
maxDepth,
[...(reRootablePaths.get(inputPath) ?? [])].sort(),
),
);
depthContinuationIds.set(inputPath, id);
return id;
};
const gaps: ExtractionGap[] = [
...dependencyGaps,
...normalized.failed_extraction_stages
.filter(({ kind }) => kind === "extractor-failure")
.map(({ input_path, provider_id, handler }): ExtractionGap => ({
kind: "extractor-failure",
input_path,
...emptyGap,
handler,
provider_id,
dependencies: [],
continuation_id: continuation(
"The producer retained this extraction stage's failure in diagnostics. Review the stage report and correct the reported cause before rerunning extract_firmware on the same immutable input with a fresh output_directory; this report establishes no byte-region ownership.",
),
})),
...chunkGaps("unknown-chunk-handler", (chunk) => chunk.handler === null),
...chunkGaps("encrypted-chunk", (chunk) => chunk.encrypted === true),
...chunkGaps(
"extractor-failure",
(chunk) =>
chunk.extraction_failed === true &&
chunk.handler !== null &&
chunk.encrypted !== true &&
// The chunk's own dependency row already carries its recovery; a
// task-wide dependency names no region and replaces nothing.
!dependencyChunkKeys.has(
JSON.stringify([chunk.input_path, chunk.provider_id]),
),
),
...normalized.depth_limited_paths.map((input_path): ExtractionGap => ({
kind: "depth-limited",
input_path,
...emptyGap,
dependencies: [],
continuation_id: depthContinuationId(input_path),
})),
...unpublished.map((entry): ExtractionGap => ({
kind: "unpublished-entry",
input_path: `$output/${entry.relative_path}`,
...emptyGap,
relative_path: entry.relative_path,
dependencies: [],
continuation_id: continuation(GAP_CONTINUATIONS["unpublished-entry"]),
})),
];
gaps.sort((left, right) =>
gapSortKey(left) < gapSortKey(right)
? -1
: gapSortKey(left) > gapSortKey(right)
? 1
: 0,
);
return {
extraction_gaps: gaps,
extraction_continuations: [...instructions].map(([instruction, id]) => ({
id,
instruction,
})),
};
};
+499 -1
View File
@@ -42,7 +42,28 @@ const root = () => ({
subtasks: [],
});
describe("firmware producer boundaries", () => {
/** A chunk whose extraction failed on a missing dependency, when asked for it. */
const nestedDependencyChunk = (id: string, offset: number) => ({
__typename__: "ChunkReport",
id,
start_offset: offset,
end_offset: offset + 10,
size: 10,
handler_name: `handler-${id}`,
is_encrypted: false,
extraction_reports:
id === "gzip-id"
? [
{
__typename__: "ExtractorDependencyNotFoundReport",
severity: "ERROR",
dependencies: ["sasquatch"],
},
]
: [],
});
describe("firmware producer chunk observations", () => {
it("retains unknown chunks and their selected original-file range", () => {
const normalized = normalizeUnblobReport([root()], context);
expect(normalized.coverage).toBe("partial");
@@ -51,6 +72,412 @@ describe("firmware producer boundaries", () => {
range: { offset: 0, length: 20 },
root_file_range: { offset: 64, length: 20 },
});
expect(normalized.missing_dependencies).toEqual([]);
});
});
describe("firmware producer dependency ownership", () => {
it("derives missing extractor dependencies from chunk and multi-file reports", () => {
const record = root();
const normalized = normalizeUnblobReport(
[
{
...record,
reports: record.reports.filter(
(item) => item.__typename__ !== "UnknownChunkReport",
),
subtasks: [
{
path: `${context.outputRoot}/rootfs`,
depth: 1,
blob_id: "sq-id",
is_multi_file: false,
},
],
},
{
task: {
path: `${context.outputRoot}/rootfs`,
depth: 1,
blob_id: "sq-id",
is_multi_file: false,
},
reports: [
{
__typename__: "StatReport",
path: `${context.outputRoot}/rootfs`,
size: 0,
is_file: false,
is_dir: true,
is_link: false,
link_target: null,
},
{
__typename__: "MultiFileReport",
id: "sq-id",
handler_name: "squashfs",
name: "rootfs",
paths: [],
extraction_reports: [
{
__typename__: "ExtractorDependencyNotFoundReport",
severity: "ERROR",
dependencies: ["sasquatch"],
},
],
},
],
subtasks: [],
},
],
context,
);
expect(normalized.missing_dependencies).toEqual([]);
expect(normalized.failed_extraction_stages).toEqual([
{
input_path: "$output/rootfs",
provider_id: "sq-id",
handler: "squashfs",
dependencies: ["sasquatch"],
kind: "missing-extractor-dependency",
},
]);
expect(normalized.chunk_missing_dependencies).toEqual([]);
});
it("orders chunk dependencies totally across provider ids", () => {
const record = root();
const chunkWithDependency = (id: string, offset: number) => ({
__typename__: "ChunkReport",
id,
start_offset: offset,
end_offset: offset + 10,
size: 10,
handler_name: `handler-${id}`,
is_encrypted: false,
extraction_reports: [
{
__typename__: "ExtractorDependencyNotFoundReport",
severity: "ERROR",
dependencies: [`${id}-tool`],
},
],
});
const normalized = normalizeUnblobReport(
[
{
...record,
reports: [
...record.reports.filter(
(item) => item.__typename__ !== "UnknownChunkReport",
),
chunkWithDependency("b-id", 0),
chunkWithDependency("a-id", 10),
],
},
],
context,
);
expect(normalized.chunk_missing_dependencies).toEqual([
{
input_path: "$input",
provider_id: "a-id",
dependencies: ["a-id-tool"],
},
{
input_path: "$input",
provider_id: "b-id",
dependencies: ["b-id-tool"],
},
]);
});
it("keeps chunk-nested dependency reports bound to their owning chunk", () => {
const record = root();
const normalized = normalizeUnblobReport(
[
{
...record,
reports: [
...record.reports.filter(
(item) => item.__typename__ !== "UnknownChunkReport",
),
nestedDependencyChunk("gzip-id", 0),
nestedDependencyChunk("other-id", 10),
],
},
],
context,
);
expect(normalized.missing_dependencies).toEqual([]);
expect(normalized.chunk_missing_dependencies).toEqual([
{
input_path: "$input",
provider_id: "gzip-id",
dependencies: ["sasquatch"],
},
]);
});
});
describe("firmware dependency failures with unavailable identities", () => {
it.each([{ dependencies: [] }, { dependencies: ["", ""] }])(
"retains unnamed chunk dependency ownership ($dependencies)",
({ dependencies }) => {
const record = root();
const normalized = normalizeUnblobReport(
[
{
...record,
reports: [
...record.reports.filter(
(item) => item.__typename__ !== "UnknownChunkReport",
),
{
...nestedDependencyChunk("gzip-id", 0),
extraction_reports: [
{
__typename__: "ExtractorDependencyNotFoundReport",
severity: "ERROR",
dependencies,
},
],
},
],
},
],
context,
);
expect(normalized.missing_dependencies).toEqual([]);
expect(normalized.chunk_missing_dependencies).toEqual([
{
input_path: "$input",
provider_id: "gzip-id",
dependencies: [],
},
]);
},
);
it("retains the missing-dependency reason on an unnamed multi-file stage", () => {
const record = root();
const normalized = normalizeUnblobReport(
[
{
...record,
reports: [
...record.reports,
{
__typename__: "MultiFileReport",
id: "multi-id",
handler_name: "multipart-firmware",
name: "parts",
paths: [],
extraction_reports: [
{
__typename__: "ExtractorDependencyNotFoundReport",
severity: "ERROR",
dependencies: [],
},
],
},
],
},
],
context,
);
expect(normalized.failed_extraction_stages).toEqual([
{
input_path: "$input",
provider_id: "multi-id",
handler: "multipart-firmware",
dependencies: [],
kind: "missing-extractor-dependency",
},
]);
});
it("retains named and unnamed dependency reports on one multi-file stage", () => {
const record = root();
const normalized = normalizeUnblobReport(
[
{
...record,
reports: [
...record.reports,
{
__typename__: "MultiFileReport",
id: "multi-id",
handler_name: "multipart-firmware",
name: "parts",
paths: [],
extraction_reports: [
{
__typename__: "ExtractorDependencyNotFoundReport",
severity: "ERROR",
dependencies: ["sasquatch"],
},
{
__typename__: "ExtractorDependencyNotFoundReport",
severity: "ERROR",
dependencies: [],
},
],
},
],
},
],
context,
);
expect(normalized.failed_extraction_stages).toEqual([
{
input_path: "$input",
provider_id: "multi-id",
handler: "multipart-firmware",
dependencies: ["sasquatch"],
kind: "missing-extractor-dependency",
},
{
input_path: "$input",
provider_id: "multi-id",
handler: "multipart-firmware",
dependencies: [],
kind: "missing-extractor-dependency",
},
]);
});
});
describe("firmware producer chunk failures", () => {
it("marks chunks whose extraction reported problems", () => {
const record = root();
const failing = {
__typename__: "ChunkReport",
id: "zip-id",
start_offset: 0,
end_offset: 20,
size: 20,
handler_name: "zip",
is_encrypted: false,
extraction_reports: [
{ __typename__: "SomeProducerFailure", severity: "ERROR" },
],
};
const normalized = normalizeUnblobReport(
[
{
...record,
reports: [
...record.reports.filter(
(item) => item.__typename__ !== "UnknownChunkReport",
),
failing,
],
},
],
context,
);
expect(normalized.chunks).toHaveLength(1);
expect(normalized.chunks[0]).toMatchObject({
handler: "zip",
extraction_failed: true,
});
expect(normalized.coverage).toBe("partial");
});
});
describe("firmware task dependency failures", () => {
it.each([{ dependencies: [] }, { dependencies: ["", ""] }])(
"retains unnamed dependency failures ($dependencies)",
({ dependencies }) => {
const record = root();
const normalized = normalizeUnblobReport(
[
{
...record,
reports: [
...record.reports,
{
__typename__: "ExtractorDependencyNotFoundReport",
severity: "ERROR",
dependencies,
},
],
},
],
context,
);
expect(normalized.missing_dependencies).toEqual([
{ input_path: "$input", dependencies: [] },
]);
expect(normalized.diagnostics).toHaveLength(1);
},
);
it("retains named and unnamed task dependency reports together", () => {
const record = root();
const normalized = normalizeUnblobReport(
[
{
...record,
reports: [
...record.reports,
{
__typename__: "ExtractorDependencyNotFoundReport",
severity: "ERROR",
dependencies: ["sasquatch"],
},
{
__typename__: "ExtractorDependencyNotFoundReport",
severity: "ERROR",
dependencies: [],
},
],
},
],
context,
);
expect(normalized.missing_dependencies).toEqual([
{ input_path: "$input", dependencies: ["sasquatch"] },
{ input_path: "$input", dependencies: [] },
]);
});
});
describe("firmware chunk dependency failures", () => {
it("retains named and unnamed dependency reports on one chunk", () => {
const record = root();
const normalized = normalizeUnblobReport(
[
{
...record,
reports: [
...record.reports.filter(
(item) => item.__typename__ !== "UnknownChunkReport",
),
{
...nestedDependencyChunk("gzip-id", 0),
extraction_reports: [
{
__typename__: "ExtractorDependencyNotFoundReport",
severity: "ERROR",
dependencies: ["sasquatch"],
},
{
__typename__: "ExtractorDependencyNotFoundReport",
severity: "ERROR",
dependencies: [],
},
],
},
],
},
],
context,
);
expect(normalized.chunk_missing_dependencies).toEqual([
{
input_path: "$input",
provider_id: "gzip-id",
dependencies: ["sasquatch"],
},
{ input_path: "$input", provider_id: "gzip-id", dependencies: [] },
]);
});
it("preserves unfamiliar report types as explicit diagnostics", () => {
const record = root();
@@ -237,6 +664,77 @@ describe("Unblob task derivation identities", () => {
report: multiFileReport,
},
]);
expect(normalized.failed_extraction_stages).toEqual([
{
input_path: "$input",
provider_id: "multi-id",
handler: "multipart-firmware",
dependencies: ["sasquatch"],
kind: "missing-extractor-dependency",
},
]);
});
});
describe("failed multi-file extraction stages", () => {
it("records a failed multi-file stage for a recovery row without chunks", () => {
const outputPath = `${context.outputRoot}/multi`;
const childTask = {
path: outputPath,
depth: 1,
blob_id: "multi-id",
is_multi_file: true,
};
const stageFailure = {
__typename__: "ExtractorExecutionErrorReport",
severity: "ERROR",
handler: "multipart-firmware",
};
const normalized = normalizeUnblobReport(
[
{
...root(),
reports: root().reports.filter(
(item) => item.__typename__ !== "UnknownChunkReport",
),
subtasks: [childTask],
},
{
task: childTask,
reports: [
{
__typename__: "StatReport",
path: outputPath,
size: 0,
is_dir: true,
is_file: false,
is_link: false,
link_target: null,
},
{
__typename__: "MultiFileReport",
id: "multi-id",
handler_name: "multipart-firmware",
name: "rootfs",
paths: [`${outputPath}/part-a`],
extraction_reports: [stageFailure],
},
],
subtasks: [],
},
],
context,
);
expect(normalized.failed_extraction_stages).toEqual([
{
input_path: "$output/multi",
provider_id: "multi-id",
handler: "multipart-firmware",
dependencies: [],
kind: "extractor-failure",
},
]);
expect(normalized.missing_dependencies).toEqual([]);
});
});
+173 -6
View File
@@ -2,6 +2,7 @@ import { z } from "zod";
import { AnalysisOutputError } from "../domain/analysisErrorCore.js";
import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js";
import { digestSchema } from "../domain/digests.js";
import { uniqueSorted } from "../domain/canonicalOrdering.js";
import { firmwareLogicalPath } from "./FirmwareFiles.js";
const integer = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER);
@@ -51,6 +52,15 @@ const multiFile = z.object({
extraction_reports: z.array(jsonValueSchema),
});
const hashes = z.object({ sha256: digestSchema });
/**
* The provider's structured missing-dependency report. Non-strict on purpose:
* unfamiliar extra fields stay in retained diagnostics, only the dependency
* names are derived here.
*/
const missingDependency = z.object({
__typename__: z.literal("ExtractorDependencyNotFoundReport"),
dependencies: z.array(z.string()),
});
const region = z.object({
id: z.string(),
start_offset: integer,
@@ -79,7 +89,9 @@ const dependencyFailure = z.object({
/** Collect the named extractors from any dependency report nested in one value. */
const missingDependencies = (value: unknown): string[] => {
const found: string[] = [];
// Producer-controlled name volume must not make deduplication quadratic;
// a Set keeps collection linear and uniqueSorted restores canonical order.
const found = new Set<string>();
const visit = (node: unknown): void => {
if (Array.isArray(node)) {
for (const item of node) visit(item);
@@ -88,14 +100,13 @@ const missingDependencies = (value: unknown): string[] => {
if (typeof node !== "object" || node === null) return;
const failure = dependencyFailure.safeParse(node);
if (failure.success) {
for (const dependency of failure.data.dependencies)
if (!found.includes(dependency)) found.push(dependency);
for (const dependency of failure.data.dependencies) found.add(dependency);
return;
}
for (const child of Object.values(node)) visit(child);
};
visit(value);
return found;
return [...found];
};
/**
@@ -194,6 +205,13 @@ export const normalizeUnblobReport = (
>();
const sizes = new Map<string, number>();
const depths: string[] = [];
const failedExtractionStages: {
input_path: string;
provider_id: string | null;
handler: string | null;
dependencies: readonly string[];
kind: "missing-extractor-dependency" | "extractor-failure";
}[] = [];
const diagnostics: JsonValue[] = [];
const blobOrigins = new Map<string, BlobOrigin>();
const addBlobOrigin = (
@@ -216,8 +234,74 @@ export const normalizeUnblobReport = (
handler: string | null;
encrypted: boolean | null;
root_file_range: { offset: number; length: number } | null;
extraction_failed: boolean;
}[] = [];
const seen = new Set<string>();
const taskFailures = new Set<string>();
type DependencyFacts = { readonly names: Set<string>; unnamed: boolean };
const missingDependencies = new Map<string, DependencyFacts>();
/** Dependencies reported by one chunk's extraction, keyed with its chunk id. */
const chunkMissingDependencies = new Map<string, DependencyFacts>();
const dependencyNames = (
reports: readonly unknown[],
): DependencyFacts | null => {
const names = new Set<string>();
let unnamed = false;
let reported = false;
for (const item of reports) {
const parsed = missingDependency.safeParse(item);
if (!parsed.success) continue;
reported = true;
if (
parsed.data.dependencies.length === 0 ||
parsed.data.dependencies.includes("")
)
unnamed = true;
for (const name of parsed.data.dependencies)
if (name !== "") names.add(name);
}
return reported ? { names, unnamed } : null;
};
const dependencyRows = (
facts: DependencyFacts,
): readonly { dependencies: readonly string[] }[] => [
...(facts.names.size > 0
? [{ dependencies: uniqueSorted([...facts.names]) }]
: []),
...(facts.unnamed ? [{ dependencies: [] }] : []),
];
const noteMissingDependencies = (
path: string,
reports: readonly unknown[],
): void => {
const names = dependencyNames(reports);
// null means no accepted failure; [] means a definite dependency failure
// whose executable identity the producer left unavailable.
if (names === null) return;
const known = missingDependencies.get(path) ?? {
names: new Set<string>(),
unnamed: false,
};
for (const name of names.names) known.names.add(name);
known.unnamed ||= names.unnamed;
missingDependencies.set(path, known);
};
const noteChunkMissingDependencies = (
path: string,
chunkId: string,
reports: readonly unknown[],
): void => {
const names = dependencyNames(reports);
if (names === null) return;
const key = JSON.stringify([path, chunkId]);
const known = chunkMissingDependencies.get(key) ?? {
names: new Set<string>(),
unnamed: false,
};
for (const name of names.names) known.names.add(name);
known.unnamed ||= names.unnamed;
chunkMissingDependencies.set(key, known);
};
for (const record of records) {
const path = logical(record.task.path);
if (seen.has(path))
@@ -273,6 +357,20 @@ export const normalizeUnblobReport = (
diagnostics.push(
jsonValueSchema.parse({ input_path: path, report: item }),
);
noteMissingDependencies(path, [item]);
if (
item.__typename__ === "ExtractorExecutionErrorReport" &&
!taskFailures.has(path)
) {
taskFailures.add(path);
failedExtractionStages.push({
input_path: path,
provider_id: null,
handler: null,
dependencies: [],
kind: "extractor-failure",
});
}
if (
item.__typename__ === "ChunkReport" ||
item.__typename__ === "UnknownChunkReport"
@@ -297,8 +395,10 @@ export const normalizeUnblobReport = (
})
.parse(item)
: null;
const extractionFailed =
detail !== null && detail.extraction_reports.length > 0;
if (detail !== null)
for (const problem of detail.extraction_reports)
for (const problem of detail.extraction_reports) {
diagnostics.push(
extractionDiagnostic(
path,
@@ -306,6 +406,9 @@ export const normalizeUnblobReport = (
jsonValueSchema.parse(problem),
),
);
// Nested under one chunk: keep the owning chunk, not the whole task.
noteChunkMissingDependencies(path, parsed.id, [problem]);
}
addBlobOrigin(path, parsed.id, {
handler: detail?.handler_name ?? null,
range: { offset: parsed.start_offset, length: parsed.size },
@@ -316,6 +419,7 @@ export const normalizeUnblobReport = (
range: { offset: parsed.start_offset, length: parsed.size },
handler: detail?.handler_name ?? null,
encrypted: detail?.is_encrypted ?? null,
extraction_failed: extractionFailed,
root_file_range:
path === "$input"
? {
@@ -331,7 +435,7 @@ export const normalizeUnblobReport = (
handler: parsed.handler_name,
range: null,
});
if (parsed.extraction_reports.length > 0)
if (parsed.extraction_reports.length > 0) {
diagnostics.push(
extractionDiagnostic(
path,
@@ -339,6 +443,28 @@ export const normalizeUnblobReport = (
jsonValueSchema.parse(item),
),
);
// The report owns its failure and dependencies. A produced subtask
// is a derivation, not a new owner for the parent's report.
const dependencies = dependencyNames(parsed.extraction_reports);
const rows =
dependencies === null
? [{ dependencies: [] as readonly string[] }]
: dependencyRows(dependencies);
for (const row of rows)
failedExtractionStages.push({
input_path: path,
provider_id: parsed.id,
handler: parsed.handler_name,
dependencies:
dependencies === null
? []
: uniqueSorted([...row.dependencies]),
kind:
dependencies === null
? "extractor-failure"
: "missing-extractor-dependency",
});
}
}
}
}
@@ -403,7 +529,48 @@ export const normalizeUnblobReport = (
chunks,
derivations,
depth_limited_paths: depths.sort(),
failed_extraction_stages: failedExtractionStages.sort((left, right) =>
`${left.input_path}\u0000${left.provider_id ?? ""}` <
`${right.input_path}\u0000${right.provider_id ?? ""}`
? -1
: `${left.input_path}\u0000${left.provider_id ?? ""}` >
`${right.input_path}\u0000${right.provider_id ?? ""}`
? 1
: 0,
),
diagnostics,
missing_dependencies: [...missingDependencies.entries()]
.flatMap(([input_path, facts]) =>
dependencyRows(facts).map(({ dependencies }) => ({
input_path,
dependencies,
})),
)
.sort((left, right) =>
left.input_path < right.input_path
? -1
: left.input_path > right.input_path
? 1
: 0,
),
chunk_missing_dependencies: [...chunkMissingDependencies.entries()]
.flatMap(([key, facts]) => {
const [input_path, provider_id] = JSON.parse(key) as [string, string];
return dependencyRows(facts).map(({ dependencies }) => ({
input_path,
provider_id,
dependencies,
}));
})
.sort((left, right) =>
`${left.input_path}\u0000${left.provider_id}` <
`${right.input_path}\u0000${right.provider_id}`
? -1
: `${left.input_path}\u0000${left.provider_id}` >
`${right.input_path}\u0000${right.provider_id}`
? 1
: 0,
),
coverage:
depths.length > 0 ||
diagnostics.length > 0 ||
+1
View File
@@ -162,6 +162,7 @@ export class SigrokRecordingProvider implements SignalRecordingPort {
this.#resources.retain(root, undefined);
const snapshotPath = join(root.path, "recording");
const execute = async (args: readonly string[]) => {
capturedOutput = undefined;
const output = await runOwnedCommand(
{
command: limiter,
+25
View File
@@ -7,6 +7,7 @@ import {
import {
AnalysisAccessDeniedError,
AnalysisArtifactChangedError,
AnalysisCapabilityUnavailableError,
AnalysisCancelledError,
AnalysisInputError,
AnalysisOutputError,
@@ -29,6 +30,15 @@ export const capturedSqliteOutput = (
truncated: snapshot.diagnosticTruncated === true,
});
/** The file-set member a filesystem error names, if it is the database, its WAL or its journal. */
const busyMember = (cause: Error, path: string): string => {
const failed = "path" in cause ? cause.path : undefined;
return typeof failed === "string" &&
[path, `${path}-wal`, `${path}-journal`].includes(failed)
? failed
: path;
};
/** Preserve artifact, caller, process, resource and output failure meanings. */
export const sqliteDatabaseFailure = (
cause: unknown,
@@ -85,6 +95,21 @@ export const sqliteDatabaseFailure = (
return new AnalysisAccessDeniedError(operation, path, cause.code, {
cause,
});
// Windows reports a sharing violation as EBUSY: another process holds the file without read
// sharing. The unchanged request succeeds once that process releases it. Name the member of the
// file set that was busy (the database, its WAL or its journal) when the error identifies it.
if (cause.code === "EBUSY")
return new AnalysisCapabilityUnavailableError(
SQLITE_PROVIDER_IDENTITY.id,
operation,
`Selected database file set is in use by another process (EBUSY): ${busyMember(cause, path)}`,
{
cause,
retryable: true,
userMessage:
"The selected SQLite database is held open by another process without read sharing. Close that application or inspect a copy, then retry.",
},
);
if (cause.code === "ENOENT" || cause.code === "ENOTDIR")
return new AnalysisInputError(operation, { cause }, [
{
+63
View File
@@ -7,11 +7,74 @@ import {
createGoBinaryByteStringFixture,
createGoBinaryDiagnosticFixtures,
createGoBinaryFixture,
createGoSplitElfFixture,
createGoOverlappingElfFixture,
createGoBinaryPathFailureFixture,
GO_MODULE_TEXT,
} from "../fixtures/go/image.js";
import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";
import { cliTest } from "../support/cli/cliFixture.js";
import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js";
import { parseMcpToolError } from "../fixtures/mcpToolError.js";
cliTest(
"bounds overlapping ELF scans through CLI and keeps MCP responsive",
async ({ cli }) => {
const root = await createTestTempDirectory("rea-go-overlap-");
const path = join(root, "overlapping.elf");
await writeFile(path, createGoOverlappingElfFixture());
const result = await cli.run({
arguments: ["inspect-go-binary", path, "--json"],
});
expect(result.exitCode).toBe(1);
const expected = {
code: "resource_constraint",
details: {
operation: "inspect_go_binary",
resource: "cpu",
reported_limits: {
boundary: "search",
maximum_bytes: 256 * 1024 * 1024,
},
},
};
expect(result.json).toMatchObject(expected);
const { client, call } = await connectLocalToolsMcp();
const response = await call("inspect_go_binary", { path });
expect(parseMcpToolError(response).error).toMatchObject(expected);
await client.ping();
},
30000,
);
cliTest(
"retains later-load sectionless ELF evidence through public CLI and stdio MCP",
async ({ cli }) => {
const root = await createTestTempDirectory("rea-go-split-elf-");
const { client, call } = await connectLocalToolsMcp();
const infos = [];
for (const sectionless of [false, true]) {
const fixture = createGoSplitElfFixture({ sectionless });
const path = join(root, sectionless ? "sectionless.elf" : "original.elf");
await writeFile(path, fixture.bytes);
const result = await cli.run({
arguments: ["inspect-go-binary", path, "--json"],
});
expect(result.exitCode, result.stderr).toBe(0);
const evidence = toolContract("inspect_go_binary").outputSchema.parse(
result.json,
);
expect(evidence.normalized_result.build_info).not.toBeNull();
infos.push(evidence.normalized_result.build_info);
const response = await call("inspect_go_binary", { path });
expect(response.isError, JSON.stringify(response)).not.toBe(true);
expect(response.structuredContent).toEqual(evidence);
}
expect(infos[1]).toEqual(infos[0]);
await client.ping();
},
30000,
);
for (const expectedErrno of ["ELOOP", "ENAMETOOLONG"] as const) {
cliTest.skipIf(expectedErrno === "ELOOP" && process.platform === "win32")(
+84 -24
View File
@@ -4,7 +4,10 @@ import {
analyzeJadxRequest,
type JadxToolPort,
} from "../../../src/android/JadxAnalysis.js";
import { androidInputSchemas } from "../../../src/domain/android/androidAnalysis.js";
import {
androidInputSchemas,
androidResultSchemas,
} from "../../../src/domain/android/androidAnalysis.js";
const summary = {
full_name: "fixture.Target",
@@ -41,6 +44,43 @@ const engine = {
heap_limit_mib: null,
};
it("requires explicit direct invoke coverage state in result evidence", () => {
const result = {
engine,
class_name: "fixture.Target",
method_name: null,
overload_index: null,
reported_signature: null,
dex_descriptor: null,
total: 1,
references: [
{
kind: "method",
name: "caller",
reported_full_name: "fixture.Other.caller",
containing_class: "fixture.Other",
top_class: "fixture.Other",
source_line: null,
dex_offset: null,
direct_invokes: null,
},
],
coverage: "complete",
reference_unit: "using_node",
};
expect(
androidResultSchemas.trace_android_references.safeParse(result).success,
).toBe(false);
expect(
androidResultSchemas.trace_android_references.safeParse({
...result,
references: [
{ ...result.references[0], direct_invokes_exhaustive: null },
],
}).success,
).toBe(true);
});
it("publishes exact DEX prototypes instead of discarding bridge metadata", async () => {
const tools: JadxToolPort = {
json: () => Promise.resolve(summary),
@@ -246,7 +286,9 @@ it.each([
async (fields, expected) => {
const result = await traceWithReference(fields as Record<string, unknown>);
expect(result).toMatchObject({
references: [{ direct_invokes: expected }],
references: [
{ direct_invokes: expected, direct_invokes_exhaustive: null },
],
});
},
);
@@ -279,26 +321,44 @@ it("preserves bounded instruction coverage separately from using-node coverage",
});
});
it("rejects malformed method descriptors at the Android adapter boundary", async () => {
const invalidSummary = {
...summary,
methods: summary.methods.map((method) => ({
...method,
dex_descriptor: "not-a-dex-descriptor",
})),
};
await expect(
analyzeJadxRequest(
{
json: () => Promise.resolve(invalidSummary),
text: () => Promise.reject(new Error("unexpected text call")),
it.each([
"not-a-dex-descriptor",
"(Lbad(name;)V",
"(Lbad)name;)V",
"()Lbad(name;",
"()Lbad)name;",
])(
"rejects malformed DEX prototype %s at the adapter boundary",
async (descriptor) => {
const invalidSummary = {
...summary,
methods: summary.methods.map((method) => ({
...method,
dex_descriptor: descriptor,
})),
};
await expect(
analyzeJadxRequest(
{
json: () => Promise.resolve(invalidSummary),
text: () => Promise.reject(new Error("unexpected text call")),
},
{
operation: "inspect_android_class",
input: { path: "/fixture.apk", class_name: "fixture.Target" },
},
{} as never,
engine,
),
).rejects.toMatchObject({
_tag: "AnalysisOutputError",
cause: {
issues: expect.arrayContaining([
expect.objectContaining({
path: ["methods", 0, "dex_descriptor"],
}),
]),
},
{
operation: "inspect_android_class",
input: { path: "/fixture.apk", class_name: "fixture.Target" },
},
{} as never,
engine,
),
).rejects.toThrow();
});
});
},
);
@@ -15,6 +15,25 @@ import { ToolResultDelivery } from "../../../src/server/toolResult.js";
import { SafeOutputTree } from "../../../src/artifacts/SafeOutputTree.js";
import { SafeOutputTreeCreationFailure } from "../../../src/artifacts/SafeOutputTreeCreationFailure.js";
// Assert recovery behavior through the public reference table, while keeping
// row identity expectations independent of the instruction's digest.
const gapsWithContinuations = (
result: ReturnType<typeof firmwareResultSchemas.extract_firmware.parse>,
) => {
const instructions = new Map(
result.extraction_continuations.map(({ id, instruction }) => [
id,
instruction,
]),
);
return result.extraction_gaps.map(({ continuation_id, ...gap }) => {
const instruction = instructions.get(continuation_id);
if (instruction === undefined)
throw new Error("Missing recovery instruction");
return { ...gap, continuation: instruction };
});
};
const directoryCloseFailure = vi.hoisted(() => ({
mode: "off" as "off" | "transient" | "persistent",
attempts: 0,
@@ -270,6 +289,247 @@ it("retains the complete firmware result when published output cleanup stays unc
);
});
it("shares recovery prose across distinct unknown chunk identities", async () => {
const fixture = await firmwareFixture("many-unknown");
const result = await fixture.service.execute("extract_firmware", {
path: fixture.path,
output_directory: fixture.output,
});
if (!result.ok) throw result.error;
const normalized = firmwareResultSchemas.extract_firmware.parse(
result.value.normalized_result,
);
expect(normalized.extraction_gaps).toHaveLength(1000);
expect(
new Set(normalized.extraction_gaps.map(({ provider_id }) => provider_id))
.size,
).toBe(1000);
expect(normalized.extraction_continuations).toHaveLength(1);
const [instruction] = normalized.extraction_continuations;
expect(instruction?.instruction).toContain("inspect_firmware_regions");
expect(
normalized.extraction_gaps.every(
(gap) => gap.continuation_id === instruction?.id,
),
).toBe(true);
});
it("retains a task dependency failure when the producer names no executable", async () => {
const fixture = await firmwareFixture("unnamed-dependency");
const result = await fixture.service.execute("extract_firmware", {
path: fixture.path,
output_directory: fixture.output,
});
if (!result.ok) throw result.error;
const normalized = firmwareResultSchemas.extract_firmware.parse(
result.value.normalized_result,
);
expect(gapsWithContinuations(normalized)).toEqual([
{
kind: "missing-extractor-dependency",
input_path: "$input",
relative_path: null,
handler: null,
provider_id: null,
range: null,
root_file_range: null,
dependencies: [],
continuation: expect.stringContaining("identity is unavailable"),
},
]);
});
it("preserves an unnamed dependency failure on its owning chunk", async () => {
const fixture = await firmwareFixture("unnamed-chunk-dependency");
const result = await fixture.service.execute("extract_firmware", {
path: fixture.path,
output_directory: fixture.output,
});
if (!result.ok) throw result.error;
const normalized = firmwareResultSchemas.extract_firmware.parse(
result.value.normalized_result,
);
expect(gapsWithContinuations(normalized)).toEqual([
{
kind: "missing-extractor-dependency",
input_path: "$input",
relative_path: null,
handler: "gzip",
provider_id: "gzip-id",
range: { offset: 0, length: 18 },
root_file_range: { offset: 0, length: 18 },
dependencies: [],
continuation: expect.stringContaining("identity is unavailable"),
},
]);
});
it("routes a nested unknown chunk through its published child", async () => {
const fixture = await firmwareFixture("nested-unknown");
const result = await fixture.service.execute("extract_firmware", {
path: fixture.path,
output_directory: fixture.output,
});
if (!result.ok) throw result.error;
const normalized = firmwareResultSchemas.extract_firmware.parse(
result.value.normalized_result,
);
expect(gapsWithContinuations(normalized)).toEqual(
expect.arrayContaining([
expect.objectContaining({
kind: "unknown-chunk-handler",
input_path: "$output/rootfs/config",
provider_id: "unknown-id",
range: { offset: 0, length: 14 },
root_file_range: null,
continuation: expect.stringContaining(
"published child named by this gap's input_path",
),
}),
]),
);
expect(gapsWithContinuations(normalized)[0]?.continuation).toContain(
"only if Binwalk is configured with REA_BINWALK_COMMAND",
);
expect(gapsWithContinuations(normalized)[0]?.continuation).toContain(
"unavailable without configured Binwalk",
);
});
it("retains every eligible child in a maximum-depth continuation", async () => {
const fixture = await firmwareFixture("depth-reentry");
const result = await fixture.service.execute("extract_firmware", {
path: fixture.path,
output_directory: fixture.output,
max_depth: 10,
});
if (!result.ok) throw result.error;
const normalized = firmwareResultSchemas.extract_firmware.parse(
result.value.normalized_result,
);
const gap = gapsWithContinuations(normalized).find(
({ kind, input_path }) =>
kind === "depth-limited" && input_path === "$output/rootfs/config",
);
expect(gap?.continuation).toContain(
JSON.stringify(`${fixture.output}/rootfs/config/inner.gz`),
);
expect(gap?.continuation).toContain(
JSON.stringify(`${fixture.output}/rootfs/config/second.gz`),
);
});
it("withholds the nested rerun advice when the published child exceeds the input limit", async () => {
const fixture = await firmwareFixture("nested-unknown-oversized");
const result = await fixture.service.execute("extract_firmware", {
path: fixture.path,
output_directory: fixture.output,
});
if (!result.ok) throw result.error;
const normalized = firmwareResultSchemas.extract_firmware.parse(
result.value.normalized_result,
);
expect(gapsWithContinuations(normalized)).toEqual([
{
kind: "unknown-chunk-handler",
input_path: "$output/rootfs/config",
relative_path: null,
handler: null,
provider_id: "unknown-id",
range: { offset: 0, length: 14 },
root_file_range: null,
dependencies: [],
continuation: expect.stringContaining(
"cannot accept it as a fresh input",
),
},
]);
});
it.each([
["depth", 10, "rerun extract_firmware"],
["depth-orphan", 10, "cannot be restarted"],
] as const)(
"bounds the maximum-depth continuation by a usable published child (%s)",
async (mode, maxDepth, expected) => {
const fixture = await firmwareFixture(mode);
const result = await fixture.service.execute("extract_firmware", {
path: fixture.path,
output_directory: fixture.output,
max_depth: maxDepth,
});
if (!result.ok) throw result.error;
const normalized = firmwareResultSchemas.extract_firmware.parse(
result.value.normalized_result,
);
expect(normalized.depth_limited_paths.length).toBeGreaterThan(0);
expect(
gapsWithContinuations(normalized).filter(
(gap) => gap.kind === "depth-limited",
),
).toEqual(
expect.arrayContaining([
expect.objectContaining({
kind: "depth-limited",
continuation: expect.stringContaining(expected),
}),
]),
);
},
);
it("reports a failed multi-file stage as an extractor-failure gap", async () => {
const fixture = await firmwareFixture("multi-file-failure");
const result = await fixture.service.execute("extract_firmware", {
path: fixture.path,
output_directory: fixture.output,
});
if (!result.ok) throw result.error;
const normalized = firmwareResultSchemas.extract_firmware.parse(
result.value.normalized_result,
);
expect(gapsWithContinuations(normalized)).toEqual(
expect.arrayContaining([
expect.objectContaining({
kind: "extractor-failure",
input_path: "$input",
handler: "tar",
provider_id: "mf-id",
range: null,
root_file_range: null,
}),
]),
);
});
it("carries the owning multi-file stage on a task-level dependency gap", async () => {
const fixture = await firmwareFixture("multi-file-dependency");
const result = await fixture.service.execute("extract_firmware", {
path: fixture.path,
output_directory: fixture.output,
});
if (!result.ok) throw result.error;
const normalized = firmwareResultSchemas.extract_firmware.parse(
result.value.normalized_result,
);
expect(gapsWithContinuations(normalized)).toEqual([
{
kind: "missing-extractor-dependency",
input_path: "$input",
relative_path: null,
handler: "squashfs",
provider_id: "sq-id",
range: null,
root_file_range: null,
dependencies: ["sasquatch"],
continuation: expect.stringContaining("installs nothing"),
},
]);
expect(normalized.diagnostics).toEqual([
expect.objectContaining({ stage: "squashfs" }),
]);
});
const injectPublishedDirectoryCloseFailure = (persistent: boolean) => {
directoryCloseFailure.attempts = 0;
const commit = SafeOutputTree.prototype.commit;
@@ -304,6 +564,22 @@ it.each(["depth", "dependency", "link"])(
expect(normalized.files).toHaveLength(1);
if (mode === "depth")
expect(normalized.depth_limited_paths).toEqual(["$output/rootfs/config"]);
if (mode === "depth")
expect(gapsWithContinuations(normalized)).toEqual([
{
kind: "depth-limited",
input_path: "$output/rootfs/config",
relative_path: null,
handler: null,
provider_id: null,
range: null,
root_file_range: null,
dependencies: [],
continuation: expect.stringMatching(
/max_depth.*fresh output_directory/u,
),
},
]);
if (mode === "dependency")
expect(normalized.diagnostics).toEqual([
{
@@ -317,6 +593,20 @@ it.each(["depth", "dependency", "link"])(
},
},
]);
if (mode === "dependency")
expect(gapsWithContinuations(normalized)).toEqual([
{
kind: "missing-extractor-dependency",
input_path: "$input",
relative_path: null,
handler: "gzip",
provider_id: "gzip-id",
range: { offset: 0, length: 18 },
root_file_range: { offset: 0, length: 18 },
dependencies: ["sasquatch"],
continuation: expect.stringContaining("installs nothing"),
},
]);
if (mode === "link") {
expect(normalized.unpublished_entries).toEqual([
{
@@ -325,6 +615,19 @@ it.each(["depth", "dependency", "link"])(
link_target: "/etc/passwd",
},
]);
expect(gapsWithContinuations(normalized)).toEqual([
{
kind: "unpublished-entry",
input_path: "$output/escape",
relative_path: "escape",
handler: null,
provider_id: null,
range: null,
root_file_range: null,
dependencies: [],
continuation: expect.stringContaining("metadata"),
},
]);
await expect(access(`${fixture.output}/escape`)).rejects.toMatchObject({
code: "ENOENT",
});
@@ -55,6 +55,8 @@ else { if(readFileSync('recording', 'utf8') !== 'recording bytes') throw Error('
},
async (input) => {
calls++;
if (mode === "launch-failure" && !input.arguments.includes("--version"))
throw Object.assign(new Error("spawn E2BIG"), { code: "E2BIG" });
if (mode === "file-limit" && !input.arguments.includes("--version")) {
const stdout = { text: "", bytes: 0, observedBytes: 0 };
const stderr = {
@@ -243,6 +245,21 @@ it.each([
},
);
it.each(["decode", "discovery"] as const)(
"does not attribute the version probe's output to a %s command that fails to launch",
async (operation) => {
const f = await fixture("launch-failure");
const result =
operation === "decode"
? await f.service.decode({ path: f.path, decoders: "uart" })
: await f.service.inspectDecoders({});
if (result.ok) throw new Error("Expected launch failure");
expect(f.calls()).toBe(2);
expect(result.error.message).toContain("spawn E2BIG");
expect(result.error.capturedOutput).toBeUndefined();
},
);
it("retains a completed observation and cleanup ownership for retry", async () => {
const f = await fixture("ok", true);
const result = await f.service.decode({ path: f.path, decoders: "uart" });
@@ -0,0 +1,159 @@
import { spawn, type ChildProcess } from "node:child_process";
import { access } from "node:fs/promises";
import { expect, it, onTestFinished, vi } from "vitest";
import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js";
import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js";
import { SqliteDatabaseProvider } from "../../../src/sqlite/SqliteDatabaseProvider.js";
import { createSqliteDatabaseFixture } from "../../fixtures/sqlite/database.js";
import {
createTestWorkspace,
removeTestWorkspace,
} from "../../support/workspace/workspaceFixture.js";
const busy = vi.hoisted(() => ({ path: "" }));
// Node opens files with full sharing, so a test process cannot hold one without read sharing
// itself. This injects the error Windows returns for that sharing violation.
vi.mock("node:fs/promises", async (importOriginal) => {
const filesystem = await importOriginal<typeof import("node:fs/promises")>();
return {
...filesystem,
open: async (...args: Parameters<typeof filesystem.open>) => {
if (busy.path !== "" && args[0] === busy.path)
throw Object.assign(
new Error(`EBUSY: resource busy or locked, open '${busy.path}'`),
{ code: "EBUSY", errno: -4082, syscall: "open", path: busy.path },
);
return filesystem.open(...args);
},
};
});
// Private roots go under the test's own workspace, so the cleanup check sees only this
// provider's root and not those of SQLite tests running in parallel.
const createProvider = (parent: string, roots: string[]) =>
new SqliteDatabaseProvider(process.env, undefined, async () => {
const root = await PrivateRuntimeRoot.create({ parent });
roots.push(root.path);
return root;
});
const expectRetryableBusy = (
error: Parameters<typeof projectAnalysisError>[0],
path: string,
) =>
expect(projectAnalysisError(error)).toMatchObject({
code: "capability_unavailable",
category: "unavailable",
retryable: true,
message:
"The selected SQLite database is held open by another process without read sharing. Close that application or inspect a copy, then retry.",
details: {
provider_id: "sqlite",
operation: "inspect_sqlite_database",
reason: `Selected database file set is in use by another process (EBUSY): ${path}`,
},
});
const expectRootsRemoved = async (roots: string[]) => {
expect(roots).toHaveLength(1);
await expect(access(roots[0]!)).rejects.toMatchObject({ code: "ENOENT" });
};
const stopHolder = async (holder: ChildProcess) => {
// A holder that never started has no pid and emits only "error", never "exit".
if (
holder.pid === undefined ||
holder.exitCode !== null ||
holder.signalCode !== null
)
return;
const exited = new Promise<void>((resolve) =>
holder.once("exit", () => resolve()),
);
holder.kill();
await exited;
};
it("reports a selected database held by another process as retryable and unavailable", async () => {
const workspace = await createTestWorkspace("rea-sqlite-busy-");
const database = createSqliteDatabaseFixture(workspace.root);
database.close();
const roots: string[] = [];
const provider = createProvider(workspace.root, roots);
onTestFinished(async () => {
busy.path = "";
await provider.close();
await removeTestWorkspace(workspace.root);
});
busy.path = database.path;
const result = await provider.inspect({ path: database.path });
if (result.ok) throw new Error("Expected a busy database rejection");
expectRetryableBusy(result.error, database.path);
await expectRootsRemoved(roots);
});
it("names the WAL when another process holds that member of the file set", async () => {
const workspace = await createTestWorkspace("rea-sqlite-busy-wal-");
const database = createSqliteDatabaseFixture(workspace.root, true);
const roots: string[] = [];
const provider = createProvider(workspace.root, roots);
onTestFinished(async () => {
busy.path = "";
await provider.close();
database.close();
await removeTestWorkspace(workspace.root);
});
busy.path = database.walPath;
const result = await provider.inspect({ path: database.path });
if (result.ok) throw new Error("Expected a busy WAL rejection");
expectRetryableBusy(result.error, database.walPath);
await expectRootsRemoved(roots);
});
it.runIf(process.platform === "win32")(
"reports a database that another Windows process holds without sharing as retryable",
async () => {
const workspace = await createTestWorkspace("rea-sqlite-busy-native-");
const database = createSqliteDatabaseFixture(workspace.root);
database.close();
const roots: string[] = [];
const provider = createProvider(workspace.root, roots);
const literal = database.path.replaceAll("'", "''");
const holder = spawn(
"powershell.exe",
[
"-NoProfile",
"-Command",
`$f = [System.IO.File]::Open('${literal}', 'Open', 'ReadWrite', 'None'); Write-Output ready; Start-Sleep -Seconds 60; $f.Close()`,
],
{ stdio: ["ignore", "pipe", "ignore"], windowsHide: true },
);
// One ordered cleanup: the holder must release the file before the workspace is removed.
onTestFinished(async () => {
await stopHolder(holder);
await provider.close();
await removeTestWorkspace(workspace.root);
});
await new Promise<void>((resolve, reject) => {
holder.stdout?.once("data", () => resolve());
holder.once("error", (error) =>
reject(new Error("Holder did not start", { cause: error })),
);
holder.once("exit", (code) =>
reject(
new Error(`Holder exited before opening the file (${String(code)})`),
),
);
});
const result = await provider.inspect({ path: database.path });
if (result.ok) throw new Error("Expected a busy database rejection");
expectRetryableBusy(result.error, database.path);
await expectRootsRemoved(roots);
},
20_000,
);
@@ -1,7 +1,7 @@
import { Cli } from "incur";
import { access, writeFile } from "node:fs/promises";
import type { FileHandle } from "node:fs/promises";
import { join } from "node:path";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { expect, it, onTestFinished, vi } from "vitest";
@@ -274,7 +274,7 @@ it("retains reply-read cancellation and retries the failed reply descriptor clos
{ signal: controller.signal },
);
if (result.ok) throw new Error("Expected cancelled reply read");
expect(projectAnalysisError(result.error)).toMatchObject({
const cancelled = {
code: "cleanup_incomplete",
details: {
execution_failure: "cancelled",
@@ -285,7 +285,20 @@ it("retains reply-read cancellation and retries the failed reply descriptor clos
truncated: false,
},
},
});
};
// Windows cannot remove a private root while the reply descriptor is open;
// the retained cancellation is then reported as the root cleanup's cause.
expect(projectAnalysisError(result.error)).toMatchObject(
process.platform === "win32"
? {
code: "cleanup_incomplete",
details: {
resources: [dirname(descriptorCleanup.path)],
diagnostics: { previous_error: cancelled },
},
}
: cancelled,
);
const handle = descriptorCleanup.handles[0];
if (handle === undefined) throw new Error("Expected owned reply descriptor");
expect(handle.fd).toBeGreaterThanOrEqual(0);
@@ -626,27 +626,32 @@ nativeTest(
const controller = new AbortController();
let ownedRoot = "";
let pid: number | undefined;
const provider = new SqliteDatabaseProvider({}, async (spawn) => {
ownedRoot = spawn.cwd ?? "";
const requestPath = spawn.arguments[2];
if (requestPath === undefined)
throw new Error("SQLite worker arguments missing");
const launched = await spawnOwnedProviderProcess({
...spawn,
arguments: [
"--input-type=module",
"-e",
`await import(${JSON.stringify(workerUrl.href)}); process.stdout.write('ready\\n'); setInterval(() => {}, 1000);`,
"sqlite-worker-test",
requestPath,
],
});
pid = launched.process.pid;
await waitForProviderProcessReady(launched.process);
await access(`${ownedRoot}/reply.json`);
setImmediate(() => controller.abort());
return launched;
});
const provider = new SqliteDatabaseProvider(
process.platform === "win32" && process.env.SystemRoot !== undefined
? { SystemRoot: process.env.SystemRoot }
: {},
async (spawn) => {
ownedRoot = spawn.cwd ?? "";
const requestPath = spawn.arguments[2];
if (requestPath === undefined)
throw new Error("SQLite worker arguments missing");
const launched = await spawnOwnedProviderProcess({
...spawn,
arguments: [
"--input-type=module",
"-e",
`await import(${JSON.stringify(workerUrl.href)}); process.stdout.write('ready\\n'); setInterval(() => {}, 1000);`,
"sqlite-worker-test",
requestPath,
],
});
pid = launched.process.pid;
await waitForProviderProcessReady(launched.process);
await access(`${ownedRoot}/reply.json`);
setImmediate(() => controller.abort());
return launched;
},
);
const result = await provider.inspect(
{ path: database.path, table: "records" },
{ signal: controller.signal },
+274 -15
View File
@@ -1,5 +1,12 @@
import { createHash } from "node:crypto";
import { chmod, mkdir, readFile, writeFile, symlink } from "node:fs/promises";
import {
chmod,
mkdir,
readFile,
truncate,
writeFile,
symlink,
} from "node:fs/promises";
import { join } from "node:path";
const args = process.argv.slice(2);
const mode = process.env.REA_FIRMWARE_FIXTURE_MODE;
@@ -61,18 +68,24 @@ if (args.includes("--version")) {
: join(output, "rootfs", "config");
await mkdir(join(output, "rootfs"));
const content = Buffer.from("firmware=true\n");
if (mode !== "missing-file")
if (
mode !== "missing-file" &&
mode !== "depth-reentry" &&
mode !== "depth-orphan"
)
await writeFile(child, mode === "budget" ? Buffer.alloc(10000) : content);
if (mode === "link") await symlink("/etc/passwd", join(output, "escape"));
const childHash = createHash("sha256").update(content).digest("hex");
const depth =
mode === "depth" ? Number(args[args.indexOf("--depth") + 1]) : 1;
const stat = (path, size) => ({
mode === "depth" || mode === "depth-reentry" || mode === "depth-orphan"
? Number(args[args.indexOf("--depth") + 1])
: 1;
const stat = (path, size, isDir = false) => ({
__typename__: "StatReport",
path,
size,
is_file: true,
is_dir: false,
is_file: !isDir,
is_dir: isDir,
is_link: false,
link_target: null,
});
@@ -84,6 +97,185 @@ if (args.includes("--version")) {
});
const hash = (value) => ({ __typename__: "HashReport", sha256: value });
const childTask = task(child, depth, "gzip-id");
if (mode === "nested-unknown") {
const nestedUnknownRecords = [
{
task: childTask,
reports: [
stat(child, content.length),
hash(childHash),
{
__typename__: "UnknownChunkReport",
id: "unknown-id",
start_offset: 0,
end_offset: content.length,
size: content.length,
},
],
subtasks: [],
},
{
task: task(input, 0, ""),
reports: [
stat(input, bytes.length),
hash(sha256),
{
__typename__: "ChunkReport",
id: "gzip-id",
start_offset: 0,
end_offset: bytes.length,
size: bytes.length,
handler_name: "gzip",
is_encrypted: false,
extraction_reports: [],
},
],
subtasks: [childTask],
},
];
await writeFile(reportPath, JSON.stringify(nestedUnknownRecords));
}
if (mode === "nested-unknown-oversized") {
// The child exceeds the firmware input limit, so a fresh run cannot
// accept it even though the unknown chunk's range is small.
const oversized = 128 * 1024 * 1024 + 1;
await truncate(child, oversized);
const oversizedHash = createHash("sha256")
.update(await readFile(child))
.digest("hex");
const nestedUnknownRecords = [
{
task: childTask,
reports: [
stat(child, oversized),
hash(oversizedHash),
{
__typename__: "UnknownChunkReport",
id: "unknown-id",
start_offset: 0,
end_offset: content.length,
size: content.length,
},
],
subtasks: [],
},
{
task: task(input, 0, ""),
reports: [
stat(input, bytes.length),
hash(sha256),
{
__typename__: "ChunkReport",
id: "gzip-id",
start_offset: 0,
end_offset: bytes.length,
size: bytes.length,
handler_name: "gzip",
is_encrypted: false,
extraction_reports: [],
},
],
subtasks: [childTask],
},
];
await writeFile(reportPath, JSON.stringify(nestedUnknownRecords));
}
if (mode === "depth-orphan") {
// A depth-limited directory with nothing published under it cannot
// re-root a fresh run.
const mid = join(output, "rootfs", "config");
await mkdir(mid, { recursive: true });
const midTask = task(mid, depth, "gzip-id");
const orphanRecords = [
{
task: midTask,
reports: [stat(mid, 0, true)],
subtasks: [],
},
{
task: task(input, 0, ""),
reports: [
stat(input, bytes.length),
hash(sha256),
{
__typename__: "ChunkReport",
id: "gzip-id",
start_offset: 0,
end_offset: bytes.length,
size: bytes.length,
handler_name: "gzip",
is_encrypted: false,
extraction_reports: [],
},
],
subtasks: [midTask],
},
];
await writeFile(reportPath, JSON.stringify(orphanRecords));
}
if (mode === "depth-reentry") {
// A depth-limited directory with two published children that can each
// re-root a fresh run, proving neither target is lost during indexing.
const mid = join(output, "rootfs", "config");
const innerPath = join(mid, "inner.gz");
const secondPath = join(mid, "second.gz");
await mkdir(mid, { recursive: true });
await writeFile(innerPath, content);
await writeFile(secondPath, content);
const innerTask = task(innerPath, depth + 1, "inner-id");
const secondTask = task(secondPath, depth + 1, "second-id");
const midTask = {
...task(mid, depth, "gzip-id"),
is_multi_file: true,
};
const depthReentryRecords = [
{
task: innerTask,
reports: [stat(innerPath, content.length), hash(childHash)],
subtasks: [],
},
{
task: secondTask,
reports: [stat(secondPath, content.length), hash(childHash)],
subtasks: [],
},
{
task: midTask,
reports: [
stat(mid, 0, true),
{
__typename__: "MultiFileReport",
id: "gzip-id",
handler_name: "gzip",
name: "config",
paths: [innerPath, secondPath],
extraction_reports: [],
},
],
subtasks: [innerTask, secondTask],
},
{
task: task(input, 0, ""),
reports: [
stat(input, bytes.length),
hash(sha256),
{
__typename__: "ChunkReport",
id: "gzip-id",
start_offset: 0,
end_offset: bytes.length,
size: bytes.length,
handler_name: "gzip",
is_encrypted: false,
extraction_reports: [],
},
],
subtasks: [midTask],
},
];
await writeFile(reportPath, JSON.stringify(depthReentryRecords));
process.exitCode = 0;
}
const records = [
{
task: childTask,
@@ -100,6 +292,41 @@ if (args.includes("--version")) {
reports: [
stat(input, bytes.length),
hash(sha256),
...(mode === "multi-file-failure" || mode === "multi-file-dependency"
? [
{
__typename__: "MultiFileReport",
id: mode === "multi-file-dependency" ? "sq-id" : "mf-id",
handler_name:
mode === "multi-file-dependency" ? "squashfs" : "tar",
name:
mode === "multi-file-dependency"
? "rootfs.squashfs"
: "rootfs.tar",
paths: [
join(
output,
mode === "multi-file-dependency"
? "rootfs.squashfs"
: "rootfs.tar",
),
],
extraction_reports: [
mode === "multi-file-dependency"
? {
__typename__: "ExtractorDependencyNotFoundReport",
severity: "ERROR",
dependencies: ["sasquatch"],
}
: {
__typename__: "ExtractorExecutionErrorReport",
severity: "ERROR",
handler: "tar",
},
],
},
]
: []),
{
__typename__: "ChunkReport",
id: "gzip-id",
@@ -109,12 +336,15 @@ if (args.includes("--version")) {
handler_name: "gzip",
is_encrypted: false,
extraction_reports:
mode === "dependency"
mode === "dependency" || mode === "unnamed-chunk-dependency"
? [
{
__typename__: "ExtractorDependencyNotFoundReport",
severity: "ERROR",
dependencies: ["sasquatch"],
dependencies:
mode === "unnamed-chunk-dependency"
? []
: ["sasquatch"],
},
]
: [],
@@ -123,13 +353,42 @@ if (args.includes("--version")) {
subtasks: [childTask],
},
];
await writeFile(
reportPath,
mode === "malformed" || mode === "workspace-cleanup-failure-malformed"
? "["
: JSON.stringify(records),
);
if (mode === "dependency") process.exitCode = 1;
if (mode === "many-unknown") {
records[1].reports = [
...records[1].reports,
...Array.from({ length: 1000 }, (_, index) => ({
__typename__: "UnknownChunkReport",
id: `unknown-${index}`,
start_offset: 0,
end_offset: 1,
size: 1,
})),
];
}
if (mode === "unnamed-dependency") {
records[1].reports = [
...records[1].reports,
{
__typename__: "ExtractorDependencyNotFoundReport",
severity: "ERROR",
dependencies: [],
},
];
}
if (
mode !== "depth-reentry" &&
mode !== "depth-orphan" &&
mode !== "nested-unknown" &&
mode !== "nested-unknown-oversized"
)
await writeFile(
reportPath,
mode === "malformed" || mode === "workspace-cleanup-failure-malformed"
? "["
: JSON.stringify(records),
);
if (mode === "dependency" || mode === "multi-file-dependency")
process.exitCode = 1;
if (
mode === "workspace-cleanup-failure" ||
mode === "workspace-cleanup-failure-malformed"
+51
View File
@@ -135,6 +135,57 @@ export const createGoBinaryFixture = (
return { bytes, headerOffset, versionOffset, moduleOffset, moduleBytes };
};
/** Two writable ELF loads, as in a PIE whose build info is in the later load. */
export const createGoSplitElfFixture = (
options: Parameters<typeof createGoBinaryFixture>[0] = {},
) => {
const fixture = createGoBinaryFixture({ ...options, format: "elf" });
const { bytes, headerOffset } = fixture;
const wide = (options.bits ?? 64) === 64;
const little = (options.byteOrder ?? "little") === "little";
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
const word = (offset: number, value: number) => {
if (wide) view.setBigUint64(offset, BigInt(value), little);
else view.setUint32(offset, value, little);
};
const second = 128 + (wide ? 56 : 32);
bytes.copy(bytes, second, 128, second);
view.setUint16(wide ? 56 : 44, 2, little);
word(128 + (wide ? 32 : 16), headerOffset);
word(128 + (wide ? 40 : 20), headerOffset);
word(second + (wide ? 8 : 4), headerOffset);
word(second + (wide ? 16 : 8), 0x10000 + headerOffset);
word(second + (wide ? 32 : 16), bytes.length - headerOffset);
word(second + (wide ? 40 : 20), bytes.length - headerOffset);
return fixture;
};
/** Many file-backed ELF loads sharing a small input, without any metadata marker. */
export const createGoOverlappingElfFixture = (duplicate = false): Buffer => {
const count = 257;
const size = 1024 * 1024;
const dataStart = 65536;
const bytes = Buffer.alloc(dataStart + size + count * 16);
createGoBinaryFixture({ sectionless: true, omitBuildInfo: true }).bytes.copy(
bytes,
0,
0,
64,
);
bytes.writeUInt16LE(count, 56);
for (let index = 0; index < count; index++) {
const header = 128 + index * 56;
const offset = dataStart + (duplicate ? 0 : index * 16);
bytes.writeUInt32LE(1, header);
bytes.writeUInt32LE(6, header + 4);
bytes.writeBigUInt64LE(BigInt(offset), header + 8);
bytes.writeBigUInt64LE(0x400000n + BigInt(offset), header + 16);
bytes.writeBigUInt64LE(BigInt(size), header + 32);
bytes.writeBigUInt64LE(BigInt(size), header + 40);
}
return bytes;
};
/** Embedded invalid UTF-8 alongside valid records, with exact independent byte expectations. */
export const createGoBinaryByteStringFixture = () => {
const version = Buffer.from("xgo1.26.0");