From fd9d362d98ae67db954a153ea83677223ba21602 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:18:00 +0800 Subject: [PATCH] fix(firmware): derive nested coverage from each reported file (#10077) * fix(firmware): account for unclassified nested file bytes * fix(firmware): describe nested gaps without invented chunk facts --- src/firmware/FirmwareNestedCoverage.test.ts | 264 ++++++++++++++++++++ src/firmware/FirmwarePublication.ts | 6 +- src/firmware/FirmwareReports.ts | 52 ++-- 3 files changed, 302 insertions(+), 20 deletions(-) create mode 100644 src/firmware/FirmwareNestedCoverage.test.ts diff --git a/src/firmware/FirmwareNestedCoverage.test.ts b/src/firmware/FirmwareNestedCoverage.test.ts new file mode 100644 index 0000000000..e10478af94 --- /dev/null +++ b/src/firmware/FirmwareNestedCoverage.test.ts @@ -0,0 +1,264 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { gzipSync } from "node:zlib"; +import { expect, it } from "vitest"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { createAnalysisExecution } from "../application/AnalysisProvider.js"; +import { FirmwareAnalysisService } from "../application/firmware/FirmwareAnalysisService.js"; +import { ok } from "../domain/result.js"; +import { startFirmwareDeadline } from "./FirmwareDeadline.js"; +import { + FirmwareFileAccess, + inventoryFirmwareOutput, +} from "./FirmwareFiles.js"; +import { publishFirmwareExtraction } from "./FirmwarePublication.js"; +import { normalizeUnblobReport } from "./FirmwareReports.js"; + +const hash = (bytes: Buffer) => + createHash("sha256").update(bytes).digest("hex"); +const known = (id: string, start: number, end: number) => ({ + __typename__: "ChunkReport", + id, + start_offset: start, + end_offset: end, + size: end - start, + handler_name: "authored-handler", + is_encrypted: false, + extraction_reports: [], +}); +const cases = [ + { name: "opaque child", spans: [], gaps: [[0, 6]], coverage: "partial" }, + { + name: "partly classified child", + spans: [[2, 4]], + gaps: [ + [0, 2], + [4, 2], + ], + coverage: "partial", + }, + { + name: "fully classified child", + spans: [[0, 6]], + gaps: [], + coverage: "complete", + }, + { + name: "explicit unknown child", + spans: [], + gaps: [], + coverage: "partial", + unknown: true, + }, + { + name: "empty child", + spans: [], + gaps: [], + coverage: "complete", + empty: true, + }, +] as const; + +/** Inert gzip bytes and matching authored Unblob task reports; no engine runs. */ +const capture = async (selection: (typeof cases)[number]) => { + const root = await createTestTempDirectory("rea-nested-firmware-coverage-"); + const staging = join(root, "staging"); + await mkdir(staging); + const child = Buffer.from("empty" in selection ? "" : "opaque"); + const compressed = gzipSync(child); + const original = Buffer.concat([ + Buffer.from("prefix!"), + compressed, + Buffer.from("suffix"), + ]); + const path = join(root, "firmware.bin"); + const inputPath = join(root, "selected.gz"); + const childPath = join(staging, "child.bin"); + await writeFile(path, original); + await writeFile(inputPath, compressed); + await writeFile(childPath, child); + const task = { + path: childPath, + depth: 1, + blob_id: "container", + is_multi_file: false, + }; + const stat = (path: string, size: number) => ({ + __typename__: "StatReport", + path, + size, + is_file: true, + is_dir: false, + is_link: false, + link_target: null, + }); + const raw = [ + { + task: { path: inputPath, depth: 0, blob_id: "", is_multi_file: false }, + reports: [ + stat(inputPath, compressed.length), + { __typename__: "HashReport", sha256: hash(compressed) }, + known("container", 0, compressed.length), + ], + subtasks: [task], + }, + { + task, + reports: [ + stat(childPath, child.length), + { __typename__: "HashReport", sha256: hash(child) }, + ...selection.spans.map(([start, end], index) => + known(String(index), start, end), + ), + ...("unknown" in selection + ? [ + { + __typename__: "UnknownChunkReport", + id: "opaque", + start_offset: 0, + end_offset: child.length, + size: child.length, + }, + ] + : []), + ], + subtasks: [], + }, + ]; + return { + root, + staging, + path, + inputPath, + child, + original, + raw, + context: { + inputPath, + outputRoot: staging, + sha256: hash(compressed), + length: compressed.length, + offset: 7, + maxDepth: 3, + }, + }; +}; + +/** Exercise actual file publication and the shared Evidence boundary. */ +const evidence = async (fixture: Awaited>) => { + const files = new FirmwareFileAccess(); + const output = join(fixture.root, "published"); + try { + const normalized = normalizeUnblobReport(fixture.raw, fixture.context); + const published = await publishFirmwareExtraction({ + files, + workspacePath: fixture.root, + deadline: startFirmwareDeadline("publication", 120_000), + analysisTimeoutMs: 120_000, + outputDirectory: output, + entries: await inventoryFirmwareOutput( + fixture.staging, + 1024, + 10, + 1024, + "extract_firmware", + ), + normalized, + selection: { + offset: 7, + length: fixture.context.length, + sha256: fixture.context.sha256, + }, + maxDepth: 3, + engine: { + name: "unblob", + version: "authored-fixture", + executable_path: "/inert/unused/unblob", + executable_sha256: "a".repeat(64), + source_revision: null, + worker_count: 1, + }, + signal: new AbortController().signal, + retainCleanup: () => { + throw new Error("Unexpected fixture cleanup failure"); + }, + }); + const execution = createAnalysisExecution( + published, + { id: "unblob", name: "Authored Unblob report", version: "fixture" }, + { + subject: { + path: fixture.path, + sha256: hash(fixture.original), + format: "file", + }, + rawResult: fixture.raw, + }, + ); + const result = await new FirmwareAnalysisService({ + execute: async () => ok(execution), + }).execute("extract_firmware", { + path: fixture.path, + range: { offset: 7, length: fixture.context.length }, + output_directory: output, + }); + if (!result.ok) throw result.error; + expect(result.value.normalized_result).toHaveProperty( + "extraction_gaps", + published.extraction_gaps, + ); + expect(result.value.normalized_result).toHaveProperty( + "extraction_continuations", + published.extraction_continuations, + ); + expect( + (await readFile(join(output, "child.bin"))).equals(fixture.child), + ).toBe(true); + expect((await readFile(fixture.path)).equals(fixture.original)).toBe(true); + expect(result.value.raw_result).toEqual(fixture.raw); + return { value: result.value, published }; + } finally { + await files.close(); + } +}; + +it.each(cases)( + "retains $name coverage and child-relative gaps in service Evidence", + async (selection) => { + const fixture = await capture(selection); + const result = await evidence(fixture); + expect(result.value.normalized_result).toMatchObject({ + coverage: selection.coverage, + }); + const implicit = result.published.extraction_gaps.filter( + (gap) => gap.kind === "unclassified-range", + ); + expect(implicit).toMatchObject( + selection.gaps.map(([offset, length]) => ({ + input_path: "$output/child.bin", + range: { offset, length }, + root_file_range: null, + })), + ); + expect(implicit).toHaveLength(selection.gaps.length); + if ("unknown" in selection) + expect(result.published.extraction_gaps).toMatchObject([ + { + kind: "unknown-chunk-handler", + input_path: "$output/child.bin", + range: { offset: 0, length: 6 }, + root_file_range: null, + }, + ]); + if (selection.gaps.length > 0) + for (const continuation of result.published.extraction_continuations) + expect(continuation.instruction).not.toContain("was carved"); + for (const gap of result.published.extraction_gaps) { + expect(result.published.extraction_continuations).toContainEqual({ + id: gap.continuation_id, + instruction: expect.stringContaining("relative to the published child"), + }); + } + }, +); diff --git a/src/firmware/FirmwarePublication.ts b/src/firmware/FirmwarePublication.ts index 1f40772ce4..0aedfe4d08 100644 --- a/src/firmware/FirmwarePublication.ts +++ b/src/firmware/FirmwarePublication.ts @@ -59,11 +59,11 @@ const depthContinuation = ( // Use a different analyzer on the actual published child; repeating the same // whole-child extraction with unchanged handlers would reproduce this gap. -const nestedUnknownContinuation = ( +const nestedRegionContinuation = ( publishedFile: { readonly path: string; readonly size: number } | undefined, ): string => { const base = - "This unknown chunk was carved from a nested extraction, so its range is relative to the published child named by this gap's input_path, not to the original input."; + "This region is within a nested extraction, so its range is relative to the published child named by this gap's input_path, not to the original input."; return publishedFile !== undefined && publishedFile.size > 0 && publishedFile.size <= FIRMWARE_LIMITS.inputBytes @@ -390,7 +390,7 @@ const extractionGaps = ( ); const nestedContinuationId = (inputPath: string): string => { return continuation( - nestedUnknownContinuation(publishedFiles.get(inputPath)), + nestedRegionContinuation(publishedFiles.get(inputPath)), ); }; const chunkGaps = ( diff --git a/src/firmware/FirmwareReports.ts b/src/firmware/FirmwareReports.ts index fe1c68758c..051a9263aa 100644 --- a/src/firmware/FirmwareReports.ts +++ b/src/firmware/FirmwareReports.ts @@ -540,28 +540,46 @@ export const normalizeUnblobReport = ( "extract_firmware", "Report contains tasks disconnected from the selected input", ); - // A producer can emit no UnknownChunkReport for an opaque file. Compute - // examined byte coverage from reported intervals rather than from empty errors. + // A producer can emit no UnknownChunkReport for an opaque file, including + // nested outputs. Each regular file owns its byte coverage; decompressed + // child offsets do not establish offsets in the original firmware. const unclassifiedRanges: { input_path: string; range: { offset: number; length: number }; - root_file_range: { offset: number; length: number }; + root_file_range: { offset: number; length: number } | null; }[] = []; - let coveredEnd = 0; - const gap = (offset: number, length: number) => - unclassifiedRanges.push({ - input_path: "$input", - range: { offset, length }, - root_file_range: { offset: context.offset + offset, length }, - }); - for (const chunk of chunks - .filter((chunk) => chunk.input_path === "$input") - .sort((left, right) => left.range.offset - right.range.offset)) { - if (chunk.range.offset > coveredEnd) - gap(coveredEnd, chunk.range.offset - coveredEnd); - coveredEnd = Math.max(coveredEnd, chunk.range.offset + chunk.range.length); + const chunksByPath = new Map(); + for (const chunk of chunks) { + const owned = chunksByPath.get(chunk.input_path) ?? []; + owned.push(chunk); + chunksByPath.set(chunk.input_path, owned); + } + for (const [path, size] of [...sizes].sort(([left], [right]) => + left < right ? -1 : left > right ? 1 : 0, + )) { + if (!files.has(path)) continue; + let coveredEnd = 0; + const gap = (offset: number, length: number) => + unclassifiedRanges.push({ + input_path: path, + range: { offset, length }, + root_file_range: + path === "$input" + ? { offset: context.offset + offset, length } + : null, + }); + for (const chunk of (chunksByPath.get(path) ?? []).sort( + (left, right) => left.range.offset - right.range.offset, + )) { + if (chunk.range.offset > coveredEnd) + gap(coveredEnd, chunk.range.offset - coveredEnd); + coveredEnd = Math.max( + coveredEnd, + chunk.range.offset + chunk.range.length, + ); + } + if (coveredEnd < size) gap(coveredEnd, size - coveredEnd); } - if (coveredEnd < context.length) gap(coveredEnd, context.length - coveredEnd); return { files, sizes,