import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; import { EspFlashImageArtifactReader } from "./EspFlashImageArtifactReader.js"; import { ArtifactResourceScope } from "../resources/ArtifactResourceScope.js"; import type { ArtifactEntry, ArtifactReaderTraversal, } from "../resources/ArtifactReader.js"; import { openRegularFile } from "../filesystem/RegularFile.js"; import { extractArtifact } from "./extraction/ArtifactExtraction.js"; import * as regionReads from "./EspFlashRegionReads.js"; import * as espImageDecoding from "../domain/firmware/espImage.js"; import { TWO_OTA_PARTITIONS, espFlashDump, otaData, otaRecord, partitionEntry, partitionTable, } from "../domain/firmware/espFlash.fixture.js"; import { ESP32C3_APP_SEGMENTS, ESP32C3_BOOTLOADER_SEGMENTS, espImage, pattern, } from "../../tests/fixtures/espImage.js"; import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js"; import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; import { scanCanonicalArtifactInventory } from "../../tests/fixtures/artifactInventory.js"; const INVENTORY_LIMITATION = "ESP flash layout inventory reads a dump's bootloader region, partition table and partitions and decodes the bootloader and app partitions as ESP images. The OTA selection is derived without app rollback or anti-rollback, and the running partition is unknown. Partitions, nvs_keys included, are extracted as ordinary data; nothing is decrypted."; const BOOTLOADER = espImage({ segments: ESP32C3_BOOTLOADER_SEGMENTS }); const APP = espImage({ segments: ESP32C3_APP_SEGMENTS }); /** A C3 dump: factory and ota_0 hold the app, ota_1 is erased, otadata selects ota_0. */ const DUMP = espFlashDump({ bootloader: { at: 0, bytes: BOOTLOADER }, parts: [ { at: 0xd000, bytes: otaData(otaRecord({ seq: 1 })) }, { at: 0x1_0000, bytes: APP }, { at: 0x2_0000, bytes: APP }, ], }); const writeDump = async (bytes: Uint8Array, name = "flash.bin") => { const path = join(await createTestTempDirectory("rea-esp-flash-"), name); await writeFile(path, bytes); return path; }; const root = ( snapshot: Awaited>, ) => snapshot.occurrences.find(({ logical_path }) => logical_path === ".") ?.format_metadata; const list = async (reader: EspFlashImageArtifactReader) => { const entries: ArtifactEntry[] = []; for await (const entry of reader.entries()) entries.push(entry); return entries; }; describe("ESP flash layout inventory", () => { it("inventories the bootloader, table and partitions with app and otadata facts", async () => { const snapshot = await scanCanonicalArtifactInventory( await writeDump(DUMP), ); expect(snapshot.manifest.root_format).toBe("esp-flash-image"); expect(snapshot.coverage).toEqual({ scope: "container-entries", status: "complete", }); expect(snapshot.limitations).toEqual([INVENTORY_LIMITATION]); expect( snapshot.occurrences.map(({ logical_path }) => logical_path), ).toEqual([ ".", "bootloader.bin", "partition-table.bin", "partitions/factory.bin", "partitions/nvs.bin", "partitions/ota_0.bin", "partitions/ota_1.bin", "partitions/otadata.bin", "partitions/phy_init.bin", ]); expect(root(snapshot)).toMatchObject({ kind: "esp-flash-image", flash_offset: { value: 0, source: "default" }, chip: { target: "esp32c3", isa: "riscv" }, bootloader: { path: "bootloader.bin", flash_address: 0, region_size: 0x8000, facts: { status: "esp-image", image: { descriptor: { kind: "bootloader" } }, coverage: "complete", }, }, partition_table: { flash_address: 0x8000, method: "default", partition_count: 6, md5: { status: "valid", entries_after_checksum: 0 }, }, partitions_outside_dump: [], otadata: { partition_index: 1 }, boot_selection: { basis: "otadata", active_record: 0, slot: 0, partition_index: 4, running_partition: "unknown", }, undescribed: [], }); const apps = ( root(snapshot) as { apps: { path: string; partition_index: number; facts: { status: string }; }[]; } ).apps; // Each image names its entry, so no lookup by index is needed. expect( apps.map(({ path, partition_index, facts }) => [ path, partition_index, facts.status, ]), ).toEqual([ ["partitions/factory.bin", 3, "esp-image"], ["partitions/ota_0.bin", 4, "esp-image"], ["partitions/ota_1.bin", 5, "erased"], ]); const factory = snapshot.occurrences.find( ({ logical_path }) => logical_path === "partitions/factory.bin", ); expect(factory).toMatchObject({ source_path: "factory", artifact_format: "esp-image", format_metadata: { kind: "esp-flash-partition", index: 3, label: "factory", type_name: "app", subtype_name: "factory", flash_address: 0x1_0000, size: 0x1_0000, }, }); }); it("finds a relocated table by scanning and says the location is derived", async () => { const snapshot = await scanCanonicalArtifactInventory( await writeDump( espFlashDump({ bootloader: { at: 0, bytes: BOOTLOADER }, tableAt: 0xa000, }), ), ); expect(root(snapshot)).toMatchObject({ partition_table: { flash_address: 0xa000, method: "scan" }, }); expect(snapshot.limitations).toContainEqual( expect.stringMatching(/0xa000.*0x1000.*MD5.*does not record/iu), ); }); it("reports partition entries after the MD5 record as a count", async () => { const [first, second, third] = TWO_OTA_PARTITIONS; const prefix = partitionTable([first!]); const snapshot = await scanCanonicalArtifactInventory( await writeDump( espFlashDump({ bootloader: { at: 0, bytes: BOOTLOADER }, table: Buffer.concat([ prefix.subarray(0, 64), partitionEntry(second!), partitionEntry(third!), prefix.subarray(64), ]), }), ), ); expect(root(snapshot)).toMatchObject({ partition_table: { flash_address: 0x8000, md5: { status: "valid", entries_after_checksum: 2 }, }, }); }); }); it("inventories a zero-byte partition without reporting missing bytes", async () => { const dump = espFlashDump({ bootloader: { at: 0, bytes: BOOTLOADER }, table: partitionTable([ ...TWO_OTA_PARTITIONS, { label: "empty", type: 1, subtype: 0x82, offset: 0xa000, size: 0 }, ]), }); const snapshot = await scanCanonicalArtifactInventory(await writeDump(dump)); expect(snapshot.coverage.status).toBe("complete"); expect( snapshot.occurrences.find( ({ logical_path }) => logical_path === "partitions/empty.bin", ), ).toMatchObject({ declared_size: 0, limitations: ["partitions/empty.bin declares zero bytes."], format_metadata: { flash_address: 0xa000, size: 0 }, }); }); it("keeps a zero-byte partition before the flash origin at a nonnegative offset", async () => { const esp32 = espImage({ segments: ESP32C3_BOOTLOADER_SEGMENTS, chipId: 0x0000, }); const whole = espFlashDump({ bootloader: { at: 0x1000, bytes: esp32 }, table: partitionTable([ { label: "empty", type: 1, subtype: 0x82, offset: 0, size: 0 }, ...TWO_OTA_PARTITIONS, ]), }); const snapshot = await scanCanonicalArtifactInventory( await writeDump(whole.subarray(0x1000)), { espFlashSelection: { flashOffset: 0x1000 } }, ); const empty = snapshot.occurrences.find( ({ logical_path }) => logical_path === "partitions/empty.bin", ); expect(empty).toMatchObject({ declared_size: 0, source_location: { offset: 0, length: 0 }, }); }); describe("ESP flash layout coverage and selection", () => { it("keeps usable partitions beside a truncated and a missing one", async () => { const cut = DUMP.subarray(0, 0x2_8000); const snapshot = await scanCanonicalArtifactInventory(await writeDump(cut)); // The table names every partition, so the end of the file hides no entry. expect(snapshot.coverage.status).toBe("complete"); const ota0 = snapshot.occurrences.find( ({ logical_path }) => logical_path === "partitions/ota_0.bin", ); expect(ota0).toMatchObject({ declared_size: 0x8000, limitations: [ "partitions/ota_0.bin is truncated: the dump holds 32768 of its 65536 bytes.", ], }); // ota_1 lies past the end: an entry holding none of its bytes. expect(root(snapshot)).toMatchObject({ partitions_outside_dump: [] }); expect( snapshot.occurrences.find( ({ logical_path }) => logical_path === "partitions/ota_1.bin", ), ).toMatchObject({ declared_size: 0, limitations: [ expect.stringMatching(/partitions\/ota_1\.bin.*outside.*65536/iu), ], format_metadata: { index: 5, written_offset: null, written_end: null, written_bytes: null, }, }); const { apps } = root(snapshot) as { apps: { partition_index: number }[]; }; // An app partition the file holds none of has no image facts. expect(apps.map(({ partition_index: index }) => index)).toEqual([3, 4]); expect( snapshot.occurrences.some( ({ logical_path }) => logical_path === "partitions/factory.bin", ), ).toBe(true); }); it("reports written bytes in an app slot the bootloader finds empty", async () => { // ota_1's header is erased, but an older image sits 0x8000 into the slot. const dump = espFlashDump({ bootloader: { at: 0, bytes: BOOTLOADER }, parts: [ { at: 0xd000, bytes: otaData(otaRecord({ seq: 1 })) }, { at: 0x1_0000, bytes: APP }, { at: 0x2_0000, bytes: APP }, { at: 0x3_8000, bytes: APP }, ], }); const snapshot = await scanCanonicalArtifactInventory( await writeDump(dump), ); const indices = [...APP.keys()].filter((index) => APP[index] !== 0xff); const start = 0x8000 + indices[0]!; const end = 0x8000 + indices.at(-1)! + 1; const row = (path: string) => snapshot.occurrences.find(({ logical_path }) => logical_path === path) ?.format_metadata; expect(row("partitions/ota_1.bin")).toMatchObject({ written_offset: start, written_end: end, written_bytes: indices.length, }); const { apps } = root(snapshot) as { apps: { partition_index: number; facts: { status: string } }[]; }; expect( apps.find(({ partition_index: index }) => index === 5), ).toMatchObject({ facts: { status: "erased" }, }); // Only the erased app slot holds bytes the bootloader does not read. const erasedSlotWarnings = snapshot.limitations.filter((text) => /bootloader.*no app/iu.test(text), ); expect(erasedSlotWarnings).toHaveLength(1); const warning = erasedSlotWarnings[0] ?? ""; expect(warning).toContain("partitions/ota_1.bin"); expect(warning).toContain(String(indices.length)); expect(warning).toContain(`0x${start.toString(16)}`); expect(warning).toContain(`0x${end.toString(16)}`); expect(warning).toMatch( /erased flash.*bootloader.*no app.*not erased.*not read as an image/iu, ); // A partition erased throughout has no written bytes. expect(row("partitions/nvs.bin")).toMatchObject({ written_offset: null, written_end: null, written_bytes: 0, }); }); it("reads a dump that starts at the caller's flash offset", async () => { const esp32 = espImage({ segments: ESP32C3_BOOTLOADER_SEGMENTS, chipId: 0x0000, }); const whole = espFlashDump({ bootloader: { at: 0x1000, bytes: esp32 } }); const snapshot = await scanCanonicalArtifactInventory( await writeDump(whole.subarray(0x1000)), { espFlashSelection: { flashOffset: 0x1000 }, }, ); expect(root(snapshot)).toMatchObject({ flash_offset: { value: 0x1000, source: "caller" }, chip: { target: "esp32" }, bootloader: { flash_address: 0x1000, region_size: 0x7000 }, partition_table: { flash_address: 0x8000, method: "default" }, }); const nvs = snapshot.occurrences.find( ({ logical_path }) => logical_path === "partitions/nvs.bin", ); expect(nvs?.source_location).toEqual({ offset: 0x8000, length: 0x4000 }); }); it("refuses a selected dump without a readable table as not recognized", async () => { const reader = new EspFlashImageArtifactReader( await writeDump(pattern(0x2_0000, 7)), ); try { await expect( reader.entries()[Symbol.asyncIterator]().next(), ).rejects.toMatchObject({ reason: "unavailable", message: expect.stringContaining( "ESP flash image layout is not recognized", ), }); } finally { await reader.close(); } // The caller's selection decides before another container's signature. const zipLike = await writeDump( Buffer.concat([Buffer.from("PK\x03\x04"), pattern(0x1_0000, 3)]), "dump.zip", ); await expect( scanCanonicalArtifactInventory(zipLike, { espFlashSelection: {} }), ).rejects.toThrow(/ESP flash image layout is not recognized/u); }); }); describe("ESP flash layout reader identity", () => { it("rejects forged entries and changed carrier identity, then closes idempotently", async () => { const path = await writeDump(DUMP); const reader = new EspFlashImageArtifactReader(path); try { const entry = (await list(reader)).find( ({ path: name }) => name === "partitions/nvs.bin", ); if (entry === undefined) throw new Error("Missing nvs partition"); await expect( reader.open({ ...entry, declaredSize: 1 }), ).rejects.toMatchObject({ reason: "integrity" }); await expect( reader.open({ ...entry, byteOffset: 0 }), ).rejects.toMatchObject({ reason: "integrity" }); await expect( reader.open({ ...entry, adapterKey: "esp-flash-image:partitions/x.bin", }), ).rejects.toMatchObject({ reason: "integrity" }); await writeFile(path, DUMP.subarray(0, 0x1_0000)); await expect(reader.open(entry)).rejects.toMatchObject({ reason: "integrity", }); } finally { await reader.close(); await reader.close(); } }); it("opens a partition past the end as the empty range at the end of the file", async () => { const cut = DUMP.subarray(0, 0x2_8000); const reader = new EspFlashImageArtifactReader(await writeDump(cut)); try { const entry = (await list(reader)).find( ({ path }) => path === "partitions/ota_1.bin", ); if (entry === undefined) throw new Error("Missing entry"); expect(entry).toMatchObject({ byteOffset: cut.length, declaredSize: 0 }); const chunks: Buffer[] = []; for await (const chunk of await reader.open(entry)) chunks.push(chunk as Buffer); expect(Buffer.concat(chunks)).toHaveLength(0); } finally { await reader.close(); } }); it("honors cancellation before admitting a file with the exact reason", async () => { const reader = new EspFlashImageArtifactReader(await writeDump(DUMP)); const controller = new AbortController(); const reason = { stage: "before admission" }; controller.abort(reason); try { await expect( reader.entries(controller.signal)[Symbol.asyncIterator]().next(), ).rejects.toMatchObject({ reason: "cancelled", cause: reason }); } finally { await reader.close(); } }); }); describe("ESP flash layout extraction and lifecycle", () => { it("extracts every partition byte-exactly, nvs_keys included", async () => { const parts = [ ...TWO_OTA_PARTITIONS.slice(0, 3), { label: "keys", type: 0x01, subtype: 0x04, offset: 0x1_0000, size: 0x1000, flags: 1, }, ]; const keys = pattern(0x1000, 3); const dump = espFlashDump({ size: 0x1_1000, bootloader: { at: 0, bytes: BOOTLOADER }, table: partitionTable(parts), parts: [{ at: 0x1_0000, bytes: keys }], }); const directory = await createTestTempDirectory("rea-esp-flash-extract-"); const path = join(directory, "flash.bin"); await writeFile(path, dump); const resourceScope = new ArtifactResourceScope(); try { const extracted = await extractArtifact({ inputPath: path, inputFormat: "esp-flash-image", outputRoot: join(directory, "out"), environment: process.env, integrityPolicy: "fail", resourceScope, }); expect(extracted.containment_verified).toBe(true); expect( extracted.artifacts.map(({ relative_path }) => relative_path).sort(), ).toEqual([ "bootloader.bin", "partition-table.bin", "partitions/keys.bin", "partitions/nvs.bin", "partitions/otadata.bin", "partitions/phy_init.bin", ]); for (const [name, offset, length] of [ ["bootloader.bin", 0, 0x8000], ["partition-table.bin", 0x8000, 0x1000], ["partitions/keys.bin", 0x1_0000, 0x1000], ] as const) expect( Buffer.compare( await readFile(join(extracted.output_root, name)), dump.subarray(offset, offset + length), ), ).toBe(0); } finally { await resourceScope.close(); } }); }); const multiChunkSize = 0x4_0000; const checkPreparationInterruption = async ( reader: EspFlashImageArtifactReader, action: "cancel" | "close", stage: string, observe: (read: () => void) => { mockRestore(): void }, ) => { const controller = new AbortController(); const reason = { stage }; let acknowledge!: () => void; const acknowledgement = new Promise((resolve) => { acknowledge = resolve; }); let reads = 0; const observer = observe(() => { if (++reads === 1) acknowledge(); }); const pending = reader .entries(controller.signal) [Symbol.asyncIterator]() .next(); try { await awaitAcknowledgement(acknowledgement, pending, stage); const requestedAt = reads; if (action === "cancel") controller.abort(reason); await Promise.all([ expect(pending).rejects.toMatchObject( action === "cancel" ? { reason: "cancelled", cause: reason } : { reason: "unavailable" }, ), action === "close" ? reader.close() : undefined, ]); expect(reads).toBe(requestedAt); await reader.close(); await expect( reader.entries()[Symbol.asyncIterator]().next(), ).rejects.toMatchObject({ reason: "unavailable" }); } finally { try { await reader.close(); } finally { await pending.catch(() => undefined); observer.mockRestore(); } } }; describe("ESP flash layout preparation interruption", () => { it.each(["cancel", "close"] as const)( "settles app decoding on %s after a decoded chunk", async (action) => { const appOffset = 0x1_0000; const partitionSize = multiChunkSize + 0x1_0000; const image = espImage({ segments: [ { address: 0x4038_0000, data: Buffer.alloc(multiChunkSize, 0x5a) }, ], }); const reader = new EspFlashImageArtifactReader( await writeDump( espFlashDump({ size: appOffset + partitionSize, bootloader: { at: 0, bytes: BOOTLOADER }, table: partitionTable([ { label: "factory", type: 0, subtype: 0, offset: appOffset, size: partitionSize, }, ]), parts: [{ at: appOffset, bytes: image }], }), ), ); const create = espImageDecoding.createEspImageDecoder; await checkPreparationInterruption( reader, action, "app decoding", (read) => vi .spyOn(espImageDecoding, "createEspImageDecoder") .mockImplementation((size) => { const decoder = create(size); return { write: (chunk) => { decoder.write(chunk); if (size === partitionSize) read(); }, get satisfied() { return decoder.satisfied; }, finish: () => decoder.finish(), }; }), ); }, ); it.each(["cancel", "close"] as const)( "settles the written-bytes scan on %s after an admitted chunk", async (action) => { // Only the written-bytes scan reads this data partition's bytes. const dataOffset = 0x1_0000; const reader = new EspFlashImageArtifactReader( await writeDump( espFlashDump({ size: dataOffset + multiChunkSize, bootloader: { at: 0, bytes: BOOTLOADER }, table: partitionTable([ { label: "storage", type: 1, subtype: 0x81, offset: dataOffset, size: multiChunkSize, }, ]), parts: [ { at: dataOffset, bytes: Buffer.alloc(multiChunkSize, 0x5a) }, ], }), ), ); const scan = regionReads.writtenExtents; await checkPreparationInterruption( reader, action, "written-bytes scan", (read) => vi .spyOn(regionReads, "writtenExtents") .mockImplementation((context, regions, assertOpen) => scan(context, regions, () => { assertOpen(); read(); }), ), ); }, ); it.each(["cancel", "close"] as const)( "settles the erased-gap scan on %s after an admitted chunk", async (action) => { const gapStart = 0xa000; const reader = new EspFlashImageArtifactReader( await writeDump( espFlashDump({ size: gapStart + multiChunkSize, bootloader: { at: 0, bytes: BOOTLOADER }, table: partitionTable([ { label: "nvs", type: 1, subtype: 2, offset: 0x9000, size: 0x1000, }, ]), }), ), ); const scan = regionReads.firstUnerasedInRange; await checkPreparationInterruption( reader, action, "erased-gap scan", (read) => vi .spyOn(regionReads, "firstUnerasedInRange") .mockImplementation((context, range, assertOpen) => scan(context, range, () => { assertOpen(); if (range.start === gapStart && range.length === multiChunkSize) read(); }), ), ); }, ); }); describe("ESP flash layout written bytes of overlapping partitions", () => { it("reads the bytes they cover once and keeps each partition's own facts", async () => { // 60 data partitions over one MiB: 40 cover all of it, 20 its first half. const parts = Array.from({ length: 60 }, (_, index) => ({ label: `p${String(index)}`, type: 1, subtype: 0x81, offset: 0x1_0000, size: index < 40 ? 0x10_0000 : 0x8_0000, })); const bytes = espFlashDump({ size: 0x11_0000, bootloader: { at: 0, bytes: BOOTLOADER }, table: partitionTable(parts), parts: [ { at: 0x1_0100, bytes: Buffer.alloc(0x100, 0x5a) }, { at: 0x9_0000, bytes: Buffer.alloc(0x100, 0x5a) }, ], }); const path = await writeDump(bytes); const admitted = await openRegularFile(path, { symlinks: "reject" }); const nativeRead = admitted.handle.read.bind(admitted.handle); let read = 0; const reads = vi.spyOn(admitted.handle, "read").mockImplementation((async ( ...args: unknown[] ) => { const result = (await Reflect.apply( nativeRead, admitted.handle, args, )) as { bytesRead: number; }; read += result.bytesRead; return result; }) as typeof admitted.handle.read); const reader = new EspFlashImageArtifactReader(path, admitted); try { let coverage: ArtifactReaderTraversal | undefined; const entries: ArtifactEntry[] = []; for await (const entry of reader.entries(undefined, { onCoverage: (value) => { coverage = value; }, })) entries.push(entry); // Each covered byte is read once, not once per partition over it. expect(read).toBeLessThan(2 * bytes.length); expect( entries.flatMap(({ path: entryPath, formatMetadata }) => formatMetadata?.kind === "esp-flash-partition" ? [[entryPath, formatMetadata]] : [], ), ).toEqual( parts.map((part, index) => [ `partitions/${part.label}.bin`, expect.objectContaining({ index, label: part.label, flash_address: 0x1_0000, size: part.size, written_offset: 0x100, written_end: index < 40 ? 0x8_0100 : 0x200, written_bytes: index < 40 ? 0x200 : 0x100, }), ]), ); expect( coverage?.limitations.filter((text) => text.includes(" overlaps ")), ).toHaveLength(59); } finally { await reader.close(); reads.mockRestore(); await admitted.handle.close(); } }); }); describe("ESP flash layout app images of overlapping partitions", () => { it("decodes each declared range once and keeps each partition's facts", async () => { // 60 app partitions over one image: 40 hold all of it, 20 cut it short. const image = espImage({ segments: [{ address: 0x3c00_0020, data: Buffer.alloc(0xf_0000, 0x5a) }], }); const parts = Array.from({ length: 60 }, (_, index) => ({ label: `a${String(index)}`, type: 0, subtype: 0x10 + (index % 16), offset: 0x1_0000, size: index < 40 ? 0x10_0000 : 0x8_0000, })); const reader = new EspFlashImageArtifactReader( await writeDump( espFlashDump({ size: 0x11_0000, bootloader: { at: 0, bytes: BOOTLOADER }, table: partitionTable(parts), parts: [{ at: 0x1_0000, bytes: image }], }), ), ); const decode = vi.spyOn(regionReads, "decodeEspFlashImage"); try { await list(reader); // The bootloader, the whole image and the image cut at 0x80000. expect(decode).toHaveBeenCalledTimes(3); const apps = reader.formatMetadata()?.apps ?? []; expect( apps.map(({ path, partition_index: index, facts }) => [ path, index, facts.status, facts.coverage, ]), ).toEqual( parts.map((part, index) => [ `partitions/${part.label}.bin`, index, "esp-image", index < 40 ? "complete" : "unknown", ]), ); } finally { decode.mockRestore(); await reader.close(); } }); });