From ebd709cef51ee0791ef9cf935da974fd51374e5d Mon Sep 17 00:00:00 2001 From: ShuaixinHuang <1836656514@qq.com> Date: Fri, 20 Nov 2026 00:59:02 +0800 Subject: [PATCH] fix(esp): disclose partition entries outside the table checksum (#12384) * fix(esp): disclose partition entries outside the table checksum * fix(esp): label the partition checksum offset origin * fix(esp): count partition entries outside the table MD5 The coverage warning was only prose, and it presented the MD5 entry's table-relative offset without a structured fact. Record entriesAfterChecksum on the MD5 result, project it as entries_after_checksum, and keep the warning as presentation. Co-authored-by: morluto * fix(esp): accept saved flash inventories without the checksum count A required entries_after_checksum rejected ESP flash Evidence written before the field existed. Keep the count exactOptional so omission stays unknown, and still emit it for new inventories. Co-authored-by: morluto --------- Co-authored-by: ShuaixinHuang <197200901+ShuaixinHuang@users.noreply.github.com> Co-authored-by: ShuaixinHuang Co-authored-by: Cursor Agent Co-authored-by: morluto --- docs/esp-flash-image.md | 13 +- .../EspFlashImageArtifactReader.test.ts | 27 +++- src/artifacts/EspFlashImageArtifactReader.ts | 1 + src/domain/artifactFormatMetadata.test.ts | 130 ++++++++++++++++++ src/domain/artifactFormatMetadata.ts | 7 + src/domain/firmware/espFlashLayout.test.ts | 27 ++++ src/domain/firmware/espPartitionTable.ts | 33 ++++- 7 files changed, 232 insertions(+), 6 deletions(-) create mode 100644 src/domain/artifactFormatMetadata.test.ts diff --git a/docs/esp-flash-image.md b/docs/esp-flash-image.md index 45a603f039..bd2b985e78 100644 --- a/docs/esp-flash-image.md +++ b/docs/esp-flash-image.md @@ -23,7 +23,10 @@ before it, and a terminating entry within `0xC00` bytes. ESP-IDF accepts a table with no partition entries, but REA requires one, so that neither erased flash nor a lone MD5 entry is read as a table. A table whose MD5 does not match, or that has a second MD5 entry, is still read, with `md5.status` `invalid` and a limitation, because the -bootloader would refuse it. The table is located in this order: +bootloader would refuse it. Partition entries after the MD5 record are +kept. The digest covers only the entries before it, and +`entries_after_checksum` counts the later ones. The table is located in +this order: 1. at the caller's `partition_table_offset`, when given; no scan follows; 2. at the default `0x8000`; @@ -89,7 +92,13 @@ The root `format_metadata` (`esp-flash-image`) records: table, with its ESP image facts; - `partition_table`: its flash address, the `method` that found it (`default`, `caller` or `scan`), the partition count and the MD5 `status` - (`valid`, `invalid` or `absent`) with the stored and computed digests; + (`valid`, `invalid` or `absent`) with the stored and computed digests. + `entries_after_checksum`, when present, is how many partition entries + follow the MD5 record and sit outside its digest. New inventories emit + 0 when the record is absent or every partition entry precedes it. + Evidence saved before the field omits it, and that omission leaves the + count unknown. When the count is greater than zero, a limitation names + it and the MD5 entry's offset from the start of the table; - `apps`: ESP image facts for each app partition the dump holds; app partitions declared over the same bytes share one decode; - `otadata`: the otadata partition's records, or null without one; diff --git a/src/artifacts/EspFlashImageArtifactReader.test.ts b/src/artifacts/EspFlashImageArtifactReader.test.ts index 7eedf94d29..7f38dc7d70 100644 --- a/src/artifacts/EspFlashImageArtifactReader.test.ts +++ b/src/artifacts/EspFlashImageArtifactReader.test.ts @@ -18,6 +18,7 @@ import { espFlashDump, otaData, otaRecord, + partitionEntry, partitionTable, } from "../domain/firmware/espFlash.fixture.js"; import { @@ -106,7 +107,7 @@ describe("ESP flash layout inventory", () => { flash_address: 0x8000, method: "default", partition_count: 6, - md5: { status: "valid" }, + md5: { status: "valid", entries_after_checksum: 0 }, }, partitions_outside_dump: [], otadata: { partition_index: 1 }, @@ -174,6 +175,30 @@ describe("ESP flash layout inventory", () => { expect.stringMatching(/0xa000.*0x1000.*MD5.*does not record/iu), ); }); + + it("reports partition entries after the MD5 record as a count", async () => { + const [first, second, third] = TWO_OTA_PARTITIONS; + const prefix = partitionTable([first!]); + const snapshot = await scanCanonicalArtifactInventory( + await writeDump( + espFlashDump({ + bootloader: { at: 0, bytes: BOOTLOADER }, + table: Buffer.concat([ + prefix.subarray(0, 64), + partitionEntry(second!), + partitionEntry(third!), + prefix.subarray(64), + ]), + }), + ), + ); + expect(root(snapshot)).toMatchObject({ + partition_table: { + flash_address: 0x8000, + md5: { status: "valid", entries_after_checksum: 2 }, + }, + }); + }); }); it("inventories a zero-byte partition without reporting missing bytes", async () => { diff --git a/src/artifacts/EspFlashImageArtifactReader.ts b/src/artifacts/EspFlashImageArtifactReader.ts index 60cfa181df..8a2bac6b25 100644 --- a/src/artifacts/EspFlashImageArtifactReader.ts +++ b/src/artifacts/EspFlashImageArtifactReader.ts @@ -334,6 +334,7 @@ const layoutMetadata = ( status: table.md5.status, stored: table.md5.stored, computed: table.md5.computed, + entries_after_checksum: table.md5.entriesAfterChecksum, }, }, partitions_outside_dump: table.partitions diff --git a/src/domain/artifactFormatMetadata.test.ts b/src/domain/artifactFormatMetadata.test.ts new file mode 100644 index 0000000000..83494d1129 --- /dev/null +++ b/src/domain/artifactFormatMetadata.test.ts @@ -0,0 +1,130 @@ +import { expect, it } from "vitest"; + +import { artifactInventoryResultSchema } from "./artifactGraph.js"; + +const digest = (character: string) => character.repeat(64); +const MD5 = "ab".repeat(16); + +/** The MD5 object Evidence stored before `entries_after_checksum` existed. */ +const historicalMd5 = { + status: "valid" as const, + stored: MD5, + computed: MD5, +}; + +const flashMetadata = (md5: { + readonly status: "valid"; + readonly stored: string; + readonly computed: string; + readonly entries_after_checksum?: number | undefined; +}) => ({ + kind: "esp-flash-image" as const, + flash_offset: { value: 0, source: "default" as const }, + chip: null, + bootloader: null, + partition_table: { + flash_address: 0x8000, + method: "default" as const, + partition_count: 1, + md5, + }, + partitions_outside_dump: [], + apps: [], + otadata: null, + boot_selection: { + basis: "none" as const, + active_record: null, + slot: null, + partition_index: null, + assumptions: [], + running_partition: "unknown" as const, + }, + undescribed: [], +}); + +/** One inventory page, the payload `inspect_analysis_view` parses. */ +const inventory = ( + metadata: ReturnType, +): Record => ({ + manifest: { + manifest_id: `agm_${digest("b")}`, + root_artifact_id: `art_${digest("a")}`, + root_sha256: digest("c"), + root_format: "esp-flash-image", + graph_sha256: digest("d"), + node_count: 1, + occurrence_count: 1, + edge_count: 0, + }, + nodes: [ + { + artifact_id: `art_${digest("a")}`, + format: "file", + sha256: digest("c"), + size: 0, + media_type: null, + architecture: null, + content_state: "materialized", + limitations: [], + }, + ], + occurrences: [ + { + occurrence_id: `occ_${digest("e")}`, + artifact_id: `art_${digest("a")}`, + parent_occurrence_id: null, + logical_path: ".", + entry_kind: "file", + artifact_kind: "container", + artifact_format: "esp-flash-image", + declared_size: 0, + compressed_size: 0, + executable: false, + encrypted: false, + hash_status: "not-hashed", + source_location: null, + format_metadata: metadata, + limitations: [], + }, + ], + edges: [], + provenance: [], + coverage: { scope: "container-entries", status: "complete" }, + integrity_contradictions: [], + limitations: [], +}); + +const parsedMd5 = (input: Record) => { + const metadata = + artifactInventoryResultSchema.parse(input).occurrences[0]?.format_metadata; + if (metadata?.kind !== "esp-flash-image") + throw new Error("expected ESP flash metadata"); + return metadata.partition_table.md5; +}; + +it("parses ESP flash Evidence saved before the checksum entry count", () => { + const md5 = parsedMd5(inventory(flashMetadata(historicalMd5))); + expect(md5).toEqual(historicalMd5); + expect(md5).not.toHaveProperty("entries_after_checksum"); +}); + +it("keeps a recorded checksum entry count on current ESP flash Evidence", () => { + expect( + parsedMd5( + inventory(flashMetadata({ ...historicalMd5, entries_after_checksum: 2 })), + ), + ).toEqual({ ...historicalMd5, entries_after_checksum: 2 }); +}); + +it("rejects an explicit undefined checksum entry count", () => { + expect( + artifactInventoryResultSchema.safeParse( + inventory( + flashMetadata({ + ...historicalMd5, + entries_after_checksum: undefined, + }), + ), + ).success, + ).toBe(false); +}); diff --git a/src/domain/artifactFormatMetadata.ts b/src/domain/artifactFormatMetadata.ts index cefd6f5822..2aaed6da2a 100644 --- a/src/domain/artifactFormatMetadata.ts +++ b/src/domain/artifactFormatMetadata.ts @@ -697,6 +697,13 @@ export const espFlashImageMetadataSchema = z.strictObject({ .string() .regex(/^[0-9a-f]{32}$/u) .nullable(), + /** + * Partition entries after the MD5 record, outside its digest. New + * inventories emit 0 when the record is absent or every partition + * entry precedes it. Omission is Evidence saved before this field + * and leaves the count unknown. + */ + entries_after_checksum: uint8.exactOptional(), }), }), /** diff --git a/src/domain/firmware/espFlashLayout.test.ts b/src/domain/firmware/espFlashLayout.test.ts index e9cdec8dd6..2e3c20408f 100644 --- a/src/domain/firmware/espFlashLayout.test.ts +++ b/src/domain/firmware/espFlashLayout.test.ts @@ -48,6 +48,7 @@ describe("ESP partition tables", () => { ); expect(decoded.md5.status).toBe("valid"); expect(decoded.md5.entryOffset).toBe(7 * 32); + expect(decoded.md5.entriesAfterChecksum).toBe(0); expect(decoded.length).toBe(9 * 32); expect(decoded.limitations).toEqual([]); expect(decoded.partitions.map(({ label }) => label)).toEqual([ @@ -79,6 +80,7 @@ describe("ESP partition tables", () => { expect(decoded.md5).toEqual({ status: "absent", entryOffset: null, + entriesAfterChecksum: 0, stored: null, computed: null, }); @@ -90,6 +92,7 @@ describe("ESP partition tables", () => { partitionTable(TWO_OTA_PARTITIONS, { md5: "corrupt" }), ); expect(decoded.md5.status).toBe("invalid"); + expect(decoded.md5.entriesAfterChecksum).toBe(0); expect(decoded.partitions).toHaveLength(6); expect(decoded.limitations).toEqual([ expect.stringMatching(/MD5.*0xc0.*does not match.*bootloader rejects/iu), @@ -491,3 +494,27 @@ describe("ESP bootloader location", () => { ]); }); }); + +it("keeps and discloses partitions after a matching prefix MD5", () => { + const [first, second, third] = TWO_OTA_PARTITIONS; + const prefix = partitionTable([first!]); + const decoded = table( + Buffer.concat([ + prefix.subarray(0, 64), + partitionEntry(second!), + partitionEntry(third!), + prefix.subarray(64), + ]), + ); + expect(decoded.md5).toMatchObject({ + status: "valid", + entryOffset: 32, + entriesAfterChecksum: 2, + }); + expect(decoded.md5.stored).toBe(decoded.md5.computed); + expect(decoded.partitions.map(({ label }) => label)).toEqual([ + first?.label, + second?.label, + third?.label, + ]); +}); diff --git a/src/domain/firmware/espPartitionTable.ts b/src/domain/firmware/espPartitionTable.ts index c3fcc6d88a..d5699c46ab 100644 --- a/src/domain/firmware/espPartitionTable.ts +++ b/src/domain/firmware/espPartitionTable.ts @@ -53,7 +53,13 @@ export interface EspPartition { export interface EspPartitionTableMd5 { readonly status: "valid" | "invalid" | "absent"; + /** Byte offset of the MD5 entry from the start of the table, or null when absent. */ readonly entryOffset: number | null; + /** + * Partition entries that follow the MD5 record and are outside its digest. + * Zero when the record is absent or every partition entry precedes it. + */ + readonly entriesAfterChecksum: number; readonly stored: string | null; readonly computed: string | null; } @@ -95,7 +101,10 @@ const partition = ( * over the entries before it, and a terminator whose magic, type and subtype * are all `0xFF`. A structural fault means the bytes are not a table. An MD5 * mismatch or a second MD5 entry, which the bootloader rejects, is reported - * as an integrity fact so that the declared entries stay visible. ESP-IDF + * as an integrity fact so that the declared entries stay visible. Partition + * entries after the MD5 record stay in the table. The digest covers only the + * entries before it; `entriesAfterChecksum` counts the later ones, and a + * limitation names that count and the record's table-relative offset. ESP-IDF * accepts a table with no partition entries, but REA requires one before * the terminator, so that neither erased flash nor a lone MD5 entry is read * as a table. @@ -108,9 +117,11 @@ export const decodeEspPartitionTable = ( let md5: EspPartitionTableMd5 = { status: "absent", entryOffset: null, + entriesAfterChecksum: 0, stored: null, computed: null, }; + let partitionsAfterMd5 = 0; for (let slot = 0; slot < MAX_ENTRIES; slot += 1) { const at = slot * ENTRY_BYTES; const entry = bytes.subarray(at, at + ENTRY_BYTES); @@ -120,6 +131,7 @@ export const decodeEspPartitionTable = ( ); const magic = u16(entry, 0); if (magic === PARTITION_MAGIC) { + if (md5.entryOffset !== null) partitionsAfterMd5 += 1; partitions.push(partition(entry, partitions.length, at)); continue; } @@ -138,6 +150,7 @@ export const decodeEspPartitionTable = ( md5 = { status: stored === computed ? "valid" : "invalid", entryOffset: at, + entriesAfterChecksum: 0, stored, computed, }; @@ -150,8 +163,22 @@ export const decodeEspPartitionTable = ( const terminator = magic === END_MAGIC && entry[2] === END_TYPE && entry[3] === END_TYPE; // An MD5 entry is not a partition: a table declares at least one. - if (terminator && partitions.length > 0) - return ok({ partitions, md5, length: at + ENTRY_BYTES, limitations }); + if (terminator && partitions.length > 0) { + const reported: EspPartitionTableMd5 = { + ...md5, + entriesAfterChecksum: partitionsAfterMd5, + }; + if (reported.entryOffset !== null && reported.entriesAfterChecksum > 0) + limitations.push( + `${String(reported.entriesAfterChecksum)} partition entries after the MD5 record at table-relative offset ${offsetText(reported.entryOffset)} are outside its checksum coverage.`, + ); + return ok({ + partitions, + md5: reported, + length: at + ENTRY_BYTES, + limitations, + }); + } return err( slot === 0 && magic === END_MAGIC ? "the first entry is empty"