diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index f0792352f2..8034918932 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -106,6 +106,8 @@ explicit path or endpoint and do not need it. `extract_firmware` when extraction is requested, with a caller-selected new absolute output directory. These tools use caller-supplied Binwalk/Unblob on Linux; see the [firmware guide](https://github.com/morluto/rea/blob/main/docs/firmware-analysis.md). +- ESP32-family app or bootloader image (`0xE9` magic): `inspect_artifact` + without a provider; see the ESP section of `references/native-and-artifacts.md`. - .NET NativeAOT PE/ELF: native Ghidra analysis. Native code can yield pseudocode; consult the NativeAOT workflow in `references/native-and-artifacts.md` for optional metadata recovery and its supported host/layout boundary. diff --git a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md index 56f9e1652b..ed858a7bb4 100644 --- a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md +++ b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md @@ -669,3 +669,24 @@ IDs. Header addresses/CPU/OS are declarations; payloads remain as stored. Multi-file components retain their aligned physical ranges; the full payload is also materialized. Unknown name text retains its original 32 bytes. See [U-Boot uImage](https://github.com/morluto/rea/blob/main/docs/uimage.md). + +## ESP images + +For an ESP32-family app or bootloader image, `open_binary` then +`inspect_artifact` with `detail: "summary"`. `root_format_metadata` holds the +observed `header` and `descriptor` (its `kind` is the app or bootloader role) +and the derived `chip`, `checksum`, `appended_sha256` and +`undescribed_trailing` ranges. Segment rows (`segment-NN.bin`) carry +`load_addr`, `classification` (`mapped`, `loaded`, `not-loaded` or `unknown`, +per ESP-IDF `should_map`/`should_load`; RAM ranges are not checked) and +`source_location`; select them with `inspect_analysis_view` and +`logical_paths`. Signature blocks are `unverified`; `public_key_sha256` +matches eFuse key digests. + +Unknown `source_coverage` means truncation or undescribed trailing bytes, such +as the flash segments of a `--ram-only-header` image; do not infer absence +from it. Erased flash is described. Integrity mismatches are facts, not +refusals. ESP8266 images are refused, and flash-encrypted images are not +recognized. Segments are not disassembled, and RISC-V or Xtensa segments +cannot be opened as raw-binary. See +[ESP images](https://github.com/morluto/rea/blob/main/docs/esp-image.md). diff --git a/docs/cli.md b/docs/cli.md index ee2bb4e9f9..877419d4e0 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -261,6 +261,17 @@ Format versions 6 through 8 retain header, block and directory facts with none/LZ4/LZ4HC block decompression. See [UnityFS](unityfs.md) for encryption and LZMA coverage boundaries. +Inspect an ESP32-family app or bootloader image without esptool: + +```sh +rea inspect-artifact firmware.bin --json +rea extract-artifact firmware.bin --json +``` + +The inventory lists each segment with its ESP-IDF load classification, the +checksum and SHA-256 results, the descriptor and any Secure Boot v2 signature +blocks. See [ESP images](esp-image.md) for recognition, coverage and refusals. + Inspect a Node.js SEA carrier without running it: ```sh diff --git a/docs/esp-image.md b/docs/esp-image.md new file mode 100644 index 0000000000..a86a8ca17f --- /dev/null +++ b/docs/esp-image.md @@ -0,0 +1,316 @@ +# ESP images + +REA reads ESP32-family firmware images through `inspect-artifact` / +`inspect_artifact` and `extract-artifact` / `extract_artifact`. OTA downloads, +app partitions and second-stage bootloaders are all ESP images. Reading is +static: no esptool, device or flash access is needed, and no code is run or +disassembled. + +```sh +rea inspect-artifact ./firmware.bin --json +rea inspect-artifact ./ota.bin --detail summary --evidence_output evidence.json --json +rea extract-artifact ./firmware.bin --json +``` + +For MCP, open the file with `open_binary`, then call `inspect_artifact`. The +summary's `root_format_metadata` holds the image facts. Use its +`parent_evidence_id` with `inspect_analysis_view` to select segment rows. +`extract_artifact` writes one file per segment and per signature block. + +For agent sessions, advertise the `session,artifact,application` tool families +and consider setting both `REA_MCP_INPUT_SCHEMA_PROFILE` and +`REA_MCP_OUTPUT_SCHEMA_PROFILE` to `compact`. The output profile bounds each +advertised output schema to 4,096 bytes by relaxing nested client validation; +REA still validates complete results against the canonical contracts. Pair +this with retained summaries and selected inventory views to limit the context +needed for one question. See [MCP contracts](mcp-contracts.md) for the profiles +and their client compatibility tradeoffs. + +The layout facts follow ESP-IDF v6.1 (`esp_app_format.h`, `esp_app_desc.h`, +`esp_bootloader_desc.h`, `esp_image_format.c`, the ROM `secure_boot.h` +headers and `docs/en/security/secure-boot-v2.rst`) and esptool v5.6.0 +(`bin_image.py`, `cmds.py`, `loader.py`, espsecure). + +## Recognition + +The container format is `esp-image`. A file is an ESP image when its bytes +show all of these; the file name does not matter: + +- magic `0xE9` at offset 0 and 1 to 16 segments. A count of 0 is valid but + loads nothing, and accepting it would admit x86 programs that begin + `E9 00`; +- a flash mode and flash speed code that ESP-IDF's enums define; +- a `hash_appended` byte of 0 or 1, as esptool requires; +- when the file holds the first segment header, a segment length that is a + multiple of 4. + +Later segments, the checksum and the digest are integrity facts. A truncated +download is still an ESP image, with its integrity reported as unavailable. + +| Input | Outcome | +| --------------------------------------------------- | ---------------------------------------------------------------- | +| ESP8266 image (v1 or v3): same magic, 8-byte header | Refused: "ESP8266 firmware images are not supported…" | +| Flash-encrypted image (ciphertext) | Not recognized: the bytes carry no magic. Unknown, not malformed | +| ESP8266 v2 image (magic `0xEA`) | Not recognized | + +An ESP8266 image has its first segment header at offset 8, and that segment +loads into ESP8266 DRAM, IRAM (Arduino's eboot links at `0x4010F000`) or the +IROM flash window, where v3 and Arduino app images begin. In an ESP32-family +header the same bytes hold `wp_pin` and the drive settings, which esptool and +ESP-IDF write as `0xEE` and zeros. Inside another container, an ESP8266 image +is classified as a plain file. + +Recognition cannot exclude every file that starts with an x86 near jump +(`0xE9`), such as a DOS `.com` program. An explicit `--target-format dos-com` +or `raw-binary` selection skips ESP recognition. + +## Image facts + +Root `format_metadata` has `kind: "esp-image"`: + +| Field | Label | Meaning | +| ------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------- | +| `header` | observed | Every `esp_image_header_t` field, with ESP-IDF's names, for example `entry_addr`, `chip_id`, `min_chip_rev_full` | +| `chip` | derived | `target`, `isa` and the table's `source` (`ESP-IDF v6.1`) from `chip_id`; `null` for an ID the table lacks | +| `checksum` | derived | `offset`, `stored`, `computed` and `status`: `valid`, `invalid` or `unavailable` | +| `appended_sha256` | derived | The digest `hash_appended` adds: `offset`, `stored`, `computed` and `status`, including `not-appended` | +| `image_end` | derived | End of the checksum block and any appended digest; `null` when the file ends before it | +| `signature_sector_offset` | derived | The Secure Boot v2 signature sector, when it holds at least one valid block | +| `undescribed_trailing` | derived | Byte ranges after the image that nothing describes; empty when every byte is described | +| `descriptor` | observed | `esp_app_desc_t` or `esp_bootloader_desc_t` fields, with ESP-IDF's names, and the descriptor's file `offset` | + +The descriptor's `kind`, derived from its magic, classifies the image as an +`app` or a `bootloader`. Without a descriptor the role is unknown. + +The ISA comes from `chip_id`, never from the series letter: ESP32-S31 is +RISC-V. The chip table covers every `esp_chip_id_t` in ESP-IDF v6.1: + +| `chip_id` | Target | ISA | `chip_id` | Target | ISA | +| --------- | --------- | ------ | --------- | ---------- | ----- | +| `0x0000` | `esp32` | xtensa | `0x0012` | `esp32p4` | riscv | +| `0x0002` | `esp32s2` | xtensa | `0x0014` | `esp32c61` | riscv | +| `0x0005` | `esp32c3` | riscv | `0x0017` | `esp32c5` | riscv | +| `0x0009` | `esp32s3` | xtensa | `0x0019` | `esp32h21` | riscv | +| `0x000C` | `esp32c2` | riscv | `0x001C` | `esp32h4` | riscv | +| `0x000D` | `esp32c6` | riscv | `0x0020` | `esp32s31` | riscv | +| `0x0010` | `esp32h2` | riscv | | | | + +Other IDs, for example esptool v5.6.0's ESP32-E22 (`0x1F`), have `chip: null` +and unclassified segments. + +**Integrity.** The checksum is the XOR of every segment data byte, seeded with +`0xEF`, in the last byte of the next 16-byte block. The appended SHA-256 covers +every byte through the checksum. Both are facts, not admission rules: a +mismatch is reported with a limitation and the segments stay readable. + +**Descriptors.** Both descriptors begin segment 0: ESP-IDF's bootloader reads +the app descriptor there, and esptool places the bootloader descriptor there. +Strings stop at the first NUL and are omitted, with a limitation, when they +are not valid UTF-8. The bootloader descriptor's magic is a single byte +(`0x50`), so its reserved bytes, which ESP-IDF zero-fills, must also be zero. +A `--ram-only-header` image puts RAM segments first, so its app descriptor is +not in segment 0 and its role is unknown. `mmu_page_size` keeps ESP-IDF's +name and value: the MMU page size as a base-2 logarithm, as +`esp_app_desc_t` declares it, so `16` means 64 KiB. It is `0` in images built +before ESP-IDF v5.4. `spi_flash_mode` is the next byte, which ESP-IDF v6.1 +names, kept as stored and not interpreted; releases that reserve it leave it +`0`. A file can end inside the app descriptor: every field it holds in full +is kept, a field it cuts off is `null`, and `observed_length` gives the +descriptor bytes the file holds. The segment's truncation is reported as usual. + +## Segments + +Each header-declared segment is one entry named `segment-.bin`, for +example `segment-00.bin`, so two builds pair segments by position. Its +`source_location` is the segment data in the image file. `format_metadata` +(`esp-image-segment`) records `index`, `load_addr`, `data_len`, +`header_offset` and `classification`. + +`classification` follows ESP-IDF's bootloader (`should_map` and +`should_load` in `esp_image_format.c`) on a cold boot: + +| Value | Meaning | +| ------------ | ------------------------------------------------------------------------------------------------- | +| `mapped` | Inside the chip's IROM, DROM or, where the MMU is per memory type, PSRAM range (`soc.h`) | +| `loaded` | `should_load` selects it: not mapped, at or above `0x10000000`. RAM ranges are not checked | +| `not-loaded` | Below `0x10000000`: the padding segments esptool writes at address 0, including `--secure-pad-v2` | +| `unknown` | The chip is unknown, or the mapping depends on the bootloader's MMU page size | + +`loaded` does not mean the bootloader accepts the segment. Before loading, +ESP-IDF's `verify_load_addresses` rejects the image when a loaded segment is +not entirely in the chip's DRAM, IRAM, RTC memory or SPM, or overlaps the +bootloader's own stack, data or loader code. For example, an ESP32-C6 +segment at `0x43000000` is `loaded` here and rejected there. REA models no +RAM ranges. + +Padding segments are inventoried and extractable, but are never load regions. +When waking from deep sleep, ESP-IDF also skips RTC-memory segments; that +runtime case is not modelled. ESP32-C6 and ESP32-H2 map flash from +`0x42000000` through 256 pages of the bootloader's configured MMU page size, +which no image records. The app descriptor's page size describes the app's +build, not the bootloader's. With page sizes from 8 KiB (which the hardware +supports and esptool builds for) to 64 KiB, segments from `0x42200000` to +`0x43000000` are `unknown`. Ranges follow ESP-IDF's `should_map`, not esptool's +wider placement ranges. + +A segment cut short by the end of the file is the bytes the file holds, with +an entry limitation that gives the declared length. + +Artifact comparison uses each segment's `load_addr` and `data_len` as its +declaration, so a segment that moves with unchanged bytes is `changed` in +`metadata`. Segments pair by index: inserting or removing one shifts the +paths of the segments after it. Header offsets and `classification` are not +compared; the image's own change covers the header and chip. The image's and +the signature blocks' `format_metadata` carry no declaration: their fields +come from their own bytes, which content comparison covers, and a block's +`image_digest_matches` follows the image's content. + +## Secure Boot v2 signatures + +A signature sector starts at the next 4 KiB boundary after the image and +holds up to three 1216-byte slots. A slot holds a block when it has the `0xE7` +magic and a matching CRC32 at offset 1196, the rule ESP-IDF and espsecure +apply. Each slot is examined independently, so an erased or damaged slot does +not hide a later block. Each block is an entry, `signature-block-.bin`, +with `esp-image-signature-block` metadata: + +| Field | Meaning | +| ---------------------------------- | ------------------------------------------------------------------------- | +| `index`, `version`, `scheme` | Slot 0 to 2, version byte, and `rsa` (`0x02`) or `ecdsa` (`0x03`) | +| `sha_version`, `curve_id` | For ECDSA, the third byte (0 is SHA-256, 1 SHA-384) and the curve ID byte | +| `digest_algorithm`, `image_digest` | The digest the block records | +| `image_digest_matches` | Whether it equals the digest of every byte before the sector | +| `public_key_sha256` | SHA-256 of the block's public key, the value eFuse key digests hold | +| `signature_status` | Always `unverified` | + +Signatures are observed, not verified: REA checks no public key, signature or +eFuse. `public_key_sha256` is the digest espsecure `signature-info-v2` +prints, so it can be compared with `espefuse summary`. An unknown version or +`sha_version` leaves the layout-dependent fields `null`. A slot with the magic +but no matching CRC32 is not a block: it is reported in a limitation, whether +or not another slot holds a block, and its bytes are undescribed. Secure Boot +v1 signatures are not decoded. + +## Coverage and limitations + +Bytes after the image are described when they are erased flash (`0xFF`) or a +signature sector's blocks; erased slots and slack in that sector count as +erased flash. Every other byte is in `undescribed_trailing`. + +| Situation | Outcome | +| ---------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| Segment data or a segment header beyond the end of the file | Truncation limitation; checksum and SHA-256 `unavailable`; coverage `unknown` | +| File ends before the checksum or the appended digest | That status is `unavailable`, with a limitation; coverage `unknown`, since a signature sector could follow | +| Checksum or SHA-256 mismatch | `invalid`, with a limitation; coverage is unchanged | +| Erased flash after the image that runs past the three slots | Described; no limitation | +| Erased flash that ends before the three slots are complete | Limitation; coverage `unknown`, since a signed image cut there would have lost its sector | +| A signature sector the file cuts short | Limitation; coverage `unknown` | +| Any other bytes after the image | Ranges in `undescribed_trailing`, a limitation, and coverage `unknown` | +| Segment length that is not a multiple of 4 | Limitation: ESP-IDF rejects such an image | +| Segment of 16 MiB or more, except on chips with 32-bit cache addresses | Limitation: ESP-IDF rejects such an image | + +ESP32-S3, ESP32-P4, ESP32-C5, ESP32-C61, ESP32-H4 and ESP32-S31 map flash +through 32-bit cache addresses (`SOC_SPI_MEM_SUPPORT_CACHE_32BIT_ADDR_MAP`), so +ESP-IDF sets no 16 MiB segment limit for them. For an unknown chip the limit +is not applied. + +Undescribed bytes include other flash contents such as an app partition's +stale data, a Secure Boot v1 signature, and the flash segments that esptool +`--ram-only-header` writes after the checksum without counting them. REA +reports their ranges and does not guess their structure. In the padding, in a +sector without a valid block, in the sector's slack and after the sector, the +first byte that is not erased starts a range to the end of that region. A +slot that is neither a block nor erased is reported whole. Adjacent ranges +are merged. + +A whole-flash dump of a chip whose bootloader is at offset 0, such as +ESP32-C3 or ESP32-S3, is recognized as that bootloader image; the partition +table, apps and data follow as undescribed bytes. + +## Limits + +An image has at most 16 segments and 3 signature blocks. Decoding reads each +byte once and retains only fixed-size fields. It reads through erased flash to +find the first byte that is not erased, and stops there. There is no +file-size limit. + +## Analyzing a segment + +Inventory does not disassemble code or build a memory map. `open_binary` +admits neither RISC-V nor Xtensa as a raw-binary architecture, so a segment +cannot yet be opened as a raw image. Each segment's `source_location` and +`load_addr` give the file offset and address to map it at by hand, for +example with the Rizin tool's `om` command. + +## Verification + +Synthetic fixtures with computed checksums, digests and CRCs cover +recognition, ESP8266 layouts, the ESP32, ESP32-C3, ESP32-C6 and ESP32-P4 +memory maps, truncation at each structure, integrity mismatches, erased and +undescribed trailing bytes, signature slots including damaged and unknown +layouts, and the refusals. CLI and stdio MCP tests exercise the same artifact +provider and advertised schemas. + +The optional real lane builds ELF files from source, converts them with +esptool and compares REA's facts with `esptool image-info`: + +```sh +REA_RISCV_CLANG=/path/to/clang REA_RISCV_LLD=/path/to/ld.lld \ +REA_ESPTOOL_PYTHON=/path/to/python-with-esptool \ + npm run verify:esp-images +``` + +It needs a Clang with the RISC-V target, `ld.lld` and a Python that can run +`esptool` and `espsecure`. It checks each tool before building anything and +names the variable to set when one is missing, and it warns when esptool is +not v5.6.0, the version whose sources the layout facts cite. It builds an +ESP32-C3 app, the same app with `--dont-append-digest`, `--secure-pad-v2`, +`--ram-only-header` and `--pad-to-size`, a bootloader, and ESP8266 v1 and v3 +images. It then checks: + +- entry point, chip ID, each segment's length, address and header offset, + checksum, digest and descriptor fields against `image-info`; +- each segment's classification in the app, its variants and the + bootloader, whose linker scripts fix every section's region, including the + `not-loaded` padding that `--secure-pad-v2` adds; +- after signing with RSA-3072 and appending ECDSA P-256 and P-384 blocks, + each block's `public_key_sha256` against espsecure `signature-info-v2`, + and that its recorded digest matches; +- extraction of the app and the signed image: each segment file equals the + bytes after esptool's segment header offset, each signature block equals + its slot in the sector espsecure appended, and the source's SHA-256 is + unchanged and equals the reported `root_sha256`; +- the `--pad-to-size` erased tail is described; the `--ram-only-header` + flash segments are undescribed; +- a flipped checksum, a flipped digest byte, a truncated segment (where + `image-info` stops with "End of file reading segment"), and both ESP8266 + refusals. + +`REA_XTENSA_ESP32_GCC` adds an ESP32 (Xtensa) app built with Espressif's +`xtensa-esp32-elf-gcc`. Without it the report marks that image as skipped. + +Each command runs as a process tree owned by the shared verifier command +owner, so a timeout also stops the linker or assembler a compiler driver +starts. The lane removes its workspace only after every tree has stopped and +keeps it when that cleanup cannot be confirmed. + +The lane ran on macOS arm64 with Node 24.22.1, Homebrew Clang and LLD 21.1.8, +and `xtensa-esp32-elf-gcc` 14.2.0 (crosstool-NG esp-14.2.0_20251107), once +with esptool and espsecure v5.6.0 and once with 4.12.0. All checks passed for +all 13 images in both runs. With v5.6.0 the complete run, including the builds +and every esptool call, took 21.7 s elapsed, 16.9 s user CPU and 6.1 s system +CPU under `/usr/bin/time -l` with warm caches. The largest single process +peaked at a 253 MiB resident set; the combined peak of all processes was not +measured. + +The lane does not establish: + +- the other chips: only ESP32-C3 (RISC-V) and ESP32 (Xtensa) images come + from a real producer; the other rows of the chip table are checked against + the ESP-IDF v6.1 headers by reading them; +- images from a real ESP-IDF project build, an OTA download or a device's + flash, none of which it inspects; +- that a signature is valid: it compares digests, and nothing verifies a + signature. + +It does not run in CI. diff --git a/docs/firmware-analysis.md b/docs/firmware-analysis.md index d6465c42ff..4fe25c7e3d 100644 --- a/docs/firmware-analysis.md +++ b/docs/firmware-analysis.md @@ -286,6 +286,15 @@ REA_FIRMWARE_VERIFY_GHIDRA=1 npm run verify:firmware generated gzip/USTAR/ext4 with a Linux x64 ELF child; SquashFS, UBIFS, proprietary firmware, other hosts and architectures remain unverified. +## Inspect ESP images + +Use `inspect-artifact` / `inspect_artifact` on an ESP32-family app or +bootloader image, such as an OTA download or app partition, for its chip, +classified segments, checksum and SHA-256 results, descriptor and Secure Boot +v2 signature blocks. `extract-artifact` / `extract_artifact` writes one file +per segment and signature block. Reading is portable and needs no external +tool. See [ESP images](esp-image.md). + ## Inspect extracted devicetrees Use [`inspect_devicetree`](devicetree.md) / `rea inspect-devicetree` on an diff --git a/docs/index.md b/docs/index.md index 8cf971e01f..18ad082ae9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -47,20 +47,20 @@ For runtime instrumentation, see the [Frida guide](./frida-instrumentation.md). ## Choose a guide -| Task | Guides | -| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Identify a desktop installation | [Runtime candidates, evidence and follow-up files](desktop-applications.md) | -| Inspect native functions, references, or types | [Native binaries](native-investigation.md), [Go metadata](go-binaries.md), [Rust dependencies](rust-binaries.md), [CPython pyc files](python-pyc.md), [PE certificates](pe-certificates.md), [PE imports](pe-imports.md), [PE TLS/load config](pe-load-config.md), [ELF notes](elf-notes.md), [Linux kernel modules](kernel-modules.md), [Apple IPS](apple-ips.md), [Erlang BEAM](erlang-beam.md), [offline ELF layout](binary-diagnostics.md), [recorded crashes](recorded-crashes.md) | -| Inspect native functions, references, or types | [Native binaries](native-investigation.md), [Go metadata](go-binaries.md), [Rust dependencies](rust-binaries.md), [CPython pyc files](python-pyc.md), [PE certificates](pe-certificates.md), [PE imports](pe-imports.md), [PE TLS/load config](pe-load-config.md), [ELF notes](elf-notes.md), [Apple IPS](apple-ips.md), [Erlang BEAM](erlang-beam.md), [offline ELF layout](binary-diagnostics.md), [recorded crashes](recorded-crashes.md) | -| Select a native provider or import format | [Ghidra semantics](ghidra-provider.md), [Windows](windows-ghidra-p0.md), [DOS](ghidra-dos.md), [raw bytes](raw-binary.md), [GameCube DOL](ghidra-dol.md), [MIPS](ghidra-mips.md), [PSP](ghidra-psp.md), [NativeAOT](ghidra-nativeaot.md), [IDA](ida-provider.md), [Kuna](kuna-provider.md), [JEB](jeb-analysis.md) | -| Analyze JavaScript or Electron artifacts | [Static analysis](javascript-artifact-reconstruction.md), [tracing and comparisons](javascript-application-workflows.md), [application graph](javascript-application-graph.md), [source recovery](javascript-recovery.md) | -| Compare static JavaScript with runtime observations | [Runtime attribution](javascript-runtime-observation.md), [reconciliation](javascript-runtime-reconciliation.md), [Electron observation](electron-observation.md), [Electron fuses](electron-fuses.md) | -| Inspect assemblies, packages, or firmware | [Godot PCK](godot-pck.md), [UnityFS](unityfs.md), [U-Boot uImage](uimage.md), [.NET](managed-code-analysis.md), [Python packages](python-packaged-applications.md), [Android](android-analysis.md), [APK signatures](apk-signature.md), [Apktool resources](apktool-resource-analysis.md), [ADB acquisition](adb-device-analysis.md), [Apple](apple-application-analysis.md), [Harmony](harmony-analysis.md), [firmware](firmware-analysis.md), [WASM](wasm-artifacts.md), [EVM](evm-bytecode.md) | -| Observe or compare a browser scenario | [CDP observation](browser-observation.md), [scenarios](browser-scenario-contract.md), [runtime evidence](web-runtime.md), [historical captures](web-network-captures.md) | -| Serve recorded behavior as an agent environment | [Agent environments](agent-environments.md) | -| Trace captured website source | [Source locations](web-source-location.md), [module traces](website-module-trace.md), [script export](website-script-export.md) | -| Capture and compare a command | [Process capture](process-capture.md) | -| Use CLI/MCP and retain evidence | [CLI](cli.md), [MCP contracts](mcp-contracts.md), [agent prompts](mcp-prompts.md), [obligation ledgers](reconstruction-obligation-ledgers.md) | +| Task | Guides | +| --------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Identify a desktop installation | [Runtime candidates, evidence and follow-up files](desktop-applications.md) | +| Inspect native functions, references, or types | [Native binaries](native-investigation.md), [Go metadata](go-binaries.md), [Rust dependencies](rust-binaries.md), [CPython pyc files](python-pyc.md), [PE certificates](pe-certificates.md), [PE imports](pe-imports.md), [PE TLS/load config](pe-load-config.md), [ELF notes](elf-notes.md), [Linux kernel modules](kernel-modules.md), [Apple IPS](apple-ips.md), [Erlang BEAM](erlang-beam.md), [offline ELF layout](binary-diagnostics.md), [recorded crashes](recorded-crashes.md) | +| Inspect native functions, references, or types | [Native binaries](native-investigation.md), [Go metadata](go-binaries.md), [Rust dependencies](rust-binaries.md), [CPython pyc files](python-pyc.md), [PE certificates](pe-certificates.md), [PE imports](pe-imports.md), [PE TLS/load config](pe-load-config.md), [ELF notes](elf-notes.md), [Apple IPS](apple-ips.md), [Erlang BEAM](erlang-beam.md), [offline ELF layout](binary-diagnostics.md), [recorded crashes](recorded-crashes.md) | +| Select a native provider or import format | [Ghidra semantics](ghidra-provider.md), [Windows](windows-ghidra-p0.md), [DOS](ghidra-dos.md), [raw bytes](raw-binary.md), [GameCube DOL](ghidra-dol.md), [MIPS](ghidra-mips.md), [PSP](ghidra-psp.md), [NativeAOT](ghidra-nativeaot.md), [IDA](ida-provider.md), [Kuna](kuna-provider.md), [JEB](jeb-analysis.md) | +| Analyze JavaScript or Electron artifacts | [Static analysis](javascript-artifact-reconstruction.md), [tracing and comparisons](javascript-application-workflows.md), [application graph](javascript-application-graph.md), [source recovery](javascript-recovery.md) | +| Compare static JavaScript with runtime observations | [Runtime attribution](javascript-runtime-observation.md), [reconciliation](javascript-runtime-reconciliation.md), [Electron observation](electron-observation.md), [Electron fuses](electron-fuses.md) | +| Inspect assemblies, packages, or firmware | [Godot PCK](godot-pck.md), [UnityFS](unityfs.md), [U-Boot uImage](uimage.md), [.NET](managed-code-analysis.md), [Python packages](python-packaged-applications.md), [Android](android-analysis.md), [APK signatures](apk-signature.md), [Apktool resources](apktool-resource-analysis.md), [ADB acquisition](adb-device-analysis.md), [Apple](apple-application-analysis.md), [Harmony](harmony-analysis.md), [firmware](firmware-analysis.md), [ESP images](esp-image.md), [WASM](wasm-artifacts.md), [EVM](evm-bytecode.md) | +| Observe or compare a browser scenario | [CDP observation](browser-observation.md), [scenarios](browser-scenario-contract.md), [runtime evidence](web-runtime.md), [historical captures](web-network-captures.md) | +| Serve recorded behavior as an agent environment | [Agent environments](agent-environments.md) | +| Trace captured website source | [Source locations](web-source-location.md), [module traces](website-module-trace.md), [script export](website-script-export.md) | +| Capture and compare a command | [Process capture](process-capture.md) | +| Use CLI/MCP and retain evidence | [CLI](cli.md), [MCP contracts](mcp-contracts.md), [agent prompts](mcp-prompts.md), [obligation ledgers](reconstruction-obligation-ledgers.md) | For development, use [testing](testing.md), [client verification](client-verification.md), [tool design](tool-design.md), [admission limits and recovery](design/limits-and-recovery.md), diff --git a/docs/testing.md b/docs/testing.md index 340cf17daf..386a694f51 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1950,6 +1950,19 @@ unknown chunks, depth limits and extracted child digests. The optional Neither optional toolchain is a base-lane prerequisite. See [firmware analysis](firmware-analysis.md) for limits and unverified formats. +`npm run verify:esp-images` requires a Clang with the RISC-V target and +`ld.lld` (`REA_RISCV_CLANG`, `REA_RISCV_LLD`) and a Python that runs esptool +and espsecure (`REA_ESPTOOL_PYTHON`). It builds ESP32-C3 app and bootloader +ELF files from `tests/conformance/esp/`, converts them with `esptool +elf2image`, including `--secure-pad-v2`, `--ram-only-header` and +`--pad-to-size`, signs one with RSA, ECDSA P-256 and P-384 keys, and compares +REA's facts with `esptool image-info` and `espsecure signature-info-v2`. It +compares extracted segments and signature blocks with the producer's bytes, +and derives malformed images and ESP8266 v1 and v3 images. It checks each +tool before building and warns when esptool is not the pinned v5.6.0. +`REA_XTENSA_ESP32_GCC` adds an ESP32 Xtensa build. See +[ESP images](esp-image.md). + ### JavaScript source recovery Build once with `npm run build:cached`, then run `npm run verify:javascript:recovery`. diff --git a/package.json b/package.json index 2d6e8b5e1e..4d9481c57b 100644 --- a/package.json +++ b/package.json @@ -239,6 +239,7 @@ "verify:managed": "npm run build:cached && node scripts/verify/managed/conformance.mjs", "verify:firmware": "npm run build:cached && node scripts/verify/firmware/analysis.mjs", "verify:devicetree": "npm run build:cached && node scripts/verify/firmware/devicetree.mjs", + "verify:esp-images": "npm run build:cached && node scripts/verify/firmware/esp-image.mjs", "fixtures:firmware": "node scripts/fixtures/firmware/generate.mjs", "verify:android": "npm run build:cached && node scripts/verify/android/analysis.mjs", "verify:jeb": "npm run build:cached && node scripts/verify-jeb.mjs", diff --git a/scripts/verify/firmware/esp-image-tools.mjs b/scripts/verify/firmware/esp-image-tools.mjs new file mode 100644 index 0000000000..69fa63504b --- /dev/null +++ b/scripts/verify/firmware/esp-image-tools.mjs @@ -0,0 +1,308 @@ +import assert from "node:assert/strict"; +import { fileURLToPath } from "node:url"; +import { createVerifierCommands } from "../../lib/verify-commands.mjs"; + +// Optional verification tools only; the runtime reader starts no tool. +const compiler = process.env.REA_RISCV_CLANG ?? "clang"; +const linker = process.env.REA_RISCV_LLD ?? "ld.lld"; +const python = process.env.REA_ESPTOOL_PYTHON ?? "python3"; +// Optional: an Xtensa ESP32 build when Espressif's GCC is supplied. +export const xtensa = process.env.REA_XTENSA_ESP32_GCC; +/** + * Every tool runs as an owned process tree, so a timeout or close also stops + * the children a compiler driver starts. Close it before removing outputs. + */ +export const commands = await createVerifierCommands(); +export const options = { maxBuffer: 16 * 1024 * 1024, timeout: 120_000 }; +export const execute = (command, args, settings = options) => + commands.exec(command, args, settings); + +/** The output of a command that exited nonzero; other failures propagate. */ +export const exitOutput = (failure) => { + if (typeof failure?.code !== "number" || failure.code === 0) throw failure; + return `${failure.snapshot.stdout.text}${failure.snapshot.stderr.text}`; +}; +export const source = (name) => + fileURLToPath( + new URL(`../../../tests/conformance/esp/${name}`, import.meta.url), + ); +const entrypoint = fileURLToPath(new URL("../../rea.mjs", import.meta.url)); + +/** The esptool release whose sources the reader's layout facts cite. */ +const PINNED_ESPTOOL = "v5.6.0"; + +/** Run esptool or espsecure through the caller-supplied Python. */ +export const tool = async (module, args) => { + const { stdout, stderr } = await execute( + python, + ["-m", module, ...args], + options, + ); + return `${stdout}${stderr}`; +}; + +/** Check one tool before any work; a failure names the variable to set. */ +const prerequisite = async (needed, variable, check) => { + try { + return await check(); + } catch (failure) { + throw new Error( + `verify:esp-images needs ${needed}; set ${variable}. ${failure.message}`, + { cause: failure }, + ); + } +}; + +const firstLine = ({ stdout }) => stdout.split("\n")[0]; + +/** Set by checkTools: esptool v5 spells commands and options with hyphens. */ +let v5; +export const spell = (v4, later) => { + assert.notEqual(v5, undefined, "checkTools must run first"); + return v5 ? later : v4; +}; + +/** Check every tool before any work and report their versions. */ +export const checkTools = async () => { + const compilerVersion = await prerequisite( + `a Clang with the RISC-V target (tried ${compiler})`, + "REA_RISCV_CLANG", + async () => { + const targets = await execute(compiler, ["--print-targets"], options); + assert.match(targets.stdout, /^\s*riscv32\b/mu, "No riscv32 target."); + return firstLine(await execute(compiler, ["--version"], options)); + }, + ); + await prerequisite(`ld.lld (tried ${linker})`, "REA_RISCV_LLD", () => + execute(linker, ["--version"], options), + ); + const versionText = await prerequisite( + `a Python that runs esptool (tried ${python} -m esptool)`, + "REA_ESPTOOL_PYTHON", + () => tool("esptool", ["version"]), + ); + const version = /\bv(\d+)\.(\d+)\.(\d+)\b/u.exec(versionText); + assert.ok(version, `Unrecognized esptool version: ${versionText}`); + v5 = Number(version[1]) >= 5; + await prerequisite( + `espsecure in the same Python (tried ${python} -m espsecure)`, + "REA_ESPTOOL_PYTHON", + async () => { + const help = await tool("espsecure", ["--help"]); + for (const command of [ + spell("sign_data", "sign-data"), + spell("signature_info_v2", "signature-info-v2"), + ]) + assert.ok(help.includes(command), `espsecure has no ${command}.`); + }, + ); + if (version[0] !== PINNED_ESPTOOL) + console.warn( + `verify:esp-images: esptool ${version[0]} is not the pinned ${PINNED_ESPTOOL}, whose sources the reader's layout facts cite.`, + ); + const xtensaVersion = + xtensa === undefined + ? undefined + : await prerequisite( + `xtensa-esp32-elf-gcc (tried ${xtensa})`, + "REA_XTENSA_ESP32_GCC to Espressif's compiler, or unset it to skip that image", + async () => firstLine(await execute(xtensa, ["--version"], options)), + ); + return { compilerVersion, esptoolVersion: version[0], xtensaVersion }; +}; + +/** Build a RISC-V ELF from a conformance source; it is never run. */ +export const buildElf = async (program, script, output) => { + await execute( + compiler, + [ + "--target=riscv32-unknown-elf", + "-march=rv32imc", + "-mabi=ilp32", + `--ld-path=${linker}`, + "-nostdlib", + "-static", + "-fno-pic", + "-Os", + `-Wl,-T,${source(script)}`, + "-Wl,--build-id=none", + source(program), + "-o", + output, + ], + options, + ); + return output; +}; + +export const elf2image = (chip, elf, output, extra = []) => + tool("esptool", [ + "--chip", + chip, + "elf2image", + spell("--flash_mode", "--flash-mode"), + "dio", + spell("--flash_size", "--flash-size"), + "4MB", + ...extra, + "-o", + output, + elf, + ]); + +/** Facts esptool image-info prints, read from its documented line formats. */ +export const imageInfo = async (file, chip) => { + const text = await tool("esptool", [ + ...(chip === undefined ? [] : ["--chip", chip]), + ...spell(["image_info", "--version", "2", file], ["image-info", file]), + ]); + const field = (label) => + new RegExp(`^${label}: (.+)$`, "mu").exec(text)?.[1]?.trim(); + const checksum = + /^Checksum: 0x([0-9a-f]+) \((valid|invalid - calculated (?:0x)?([0-9a-f]+))\)$/mu.exec( + text, + ); + const hash = /^Validation hash: ([0-9a-f]{64}) \((valid|invalid)\)$/mu.exec( + text, + ); + return { + detected: field("Detected image type"), + entry: Number(field("Entry point")), + chipId: Number(field("Chip ID")?.split(" ")[0]), + segments: [ + ...text.matchAll( + /^\s*(\d+)\s+0x([0-9a-f]+)\s+0x([0-9a-f]+)\s+0x([0-9a-f]+)\s+\S.*$/gmu, + ), + ].map((row) => ({ + index: Number(row[1]), + length: Number.parseInt(row[2], 16), + address: Number.parseInt(row[3], 16), + headerOffset: Number.parseInt(row[4], 16), + })), + checksum: + checksum === null + ? undefined + : { + stored: Number.parseInt(checksum[1], 16), + valid: checksum[2] === "valid", + computed: Number.parseInt(checksum[3] ?? checksum[1], 16), + }, + hash: + hash === null + ? undefined + : { computed: hash[1], valid: hash[2] === "valid" }, + project: field("Project name"), + appVersion: field("App version"), + elfSha256: field("ELF file SHA256"), + idfVer: field("ESP-IDF"), + secureVersion: field("Secure version"), + bootloaderVersion: field("Bootloader version"), + compileTime: field("Compile time"), + }; +}; + +/** REA's complete inline inventory for one file through the built CLI. */ +export const inspect = async (file) => { + const result = await execute( + process.execPath, + [entrypoint, "inspect-artifact", file, "--json"], + { ...options, env: { ...process.env, REA_LOG_LEVEL: "silent" } }, + ).then( + ({ stdout }) => ({ ok: true, stdout }), + (failure) => ({ ok: false, stdout: exitOutput(failure) }), + ); + if (!result.ok) return { refused: result.stdout }; + const inventory = JSON.parse(result.stdout).normalized_result.substeps[0] + .evidence.normalized_result; + const rows = inventory.occurrences.filter((row) => row.logical_path !== "."); + return { + inventory, + root: inventory.occurrences.find((row) => row.logical_path === ".") + .format_metadata, + segments: rows.filter( + (row) => row.format_metadata?.kind === "esp-image-segment", + ), + signatures: rows.filter( + (row) => row.format_metadata?.kind === "esp-image-signature-block", + ), + }; +}; + +const oracleDescriptorKind = (oracle) => { + if (oracle.project !== undefined) return "app"; + if (oracle.bootloaderVersion !== undefined) return "bootloader"; + return null; +}; + +/** Extract every entry through the built CLI into a new directory in `root`. */ +export const extract = async (file, root) => { + const { stdout } = await execute( + process.execPath, + [entrypoint, "extract-artifact", file, "--json"], + { + ...options, + env: { + ...process.env, + REA_LOG_LEVEL: "silent", + TMPDIR: root, + TMP: root, + TEMP: root, + }, + }, + ); + return JSON.parse(stdout).normalized_result; +}; + +const assertDescriptorAgrees = (name, oracle, descriptor) => { + // Both directions, so a descriptor esptool does not print fails the lane. + assert.equal( + descriptor?.kind ?? null, + oracleDescriptorKind(oracle), + `${name} descriptor`, + ); + if (oracle.project !== undefined) { + assert.equal(descriptor.project_name, oracle.project, name); + assert.equal(descriptor.version, oracle.appVersion, name); + assert.equal(descriptor.app_elf_sha256, oracle.elfSha256, name); + assert.equal(descriptor.idf_ver, oracle.idfVer, name); + assert.equal(String(descriptor.secure_version), oracle.secureVersion, name); + assert.equal( + `${descriptor.date} ${descriptor.time}`, + oracle.compileTime, + name, + ); + } + if (oracle.bootloaderVersion !== undefined) { + assert.equal(String(descriptor.version), oracle.bootloaderVersion, name); + assert.equal(descriptor.idf_ver, oracle.idfVer, name); + assert.equal(descriptor.date_time, oracle.compileTime, name); + } +}; + +/** Every fact both tools report for an image esptool can load. */ +export const assertAgrees = (name, oracle, rea) => { + assert.equal(rea.inventory.manifest.root_format, "esp-image", name); + assert.equal(rea.root.header.entry_addr, oracle.entry, `${name} entry`); + assert.equal(rea.root.header.chip_id, oracle.chipId, `${name} chip_id`); + assert.deepEqual( + rea.segments.map(({ format_metadata: segment }) => ({ + index: segment.index, + length: segment.data_len, + address: segment.load_addr, + headerOffset: segment.header_offset, + })), + oracle.segments, + `${name} segments`, + ); + const { checksum, appended_sha256: digest } = rea.root; + assert.equal(checksum.stored, oracle.checksum.stored, `${name} checksum`); + assert.equal(checksum.computed, oracle.checksum.computed, name); + assert.equal(checksum.status === "valid", oracle.checksum.valid, name); + if (oracle.hash === undefined) + assert.equal(digest.status, "not-appended", `${name} digest`); + else { + assert.equal(digest.computed, oracle.hash.computed, `${name} digest`); + assert.equal(digest.status === "valid", oracle.hash.valid, name); + } + assertDescriptorAgrees(name, oracle, rea.root.descriptor); +}; diff --git a/scripts/verify/firmware/esp-image.mjs b/scripts/verify/firmware/esp-image.mjs new file mode 100644 index 0000000000..d98874e7ff --- /dev/null +++ b/scripts/verify/firmware/esp-image.mjs @@ -0,0 +1,418 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { withVerifierCleanup } from "../../lib/verifier-cleanup.mjs"; +import { + assertAgrees, + buildElf, + checkTools, + commands, + elf2image, + execute, + exitOutput, + extract, + imageInfo, + inspect, + options, + source, + spell, + tool, + xtensa, +} from "./esp-image-tools.mjs"; + +let workspace; +const path = (name) => join(workspace, name); + +await withVerifierCleanup(async () => { + const { compilerVersion, esptoolVersion, xtensaVersion } = await checkTools(); + workspace = await mkdtemp(join(tmpdir(), "rea-esp-image-proof-")); + const report = { + esptool: esptoolVersion, + compiler: compilerVersion, + images: [], + extractions: [], + }; + const app = await buildElf("app.c", "app.ld", path("app.elf")); + const boot = await buildElf( + "bootloader.c", + "bootloader.ld", + path("boot.elf"), + ); + const esp8266 = await buildElf( + "bootloader.c", + "esp8266.ld", + path("esp8266.elf"), + ); + const appDescriptor = ["--elf-sha256-offset", "0xb0"]; + await elf2image("esp32c3", app, path("app.bin"), appDescriptor); + await elf2image("esp32c3", app, path("app-nodigest.bin"), [ + ...appDescriptor, + "--dont-append-digest", + ]); + await elf2image("esp32c3", app, path("app-pad.bin"), [ + ...appDescriptor, + "--secure-pad-v2", + ]); + await elf2image("esp32c3", app, path("app-ram.bin"), ["--ram-only-header"]); + await elf2image("esp32c3", app, path("app-filled.bin"), [ + ...appDescriptor, + "--pad-to-size", + "1MB", + ]); + await elf2image("esp32c3", boot, path("boot.bin")); + await tool("esptool", [ + "--chip", + "esp8266", + "elf2image", + "--version", + "1", + "-o", + path("esp8266-"), + esp8266, + ]); + + // Producer goldens: REA and esptool image-info agree field by field. Each + // class follows from the region app.ld or bootloader.ld places a section in; + // esptool adds the address-0 padding that aligns the flash text. + const appClasses = ["mapped", "loaded", "loaded", "not-loaded", "mapped"]; + for (const [name, classes] of [ + ["app.bin", appClasses], + ["app-nodigest.bin", appClasses], + ["app-pad.bin", [...appClasses, "not-loaded"]], + ["boot.bin", ["loaded", "loaded"]], + ]) { + const [oracle, rea] = await Promise.all([ + imageInfo(path(name)), + inspect(path(name)), + ]); + assertAgrees(name, oracle, rea); + assert.equal(rea.inventory.coverage.status, "complete", name); + assert.deepEqual( + rea.segments.map( + ({ format_metadata: segment }) => segment.classification, + ), + classes, + `${name} classes`, + ); + report.images.push({ + name, + role: rea.root.descriptor?.kind ?? "unknown", + segments: rea.segments.map( + ({ format_metadata: segment }) => + `${segment.index}:0x${segment.load_addr.toString(16)}:${segment.classification}`, + ), + checksum: rea.root.checksum.status, + sha256: rea.root.appended_sha256.status, + }); + } + const padded = await inspect(path("app-pad.bin")); + // --secure-pad-v2 ends the digest on a 64 KiB boundary with an address-0 + // padding segment, which ESP-IDF neither maps nor loads. + assert.equal(padded.root.image_end % 0x1_0000, 0); + assert.equal(padded.segments.at(-1).format_metadata.load_addr, 0); + + // Signed: blocks are observed; espsecure is the oracle for their key digests. + const generate = spell("generate_signing_key", "generate-signing-key"); + const sign = spell("sign_data", "sign-data"); + await tool("espsecure", [ + generate, + "--version", + "2", + "--scheme", + "rsa3072", + path("rsa.pem"), + ]); + for (const scheme of ["ecdsa256", "ecdsa384"]) + await tool("espsecure", [ + generate, + "--version", + "2", + "--scheme", + scheme, + path(`${scheme}.pem`), + ]); + await tool("espsecure", [ + sign, + "--version", + "2", + "--keyfile", + path("rsa.pem"), + "--output", + path("signed-rsa.bin"), + path("app-pad.bin"), + ]); + // Append one ECDSA P-256 and one P-384 block after the RSA block. + for (const [scheme, input, output] of [ + ["ecdsa256", "signed-rsa.bin", "signed-p256.bin"], + ["ecdsa384", "signed-p256.bin", "signed.bin"], + ]) + await tool("espsecure", [ + sign, + "--version", + "2", + spell("--append_signatures", "--append-signatures"), + "--keyfile", + path(`${scheme}.pem`), + "--output", + path(output), + path(input), + ]); + const signatureInfo = await tool("espsecure", [ + spell("signature_info_v2", "signature-info-v2"), + path("signed.bin"), + ]); + const keyDigests = [ + ...signatureInfo.matchAll( + /Public key digest for block (\d+): ((?:[0-9a-f]{2} ?){32})/gu, + ), + ].map((match) => match[2].replaceAll(" ", "")); + const signed = await inspect(path("signed.bin")); + assertAgrees("signed.bin", await imageInfo(path("signed.bin")), signed); + assert.equal(signed.root.signature_sector_offset, padded.root.image_end); + assert.deepEqual(signed.root.undescribed_trailing, []); + assert.equal(signed.inventory.coverage.status, "complete"); + assert.deepEqual( + signed.signatures.map(({ format_metadata: block }) => [ + block.index, + block.scheme, + block.digest_algorithm, + block.image_digest_matches, + block.public_key_sha256, + block.signature_status, + ]), + [ + [0, "rsa", "sha256", true, keyDigests[0], "unverified"], + [1, "ecdsa", "sha256", true, keyDigests[1], "unverified"], + [2, "ecdsa", "sha384", true, keyDigests[2], "unverified"], + ], + ); + report.images.push({ + name: "signed.bin", + signatures: signed.signatures.map( + ({ format_metadata: block }) => + `${block.scheme}:${block.public_key_sha256}`, + ), + }); + + // Extraction writes the producer's bytes and leaves the source unchanged. + const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex"); + // espsecure appends the signature sector to the --secure-pad-v2 image. + const sectorOffset = (await readFile(path("app-pad.bin"))).length; + for (const name of ["app.bin", "signed.bin"]) { + const bytes = await readFile(path(name)); + const digest = sha256(bytes); + const expected = new Map( + (await imageInfo(path(name))).segments.map( + ({ index, headerOffset, length }) => [ + `segment-${String(index).padStart(2, "0")}.bin`, + bytes.subarray(headerOffset + 8, headerOffset + 8 + length), + ], + ), + ); + if (name === "signed.bin") { + assert.equal(bytes.length, sectorOffset + 0x1000, "signature sector"); + for (const slot of [0, 1, 2]) + expected.set( + `signature-block-${String(slot)}.bin`, + bytes.subarray( + sectorOffset + slot * 1216, + sectorOffset + (slot + 1) * 1216, + ), + ); + } + const extraction = await extract(path(name), workspace); + assert.equal(extraction.containment_verified, true, name); + assert.equal(extraction.manifest.root_sha256, digest, `${name} source`); + assert.deepEqual( + extraction.artifacts.map(({ relative_path }) => relative_path).sort(), + [...expected.keys()].sort(), + `${name} entries`, + ); + for (const { + relative_path: entry, + sha256: written, + } of extraction.artifacts) { + const content = await readFile(join(extraction.output_root, entry)); + assert.ok(content.equals(expected.get(entry)), `${name} ${entry}`); + assert.equal(written, sha256(content), `${name} ${entry} digest`); + } + assert.equal(sha256(await readFile(path(name))), digest, `${name} source`); + report.extractions.push({ name, entries: extraction.artifacts.length }); + } + + // --ram-only-header counts only RAM segments; the rest is coverage. + const ram = await inspect(path("app-ram.bin")); + const ramOracle = await imageInfo(path("app-ram.bin")); + assertAgrees("app-ram.bin", ramOracle, ram); + const ramBytes = await readFile(path("app-ram.bin")); + assert.equal(ram.inventory.coverage.status, "unknown"); + assert.deepEqual(ram.root.undescribed_trailing, [ + { + offset: ram.root.image_end, + length: ramBytes.length - ram.root.image_end, + }, + ]); + report.images.push({ + name: "app-ram.bin", + undescribed_trailing: ram.root.undescribed_trailing, + }); + + // --pad-to-size fills to the size with 0xFF, which describes erased flash. + const filled = await inspect(path("app-filled.bin")); + assertAgrees( + "app-filled.bin", + await imageInfo(path("app-filled.bin")), + filled, + ); + assert.equal((await readFile(path("app-filled.bin"))).length, 0x10_0000); + assert.deepEqual(filled.root.undescribed_trailing, []); + assert.equal(filled.inventory.coverage.status, "complete"); + report.images.push({ name: "app-filled.bin", coverage: "complete" }); + + // Malformed cases derived from producer output. + const image = await readFile(path("app.bin")); + const noDigest = await readFile(path("app-nodigest.bin")); + const flipLast = (bytes) => { + const copy = Buffer.from(bytes); + copy[copy.length - 1] ^= 0x01; + return copy; + }; + await writeFile(path("bad-checksum.bin"), flipLast(noDigest)); + await writeFile(path("bad-sha256.bin"), flipLast(image)); + await writeFile(path("truncated.bin"), image.subarray(0, 0x1000)); + for (const [name, checksum, sha256] of [ + ["bad-checksum.bin", "invalid", "not-appended"], + ["bad-sha256.bin", "valid", "invalid"], + ]) { + const [oracle, rea] = await Promise.all([ + imageInfo(path(name)), + inspect(path(name)), + ]); + assertAgrees(name, oracle, rea); + assert.equal(rea.root.checksum.status, checksum, name); + assert.equal(rea.root.appended_sha256.status, sha256, name); + report.images.push({ name, checksum, sha256 }); + } + const truncatedOracle = await imageInfo(path("truncated.bin")).then( + () => "loaded", + exitOutput, + ); + assert.match(truncatedOracle, /End of file reading segment/u); + const truncated = await inspect(path("truncated.bin")); + assert.equal(truncated.inventory.manifest.root_format, "esp-image"); + assert.equal(truncated.inventory.coverage.status, "unknown"); + assert.equal(truncated.root.checksum.status, "unavailable"); + const truncatedSegment = truncated.segments.at(-1); + assert.ok(truncatedSegment); + assert.ok(Number.isSafeInteger(truncatedSegment.declared_size)); + assert.ok(truncatedSegment.source_location); + assert.equal( + truncatedSegment.source_location.offset + truncatedSegment.declared_size, + 0x1000, + ); + assert.ok( + truncatedSegment.format_metadata.data_len > truncatedSegment.declared_size, + ); + assert.ok(truncatedSegment.limitations.length > 0); + report.images.push({ name: "truncated.bin", coverage: "unknown" }); + + if (xtensa === undefined) + report.images.push({ + name: "app-esp32.bin", + skipped: "REA_XTENSA_ESP32_GCC is unset", + }); + else { + await execute( + xtensa, + [ + "-nostdlib", + "-static", + "-Os", + "-mlongcalls", + "-mtext-section-literals", + `-Wl,-T,${source("esp32-app.ld")}`, + "-Wl,--build-id=none", + source("app.c"), + "-o", + path("app-esp32.elf"), + ], + options, + ); + await elf2image( + "esp32", + path("app-esp32.elf"), + path("app-esp32.bin"), + appDescriptor, + ); + const [oracle, rea] = await Promise.all([ + imageInfo(path("app-esp32.bin")), + inspect(path("app-esp32.bin")), + ]); + assertAgrees("app-esp32.bin", oracle, rea); + assert.deepEqual(rea.root.chip, { + target: "esp32", + isa: "xtensa", + source: "ESP-IDF v6.1", + }); + assert.deepEqual( + rea.segments.map( + ({ format_metadata: segment }) => segment.classification, + ), + ["mapped", "loaded", "loaded", "not-loaded", "mapped"], + ); + report.images.push({ + name: "app-esp32.bin", + compiler: xtensaVersion, + segments: rea.segments.map( + ({ format_metadata: segment }) => + `${segment.index}:0x${segment.load_addr.toString(16)}:${segment.classification}`, + ), + }); + } + + // ESP8266 v3 (RTOS SDK) images begin with their IROM segment. + const esp8266App = await buildElf( + "app.c", + "esp8266-app.ld", + path("esp8266-app.elf"), + ); + await tool("esptool", [ + "--chip", + "esp8266", + "elf2image", + "--version", + "3", + "-o", + path("esp8266-v3.bin"), + esp8266App, + ]); + const v3Oracle = await imageInfo(path("esp8266-v3.bin"), "esp8266"); + assert.ok((v3Oracle.segments[0]?.address ?? 0) >= 0x4020_0000); + assert.match( + (await inspect(path("esp8266-v3.bin"))).refused ?? "", + /ESP8266 firmware images are not supported/u, + ); + report.images.push({ name: "esp8266-v3.bin", refused: true }); + + // ESP8266 v1: esptool falls back to ESP8266; REA refuses it by name. + const esp8266Image = path("esp8266-0x00000.bin"); + assert.equal((await imageInfo(esp8266Image)).detected, "ESP8266"); + const refused = await inspect(esp8266Image); + assert.match( + refused.refused ?? "", + /ESP8266 firmware images are not supported/u, + ); + report.images.push({ name: "esp8266-0x00000.bin", refused: true }); + console.log(JSON.stringify(report, null, 2)); +}, [ + // Remove outputs only after every owned command tree has stopped; a failed + // close keeps the workspace for inspection. + async () => { + await commands.close(); + if (workspace !== undefined) + await rm(workspace, { recursive: true, force: true }); + }, +]); diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.ts index f8ef0c4932..d9acf19f94 100644 --- a/src/application/BinaryTargetResolver.ts +++ b/src/application/BinaryTargetResolver.ts @@ -78,6 +78,10 @@ import { hasGodotPckHeader } from "../domain/godotPckFormat.js"; import { hasUImageMagic } from "../domain/uimageFormat.js"; import { hasUnityFsHeader } from "../domain/unityfsFormat.js"; import { detectHexRecordEncoding } from "../domain/firmware/hexRecords.js"; +import { + ESP8266_REFUSAL, + recognizeEspImage, +} from "../domain/firmware/espImage.js"; import { hasZipSignature, locateTrailingZipArchive, @@ -672,6 +676,12 @@ const detectArtifactFormat = async ( if (hasUImageMagic(magic)) return "uimage"; const hexRecords = detectHexRecordEncoding(magic); if (hexRecords !== undefined) return hexRecords; + const esp = recognizeEspImage(magic, initial.size); + if (esp === "esp8266") + throw new BinaryTargetError(path, ESP8266_REFUSAL, { + constraint: "file_format_rejected", + }); + if (esp === "esp-image") return "esp-image"; // Binary plists, including keyed archives, often use other extensions. if (hasBinaryPlistSignature(magic)) return "plist"; const named = namedArtifactFormat(lower); @@ -748,6 +758,7 @@ const isArchiveFormat = ( "pkg", "intel-hex", "srec", + "esp-image", ].includes(format); /** Whether the leading bytes begin any executable header family, valid or not. */ diff --git a/src/artifacts/ArtifactReader.ts b/src/artifacts/ArtifactReader.ts index ace8eff0d5..1d64f6cc4c 100644 --- a/src/artifacts/ArtifactReader.ts +++ b/src/artifacts/ArtifactReader.ts @@ -183,6 +183,7 @@ export interface ArtifactReader { | "node-sea" | "intel-hex" | "srec" + | "esp-image" | "file"; entries( signal?: AbortSignal, diff --git a/src/artifacts/EspImageArtifactReader.boundaries.test.ts b/src/artifacts/EspImageArtifactReader.boundaries.test.ts new file mode 100644 index 0000000000..cb7c122678 --- /dev/null +++ b/src/artifacts/EspImageArtifactReader.boundaries.test.ts @@ -0,0 +1,11 @@ +import { artifactReaderLifecycleCases } from "../../tests/fixtures/artifactReaderLifecycle.js"; +import { espImage } from "../../tests/fixtures/espImage.js"; +import { EspImageArtifactReader } from "./EspImageArtifactReader.js"; + +artifactReaderLifecycleCases({ + image: () => + espImage({ + segments: [{ address: 0x40380000, data: Buffer.alloc(8, 0x61) }], + }), + createReader: (path, admitted) => new EspImageArtifactReader(path, admitted), +}); diff --git a/src/artifacts/EspImageArtifactReader.test.ts b/src/artifacts/EspImageArtifactReader.test.ts new file mode 100644 index 0000000000..53ce6d91a1 --- /dev/null +++ b/src/artifacts/EspImageArtifactReader.test.ts @@ -0,0 +1,363 @@ +import { ESP_IMAGE_LIMITATIONS } from "../domain/firmware/espImageFacts.js"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it, vi } from "vitest"; + +import { EspImageArtifactReader } from "./EspImageArtifactReader.js"; +import { ArtifactResourceScope } from "./ArtifactResourceScope.js"; +import type { ArtifactEntry } from "./ArtifactReader.js"; +import { extractArtifact } from "./extraction/ArtifactExtraction.js"; +import * as espImageDecoding from "../domain/firmware/espImage.js"; +import { + ESP32C3_APP_SEGMENTS, + esp8266Image, + espImage, + signedEspImage, +} from "../../tests/fixtures/espImage.js"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { scanCanonicalArtifactInventory } from "../../tests/fixtures/artifactInventory.js"; +import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js"; + +const APP = espImage({ segments: ESP32C3_APP_SEGMENTS }); +// An erased slot between two blocks must not hide the later block. +const SIGNED = signedEspImage(APP, ["rsa", null, "ecdsa-p256"]); + +const writeImage = async (bytes: Uint8Array, name = "app.bin") => { + const path = join(await createTestTempDirectory("rea-esp-image-"), name); + await writeFile(path, bytes); + return path; +}; + +const list = async (reader: EspImageArtifactReader) => { + const entries: ArtifactEntry[] = []; + for await (const entry of reader.entries()) entries.push(entry); + return entries; +}; + +it("inventories segments and signature blocks with root and entry facts", async () => { + const snapshot = await scanCanonicalArtifactInventory( + await writeImage(SIGNED), + ); + expect(snapshot.manifest.root_format).toBe("esp-image"); + expect(snapshot.coverage).toEqual({ + scope: "container-entries", + status: "complete", + }); + expect(snapshot.limitations).toEqual([ESP_IMAGE_LIMITATIONS.inventoryScope]); + const root = snapshot.occurrences.find((row) => row.logical_path === "."); + expect(root).toMatchObject({ + artifact_kind: "container", + artifact_format: "esp-image", + format_metadata: { + kind: "esp-image", + descriptor: { kind: "app" }, + chip: { target: "esp32c3", isa: "riscv" }, + checksum: { status: "valid" }, + appended_sha256: { status: "valid" }, + signature_sector_offset: 4096, + undescribed_trailing: [], + }, + }); + expect( + snapshot.occurrences + .filter((row) => row.logical_path !== ".") + .map((row) => [ + row.logical_path, + row.declared_size, + row.source_location, + row.hash_status, + row.format_metadata?.kind, + ]), + ).toEqual([ + [ + "segment-00.bin", + 288, + { offset: 32, length: 288 }, + "verified", + "esp-image-segment", + ], + [ + "segment-01.bin", + 16, + { offset: 328, length: 16 }, + "verified", + "esp-image-segment", + ], + [ + "segment-02.bin", + 32, + { offset: 352, length: 32 }, + "verified", + "esp-image-segment", + ], + [ + "segment-03.bin", + 64, + { offset: 392, length: 64 }, + "verified", + "esp-image-segment", + ], + [ + "signature-block-0.bin", + 1216, + { offset: 4096, length: 1216 }, + "verified", + "esp-image-signature-block", + ], + [ + "signature-block-2.bin", + 1216, + { offset: 6528, length: 1216 }, + "verified", + "esp-image-signature-block", + ], + ]); + expect( + snapshot.occurrences.find((row) => row.logical_path === "segment-03.bin") + ?.format_metadata, + ).toEqual({ + kind: "esp-image-segment", + index: 3, + load_addr: 0x4200_0020, + data_len: 64, + header_offset: 384, + classification: "mapped", + }); +}); + +it("reports truncation and undescribed trailing bytes as coverage", async () => { + const bytes = APP.subarray(0, 360); + const truncated = await scanCanonicalArtifactInventory( + await writeImage(bytes), + ); + expect(truncated.manifest.root_format).toBe("esp-image"); + expect(truncated.coverage.status).toBe("unknown"); + const decoder = espImageDecoding.createEspImageDecoder(bytes.length); + decoder.write(bytes); + const decoded = decoder.finish(); + if (!decoded.ok) + throw new Error("Expected partial fixture facts", { cause: decoded.error }); + expect(truncated.limitations).toEqual( + expect.arrayContaining([ + ESP_IMAGE_LIMITATIONS.inventoryScope, + ...decoded.value.limitations, + ]), + ); + const segment = truncated.occurrences.find( + (row) => row.logical_path === "segment-02.bin", + ); + expect(segment).toMatchObject({ + declared_size: 8, + limitations: [ESP_IMAGE_LIMITATIONS.segmentEntryTruncated(2, 32, 8)], + source_location: { offset: 352, length: 8 }, + format_metadata: { + kind: "esp-image-segment", + index: 2, + header_offset: 344, + data_len: 32, + }, + }); + const extended = await scanCanonicalArtifactInventory( + await writeImage(Buffer.concat([APP, Buffer.from("trailing data")])), + ); + expect(extended.coverage.status).toBe("unknown"); + expect( + extended.occurrences.find((row) => row.logical_path === ".") + ?.format_metadata, + ).toMatchObject({ + undescribed_trailing: [{ offset: APP.length, length: 13 }], + }); +}); + +it("extracts byte-exact segments that re-inventory as raw files", async () => { + const directory = await createTestTempDirectory("rea-esp-extract-"); + const path = join(directory, "signed.bin"); + await writeFile(path, SIGNED); + const resourceScope = new ArtifactResourceScope(); + try { + const extracted = await extractArtifact({ + inputPath: path, + inputFormat: "esp-image", + outputRoot: join(directory, "out"), + environment: process.env, + integrityPolicy: "fail", + resourceScope, + }); + expect(extracted.containment_verified).toBe(true); + expect(extracted.artifacts.map((row) => row.relative_path).sort()).toEqual([ + "segment-00.bin", + "segment-01.bin", + "segment-02.bin", + "segment-03.bin", + "signature-block-0.bin", + "signature-block-2.bin", + ]); + for (const [name, offset, length] of [ + ["segment-00.bin", 32, 288], + ["segment-03.bin", 392, 64], + ["signature-block-0.bin", 4096, 1216], + ["signature-block-2.bin", 6528, 1216], + ] as const) + expect( + Buffer.compare( + await readFile(join(extracted.output_root, name)), + SIGNED.subarray(offset, offset + length), + ), + ).toBe(0); + const reinventoried = await scanCanonicalArtifactInventory( + extracted.output_root, + ); + expect(reinventoried.manifest.root_format).toBe("directory"); + } finally { + await resourceScope.close(); + } +}); + +it("refuses an ESP8266 image with its specific reason", async () => { + const reader = new EspImageArtifactReader( + await writeImage(esp8266Image(), "esp8266.bin"), + ); + try { + await expect(list(reader)).rejects.toMatchObject({ + reason: "unavailable", + message: espImageDecoding.ESP8266_REFUSAL, + }); + } finally { + await reader.close(); + } +}); + +it("rejects a changed carrier during extraction preparation", async () => { + const reader = new EspImageArtifactReader(await writeImage(APP)); + try { + await expect(reader.prepareContainer("0".repeat(64))).rejects.toMatchObject( + { reason: "integrity" }, + ); + } finally { + await reader.close(); + } +}); + +it("rejects forged entries and changed carrier identity, then closes idempotently", async () => { + const path = await writeImage(APP); + const reader = new EspImageArtifactReader(path); + try { + const [entry] = await list(reader); + if (entry === undefined) throw new Error("Missing segment"); + await expect( + reader.open({ ...entry, declaredSize: 1 }), + ).rejects.toMatchObject({ reason: "integrity" }); + await expect( + reader.open({ ...entry, byteOffset: 0 }), + ).rejects.toMatchObject({ reason: "integrity" }); + await expect( + reader.open({ ...entry, adapterKey: "esp-image:segment:9" }), + ).rejects.toMatchObject({ reason: "integrity" }); + await writeFile(path, APP.subarray(0, 300)); + await expect(reader.open(entry)).rejects.toMatchObject({ + reason: "integrity", + }); + } finally { + await reader.close(); + await reader.close(); + } +}); + +it("settles an active entry stream when closed", async () => { + const reader = new EspImageArtifactReader(await writeImage(APP)); + try { + const [entry] = await list(reader); + if (entry === undefined) throw new Error("Missing segment"); + const stream = await reader.open(entry); + await reader.close(); + expect(stream.destroyed).toBe(true); + await expect( + reader.entries()[Symbol.asyncIterator]().next(), + ).rejects.toMatchObject({ reason: "unavailable" }); + } finally { + await reader.close(); + } +}); + +it("honors cancellation before admitting a file with the exact reason", async () => { + const reader = new EspImageArtifactReader(await writeImage(APP)); + const controller = new AbortController(); + const reason = { stage: "before admission" }; + controller.abort(reason); + try { + await expect( + reader.entries(controller.signal)[Symbol.asyncIterator]().next(), + ).rejects.toMatchObject({ reason: "cancelled", cause: reason }); + } finally { + await reader.close(); + } +}); + +it.each(["cancel", "close"] as const)( + "settles decoding when the caller requests %s", + async (action) => { + // Several MiB of segment data take many chunked reads to decode. + const large = espImage({ + segments: [{ address: 0x4038_0000, data: Buffer.alloc(8 << 20, 0x5a) }], + }); + const reader = new EspImageArtifactReader(await writeImage(large)); + const controller = new AbortController(); + const reason = { stage: "decoding" }; + let completion: Promise | undefined; + let pending: Promise | undefined; + // The decoder acknowledges its first chunk, so the request arrives while + // decoding is active rather than while the file is still being admitted. + let decoded = 0; + let acknowledge = (): void => undefined; + const decoding = new Promise((resolve) => { + acknowledge = resolve; + }); + const create = espImageDecoding.createEspImageDecoder; + const observer = vi + .spyOn(espImageDecoding, "createEspImageDecoder") + .mockImplementation((size) => { + const decoder = create(size); + return { + write: (chunk) => { + decoder.write(chunk); + decoded += 1; + acknowledge(); + }, + get satisfied() { + return decoder.satisfied; + }, + finish: () => decoder.finish(), + }; + }); + try { + pending = reader + .entries(controller.signal) + [Symbol.asyncIterator]() + .next(); + await awaitAcknowledgement(decoding, pending, "ESP image decoding"); + const requestedAt = decoded; + if (action === "cancel") controller.abort(reason); + else completion = reader.close(); + await expect(pending).rejects.toMatchObject( + action === "cancel" + ? { reason: "cancelled", cause: reason } + : { reason: "unavailable" }, + ); + await completion; + // The request stops decoding; no later chunk reaches the decoder. + expect(decoded).toBe(requestedAt); + await reader.close(); + await expect( + reader.entries()[Symbol.asyncIterator]().next(), + ).rejects.toMatchObject({ reason: "unavailable" }); + } finally { + try { + await reader.close(); + } finally { + await pending?.catch(() => undefined); + observer.mockRestore(); + } + } + }, +); diff --git a/src/artifacts/EspImageArtifactReader.ts b/src/artifacts/EspImageArtifactReader.ts new file mode 100644 index 0000000000..b23625ff06 --- /dev/null +++ b/src/artifacts/EspImageArtifactReader.ts @@ -0,0 +1,311 @@ +import { Readable, Transform, pipeline } from "node:stream"; + +import type { + EspImageMetadata, + EspImageSegmentMetadata, + EspImageSignatureBlockMetadata, +} from "../domain/artifactFormatMetadata.js"; +import { + createEspImageDecoder, + type EspImageFailure, +} from "../domain/firmware/espImage.js"; +import { + ESP_IMAGE_LIMITATIONS, + type EspImage, +} from "../domain/firmware/espImageFacts.js"; +import { ESP_SIGNATURE_BLOCK_BYTES } from "../domain/firmware/espImageStructures.js"; +import { + ArtifactReaderFailure, + copyArtifactEntry, + sameArtifactEntry, + type ArtifactEntry, + type ArtifactReader, + type ArtifactReaderTraversal, + type ArtifactTraversalOptions, +} from "./ArtifactReader.js"; +import { ArtifactResourceScope } from "./ArtifactResourceScope.js"; +import { assertArtifactReadActive } from "./ArtifactFileOperation.js"; +import { ArtifactFileSource } from "./ArtifactFileSource.js"; +import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js"; +import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js"; + +const LABEL = "ESP image"; +type EspImageEntry = ArtifactEntry & { + readonly byteOffset: number; + readonly declaredSize: number; + readonly formatMetadata: + | EspImageSegmentMetadata + | EspImageSignatureBlockMetadata; +}; + +/** + * Inventory one ESP32-family image as its header-declared segments and + * Secure Boot v2 signature blocks. Decoding reads the file once; each entry + * is a positional read of the carrier, rechecked against its identity. + */ +export class EspImageArtifactReader implements ArtifactReader { + readonly format = "esp-image" as const; + readonly #scope = new ArtifactResourceScope(); + readonly #entries = new Map(); + readonly #streams = new Set(); + readonly #source: ArtifactFileSource; + #prepared: Promise | undefined; + #metadata: EspImageMetadata | undefined; + #coverage: ArtifactReaderTraversal | undefined; + #closing = false; + + constructor(path: string, admitted?: StableRegularFileDescriptor) { + this.#source = new ArtifactFileSource(path, LABEL, admitted); + } + + formatMetadata(): EspImageMetadata | undefined { + return this.#metadata === undefined + ? undefined + : structuredClone(this.#metadata); + } + + async prepareContainer( + expectedSha256: string, + signal?: AbortSignal, + ): Promise { + await this.#scope.run(async () => { + this.#assertOpen(signal); + await this.#source.bindSha256(expectedSha256, signal); + }); + } + + async *entries( + signal?: AbortSignal, + options?: ArtifactTraversalOptions, + ): AsyncIterable { + await this.#scope.run(async () => { + this.#assertOpen(signal); + this.#prepared ??= this.#prepare(signal); + await this.#prepared; + await this.#source.read(signal); + }); + this.#assertOpen(signal); + if (this.#coverage !== undefined) + options?.onCoverage?.({ + ...this.#coverage, + limitations: [...this.#coverage.limitations], + }); + for (const entry of this.#entries.values()) { + this.#assertOpen(signal); + yield copyArtifactEntry(entry); + } + this.#assertOpen(signal); + } + + async open(entry: ArtifactEntry, signal?: AbortSignal): Promise { + return this.#scope.run(async () => { + this.#assertOpen(signal); + const stored = this.#entries.get(entry.adapterKey); + if (stored === undefined || !sameArtifactEntry(entry, stored)) + throw new ArtifactReaderFailure( + "integrity", + `${LABEL} entry does not match its owning inventory`, + ); + const { handle } = await this.#source.read(signal); + this.#assertOpen(signal); + const source = + stored.declaredSize === 0 + ? Readable.from([]) + : readFileHandleChunks(handle, { + start: stored.byteOffset, + end: stored.byteOffset + stored.declaredSize - 1, + signal, + cancellationFailure: (reason) => + new ArtifactReaderFailure( + "cancelled", + `${LABEL} read cancelled`, + { cause: reason }, + ), + }); + const output = this.#entryStream(stored, signal); + // Retirement can destroy the stream before its consumer listens. + output.on("error", () => undefined); + this.#streams.add(output); + let producerComplete = false; + const onAbort = (): void => { + // Preserve active read/validation errors; after production, the caller + // still owns buffered bytes and cancellation must retire that result. + if (!producerComplete || output.destroyed) return; + output.destroy( + new ArtifactReaderFailure("cancelled", `${LABEL} read cancelled`, { + cause: signal?.reason, + }), + ); + }; + const retired = new Promise((resolve) => { + output.once("close", () => { + this.#streams.delete(output); + signal?.removeEventListener("abort", onAbort); + resolve(); + }); + }); + signal?.addEventListener("abort", onAbort, { once: true }); + if (signal?.aborted) onAbort(); + void this.#scope + .run(async () => { + await Promise.all([ + new Promise((resolve) => { + pipeline(source, output, () => { + producerComplete = true; + if (signal?.aborted) onAbort(); + resolve(); + }); + }), + retired, + ]); + }) + .catch((cause: unknown) => + output.destroy( + cause instanceof Error + ? cause + : new Error(`${LABEL} stream ownership failed`, { cause }), + ), + ); + return output; + }); + } + + async close(): Promise { + this.#closing = true; + for (const stream of this.#streams) + stream.destroy( + new ArtifactReaderFailure("unavailable", `${LABEL} reader closed`), + ); + await this.#scope.close(); + await this.#source.close(); + this.#entries.clear(); + } + + provenance(): readonly [] { + return []; + } + + #assertOpen(signal?: AbortSignal): void { + assertArtifactReadActive(signal); + if (this.#closing) + throw new ArtifactReaderFailure( + "unavailable", + `${LABEL} reader is closed`, + ); + } + + async #prepare(signal?: AbortSignal): Promise { + const image = await this.#decode(signal); + for (const segment of image.segments) { + const { index, data_len } = segment.metadata; + const entry = { + path: `segment-${String(index).padStart(2, "0")}.bin`, + kind: "file" as const, + declaredSize: segment.availableLength, + compressedSize: null, + executable: false, + encrypted: false, + byteOffset: segment.dataOffset, + declaredIntegrity: null, + unpacked: false, + limitations: + segment.availableLength < data_len + ? [ + ESP_IMAGE_LIMITATIONS.segmentEntryTruncated( + index, + data_len, + segment.availableLength, + ), + ] + : [], + adapterKey: `esp-image:segment:${String(index)}`, + formatMetadata: { ...segment.metadata }, + } satisfies EspImageEntry; + this.#entries.set(entry.adapterKey, entry); + } + for (const block of image.signatureBlocks) { + const entry = { + path: `signature-block-${String(block.metadata.index)}.bin`, + kind: "file" as const, + declaredSize: ESP_SIGNATURE_BLOCK_BYTES, + compressedSize: null, + executable: false, + encrypted: false, + byteOffset: block.offset, + declaredIntegrity: null, + unpacked: false, + limitations: [], + adapterKey: `esp-image:signature-block:${String(block.metadata.index)}`, + formatMetadata: { ...block.metadata }, + } satisfies EspImageEntry; + this.#entries.set(entry.adapterKey, entry); + } + this.#metadata = image.metadata; + if (image.coverage === "unknown" || image.limitations.length > 0) + this.#coverage = { + status: image.coverage, + limitations: [...image.limitations], + }; + } + + /** Stream the carrier once, stopping when later bytes cannot matter. */ + async #decode(signal?: AbortSignal): Promise { + const { handle, initial } = await this.#source.read(signal); + const decoder = createEspImageDecoder(initial.size); + if (initial.size > 0) + for await (const chunk of readFileHandleChunks(handle, { + end: initial.size - 1, + signal, + cancellationFailure: (reason) => + new ArtifactReaderFailure("cancelled", `${LABEL} read cancelled`, { + cause: reason, + }), + }) as AsyncIterable) { + this.#assertOpen(signal); + decoder.write(chunk); + if (decoder.satisfied) break; + } + // A changed carrier is an identity failure, not a malformed image. + await this.#source.read(signal); + const image = decoder.finish(); + if (!image.ok) throw decodeFailure(image.error); + return image.value; + } + + #entryStream(entry: EspImageEntry, signal?: AbortSignal): Transform { + let bytes = 0; + const check = async (): Promise => { + if (bytes !== entry.declaredSize) + throw new ArtifactReaderFailure( + "integrity", + `${LABEL} entry ended early: ${entry.path}`, + ); + await this.#source.read(signal); + }; + return new Transform({ + transform(chunk: Buffer, _encoding, callback) { + bytes += chunk.length; + callback(null, chunk); + }, + flush(callback) { + void check().then( + () => callback(), + (cause: unknown) => + callback( + cause instanceof Error + ? cause + : new Error(`${LABEL} read failed`, { cause }), + ), + ); + }, + }); + } +} + +const decodeFailure = (failure: EspImageFailure): ArtifactReaderFailure => + failure.kind === "unsupported" + ? new ArtifactReaderFailure("unavailable", failure.message) + : new ArtifactReaderFailure( + "format", + `${LABEL} is malformed: ${failure.message}`, + ); diff --git a/src/artifacts/UImageArtifactReader.boundaries.test.ts b/src/artifacts/UImageArtifactReader.boundaries.test.ts index 4ce08a8fed..d4a711dbc3 100644 --- a/src/artifacts/UImageArtifactReader.boundaries.test.ts +++ b/src/artifacts/UImageArtifactReader.boundaries.test.ts @@ -1,14 +1,10 @@ +import { artifactReaderLifecycleCases } from "../../tests/fixtures/artifactReaderLifecycle.js"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { buffer } from "node:stream/consumers"; -import { finished } from "node:stream/promises"; -import { setImmediate } from "node:timers/promises"; import { expect, it, vi } from "vitest"; import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; -import { createDeferred } from "../../tests/fixtures/binarySession.js"; -import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js"; import { openRegularFile } from "../filesystem/RegularFile.js"; -import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; import { UImageArtifactReader } from "./UImageArtifactReader.js"; import { buildUImage, buildMultiPayload } from "../../tests/fixtures/uimage.js"; import { @@ -122,207 +118,6 @@ it("admits a multi-image count before scanning the remaining payload", async () } }); -it("does not complete paused inventory after its reader closes", async () => { - const reader = new UImageArtifactReader( - await fixture( - buildUImage({ - type: 4, - payload: buildMultiPayload([Buffer.from("a"), Buffer.from("b")]), - }), - ), - ); - const iterator = reader.entries()[Symbol.asyncIterator](); - try { - expect((await iterator.next()).done).toBe(false); - await reader.close(); - await expect(iterator.next()).rejects.toMatchObject({ - reason: "unavailable", - }); - } finally { - await reader.close(); - } -}); - -it.each(["cancel", "close"] as const)( - "retires fully buffered, unconsumed member bytes on %s", - async (action) => { - const reader = new UImageArtifactReader(await fixture(buildUImage())); - const controller = new AbortController(); - const reason = new Error("cancel buffered member"); - try { - const [entry] = await entries(reader); - const stream = await reader.open(entry!, controller.signal); - await finished(stream, { readable: false, cleanup: true }); - await setImmediate(); - expect(stream.readableLength).toBeGreaterThan(0); - if (action === "cancel") controller.abort(reason); - else await reader.close(); - await expect(buffer(stream)).rejects.toMatchObject( - action === "cancel" - ? { reason: "cancelled", cause: reason } - : { reason: "unavailable" }, - ); - expect(stream.destroyed).toBe(true); - } finally { - await reader.close(); - } - }, -); - -it("joins a member's outstanding positional read before descriptor closure", async () => { - const path = await fixture(buildUImage()); - const admitted = await openRegularFile(path, { symlinks: "reject" }); - const reader = new UImageArtifactReader(path, admitted); - const reached = createDeferred(), - release = createDeferred(); - const nativeRead = admitted.handle.read.bind(admitted.handle); - const reads = vi.spyOn(admitted.handle, "read"); - let outcome: Promise | undefined; - let closing: Promise | undefined; - try { - const [entry] = await entries(reader); - reads.mockImplementation((async (...args: unknown[]) => { - reached.resolve(); - await release.promise; - return Reflect.apply(nativeRead, admitted.handle, args); - }) as typeof admitted.handle.read); - const stream = await reader.open(entry!); - outcome = buffer(stream).then( - () => ({ ok: true }), - (cause: unknown) => ({ ok: false, cause }), - ); - await awaitAcknowledgement( - reached.promise, - outcome, - "member positional read", - ); - let closed = false; - closing = reader.close().then(() => { - closed = true; - }); - await setImmediate(); - expect(closed).toBe(false); - release.resolve(); - await closing; - expect(stream.closed).toBe(true); - } finally { - release.resolve(); - await outcome; - await closing; - await reader.close(); - reads.mockRestore(); - await admitted.handle.close(); - } -}); - -it("retains failed descriptor cleanup for retry", async () => { - const reader = new UImageArtifactReader(await fixture(buildUImage())); - const [entry] = await entries(reader); - const stream = await reader.open(entry!); - const failure = new Error("uImage close denied"); - const close = vi - .spyOn(OwnedFileHandle.prototype, "close") - .mockRejectedValueOnce(failure); - try { - await expect(reader.close()).rejects.toBe(failure); - await expect(reader.open(entry!)).rejects.toMatchObject({ - reason: "unavailable", - }); - expect(stream.closed).toBe(true); - await reader.close(); - expect(close).toHaveBeenCalledTimes(2); - } finally { - close.mockRestore(); - await reader.close(); - } -}); - -it.each(["cancel", "cancel-with-validation-error", "close"] as const)( - "joins final freshness validation during %s", - async (action) => { - const path = await fixture(buildUImage()); - const admitted = await openRegularFile(path, { symlinks: "reject" }); - const reader = new UImageArtifactReader(path, admitted); - const reached = createDeferred(), - release = createDeferred(); - const nativeStat = admitted.handle.stat.bind(admitted.handle); - const nativeRead = admitted.handle.read.bind(admitted.handle); - const reads = vi.spyOn(admitted.handle, "read"); - const stats = vi.spyOn(admitted.handle, "stat"); - const controller = new AbortController(), - reason = new Error("cancel final validation"); - const validationFailure = new Error("final descriptor stat failed"); - let outcome: Promise | undefined; - let closing: Promise | undefined; - try { - const [entry] = await entries(reader); - const memberEnd = BigInt(entry!.byteOffset! + entry!.declaredSize!); - let memberReadComplete = false; - reads.mockImplementation((async (...args: unknown[]) => { - const result = (await Reflect.apply( - nativeRead, - admitted.handle, - args, - )) as { - bytesRead: number; - buffer: Buffer; - }; - if (typeof args[3] === "bigint") - memberReadComplete = args[3] + BigInt(result.bytesRead) === memberEnd; - return result; - }) as typeof admitted.handle.read); - stats.mockImplementation((async (...args: unknown[]) => { - const result = await Reflect.apply(nativeStat, admitted.handle, args); - if (memberReadComplete) { - memberReadComplete = false; - reached.resolve(); - await release.promise; - if (action === "cancel-with-validation-error") - throw validationFailure; - } - return result; - }) as typeof admitted.handle.stat); - const stream = await reader.open(entry!, controller.signal); - outcome = buffer(stream).then( - () => ({ ok: true }) as const, - (cause: unknown) => ({ ok: false, cause }) as const, - ); - await awaitAcknowledgement( - reached.promise, - outcome, - "post-read freshness validation", - ); - let closed = false; - if (action !== "close") controller.abort(reason); - else - closing = reader.close().then(() => { - closed = true; - }); - await setImmediate(); - expect(closed).toBe(false); - release.resolve(); - expect(await outcome).toMatchObject({ - ok: false, - cause: - action === "cancel-with-validation-error" - ? validationFailure - : action === "cancel" - ? { reason: "cancelled", cause: reason } - : { reason: "unavailable" }, - }); - await closing; - } finally { - release.resolve(); - await outcome; - await closing; - await reader.close(); - stats.mockRestore(); - reads.mockRestore(); - await admitted.handle.close(); - } - }, -); - it("rejects a source changed after its header and CRC were prepared", async () => { const bytes = buildUImage(); const path = await fixture(bytes), @@ -361,3 +156,12 @@ it("checks a multi-chunk stored payload incrementally and preserves read cancell await reader.close(); } }); + +artifactReaderLifecycleCases({ + image: () => + buildUImage({ + type: 4, + payload: buildMultiPayload([Buffer.from("a"), Buffer.from("b")]), + }), + createReader: (path, admitted) => new UImageArtifactReader(path, admitted), +}); diff --git a/src/artifacts/createArtifactReader.ts b/src/artifacts/createArtifactReader.ts index 44e97a6193..67a572fb8e 100644 --- a/src/artifacts/createArtifactReader.ts +++ b/src/artifacts/createArtifactReader.ts @@ -6,6 +6,7 @@ import { AsarArtifactReader } from "./AsarArtifactReader.js"; import type { ArtifactReader } from "./ArtifactReader.js"; import { HexRecordArtifactReader } from "./HexRecordArtifactReader.js"; import { DirectoryArtifactReader } from "./DirectoryArtifactReader.js"; +import { EspImageArtifactReader } from "./EspImageArtifactReader.js"; import { GodotPckArtifactReader } from "./GodotPckArtifactReader.js"; import { MachOSliceArtifactReader } from "./MachOSliceArtifactReader.js"; import { NativeDmgArtifactReader } from "./NativeDmgArtifactReader.js"; @@ -34,6 +35,7 @@ export type ArtifactReaderFormat = | "srec" | "mach-o-universal" | "dmg" + | "esp-image" | ZipPackageFormat; interface ArtifactReaderConstruction { @@ -128,6 +130,8 @@ export function createArtifactReader( ); case "dmg": return new NativeDmgArtifactReader(path, options.environment); + case "esp-image": + return new EspImageArtifactReader(path, rootSource); default: return undefined; } diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts index 9caa6883fe..85a488b1e3 100644 --- a/src/artifacts/extraction/ArtifactExtraction.ts +++ b/src/artifacts/extraction/ArtifactExtraction.ts @@ -590,6 +590,7 @@ const requireExtractableFormat = async ( format !== "uimage" && format !== "intel-hex" && format !== "srec" && + format !== "esp-image" && format !== "mach-o-universal" && !isZipPackageFormat(format) ) { diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts index dd4f60444f..ba001a6d75 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts @@ -19,6 +19,11 @@ import { pe } from "../../domain/binaryTarget.fixture.js"; import type { ArtifactEntry } from "../ArtifactReader.js"; import { classifyArtifactBytesAtPath } from "../../../tests/fixtures/artifactClassification.js"; +import { + ESP32C3_APP_SEGMENTS, + esp8266Image, + espImage, +} from "../../../tests/fixtures/espImage.js"; it("joins only observed PE signature bytes across stream chunks", () => { const bytes = pe(0x8664, 8190); @@ -473,6 +478,19 @@ it("classifies nested HEX and S-record content as containers by bytes", () => { ).toEqual({ kind: "resource", format: "file" }); }); +it("classifies nested ESP images as containers by bytes", () => { + const image = espImage({ segments: ESP32C3_APP_SEGMENTS }); + expect(classifyArtifactBytesAtPath("ota/update.dat", image)).toEqual({ + kind: "container", + format: "esp-image", + }); + // An unsupported ESP8266 image is a plain file, not a container. + expect(classifyArtifactBytesAtPath("esp8266.bin", esp8266Image())).toEqual({ + kind: "resource", + format: "file", + }); +}); + describe("FAT prefix observation coverage", () => { it("does not invent an observation for a header beyond the retained prefix", () => { const bytes = Buffer.alloc(8_224); diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.ts b/src/artifacts/inventory/ArtifactGraphConstruction.ts index 1be664a8a8..685ec8f741 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.ts @@ -2,6 +2,7 @@ import { hasGodotPckHeader } from "../../domain/godotPckFormat.js"; import { hasUImageMagic } from "../../domain/uimageFormat.js"; import { hasUnityFsHeader } from "../../domain/unityfsFormat.js"; import { detectHexRecordEncoding } from "../../domain/firmware/hexRecords.js"; +import { recognizeEspImage } from "../../domain/firmware/espImage.js"; import type { ArtifactClassificationEvidence } from "../ArtifactClassificationProbe.js"; import { asarHeaderAdmissionFailure } from "../AsarHeader.js"; import { hasDestinationPath } from "../ArtifactPaths.js"; @@ -289,6 +290,7 @@ export const rootOccurrenceFor = ( "node-sea", "intel-hex", "srec", + "esp-image", ].includes(metadata.format) ? "container" : artifactRoleForFormat(metadata.path, metadata.format).kind, @@ -585,7 +587,8 @@ const artifactRoleForFormat = ( format === "unityfs" || format === "uimage" || format === "intel-hex" || - format === "srec" + format === "srec" || + format === "esp-image" ) return { kind: "container", format }; return { @@ -608,6 +611,7 @@ export const classifyArtifactBytes = ( if (hasUImageMagic(prefix)) return "uimage"; const hexRecords = detectHexRecordEncoding(prefix); if (hexRecords !== undefined) return hexRecords; + if (recognizeEspImage(prefix, fileSize) === "esp-image") return "esp-image"; if (hasBinaryPlistSignature(prefix)) return "plist"; if (hasXmlPlistSignature(prefix)) return "plist"; const fatFormat = classifyFatArtifact(evidence, fileSize); diff --git a/src/artifacts/inventory/classify.ts b/src/artifacts/inventory/classify.ts index 5a47196c97..a51b95aad9 100644 --- a/src/artifacts/inventory/classify.ts +++ b/src/artifacts/inventory/classify.ts @@ -170,7 +170,8 @@ export const classifyAndHashRootForInventory = async ( format === "pyinstaller" || format === "node-sea" || format === "intel-hex" || - format === "srec"; + format === "srec" || + format === "esp-image"; outcome = { kind: "completed", value: { diff --git a/src/artifacts/inventory/reader.ts b/src/artifacts/inventory/reader.ts index ec312fb73c..a0cf09997f 100644 --- a/src/artifacts/inventory/reader.ts +++ b/src/artifacts/inventory/reader.ts @@ -16,6 +16,7 @@ import { ZIP_NON_ENTRY_LIMITATION, } from "../../domain/zipPackageFormat.js"; import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js"; +import { ESP_IMAGE_LIMITATIONS } from "../../domain/firmware/espImageFacts.js"; export const createReader = ( path: string, @@ -79,6 +80,9 @@ export const inventoryCoverage = ( "Intel HEX/S-record inventory decodes data records into address-contiguous regions. It does not establish CPU, byte order, memory map, or that a declared start address is executable; gaps are unwritten, not zero.", ] : []), + ...(reader.format === "esp-image" + ? [ESP_IMAGE_LIMITATIONS.inventoryScope] + : []), ...(reader.format === "uimage" ? [ "uImage inventory establishes the 64-byte legacy header facts and stored payload bytes. Compressed payloads are not decompressed, and the image is not executed.", @@ -111,7 +115,8 @@ export const inventoryCoverage = ( format === "uimage" || format === "node-sea" || format === "intel-hex" || - format === "srec" + format === "srec" || + format === "esp-image" ) return { coverage: { scope: "container-entries", status: "unknown" }, diff --git a/src/domain/analysisSnapshot.ts b/src/domain/analysisSnapshot.ts index c805677ad5..b470aa9067 100644 --- a/src/domain/analysisSnapshot.ts +++ b/src/domain/analysisSnapshot.ts @@ -79,6 +79,7 @@ const formatSchema = z.enum([ "pkg", "intel-hex", "srec", + "esp-image", "plist", "javascript", "source-map", diff --git a/src/domain/analysisView/artifactInventorySummaryProjection.ts b/src/domain/analysisView/artifactInventorySummaryProjection.ts index cddfa2c73c..e8cd4b54ba 100644 --- a/src/domain/analysisView/artifactInventorySummaryProjection.ts +++ b/src/domain/analysisView/artifactInventorySummaryProjection.ts @@ -18,7 +18,8 @@ export const artifactInventorySummaryProjectionIds = ( format === "godot-pck" || format === "unityfs" || format === "pyinstaller" || - format === "node-sea" + format === "node-sea" || + format === "esp-image" ) return [current.evidence_id]; const { view_digest: _digest, ...projection } = result; diff --git a/src/domain/analysisView/artifactInventoryView.ts b/src/domain/analysisView/artifactInventoryView.ts index 3dfdfdc9ee..2b35813ccb 100644 --- a/src/domain/analysisView/artifactInventoryView.ts +++ b/src/domain/analysisView/artifactInventoryView.ts @@ -269,7 +269,8 @@ export const projectArtifactInventoryView = ( inventory.manifest.root_format === "pyinstaller" || inventory.manifest.root_format === "node-sea" || inventory.manifest.root_format === "intel-hex" || - inventory.manifest.root_format === "srec"; + inventory.manifest.root_format === "srec" || + inventory.manifest.root_format === "esp-image"; const counts = inventoryCounts(source); const limitations = uniqueSorted([ ...parent.limitations, diff --git a/src/domain/artifactComparison.ts b/src/domain/artifactComparison.ts index 78fdedf3e8..d8fafc7356 100644 --- a/src/domain/artifactComparison.ts +++ b/src/domain/artifactComparison.ts @@ -15,6 +15,7 @@ import { type InventorySet, } from "./artifactInventoryEvidence.js"; import { + espImageSegmentDeclaration, nodeSeaEntryDeclaration, pyInstallerEntryDeclaration, } from "./artifactFormatMetadata.js"; @@ -414,8 +415,7 @@ const classifyPath = (input: ClassifyPathInput): ArtifactChange | null => { if (declaration.handled) return declaration.change; const availability = classifyAvailability(input, base); if (availability !== null) - return input.metadata.asarPacking === "changed" || - input.metadata.nodeSea === "changed" + return declarationChanged(input.metadata) ? { ...base, classification: "changed", @@ -447,9 +447,16 @@ type MetadataComparison = "unchanged" | "changed" | "unknown"; interface OccurrenceMetadata { readonly asarPacking: MetadataComparison; readonly nodeSea: MetadataComparison; + readonly espImageSegment: MetadataComparison; readonly pyInstaller: MetadataComparison; } +/** A declaration that differs even when content is unavailable. */ +const declarationChanged = (metadata: OccurrenceMetadata): boolean => + metadata.asarPacking === "changed" || + metadata.nodeSea === "changed" || + metadata.espImageSegment === "changed"; + const compareOptionalMetadata = ( left: T | undefined, right: T | undefined, @@ -481,6 +488,15 @@ const occurrenceMetadata = (input: OccurrencePair): OccurrenceMetadata => { : undefined, nodeSeaEntryDeclaration, ), + espImageSegment: compareOptionalMetadata( + left?.format_metadata?.kind === "esp-image-segment" + ? left.format_metadata + : undefined, + right?.format_metadata?.kind === "esp-image-segment" + ? right.format_metadata + : undefined, + espImageSegmentDeclaration, + ), pyInstaller: compareOptionalMetadata( isPyInstallerEntry(left) ? left.format_metadata : undefined, isPyInstallerEntry(right) ? right.format_metadata : undefined, @@ -590,8 +606,7 @@ const metadataChanged = ( left: ArtifactNode, right: ArtifactNode, ): boolean => - input.metadata.asarPacking === "changed" || - input.metadata.nodeSea === "changed" || + declarationChanged(input.metadata) || left.media_type !== right.media_type || left.architecture !== right.architecture || left.content_state !== right.content_state || diff --git a/src/domain/artifactFormatMetadata.ts b/src/domain/artifactFormatMetadata.ts index 7bb991391f..60302dad88 100644 --- a/src/domain/artifactFormatMetadata.ts +++ b/src/domain/artifactFormatMetadata.ts @@ -205,6 +205,158 @@ export const nodeSeaEntryDeclaration = ( metadata: z.infer, ) => ({ role: metadata.role, source_path: metadata.source_path }); +const uint8 = z.number().int().min(0).max(0xff); +const uint16 = z.number().int().min(0).max(0xffff); +const sha256Hex = z.string().regex(/^[0-9a-f]{64}$/u); +const byteRange = z.strictObject({ offset, length: offset }); + +/** + * `esp_app_desc_t` fields, as ESP-IDF names them; strings stop at NUL. A field + * the file cuts off is null; `observed_length` says where the descriptor ends. + */ +const espAppDescriptorSchema = z.strictObject({ + kind: z.literal("app"), + offset, + /** Descriptor bytes the file holds, at most the 256 of `esp_app_desc_t`. */ + observed_length: z.int().min(4).max(256), + secure_version: uint32.nullable(), + version: z.string().nullable(), + project_name: z.string().nullable(), + time: z.string().nullable(), + date: z.string().nullable(), + idf_ver: z.string().nullable(), + app_elf_sha256: sha256Hex.nullable(), + min_efuse_blk_rev_full: uint16.nullable(), + max_efuse_blk_rev_full: uint16.nullable(), + /** + * ESP-IDF's value: the MMU page size as a base-2 logarithm (16 is 64 KiB); + * 0 in images built before ESP-IDF v5.4. + */ + mmu_page_size: uint8.nullable(), + /** + * The byte ESP-IDF v6.1 names `spi_flash_mode`, as stored; REA does not + * interpret it. Releases that reserve this byte leave it zero. + */ + spi_flash_mode: uint8.nullable(), +}); + +/** `esp_bootloader_desc_t` fields, as ESP-IDF names them. */ +const espBootloaderDescriptorSchema = z.strictObject({ + kind: z.literal("bootloader"), + offset, + secure_version: uint8, + version: uint32, + idf_ver: z.string().nullable(), + date_time: z.string().nullable(), +}); + +/** + * ESP32-family image facts. `header` and the descriptor fields are observed; + * the descriptor kind, `chip`, the integrity statuses and the trailing-byte + * ranges are derived. + */ +export const espImageMetadataSchema = z.strictObject({ + kind: z.literal("esp-image"), + /** `esp_image_header_t` fields, as ESP-IDF names them. */ + header: z.strictObject({ + segment_count: uint8, + spi_mode: uint8, + spi_speed: uint8, + spi_size: uint8, + entry_addr: uint32, + wp_pin: uint8, + spi_pin_drv: z.string().regex(/^[0-9a-f]{6}$/u), + chip_id: uint16, + min_chip_rev: uint8, + min_chip_rev_full: uint16, + max_chip_rev_full: uint16, + reserved: z.string().regex(/^[0-9a-f]{8}$/u), + hash_appended: z.boolean(), + }), + /** Derived from `chip_id` through the cited table; null for unknown IDs. */ + chip: z + .strictObject({ + target: z.string(), + isa: z.enum(["xtensa", "riscv"]), + source: z.string(), + }) + .nullable(), + checksum: z.strictObject({ + offset: offset.nullable(), + stored: uint8.nullable(), + computed: uint8.nullable(), + status: z.enum(["valid", "invalid", "unavailable"]), + }), + /** The SHA-256 the header's `hash_appended` adds after the checksum. */ + appended_sha256: z.strictObject({ + offset: offset.nullable(), + stored: sha256Hex.nullable(), + computed: sha256Hex.nullable(), + status: z.enum(["valid", "invalid", "unavailable", "not-appended"]), + }), + /** End of the checksum block and any digest; null when truncated before it. */ + image_end: offset.nullable(), + signature_sector_offset: offset.nullable(), + /** Bytes after the image that no padding, signature or erased flash describes. */ + undescribed_trailing: z.array(byteRange), + /** Its kind classifies the image as an app or a bootloader. */ + descriptor: z + .discriminatedUnion("kind", [ + espAppDescriptorSchema, + espBootloaderDescriptorSchema, + ]) + .nullable(), +}); +export type EspImageMetadata = z.infer; + +/** One header-declared segment; `data_len` is declared, not available. */ +export const espImageSegmentMetadataSchema = z.strictObject({ + kind: z.literal("esp-image-segment"), + index: uint8, + load_addr: uint32, + data_len: uint32, + header_offset: offset, + /** ESP-IDF's cold-boot `should_map` and `should_load`; derived. */ + classification: z.enum(["mapped", "loaded", "not-loaded", "unknown"]), +}); +export type EspImageSegmentMetadata = z.infer< + typeof espImageSegmentMetadataSchema +>; + +/** + * What a segment header declares: where its bytes load and how many there + * are. Header offsets are layout, and the classification is derived. + */ +export const espImageSegmentDeclaration = ( + metadata: EspImageSegmentMetadata, +) => ({ load_addr: metadata.load_addr, data_len: metadata.data_len }); + +/** One Secure Boot v2 block with its 0xE7 magic and a matching CRC32. */ +export const espImageSignatureBlockMetadataSchema = z.strictObject({ + kind: z.literal("esp-image-signature-block"), + /** Slot in the signature sector, 0 to 2. */ + index: uint8, + version: uint8, + scheme: z.enum(["rsa", "ecdsa"]).nullable(), + /** Third block byte for ECDSA: 0 selects SHA-256, 1 SHA-384. */ + sha_version: uint8.nullable(), + curve_id: uint8.nullable(), + digest_algorithm: z.enum(["sha256", "sha384"]).nullable(), + image_digest: z + .string() + .regex(/^(?:[0-9a-f]{64}|[0-9a-f]{96})$/u) + .nullable(), + /** Whether `image_digest` equals the digest of the bytes before the sector. */ + image_digest_matches: z.boolean().nullable(), + /** SHA-256 of the block's public key, the value eFuse key digests hold. */ + public_key_sha256: sha256Hex.nullable(), + /** REA checks no key, signature or eFuse. */ + signature_status: z.literal("unverified"), +}); +export type EspImageSignatureBlockMetadata = z.infer< + typeof espImageSignatureBlockMetadataSchema +>; + /** Declaration identity that is stable across archive layout changes. */ export const pyInstallerEntryDeclaration = ( metadata: PyInstallerEntryMetadata, @@ -265,6 +417,9 @@ export const artifactFormatMetadataSchema = z.discriminatedUnion("kind", [ hexRecordRegionMetadataSchema, uImageMetadataSchema, uImageEntryMetadataSchema, + espImageMetadataSchema, + espImageSegmentMetadataSchema, + espImageSignatureBlockMetadataSchema, ]); export type ArtifactFormatMetadata = z.infer< typeof artifactFormatMetadataSchema diff --git a/src/domain/artifactGraph.ts b/src/domain/artifactGraph.ts index 5faa354671..ee6b8ead33 100644 --- a/src/domain/artifactGraph.ts +++ b/src/domain/artifactGraph.ts @@ -90,6 +90,7 @@ const artifactFormatSchema = z.enum([ "node-sea", "intel-hex", "srec", + "esp-image", "unknown", ]); @@ -101,6 +102,7 @@ const artifactContentFormatSchema = artifactFormatSchema.extract([ "uimage", "intel-hex", "srec", + "esp-image", "mach-o-universal", "mach-o", "elf", diff --git a/src/domain/binaryTargetTypes.ts b/src/domain/binaryTargetTypes.ts index d1fbfe594a..9fd566512c 100644 --- a/src/domain/binaryTargetTypes.ts +++ b/src/domain/binaryTargetTypes.ts @@ -263,7 +263,8 @@ export type BinaryTarget = | "dmg" | "pkg" | "intel-hex" - | "srec"; + | "srec" + | "esp-image"; }) | (BinaryTargetIdentity & NonExecutableMetadata & { diff --git a/src/domain/evidenceSubject.ts b/src/domain/evidenceSubject.ts index facc57c0e3..65f7eca0d6 100644 --- a/src/domain/evidenceSubject.ts +++ b/src/domain/evidenceSubject.ts @@ -42,6 +42,7 @@ export const evidenceSubjectFormatSchema = z.enum([ "node-sea", "intel-hex", "srec", + "esp-image", ]); /** Format-owned architecture declarations shared by Evidence and persisted executions. */ diff --git a/src/domain/firmware/espChips.ts b/src/domain/firmware/espChips.ts new file mode 100644 index 0000000000..bd0a3758ea --- /dev/null +++ b/src/domain/firmware/espChips.ts @@ -0,0 +1,221 @@ +/** + * ESP chip identities and flash-mapped address ranges from ESP-IDF v6.1: + * - `chip_id` values: `components/bootloader_support/include/esp_app_format.h` + * (`esp_chip_id_t`) and `Kconfig` (`IDF_FIRMWARE_CHIP_ID`); + * - ISA: `Kconfig`, where each `IDF_TARGET_*` selects `IDF_TARGET_ARCH_XTENSA` + * or `IDF_TARGET_ARCH_RISCV`; + * - mapped ranges: `SOC_IROM_*`, `SOC_DROM_*` and, where `soc_caps.h` sets + * `SOC_MMU_PER_EXT_MEM_TARGET`, `SOC_EXTRAM_*` in + * `components/soc//include/soc/soc.h`; + * - page sizes: `SOC_MMU_PAGE_SIZE_8KB_SUPPORTED` in `soc_caps.h` and + * `components/soc//Kconfig.mmu`; + * - segment length limit: `SOC_SPI_MEM_SUPPORT_CACHE_32BIT_ADDR_MAP` in + * `soc_caps.h`, which sets `ESP_IMAGE_MAX_FLASH_ADDR_SIZE` in + * `components/bootloader_support/src/esp_image_format.c`. + * + * Every value is a derivation from the observed `chip_id`, not a fact the + * image records. + */ +export const ESP_CHIP_TABLE_SOURCE = "ESP-IDF v6.1"; + +export type EspChipIsa = "xtensa" | "riscv"; + +/** Half-open `[low, high)` address range. */ +interface EspAddressRange { + readonly low: number; + readonly high: number; +} + +export interface EspChip { + readonly chipId: number; + /** ESP-IDF `IDF_TARGET` name, as `esptool --chip` also spells it. */ + readonly target: string; + readonly isa: EspChipIsa; + /** `SOC_SPI_MEM_SUPPORT_CACHE_32BIT_ADDR_MAP`, which lifts the 16 MiB segment limit. */ + readonly cache32BitAddressMap: boolean; + /** `should_map` ranges with fixed bounds. */ + readonly mapped: readonly EspAddressRange[]; + /** + * Shared IROM/DROM range ending at `low + (SOC_MMU_PAGE_SIZE << 8)`. The + * page size is the bootloader build's `CONFIG_MMU_PAGE_SIZE`, which no + * image records, so only bounds common to every page size are known. + */ + readonly pageSizedMapping?: { + readonly low: number; + readonly pageSizes: readonly number[]; + }; +} + +const range = (low: number, high: number): EspAddressRange => ({ low, high }); + +/** 8 KiB is a hardware page size esptool builds for; Kconfig selects 16-64. */ +const C6_H2_PAGE_SIZES = [0x2000, 0x4000, 0x8000, 0x1_0000] as const; + +const CHIPS: readonly EspChip[] = [ + { + chipId: 0x0000, + target: "esp32", + isa: "xtensa", + cache32BitAddressMap: false, + mapped: [range(0x400d_0000, 0x4040_0000), range(0x3f40_0000, 0x3f80_0000)], + }, + { + chipId: 0x0002, + target: "esp32s2", + isa: "xtensa", + cache32BitAddressMap: false, + mapped: [range(0x4008_0000, 0x4080_0000), range(0x3f00_0000, 0x3ff8_0000)], + }, + { + chipId: 0x0005, + target: "esp32c3", + isa: "riscv", + cache32BitAddressMap: false, + mapped: [range(0x4200_0000, 0x4280_0000), range(0x3c00_0000, 0x3c80_0000)], + }, + { + chipId: 0x0009, + target: "esp32s3", + isa: "xtensa", + cache32BitAddressMap: true, + mapped: [range(0x4200_0000, 0x4400_0000), range(0x3c00_0000, 0x3e00_0000)], + }, + { + chipId: 0x000c, + target: "esp32c2", + isa: "riscv", + cache32BitAddressMap: false, + mapped: [range(0x4200_0000, 0x4240_0000), range(0x3c00_0000, 0x3c40_0000)], + }, + { + chipId: 0x000d, + target: "esp32c6", + isa: "riscv", + cache32BitAddressMap: false, + mapped: [], + pageSizedMapping: { low: 0x4200_0000, pageSizes: C6_H2_PAGE_SIZES }, + }, + { + chipId: 0x0010, + target: "esp32h2", + isa: "riscv", + cache32BitAddressMap: false, + mapped: [], + pageSizedMapping: { low: 0x4200_0000, pageSizes: C6_H2_PAGE_SIZES }, + }, + { + chipId: 0x0012, + target: "esp32p4", + isa: "riscv", + cache32BitAddressMap: true, + mapped: [range(0x4000_0000, 0x4400_0000), range(0x4800_0000, 0x4c00_0000)], + }, + { + chipId: 0x0014, + target: "esp32c61", + isa: "riscv", + cache32BitAddressMap: true, + mapped: [range(0x4200_0000, 0x4400_0000)], + }, + { + chipId: 0x0017, + target: "esp32c5", + isa: "riscv", + cache32BitAddressMap: true, + mapped: [range(0x4200_0000, 0x4400_0000)], + }, + { + chipId: 0x0019, + target: "esp32h21", + isa: "riscv", + cache32BitAddressMap: false, + mapped: [range(0x4200_0000, 0x4300_0000)], + }, + { + chipId: 0x001c, + target: "esp32h4", + isa: "riscv", + cache32BitAddressMap: true, + mapped: [range(0x4200_0000, 0x4400_0000)], + }, + { + chipId: 0x0020, + target: "esp32s31", + isa: "riscv", + cache32BitAddressMap: true, + mapped: [range(0x4000_0000, 0x4400_0000), range(0x5000_0000, 0x5400_0000)], + }, +]; + +const BY_ID = new Map(CHIPS.map((chip) => [chip.chipId, chip])); + +export const espChipForId = (chipId: number): EspChip | undefined => + BY_ID.get(chipId); + +const SIXTEEN_MIB = 0x100_0000; + +/** + * The segment length ESP-IDF's `verify_segment_header` rejects at or above, + * `ESP_IMAGE_MAX_FLASH_ADDR_SIZE`. With 32-bit cache addresses the limit is + * `UINT32_MAX`, which the 4-byte alignment rule already excludes. + */ +export const espSegmentLengthLimit = (chip: EspChip): number | undefined => + chip.cache32BitAddressMap ? undefined : SIXTEEN_MIB; + +/** `should_load` treats every address below this as reserved (padding, MD5). */ +const RESERVED_ADDRESS_LIMIT = 0x1000_0000; + +/** + * How ESP-IDF's bootloader treats a segment on a cold boot + * (`esp_image_format.c` `should_map` and `should_load`). `loaded` means + * `should_load` selects the segment, not that the bootloader accepts it: + * `verify_load_addresses` then rejects a segment outside the chip's DRAM, + * IRAM, RTC memory or SPM, or over the bootloader's own stack, data or loader + * code. No RAM ranges are modelled here. + */ +export type EspSegmentClassification = + | "mapped" + | "loaded" + | "not-loaded" + | "unknown"; + +/** Classify one load address; a mapping that depends on the page size stays unknown. */ +export const classifyEspSegment = ( + chip: EspChip | undefined, + address: number, +): EspSegmentClassification => { + if (chip === undefined) return "unknown"; + const mapped = isMapped(chip, address); + if (mapped === undefined) return "unknown"; + if (mapped) return "mapped"; + return address < RESERVED_ADDRESS_LIMIT ? "not-loaded" : "loaded"; +}; + +/** Addresses whose mapping depends on the bootloader's MMU page size. */ +export const espPageSizedMappingAmbiguity = ( + chip: EspChip, +): EspAddressRange | undefined => { + const paged = chip.pageSizedMapping; + return paged === undefined + ? undefined + : range( + pageSizedHigh(paged.low, Math.min(...paged.pageSizes)), + pageSizedHigh(paged.low, Math.max(...paged.pageSizes)), + ); +}; + +const isMapped = (chip: EspChip, address: number): boolean | undefined => { + if (chip.mapped.some(({ low, high }) => address >= low && address < high)) + return true; + const paged = chip.pageSizedMapping; + if (paged === undefined || address < paged.low) return false; + if (address < pageSizedHigh(paged.low, Math.min(...paged.pageSizes))) + return true; + return address < pageSizedHigh(paged.low, Math.max(...paged.pageSizes)) + ? undefined + : false; +}; + +/** `SOC_IROM_HIGH = SOC_IROM_LOW + (SOC_MMU_PAGE_SIZE << 8)`. */ +const pageSizedHigh = (low: number, pageSize: number): number => + low + pageSize * 256; diff --git a/src/domain/firmware/espImage.test.ts b/src/domain/firmware/espImage.test.ts new file mode 100644 index 0000000000..706861f0ad --- /dev/null +++ b/src/domain/firmware/espImage.test.ts @@ -0,0 +1,923 @@ +import { createHash } from "node:crypto"; +import { crc32 } from "node:zlib"; + +import { describe, expect, it } from "vitest"; +import { ESP_IMAGE_LIMITATIONS } from "./espImageFacts.js"; +import { ESP_DESCRIPTOR_LIMITATIONS } from "./espImageStructures.js"; + +import { + ESP8266_REFUSAL, + createEspImageDecoder, + recognizeEspImage, +} from "./espImage.js"; +import { + classifyEspSegment, + espChipForId, + espSegmentLengthLimit, + espPageSizedMappingAmbiguity, +} from "./espChips.js"; +import { + ESP32C3_APP_SEGMENTS, + ESP32C3_BOOTLOADER_SEGMENTS, + appDescriptor, + bootloaderDescriptor, + erasedTo4KiB, + esp8266Image, + espChecksum, + espImage, + pattern, + signatureBlock, + signedEspImage, +} from "../../../tests/fixtures/espImage.js"; + +const decode = (bytes: Uint8Array, chunkBytes = 7) => { + const decoder = createEspImageDecoder(bytes.length); + for ( + let offset = 0; + offset < bytes.length && !decoder.satisfied; + offset += chunkBytes + ) + decoder.write(bytes.subarray(offset, offset + chunkBytes)); + return decoder.finish(); +}; + +const image = (bytes: Uint8Array, chunkBytes?: number) => { + const decoded = decode(bytes, chunkBytes); + if (!decoded.ok) throw new Error(decoded.error.message); + return decoded.value; +}; + +const sha256 = (bytes: Uint8Array): string => + createHash("sha256").update(bytes).digest("hex"); + +const classifications = (bytes: Uint8Array) => + image(bytes).segments.map(({ metadata }) => metadata.classification); + +const APP = espImage({ segments: ESP32C3_APP_SEGMENTS }); +const PADDED = erasedTo4KiB(APP); + +/** A sector holding `slots` at their indices, the rest erased. */ +const sector = (slots: readonly (Buffer | null)[]): Buffer => { + const bytes = Buffer.alloc(4096, 0xff); + slots.forEach((slot, index) => slot?.copy(bytes, index * 1216)); + return bytes; +}; + +describe("ESP image recognition", () => { + it.each([ + ["an ESP32-C3 app", APP, "esp-image"], + ["an image cut after its first header", APP.subarray(0, 40), "esp-image"], + ["a header alone", APP.subarray(0, 24), "esp-image"], + ["an ESP8266 v1 image", esp8266Image(), "esp8266"], + ["an ESP8266 image led by eboot", esp8266Image(0x4010_f000), "esp8266"], + ["an ESP8266 v3 image led by IROM", esp8266Image(0x4020_1010), "esp8266"], + ] as const)("recognizes %s", (_name, bytes, expected) => { + expect(recognizeEspImage(bytes, bytes.length)).toBe(expected); + }); + + const changed = (offset: number, value: number): Buffer => { + const bytes = Buffer.from(APP); + bytes[offset] = value; + return bytes; + }; + + it.each([ + ["another magic", changed(0, 0xea)], + // Valid, but loads nothing; the decoder still reads one. + ["an image with no segments", espImage({ segments: [] })], + ["17 segments", changed(1, 17)], + ["a flash mode outside the enum", changed(2, 6)], + ["a flash speed outside the enum", changed(3, 0x23)], + ["a hash_appended byte of 2", changed(23, 2)], + ["an unaligned first segment length", changed(28, 0x21)], + ["fewer than 24 bytes", APP.subarray(0, 23)], + // DOS .COM near jumps, over code or over a zeroed data area. + [ + "an x86 jump stub", + Buffer.from([0xe9, 0x0d, 0x01, 0xb4, 0x09, ...Buffer.alloc(59)]), + ], + [ + "an x86 jump over zeros", + Buffer.from([0xe9, 0x00, 0x01, ...Buffer.alloc(61)]), + ], + ["encrypted flash", pattern(4096, 0x5a)], + ["nothing", Buffer.alloc(0)], + ] as const)("does not recognize %s", (_name, bytes) => { + expect(recognizeEspImage(bytes, bytes.length)).toBeUndefined(); + }); +}); + +describe("ESP image decoding", () => { + it("reports header, chip, segment, integrity and descriptor facts", () => { + const decoded = image(APP); + expect(decoded.metadata).toEqual({ + kind: "esp-image", + header: { + segment_count: 4, + spi_mode: 2, + spi_speed: 0, + spi_size: 2, + entry_addr: 0x4038_0000, + wp_pin: 0xee, + spi_pin_drv: "000000", + chip_id: 5, + min_chip_rev: 0, + min_chip_rev_full: 0, + max_chip_rev_full: 199, + reserved: "00000000", + hash_appended: true, + }, + chip: { target: "esp32c3", isa: "riscv", source: "ESP-IDF v6.1" }, + // Segments end at 456; the checksum ends the next 16-byte block. + checksum: { + offset: 463, + stored: espChecksum(ESP32C3_APP_SEGMENTS), + computed: espChecksum(ESP32C3_APP_SEGMENTS), + status: "valid", + }, + appended_sha256: { + offset: 464, + stored: sha256(APP.subarray(0, 464)), + computed: sha256(APP.subarray(0, 464)), + status: "valid", + }, + image_end: 496, + signature_sector_offset: null, + undescribed_trailing: [], + descriptor: { + kind: "app", + offset: 32, + observed_length: 256, + secure_version: 0, + version: "1.2.3", + project_name: "rea_esp_fixture", + time: "12:34:56", + date: "Oct 27 2026", + idf_ver: "v6.1", + app_elf_sha256: "ab".repeat(32), + min_efuse_blk_rev_full: 0, + max_efuse_blk_rev_full: 199, + mmu_page_size: 16, + spi_flash_mode: 2, + }, + }); + expect( + decoded.segments.map(({ dataOffset, availableLength, metadata }) => [ + metadata.index, + metadata.load_addr, + metadata.data_len, + metadata.header_offset, + dataOffset, + availableLength, + metadata.classification, + ]), + ).toEqual([ + [0, 0x3c00_0020, 288, 24, 32, 288, "mapped"], + [1, 0x3fc8_8000, 16, 320, 328, 16, "loaded"], + [2, 0x4038_0000, 32, 344, 352, 32, "loaded"], + [3, 0x4200_0020, 64, 384, 392, 64, "mapped"], + ]); + expect(decoded.coverage).toBe("complete"); + expect(decoded.limitations).toEqual([]); + expect(decoded.signatureBlocks).toEqual([]); + }); + + it("decodes identically from any chunking", () => { + const signed = signedEspImage(APP, ["rsa", null, "ecdsa-p384"]); + const expected = image(signed, signed.length); + for (const chunkBytes of [1, 3, 16, 4096]) + expect(image(signed, chunkBytes)).toEqual(expected); + }); + + it("reads an image with no segments", () => { + const decoded = image(espImage({ segments: [] })); + expect(decoded.metadata).toMatchObject({ + checksum: { offset: 31, stored: 0xef, computed: 0xef, status: "valid" }, + appended_sha256: { offset: 32, status: "valid" }, + image_end: 64, + descriptor: null, + }); + expect(decoded.segments).toEqual([]); + expect(decoded.coverage).toBe("complete"); + }); + + it("reads a bootloader descriptor only with zero reserved bytes", () => { + expect( + image(espImage({ segments: ESP32C3_BOOTLOADER_SEGMENTS })).metadata + .descriptor, + ).toEqual({ + kind: "bootloader", + offset: 32, + secure_version: 1, + version: 1, + idf_ver: "v6.1", + date_time: "Oct 27 2026 12:34:56", + }); + const code = bootloaderDescriptor(); + code[1] = 0x41; + const decoded = image( + espImage({ segments: [{ address: 0x4038_0000, data: code }] }), + ); + expect(decoded.metadata.descriptor).toBeNull(); + expect(decoded.limitations).toEqual([ + ESP_DESCRIPTOR_LIMITATIONS.reservedBytesNotZero, + ]); + }); + + it("keeps invalid descriptor text unknown", () => { + const descriptor = appDescriptor(); + descriptor[16] = 0xff; + const decoded = image( + espImage({ segments: [{ address: 0x3c00_0020, data: descriptor }] }), + ); + expect(decoded.metadata.descriptor).toMatchObject({ + version: null, + project_name: "rea_esp_fixture", + }); + expect(decoded.limitations).toEqual([ + ESP_DESCRIPTOR_LIMITATIONS.fieldNotUtf8("version"), + ]); + }); +}); + +describe("ESP image segment classification", () => { + it("classifies by the ESP32 memory map and keeps padding unloaded", () => { + const bytes = espImage({ + chipId: 0, + segments: [ + { address: 0x3f40_0020, data: appDescriptor() }, + { address: 0, data: pattern(16, 1) }, + { address: 0x3ffb_0000, data: pattern(16, 2) }, + { address: 0x4008_0000, data: pattern(16, 3) }, + { address: 0x400d_0020, data: pattern(16, 4) }, + { address: 0x5000_0000, data: pattern(16, 5) }, + ], + }); + expect(image(bytes).metadata.chip).toEqual({ + target: "esp32", + isa: "xtensa", + source: "ESP-IDF v6.1", + }); + expect(classifications(bytes)).toEqual([ + "mapped", + "not-loaded", + "loaded", + "loaded", + "mapped", + "loaded", + ]); + }); + + it("reports unknown chips without guessing a memory map", () => { + const decoded = image( + espImage({ chipId: 0x1f, segments: ESP32C3_APP_SEGMENTS }), + ); + expect(decoded.metadata.chip).toBeNull(); + expect(decoded.metadata.header.chip_id).toBe(0x1f); + expect( + new Set(decoded.segments.map(({ metadata }) => metadata.classification)), + ).toEqual(new Set(["unknown"])); + expect(decoded.limitations).toEqual([ + ESP_IMAGE_LIMITATIONS.unknownChip(0x1f), + ]); + }); + + it("leaves ESP32-C6 mapping unknown where the bootloader page size decides it", () => { + // The app's page size does not establish the bootloader's. + const bytes = espImage({ + chipId: 0x0d, + segments: [ + { address: 0x4200_0020, data: appDescriptor({ mmuPageSizeLog2: 16 }) }, + { address: 0x4210_0000, data: pattern(16, 1) }, + { address: 0x4230_0000, data: pattern(16, 2) }, + { address: 0x4300_0000, data: pattern(16, 3) }, + ], + }); + const chip = espChipForId(0x0d); + if (chip === undefined) throw new Error("Expected ESP32-C6 fixture chip"); + expect(espPageSizedMappingAmbiguity(chip)).toEqual({ + low: 0x4220_0000, + high: 0x4300_0000, + }); + expect(classifications(bytes)).toEqual([ + "mapped", + "mapped", + "unknown", + "loaded", + ]); + expect(image(bytes).limitations).toEqual([ + ESP_IMAGE_LIMITATIONS.unclassifiedPages( + "esp32c6", + 0x4220_0000, + 0x4300_0000, + ), + ]); + }); + + it("derives the ISA from chip_id, not from the series letter", () => { + expect(espChipForId(0x20)).toMatchObject({ + target: "esp32s31", + isa: "riscv", + }); + expect(espChipForId(0x09)).toMatchObject({ + target: "esp32s3", + isa: "xtensa", + }); + // ESP32-P4 maps PSRAM through its own MMU. + expect(classifyEspSegment(espChipForId(0x12), 0x4800_0000)).toBe("mapped"); + }); +}); + +describe("ESP image integrity", () => { + it("reports a checksum mismatch as an integrity fact", () => { + const computed = espChecksum(ESP32C3_APP_SEGMENTS); + const stored = computed ^ 0x5a; + const decoded = image( + espImage({ segments: ESP32C3_APP_SEGMENTS, checksum: stored }), + ); + expect(decoded.metadata.checksum).toMatchObject({ + offset: 463, + stored, + computed, + status: "invalid", + }); + expect(decoded.metadata.appended_sha256.status).toBe("valid"); + expect(decoded.coverage).toBe("complete"); + expect(decoded.limitations).toEqual([ + ESP_IMAGE_LIMITATIONS.checksumMismatch(stored, 0x1cf, computed), + ]); + }); + + it("reports an appended SHA-256 mismatch", () => { + const bytes = Buffer.from(APP); + bytes.writeUInt8(bytes.readUInt8(bytes.length - 1) ^ 1, bytes.length - 1); + const decoded = image(bytes); + expect(decoded.metadata.checksum.status).toBe("valid"); + expect(decoded.metadata.appended_sha256).toMatchObject({ + offset: 464, + status: "invalid", + stored: bytes.subarray(464).toString("hex"), + computed: createHash("sha256") + .update(bytes.subarray(0, 464)) + .digest("hex"), + }); + expect(decoded.limitations).toEqual([ + ESP_IMAGE_LIMITATIONS.digestMismatch(0x1d0), + ]); + }); + + it("marks an image without a digest", () => { + expect( + image(espImage({ segments: ESP32C3_APP_SEGMENTS, hashAppended: false })) + .metadata.appended_sha256, + ).toEqual({ + offset: null, + stored: null, + computed: null, + status: "not-appended", + }); + }); +}); + +describe("ESP image framing and segment limits", () => { + it("keeps the app descriptor's fields when the file ends in its reserved tail", () => { + // Segment 0's data, and with it the descriptor, starts at offset 32. + const decoded = image(APP.subarray(0, 32 + 200)); + expect(decoded.metadata.descriptor).toMatchObject({ + kind: "app", + offset: 32, + observed_length: 200, + version: "1.2.3", + mmu_page_size: 16, + spi_flash_mode: 2, + }); + expect(decoded.segments[0]?.availableLength).toBe(200); + expect(decoded.coverage).toBe("unknown"); + }); + + it.each([ + [20, { secure_version: 0, version: null, app_elf_sha256: null }], + [ + 176, + { + idf_ver: "v6.1", + app_elf_sha256: "ab".repeat(32), + min_efuse_blk_rev_full: null, + max_efuse_blk_rev_full: null, + mmu_page_size: null, + spi_flash_mode: null, + }, + ], + [ + 181, + { max_efuse_blk_rev_full: 199, mmu_page_size: 16, spi_flash_mode: null }, + ], + ] as const)( + "keeps the descriptor fields a file cut at byte %i holds in full", + (length, fields) => { + const decoded = image(APP.subarray(0, 32 + length)); + expect(decoded.metadata.descriptor).toMatchObject({ + kind: "app", + observed_length: length, + ...fields, + }); + // A cut field is unavailable, not malformed text. + expect(decoded.limitations).toEqual([ + ESP_IMAGE_LIMITATIONS.truncatedSegment(0, 288, 32, length), + ]); + }, + ); + + it("keeps a truncated segment as an image with unavailable integrity", () => { + const decoded = image(APP.subarray(0, 360)); + expect(decoded.metadata).toMatchObject({ + checksum: { offset: null, computed: null, status: "unavailable" }, + appended_sha256: { offset: null, computed: null, status: "unavailable" }, + image_end: null, + }); + expect( + decoded.segments.map(({ availableLength }) => availableLength), + ).toEqual([288, 16, 8]); + expect(decoded.metadata.header.segment_count).toBe(4); + expect(decoded.segments[2]).toMatchObject({ + dataOffset: 352, + availableLength: 8, + metadata: { index: 2, header_offset: 344, data_len: 32 }, + }); + expect(decoded.coverage).toBe("unknown"); + expect(decoded.limitations).toEqual([ + ESP_IMAGE_LIMITATIONS.truncatedSegment(2, 32, 0x160, 8), + ]); + }); + + it("reports a file that ends before a declared segment header", () => { + const decoded = image(APP.subarray(0, 344)); + expect(decoded.segments).toHaveLength(2); + expect(decoded.metadata).toMatchObject({ + header: { segment_count: 4 }, + checksum: { status: "unavailable" }, + appended_sha256: { status: "unavailable" }, + image_end: null, + }); + expect(decoded.coverage).toBe("unknown"); + expect(decoded.limitations).toEqual([ + ESP_IMAGE_LIMITATIONS.truncatedSegmentHeaders(4, 2), + ]); + }); + + it.each([ + ["checksum", 460, "unavailable"], + ["appended SHA-256", 470, "valid"], + ] as const)( + "keeps coverage unknown when the file ends before its %s", + (_name, end, checksumStatus) => { + const decoded = image(APP.subarray(0, end)); + expect(decoded.metadata).toMatchObject({ + checksum: { offset: 463, status: checksumStatus }, + appended_sha256: { + offset: 464, + stored: null, + status: "unavailable", + computed: + end < 464 + ? null + : createHash("sha256").update(APP.subarray(0, 464)).digest("hex"), + }, + image_end: null, + }); + expect(decoded.limitations).toEqual([ + checksumStatus === "unavailable" + ? ESP_IMAGE_LIMITATIONS.truncatedBeforeChecksum(463) + : ESP_IMAGE_LIMITATIONS.truncatedBeforeDigest(464), + ]); + expect(decoded.coverage).toBe("unknown"); + }, + ); +}); + +describe("ESP image segment constraints", () => { + it("reports an unaligned segment length that ESP-IDF rejects", () => { + const decoded = image( + espImage({ + segments: [ + { address: 0x3c00_0020, data: appDescriptor() }, + { address: 0x4038_0000, data: pattern(6, 1) }, + { address: 0x3fc8_8000, data: Buffer.alloc(0) }, + ], + }), + ); + expect(decoded.metadata.checksum.status).toBe("valid"); + expect(decoded.segments[1]).toMatchObject({ + availableLength: 6, + metadata: { index: 1, data_len: 6 }, + }); + expect(decoded.segments[2]).toMatchObject({ + availableLength: 0, + metadata: { index: 2, data_len: 0 }, + }); + expect(decoded.coverage).toBe("complete"); + expect(decoded.limitations).toEqual([ + ESP_IMAGE_LIMITATIONS.segmentLengthUnaligned(1, 6), + ]); + }); + + it.each([ + ["esp32c3", 0x0005, true], + ["esp32s3", 0x0009, false], + ] as const)( + "applies ESP-IDF's 16 MiB segment limit to %s by its cache address width", + (target, chipId, limited) => { + const bytes = Buffer.from( + espImage({ + chipId, + segments: [{ address: 0x3fc8_8000, data: pattern(16, 1) }], + }), + ); + // Segment 0 declares 16 MiB; the file holds only its first bytes. + bytes.writeUInt32LE(0x100_0000, 28); + const decoded = image(bytes); + const chip = espChipForId(chipId); + expect(chip?.target).toBe(target); + if (chip === undefined) throw new Error("Expected a known fixture chip"); + expect(espSegmentLengthLimit(chip)).toBe( + limited ? 0x100_0000 : undefined, + ); + expect(decoded.segments[0]).toMatchObject({ + dataOffset: 32, + availableLength: bytes.length - 32, + metadata: { data_len: 0x100_0000 }, + }); + expect(decoded.metadata.checksum.status).toBe("unavailable"); + expect(decoded.coverage).toBe("unknown"); + expect(decoded.limitations).toEqual([ + ...(limited + ? [ + ESP_IMAGE_LIMITATIONS.segmentLengthTooLarge( + 0, + 0x100_0000, + 0x100_0000, + target, + ), + ] + : []), + ESP_IMAGE_LIMITATIONS.truncatedSegment( + 0, + 0x100_0000, + 0x20, + bytes.length - 0x20, + ), + ]); + }, + ); +}); + +describe("ESP image trailing bytes", () => { + it("describes erased flash that runs past the signature slots", () => { + const decoded = image( + Buffer.concat([APP, Buffer.alloc(0x1_0000 - APP.length, 0xff)]), + ); + expect(decoded.metadata.undescribed_trailing).toEqual([]); + expect(decoded.coverage).toBe("complete"); + }); + + it.each([ + ["inside the padding", Buffer.concat([APP, Buffer.alloc(1000, 0xff)])], + ["at the sector boundary", PADDED], + ] as const)( + "keeps coverage unknown when erased flash ends %s", + (_name, bytes) => { + // A signed download cut here would have lost its signature sector. + const decoded = image(bytes); + expect(decoded.metadata.undescribed_trailing).toEqual([]); + expect(decoded.metadata.image_end).toBe(APP.length); + expect(decoded.metadata.signature_sector_offset).toBeNull(); + expect(decoded.signatureBlocks).toEqual([]); + expect(decoded.coverage).toBe("unknown"); + expect(decoded.limitations).not.toHaveLength(0); + }, + ); + + it("reports flash segments that a RAM-only header leaves uncounted", () => { + const headerOnly = espImage({ + segments: ESP32C3_APP_SEGMENTS.slice(1, 3), + hashAppended: false, + }); + const flash = Buffer.concat([ + Buffer.from([0, 0, 0, 0, 16, 0, 0, 0]), + Buffer.alloc(16), + ]); + const decoded = image(Buffer.concat([headerOnly, flash])); + expect(decoded.metadata).toMatchObject({ + checksum: { status: "valid" }, + appended_sha256: { status: "not-appended" }, + image_end: 96, + undescribed_trailing: [{ offset: 96, length: 24 }], + }); + expect(decoded.coverage).toBe("unknown"); + expect(decoded.limitations).toEqual([ + ESP_IMAGE_LIMITATIONS.undescribedTrailing(24, 1, 0x60), + ]); + }); + + it("starts an undescribed range at the first byte that is not erased", () => { + const bytes = Buffer.concat([ + PADDED, + Buffer.alloc(4096, 0xff), + pattern(64, 1), + ]); + expect(image(bytes).metadata.undescribed_trailing).toEqual([ + { offset: 8192, length: 64 }, + ]); + }); + + it("reports disjoint undescribed ranges around a signature sector", () => { + const padded = Buffer.concat([APP, Buffer.alloc(4096 - APP.length)]); + const bytes = Buffer.concat([ + padded, + sector([signatureBlock("rsa", padded)]), + Buffer.alloc(64, 0x11), + ]); + const decoded = image(bytes); + expect(decoded.signatureBlocks).toHaveLength(1); + expect(decoded.metadata.undescribed_trailing).toEqual([ + { offset: 496, length: 3600 }, + { offset: 8192, length: 64 }, + ]); + expect(decoded.coverage).toBe("unknown"); + }); +}); + +describe("Secure Boot v2 signature blocks", () => { + it("observes blocks without verifying them", () => { + const signed = signedEspImage(APP, ["rsa", "ecdsa-p256", "ecdsa-p384"]); + const decoded = image(signed); + const content = signed.subarray(0, 4096); + expect(decoded.metadata).toMatchObject({ + signature_sector_offset: 4096, + undescribed_trailing: [], + }); + expect(decoded.coverage).toBe("complete"); + expect(decoded.limitations).toEqual([]); + const block = ( + offset: number, + facts: Record, + keyStart: number, + keyEnd: number, + ) => [ + offset, + { + kind: "esp-image-signature-block", + ...facts, + public_key_sha256: sha256( + signed.subarray(offset + keyStart, offset + keyEnd), + ), + signature_status: "unverified", + }, + ]; + const sha256Fields = { + digest_algorithm: "sha256", + image_digest: sha256(content), + image_digest_matches: true, + }; + expect( + decoded.signatureBlocks.map(({ offset, metadata }) => [offset, metadata]), + ).toEqual([ + block( + 4096, + { + index: 0, + version: 2, + scheme: "rsa", + sha_version: null, + curve_id: null, + ...sha256Fields, + }, + 36, + 812, + ), + block( + 5312, + { + index: 1, + version: 3, + scheme: "ecdsa", + sha_version: 0, + curve_id: 2, + ...sha256Fields, + }, + 36, + 101, + ), + block( + 6528, + { + index: 2, + version: 3, + scheme: "ecdsa", + sha_version: 1, + curve_id: 3, + digest_algorithm: "sha384", + image_digest: createHash("sha384").update(content).digest("hex"), + image_digest_matches: true, + }, + 52, + 149, + ), + ]); + }); + + it("examines every slot, so an erased slot does not hide a later block", () => { + const decoded = image(signedEspImage(APP, ["rsa", null, "ecdsa-p256"])); + expect( + decoded.signatureBlocks.map(({ offset, metadata }) => [ + offset, + metadata.index, + ]), + ).toEqual([ + [4096, 0], + [6528, 2], + ]); + expect(decoded.coverage).toBe("complete"); + }); + + it("does not admit a slot whose CRC32 does not match", () => { + const decoded = image( + Buffer.concat([ + PADDED, + sector([ + signatureBlock("rsa", PADDED), + signatureBlock("rsa", PADDED, { crc: 0 }), + ]), + ]), + ); + expect( + decoded.signatureBlocks.map(({ metadata }) => metadata.index), + ).toEqual([0]); + expect(decoded.metadata.undescribed_trailing).toEqual([ + { offset: 5312, length: 1216 }, + ]); + expect(decoded.coverage).toBe("unknown"); + expect(decoded.limitations).not.toHaveLength(0); + // A stray 0xE7 in trailing data, without a valid block, is not a sector. + const stray = image( + Buffer.concat([PADDED, Buffer.from([0xe7]), pattern(100, 3)]), + ); + expect(stray.metadata.signature_sector_offset).toBeNull(); + expect(stray.signatureBlocks).toEqual([]); + expect(stray.coverage).toBe("unknown"); + expect(stray.metadata.undescribed_trailing).toEqual([ + { offset: 4096, length: 101 }, + ]); + expect(stray.limitations).not.toHaveLength(0); + }); + + it("reports a damaged slot when no slot holds a valid block", () => { + const decoded = image( + Buffer.concat([ + PADDED, + sector([signatureBlock("rsa", PADDED, { crc: 0 })]), + ]), + ); + expect(decoded.signatureBlocks).toEqual([]); + expect(decoded.metadata.undescribed_trailing).toEqual([ + { offset: 4096, length: 4096 }, + ]); + expect(decoded.limitations).not.toHaveLength(0); + expect(decoded.coverage).toBe("unknown"); + }); +}); + +describe("Secure Boot v2 block layouts and sector bounds", () => { + it.each([ + [ + "an unknown ECDSA sha_version", + { shaVersion: 2 }, + { version: 3, scheme: "ecdsa", sha_version: 2 }, + ], + [ + "an unknown version", + { version: 4 }, + { version: 4, scheme: null, sha_version: null }, + ], + ] as const)("leaves layout facts unknown for %s", (_name, options, known) => { + const decoded = image( + Buffer.concat([ + PADDED, + sector([signatureBlock("ecdsa-p256", PADDED, options)]), + ]), + ); + expect(decoded.signatureBlocks[0]?.metadata).toEqual({ + kind: "esp-image-signature-block", + index: 0, + ...known, + curve_id: null, + digest_algorithm: null, + image_digest: null, + image_digest_matches: null, + public_key_sha256: null, + signature_status: "unverified", + }); + }); + + it("hashes a P-192 key as espsecure does, with its point zero-padded to 64 bytes", () => { + // espsecure packs ECDSA blocks as " { + const decoded = image( + Buffer.concat([ + PADDED, + sector([ + signatureBlock("rsa", PADDED, { imageDigest: Buffer.alloc(32) }), + ]), + ]), + ); + expect(decoded.signatureBlocks[0]?.metadata).toMatchObject({ + image_digest_matches: false, + signature_status: "unverified", + }); + // An integrity contradiction leaves coverage complete, as a checksum + // mismatch does. + expect(decoded.coverage).toBe("complete"); + expect(decoded.limitations).toEqual([ + ESP_IMAGE_LIMITATIONS.signatureDigestMismatch(0, 4096), + ]); + }); + + it("reports a digest that does not match and bytes in the sector slack", () => { + const slack = sector([ + signatureBlock("rsa", PADDED, { imageDigest: Buffer.alloc(32) }), + ]); + slack[3700] = 0; + const decoded = image(Buffer.concat([PADDED, slack])); + expect(decoded.signatureBlocks[0]?.metadata.image_digest_matches).toBe( + false, + ); + expect(decoded.metadata.undescribed_trailing).toEqual([ + { offset: 4096 + 3700, length: 396 }, + ]); + }); + + it("reports a truncated sector and describes an erased tail after it", () => { + const signed = signedEspImage(APP, ["rsa"]); + const cut = image(signed.subarray(0, 4096 + 1300)); + expect(cut.coverage).toBe("unknown"); + expect(cut.signatureBlocks).toHaveLength(1); + expect(cut.metadata.signature_sector_offset).toBe(4096); + expect(cut.signatureBlocks[0]?.offset).toBe(4096); + expect(cut.limitations).not.toHaveLength(0); + expect( + image(Buffer.concat([signed, Buffer.alloc(32, 0xff)])).coverage, + ).toBe("complete"); + expect( + image(Buffer.concat([signed, Buffer.alloc(32)])).metadata + .undescribed_trailing, + ).toEqual([{ offset: 8192, length: 32 }]); + }); +}); + +describe("ESP image refusals", () => { + it.each([0x4010_0000, 0x4010_f000, 0x4020_1010])( + "refuses an ESP8266 image whose first segment loads at %i", + (address) => { + expect(decode(esp8266Image(address))).toEqual({ + ok: false, + error: { kind: "unsupported", message: ESP8266_REFUSAL }, + }); + }, + ); + + it("refuses bytes that are not an ESP image header", () => { + expect(decode(pattern(64, 0x5a))).toMatchObject({ + ok: false, + error: { kind: "format" }, + }); + }); + + it("refuses input that stops before the admitted size", () => { + expect(image(APP.subarray(0, 100)).coverage).toBe("unknown"); + const decoder = createEspImageDecoder(APP.length); + decoder.write(APP.subarray(0, 100)); + expect(decoder.finish()).toMatchObject({ + ok: false, + error: { + kind: "format", + }, + }); + }); +}); diff --git a/src/domain/firmware/espImage.ts b/src/domain/firmware/espImage.ts new file mode 100644 index 0000000000..cf98a802e2 --- /dev/null +++ b/src/domain/firmware/espImage.ts @@ -0,0 +1,375 @@ +import { createHash, type Hash } from "node:crypto"; + +import { err, ok, type Result } from "../result.js"; +import { + deriveEspImage, + type EspImage, + type ObservedSegmentHeader, +} from "./espImageFacts.js"; +import { + ESP_APP_DESC_BYTES, + ESP_FLASH_SECTOR_BYTES, + ESP_IMAGE_DIGEST_BYTES, + ESP_IMAGE_SEGMENT_HEADER_BYTES, + firstUnerased, + offsetText, + u32, +} from "./espImageStructures.js"; + +/* + * Layout facts from ESP-IDF v6.1 (`esp_app_format.h`, `esp_image_format.c` + * and `docs/en/security/secure-boot-v2.rst`) and esptool v5.6.0 + * (`bin_image.py`, `cmds.py`, `loader.py`, `targets/esp8266.py`). + */ +export const ESP_IMAGE_MAGIC = 0xe9; +export const ESP_IMAGE_HEADER_BYTES = 24; +export const ESP_IMAGE_MAX_SEGMENTS = 16; +const CHECKSUM_SEED = 0xef; +/** `esp_image_spi_freq_t` values; every esptool target uses only these. */ +const SPI_SPEEDS = new Set([0x0, 0x1, 0x2, 0xf]); +/** The last `esp_image_spi_mode_t` value, `ESP_IMAGE_SPI_MODE_SLOW_READ`. */ +const LAST_SPI_MODE = 5; +/** + * Where an ESP8266 image's first segment loads: DRAM and IRAM from esptool + * `ESP8266ROM.MEMORY_MAP`, IRAM extended through the cache area Arduino's + * eboot links at 0x4010F000, and esptool's IROM window + * (`IROM_MAP_START`-`IROM_MAP_END`), where v3 and Arduino images begin. + */ +const ESP8266_LOAD_REGIONS = [ + [0x3ffe_8000, 0x4000_0000], + [0x4010_0000, 0x4011_0000], + [0x4020_0000, 0x4030_0000], +] as const; + +export const ESP8266_REFUSAL = + "ESP8266 firmware images are not supported: they share the 0xE9 magic but have an 8-byte header without the ESP32 extended header"; + +export type EspImageRecognition = "esp-image" | "esp8266"; + +/** A whole-image refusal; no partial facts accompany it. */ +export interface EspImageFailure { + readonly kind: "format" | "unsupported"; + readonly message: string; +} + +export interface EspImageDecoder { + /** Consume the next file bytes in order; chunks may split any field. */ + write(chunk: Uint8Array): void; + /** True once later bytes can no longer change the result. */ + readonly satisfied: boolean; + /** Decode after `satisfied` or after the last byte of the file. */ + finish(): Result; +} + +const alignUp = (value: number, alignment: number): number => + Math.ceil(value / alignment) * alignment; + +/** + * Header-level recognition. An ESP8266 v1 or v3 image has its first segment + * header at offset 8, loading into ESP8266 memory; an ESP32-family header + * holds `wp_pin` and drive settings there, which esptool and ESP-IDF write + * as 0xEE and zeros. + */ +const recognizeHeader = ( + prefix: Uint8Array, + fileSize: number, +): EspImageRecognition | undefined => { + if (!Number.isSafeInteger(fileSize) || prefix[0] !== ESP_IMAGE_MAGIC) + return undefined; + const count = prefix[1] ?? 0; + if (count > ESP_IMAGE_MAX_SEGMENTS) return undefined; + if (count > 0 && prefix.length >= 16 && fileSize >= 16) { + const firstLoad = u32(prefix, 8); + if ( + ESP8266_LOAD_REGIONS.some( + ([low, high]) => firstLoad >= low && firstLoad < high, + ) && + u32(prefix, 12) % 4 === 0 + ) + return "esp8266"; + } + if ( + fileSize < ESP_IMAGE_HEADER_BYTES || + prefix.length < ESP_IMAGE_HEADER_BYTES + ) + return undefined; + // Values outside ESP-IDF's enums, and a hash_appended byte esptool refuses. + return (prefix[2] ?? 0) > LAST_SPI_MODE || + !SPI_SPEEDS.has((prefix[3] ?? 0) & 0x0f) || + (prefix[23] ?? 0) > 1 + ? undefined + : "esp-image"; +}; + +/** + * Recognize an image by its leading bytes and size, never by name: the + * header checks, at least one segment, and a word-aligned first segment + * length when the file holds that header. A zero count is valid but loads + * nothing and would admit x86 programs that begin `E9 00`. Later segments, + * the checksum and the digest are integrity facts, so truncation after the + * first segment header leaves the image recognized. + */ +export const recognizeEspImage = ( + prefix: Uint8Array, + fileSize: number, +): EspImageRecognition | undefined => { + const recognized = recognizeHeader(prefix, fileSize); + const firstHeaderEnd = + ESP_IMAGE_HEADER_BYTES + ESP_IMAGE_SEGMENT_HEADER_BYTES; + if (recognized !== "esp-image") return recognized; + if (prefix[1] === 0) return undefined; + if (fileSize < firstHeaderEnd) return recognized; + if (prefix.length < firstHeaderEnd) return undefined; + return u32(prefix, 28) % 4 === 0 ? "esp-image" : undefined; +}; + +/** Start a decoder that reads each byte once and retains bounded state. */ +export const createEspImageDecoder = (fileSize: number): EspImageDecoder => + new StreamingEspImageDecoder(fileSize); + +/** File regions in order; each ends where the next begins. */ +type Stage = + | "header" + | "segment-header" + | "segment-data" + | "padding" + | "checksum" + | "digest" + | "gap" + | "sector" + | "rest"; + +class StreamingEspImageDecoder implements EspImageDecoder { + readonly #fileSize: number; + readonly #header = new Uint8Array(ESP_IMAGE_HEADER_BYTES); + readonly #segmentHeader = new Uint8Array(ESP_IMAGE_SEGMENT_HEADER_BYTES); + readonly #segments: ObservedSegmentHeader[] = []; + readonly #descriptor = new Uint8Array(ESP_APP_DESC_BYTES); + readonly #digest = new Uint8Array(ESP_IMAGE_DIGEST_BYTES); + readonly #sector = new Uint8Array(ESP_FLASH_SECTOR_BYTES); + /** Hash every byte before the sector; signatures cover all of them. */ + readonly #sha256: Hash = createHash("sha256"); + readonly #sha384: Hash = createHash("sha384"); + #stage: Stage = "header"; + #position = 0; + #regionStart = 0; + #regionEnd = ESP_IMAGE_HEADER_BYTES; + #segmentCount = 0; + #checksum = CHECKSUM_SEED; + #descriptorBytes = 0; + #checksumOffset: number | undefined; + #storedChecksum: number | undefined; + #imageSha256: string | undefined; + #imageEnd: number | undefined; + #sectorOffset: number | undefined; + #gapFirstUnerased: number | undefined; + #restFirstUnerased: number | undefined; + #contentSha256: string | undefined; + #contentSha384: string | undefined; + #failure: EspImageFailure | undefined; + + constructor(fileSize: number) { + this.#fileSize = fileSize; + } + + /** Reading stops at the first byte after the sector that is not erased. */ + get satisfied(): boolean { + return ( + this.#failure !== undefined || + (this.#stage === "rest" && this.#restFirstUnerased !== undefined) + ); + } + + write(chunk: Uint8Array): void { + let index = 0; + for (;;) { + // Zero-length regions, such as an empty segment, end immediately. + while (!this.satisfied && this.#position >= this.#regionEnd) + this.#advance(); + if (this.satisfied || index >= chunk.length) return; + const length = Math.min( + chunk.length - index, + this.#regionEnd - this.#position, + ); + this.#consume(chunk.subarray(index, index + length)); + index += length; + } + } + + finish(): Result { + if (this.#failure !== undefined) return err(this.#failure); + if (!this.satisfied && this.#position !== this.#fileSize) + return err({ + kind: "format", + message: `ESP image input ended at offset ${offsetText(this.#position)}, before the admitted size ${offsetText(this.#fileSize)}`, + }); + if (this.#stage === "header") + return err({ + kind: "format", + message: `ESP image header needs ${String(ESP_IMAGE_HEADER_BYTES)} bytes; the file holds ${String(this.#position)}`, + }); + return ok( + deriveEspImage({ + fileSize: this.#fileSize, + observedEnd: this.#position, + header: this.#header, + segments: this.#segments, + descriptorBytes: this.#descriptor.subarray(0, this.#descriptorBytes), + checksum: this.#checksum, + checksumOffset: this.#checksumOffset, + storedChecksum: this.#storedChecksum, + imageSha256: this.#imageSha256, + storedDigest: this.#digest, + imageEnd: this.#imageEnd, + sectorOffset: this.#sectorOffset, + gapFirstUnerased: this.#gapFirstUnerased, + sector: this.#sector, + restFirstUnerased: this.#restFirstUnerased, + signedContent: { + sha256: this.#contentSha256, + sha384: this.#contentSha384, + }, + }), + ); + } + + #consume(bytes: Uint8Array): void { + const within = this.#position - this.#regionStart; + switch (this.#stage) { + case "header": + this.#header.set(bytes, this.#position); + break; + case "segment-header": + this.#segmentHeader.set(bytes, within); + break; + case "segment-data": { + let checksum = this.#checksum; + for (let index = 0; index < bytes.length; index += 1) + checksum ^= bytes[index] ?? 0; + this.#checksum = checksum; + if ( + this.#segments.length === 1 && + this.#descriptorBytes < ESP_APP_DESC_BYTES + ) { + const captured = bytes.subarray( + 0, + ESP_APP_DESC_BYTES - this.#descriptorBytes, + ); + this.#descriptor.set(captured, this.#descriptorBytes); + this.#descriptorBytes += captured.length; + } + break; + } + case "checksum": + this.#storedChecksum = bytes[0]; + break; + case "digest": + this.#digest.set(bytes, within); + break; + case "gap": + this.#gapFirstUnerased ??= firstUnerased(bytes, this.#position); + break; + case "sector": + this.#sector.set(bytes, within); + break; + case "rest": + this.#restFirstUnerased ??= firstUnerased(bytes, this.#position); + break; + case "padding": + break; + } + // Regions never span the sector boundary, so this is exact. + if ( + this.#sectorOffset === undefined || + this.#position < this.#sectorOffset + ) { + this.#sha256.update(bytes); + this.#sha384.update(bytes); + } + this.#position += bytes.length; + } + + /** Enter the next region once the current one is complete. */ + #advance(): void { + switch (this.#stage) { + case "header": { + const recognized = recognizeHeader(this.#header, this.#fileSize); + if (recognized !== "esp-image") { + this.#failure = + recognized === "esp8266" + ? { kind: "unsupported", message: ESP8266_REFUSAL } + : { + kind: "format", + message: + "ESP image header is not recognized: it needs magic 0xe9, at most 16 segments, ESP-IDF flash mode and speed codes, and a hash_appended byte of 0 or 1", + }; + return; + } + this.#segmentCount = this.#header[1] ?? 0; + this.#nextSegment(); + return; + } + case "segment-header": { + const length = u32(this.#segmentHeader, 4); + this.#segments.push({ + index: this.#segments.length, + headerOffset: this.#regionStart, + loadAddress: u32(this.#segmentHeader, 0), + length, + }); + this.#enter("segment-data", length); + return; + } + case "segment-data": + this.#nextSegment(); + return; + case "padding": + this.#enter("checksum", 1); + return; + case "checksum": + // The appended digest covers every byte through the checksum. + this.#imageSha256 = this.#sha256.copy().digest("hex"); + if (this.#header[23] === 1) + this.#enter("digest", ESP_IMAGE_DIGEST_BYTES); + else this.#endImage(); + return; + case "digest": + this.#endImage(); + return; + case "gap": + this.#contentSha256 = this.#sha256.digest("hex"); + this.#contentSha384 = this.#sha384.digest("hex"); + this.#enter("sector", ESP_FLASH_SECTOR_BYTES); + return; + case "sector": + this.#enter("rest", Number.POSITIVE_INFINITY); + return; + case "rest": + return; + } + } + + #enter(stage: Stage, length: number): void { + this.#stage = stage; + this.#regionStart = this.#position; + this.#regionEnd = this.#position + length; + } + + /** Read the next declared segment, or the checksum after the last one. */ + #nextSegment(): void { + if (this.#segments.length < this.#segmentCount) { + this.#enter("segment-header", ESP_IMAGE_SEGMENT_HEADER_BYTES); + return; + } + // The checksum is the last byte of the next 16-byte block. + this.#checksumOffset = alignUp(this.#position + 1, 16) - 1; + this.#enter("padding", this.#checksumOffset - this.#position); + } + + #endImage(): void { + this.#imageEnd = this.#position; + // A signature sector begins at the next 4 KiB flash sector boundary. + this.#sectorOffset = alignUp(this.#position, ESP_FLASH_SECTOR_BYTES); + this.#enter("gap", this.#sectorOffset - this.#position); + } +} diff --git a/src/domain/firmware/espImageFacts.ts b/src/domain/firmware/espImageFacts.ts new file mode 100644 index 0000000000..c31d170d85 --- /dev/null +++ b/src/domain/firmware/espImageFacts.ts @@ -0,0 +1,511 @@ +import type { + EspImageMetadata, + EspImageSegmentMetadata, + EspImageSignatureBlockMetadata, +} from "../artifactFormatMetadata.js"; +import { + ESP_CHIP_TABLE_SOURCE, + classifyEspSegment, + espChipForId, + espPageSizedMappingAmbiguity, + espSegmentLengthLimit, + type EspChip, +} from "./espChips.js"; +import { + ESP_FLASH_SECTOR_BYTES, + ESP_IMAGE_DIGEST_BYTES, + ESP_IMAGE_SEGMENT_HEADER_BYTES, + ESP_SIGNATURE_BLOCK_BYTES, + decodeEspDescriptor, + decodeEspSignatureBlock, + espSignatureSlot, + firstUnerased, + hex, + offsetText, + u16, + u32, + type EspSignedContentDigests, +} from "./espImageStructures.js"; + +const SIGNATURE_SLOTS = 3; + +/** One header-declared segment; `availableLength` is what the file holds. */ +export interface EspImageSegment { + readonly dataOffset: number; + readonly availableLength: number; + readonly metadata: EspImageSegmentMetadata; +} + +export interface EspImageSignatureBlock { + readonly offset: number; + readonly metadata: EspImageSignatureBlockMetadata; +} + +export interface EspImage { + readonly metadata: EspImageMetadata; + readonly segments: readonly EspImageSegment[]; + readonly signatureBlocks: readonly EspImageSignatureBlock[]; + /** Unknown when truncation or undescribed bytes may hide content. */ + readonly coverage: "complete" | "unknown"; + readonly limitations: readonly string[]; +} + +export interface ObservedSegmentHeader { + readonly index: number; + readonly headerOffset: number; + readonly loadAddress: number; + readonly length: number; +} + +/** + * What one in-order pass over an image observed. `observedEnd` is the end of + * the last chunk read: the file size, or the end of the chunk that holds the + * first non-erased byte after the signature sector, where reading stops. + */ +export interface EspImageObservations { + readonly fileSize: number; + readonly observedEnd: number; + readonly header: Uint8Array; + readonly segments: readonly ObservedSegmentHeader[]; + /** Up to the first 256 data bytes of segment 0. */ + readonly descriptorBytes: Uint8Array; + /** XOR of the observed segment data bytes, seeded with 0xEF. */ + readonly checksum: number; + readonly checksumOffset: number | undefined; + readonly storedChecksum: number | undefined; + /** SHA-256 of every byte through the checksum. */ + readonly imageSha256: string | undefined; + readonly storedDigest: Uint8Array; + readonly imageEnd: number | undefined; + readonly sectorOffset: number | undefined; + /** First byte between the image and its sector that is not 0xFF. */ + readonly gapFirstUnerased: number | undefined; + /** The 4 KiB from the sector boundary, as far as the file holds them. */ + readonly sector: Uint8Array; + /** First byte after that sector that is not 0xFF. */ + readonly restFirstUnerased: number | undefined; + readonly signedContent: EspSignedContentDigests; +} + +type ByteRange = EspImageMetadata["undescribed_trailing"][number]; + +const hexByte = (value: number): string => + `0x${value.toString(16).padStart(2, "0")}`; + +/** + * How each ESP image fact REA could not establish is reported. Decoding, + * inventory and tests use these, so wording has one owner. + */ +export const ESP_IMAGE_LIMITATIONS = { + /** What an ESP image inventory reads, and what it leaves unverified. */ + inventoryScope: + "ESP image inventory reads the header-declared segments, checksum, appended SHA-256, descriptor and Secure Boot v2 signature blocks. Segment classification follows ESP-IDF v6.1 should_map and should_load for a cold boot and does not check RAM ranges; signatures are not cryptographically verified, and code is not disassembled.", + undescribedTrailing: ( + bytes: number, + ranges: number, + firstOffset: number, + ): string => + `${String(bytes)} bytes after the ESP image, in ${String(ranges)} range(s) from offset ${offsetText(firstOffset)}, are not described by its header, erased flash or a signature sector. They may be other flash contents, a Secure Boot v1 signature, or flash segments that esptool --ram-only-header leaves out of the segment count.`, + unknownChip: (chipId: number): string => + `ESP image chip_id 0x${chipId.toString(16).padStart(4, "0")} is not in the ${ESP_CHIP_TABLE_SOURCE} chip table; chip, ISA and segment classification are unknown.`, + unclassifiedPages: (target: string, low: number, high: number): string => + `${target} maps flash through 256 MMU pages of the bootloader's configured size, which the image does not record; segments from ${offsetText(low)} to ${offsetText(high)} are unclassified.`, + segmentLengthUnaligned: (index: number, length: number): string => + `ESP image segment ${String(index)} length ${offsetText(length)} is not a multiple of 4; ESP-IDF rejects such an image.`, + segmentLengthTooLarge: ( + index: number, + length: number, + limit: number, + target: string, + ): string => + `ESP image segment ${String(index)} length ${offsetText(length)} is ${String(limit / 0x10_0000)} MiB or more; ESP-IDF rejects such an image for ${target}.`, + truncatedSegment: ( + index: number, + declared: number, + offset: number, + available: number, + ): string => + `ESP image is truncated: segment ${String(index)} declares ${String(declared)} bytes at offset ${offsetText(offset)}, but the file holds ${String(available)}.`, + truncatedSegmentHeaders: (declared: number, present: number): string => + `ESP image is truncated: the header declares ${String(declared)} segments, but the file ends before the header of segment ${String(present)}.`, + truncatedBeforeChecksum: (offset: number): string => + `ESP image is truncated before its checksum at offset ${offsetText(offset)}.`, + checksumMismatch: ( + stored: number, + offset: number, + computed: number, + ): string => + `ESP image checksum ${hexByte(stored)} at offset ${offsetText(offset)} does not match the computed ${hexByte(computed)}.`, + truncatedBeforeDigest: (offset: number): string => + `ESP image is truncated before its appended SHA-256 at offset ${offsetText(offset)}.`, + digestMismatch: (offset: number): string => + `ESP image appended SHA-256 at offset ${offsetText(offset)} does not match the SHA-256 of the bytes before it.`, + segmentEntryTruncated: ( + index: number, + declared: number, + available: number, + ): string => + `ESP image segment ${String(index)} is truncated: its header declares ${String(declared)} bytes; the file holds ${String(available)}.`, + signatureDigestMismatch: (slot: number, offset: number): string => + `Secure Boot v2 signature block in slot ${String(slot)} at offset ${offsetText(offset)} records an image digest that does not match the bytes before its sector.`, + invalidSignatureSlot: (slot: number, offset: number): string => + `Secure Boot v2 slot ${String(slot)} at offset ${offsetText(offset)} has the 0xE7 magic but not a matching CRC32 or a complete block; it is not a valid signature block.`, + signatureSlotsCut: (fileSize: number, sector: number): string => + `The file ends at ${offsetText(fileSize)} in erased flash, before the Secure Boot v2 signature slots at ${offsetText(sector)} are complete; signature blocks may be missing.`, + signatureSectorTruncated: ( + sector: number, + fileSize: number, + sectorEnd: number, + ): string => + `Secure Boot v2 signature sector at offset ${offsetText(sector)} is truncated: the file ends at ${offsetText(fileSize)}, before ${offsetText(sectorEnd)}.`, +} as const; + +/** Derive chip, classification, integrity, descriptor and coverage facts. */ +export const deriveEspImage = (observed: EspImageObservations): EspImage => { + const limitations: string[] = []; + const chipId = u16(observed.header, 12); + const chip = espChipForId(chipId); + const first = observed.segments[0]; + const descriptor = + first === undefined + ? null + : decodeEspDescriptor( + observed.descriptorBytes, + first.headerOffset + ESP_IMAGE_SEGMENT_HEADER_BYTES, + limitations, + ); + const segments = observed.segments.map((segment) => + segmentFacts(observed, segment, chip), + ); + limitations.push(...segmentLimitations(chipId, chip, segments)); + const truncated = truncation(observed, segments, limitations); + const checksum = checksumFacts(observed, truncated, limitations); + const appendedSha256 = digestFacts(observed, limitations); + const trailing = trailingFacts(observed, limitations); + const undescribedBytes = trailing.undescribed.reduce( + (total, { length }) => total + length, + 0, + ); + const firstUndescribed = trailing.undescribed[0]; + if (firstUndescribed !== undefined) + limitations.push( + ESP_IMAGE_LIMITATIONS.undescribedTrailing( + undescribedBytes, + trailing.undescribed.length, + firstUndescribed.offset, + ), + ); + return { + metadata: { + kind: "esp-image", + header: headerFacts(observed.header), + chip: + chip === undefined + ? null + : { + target: chip.target, + isa: chip.isa, + source: ESP_CHIP_TABLE_SOURCE, + }, + checksum, + appended_sha256: appendedSha256, + image_end: observed.imageEnd ?? null, + signature_sector_offset: + trailing.blocks.length > 0 ? (observed.sectorOffset ?? null) : null, + undescribed_trailing: trailing.undescribed, + descriptor, + }, + segments, + signatureBlocks: trailing.blocks, + // Signature blocks follow the image end, so a file cut before it may + // hide them even when every declared segment is present. + coverage: + truncated || + !imageComplete(observed) || + trailing.undescribed.length > 0 || + trailing.truncated + ? "unknown" + : "complete", + limitations, + }; +}; + +/** Whether the file holds the checksum and any appended digest. */ +const imageComplete = (observed: EspImageObservations): boolean => + observed.imageEnd !== undefined; + +/** `esp_image_header_t` fields, observed as written. */ +const headerFacts = (header: Uint8Array): EspImageMetadata["header"] => ({ + segment_count: header[1] ?? 0, + spi_mode: header[2] ?? 0, + spi_speed: (header[3] ?? 0) & 0x0f, + spi_size: (header[3] ?? 0) >> 4, + entry_addr: u32(header, 4), + wp_pin: header[8] ?? 0, + spi_pin_drv: hex(header.subarray(9, 12)), + chip_id: u16(header, 12), + min_chip_rev: header[14] ?? 0, + min_chip_rev_full: u16(header, 15), + max_chip_rev_full: u16(header, 17), + reserved: hex(header.subarray(19, 23)), + hash_appended: header[23] === 1, +}); + +const segmentFacts = ( + observed: EspImageObservations, + segment: ObservedSegmentHeader, + chip: EspChip | undefined, +): EspImageSegment => { + const dataOffset = segment.headerOffset + ESP_IMAGE_SEGMENT_HEADER_BYTES; + return { + dataOffset, + availableLength: Math.max( + 0, + Math.min(segment.length, observed.observedEnd - dataOffset), + ), + metadata: { + kind: "esp-image-segment", + index: segment.index, + load_addr: segment.loadAddress, + data_len: segment.length, + header_offset: segment.headerOffset, + classification: classifyEspSegment(chip, segment.loadAddress), + }, + }; +}; + +/** Classification gaps and segment lengths ESP-IDF rejects. */ +const segmentLimitations = ( + chipId: number, + chip: EspChip | undefined, + segments: readonly EspImageSegment[], +): string[] => { + const limitations: string[] = []; + const ambiguity = + chip === undefined ? undefined : espPageSizedMappingAmbiguity(chip); + if (chip === undefined) + limitations.push(ESP_IMAGE_LIMITATIONS.unknownChip(chipId)); + else if ( + ambiguity !== undefined && + segments.some(({ metadata }) => metadata.classification === "unknown") + ) + limitations.push( + ESP_IMAGE_LIMITATIONS.unclassifiedPages( + chip.target, + ambiguity.low, + ambiguity.high, + ), + ); + const lengthLimit = + chip === undefined ? undefined : espSegmentLengthLimit(chip); + for (const { metadata } of segments) { + if (metadata.data_len % 4 !== 0) + limitations.push( + ESP_IMAGE_LIMITATIONS.segmentLengthUnaligned( + metadata.index, + metadata.data_len, + ), + ); + if ( + chip !== undefined && + lengthLimit !== undefined && + metadata.data_len >= lengthLimit + ) + limitations.push( + ESP_IMAGE_LIMITATIONS.segmentLengthTooLarge( + metadata.index, + metadata.data_len, + lengthLimit, + chip.target, + ), + ); + } + return limitations; +}; + +/** Report the first missing structure; later structures cannot be located. */ +const truncation = ( + observed: EspImageObservations, + segments: readonly EspImageSegment[], + limitations: string[], +): boolean => { + const declared = observed.header[1] ?? 0; + const short = segments.find( + ({ availableLength, metadata }) => availableLength < metadata.data_len, + ); + if (short !== undefined) { + limitations.push( + ESP_IMAGE_LIMITATIONS.truncatedSegment( + short.metadata.index, + short.metadata.data_len, + short.dataOffset, + short.availableLength, + ), + ); + return true; + } + if (segments.length < declared) { + limitations.push( + ESP_IMAGE_LIMITATIONS.truncatedSegmentHeaders(declared, segments.length), + ); + return true; + } + return false; +}; + +/** XOR of every segment data byte, seeded with 0xEF. */ +const checksumFacts = ( + observed: EspImageObservations, + truncated: boolean, + limitations: string[], +): EspImageMetadata["checksum"] => { + const offset = observed.checksumOffset ?? null; + const computed = truncated ? null : observed.checksum; + const stored = observed.storedChecksum ?? null; + if (offset === null || computed === null || stored === null) { + if (!truncated && offset !== null) + limitations.push(ESP_IMAGE_LIMITATIONS.truncatedBeforeChecksum(offset)); + return { offset, stored, computed, status: "unavailable" }; + } + if (stored !== computed) + limitations.push( + ESP_IMAGE_LIMITATIONS.checksumMismatch(stored, offset, computed), + ); + return { + offset, + stored, + computed, + status: stored === computed ? "valid" : "invalid", + }; +}; + +/** SHA-256 of every byte through the checksum, appended after it. */ +const digestFacts = ( + observed: EspImageObservations, + limitations: string[], +): EspImageMetadata["appended_sha256"] => { + if (observed.header[23] !== 1) + return { + offset: null, + stored: null, + computed: null, + status: "not-appended", + }; + const offset = + observed.checksumOffset === undefined ? null : observed.checksumOffset + 1; + const computed = observed.imageSha256 ?? null; + if ( + offset === null || + computed === null || + observed.observedEnd < offset + ESP_IMAGE_DIGEST_BYTES + ) { + if (computed !== null && offset !== null) + limitations.push(ESP_IMAGE_LIMITATIONS.truncatedBeforeDigest(offset)); + return { offset, stored: null, computed, status: "unavailable" }; + } + const stored = hex(observed.storedDigest); + if (stored !== computed) + limitations.push(ESP_IMAGE_LIMITATIONS.digestMismatch(offset)); + return { + offset, + stored, + computed, + status: stored === computed ? "valid" : "invalid", + }; +}; + +/** + * Describe every byte after the image: erased flash (0xFF) and, at the next + * 4 KiB boundary, a signature sector holding at least one valid block. Each + * of the three slots is examined independently, as ESP-IDF's bootloader + * does. In unstructured regions (the padding, the sector without a block or + * its slack, the bytes after it) the first non-erased byte starts a range to + * the region's end; a slot that is neither a block nor erased is reported + * whole. Ranges are merged when adjacent. + */ +const trailingFacts = ( + observed: EspImageObservations, + limitations: string[], +): { + readonly blocks: EspImageSignatureBlock[]; + readonly undescribed: ByteRange[]; + readonly truncated: boolean; +} => { + const { imageEnd: end, sectorOffset: sector, fileSize } = observed; + const blocks: EspImageSignatureBlock[] = []; + const undescribed: ByteRange[] = []; + if (end === undefined || sector === undefined || fileSize <= end) + return { blocks, undescribed, truncated: false }; + const add = (from: number | undefined, to: number): void => { + if (from === undefined || to <= from) return; + const last = undescribed.at(-1); + if (last !== undefined && last.offset + last.length === from) + undescribed[undescribed.length - 1] = { + offset: last.offset, + length: to - last.offset, + }; + else undescribed.push({ offset: from, length: to - from }); + }; + add(observed.gapFirstUnerased, Math.min(sector, fileSize)); + const held = observed.sector.subarray( + 0, + Math.max(0, Math.min(ESP_FLASH_SECTOR_BYTES, fileSize - sector)), + ); + const slots = Array.from({ length: SIGNATURE_SLOTS }, (_, index) => { + const from = index * ESP_SIGNATURE_BLOCK_BYTES; + const bytes = held.subarray(from, from + ESP_SIGNATURE_BLOCK_BYTES); + return { + index, + offset: sector + from, + bytes, + kind: bytes.length === 0 ? undefined : espSignatureSlot(bytes), + }; + }); + for (const { index, offset, bytes, kind } of slots) { + if (kind === "block") { + const metadata = decodeEspSignatureBlock( + bytes, + index, + observed.signedContent, + ); + blocks.push({ offset, metadata }); + // A definite contradiction, like a checksum mismatch; the signature + // itself stays unverified. + if (metadata.image_digest_matches === false) + limitations.push( + ESP_IMAGE_LIMITATIONS.signatureDigestMismatch(index, offset), + ); + } else if (kind === "other" && bytes[0] === 0xe7) + limitations.push( + ESP_IMAGE_LIMITATIONS.invalidSignatureSlot(index, offset), + ); + } + const sectorEnd = sector + ESP_FLASH_SECTOR_BYTES; + const slotsEnd = sector + SIGNATURE_SLOTS * ESP_SIGNATURE_BLOCK_BYTES; + if (blocks.length === 0) { + // Without a block, the sector is ordinary trailing bytes. + add(firstUnerased(held, sector), sector + held.length); + add(observed.restFirstUnerased, fileSize); + if (undescribed.length > 0 || fileSize >= slotsEnd) + return { blocks, undescribed, truncated: false }; + // Erased padding cut short may be a signed image that lost its sector. + limitations.push(ESP_IMAGE_LIMITATIONS.signatureSlotsCut(fileSize, sector)); + return { blocks, undescribed, truncated: true }; + } + for (const { offset, bytes, kind } of slots) + if (kind === "other") add(offset, offset + bytes.length); + const slack = held.subarray(SIGNATURE_SLOTS * ESP_SIGNATURE_BLOCK_BYTES); + add(firstUnerased(slack, slotsEnd), sector + held.length); + add(observed.restFirstUnerased, fileSize); + if (fileSize < sectorEnd) { + limitations.push( + ESP_IMAGE_LIMITATIONS.signatureSectorTruncated( + sector, + fileSize, + sectorEnd, + ), + ); + return { blocks, undescribed, truncated: true }; + } + return { blocks, undescribed, truncated: false }; +}; diff --git a/src/domain/firmware/espImageStructures.ts b/src/domain/firmware/espImageStructures.ts new file mode 100644 index 0000000000..2f2ef7276a --- /dev/null +++ b/src/domain/firmware/espImageStructures.ts @@ -0,0 +1,236 @@ +import { createHash } from "node:crypto"; +import { crc32 } from "node:zlib"; + +import type { + EspImageMetadata, + EspImageSignatureBlockMetadata, +} from "../artifactFormatMetadata.js"; + +/* + * Fixed-layout structures inside an ESP image, from ESP-IDF v6.1 + * (`esp_app_format.h`, `esp_image_format.h`, `esp_app_desc.h`, + * `esp_bootloader_desc.h`, `esp_bootloader_desc.c`, the ROM `secure_boot.h` + * headers, `docs/en/security/secure-boot-v2.rst`) and esptool v5.6.0's + * espsecure. + */ +export const ESP_IMAGE_SEGMENT_HEADER_BYTES = 8; +/** `ESP_IMAGE_HASH_LEN`: the SHA-256 appended after the checksum. */ +export const ESP_IMAGE_DIGEST_BYTES = 32; +/** Flash sector; a signature sector begins at the next such boundary. */ +export const ESP_FLASH_SECTOR_BYTES = 4096; +export const ESP_APP_DESC_BYTES = 256; +export const ESP_SIGNATURE_BLOCK_BYTES = 1216; +const ERASED = 0xff; +const APP_DESC_MAGIC = 0xabcd_5432; +const BOOTLOADER_DESC_MAGIC = 0x50; +const BOOTLOADER_DESC_BYTES = 80; +const SIGNATURE_MAGIC = 0xe7; +const SIGNATURE_CRC_OFFSET = 1196; + +export const u32 = (bytes: Uint8Array, at: number): number => + ((bytes[at] ?? 0) | + ((bytes[at + 1] ?? 0) << 8) | + ((bytes[at + 2] ?? 0) << 16) | + ((bytes[at + 3] ?? 0) << 24)) >>> + 0; + +export const u16 = (bytes: Uint8Array, at: number): number => + (bytes[at] ?? 0) | ((bytes[at + 1] ?? 0) << 8); + +export const hex = (bytes: Uint8Array): string => + Buffer.from(bytes).toString("hex"); + +export const offsetText = (value: number): string => `0x${value.toString(16)}`; + +/** File offset of the first byte of `bytes` that is not erased flash, if any. */ +export const firstUnerased = ( + bytes: Uint8Array, + offset: number, +): number | undefined => { + const index = bytes.findIndex((byte) => byte !== ERASED); + return index < 0 ? undefined : offset + index; +}; + +const allZero = (bytes: Uint8Array): boolean => + bytes.every((byte) => byte === 0); + +const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + +/** A NUL-terminated C string in a fixed array, or null when not UTF-8. */ +const cString = (bytes: Uint8Array): string | null => { + const end = bytes.indexOf(0); + try { + return UTF8.decode(end < 0 ? bytes : bytes.subarray(0, end)); + } catch { + return null; + } +}; + +/** How a descriptor REA could not read is reported; tests compare with these. */ +export const ESP_DESCRIPTOR_LIMITATIONS = { + fieldNotUtf8: (field: string): string => + `ESP image descriptor field ${field} is not valid UTF-8; its text is unknown.`, + reservedBytesNotZero: + "ESP image segment 0 begins with the bootloader descriptor magic 0x50, but its reserved bytes are not zero; it is not read as an esp_bootloader_desc_t.", +} as const; + +/** + * Decode the descriptor that begins segment 0: `esp_app_desc_t`, which + * ESP-IDF's bootloader reads there, or `esp_bootloader_desc_t`, which esptool + * places there. The bootloader descriptor's magic is one byte, so its + * reserved fields, which ESP-IDF zero-fills, must also be zero. + */ +export const decodeEspDescriptor = ( + bytes: Uint8Array, + offset: number, + limitations: string[], +): EspImageMetadata["descriptor"] => { + const text = (from: number, length: number, field: string): string | null => { + const value = cString(bytes.subarray(from, from + length)); + if (value === null) + limitations.push(ESP_DESCRIPTOR_LIMITATIONS.fieldNotUtf8(field)); + return value; + }; + if (bytes.length >= 4 && u32(bytes, 0) === APP_DESC_MAGIC) { + // A file can end inside the descriptor. Each field is kept when the file + // holds all of its bytes; a field it cuts off is null. + const held = (end: number): boolean => bytes.length >= end; + return { + kind: "app", + offset, + observed_length: Math.min(bytes.length, ESP_APP_DESC_BYTES), + secure_version: held(8) ? u32(bytes, 4) : null, + version: held(48) ? text(16, 32, "version") : null, + project_name: held(80) ? text(48, 32, "project_name") : null, + time: held(96) ? text(80, 16, "time") : null, + date: held(112) ? text(96, 16, "date") : null, + idf_ver: held(144) ? text(112, 32, "idf_ver") : null, + app_elf_sha256: held(176) ? hex(bytes.subarray(144, 176)) : null, + min_efuse_blk_rev_full: held(178) ? u16(bytes, 176) : null, + max_efuse_blk_rev_full: held(180) ? u16(bytes, 178) : null, + mmu_page_size: bytes[180] ?? null, + spi_flash_mode: bytes[181] ?? null, + }; + } + if ( + bytes.length < BOOTLOADER_DESC_BYTES || + bytes[0] !== BOOTLOADER_DESC_MAGIC + ) + return null; + if (!allZero(bytes.subarray(1, 3)) || !allZero(bytes.subarray(64, 80))) { + limitations.push(ESP_DESCRIPTOR_LIMITATIONS.reservedBytesNotZero); + return null; + } + return { + kind: "bootloader", + offset, + secure_version: bytes[3] ?? 0, + version: u32(bytes, 4), + idf_ver: text(8, 32, "idf_ver"), + date_time: text(40, 24, "date_time"), + }; +}; + +/** The bytes of one sector slot: a valid block, erased flash, or neither. */ +export type EspSignatureSlot = "block" | "erased" | "other"; + +/** + * A slot holds a block when it has the 0xE7 magic and a CRC32 of its first + * 1196 bytes at offset 1196, the validity rule of ESP-IDF and espsecure. + */ +export const espSignatureSlot = (slot: Uint8Array): EspSignatureSlot => { + if ( + slot.length === ESP_SIGNATURE_BLOCK_BYTES && + slot[0] === SIGNATURE_MAGIC && + crc32(slot.subarray(0, SIGNATURE_CRC_OFFSET)) === + u32(slot, SIGNATURE_CRC_OFFSET) + ) + return "block"; + return slot.every((byte) => byte === ERASED) ? "erased" : "other"; +}; + +/** Field offsets inside a signature block, from the ROM `secure_boot.h`. */ +interface SignatureLayout { + readonly digestBytes: 32 | 48; + readonly keyStart: number; + readonly keyEnd: number; + readonly curveAt: number | undefined; +} + +const RSA_LAYOUT: SignatureLayout = { + digestBytes: 32, + keyStart: 36, + keyEnd: 812, + curveAt: undefined, +}; +/** ECDSA layouts by `sha_version`: 0 is P-192/P-256, 1 is P-384. */ +const ECDSA_LAYOUTS: Readonly> = { + 0: { digestBytes: 32, keyStart: 36, keyEnd: 101, curveAt: 36 }, + 1: { digestBytes: 48, keyStart: 52, keyEnd: 149, curveAt: 52 }, +}; + +/** Digests of every byte before the signature sector, when present. */ +export interface EspSignedContentDigests { + readonly sha256: string | undefined; + readonly sha384: string | undefined; +} + +/** + * Decode a valid block's declared facts without verifying the signature. + * Version 0x02 is RSA-PSS and 0x03 ECDSA; any other version or ECDSA + * `sha_version` leaves the layout, and so every layout-dependent fact, + * unknown. The public-key digest covers the bytes espsecure + * `signature-info-v2` hashes. + */ +export const decodeEspSignatureBlock = ( + block: Uint8Array, + index: number, + content: EspSignedContentDigests, +): EspImageSignatureBlockMetadata => { + const version = block[1] ?? 0; + const scheme = version === 0x02 ? "rsa" : version === 0x03 ? "ecdsa" : null; + const shaVersion = scheme === "ecdsa" ? (block[2] ?? 0) : null; + const layout = + scheme === "rsa" + ? RSA_LAYOUT + : shaVersion === null + ? undefined + : ECDSA_LAYOUTS[shaVersion]; + const unknown = { + curve_id: null, + digest_algorithm: null, + image_digest: null, + image_digest_matches: null, + public_key_sha256: null, + }; + if (layout === undefined) + return { + kind: "esp-image-signature-block", + index, + version, + scheme, + sha_version: shaVersion, + ...unknown, + signature_status: "unverified", + }; + const digestAlgorithm = layout.digestBytes === 48 ? "sha384" : "sha256"; + const imageDigest = hex(block.subarray(4, 4 + layout.digestBytes)); + const expected = content[digestAlgorithm]; + return { + kind: "esp-image-signature-block", + index, + version, + scheme, + sha_version: shaVersion, + curve_id: + layout.curveAt === undefined ? null : (block[layout.curveAt] ?? 0), + digest_algorithm: digestAlgorithm, + image_digest: imageDigest, + image_digest_matches: + expected === undefined ? null : expected === imageDigest, + public_key_sha256: createHash("sha256") + .update(block.subarray(layout.keyStart, layout.keyEnd)) + .digest("hex"), + signature_status: "unverified", + }; +}; diff --git a/src/domain/liveBinaryTargetSummary.ts b/src/domain/liveBinaryTargetSummary.ts index 23185fbe4d..f34e5c8013 100644 --- a/src/domain/liveBinaryTargetSummary.ts +++ b/src/domain/liveBinaryTargetSummary.ts @@ -77,6 +77,7 @@ export const liveBinaryTargetSummaryVariants = [ "pkg", "intel-hex", "srec", + "esp-image", ]), architecture: z.null(), }), diff --git a/tests/acceptance/artifactComparisonEspImage.test.ts b/tests/acceptance/artifactComparisonEspImage.test.ts new file mode 100644 index 0000000000..d3c3a15f52 --- /dev/null +++ b/tests/acceptance/artifactComparisonEspImage.test.ts @@ -0,0 +1,85 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; + +import { ESP32C3_APP_SEGMENTS, espImage } from "../fixtures/espImage.js"; +import { artifactComparisonResultSchema } from "../../src/domain/artifactComparison.js"; +import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js"; +import { artifactInventoryResultSchema } from "../../src/domain/artifactGraph.js"; +import { parseEvidence, type Evidence } from "../../src/domain/evidence.js"; +import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +it("reports an ESP segment whose load address alone changed", async () => { + const root = await createTestTempDirectory("rea-esp-comparison-"); + const { call } = await connectLocalToolsMcp(); + const inspect = async ( + name: string, + segments: typeof ESP32C3_APP_SEGMENTS, + ): Promise => { + const path = join(root, name); + await writeFile(path, espImage({ segments })); + const opened = await call("open_binary", { path }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const response = await call("inspect_artifact", {}); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const inventory = artifactInspectionResultSchema + .parse(parseEvidence(response.structuredContent).normalized_result) + .substeps.find( + ({ operation }) => operation === "inventory_artifact", + )?.evidence; + if (inventory === undefined) throw new Error("Missing inventory Evidence"); + return parseEvidence(inventory); + }; + const compare = async (left: Evidence, right: Evidence) => { + const response = await call("compare_artifacts", { left, right }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + return artifactComparisonResultSchema.parse( + parseEvidence(response.structuredContent).normalized_result, + ); + }; + const segment = (evidence: Evidence, path: string) => { + const occurrence = artifactInventoryResultSchema + .parse(evidence.normalized_result) + .occurrences.find(({ logical_path }) => logical_path === path); + if (occurrence === undefined) throw new Error(`Missing ${path}`); + return occurrence; + }; + + // The same bytes, with the DRAM segment loading 8 KiB higher. + const moved = ESP32C3_APP_SEGMENTS.map((item, index) => + index === 1 ? { ...item, address: 0x3fc8_a000 } : item, + ); + const left = await inspect("app.bin", ESP32C3_APP_SEGMENTS); + const right = await inspect("moved.bin", moved); + const leftSegment = segment(left, "segment-01.bin"); + const rightSegment = segment(right, "segment-01.bin"); + expect(rightSegment.artifact_id).toBe(leftSegment.artifact_id); + expect(leftSegment.format_metadata).toMatchObject({ load_addr: 0x3fc8_8000 }); + expect(rightSegment.format_metadata).toMatchObject({ + load_addr: 0x3fc8_a000, + }); + + const changed = await compare(left, right); + expect( + changed.changes.filter(({ logical_path }) => logical_path !== "."), + ).toEqual([ + { + classification: "changed", + logical_path: "segment-01.bin", + dimensions: ["metadata"], + left_occurrence_id: leftSegment.occurrence_id, + right_occurrence_id: rightSegment.occurrence_id, + left_artifact_id: leftSegment.artifact_id, + right_artifact_id: rightSegment.artifact_id, + evidence_links: [left.evidence_id, right.evidence_id], + }, + ]); + + // Equal declarations leave every segment unchanged. + const same = await compare( + left, + await inspect("copy.bin", ESP32C3_APP_SEGMENTS), + ); + expect(same.changes).toEqual([]); +}); diff --git a/tests/acceptance/espImageCli.test.ts b/tests/acceptance/espImageCli.test.ts new file mode 100644 index 0000000000..21a330a480 --- /dev/null +++ b/tests/acceptance/espImageCli.test.ts @@ -0,0 +1,105 @@ +import { readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, onTestFinished } from "vitest"; +import { cliTest } from "../support/cli/cliFixture.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { + ESP32C3_APP_SEGMENTS, + ESP32C3_BOOTLOADER_SEGMENTS, + esp8266Image, + espImage, +} from "../fixtures/espImage.js"; +import { ESP8266_REFUSAL } from "../../src/domain/firmware/espImage.js"; +import { parseEvidence } from "../../src/domain/evidence.js"; +import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js"; +import { + artifactExtractionResultSchema, + artifactInventoryResultSchema, +} from "../../src/domain/artifactGraph.js"; + +const cases = [ + { + role: "app", + bytes: espImage({ segments: ESP32C3_APP_SEGMENTS }), + segments: [ + "segment-00.bin", + "segment-01.bin", + "segment-02.bin", + "segment-03.bin", + ], + }, + { + role: "bootloader", + bytes: espImage({ segments: ESP32C3_BOOTLOADER_SEGMENTS }), + segments: ["segment-00.bin", "segment-01.bin"], + }, +] as const; + +cliTest.for(cases)( + "inventories and extracts an ESP $role image through the compiled CLI", + async ({ role, bytes, segments }, { cli }) => { + const directory = await createTestTempDirectory("rea-esp-cli-"); + const source = join(directory, `${role}.bin`); + await writeFile(source, bytes); + const inspected = await cli.run({ + arguments: ["inspect-artifact", source, "--json"], + }); + expect(inspected.exitCode, inspected.stdout + inspected.stderr).toBe(0); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(inspected.json).normalized_result, + ); + const graph = artifactInventoryResultSchema.parse( + inspection.substeps[0]?.evidence.normalized_result, + ); + expect(graph.manifest.root_format).toBe("esp-image"); + expect(graph.coverage.status).toBe("complete"); + expect( + graph.occurrences.find((row) => row.logical_path === ".") + ?.format_metadata, + ).toMatchObject({ + kind: "esp-image", + descriptor: { kind: role }, + checksum: { status: "valid" }, + }); + expect( + graph.occurrences + .filter((row) => row.logical_path !== ".") + .map((row) => row.logical_path), + ).toEqual(segments); + expect(inspection.next_probes).toEqual([]); + const extracted = await cli.run({ + arguments: ["extract-artifact", source, "--json"], + environment: { TMPDIR: directory, TMP: directory, TEMP: directory }, + }); + expect(extracted.exitCode, extracted.stdout + extracted.stderr).toBe(0); + const result = artifactExtractionResultSchema.parse( + parseEvidence(extracted.json).normalized_result, + ); + onTestFinished(() => + rm(result.output_root, { recursive: true, force: true }), + ); + expect(result.containment_verified).toBe(true); + const first = graph.occurrences.find( + (row) => row.logical_path === segments[0], + )?.source_location; + if (first === null || first === undefined) + throw new Error("Missing segment location"); + expect( + Buffer.compare( + await readFile(join(result.output_root, segments[0])), + bytes.subarray(first.offset, first.offset + first.length), + ), + ).toBe(0); + }, +); + +cliTest("refuses an ESP8266 image through the CLI", async ({ cli }) => { + const directory = await createTestTempDirectory("rea-esp8266-cli-"); + const source = join(directory, "esp8266.bin"); + await writeFile(source, esp8266Image()); + const result = await cli.run({ + arguments: ["inspect-artifact", source, "--json"], + }); + expect(result.exitCode).not.toBe(0); + expect(result.stdout + result.stderr).toContain(ESP8266_REFUSAL); +}); diff --git a/tests/boundary/filesystem/binaryTarget.test.ts b/tests/boundary/filesystem/binaryTarget.test.ts index f064b231e8..f4ef009fd0 100644 --- a/tests/boundary/filesystem/binaryTarget.test.ts +++ b/tests/boundary/filesystem/binaryTarget.test.ts @@ -32,6 +32,12 @@ import { hasXmlPlistSignature } from "../../../src/domain/apple/appleArchiveForm import { parseExecutableHeader } from "../../../src/domain/binaryTarget.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { ESP8266_REFUSAL } from "../../../src/domain/firmware/espImage.js"; +import { + ESP32C3_APP_SEGMENTS, + esp8266Image, + espImage, +} from "../../fixtures/espImage.js"; import { javaClassFile } from "../../fixtures/javaClassFile.js"; import { MAX_JVM_CLASS_IDENTITY_BYTES, @@ -541,6 +547,38 @@ describe("binary target I/O: package classification", () => { }); }); + it("classifies ESP images by bytes and refuses ESP8266 v1 images", async () => { + const directory = await createTestTempDirectory("rea-esp-target-"); + const app = join(directory, "firmware.ota"); + const truncated = join(directory, "download.bin"); + const esp8266 = join(directory, "esp8266.bin"); + const encrypted = join(directory, "encrypted.bin"); + const image = espImage({ segments: ESP32C3_APP_SEGMENTS }); + await Promise.all([ + writeFile(app, image), + writeFile(truncated, image.subarray(0, 40)), + writeFile(esp8266, esp8266Image()), + writeFile(encrypted, Buffer.alloc(4096, 0x5a)), + ]); + for (const path of [app, truncated]) + expect(await parseBinaryTarget(path)).toMatchObject({ + ok: true, + value: { kind: "archive", format: "esp-image" }, + }); + expect(await parseBinaryTarget(esp8266)).toMatchObject({ + ok: false, + error: { + _tag: "BinaryTargetError", + constraint: "file_format_rejected", + reason: ESP8266_REFUSAL, + }, + }); + // Flash-encrypted bytes carry no magic; they stay unrecognized, not ESP. + const unrecognized = await parseBinaryTarget(encrypted); + expect(unrecognized.ok).toBe(false); + expect(JSON.stringify(unrecognized)).not.toContain("ESP"); + }); + it("does not trust a ZIP-family extension without ZIP magic", async () => { const directory = await createTestTempDirectory("rea-fake-archive-"); const path = join(directory, "fake.zip"); diff --git a/tests/boundary/mcp/espImageMcp.test.ts b/tests/boundary/mcp/espImageMcp.test.ts new file mode 100644 index 0000000000..eb18906655 --- /dev/null +++ b/tests/boundary/mcp/espImageMcp.test.ts @@ -0,0 +1,97 @@ +import { readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { z } from "zod"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { + ESP32C3_APP_SEGMENTS, + esp8266Image, + espImage, + signedEspImage, +} from "../../fixtures/espImage.js"; +import { ESP8266_REFUSAL } from "../../../src/domain/firmware/espImage.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { artifactExtractionResultSchema } from "../../../src/domain/artifactGraph.js"; + +// The compiled stdio fixture checks advertised schemas against their dialect and +// validates actual tool responses, avoiding a second source server module graph. +// Full-profile retained-view schema work needs the compiled MCP boundary budget. +it.each(["full", "compact"] as const)( + "exposes ESP image facts and segment extraction through MCP (%s)", + async (profile) => { + const root = await createTestTempDirectory("rea-esp-mcp-"); + const path = join(root, "firmware.bin"); + const image = signedEspImage(espImage({ segments: ESP32C3_APP_SEGMENTS }), [ + "ecdsa-p256", + ]); + await writeFile(path, image); + const { call } = await connectLocalToolsMcp({ + REA_MCP_TOOL_FAMILIES: "session,artifact,application", + REA_MCP_INPUT_SCHEMA_PROFILE: profile, + REA_MCP_OUTPUT_SCHEMA_PROFILE: profile, + }); + let outputRoot: string | undefined; + try { + const opened = await call("open_binary", { path }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const reply = await call("inspect_artifact", { detail: "summary" }); + expect(reply.isError, JSON.stringify(reply)).not.toBe(true); + const summary = z + .object({ + parent_evidence_id: z.string(), + summary: z.object({ root_format_metadata: z.unknown() }), + }) + .parse(parseEvidence(reply.structuredContent).normalized_result); + expect(summary.summary.root_format_metadata).toMatchObject({ + kind: "esp-image", + chip: { target: "esp32c3", isa: "riscv", source: "ESP-IDF v6.1" }, + checksum: { status: "valid" }, + appended_sha256: { status: "valid" }, + undescribed_trailing: [], + signature_sector_offset: 4096, + descriptor: { kind: "app", project_name: "rea_esp_fixture" }, + }); + const retained = await call("inspect_analysis_view", { + source: { + kind: "retained-evidence", + evidence_id: summary.parent_evidence_id, + }, + view: { + kind: "inventory", + collection: "occurrences", + logical_paths: ["segment-03.bin", "signature-block-0.bin"], + fields: ["format_metadata", "source_location"], + }, + }); + expect(retained.isError, JSON.stringify(retained)).not.toBe(true); + const rows = JSON.stringify(retained.structuredContent); + expect(rows).toContain('"classification":"mapped"'); + expect(rows).toContain('"scheme":"ecdsa"'); + expect(rows).toContain('"signature_status":"unverified"'); + const extracted = await call("extract_artifact", {}); + expect(extracted.isError, JSON.stringify(extracted)).not.toBe(true); + const result = artifactExtractionResultSchema.parse( + parseEvidence(extracted.structuredContent).normalized_result, + ); + outputRoot = result.output_root; + expect( + Buffer.compare( + await readFile(join(result.output_root, "segment-03.bin")), + image.subarray(392, 456), + ), + ).toBe(0); + await call("close_binary", {}); + await writeFile(path, esp8266Image()); + const refused = await call("open_binary", { path }); + expect(JSON.stringify(parseMcpToolError(refused))).toContain( + ESP8266_REFUSAL, + ); + } finally { + if (outputRoot !== undefined) + await rm(outputRoot, { recursive: true, force: true }); + } + }, + 60_000, +); diff --git a/tests/boundary/mcp/uImageMcp.test.ts b/tests/boundary/mcp/uImageMcp.test.ts index a8715ff805..9d88e9a86d 100644 --- a/tests/boundary/mcp/uImageMcp.test.ts +++ b/tests/boundary/mcp/uImageMcp.test.ts @@ -24,7 +24,9 @@ it.each(["full", "compact"] as const)( }); await writeFile(path, bytes); const { call } = await connectLocalToolsMcp({ + REA_MCP_TOOL_FAMILIES: "session,artifact", REA_MCP_INPUT_SCHEMA_PROFILE: profile, + REA_MCP_OUTPUT_SCHEMA_PROFILE: profile, }); let outputRoot: string | undefined; try { diff --git a/tests/conformance/esp/app.c b/tests/conformance/esp/app.c new file mode 100644 index 0000000000..899312f42f --- /dev/null +++ b/tests/conformance/esp/app.c @@ -0,0 +1,50 @@ +#include + +/* esp_app_desc_t layout from ESP-IDF v6.1 esp_app_desc.h. */ +typedef struct { + uint32_t magic_word; + uint32_t secure_version; + uint32_t reserv1[2]; + char version[32]; + char project_name[32]; + char time[16]; + char date[16]; + char idf_ver[32]; + uint8_t app_elf_sha256[32]; + uint16_t min_efuse_blk_rev_full; + uint16_t max_efuse_blk_rev_full; + uint8_t mmu_page_size; + uint8_t spi_flash_mode; + uint8_t reserv3[2]; + uint32_t reserv2[18]; +} rea_app_desc_t; + +_Static_assert(sizeof(rea_app_desc_t) == 256, "esp_app_desc_t is 256 bytes"); + +__attribute__((section(".flash.appdesc"), used)) +const rea_app_desc_t rea_app_desc = { + .magic_word = 0xABCD5432, + .secure_version = 3, + .version = "rea-verify-1", + .project_name = "rea_esp_image", + .time = "00:00:00", + .date = "Jan 1 2026", + .idf_ver = "v6.1-rea", + .max_efuse_blk_rev_full = 199, + .mmu_page_size = 16, +}; + +__attribute__((section(".flash.rodata"), used)) +const char rea_message[] = "REA ESP image verification"; + +__attribute__((section(".flash.text"), used, noinline)) int +rea_flash_function(int value) { + return value * 3 + rea_message[value & 7]; +} + +__attribute__((section(".dram0.data"), used)) volatile int rea_counter = 7; + +__attribute__((section(".iram0.text"), used, noreturn)) void rea_entry(void) { + for (;;) + rea_counter = rea_flash_function(rea_counter); +} diff --git a/tests/conformance/esp/app.ld b/tests/conformance/esp/app.ld new file mode 100644 index 0000000000..3cac667f24 --- /dev/null +++ b/tests/conformance/esp/app.ld @@ -0,0 +1,22 @@ +ENTRY(rea_entry) +/* + * ESP32-C3 address ranges from ESP-IDF v6.1 + * components/soc/esp32c3/include/soc/soc.h. One program header per + * memory keeps the ELF small; esptool reads sections. + */ +PHDRS +{ + drom PT_LOAD; + dram PT_LOAD; + iram PT_LOAD; + irom PT_LOAD; +} +SECTIONS +{ + .flash.appdesc 0x3C000020 : { KEEP(*(.flash.appdesc)) } :drom + .flash.rodata : { *(.flash.rodata) *(.rodata .rodata.* .srodata .srodata.*) } :drom + .dram0.data 0x3FC88000 : { *(.dram0.data) *(.data .data.* .sdata .sdata.*) } :dram + .iram0.text 0x40380000 : { *(.iram0.text) *(.text .text.*) } :iram + .flash.text 0x42000020 : { *(.flash.text) } :irom + /DISCARD/ : { *(.comment) *(.note.*) } +} diff --git a/tests/conformance/esp/bootloader.c b/tests/conformance/esp/bootloader.c new file mode 100644 index 0000000000..2b96ff790b --- /dev/null +++ b/tests/conformance/esp/bootloader.c @@ -0,0 +1,32 @@ +#include + +/* esp_bootloader_desc_t layout from ESP-IDF v6.1 esp_bootloader_desc.h. */ +typedef struct { + uint8_t magic_byte; + uint8_t reserved[2]; + uint8_t secure_version; + uint32_t version; + char idf_ver[32]; + char date_time[24]; + uint8_t reserved2[16]; +} rea_bootloader_desc_t; + +_Static_assert(sizeof(rea_bootloader_desc_t) == 80, + "esp_bootloader_desc_t is 80 bytes"); + +__attribute__((section(".dram0.bootdesc"), used)) +const rea_bootloader_desc_t rea_bootloader_desc = { + .magic_byte = 80, + .secure_version = 2, + .version = 7, + .idf_ver = "v6.1-rea", + .date_time = "Jan 1 2026 00:00:00", +}; + +__attribute__((section(".dram0.data"), used)) volatile int rea_counter = 7; + +__attribute__((section(".iram_loader.text"), used, noreturn)) void +rea_entry(void) { + for (;;) + rea_counter += 1; +} diff --git a/tests/conformance/esp/bootloader.ld b/tests/conformance/esp/bootloader.ld new file mode 100644 index 0000000000..ee7215cb17 --- /dev/null +++ b/tests/conformance/esp/bootloader.ld @@ -0,0 +1,18 @@ +ENTRY(rea_entry) +/* + * RAM-only ESP32-C3 layout, as a second-stage bootloader has. The addresses + * are in the DRAM and IRAM ranges of ESP-IDF v6.1 + * components/soc/esp32c3/include/soc/soc.h. + */ +PHDRS +{ + dram PT_LOAD; + iram PT_LOAD; +} +SECTIONS +{ + .dram0.bootdesc 0x3FCD5820 : { KEEP(*(.dram0.bootdesc)) } :dram + .dram0.data : { *(.dram0.data) *(.data .data.* .sdata .sdata.*) *(.rodata .rodata.* .srodata .srodata.*) } :dram + .iram_loader.text 0x403CC710 : { *(.iram_loader.text) *(.text .text.*) } :iram + /DISCARD/ : { *(.comment) *(.note.*) } +} diff --git a/tests/conformance/esp/esp32-app.ld b/tests/conformance/esp/esp32-app.ld new file mode 100644 index 0000000000..95db31e16a --- /dev/null +++ b/tests/conformance/esp/esp32-app.ld @@ -0,0 +1,22 @@ +ENTRY(rea_entry) +/* + * ESP32 (Xtensa) address ranges from ESP-IDF v6.1 + * components/soc/esp32/include/soc/soc.h, for app.c built with + * xtensa-esp32-elf-gcc -mtext-section-literals. + */ +PHDRS +{ + drom PT_LOAD; + dram PT_LOAD; + iram PT_LOAD; + irom PT_LOAD; +} +SECTIONS +{ + .flash.appdesc 0x3F400020 : { KEEP(*(.flash.appdesc)) } :drom + .flash.rodata : { *(.flash.rodata) *(.rodata .rodata.*) } :drom + .dram0.data 0x3FFB0000 : { *(.dram0.data) *(.data .data.*) } :dram + .iram0.text 0x40080000 : { *(.iram0.literal .iram0.text) *(.literal .text .literal.* .text.*) } :iram + .flash.text 0x400D0020 : { *(.flash.literal .flash.text) } :irom + /DISCARD/ : { *(.comment) *(.note.*) *(.xtensa.info) *(.xt.*) } +} diff --git a/tests/conformance/esp/esp8266-app.ld b/tests/conformance/esp/esp8266-app.ld new file mode 100644 index 0000000000..036aed7645 --- /dev/null +++ b/tests/conformance/esp/esp8266-app.ld @@ -0,0 +1,19 @@ +ENTRY(rea_entry) +/* + * app.c placed in ESP8266 DRAM, IRAM and the IROM flash window (esptool + * v5.6.0 targets/esp8266.py MEMORY_MAP), so esptool --version 3 writes an + * image whose first segment is IROM. + */ +PHDRS +{ + dram PT_LOAD; + iram PT_LOAD; + irom PT_LOAD; +} +SECTIONS +{ + .data 0x3FFE8000 : { *(.flash.appdesc) *(.flash.rodata) *(.dram0.data) *(.data .data.* .sdata .sdata.*) *(.rodata .rodata.* .srodata .srodata.*) } :dram + .text 0x40100000 : { *(.iram0.text) *(.text .text.*) } :iram + .irom0.text 0x40201010 : { *(.flash.text) } :irom + /DISCARD/ : { *(.comment) *(.note.*) } +} diff --git a/tests/conformance/esp/esp8266.ld b/tests/conformance/esp/esp8266.ld new file mode 100644 index 0000000000..a0c4d07b5b --- /dev/null +++ b/tests/conformance/esp/esp8266.ld @@ -0,0 +1,17 @@ +ENTRY(rea_entry) +/* + * The bootloader fixture placed in ESP8266 DRAM and IRAM + * (esptool v5.6.0 targets/esp8266.py MEMORY_MAP), so esptool writes an + * ESP8266 v1 image. esptool accepts RISC-V ELF files for every chip. + */ +PHDRS +{ + dram PT_LOAD; + iram PT_LOAD; +} +SECTIONS +{ + .data 0x3FFE8000 : { *(.dram0.bootdesc) *(.dram0.data) *(.data .data.* .sdata .sdata.*) *(.rodata .rodata.* .srodata .srodata.*) } :dram + .text 0x40100000 : { *(.iram_loader.text) *(.text .text.*) } :iram + /DISCARD/ : { *(.comment) *(.note.*) } +} diff --git a/tests/fixtures/artifactReaderLifecycle.ts b/tests/fixtures/artifactReaderLifecycle.ts new file mode 100644 index 0000000000..ec8ff9e537 --- /dev/null +++ b/tests/fixtures/artifactReaderLifecycle.ts @@ -0,0 +1,254 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { buffer } from "node:stream/consumers"; +import { finished } from "node:stream/promises"; +import { setImmediate } from "node:timers/promises"; +import { describe, expect, it, vi } from "vitest"; +import type { + ArtifactReader, + ArtifactEntry, +} from "../../src/artifacts/ArtifactReader.js"; +import { + openRegularFile, + type StableRegularFileDescriptor, +} from "../../src/filesystem/RegularFile.js"; +import { OwnedFileHandle } from "../../src/filesystem/OwnedFileHandle.js"; +import { createTestTempDirectory } from "./temporaryDirectory.js"; +import { createDeferred } from "./binarySession.js"; +import { awaitAcknowledgement } from "./lifecycleAcknowledgement.js"; + +interface LifecycleFixture { + readonly image: () => Buffer; + readonly createReader: ( + path: string, + admitted?: StableRegularFileDescriptor, + ) => ArtifactReader; +} + +const fixture = async (bytes: Buffer): Promise => { + const path = join( + await createTestTempDirectory("rea-reader-lifecycle-"), + "image", + ); + await writeFile(path, bytes); + return path; +}; +const entries = async (reader: ArtifactReader): Promise => { + const result: ArtifactEntry[] = []; + for await (const entry of reader.entries()) result.push(entry); + return result; +}; + +const consumptionCases = ({ image, createReader }: LifecycleFixture): void => { + it("does not complete paused inventory after its reader closes", async () => { + const reader = createReader(await fixture(image())); + const iterator = reader.entries()[Symbol.asyncIterator](); + try { + expect((await iterator.next()).done).toBe(false); + await reader.close(); + await expect(iterator.next()).rejects.toMatchObject({ + reason: "unavailable", + }); + } finally { + await reader.close(); + } + }); + it.each(["cancel", "close"] as const)( + "retires fully buffered, unconsumed member bytes on %s", + async (action) => { + const reader = createReader(await fixture(image())); + const controller = new AbortController(); + const reason = new Error("cancel buffered member"); + try { + const [entry] = await entries(reader); + const stream = await reader.open(entry!, controller.signal); + await finished(stream, { readable: false, cleanup: true }); + await setImmediate(); + expect(stream.readableLength).toBeGreaterThan(0); + if (action === "cancel") controller.abort(reason); + else await reader.close(); + await expect(buffer(stream)).rejects.toMatchObject( + action === "cancel" + ? { reason: "cancelled", cause: reason } + : { reason: "unavailable" }, + ); + expect(stream.destroyed).toBe(true); + } finally { + await reader.close(); + } + }, + ); +}; + +const descriptorCases = ({ image, createReader }: LifecycleFixture): void => { + it("joins a member's outstanding positional read before descriptor closure", async () => { + const path = await fixture(image()); + const admitted = await openRegularFile(path, { symlinks: "reject" }); + const reader = createReader(path, admitted); + const reached = createDeferred(), + release = createDeferred(); + const nativeRead = admitted.handle.read.bind(admitted.handle); + const reads = vi.spyOn(admitted.handle, "read"); + let outcome: Promise | undefined; + let closing: Promise | undefined; + try { + const [entry] = await entries(reader); + reads.mockImplementation((async (...args: unknown[]) => { + reached.resolve(); + await release.promise; + return Reflect.apply(nativeRead, admitted.handle, args); + }) as typeof admitted.handle.read); + const stream = await reader.open(entry!); + outcome = buffer(stream).then( + () => ({ ok: true }), + (cause: unknown) => ({ ok: false, cause }), + ); + await awaitAcknowledgement( + reached.promise, + outcome, + "member positional read", + ); + let closed = false; + closing = reader.close().then(() => { + closed = true; + }); + await setImmediate(); + expect(closed).toBe(false); + release.resolve(); + await closing; + expect(stream.closed).toBe(true); + } finally { + release.resolve(); + await outcome; + await closing; + await reader.close(); + reads.mockRestore(); + await admitted.handle.close(); + } + }); + + it("retains failed descriptor cleanup for retry", async () => { + const reader = createReader(await fixture(image())); + try { + const [entry] = await entries(reader); + const stream = await reader.open(entry!); + const failure = new Error("artifact close denied"); + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockRejectedValueOnce(failure); + try { + await expect(reader.close()).rejects.toBe(failure); + await expect(reader.open(entry!)).rejects.toMatchObject({ + reason: "unavailable", + }); + expect(stream.closed).toBe(true); + await reader.close(); + expect(close).toHaveBeenCalledTimes(2); + } finally { + close.mockRestore(); + } + } finally { + await reader.close(); + } + }); +}; + +const validationCases = ({ image, createReader }: LifecycleFixture): void => { + it.each(["cancel", "cancel-with-validation-error", "close"] as const)( + "joins final freshness validation during %s", + async (action) => { + const path = await fixture(image()); + const admitted = await openRegularFile(path, { symlinks: "reject" }); + const reader = createReader(path, admitted); + const reached = createDeferred(), + release = createDeferred(); + const nativeStat = admitted.handle.stat.bind(admitted.handle); + const nativeRead = admitted.handle.read.bind(admitted.handle); + const reads = vi.spyOn(admitted.handle, "read"); + const stats = vi.spyOn(admitted.handle, "stat"); + const controller = new AbortController(), + reason = new Error("cancel final validation"); + const validationFailure = new Error("final descriptor stat failed"); + let outcome: Promise | undefined; + let closing: Promise | undefined; + try { + const [entry] = await entries(reader); + const memberEnd = BigInt(entry!.byteOffset! + entry!.declaredSize!); + let memberReadComplete = false; + reads.mockImplementation((async (...args: unknown[]) => { + const result = (await Reflect.apply( + nativeRead, + admitted.handle, + args, + )) as { + bytesRead: number; + buffer: Buffer; + }; + if (typeof args[3] === "bigint") + memberReadComplete = + args[3] + BigInt(result.bytesRead) === memberEnd; + return result; + }) as typeof admitted.handle.read); + stats.mockImplementation((async (...args: unknown[]) => { + const result = await Reflect.apply(nativeStat, admitted.handle, args); + if (memberReadComplete) { + memberReadComplete = false; + reached.resolve(); + await release.promise; + if (action === "cancel-with-validation-error") + throw validationFailure; + } + return result; + }) as typeof admitted.handle.stat); + const stream = await reader.open(entry!, controller.signal); + outcome = buffer(stream).then( + () => ({ ok: true }) as const, + (cause: unknown) => ({ ok: false, cause }) as const, + ); + await awaitAcknowledgement( + reached.promise, + outcome, + "post-read freshness validation", + ); + let closed = false; + if (action !== "close") controller.abort(reason); + else + closing = reader.close().then(() => { + closed = true; + }); + await setImmediate(); + expect(closed).toBe(false); + release.resolve(); + expect(await outcome).toMatchObject({ + ok: false, + cause: + action === "cancel-with-validation-error" + ? validationFailure + : action === "cancel" + ? { reason: "cancelled", cause: reason } + : { reason: "unavailable" }, + }); + await closing; + } finally { + release.resolve(); + await outcome; + await closing; + await reader.close(); + stats.mockRestore(); + reads.mockRestore(); + await admitted.handle.close(); + } + }, + ); +}; + +/** Stored-member readers share ownership and native I/O regression oracles. + * The fixture must have a nonempty first entry with a physical byte range. + */ +export const artifactReaderLifecycleCases = (setup: LifecycleFixture): void => { + describe("stored member lifecycle", () => { + consumptionCases(setup); + descriptorCases(setup); + validationCases(setup); + }); +}; diff --git a/tests/fixtures/espImage.ts b/tests/fixtures/espImage.ts new file mode 100644 index 0000000000..caf14474f0 --- /dev/null +++ b/tests/fixtures/espImage.ts @@ -0,0 +1,220 @@ +import { createHash } from "node:crypto"; +import { crc32 } from "node:zlib"; + +/** One segment as `esptool elf2image` writes it: load address and data. */ +export interface EspFixtureSegment { + readonly address: number; + readonly data: Uint8Array; +} + +export interface EspImageFixtureOptions { + readonly segments: readonly EspFixtureSegment[]; + readonly chipId?: number; + readonly entry?: number; + readonly hashAppended?: boolean; + /** Header count; esptool --ram-only-header counts only RAM segments. */ + readonly segmentCount?: number; + /** Replace the computed checksum before any digest is appended. */ + readonly checksum?: number; +} + +const word = (value: number): Buffer => { + const bytes = Buffer.alloc(4); + bytes.writeUInt32LE(value >>> 0); + return bytes; +}; + +const halfWord = (value: number): Buffer => { + const bytes = Buffer.alloc(2); + bytes.writeUInt16LE(value); + return bytes; +}; + +/** XOR of every segment data byte, seeded with 0xEF. */ +export const espChecksum = (segments: readonly EspFixtureSegment[]): number => + segments.reduce( + (value, { data }) => data.reduce((inner, byte) => inner ^ byte, value), + 0xef, + ); + +/** A 24-byte header, the segments, the 16-byte-aligned checksum and digest. */ +export const espImage = (options: EspImageFixtureOptions): Buffer => { + const hashAppended = options.hashAppended ?? true; + const header = Buffer.concat([ + Buffer.from([ + 0xe9, + options.segmentCount ?? options.segments.length, + 0x02, + 0x20, + ]), + word(options.entry ?? 0x4038_0000), + Buffer.from([0xee, 0, 0, 0]), + halfWord(options.chipId ?? 0x0005), + Buffer.from([0]), + halfWord(0), + halfWord(199), + Buffer.from([0, 0, 0, 0, hashAppended ? 1 : 0]), + ]); + const parts: Buffer[] = [header]; + for (const { address, data } of options.segments) + parts.push(word(address), word(data.length), Buffer.from(data)); + const body = Buffer.concat(parts); + const padding = 15 - (body.length % 16); + const image = Buffer.concat([ + body, + Buffer.alloc(padding), + Buffer.from([options.checksum ?? espChecksum(options.segments)]), + ]); + return hashAppended + ? Buffer.concat([image, createHash("sha256").update(image).digest()]) + : image; +}; + +const fixed = (text: string, length: number): Buffer => { + const bytes = Buffer.alloc(length); + bytes.write(text, "utf8"); + return bytes; +}; + +export interface AppDescriptorFields { + readonly version?: string; + readonly projectName?: string; + readonly time?: string; + readonly date?: string; + readonly idfVer?: string; + readonly elfSha256?: string; + readonly mmuPageSizeLog2?: number; + readonly secureVersion?: number; +} + +/** A 256-byte `esp_app_desc_t`. */ +export const appDescriptor = (fields: AppDescriptorFields = {}): Buffer => + Buffer.concat([ + word(0xabcd_5432), + word(fields.secureVersion ?? 0), + Buffer.alloc(8), + fixed(fields.version ?? "1.2.3", 32), + fixed(fields.projectName ?? "rea_esp_fixture", 32), + fixed(fields.time ?? "12:34:56", 16), + fixed(fields.date ?? "Oct 27 2026", 16), + fixed(fields.idfVer ?? "v6.1", 32), + Buffer.from(fields.elfSha256 ?? "ab".repeat(32), "hex"), + halfWord(0), + halfWord(199), + Buffer.from([fields.mmuPageSizeLog2 ?? 16, 2, 0, 0]), + Buffer.alloc(72), + ]); + +/** An 80-byte `esp_bootloader_desc_t`. */ +export const bootloaderDescriptor = (): Buffer => + Buffer.concat([ + Buffer.from([0x50, 0, 0, 1]), + word(1), + fixed("v6.1", 32), + fixed("Oct 27 2026 12:34:56", 24), + Buffer.alloc(16), + ]); + +/** Words of distinct bytes, so XOR folds and offsets are observable. */ +export const pattern = (length: number, seed: number): Buffer => + Buffer.from(Array.from({ length }, (_, index) => (seed + index * 7) & 0xff)); + +/** An ESP32-C3 app: DROM with the descriptor, DRAM, IRAM and IROM. */ +export const ESP32C3_APP_SEGMENTS: readonly EspFixtureSegment[] = [ + { + address: 0x3c00_0020, + data: Buffer.concat([appDescriptor(), pattern(32, 1)]), + }, + { address: 0x3fc8_8000, data: pattern(16, 2) }, + { address: 0x4038_0000, data: pattern(32, 3) }, + { address: 0x4200_0020, data: pattern(64, 4) }, +]; + +/** An ESP32-C3 bootloader: DRAM with the descriptor, then IRAM. */ +export const ESP32C3_BOOTLOADER_SEGMENTS: readonly EspFixtureSegment[] = [ + { + address: 0x3fcd_5820, + data: Buffer.concat([bootloaderDescriptor(), pattern(16, 5)]), + }, + { address: 0x403c_c710, data: pattern(48, 6) }, +]; + +export type SignatureBlockKind = "rsa" | "ecdsa-p256" | "ecdsa-p384"; + +/** + * A 1216-byte Secure Boot v2 block with a key pattern and the CRC32 that + * espsecure writes. The image digest covers `content` unless supplied. + */ +export const signatureBlock = ( + kind: SignatureBlockKind, + content: Uint8Array, + options: { + readonly imageDigest?: Buffer; + readonly crc?: number; + readonly version?: number; + readonly shaVersion?: number; + } = {}, +): Buffer => { + const block = Buffer.alloc(1216); + block[0] = 0xe7; + block[1] = options.version ?? (kind === "rsa" ? 0x02 : 0x03); + block[2] = options.shaVersion ?? (kind === "ecdsa-p384" ? 1 : 0); + const digest = + options.imageDigest ?? + createHash(kind === "ecdsa-p384" ? "sha384" : "sha256") + .update(content) + .digest(); + digest.copy(block, 4); + const keyStart = kind === "ecdsa-p384" ? 52 : 36; + const keyEnd = kind === "rsa" ? 812 : kind === "ecdsa-p256" ? 101 : 149; + pattern(keyEnd - keyStart, 9).copy(block, keyStart); + if (kind !== "rsa") block[keyStart] = kind === "ecdsa-p384" ? 3 : 2; + block.writeUInt32LE(options.crc ?? crc32(block.subarray(0, 1196)), 1196); + return block; +}; + +/** Pad with erased flash to 4 KiB, as espsecure does before its sector. */ +export const erasedTo4KiB = (image: Buffer): Buffer => + Buffer.concat([ + image, + Buffer.alloc((4096 - (image.length % 4096)) % 4096, 0xff), + ]); + +/** + * Append a signature sector; each kind fills the slot of its index and + * `null` leaves that slot erased. + */ +export const signedEspImage = ( + image: Buffer, + kinds: readonly (SignatureBlockKind | null)[], +): Buffer => { + const padded = erasedTo4KiB(image); + const sector = Buffer.alloc(4096, 0xff); + kinds.forEach((kind, index) => { + if (kind !== null) signatureBlock(kind, padded).copy(sector, index * 1216); + }); + return Buffer.concat([padded, sector]); +}; + +/** + * An ESP8266 image: an 8-byte header without an extended header. A v1 image + * starts in IRAM, Arduino's eboot at 0x4010F000, a v3 image in IROM. + */ +export const esp8266Image = (firstAddress = 0x4010_0000): Buffer => { + const segments = [ + { address: firstAddress, data: pattern(32, 7) }, + { address: 0x3ffe_8000, data: pattern(16, 8) }, + ]; + const parts: Buffer[] = [ + Buffer.from([0xe9, segments.length, 0x00, 0x20]), + word(0x4010_0004), + ]; + for (const { address, data } of segments) + parts.push(word(address), word(data.length), data); + const body = Buffer.concat(parts); + return Buffer.concat([ + body, + Buffer.alloc(15 - (body.length % 16)), + Buffer.from([espChecksum(segments)]), + ]); +};