diff --git a/docs/esp-image.md b/docs/esp-image.md index c6da05d82e..58fb257fbe 100644 --- a/docs/esp-image.md +++ b/docs/esp-image.md @@ -271,11 +271,16 @@ images. It then checks: `REA_XTENSA_ESP32_GCC` adds an ESP32 (Xtensa) app built with Espressif's `xtensa-esp32-elf-gcc`. Without it the report marks that image as skipped. +Each command runs as a process tree owned by the shared verifier command +owner, so a timeout also stops the linker or assembler a compiler driver +starts. The lane removes its workspace only after every tree has stopped and +keeps it when that cleanup cannot be confirmed. + The lane ran on macOS arm64 with Node 24.22.1, Homebrew Clang and LLD 21.1.8, and `xtensa-esp32-elf-gcc` 14.2.0 (crosstool-NG esp-14.2.0_20251107), once with esptool and espsecure v5.6.0 and once with 4.12.0. All checks passed for all 13 images in both runs. With v5.6.0 the complete run, including the builds -and every esptool call, took 10.4 s elapsed, 12.1 s user CPU and 1.4 s system +and every esptool call, took 21.7 s elapsed, 16.9 s user CPU and 6.1 s system CPU under `/usr/bin/time -l` with warm caches. The largest single process peaked at a 253 MiB resident set; the combined peak of all processes was not measured. diff --git a/scripts/verify/firmware/esp-image-tools.mjs b/scripts/verify/firmware/esp-image-tools.mjs index 2af33ae068..69fa63504b 100644 --- a/scripts/verify/firmware/esp-image-tools.mjs +++ b/scripts/verify/firmware/esp-image-tools.mjs @@ -1,7 +1,6 @@ import assert from "node:assert/strict"; -import { execFile } from "node:child_process"; import { fileURLToPath } from "node:url"; -import { promisify } from "node:util"; +import { createVerifierCommands } from "../../lib/verify-commands.mjs"; // Optional verification tools only; the runtime reader starts no tool. const compiler = process.env.REA_RISCV_CLANG ?? "clang"; @@ -9,8 +8,20 @@ const linker = process.env.REA_RISCV_LLD ?? "ld.lld"; const python = process.env.REA_ESPTOOL_PYTHON ?? "python3"; // Optional: an Xtensa ESP32 build when Espressif's GCC is supplied. export const xtensa = process.env.REA_XTENSA_ESP32_GCC; -export const execute = promisify(execFile); +/** + * Every tool runs as an owned process tree, so a timeout or close also stops + * the children a compiler driver starts. Close it before removing outputs. + */ +export const commands = await createVerifierCommands(); export const options = { maxBuffer: 16 * 1024 * 1024, timeout: 120_000 }; +export const execute = (command, args, settings = options) => + commands.exec(command, args, settings); + +/** The output of a command that exited nonzero; other failures propagate. */ +export const exitOutput = (failure) => { + if (typeof failure?.code !== "number" || failure.code === 0) throw failure; + return `${failure.snapshot.stdout.text}${failure.snapshot.stderr.text}`; +}; export const source = (name) => fileURLToPath( new URL(`../../../tests/conformance/esp/${name}`, import.meta.url), @@ -44,53 +55,61 @@ const prerequisite = async (needed, variable, check) => { const firstLine = ({ stdout }) => stdout.split("\n")[0]; -export const compilerVersion = await prerequisite( - `a Clang with the RISC-V target (tried ${compiler})`, - "REA_RISCV_CLANG", - async () => { - const targets = await execute(compiler, ["--print-targets"], options); - assert.match(targets.stdout, /^\s*riscv32\b/mu, "No riscv32 target."); - return firstLine(await execute(compiler, ["--version"], options)); - }, -); -await prerequisite(`ld.lld (tried ${linker})`, "REA_RISCV_LLD", () => - execute(linker, ["--version"], options), -); -const versionText = await prerequisite( - `a Python that runs esptool (tried ${python} -m esptool)`, - "REA_ESPTOOL_PYTHON", - () => tool("esptool", ["version"]), -); -const version = /\bv(\d+)\.(\d+)\.(\d+)\b/u.exec(versionText); -assert.ok(version, `Unrecognized esptool version: ${versionText}`); -export const esptoolVersion = version[0]; -const v5 = Number(version[1]) >= 5; -/** esptool v5 spells commands and options with hyphens. */ -export const spell = (v4, later) => (v5 ? later : v4); -await prerequisite( - `espsecure in the same Python (tried ${python} -m espsecure)`, - "REA_ESPTOOL_PYTHON", - async () => { - const help = await tool("espsecure", ["--help"]); - for (const command of [ - spell("sign_data", "sign-data"), - spell("signature_info_v2", "signature-info-v2"), - ]) - assert.ok(help.includes(command), `espsecure has no ${command}.`); - }, -); -if (esptoolVersion !== PINNED_ESPTOOL) - console.warn( - `verify:esp-images: esptool ${esptoolVersion} is not the pinned ${PINNED_ESPTOOL}, whose sources the reader's layout facts cite.`, +/** Set by checkTools: esptool v5 spells commands and options with hyphens. */ +let v5; +export const spell = (v4, later) => { + assert.notEqual(v5, undefined, "checkTools must run first"); + return v5 ? later : v4; +}; + +/** Check every tool before any work and report their versions. */ +export const checkTools = async () => { + const compilerVersion = await prerequisite( + `a Clang with the RISC-V target (tried ${compiler})`, + "REA_RISCV_CLANG", + async () => { + const targets = await execute(compiler, ["--print-targets"], options); + assert.match(targets.stdout, /^\s*riscv32\b/mu, "No riscv32 target."); + return firstLine(await execute(compiler, ["--version"], options)); + }, ); -export const xtensaVersion = - xtensa === undefined - ? undefined - : await prerequisite( - `xtensa-esp32-elf-gcc (tried ${xtensa})`, - "REA_XTENSA_ESP32_GCC to Espressif's compiler, or unset it to skip that image", - async () => firstLine(await execute(xtensa, ["--version"], options)), - ); + await prerequisite(`ld.lld (tried ${linker})`, "REA_RISCV_LLD", () => + execute(linker, ["--version"], options), + ); + const versionText = await prerequisite( + `a Python that runs esptool (tried ${python} -m esptool)`, + "REA_ESPTOOL_PYTHON", + () => tool("esptool", ["version"]), + ); + const version = /\bv(\d+)\.(\d+)\.(\d+)\b/u.exec(versionText); + assert.ok(version, `Unrecognized esptool version: ${versionText}`); + v5 = Number(version[1]) >= 5; + await prerequisite( + `espsecure in the same Python (tried ${python} -m espsecure)`, + "REA_ESPTOOL_PYTHON", + async () => { + const help = await tool("espsecure", ["--help"]); + for (const command of [ + spell("sign_data", "sign-data"), + spell("signature_info_v2", "signature-info-v2"), + ]) + assert.ok(help.includes(command), `espsecure has no ${command}.`); + }, + ); + if (version[0] !== PINNED_ESPTOOL) + console.warn( + `verify:esp-images: esptool ${version[0]} is not the pinned ${PINNED_ESPTOOL}, whose sources the reader's layout facts cite.`, + ); + const xtensaVersion = + xtensa === undefined + ? undefined + : await prerequisite( + `xtensa-esp32-elf-gcc (tried ${xtensa})`, + "REA_XTENSA_ESP32_GCC to Espressif's compiler, or unset it to skip that image", + async () => firstLine(await execute(xtensa, ["--version"], options)), + ); + return { compilerVersion, esptoolVersion: version[0], xtensaVersion }; +}; /** Build a RISC-V ELF from a conformance source; it is never run. */ export const buildElf = async (program, script, output) => { @@ -135,7 +154,7 @@ export const elf2image = (chip, elf, output, extra = []) => export const imageInfo = async (file, chip) => { const text = await tool("esptool", [ ...(chip === undefined ? [] : ["--chip", chip]), - ...(v5 ? ["image-info", file] : ["image_info", "--version", "2", file]), + ...spell(["image_info", "--version", "2", file], ["image-info", file]), ]); const field = (label) => new RegExp(`^${label}: (.+)$`, "mu").exec(text)?.[1]?.trim(); @@ -190,7 +209,7 @@ export const inspect = async (file) => { { ...options, env: { ...process.env, REA_LOG_LEVEL: "silent" } }, ).then( ({ stdout }) => ({ ok: true, stdout }), - (failure) => ({ ok: false, stdout: `${failure.stdout}${failure.stderr}` }), + (failure) => ({ ok: false, stdout: exitOutput(failure) }), ); if (!result.ok) return { refused: result.stdout }; const inventory = JSON.parse(result.stdout).normalized_result.substeps[0] diff --git a/scripts/verify/firmware/esp-image.mjs b/scripts/verify/firmware/esp-image.mjs index 9f5f20db26..ad8accc8bf 100644 --- a/scripts/verify/firmware/esp-image.mjs +++ b/scripts/verify/firmware/esp-image.mjs @@ -4,13 +4,15 @@ import { createHash } from "node:crypto"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { withVerifierCleanup } from "../../lib/verifier-cleanup.mjs"; import { assertAgrees, buildElf, - compilerVersion, + checkTools, + commands, elf2image, - esptoolVersion, execute, + exitOutput, extract, imageInfo, inspect, @@ -19,19 +21,20 @@ import { spell, tool, xtensa, - xtensaVersion, } from "./esp-image-tools.mjs"; -const workspace = await mkdtemp(join(tmpdir(), "rea-esp-image-proof-")); +let workspace; const path = (name) => join(workspace, name); -const report = { - esptool: esptoolVersion, - compiler: compilerVersion, - images: [], - extractions: [], -}; -try { +await withVerifierCleanup(async () => { + const { compilerVersion, esptoolVersion, xtensaVersion } = await checkTools(); + workspace = await mkdtemp(join(tmpdir(), "rea-esp-image-proof-")); + const report = { + esptool: esptoolVersion, + compiler: compilerVersion, + images: [], + extractions: [], + }; const app = await buildElf("app.c", "app.ld", path("app.elf")); const boot = await buildElf( "bootloader.c", @@ -295,7 +298,7 @@ try { } const truncatedOracle = await imageInfo(path("truncated.bin")).then( () => "loaded", - (failure) => `${failure.stdout}${failure.stderr}`, + exitOutput, ); assert.match(truncatedOracle, /End of file reading segment/u); const truncated = await inspect(path("truncated.bin")); @@ -397,6 +400,12 @@ try { ); report.images.push({ name: "esp8266-0x00000.bin", refused: true }); console.log(JSON.stringify(report, null, 2)); -} finally { - await rm(workspace, { recursive: true, force: true }); -} +}, [ + // Remove outputs only after every owned command tree has stopped; a failed + // close keeps the workspace for inspection. + async () => { + await commands.close(); + if (workspace !== undefined) + await rm(workspace, { recursive: true, force: true }); + }, +]);