mirror of
https://github.com/morluto/rea.git
synced 2026-11-18 04:34:13 +00:00
Add bounded offline DWARF compilation-unit inspection with source provenance, explicit coverage, caller-selected controls and shared ELF compression framing. Assisted-by: Suika (gensokyo-skills:suika-scatter-gather-planning) Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com>
599 lines
18 KiB
JavaScript
599 lines
18 KiB
JavaScript
/** Compile a two-file C fixture and compare compile-unit facts with a DWARF dumper. */
|
|
import { mkdir, readFile, readdir, rename, writeFile } from "node:fs/promises";
|
|
import { delimiter, join } from "node:path";
|
|
import { accessSync, constants, realpathSync } from "node:fs";
|
|
import { pathToFileURL } from "node:url";
|
|
import { createVerifierCommands } from "./lib/verify-commands.mjs";
|
|
import { VerifierFixture } from "./lib/verifier-fixture.mjs";
|
|
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
|
|
import { importVerifierSource } from "./lib/verifier-source-loader.mjs";
|
|
|
|
const commandExists = (name) =>
|
|
(name.includes("/")
|
|
? [name]
|
|
: (process.env.PATH ?? "")
|
|
.split(delimiter)
|
|
.map((directory) => join(directory, name))
|
|
).some((path) => {
|
|
try {
|
|
accessSync(path, constants.X_OK);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
});
|
|
|
|
const compiler = ["clang", "gcc"].find((name) => commandExists(name));
|
|
const dumper = [
|
|
process.env.REA_DWARFDUMP,
|
|
"llvm-dwarfdump",
|
|
"/opt/homebrew/opt/llvm/bin/llvm-dwarfdump",
|
|
"/usr/bin/llvm-dwarfdump",
|
|
"/usr/bin/dwarfdump",
|
|
"dwarfdump",
|
|
"readelf",
|
|
].find((name) => name !== undefined && commandExists(name));
|
|
|
|
const ELF_TARGET = ["-target", "x86_64-unknown-linux-gnu"];
|
|
const PE_TARGET = ["-target", "x86_64-w64-mingw32"];
|
|
|
|
/**
|
|
* `expect` fields make a variant fail when it did not exercise what it names.
|
|
* Apple clang accepts `-gz` and `-gsplit-dwarf` on Mach-O and ignores them, so
|
|
* those rows carry no expectation.
|
|
*/
|
|
const variants = [
|
|
{ id: "dwarf4", args: ["-gdwarf-4", "-g"], expect: { version: 4 } },
|
|
{ id: "dwarf5", args: ["-gdwarf-5", "-g"], expect: { version: 5 } },
|
|
{ id: "zlib", args: ["-gdwarf-5", "-g", "-gz=zlib"] },
|
|
{ id: "zstd", args: ["-gdwarf-5", "-g", "-gz=zstd"] },
|
|
{ id: "split", args: ["-gdwarf-5", "-g", "-gsplit-dwarf"] },
|
|
{
|
|
id: "elf-dwarf2",
|
|
args: [...ELF_TARGET, "-gdwarf-2", "-g"],
|
|
expect: { version: 2 },
|
|
},
|
|
{
|
|
id: "elf-dwarf3",
|
|
args: [...ELF_TARGET, "-gdwarf-3", "-g"],
|
|
expect: { version: 3 },
|
|
},
|
|
{
|
|
id: "elf-dwarf4",
|
|
args: [...ELF_TARGET, "-gdwarf-4", "-g"],
|
|
expect: { version: 4 },
|
|
},
|
|
{
|
|
id: "elf-dwarf5",
|
|
args: [...ELF_TARGET, "-gdwarf-5", "-g"],
|
|
expect: { version: 5 },
|
|
},
|
|
{
|
|
id: "elf-dwarf64",
|
|
args: [...ELF_TARGET, "-gdwarf-5", "-gdwarf64", "-g"],
|
|
expect: { version: 5, offsetSize: 8 },
|
|
},
|
|
{
|
|
id: "elf-zlib",
|
|
args: [...ELF_TARGET, "-gdwarf-5", "-g", "-gz=zlib"],
|
|
expect: { version: 5, compression: "zlib" },
|
|
},
|
|
{
|
|
id: "elf-zstd",
|
|
args: [...ELF_TARGET, "-gdwarf-5", "-g", "-gz=zstd"],
|
|
expect: { version: 5, compression: "zstd" },
|
|
},
|
|
{
|
|
id: "elf-split",
|
|
args: [...ELF_TARGET, "-gdwarf-5", "-g", "-gsplit-dwarf"],
|
|
expect: { version: 5, dwoNames: true },
|
|
},
|
|
{
|
|
id: "elf-dwarf4-split",
|
|
args: [...ELF_TARGET, "-gdwarf-4", "-g", "-gsplit-dwarf"],
|
|
expect: { version: 4, dwoNames: true },
|
|
},
|
|
{
|
|
id: "elf-big-endian",
|
|
args: ["-target", "powerpc-unknown-linux-gnu", "-gdwarf-4", "-g"],
|
|
expect: { version: 4 },
|
|
},
|
|
{
|
|
id: "elf-big-endian-aarch64",
|
|
args: ["-target", "aarch64_be-unknown-linux-gnu", "-gdwarf-4", "-g"],
|
|
expect: { version: 4 },
|
|
},
|
|
{
|
|
id: "pe-dwarf4",
|
|
args: [...PE_TARGET, "-gdwarf-4", "-g"],
|
|
expect: { version: 4 },
|
|
},
|
|
{
|
|
id: "pe-dwarf5",
|
|
args: [...PE_TARGET, "-gdwarf-5", "-g"],
|
|
expect: { version: 5 },
|
|
},
|
|
];
|
|
|
|
/**
|
|
* llvm-dwarfdump prints `DW_AT_x ("escaped")`; readelf prints `DW_AT_x : raw`
|
|
* with quotes left as they are, so its whole tail is the value.
|
|
*/
|
|
const quoted = (line) => {
|
|
const match = /\b(?:DW_)?AT_\w+\s*\(\s*"((?:\\.|[^"\\])*)"\s*\)\s*$/u.exec(
|
|
line,
|
|
);
|
|
if (match?.[1] !== undefined) return match[1].replace(/\\(["\\])/gu, "$1");
|
|
const attribute = /\bDW_AT_\w+\s*:\s*(.*)$/u.exec(line);
|
|
if (attribute === null) return null;
|
|
const tail = attribute[1]
|
|
.replace(
|
|
/^\((?:indirect (?:line )?string, offset|indexed string): (?:0x)?[0-9a-f]+\):\s*/iu,
|
|
"",
|
|
)
|
|
.trim();
|
|
return tail === "" ? null : tail;
|
|
};
|
|
|
|
const attribute = (line) => {
|
|
const match = /(?:DW_)?AT_(?:GNU_)?(dwo_name|comp_dir|producer|name)\b/u.exec(
|
|
line,
|
|
);
|
|
if (match?.[1] === undefined) return null;
|
|
const value = quoted(line);
|
|
return value === null ? null : { key: match[1], value };
|
|
};
|
|
|
|
const versionOf = (line) => {
|
|
const match =
|
|
/version\s*=\s*0x([0-9a-f]+)/iu.exec(line) ??
|
|
/\bVersion:\s+([0-9]+)/u.exec(line);
|
|
if (match?.[1] === undefined) return null;
|
|
return Number.parseInt(match[1], match[1].length > 2 ? 16 : 10);
|
|
};
|
|
|
|
/**
|
|
* Pull unit version, name, comp_dir, and producer in file order.
|
|
* `readelf --debug-dump=info` prints the compilation-unit marker and
|
|
* `Version:` on separate lines. llvm-dwarfdump keeps both on one line.
|
|
*/
|
|
export const parseDumperUnits = (text) => {
|
|
const units = [];
|
|
let current = null;
|
|
let collecting = false;
|
|
const push = () => {
|
|
if (current !== null) units.push(current);
|
|
};
|
|
for (const line of text.split(/\r?\n/u)) {
|
|
// Attribute values are producer text, never unit structure.
|
|
const isAttribute = attribute(line) !== null;
|
|
const version = isAttribute ? null : versionOf(line);
|
|
const header =
|
|
!isAttribute &&
|
|
/^\s*Compilation Unit @|^\s*0x[0-9a-f]+:\s+Compile Unit:/u.test(line);
|
|
const tagged =
|
|
!isAttribute &&
|
|
/TAG_compile_unit|DW_TAG_compile_unit|DW_TAG_skeleton_unit/u.test(line);
|
|
if (header || (tagged && version !== null)) {
|
|
push();
|
|
collecting = true;
|
|
current = {
|
|
version,
|
|
name: null,
|
|
comp_dir: null,
|
|
producer: null,
|
|
dwo_name: null,
|
|
};
|
|
continue;
|
|
}
|
|
if (current === null) continue;
|
|
if (current.version === null && version !== null) {
|
|
current.version = version;
|
|
continue;
|
|
}
|
|
const readelfDie = /^\s*<(\d+)><[0-9a-f]+>:/iu.exec(line);
|
|
const llvmDie = /^\s*0x[0-9a-f]+:\s+(DW_TAG_\w+)/iu.exec(line);
|
|
if (
|
|
(readelfDie !== null && Number(readelfDie[1]) > 0) ||
|
|
(llvmDie !== null &&
|
|
!["DW_TAG_compile_unit", "DW_TAG_skeleton_unit"].includes(llvmDie[1]))
|
|
)
|
|
collecting = false;
|
|
if (!collecting) continue;
|
|
const fact = attribute(line);
|
|
if (fact !== null && current[fact.key] === null)
|
|
current[fact.key] = fact.value;
|
|
}
|
|
push();
|
|
return units;
|
|
};
|
|
|
|
/** A variant this host cannot build; a timeout or crash is not one. */
|
|
class UnavailableVariant extends Error {}
|
|
|
|
const run = async (commands, command, args, cwd) => {
|
|
try {
|
|
const { stdout } = await commands.exec(command, args, {
|
|
cwd,
|
|
timeout: 120000,
|
|
stdin: "closed-pipe",
|
|
});
|
|
return stdout;
|
|
} catch (cause) {
|
|
if (!(cause instanceof Error) || cause.name !== "VerifierCommandExitError")
|
|
throw cause;
|
|
const stderr =
|
|
cause instanceof Error && "snapshot" in cause
|
|
? cause.snapshot.stderr.text.trim()
|
|
: "";
|
|
const stdout =
|
|
cause instanceof Error && "snapshot" in cause
|
|
? cause.snapshot.stdout.text.trim()
|
|
: "";
|
|
const detail = [stderr, stdout].filter((part) => part !== "").join("\n");
|
|
const message = cause instanceof Error ? cause.message : String(cause);
|
|
throw new UnavailableVariant(detail === "" ? message : detail);
|
|
}
|
|
};
|
|
|
|
const dumpUnits = async (commands, binary) => {
|
|
if (dumper === undefined) throw new Error("No DWARF dumper is installed");
|
|
const base = dumper.split("/").at(-1);
|
|
const args =
|
|
base === "readelf"
|
|
? ["--debug-dump=info", binary]
|
|
: ["--debug-info", binary];
|
|
return parseDumperUnits(await run(commands, dumper, args, undefined));
|
|
};
|
|
|
|
const inspectUnits = async (binary) => {
|
|
const { createDwarfUnitsService } = await importVerifierSource(
|
|
"../src/composition/binaryDiagnostics.ts",
|
|
import.meta.url,
|
|
);
|
|
const service = createDwarfUnitsService();
|
|
try {
|
|
const result = await service.inspect({ path: binary, limit: 256 });
|
|
if (!result.ok) throw result.error;
|
|
const report = result.value.normalized_result;
|
|
if (report.total_units > report.units.length)
|
|
throw new Error(
|
|
`${binary} has ${String(report.total_units)} units and the page returned ${String(report.units.length)}`,
|
|
);
|
|
return {
|
|
report,
|
|
units: report.units.map((unit) => ({
|
|
version: unit.version,
|
|
name: unit.name.value,
|
|
comp_dir: unit.comp_dir.value,
|
|
producer: unit.producer.value,
|
|
dwo_name: unit.dwo_name.value,
|
|
})),
|
|
offsetSizes: report.units.map((unit) => unit.offset_size),
|
|
infoCompression: report.sections
|
|
.filter((section) => section.role === ".debug_info")
|
|
.map((section) => section.compression),
|
|
};
|
|
} finally {
|
|
await service.close();
|
|
}
|
|
};
|
|
|
|
const ELF_MAGIC = "\u007fELF";
|
|
|
|
const isElf = async (path) =>
|
|
(await readFile(path)).subarray(0, 4).toString("binary") === ELF_MAGIC;
|
|
|
|
/**
|
|
* readelf reads ELF files only, so a PE or Mach-O image under it is a host gap
|
|
* to skip, not a disagreement. Returns the skip reason, or null when it reads.
|
|
*/
|
|
export const dumperFormatGap = (dumperPath, header) =>
|
|
dumperPath.split("/").at(-1) === "readelf" &&
|
|
header.subarray(0, 4).toString("binary") !== ELF_MAGIC
|
|
? "readelf reads ELF files only, so this image was not dumped"
|
|
: null;
|
|
|
|
const linkPe = async (commands, directory, objects) => {
|
|
if (!commandExists("ld.lld"))
|
|
throw new UnavailableVariant("ld.lld is absent, so no PE image was linked");
|
|
const output = join(directory, "program.exe");
|
|
await run(
|
|
commands,
|
|
"ld.lld",
|
|
[
|
|
"-m",
|
|
"i386pep",
|
|
"--entry",
|
|
"alpha",
|
|
"--subsystem",
|
|
"console",
|
|
...objects,
|
|
"-o",
|
|
output,
|
|
],
|
|
directory,
|
|
);
|
|
return [output];
|
|
};
|
|
|
|
const linkArgs = (variant) => {
|
|
if (variant.args.includes("-gz=zstd"))
|
|
return ["--compress-debug-sections=zstd"];
|
|
if (variant.args.includes("-gz=zlib"))
|
|
return ["--compress-debug-sections=zlib"];
|
|
return [];
|
|
};
|
|
|
|
const compileVariant = async (commands, directory, variant) => {
|
|
if (compiler === undefined)
|
|
throw new UnavailableVariant("clang or gcc is not installed");
|
|
await writeFixture(directory);
|
|
const objects = [join(directory, "alpha.o"), join(directory, "beta.o")];
|
|
await run(
|
|
commands,
|
|
compiler,
|
|
[...variant.args, "-c", "alpha.c", "-o", "alpha.o"],
|
|
directory,
|
|
);
|
|
await run(
|
|
commands,
|
|
compiler,
|
|
[...variant.args, "-c", "beta.c", "-o", "beta.o"],
|
|
directory,
|
|
);
|
|
let binaries = objects;
|
|
if (variant.args.includes(PE_TARGET[1])) {
|
|
binaries = await linkPe(commands, directory, objects);
|
|
} else if (await isElf(objects[0])) {
|
|
if (!commandExists("ld.lld"))
|
|
throw new UnavailableVariant(
|
|
"ld.lld is absent, so ELF relocations were not applied",
|
|
);
|
|
const output = join(directory, "program");
|
|
await run(
|
|
commands,
|
|
"ld.lld",
|
|
["-e", "alpha", ...linkArgs(variant), ...objects, "-o", output],
|
|
directory,
|
|
);
|
|
binaries = [output];
|
|
}
|
|
const aside = join(directory, "dwo");
|
|
await mkdir(aside);
|
|
for (const name of await readdir(directory)) {
|
|
if (name.endsWith(".dwo"))
|
|
await rename(join(directory, name), join(aside, name));
|
|
}
|
|
for (const binary of binaries) {
|
|
const gap = dumperFormatGap(dumper ?? "", await readFile(binary));
|
|
if (gap !== null) throw new UnavailableVariant(gap);
|
|
}
|
|
return binaries;
|
|
};
|
|
|
|
const writeFixture = async (directory) => {
|
|
await writeFile(
|
|
join(directory, "alpha.c"),
|
|
"int alpha(void) { return 1; }\n",
|
|
);
|
|
await writeFile(join(directory, "beta.c"), "int beta(void) { return 2; }\n");
|
|
};
|
|
|
|
const issueCodes = (unit) => unit.issues.map(({ code }) => code);
|
|
|
|
/** Files the dumper reads and `inspect_dwarf_units` must not claim to resolve. */
|
|
const checkBoundaries = async (commands, root) => {
|
|
const directory = join(root, "boundaries");
|
|
await mkdir(directory);
|
|
await writeFixture(directory);
|
|
const elf = [
|
|
...(compiler === "clang" ? ELF_TARGET : []),
|
|
"-gdwarf-5",
|
|
"-g",
|
|
"-c",
|
|
];
|
|
await run(
|
|
commands,
|
|
compiler,
|
|
[...elf, "alpha.c", "-o", "alpha.o"],
|
|
directory,
|
|
);
|
|
await run(commands, compiler, [...elf, "beta.c", "-o", "beta.o"], directory);
|
|
await run(
|
|
commands,
|
|
compiler,
|
|
[...elf, "-gsplit-dwarf", "alpha.c", "-o", "split.o"],
|
|
directory,
|
|
);
|
|
await run(
|
|
commands,
|
|
"ld.lld",
|
|
["-r", "alpha.o", "beta.o", "-o", "merged.o"],
|
|
directory,
|
|
);
|
|
const rows = [];
|
|
|
|
const single = (await inspectUnits(join(directory, "alpha.o"))).report;
|
|
const [object] = single.units;
|
|
if (
|
|
single.total_units !== 1 ||
|
|
object?.abbrev_offset !== 0 ||
|
|
object.language.state !== "present" ||
|
|
object.name.state !== "unresolved" ||
|
|
object.stmt_list.state !== "unresolved"
|
|
)
|
|
throw new Error(`ET_REL object: ${JSON.stringify(single)}`);
|
|
rows.push({ id: "elf-relocatable-object", units: 1, coverage: "complete" });
|
|
|
|
const merged = (await inspectUnits(join(directory, "merged.o"))).report;
|
|
if (
|
|
merged.total_units !==
|
|
(await dumpUnits(commands, join(directory, "merged.o"))).length ||
|
|
merged.units.some(
|
|
(unit) =>
|
|
unit.abbrev_offset !== null ||
|
|
unit.name.state !== "unknown" ||
|
|
!issueCodes(unit).includes("unapplied_relocation"),
|
|
)
|
|
)
|
|
throw new Error(`ld -r object: ${JSON.stringify(merged)}`);
|
|
rows.push({
|
|
id: "elf-relocatable-merged",
|
|
units: merged.total_units,
|
|
coverage: merged.coverage.status,
|
|
});
|
|
|
|
const dwo = (await inspectUnits(join(directory, "split.dwo"))).report;
|
|
if (
|
|
dwo.coverage.status !== "partial" ||
|
|
dwo.coverage.reason !== "Split DWARF section .debug_info.dwo is not read"
|
|
)
|
|
throw new Error(`Split DWARF object: ${JSON.stringify(dwo)}`);
|
|
rows.push({ id: "elf-dwo-object", units: 0, coverage: dwo.coverage.status });
|
|
return rows;
|
|
};
|
|
|
|
const expectationFailure = (expected, row) => {
|
|
if (expected === undefined) return null;
|
|
if (expected.version !== undefined && row.version !== expected.version)
|
|
return `version ${String(row.version)}`;
|
|
if (
|
|
expected.compression !== undefined &&
|
|
!row.info_compression.includes(expected.compression)
|
|
)
|
|
return `.debug_info compression ${row.info_compression.join(",")}`;
|
|
if (expected.dwoNames === true && row.dwo_names !== row.units)
|
|
return `${String(row.dwo_names)} dwo_name of ${String(row.units)} units`;
|
|
if (
|
|
expected.offsetSize !== undefined &&
|
|
!row.offset_sizes.includes(expected.offsetSize)
|
|
)
|
|
return `offset sizes ${row.offset_sizes.join(",")}`;
|
|
return null;
|
|
};
|
|
|
|
const sameUnit = (left, right) =>
|
|
left.version === right.version &&
|
|
left.name === right.name &&
|
|
left.comp_dir === right.comp_dir &&
|
|
left.producer === right.producer &&
|
|
left.dwo_name === right.dwo_name;
|
|
|
|
const invokedDirectly = () => {
|
|
const entry = process.argv[1];
|
|
if (entry === undefined) return false;
|
|
try {
|
|
return import.meta.url === pathToFileURL(realpathSync(entry)).href;
|
|
} catch {
|
|
return false;
|
|
}
|
|
};
|
|
|
|
const main = async () => {
|
|
const commands = await createVerifierCommands();
|
|
let root;
|
|
let fixture;
|
|
const checked = [];
|
|
const skipped = [];
|
|
const boundaries = [];
|
|
await withVerifierCleanup(async () => {
|
|
fixture = await VerifierFixture.open({ prefix: "rea-dwarf-units-" });
|
|
root = fixture.path;
|
|
if (compiler === undefined || dumper === undefined) {
|
|
skipped.push({
|
|
id: "all",
|
|
reason:
|
|
compiler === undefined
|
|
? "clang and gcc are absent"
|
|
: "llvm-dwarfdump, dwarfdump, and readelf are absent",
|
|
});
|
|
} else {
|
|
for (const variant of variants) {
|
|
const directory = join(root, variant.id);
|
|
await mkdir(directory);
|
|
let binaries;
|
|
try {
|
|
binaries = await compileVariant(commands, directory, variant);
|
|
} catch (cause) {
|
|
if (!(cause instanceof UnavailableVariant)) throw cause;
|
|
const reason = cause instanceof Error ? cause.message : String(cause);
|
|
skipped.push({
|
|
id: variant.id,
|
|
reason:
|
|
reason.split("\n").find((line) => line.trim() !== "") ?? reason,
|
|
});
|
|
continue;
|
|
}
|
|
let units = 0;
|
|
let dwoNames = 0;
|
|
let version = null;
|
|
const infoCompression = new Set();
|
|
const offsetSizes = new Set();
|
|
for (const binary of binaries) {
|
|
const dumped = await dumpUnits(commands, binary);
|
|
const inspected = await inspectUnits(binary);
|
|
if (
|
|
dumped.length === 0 ||
|
|
dumped.length !== inspected.units.length ||
|
|
dumped.some(
|
|
(unit, index) => !sameUnit(unit, inspected.units[index]),
|
|
)
|
|
) {
|
|
console.error(
|
|
JSON.stringify(
|
|
{ id: variant.id, binary, dumped, inspected },
|
|
null,
|
|
2,
|
|
),
|
|
);
|
|
throw new Error(`${variant.id} disagreed with ${dumper}`);
|
|
}
|
|
units += inspected.units.length;
|
|
dwoNames += inspected.units.filter(
|
|
(unit) => unit.dwo_name !== null,
|
|
).length;
|
|
for (const size of inspected.offsetSizes) offsetSizes.add(size);
|
|
version ??= inspected.units[0]?.version ?? null;
|
|
for (const compression of inspected.infoCompression)
|
|
infoCompression.add(compression);
|
|
}
|
|
const row = {
|
|
id: variant.id,
|
|
binaries: binaries.length,
|
|
units,
|
|
dwo_names: dwoNames,
|
|
version,
|
|
offset_sizes: [...offsetSizes],
|
|
info_compression: [...infoCompression],
|
|
dumper,
|
|
};
|
|
const failure = expectationFailure(variant.expect, row);
|
|
if (failure !== null)
|
|
throw new Error(`${variant.id} did not exercise it: ${failure}`);
|
|
checked.push(row);
|
|
}
|
|
boundaries.push(...(await checkBoundaries(commands, root)));
|
|
}
|
|
if (checked.length === 0)
|
|
throw new Error(
|
|
`No DWARF variant was compared. Skipped: ${JSON.stringify(skipped)}`,
|
|
);
|
|
console.log(
|
|
JSON.stringify(
|
|
{ compiler, dumper, checked, boundaries, skipped },
|
|
null,
|
|
2,
|
|
),
|
|
);
|
|
}, [
|
|
// The directory stays when a command tree could not be stopped.
|
|
async () => {
|
|
await commands.close();
|
|
await fixture?.release();
|
|
},
|
|
]);
|
|
};
|
|
|
|
if (invokedDirectly()) await main();
|