diff --git a/scripts/build-conformance-fixtures.mjs b/scripts/build-conformance-fixtures.mjs index 557b7a7d69..c5dbe5cc8e 100644 --- a/scripts/build-conformance-fixtures.mjs +++ b/scripts/build-conformance-fixtures.mjs @@ -37,6 +37,7 @@ await buildPortableFixtures(); if (hostPlatform === "darwin") await buildDarwinFixtures(); const manifest = { + schemaVersion: 2, platform: hostPlatform, architecture, generatedAt: new Date(0).toISOString(), @@ -64,7 +65,7 @@ async function buildPortableFixtures() { symbols: ["rea_entry", "rea_branch", "rea_leaf", "rea_c_global"].map( (name) => `${symbolPrefix}${name}`, ), - strings: ["REA_C_ENTRY", "REA_C_LEAF"], + stringMarkers: ["REA_C_ENTRY", "REA_C_LEAF"], hopperOracle: HOPPER_C_ORACLE, }, }); @@ -85,7 +86,7 @@ async function buildPortableFixtures() { `${symbolPrefix}rea_version_leaf`, ...(second ? [`${symbolPrefix}rea_added`] : []), ], - strings: [second ? "REA_VERSION_TWO" : "REA_VERSION_ONE"], + stringMarkers: [second ? "REA_VERSION_TWO" : "REA_VERSION_ONE"], }, }); } @@ -127,7 +128,11 @@ async function buildDarwinFixtures() { ], expectations: { symbols: ["_OBJC_CLASS_$_REAWidget"], - strings: ["REAWidget", "REAWidgetDelegate", "performAction:error:"], + stringFragments: [ + "REAWidget", + "REAWidgetDelegate", + "performAction:error:", + ], }, }); await build({ @@ -144,7 +149,7 @@ async function buildDarwinFixtures() { ], expectations: { symbols: ["_napi_register_module_v1"], - strings: ["reaProbeOne", "reaProbeTwo", "reaProbeThree"], + stringMarkers: ["reaProbeOne", "reaProbeTwo", "reaProbeThree"], }, }); try { @@ -163,13 +168,8 @@ async function buildDarwinFixtures() { join(outputRoot, "swift"), ], expectations: { - strings: [ - "REA_SWIFT_EXECUTE", - "REAService", - "REARecord", - "REAState", - "REAProtocol", - ], + stringMarkers: ["REA_SWIFT_EXECUTE"], + stringFragments: ["REAService", "REARecord", "REAState", "REAProtocol"], }, compiler: swiftc, compilerVersion: swiftVersion, diff --git a/scripts/lib/conformance-fixtures.d.mts b/scripts/lib/conformance-fixtures.d.mts index 4c34ecb5a0..947442e393 100644 --- a/scripts/lib/conformance-fixtures.d.mts +++ b/scripts/lib/conformance-fixtures.d.mts @@ -1,4 +1,9 @@ export const LARGE_FIXTURE_COUNT: number; +export function parseConformanceFixtureManifest(value: unknown): { + readonly schemaVersion: 2; + readonly platform: "darwin" | "linux"; + readonly fixtures: readonly Record[]; +}; export const HOPPER_C_ORACLE: Readonly<{ mainProcedure: "main"; entryProcedure: "rea_entry"; @@ -9,6 +14,8 @@ export const HOPPER_C_ORACLE: Readonly<{ globalName: "rea_c_global"; }>; export function sha256(value: string | Uint8Array): string; +/** `nm -j` ends each name with LF. A carriage return before that LF belongs to the name. */ +export function splitNmSymbolRecords(output: string): string[]; export function generateLargeFixture(): string; export function sourceDigest( sources: readonly { readonly path: string; readonly content: string }[], diff --git a/scripts/lib/conformance-fixtures.mjs b/scripts/lib/conformance-fixtures.mjs index 5c54f6edf4..69d8a79df9 100644 --- a/scripts/lib/conformance-fixtures.mjs +++ b/scripts/lib/conformance-fixtures.mjs @@ -4,6 +4,37 @@ import { isAbsolute, relative, resolve, sep } from "node:path"; export const LARGE_FIXTURE_COUNT = 1_205; +/** Admit the current producer contract; unversioned substring checks are retired. */ +export const parseConformanceFixtureManifest = (manifest) => { + if ( + manifest?.schemaVersion !== 2 || + !["darwin", "linux"].includes(manifest.platform) || + !Array.isArray(manifest.fixtures) + ) + throw new Error( + "Unsupported conformance manifest; regenerate with npm run build:fixtures", + ); + for (const fixture of manifest.fixtures) { + if (fixture === null || typeof fixture !== "object") + throw new Error("Invalid conformance fixture entry"); + const expectations = fixture.expectations; + if (expectations === undefined) continue; + if ( + expectations === null || + typeof expectations !== "object" || + Object.hasOwn(expectations, "strings") || + ["stringMarkers", "stringFragments"].some( + (key) => + expectations[key] !== undefined && + (!Array.isArray(expectations[key]) || + expectations[key].some((value) => typeof value !== "string")), + ) + ) + throw new Error("Invalid conformance fixture string expectations"); + } + return manifest; +}; + export const HOPPER_C_ORACLE = Object.freeze({ mainProcedure: "main", entryProcedure: "rea_entry", @@ -17,6 +48,10 @@ export const HOPPER_C_ORACLE = Object.freeze({ export const sha256 = (value) => createHash("sha256").update(value).digest("hex"); +/** `nm -j` ends each name with LF. A carriage return before that LF belongs to the name. */ +export const splitNmSymbolRecords = (output) => + output.split("\n").filter((record) => record.length > 0); + export const generateLargeFixture = () => { const declarations = Array.from( { length: LARGE_FIXTURE_COUNT }, diff --git a/scripts/lib/real-hopper-fixture.mjs b/scripts/lib/real-hopper-fixture.mjs index b12989f7f0..af0144a6e6 100644 --- a/scripts/lib/real-hopper-fixture.mjs +++ b/scripts/lib/real-hopper-fixture.mjs @@ -3,14 +3,15 @@ import { dirname } from "node:path"; import { bindConformanceFixtureArtifact, resolveConformanceFixturePath, + parseConformanceFixtureManifest, } from "./conformance-fixtures.mjs"; /** Bind the real-Hopper semantic verifier to source-owned manifest artifacts. */ export async function loadRealHopperFixtureTargets(manifestPath) { const canonicalManifest = await realpath(manifestPath); - const manifest = JSON.parse(await readFile(canonicalManifest, "utf8")); - if (!Array.isArray(manifest?.fixtures)) - throw new Error("Invalid conformance fixture manifest"); + const manifest = parseConformanceFixtureManifest( + JSON.parse(await readFile(canonicalManifest, "utf8")), + ); const primary = fixtureEntry(manifest.fixtures, "c"); const secondary = fixtureEntry(manifest.fixtures, "version-v2"); const large = fixtureEntry(manifest.fixtures, "large"); diff --git a/scripts/verify-conformance-fixtures.mjs b/scripts/verify-conformance-fixtures.mjs index 93efb6c3fe..856f662030 100644 --- a/scripts/verify-conformance-fixtures.mjs +++ b/scripts/verify-conformance-fixtures.mjs @@ -9,6 +9,8 @@ import { generateLargeFixture, sha256, sourceDigest, + parseConformanceFixtureManifest, + splitNmSymbolRecords, } from "./lib/conformance-fixtures.mjs"; import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; @@ -17,19 +19,18 @@ const verifierRun = createVerifierRun(); const manifestArgument = process.argv[2] ?? "build/conformance/manifest.json"; const manifestPath = await realpath(resolve(manifestArgument)); const sourceRoot = await resolveSourceRoot(manifestPath, process.argv[3]); -const manifest = JSON.parse(await readFile(manifestPath, "utf8")); -if ( - !Array.isArray(manifest.fixtures) || - (manifest.platform !== "darwin" && manifest.platform !== "linux") -) - throw new Error("Unsupported conformance manifest"); +const manifest = parseConformanceFixtureManifest( + JSON.parse(await readFile(manifestPath, "utf8")), +); if (manifest.platform !== process.platform) throw new Error( `Conformance manifest platform ${manifest.platform} does not match ${process.platform}`, ); const root = dirname(manifestPath); const fileMarker = manifest.platform === "darwin" ? "Mach-O" : "ELF"; -const nmArguments = manifest.platform === "darwin" ? ["-gU"] : ["-g"]; +// Both GNU and LLVM nm define -j as names-only output, one symbol per line. +const nmArguments = + manifest.platform === "darwin" ? ["-gU", "-j"] : ["-g", "-j"]; for (const fixture of manifest.fixtures) await verifyFixture(fixture); @@ -91,21 +92,28 @@ async function verifyFixture(fixture) { const fileOutput = (await exec("file", [artifactPath])).stdout; if (!fileOutput.includes(fileMarker)) throw new Error(`${fixture.name}: not ${fileMarker}`); - const symbols = ( - await exec("nm", [...nmArguments, artifactPath], { - maxBuffer: 20 * 1024 * 1024, - }) - ).stdout; + const symbols = splitNmSymbolRecords( + ( + await exec("nm", [...nmArguments, artifactPath], { + maxBuffer: 20 * 1024 * 1024, + }) + ).stdout, + ); const strings = ( await exec("strings", [artifactPath], { maxBuffer: 20 * 1024 * 1024 }) - ).stdout; + ).stdout + .split(/\r?\n/u) + .filter(Boolean); const expectations = fixture.expectations ?? {}; for (const symbol of expectations.symbols ?? []) if (!symbols.includes(symbol)) throw new Error(`${fixture.name}: missing symbol ${symbol}`); - for (const value of expectations.strings ?? []) + for (const value of expectations.stringMarkers ?? []) if (!strings.includes(value)) - throw new Error(`${fixture.name}: missing string ${value}`); + throw new Error(`${fixture.name}: missing string marker ${value}`); + for (const value of expectations.stringFragments ?? []) + if (!strings.some((record) => record.includes(value))) + throw new Error(`${fixture.name}: missing string fragment ${value}`); verifyCount( symbols, expectations.symbolPrefix, @@ -122,9 +130,7 @@ async function verifyFixture(fixture) { function verifyCount(output, prefix, expected, fixture) { if (prefix === undefined || expected === undefined) return; - const count = output - .split("\n") - .filter((line) => line.includes(prefix)).length; + const count = output.filter((record) => record.startsWith(prefix)).length; if (count !== expected) throw new Error( `${fixture}: expected ${expected} ${prefix} records, received ${count}`, diff --git a/tests/conformance/package/conformanceFixtureVerifier.test.ts b/tests/conformance/package/conformanceFixtureVerifier.test.ts new file mode 100644 index 0000000000..3c17c20506 --- /dev/null +++ b/tests/conformance/package/conformanceFixtureVerifier.test.ts @@ -0,0 +1,181 @@ +import { createHash } from "node:crypto"; +import { + mkdir, + mkdtemp, + readFile, + realpath, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { expect, it, onTestFinished } from "vitest"; + +import { splitNmSymbolRecords } from "../../../scripts/lib/conformance-fixtures.mjs"; +import { createVerifierCommands } from "../../../scripts/lib/verify-commands.mjs"; +import { removeTestWorkspace } from "../../support/workspace/workspaceFixture.js"; + +const digest = (bytes: string | Buffer) => + createHash("sha256").update(bytes).digest("hex"); +const verifier = fileURLToPath( + new URL("../../../scripts/verify-conformance-fixtures.mjs", import.meta.url), +); + +const ownedFixture = async (prefix: string) => { + const commands = await createVerifierCommands(); + let root: string | undefined; + onTestFinished(async () => { + await commands.close(); + if (root !== undefined) await removeTestWorkspace(root); + }); + root = await realpath(await mkdtemp(join(await realpath(tmpdir()), prefix))); + return { commands, root }; +}; + +it.skipIf(process.platform !== "darwin" && process.platform !== "linux").each([ + { mode: "complete", marker: "REA_C_ENTRY" }, + { mode: "renamed", marker: "REA_C_ENTRY" }, + { mode: "wrong-prefix", marker: "REA_C_ENTRY" }, + { mode: "prefix-complete", marker: "REA_C_ENTRY" }, + { mode: "marker", marker: "REA_C_ENTRY" }, + { mode: "marker", marker: "REA_VERSION_ONE" }, + { mode: "marker", marker: "reaProbeOne" }, + { mode: "marker", marker: "REA_SWIFT_EXECUTE" }, + { mode: "fragment", marker: "REAService" }, + { mode: "fragment", marker: "performAction:error:" }, +])( + "verifies compiler-produced fixture records: $mode / $marker", + async ({ mode, marker }) => { + const { commands, root } = await ownedFixture("rea-fixture-symbols-"); + const sources = join(root, "sources"); + const output = join(root, "output"); + await mkdir(sources); + await mkdir(output); + const fixtures = []; + const prefix = process.platform === "darwin" ? "_" : ""; + for (const version of [1, 2]) { + const name = `version-v${version}`; + const path = `${name}.c`; + const artifact = `${name}.o`; + const suffix = mode === "renamed" ? "_renamed" : ""; + const literal = + mode === "marker" + ? `${marker}_renamed` + : mode === "fragment" + ? `embedded_${marker}_suffix` + : marker; + const counted = mode === "prefix-complete" ? "" : "unrelated_"; + const content = `int rea_version_entry${suffix}(void) { return ${version}; }\nint ${counted}rea_inventory_0000(void) { return 0; }\nconst char *marker = "${literal}";\n`; + await writeFile(join(sources, path), content); + await commands.exec( + "cc", + ["-c", join(sources, path), "-o", join(output, artifact)], + { timeout: 10_000 }, + ); + const bytes = await readFile(join(output, artifact)); + fixtures.push({ + name, + artifact, + artifactSha256: digest(bytes), + sources: [{ path, sha256: digest(content) }], + sourceSha256: digest(`${path}\0${digest(content)}\n`), + expectations: { + symbols: [`${prefix}rea_version_entry`], + ...(mode === "fragment" + ? { stringFragments: [marker] } + : { stringMarkers: [marker] }), + ...(mode === "wrong-prefix" || mode === "prefix-complete" + ? { symbolPrefix: `${prefix}rea_inventory_`, symbolCount: 1 } + : {}), + }, + }); + } + const manifest = join(output, "manifest.json"); + await writeFile( + manifest, + JSON.stringify({ + schemaVersion: 2, + platform: process.platform, + fixtures, + }), + ); + const result = commands.exec( + process.execPath, + [verifier, manifest, sources], + { + cwd: root, + timeout: 25_000, + }, + ); + if (!["complete", "fragment", "prefix-complete"].includes(mode)) + await expect(result).rejects.toMatchObject({ + code: 1, + snapshot: { + stderr: { + text: expect.stringContaining( + mode === "renamed" + ? `missing symbol ${prefix}rea_version_entry` + : mode === "wrong-prefix" + ? `expected 1 ${prefix}rea_inventory_ records, received 0` + : `missing string marker ${marker}`, + ), + }, + }, + }); + else { + const completed = await result; + expect(completed.stderr).toBe(""); + expect(JSON.parse(completed.stdout)).toMatchObject({ verified: 2 }); + } + }, + 120_000, +); + +it.each([ + { schemaVersion: undefined, expectations: {} }, + { schemaVersion: 1, expectations: {} }, + { schemaVersion: 2, expectations: { strings: ["ambiguous"] } }, +])( + "refuses retired manifest semantics: %j", + async (legacy) => { + const { commands, root } = await ownedFixture( + "rea-fixture-manifest-version-", + ); + const manifest = join(root, "manifest.json"); + await writeFile( + manifest, + JSON.stringify({ + schemaVersion: legacy.schemaVersion, + platform: "linux", + fixtures: [{ expectations: legacy.expectations }], + }), + ); + await expect( + commands.exec(process.execPath, [verifier, manifest, root], { + cwd: root, + timeout: 10_000, + }), + ).rejects.toMatchObject({ + code: 1, + snapshot: { + stderr: { + text: expect.stringContaining( + legacy.schemaVersion === 2 + ? "Invalid conformance fixture string expectations" + : "Unsupported conformance manifest", + ), + }, + }, + }); + }, + 30_000, +); + +it("preserves a carriage return that belongs to an nm symbol name", () => { + expect(splitNmSymbolRecords("rea_entry\r\nrea_inventory_0000\n")).toEqual([ + "rea_entry\r", + "rea_inventory_0000", + ]); + expect(splitNmSymbolRecords("rea_entry\r\n")).toEqual(["rea_entry\r"]); +}); diff --git a/tests/conformance/providers/hopper/realHopperFixture.test.ts b/tests/conformance/providers/hopper/realHopperFixture.test.ts index 506f3e9ece..1bee8774d5 100644 --- a/tests/conformance/providers/hopper/realHopperFixture.test.ts +++ b/tests/conformance/providers/hopper/realHopperFixture.test.ts @@ -14,6 +14,16 @@ afterEach(async () => { }); describe("real Hopper fixture binding", () => { + it("refuses the retired unversioned manifest before artifact admission", async () => { + const manifest = await fixtureManifest(); + const parsed = JSON.parse(await readFile(manifest, "utf8")); + delete parsed.schemaVersion; + await writeFile(manifest, JSON.stringify(parsed)); + await expect(loadRealHopperFixtureTargets(manifest)).rejects.toThrow( + "regenerate with npm run build:fixtures", + ); + }); + it("binds exact source-owned artifacts and their semantic oracle", async () => { const manifest = await fixtureManifest(); await expect(loadRealHopperFixtureTargets(manifest)).resolves.toMatchObject( @@ -92,6 +102,8 @@ const fixtureManifest = async (): Promise => { await writeFile( manifest, JSON.stringify({ + schemaVersion: 2, + platform: "darwin", fixtures: [ { name: "c",