fix(ci)!: verify exact conformance fixture records (#9977)

* fix(ci)!: verify exact conformance fixture records

BREAKING CHANGE: Regenerate unversioned conformance manifests with npm run build:fixtures. Schema version 2 separates exact string markers from intentional fragments and rejects the ambiguous strings field.

* test(conformance): wait out a cold verifier instead of treating SIGTERM as rejection

The compiler-produced complete/REA_C_ENTRY run was killed by its 10s
execFile timeout on the loaded source-test runner, with empty stderr and
SIGTERM. The stricter record checks already passed locally and for the
other compiler-produced cases. Assert the process status and give that
cold run room inside the test budget.

* fix(ci): preserve carriage returns in nm symbol records

nm -j terminates each name with LF. A carriage return before that LF is
part of the symbol. Splitting on CR LF dropped it, so a renamed symbol
ending in CR matched the name without it.

* test(ci): supervise conformance verifier subprocesses

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>

scripts/build-conformance-fixtures.mjs

@@ -37,6 +37,7 @@ await buildPortableFixtures();
 if (hostPlatform === "darwin") await buildDarwinFixtures();
 
 const manifest = {
+  schemaVersion: 2,
   platform: hostPlatform,
   architecture,
   generatedAt: new Date(0).toISOString(),
@@ -64,7 +65,7 @@ async function buildPortableFixtures() {
       symbols: ["rea_entry", "rea_branch", "rea_leaf", "rea_c_global"].map(
         (name) => `${symbolPrefix}${name}`,
       ),
-      strings: ["REA_C_ENTRY", "REA_C_LEAF"],
+      stringMarkers: ["REA_C_ENTRY", "REA_C_LEAF"],
       hopperOracle: HOPPER_C_ORACLE,
     },
   });
@@ -85,7 +86,7 @@ async function buildPortableFixtures() {
           `${symbolPrefix}rea_version_leaf`,
           ...(second ? [`${symbolPrefix}rea_added`] : []),
         ],
-        strings: [second ? "REA_VERSION_TWO" : "REA_VERSION_ONE"],
+        stringMarkers: [second ? "REA_VERSION_TWO" : "REA_VERSION_ONE"],
       },
     });
   }
@@ -127,7 +128,11 @@ async function buildDarwinFixtures() {
     ],
     expectations: {
       symbols: ["_OBJC_CLASS_$_REAWidget"],
-      strings: ["REAWidget", "REAWidgetDelegate", "performAction:error:"],
+      stringFragments: [
+        "REAWidget",
+        "REAWidgetDelegate",
+        "performAction:error:",
+      ],
     },
   });
   await build({
@@ -144,7 +149,7 @@ async function buildDarwinFixtures() {
     ],
     expectations: {
       symbols: ["_napi_register_module_v1"],
-      strings: ["reaProbeOne", "reaProbeTwo", "reaProbeThree"],
+      stringMarkers: ["reaProbeOne", "reaProbeTwo", "reaProbeThree"],
     },
   });
   try {
@@ -163,13 +168,8 @@ async function buildDarwinFixtures() {
         join(outputRoot, "swift"),
       ],
       expectations: {
-        strings: [
-          "REA_SWIFT_EXECUTE",
-          "REAService",
-          "REARecord",
-          "REAState",
-          "REAProtocol",
-        ],
+        stringMarkers: ["REA_SWIFT_EXECUTE"],
+        stringFragments: ["REAService", "REARecord", "REAState", "REAProtocol"],
       },
       compiler: swiftc,
       compilerVersion: swiftVersion,

scripts/lib/conformance-fixtures.d.mts

@@ -1,4 +1,9 @@
 export const LARGE_FIXTURE_COUNT: number;
+export function parseConformanceFixtureManifest(value: unknown): {
+  readonly schemaVersion: 2;
+  readonly platform: "darwin" | "linux";
+  readonly fixtures: readonly Record<string, unknown>[];
+};
 export const HOPPER_C_ORACLE: Readonly<{
   mainProcedure: "main";
   entryProcedure: "rea_entry";
@@ -9,6 +14,8 @@ export const HOPPER_C_ORACLE: Readonly<{
   globalName: "rea_c_global";
 }>;
 export function sha256(value: string | Uint8Array): string;
+/** `nm -j` ends each name with LF. A carriage return before that LF belongs to the name. */
+export function splitNmSymbolRecords(output: string): string[];
 export function generateLargeFixture(): string;
 export function sourceDigest(
   sources: readonly { readonly path: string; readonly content: string }[],

scripts/lib/conformance-fixtures.mjs

@@ -4,6 +4,37 @@ import { isAbsolute, relative, resolve, sep } from "node:path";
 
 export const LARGE_FIXTURE_COUNT = 1_205;
 
+/** Admit the current producer contract; unversioned substring checks are retired. */
+export const parseConformanceFixtureManifest = (manifest) => {
+  if (
+    manifest?.schemaVersion !== 2 ||
+    !["darwin", "linux"].includes(manifest.platform) ||
+    !Array.isArray(manifest.fixtures)
+  )
+    throw new Error(
+      "Unsupported conformance manifest; regenerate with npm run build:fixtures",
+    );
+  for (const fixture of manifest.fixtures) {
+    if (fixture === null || typeof fixture !== "object")
+      throw new Error("Invalid conformance fixture entry");
+    const expectations = fixture.expectations;
+    if (expectations === undefined) continue;
+    if (
+      expectations === null ||
+      typeof expectations !== "object" ||
+      Object.hasOwn(expectations, "strings") ||
+      ["stringMarkers", "stringFragments"].some(
+        (key) =>
+          expectations[key] !== undefined &&
+          (!Array.isArray(expectations[key]) ||
+            expectations[key].some((value) => typeof value !== "string")),
+      )
+    )
+      throw new Error("Invalid conformance fixture string expectations");
+  }
+  return manifest;
+};
+
 export const HOPPER_C_ORACLE = Object.freeze({
   mainProcedure: "main",
   entryProcedure: "rea_entry",
@@ -17,6 +48,10 @@ export const HOPPER_C_ORACLE = Object.freeze({
 export const sha256 = (value) =>
   createHash("sha256").update(value).digest("hex");
 
+/** `nm -j` ends each name with LF. A carriage return before that LF belongs to the name. */
+export const splitNmSymbolRecords = (output) =>
+  output.split("\n").filter((record) => record.length > 0);
+
 export const generateLargeFixture = () => {
   const declarations = Array.from(
     { length: LARGE_FIXTURE_COUNT },

scripts/lib/real-hopper-fixture.mjs

@@ -3,14 +3,15 @@ import { dirname } from "node:path";
 import {
   bindConformanceFixtureArtifact,
   resolveConformanceFixturePath,
+  parseConformanceFixtureManifest,
 } from "./conformance-fixtures.mjs";
 
 /** Bind the real-Hopper semantic verifier to source-owned manifest artifacts. */
 export async function loadRealHopperFixtureTargets(manifestPath) {
   const canonicalManifest = await realpath(manifestPath);
-  const manifest = JSON.parse(await readFile(canonicalManifest, "utf8"));
-  if (!Array.isArray(manifest?.fixtures))
-    throw new Error("Invalid conformance fixture manifest");
+  const manifest = parseConformanceFixtureManifest(
+    JSON.parse(await readFile(canonicalManifest, "utf8")),
+  );
   const primary = fixtureEntry(manifest.fixtures, "c");
   const secondary = fixtureEntry(manifest.fixtures, "version-v2");
   const large = fixtureEntry(manifest.fixtures, "large");

scripts/verify-conformance-fixtures.mjs

@@ -9,6 +9,8 @@ import {
   generateLargeFixture,
   sha256,
   sourceDigest,
+  parseConformanceFixtureManifest,
+  splitNmSymbolRecords,
 } from "./lib/conformance-fixtures.mjs";
 import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
 
@@ -17,19 +19,18 @@ const verifierRun = createVerifierRun();
 const manifestArgument = process.argv[2] ?? "build/conformance/manifest.json";
 const manifestPath = await realpath(resolve(manifestArgument));
 const sourceRoot = await resolveSourceRoot(manifestPath, process.argv[3]);
-const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
-if (
-  !Array.isArray(manifest.fixtures) ||
-  (manifest.platform !== "darwin" && manifest.platform !== "linux")
-)
-  throw new Error("Unsupported conformance manifest");
+const manifest = parseConformanceFixtureManifest(
+  JSON.parse(await readFile(manifestPath, "utf8")),
+);
 if (manifest.platform !== process.platform)
   throw new Error(
     `Conformance manifest platform ${manifest.platform} does not match ${process.platform}`,
   );
 const root = dirname(manifestPath);
 const fileMarker = manifest.platform === "darwin" ? "Mach-O" : "ELF";
-const nmArguments = manifest.platform === "darwin" ? ["-gU"] : ["-g"];
+// Both GNU and LLVM nm define -j as names-only output, one symbol per line.
+const nmArguments =
+  manifest.platform === "darwin" ? ["-gU", "-j"] : ["-g", "-j"];
 
 for (const fixture of manifest.fixtures) await verifyFixture(fixture);
 
@@ -91,21 +92,28 @@ async function verifyFixture(fixture) {
   const fileOutput = (await exec("file", [artifactPath])).stdout;
   if (!fileOutput.includes(fileMarker))
     throw new Error(`${fixture.name}: not ${fileMarker}`);
-  const symbols = (
-    await exec("nm", [...nmArguments, artifactPath], {
-      maxBuffer: 20 * 1024 * 1024,
-    })
-  ).stdout;
+  const symbols = splitNmSymbolRecords(
+    (
+      await exec("nm", [...nmArguments, artifactPath], {
+        maxBuffer: 20 * 1024 * 1024,
+      })
+    ).stdout,
+  );
   const strings = (
     await exec("strings", [artifactPath], { maxBuffer: 20 * 1024 * 1024 })
-  ).stdout;
+  ).stdout
+    .split(/\r?\n/u)
+    .filter(Boolean);
   const expectations = fixture.expectations ?? {};
   for (const symbol of expectations.symbols ?? [])
     if (!symbols.includes(symbol))
       throw new Error(`${fixture.name}: missing symbol ${symbol}`);
-  for (const value of expectations.strings ?? [])
+  for (const value of expectations.stringMarkers ?? [])
     if (!strings.includes(value))
-      throw new Error(`${fixture.name}: missing string ${value}`);
+      throw new Error(`${fixture.name}: missing string marker ${value}`);
+  for (const value of expectations.stringFragments ?? [])
+    if (!strings.some((record) => record.includes(value)))
+      throw new Error(`${fixture.name}: missing string fragment ${value}`);
   verifyCount(
     symbols,
     expectations.symbolPrefix,
@@ -122,9 +130,7 @@ async function verifyFixture(fixture) {
 
 function verifyCount(output, prefix, expected, fixture) {
   if (prefix === undefined || expected === undefined) return;
-  const count = output
-    .split("\n")
-    .filter((line) => line.includes(prefix)).length;
+  const count = output.filter((record) => record.startsWith(prefix)).length;
   if (count !== expected)
     throw new Error(
       `${fixture}: expected ${expected} ${prefix} records, received ${count}`,

tests/conformance/package/conformanceFixtureVerifier.test.ts

@@ -0,0 +1,181 @@
+import { createHash } from "node:crypto";
+import {
+  mkdir,
+  mkdtemp,
+  readFile,
+  realpath,
+  writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { expect, it, onTestFinished } from "vitest";
+
+import { splitNmSymbolRecords } from "../../../scripts/lib/conformance-fixtures.mjs";
+import { createVerifierCommands } from "../../../scripts/lib/verify-commands.mjs";
+import { removeTestWorkspace } from "../../support/workspace/workspaceFixture.js";
+
+const digest = (bytes: string | Buffer) =>
+  createHash("sha256").update(bytes).digest("hex");
+const verifier = fileURLToPath(
+  new URL("../../../scripts/verify-conformance-fixtures.mjs", import.meta.url),
+);
+
+const ownedFixture = async (prefix: string) => {
+  const commands = await createVerifierCommands();
+  let root: string | undefined;
+  onTestFinished(async () => {
+    await commands.close();
+    if (root !== undefined) await removeTestWorkspace(root);
+  });
+  root = await realpath(await mkdtemp(join(await realpath(tmpdir()), prefix)));
+  return { commands, root };
+};
+
+it.skipIf(process.platform !== "darwin" && process.platform !== "linux").each([
+  { mode: "complete", marker: "REA_C_ENTRY" },
+  { mode: "renamed", marker: "REA_C_ENTRY" },
+  { mode: "wrong-prefix", marker: "REA_C_ENTRY" },
+  { mode: "prefix-complete", marker: "REA_C_ENTRY" },
+  { mode: "marker", marker: "REA_C_ENTRY" },
+  { mode: "marker", marker: "REA_VERSION_ONE" },
+  { mode: "marker", marker: "reaProbeOne" },
+  { mode: "marker", marker: "REA_SWIFT_EXECUTE" },
+  { mode: "fragment", marker: "REAService" },
+  { mode: "fragment", marker: "performAction:error:" },
+])(
+  "verifies compiler-produced fixture records: $mode / $marker",
+  async ({ mode, marker }) => {
+    const { commands, root } = await ownedFixture("rea-fixture-symbols-");
+    const sources = join(root, "sources");
+    const output = join(root, "output");
+    await mkdir(sources);
+    await mkdir(output);
+    const fixtures = [];
+    const prefix = process.platform === "darwin" ? "_" : "";
+    for (const version of [1, 2]) {
+      const name = `version-v${version}`;
+      const path = `${name}.c`;
+      const artifact = `${name}.o`;
+      const suffix = mode === "renamed" ? "_renamed" : "";
+      const literal =
+        mode === "marker"
+          ? `${marker}_renamed`
+          : mode === "fragment"
+            ? `embedded_${marker}_suffix`
+            : marker;
+      const counted = mode === "prefix-complete" ? "" : "unrelated_";
+      const content = `int rea_version_entry${suffix}(void) { return ${version}; }\nint ${counted}rea_inventory_0000(void) { return 0; }\nconst char *marker = "${literal}";\n`;
+      await writeFile(join(sources, path), content);
+      await commands.exec(
+        "cc",
+        ["-c", join(sources, path), "-o", join(output, artifact)],
+        { timeout: 10_000 },
+      );
+      const bytes = await readFile(join(output, artifact));
+      fixtures.push({
+        name,
+        artifact,
+        artifactSha256: digest(bytes),
+        sources: [{ path, sha256: digest(content) }],
+        sourceSha256: digest(`${path}\0${digest(content)}\n`),
+        expectations: {
+          symbols: [`${prefix}rea_version_entry`],
+          ...(mode === "fragment"
+            ? { stringFragments: [marker] }
+            : { stringMarkers: [marker] }),
+          ...(mode === "wrong-prefix" || mode === "prefix-complete"
+            ? { symbolPrefix: `${prefix}rea_inventory_`, symbolCount: 1 }
+            : {}),
+        },
+      });
+    }
+    const manifest = join(output, "manifest.json");
+    await writeFile(
+      manifest,
+      JSON.stringify({
+        schemaVersion: 2,
+        platform: process.platform,
+        fixtures,
+      }),
+    );
+    const result = commands.exec(
+      process.execPath,
+      [verifier, manifest, sources],
+      {
+        cwd: root,
+        timeout: 25_000,
+      },
+    );
+    if (!["complete", "fragment", "prefix-complete"].includes(mode))
+      await expect(result).rejects.toMatchObject({
+        code: 1,
+        snapshot: {
+          stderr: {
+            text: expect.stringContaining(
+              mode === "renamed"
+                ? `missing symbol ${prefix}rea_version_entry`
+                : mode === "wrong-prefix"
+                  ? `expected 1 ${prefix}rea_inventory_ records, received 0`
+                  : `missing string marker ${marker}`,
+            ),
+          },
+        },
+      });
+    else {
+      const completed = await result;
+      expect(completed.stderr).toBe("");
+      expect(JSON.parse(completed.stdout)).toMatchObject({ verified: 2 });
+    }
+  },
+  120_000,
+);
+
+it.each([
+  { schemaVersion: undefined, expectations: {} },
+  { schemaVersion: 1, expectations: {} },
+  { schemaVersion: 2, expectations: { strings: ["ambiguous"] } },
+])(
+  "refuses retired manifest semantics: %j",
+  async (legacy) => {
+    const { commands, root } = await ownedFixture(
+      "rea-fixture-manifest-version-",
+    );
+    const manifest = join(root, "manifest.json");
+    await writeFile(
+      manifest,
+      JSON.stringify({
+        schemaVersion: legacy.schemaVersion,
+        platform: "linux",
+        fixtures: [{ expectations: legacy.expectations }],
+      }),
+    );
+    await expect(
+      commands.exec(process.execPath, [verifier, manifest, root], {
+        cwd: root,
+        timeout: 10_000,
+      }),
+    ).rejects.toMatchObject({
+      code: 1,
+      snapshot: {
+        stderr: {
+          text: expect.stringContaining(
+            legacy.schemaVersion === 2
+              ? "Invalid conformance fixture string expectations"
+              : "Unsupported conformance manifest",
+          ),
+        },
+      },
+    });
+  },
+  30_000,
+);
+
+it("preserves a carriage return that belongs to an nm symbol name", () => {
+  expect(splitNmSymbolRecords("rea_entry\r\nrea_inventory_0000\n")).toEqual([
+    "rea_entry\r",
+    "rea_inventory_0000",
+  ]);
+  expect(splitNmSymbolRecords("rea_entry\r\n")).toEqual(["rea_entry\r"]);
+});

tests/conformance/providers/hopper/realHopperFixture.test.ts

@@ -14,6 +14,16 @@ afterEach(async () => {
 });
 
 describe("real Hopper fixture binding", () => {
+  it("refuses the retired unversioned manifest before artifact admission", async () => {
+    const manifest = await fixtureManifest();
+    const parsed = JSON.parse(await readFile(manifest, "utf8"));
+    delete parsed.schemaVersion;
+    await writeFile(manifest, JSON.stringify(parsed));
+    await expect(loadRealHopperFixtureTargets(manifest)).rejects.toThrow(
+      "regenerate with npm run build:fixtures",
+    );
+  });
+
   it("binds exact source-owned artifacts and their semantic oracle", async () => {
     const manifest = await fixtureManifest();
     await expect(loadRealHopperFixtureTargets(manifest)).resolves.toMatchObject(
@@ -92,6 +102,8 @@ const fixtureManifest = async (): Promise<string> => {
   await writeFile(
     manifest,
     JSON.stringify({
+      schemaVersion: 2,
+      platform: "darwin",
       fixtures: [
         {
           name: "c",