fix(managed): admit ReadyToRun images built for non-Windows hosts (#1985)

ReadyToRun images compiled for Linux, macOS or the BSDs store the COFF machine XORed with IMAGE_FILE_MACHINE_NATIVE_OS_OVERRIDE (dotnet/runtime src/coreclr/inc/pedecoder.h). Every framework assembly in a Linux .NET install has machine 0xFD1D, so target admission and the managed PE reader rejected them as an unsupported architecture.

Decode the PE machine once in src/domain/peMachine.ts, replacing the two duplicated switches, and accept an OS override only on images with a CLI directory.

src/domain/binaryTarget.test.ts

@@ -50,6 +50,41 @@ describe("executable header parsing", () => {
     });
   });
 
+  it.each([
+    ["Linux x86_64", 0x8664 ^ 0x7b79, "x86_64"],
+    ["Linux arm64", 0xaa64 ^ 0x7b79, "arm64"],
+    ["Apple x86_64", 0x8664 ^ 0x4644, "x86_64"],
+    ["Apple arm64", 0xaa64 ^ 0x4644, "arm64"],
+    ["FreeBSD x86_64", 0x8664 ^ 0xadc4, "x86_64"],
+    ["OpenBSD x86_64", 0x8664 ^ 0xadc5, "x86_64"],
+    ["NetBSD x86_64", 0x8664 ^ 0x1993, "x86_64"],
+    ["SunOS x86_64", 0x8664 ^ 0x1992, "x86_64"],
+  ] as const)(
+    "decodes the %s ReadyToRun OS-override machine on a managed image",
+    (_label, machine, architecture) => {
+      const result = parseExecutableHeader(
+        pe(machine, 64, 0x2002, true),
+        "x64",
+      );
+      expect(result.ok && result.value).toMatchObject({
+        format: "pe",
+        architecture,
+        managed: true,
+      });
+    },
+  );
+
+  it("rejects an OS-override machine on a native image", () => {
+    const result = parseExecutableHeader(
+      pe(0x8664 ^ 0x7b79, 64, 0x2002),
+      "x64",
+    );
+    expect(result).toEqual({
+      ok: false,
+      error: "unsupported PE architecture",
+    });
+  });
+
   it("selects the host architecture from a FAT table", () => {
     const bytes = fat([0x01000007, 0x0100000c]);
     const arm = parseExecutableHeader(bytes, "arm64");

src/domain/binaryTarget.ts

@@ -4,6 +4,7 @@ import type {
 } from "./binaryTargetTypes.js";
 import { err, ok, type Result } from "./result.js";
 import { mzWindowsHeaderOffset, parseDosMzHeader } from "./dosMz.js";
+import { decodePeMachine } from "./peMachine.js";
 
 /** Format and architecture facts recovered from an executable header. */
 export type ExecutableMetadata =
@@ -216,14 +217,17 @@ export const parsePeRecord = (
     record.toString("binary", 0, 4) !== "PE\u0000\u0000"
   )
     return err("invalid or truncated PE header");
-  const architecture = peArchitecture(record.readUInt16LE(4));
-  if (architecture === undefined) return err("unsupported PE architecture");
+  const machine = decodePeMachine(record.readUInt16LE(4));
+  if (machine === undefined) return err("unsupported PE architecture");
+  const { architecture } = machine;
   const optionalHeaderSize = record.readUInt16LE(20);
   if (optionalHeaderSize > 4096 || record.length < 24 + optionalHeaderSize)
     return err("invalid or truncated PE optional header");
   const optionalHeader = record.subarray(24, 24 + optionalHeaderSize);
   const managed = peManagedStatus(optionalHeader, architecture);
   if (!managed.ok) return managed;
+  if (machine.nativeOsOverride && !managed.value)
+    return err("unsupported PE architecture");
   const characteristics = record.readUInt16LE(22);
   return ok({
     format: "pe",
@@ -299,21 +303,3 @@ const elfArchitecture = (machine: number): BinaryArchitecture | undefined => {
   }
   return undefined;
 };
-
-const peArchitecture = (
-  machine: number,
-): Exclude<BinaryArchitecture, "mips"> | undefined => {
-  switch (machine) {
-    case 0x14c:
-      return "x86";
-    case 0x8664:
-      return "x86_64";
-    case 0x1c0:
-    case 0x1c2:
-    case 0x1c4:
-      return "arm";
-    case 0xaa64:
-      return "arm64";
-  }
-  return undefined;
-};

src/domain/peMachine.ts

@@ -0,0 +1,50 @@
+import type { BinaryArchitecture } from "./binaryTargetTypes.js";
+
+export type PeArchitecture = Exclude<BinaryArchitecture, "mips">;
+
+/** A decoded COFF machine and whether it carried a ReadyToRun OS override. */
+export interface PeMachine {
+  readonly architecture: PeArchitecture;
+  readonly nativeOsOverride: boolean;
+}
+
+const architectureForMachine = (
+  machine: number,
+): PeArchitecture | undefined => {
+  switch (machine) {
+    case 0x014c:
+      return "x86";
+    case 0x8664:
+      return "x86_64";
+    case 0x01c0:
+    case 0x01c2:
+    case 0x01c4:
+      return "arm";
+    case 0xaa64:
+      return "arm64";
+  }
+  return undefined;
+};
+
+/**
+ * ReadyToRun images compiled for a non-Windows OS store the COFF machine XORed
+ * with IMAGE_FILE_MACHINE_NATIVE_OS_OVERRIDE (dotnet/runtime
+ * src/coreclr/inc/pedecoder.h): Apple, FreeBSD, OpenBSD, Linux, NetBSD, SunOS.
+ */
+const NATIVE_OS_OVERRIDES = [0x4644, 0xadc4, 0xadc5, 0x7b79, 0x1993, 0x1992];
+
+/**
+ * Decode a COFF machine. Only managed images carry an OS override, so callers
+ * must reject `nativeOsOverride` when the image has no CLI header.
+ */
+export const decodePeMachine = (machine: number): PeMachine | undefined => {
+  const direct = architectureForMachine(machine);
+  if (direct !== undefined)
+    return { architecture: direct, nativeOsOverride: false };
+  for (const override of NATIVE_OS_OVERRIDES) {
+    const architecture = architectureForMachine(machine ^ override);
+    if (architecture !== undefined)
+      return { architecture, nativeOsOverride: true };
+  }
+  return undefined;
+};

src/dotnet/ManagedArtifactFailures.test.ts

@@ -46,6 +46,14 @@ describe("managed artifact failure classification", () => {
     ]);
   });
 
+  it("rejects a ReadyToRun OS-override machine without a CLI directory", () => {
+    const bytes = buildNativePeFixture();
+    bytes.writeUInt16LE(0x014c ^ 0x7b79, bytes.readUInt32LE(0x3c) + 4);
+    expect(() =>
+      inspectManagedArtifactBytes(bytes, managedPeFixtureTarget(bytes)),
+    ).toThrow("Unsupported PE machine 0x7a35");
+  });
+
   it("keeps malformed reference and attribute issues across inventory callers", () => {
     const bytes = buildManagedPeFixture({
       references: [

src/dotnet/ManagedPeReader.ts

@@ -1,4 +1,5 @@
 import type { ManagedParseIssue } from "../domain/managed/managedArtifact.js";
+import { decodePeMachine } from "../domain/peMachine.js";
 import {
   managedFailure,
   ManagedReaderFailure,
@@ -100,25 +101,18 @@ const add = (left: number, right: number, scope: string): number => {
   return result;
 };
 
-const architectureFor = (machine: number): ManagedPeLayout["architecture"] => {
-  switch (machine) {
-    case 0x014c:
-      return "x86";
-    case 0x8664:
-      return "x86_64";
-    case 0x01c0:
-    case 0x01c2:
-    case 0x01c4:
-      return "arm";
-    case 0xaa64:
-      return "arm64";
-    default:
-      throw managedFailure(
-        "invalid-directory",
-        "pe.machine",
-        `Unsupported PE machine 0x${machine.toString(16).padStart(4, "0")}`,
-      );
-  }
+const architectureFor = (
+  machine: number,
+  managed: boolean,
+): ManagedPeLayout["architecture"] => {
+  const decoded = decodePeMachine(machine);
+  if (decoded === undefined || (decoded.nativeOsOverride && !managed))
+    throw managedFailure(
+      "invalid-directory",
+      "pe.machine",
+      `Unsupported PE machine 0x${machine.toString(16).padStart(4, "0")}`,
+    );
+  return decoded.architecture;
 };
 
 /** Resolve one section identity for both exact reads and available-byte queries. */
@@ -398,7 +392,7 @@ export const readManagedPeLayout = (bytes: Buffer): ManagedPeLayout => {
     }
   return {
     machine: coff.machine,
-    architecture: architectureFor(coff.machine),
+    architecture: architectureFor(coff.machine, cliDirectoryPresent),
     optionalHeader: optional.optionalHeader,
     sectionCount: coff.sectionCount,
     characteristics: coff.characteristics,

tests/boundary/providers/managed/managedNativeOsMachine.test.ts

@@ -0,0 +1,55 @@
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { expect, it } from "vitest";
+import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js";
+import { parseBinaryTarget } from "../../../../src/application/BinaryTargetResolver.js";
+import { buildManagedPeFixture } from "../../../../src/dotnet/ManagedPe.fixture.js";
+import { ManagedStaticProvider } from "../../../../src/dotnet/ManagedStaticProvider.js";
+
+// ReadyToRun images compiled for a non-Windows OS store the machine XORed with
+// IMAGE_FILE_MACHINE_NATIVE_OS_OVERRIDE (dotnet/runtime src/coreclr/inc/pedecoder.h).
+it.each([
+  ["Linux x86", 0x014c ^ 0x7b79, 0, "x86"],
+  ["Linux ARM", 0x01c4 ^ 0x7b79, 0, "arm"],
+  ["Apple ARM", 0x01c4 ^ 0x4644, 0, "arm"],
+  ["FreeBSD x86", 0x014c ^ 0xadc4, 0, "x86"],
+] as const)(
+  "opens and inspects the %s ReadyToRun machine through the public path",
+  async (_label, machine, cliFlags, architecture) => {
+    const bytes = buildManagedPeFixture({ cliFlags, readyToRun: true });
+    bytes.writeUInt16LE(machine, bytes.readUInt32LE(0x3c) + 4);
+    const directory = await createTestTempDirectory("rea-managed-os-machine-");
+    const path = join(directory, "fixture.dll");
+    await writeFile(path, bytes);
+    const parsed = await parseBinaryTarget(path);
+    if (!parsed.ok) throw parsed.error;
+    expect(parsed.value).toMatchObject({
+      kind: "executable",
+      format: "pe",
+      architecture,
+      managed: true,
+    });
+    const client = new ManagedStaticProvider().createClient(parsed.value);
+    try {
+      for (const operation of [
+        "inspect_managed_artifact",
+        "inspect_managed_members",
+        "inspect_managed_native_boundaries",
+      ] as const) {
+        const observed = await client.execute(operation, {});
+        if (!observed.ok) throw observed.error;
+        expect(observed.value.result).toMatchObject({
+          module: { name: "Fixture.dll" },
+        });
+      }
+      const artifact = await client.execute("inspect_managed_artifact", {});
+      if (!artifact.ok) throw artifact.error;
+      expect(artifact.value.result).toMatchObject({
+        pe: { machine, architecture },
+        classification: { managed_architecture: architecture },
+      });
+    } finally {
+      await client.close();
+    }
+  },
+);