From 635f673a8c281cc4f7424ca571094093d8f4a1ab Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:32:54 +0800 Subject: [PATCH] fix(gpu): own CCOB boundaries and payload representations Use completed codec input to establish legacy CCOB extents, preserving declaration/integrity failures independently. Reuse bounded input windows, charge actual reads and decoding attempts, and qualify recovery after incomplete decoding. Publish only established or qualified terminal extents so interruption retains completed declarations. Recognize bitcode representation in HIP bundles while leaving inner facts unsupported. Fixes #6887 Fixes #6880 --- docs/gpu-code-objects.md | 21 ++- src/gpu/ClangOffloadBundle.ts | 4 +- src/gpu/GpuInventory.test.ts | 109 +++++++++++++++- src/gpu/GpuInventory.ts | 144 ++++++++++++++++++--- src/gpu/GpuPayloadCodec.test.ts | 26 +++- src/gpu/GpuPayloadCodec.ts | 43 ++++-- tests/acceptance/gpuCodeObjectsCli.test.ts | 144 +++++++++++++++++++++ tests/boundary/gpu/codeObjects.test.ts | 116 ++++++++++++++--- tests/fixtures/gpu/codeObjects.ts | 10 ++ 9 files changed, 553 insertions(+), 64 deletions(-) diff --git a/docs/gpu-code-objects.md b/docs/gpu-code-objects.md index 0171b1b0c4..fb02ac7bf7 100644 --- a/docs/gpu-code-objects.md +++ b/docs/gpu-code-objects.md @@ -32,6 +32,12 @@ unexamined payloads remain `unexamined`. An exact AMDGPU target is normalized at its note owner for comparison with a Clang bundle declaration; raw values remain intact. Unresolved feature comparisons stay `unknown`. +A HIP target ID identifies a toolchain target, not its payload representation. +Recognized raw or wrapped LLVM bitcode stays `unsupported` for kernel metadata +inspection, with the bundle ID, range and selected payload digest retained. +Bitcode validity and inner targets remain unexamined; malformed recognized ELF +still reports `malformed`. Standalone bitcode is outside GPU container coverage. + HIP bundle and metadata note recognition accept both the legacy `amdgcn` triple and LLVM's `amdgpu` subarchitecture triples, including declared generic processors such as `gfx11-generic` and `gfx9-4-generic`. Notes with the empty or explicit @@ -40,9 +46,18 @@ retaining their raw triple spelling. CUDA's legacy fatbinary compression flag is retained as an unsupported codec with its encoded and decoded extents; compressed bytes are not treated as PTX text. CCOB is recognized by the exact four signature bytes `CCOB`; high-bit-corrupted bytes are not decoded as ASCII -and accepted as a bundle. CCOB v1 has no stored total size, so a following -legacy bundle is located by the next `CCOB` signature and zero-byte alignment -padding remains part of the first container extent. +and accepted as a bundle. CCOB v1 has no stored total size: the completed zlib +or Zstd stream determines its encoded extent. Embedded `CCOB` text is ordinary +compressed data; zero-byte alignment after the stream belongs to neither +payload. If decoding fails or its codec is unsupported, a later `CCOB` marker +is only an uncertain recovery candidate. The affected row retains that +limitation and discovery remains partial. + +The declared decoded length bounds decompression. An understated length can +stop decoding before codec completion, so any subsequent recovery extent stays +unknown; the decoder does not expand its allowance to prove the bad declaration. +Interrupted observations omit unfinished v1 rows. A terminal capacity row +qualifies its retained range as the containing tail and does not infer siblings. `cuda_target_header` retains the OS ABI, normalized raw `e_flags` and exact header location. CUDA OS ABI `0x33` and the observed `0x41` flags layout use diff --git a/src/gpu/ClangOffloadBundle.ts b/src/gpu/ClangOffloadBundle.ts index 0a42368f5b..fe76dfc9d4 100644 --- a/src/gpu/ClangOffloadBundle.ts +++ b/src/gpu/ClangOffloadBundle.ts @@ -118,8 +118,8 @@ export const readCcobHeader = (bytes: Buffer, remaining: number) => { return { version, method, width, total, decoded, hash }; }; -/** Legacy v1 has no size field; LLVM finds the next CCOB magic as its boundary. */ -export const readLegacyCcobTotal = async ( +/** After failed/unsupported v1 decoding only: a marker is an uncertain recovery candidate. */ +export const findLegacyCcobRecoveryTotal = async ( source: GpuSource, base: number, ): Promise => { diff --git a/src/gpu/GpuInventory.test.ts b/src/gpu/GpuInventory.test.ts index e98be43042..269b14eb22 100644 --- a/src/gpu/GpuInventory.test.ts +++ b/src/gpu/GpuInventory.test.ts @@ -1,24 +1,29 @@ import { expect, it } from "vitest"; import { GpuInventory } from "./GpuInventory.js"; import { GpuBufferSource } from "./GpuSource.js"; +import { assertArtifactReadActive } from "../artifacts/ArtifactFileOperation.js"; import { ccobFixture, cubinFixture, cudaFixture, } from "../../tests/fixtures/gpu/codeObjects.js"; -it.each(["cuda", "ccob"] as const)( +it.each(["cuda", "ccob", "ccob-v1"] as const)( "refuses oversized %s output before reading compressed input", async (kind) => { const bytes = kind === "cuda" ? cudaFixture("zstd", [{ kind: 2, target: 80, bytes: cubinFixture() }]) - : ccobFixture(3); - const payloadOffset = kind === "cuda" ? 80 : 32; - bytes.writeBigUInt64LE( - BigInt(128 * 1024 * 1024 + 1), - kind === "cuda" ? 72 : 16, - ); + : kind === "ccob" + ? ccobFixture(3) + : Buffer.concat([ccobFixture(1), ccobFixture(2)]); + const payloadOffset = kind === "cuda" ? 80 : kind === "ccob" ? 32 : 20; + if (kind === "ccob-v1") bytes.writeUInt32LE(128 * 1024 * 1024 + 1, 8); + else + bytes.writeBigUInt64LE( + BigInt(128 * 1024 * 1024 + 1), + kind === "cuda" ? 72 : 16, + ); const source = new GpuBufferSource(bytes, "fixture"), reads: number[] = [], inventory = new GpuInventory({ path: "/fixture", analysis: "kernels" }); @@ -36,6 +41,65 @@ it.each(["cuda", "ccob"] as const)( sha256: null, }); expect(inventory.kernels).toEqual([]); + if (kind === "ccob-v1") { + expect(inventory.objects).toHaveLength(1); + expect(inventory.objects[0]!.location.size).toBe(bytes.length - 20); + expect(inventory.objects[0]!.limitations.join(" ")).toContain( + "only the containing tail", + ); + expect(inventory.objects[0]!.limitations.join(" ")).toContain( + "Later siblings were not inferred", + ); + } + }, +); +it.each(["zlib", "zstd"] as const)( + "omits an unfinished v1 %s extent when cancellation interrupts its source read", + async (codec) => { + const first = ccobFixture(2, codec), + second = ccobFixture(1, codec), + source = new GpuBufferSource(Buffer.concat([first, second]), "fixture"), + controller = new AbortController(), + reason = { selected: "cancel-v1-payload-read" }, + inventory = new GpuInventory( + { path: "/fixture", analysis: "inventory" }, + controller.signal, + ); + let interrupted = false; + const result = inventory.inspect({ + size: source.size, + containerId: source.containerId, + read: async (offset, size) => { + if (offset === first.length + 20) { + interrupted = true; + controller.abort(reason); + assertArtifactReadActive(controller.signal); + } + return source.read(offset, size); + }, + }); + await expect(result).rejects.toMatchObject({ + reason: "cancelled", + cause: reason, + }); + expect(interrupted).toBe(true); + const wrappers = inventory.objects.filter( + ({ kind }) => kind === "hip-bundle", + ); + expect(wrappers).toHaveLength(1); + expect(wrappers[0]).toMatchObject({ + id: "fixture@0", + inspection: "complete", + encoded_size: first.length - 24, + location: { offset: 24, size: first.length - 24 }, + }); + expect( + inventory.objects.every( + ({ id, parent_id }) => + id !== `fixture@${first.length.toString(16)}` && + parent_id !== `fixture@${first.length.toString(16)}`, + ), + ).toBe(true); }, ); it("observes cancellation while walking flat empty bundles in a decoded buffer", async () => { @@ -55,3 +119,34 @@ it("observes cancellation while walking flat empty bundles in a decoded buffer", cause: reason, }); }); +it.each(["zlib", "zstd"] as const)( + "reuses observed %s input across multiple valid legacy CCOB siblings", + async (codec) => { + const count = 16, + bytes = Buffer.concat( + Array.from({ length: count }, () => ccobFixture(1, codec)), + ), + source = new GpuBufferSource(bytes, "fixture"), + inventory = new GpuInventory({ path: "/fixture", analysis: "inventory" }); + let observedReadBytes = 0; + await inventory.inspect({ + size: source.size, + containerId: source.containerId, + read: async (offset, size) => { + observedReadBytes += size; + return source.read(offset, size); + }, + }); + const bundles = inventory.objects.filter( + ({ kind }) => kind === "hip-bundle", + ); + expect(bundles).toHaveLength(count); + expect(bundles.every(({ inspection }) => inspection === "complete")).toBe( + true, + ); + expect(inventory.discovery).toBe("complete"); + // Includes prefix/alignment reads. Re-reading every suffix grows with the + // square of the sibling count and exceeds this physical-input allowance. + expect(observedReadBytes).toBeLessThan(2 * bytes.length); + }, +); diff --git a/src/gpu/GpuInventory.ts b/src/gpu/GpuInventory.ts index 8926f0e66e..3b1c310459 100644 --- a/src/gpu/GpuInventory.ts +++ b/src/gpu/GpuInventory.ts @@ -1,14 +1,14 @@ import { CCOB_MAGIC, CLANG_BUNDLE_MAGIC, - readLegacyCcobTotal, + findLegacyCcobRecoveryTotal, readClangBundleDirectory, readCcobHeader, } from "./ClangOffloadBundle.js"; import { gpuObject, gpuLocation, type GpuPayloadTarget } from "./GpuObject.js"; import { readCudaTarget } from "./CudaTarget.js"; import { readCudaFatbinary } from "./CudaFatbinary.js"; -import { decodeGpuPayload } from "./GpuPayloadCodec.js"; +import { decodeGpuPayload, decodeGpuPayloadStream } from "./GpuPayloadCodec.js"; import { createHash } from "node:crypto"; import { scheduler } from "node:timers/promises"; import type { @@ -99,6 +99,11 @@ export class GpuInventory { row.sha256 = sha256; row.decoded_size = size; } + #encodedExtent(row: GpuObject, size: number): void { + this.#retain({ encoded_size: size, size }); + row.encoded_size = size; + row.location.size = size; + } #objectLimitations(row: GpuObject, messages: string[]): void { this.#retain(messages); row.limitations.push(...messages); @@ -394,6 +399,8 @@ export class GpuInventory { this.kernels.push(kernel); } } else if (row.kind === "cubin" || row.kind === "amdgpu") { + if (row.kind === "amdgpu" && isLlvmBitcode(bytes)) + throw new BinaryFormatFailure("unsupported", LLVM_BITCODE_LIMITATION); coverage = await this.#elfFacts(decoded, row, row.kind, preparedAmd); } else { row.inspection = "unsupported"; @@ -464,6 +471,19 @@ export class GpuInventory { }; try { + // An AMD target declaration does not establish an ELF representation. + // Without an outer target, inventory must still avoid an ELF corruption + // claim for recognizable bitcode; its inner target remains unexamined. + if ( + directory === undefined && + isLlvmBitcode(await child.read(0, Math.min(4, child.size))) + ) { + this.discovery = "partial"; + if (this.input.analysis !== "kernels" || !this.#selected(row)) + this.#objectLimitations(row, [LLVM_BITCODE_LIMITATION]); + await this.#payload(source, row); + return; + } const native = directory ?? (await readGpuNativeDirectory( @@ -528,6 +548,7 @@ export class GpuInventory { async #hip(source: GpuSource, depth: number): Promise { let base = 0; let bundles = 0; + let legacyWindow: { offset: number; bytes: Buffer } | undefined; while (base < source.size) { if (++bundles % 128 === 0) { await scheduler.yield(); @@ -552,12 +573,10 @@ export class GpuInventory { } if (!prefix.subarray(0, CCOB_MAGIC.length).equals(CCOB_MAGIC)) rejectGpu("Unexpected bytes between Clang offload bundles"); - const legacyTotal = - prefix.length >= 6 && prefix.readUInt16LE(4) === 1 - ? await readLegacyCcobTotal(source, base) - : source.size - base; - const header = readCcobHeader(prefix, legacyTotal), - { width, total, decoded, hash, version, method } = header; + const header = readCcobHeader(prefix, source.size - base), + { width, decoded, hash, version, method } = header; + let total = header.total; + let extentKnown = version !== 1; const row = gpuObject(source, { offset: base + width, size: total - width, @@ -568,21 +587,81 @@ export class GpuInventory { method === 0 ? "zlib" : method === 1 ? "zstd" : "unsupported"; row.compression_id = `ccob:${version}:${method}`; row.decoded_size = decoded; - this.#add(row); + if (extentKnown) this.#add(row); if (row.compression === "unsupported") { row.inspection = "unsupported"; this.#objectLimitations(row, ["Unsupported CCOB compression method"]); this.discovery = "partial"; } else try { - this.#budgetWork(total - width, decoded); - const data = await decodeGpuPayload( - await source.read(base + width, total - width), - row.compression, - decoded, - this.signal, - version === 1, - ); + // v1 has no encoded size. Supply a bounded suffix, then retain only + // the codec's completed extent, never an embedded CCOB marker. + const inputSize = + version === 1 + ? Math.min(total - width, GPU_OBJECT_BYTES) + : total - width; + let data: Buffer; + if (version === 1) { + const start = base + width; + const cached = + legacyWindow !== undefined && + start >= legacyWindow.offset && + start < legacyWindow.offset + legacyWindow.bytes.length; + const readWindow = async (): Promise => { + this.#budgetWork(inputSize, decoded); + legacyWindow = undefined; + const bytes = await source.read(start, inputSize); + legacyWindow = { offset: start, bytes }; + return bytes; + }; + if (cached) this.#budgetWork(decoded); + let encoded: Buffer | undefined = cached + ? legacyWindow!.bytes.subarray(start - legacyWindow!.offset) + : await readWindow(); + const available = encoded.length; + const compression = row.compression; + const completed = await decodeGpuPayloadStream( + encoded, + compression, + decoded, + this.signal, + ).catch(async (cause: unknown) => { + // A cached window may end inside this sibling. Refill once + // before treating that incomplete view as a malformed stream. + if ( + !(cause instanceof BinaryFormatFailure) || + !cached || + available >= inputSize + ) + throw cause; + // Retire the previous window before allocating its replacement. + encoded = undefined; + return decodeGpuPayloadStream( + await readWindow(), + compression, + decoded, + this.signal, + ); + }); + total = width + completed.consumedBytes; + extentKnown = true; + this.#encodedExtent(row, completed.consumedBytes); + // Publish the completed parent extent before inspecting its children. + this.#add(row); + data = completed.buffer; + if (data.length !== decoded) + rejectGpu( + "GPU decompressed length disagrees with its declaration", + ); + } else { + this.#budgetWork(inputSize, decoded); + data = await decodeGpuPayload( + await source.read(base + width, inputSize), + row.compression, + decoded, + this.signal, + ); + } if ( createHash("md5").update(data).digest().readBigUInt64LE(0) !== hash ) @@ -594,10 +673,41 @@ export class GpuInventory { this.#retainFailure(row, cause); this.discovery = "partial"; } + if (!extentKnown) { + if (row.inspection !== "capacity") + try { + total = await findLegacyCcobRecoveryTotal(source, base); + } catch (cause) { + if (!(cause instanceof BinaryResourceFailure)) throw cause; + this.#retainFailure(row, cause); + } + // A capacity refusal cannot establish a boundary for later siblings. + if (row.inspection === "capacity") { + this.#objectLimitations(row, [ + "CCOB v1 codec extent is unknown; the retained size is only the containing tail, not a completed codec extent. Later siblings were not inferred.", + ]); + this.#add(row); + return; + } + this.#encodedExtent(row, total - width); + this.#objectLimitations(row, [ + total < source.size - base + ? "CCOB v1 codec extent is unknown; the next CCOB marker is only an uncertain recovery candidate." + : "CCOB v1 codec extent is unknown; the retained range extends to the container end.", + ]); + this.#add(row); + } base = await nextGpuContainerOffset(source, base + total); } } } +const LLVM_BITCODE_LIMITATION = + "LLVM bitcode kernel metadata is outside current coverage; bitcode validity and inner target were not inspected."; +/** LLVM raw and wrapper signatures identify representation, not bitcode validity. */ +const isLlvmBitcode = (bytes: Buffer): boolean => + bytes.length >= 4 && + (bytes.readUInt32LE(0) === 0xdec04342 || + bytes.readUInt32LE(0) === 0x0b17c0de); const isGpuSection = (name: string): boolean => [ ".nv_fatbin", diff --git a/src/gpu/GpuPayloadCodec.test.ts b/src/gpu/GpuPayloadCodec.test.ts index 97fc039ee8..465fc725c4 100644 --- a/src/gpu/GpuPayloadCodec.test.ts +++ b/src/gpu/GpuPayloadCodec.test.ts @@ -1,6 +1,6 @@ import { expect, it } from "vitest"; import { deflateSync, zstdCompressSync } from "node:zlib"; -import { decodeGpuPayload } from "./GpuPayloadCodec.js"; +import { decodeGpuPayload, decodeGpuPayloadStream } from "./GpuPayloadCodec.js"; it.each(["zlib", "zstd"] as const)( "requires all declared %s input bytes to belong to the codec stream", async (codec) => { @@ -19,3 +19,27 @@ it.each(["zlib", "zstd"] as const)( ).rejects.toMatchObject({ kind: "malformed" }); }, ); +it.each(["zlib", "zstd"] as const)( + "retains completed %s input accounting before trailing container bytes", + async (codec) => { + const source = Buffer.from("ordinary CCOB marker"), + compressed = + codec === "zstd" + ? zstdCompressSync(source) + : deflateSync(source, { level: 0 }), + input = Buffer.concat([ + compressed, + Buffer.from("CCOB following container"), + ]); + expect(await decodeGpuPayloadStream(input, codec, source.length)).toEqual({ + buffer: source, + consumedBytes: compressed.length, + }); + await expect( + decodeGpuPayloadStream(compressed.subarray(0, -1), codec, source.length), + ).rejects.toMatchObject({ kind: "malformed" }); + await expect( + decodeGpuPayload(compressed, codec, source.length + 1), + ).rejects.toMatchObject({ kind: "malformed" }); + }, +); diff --git a/src/gpu/GpuPayloadCodec.ts b/src/gpu/GpuPayloadCodec.ts index e84a94ad86..29248d5ecc 100644 --- a/src/gpu/GpuPayloadCodec.ts +++ b/src/gpu/GpuPayloadCodec.ts @@ -6,18 +6,17 @@ import { decodeLz4Block } from "./Lz4Block.js"; import { rejectGpu } from "./GpuSource.js"; const unzip = promisify(inflate); const unzstd = promisify(zstdDecompress); -export const decodeGpuPayload = async ( +/** Completed codec bytes and their encoded extent share one completion owner. */ +export const decodeGpuPayloadStream = async ( bytes: Buffer, - compression: "zlib" | "zstd" | "lz4", - expected: number, + compression: "zlib" | "zstd", + maximumDecodedSize: number, signal?: AbortSignal, - allowZeroPadding = false, -): Promise => { +): Promise<{ buffer: Buffer; consumedBytes: number }> => { assertArtifactReadActive(signal); - if (compression === "lz4") return decodeLz4Block(bytes, expected, signal); let result: unknown; try { - const maxOutputLength = Math.max(1, expected); + const maxOutputLength = Math.max(1, maximumDecodedSize); result = await (compression === "zstd" ? unzstd(bytes, { maxOutputLength, @@ -34,16 +33,32 @@ export const decodeGpuPayload = async ( assertArtifactReadActive(signal); const { buffer: decoded, consumedBytes: consumed } = nodeCodecResult(result); if ( - consumed !== bytes.length && - (!allowZeroPadding || - consumed < 0 || - consumed > bytes.length || - !bytes.subarray(consumed).every((byte) => byte === 0)) + !Number.isSafeInteger(consumed) || + consumed <= 0 || + consumed > bytes.length ) + rejectGpu("GPU decoder returned an invalid complete codec extent"); + return { buffer: decoded, consumedBytes: consumed }; +}; +export const decodeGpuPayload = async ( + bytes: Buffer, + compression: "zlib" | "zstd" | "lz4", + expected: number, + signal?: AbortSignal, +): Promise => { + assertArtifactReadActive(signal); + if (compression === "lz4") return decodeLz4Block(bytes, expected, signal); + const decoded = await decodeGpuPayloadStream( + bytes, + compression, + expected, + signal, + ); + if (decoded.consumedBytes !== bytes.length) rejectGpu( "GPU compressed payload contains bytes outside its complete codec stream", ); - if (decoded.length !== expected) + if (decoded.buffer.length !== expected) rejectGpu("GPU decompressed length disagrees with its declaration"); - return decoded; + return decoded.buffer; }; diff --git a/tests/acceptance/gpuCodeObjectsCli.test.ts b/tests/acceptance/gpuCodeObjectsCli.test.ts index b9f5ea8165..7d9c3e73ad 100644 --- a/tests/acceptance/gpuCodeObjectsCli.test.ts +++ b/tests/acceptance/gpuCodeObjectsCli.test.ts @@ -2,6 +2,7 @@ import { expect, it } from "vitest"; import { writeFile, unlink } from "node:fs/promises"; import { join } from "node:path"; import { createHash } from "node:crypto"; +import { deflateSync } from "node:zlib"; import { toolContract } from "../../src/contracts/toolContracts.js"; import { gpuCodeObjectsSchema } from "../../src/domain/gpu/gpuCodeObjects.js"; import { @@ -21,6 +22,7 @@ import { cubinFixture, hipFixture, elfFixture, + llvmBitcodeFixture, } from "../fixtures/gpu/codeObjects.js"; cliTest( @@ -65,6 +67,148 @@ cliTest( }, ); +cliTest( + "inventories a complete CCOB stream containing ordinary CCOB payload text", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-ccob-embedded-marker-"); + const bundle = hipFixture( + Buffer.from("ordinary CCOB marker"), + "host-other", + ), + compressed = deflateSync(bundle, { level: 0 }); + expect(compressed.indexOf("CCOB")).toBeGreaterThan(0); + for (const version of [1, 2] as const) { + const width = version === 1 ? 20 : 24, + header = Buffer.from( + ccobFixture(version, "zlib", bundle).subarray(0, width), + ); + if (version === 2) header.writeUInt32LE(width + compressed.length, 8); + const bytes = Buffer.concat([header, compressed]), + path = join(root, `v${version}`); + await writeFile(path, bytes); + const response = await cli.run({ + arguments: [ + "inspect-gpu-code-objects", + path, + "--analysis", + "inventory", + "--detail", + "complete", + "--json", + ], + }); + expect(response.exitCode, response.stderr).toBe(0); + const evidence = parseEvidence( + toolContract("inspect_gpu_code_objects").outputSchema.parse( + response.json, + ), + ); + const report = gpuCodeObjectsSchema.parse(evidence.normalized_result); + expect(report.coverage).toMatchObject({ + discovery: "complete", + payloads: "not-requested", + }); + expect(report.objects).toHaveLength(3); + expect(report.objects[0]).toMatchObject({ + kind: "hip-bundle", + inspection: "complete", + encoded_size: compressed.length, + decoded_size: bundle.length, + sha256: createHash("sha256").update(bundle).digest("hex"), + location: { offset: width, size: compressed.length }, + }); + expect( + report.objects + .slice(1) + .map(({ bundle_id, inspection }) => [bundle_id, inspection]), + ).toEqual([ + ["host-x86_64-unknown-linux", "complete"], + ["host-other", "complete"], + ]); + } + }, +); + +cliTest( + "distinguishes unsupported HIP bitcode from corrupted ELF through the public CLI", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-hip-bitcode-cli-"), + bitcode = llvmBitcodeFixture(), + corruptedElf = amdFixture(), + bundleId = "hip-amdgcn-amd-amdhsa--gfx900"; + corruptedElf.writeBigUInt64LE(BigInt(corruptedElf.length + 1), 40); + for (const [name, payload, analysis, inspection] of [ + ["inventory", bitcode, "inventory", "inventory"], + ["bitcode", bitcode, "kernels", "unsupported"], + ["corrupt-elf", corruptedElf, "kernels", "malformed"], + ] as const) { + const bytes = hipFixture(payload, bundleId), + path = join(root, name); + await writeFile(path, bytes); + const response = await cli.run({ + arguments: [ + "inspect-gpu-code-objects", + path, + "--analysis", + analysis, + "--targets", + '["gfx900"]', + "--detail", + "complete", + "--json", + ], + }); + expect(response.exitCode, response.stderr).toBe(0); + const evidence = parseEvidence( + toolContract("inspect_gpu_code_objects").outputSchema.parse( + response.json, + ), + ); + const report = gpuCodeObjectsSchema.parse(evidence.normalized_result); + const row = report.objects.find(({ kind }) => kind === "amdgpu")!; + expect(row).toMatchObject({ + bundle_id: bundleId, + target: "gfx900", + payload_target: null, + inspection, + encoded_size: payload.length, + location: { + container_id: null, + offset: bytes.length - payload.length, + size: payload.length, + }, + sha256: + analysis === "inventory" + ? null + : createHash("sha256").update(payload).digest("hex"), + }); + expect(report.coverage.payloads).toBe( + analysis === "inventory" ? "not-requested" : "partial", + ); + expect(report.kernels).toEqual([]); + if (name === "bitcode") { + expect(row.limitations.join(" ")).toContain("LLVM bitcode"); + expect(row.limitations.join(" ")).toContain( + "validity and inner target were not inspected", + ); + } + } + const path = join(root, "raw-bitcode"); + await writeFile(path, bitcode); + const raw = await cli.run({ + arguments: [ + "inspect-gpu-code-objects", + path, + "--analysis", + "kernels", + "--json", + ], + }); + expect(raw.exitCode).toBe(1); + expect(raw.json).toMatchObject({ code: "unsupported_target" }); + }, +); + cliTest( "keeps embedded Clang bundle reads inside their native section", async ({ cli }) => { diff --git a/tests/boundary/gpu/codeObjects.test.ts b/tests/boundary/gpu/codeObjects.test.ts index 029cdb14b8..f172ff2786 100644 --- a/tests/boundary/gpu/codeObjects.test.ts +++ b/tests/boundary/gpu/codeObjects.test.ts @@ -2,6 +2,7 @@ import { expect, it, onTestFinished } from "vitest"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { deflateSync } from "node:zlib"; +import { createHash } from "node:crypto"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { gpuCodeObjectsPartialSchema } from "../../../src/domain/gpu/gpuCodeObjects.js"; import { GpuCodeObjectsProvider } from "../../../src/gpu/GpuCodeObjectsProvider.js"; @@ -15,6 +16,7 @@ import { elfFixture, ptxFixture, amdFixture, + llvmBitcodeFixture, } from "../../fixtures/gpu/codeObjects.js"; const inspect = async ( bytes: Buffer, @@ -278,6 +280,37 @@ it("keeps AMDGPU processor identity unknown when ELF flags are unassigned", asyn inspection: "inventory", }); }); +it.each([false, true])( + "keeps HIP bitcode representation unsupported with wrapped=%s and no declared processor", + async (wrapped) => { + const payload = llvmBitcodeFixture(wrapped), + bytes = hipFixture(payload, "hip-amdgcn-amd-amdhsa--"); + for (const analysis of ["inventory", "kernels"] as const) { + const result = await inspect(bytes, { analysis }); + if (!result.ok) throw result.error; + const row = result.value.objects.find(({ kind }) => kind === "amdgpu")!; + expect(row).toMatchObject({ + bundle_id: "hip-amdgcn-amd-amdhsa--", + target: null, + payload_target: null, + inspection: analysis === "inventory" ? "inventory" : "unsupported", + encoded_size: payload.length, + location: { size: payload.length }, + sha256: + analysis === "inventory" + ? null + : createHash("sha256").update(payload).digest("hex"), + }); + expect(row.limitations.join(" ")).toContain("LLVM bitcode"); + expect(row.limitations.join(" ")).not.toContain("not an ELF image"); + expect(result.value.coverage).toMatchObject({ + discovery: "partial", + payloads: analysis === "inventory" ? "not-requested" : "partial", + }); + expect(result.value.kernels).toEqual([]); + } + }, +); it.each([ ["amdgcn", "gfx900:xnack+"], ["amdgcn", "gfx900+xnack"], @@ -930,26 +963,34 @@ it("does not dispatch a high-bit-corrupted DOS signature to the PE parser", asyn error: { _tag: "AnalysisUnsupportedTargetError" }, }); }); -it("uses each legacy CCOB v1 stream boundary across alignment padding", async () => { - const first = ccobFixture(1, "zlib"), - second = ccobFixture(2, "zlib"), - padding = Buffer.alloc((8 - (first.length % 8)) % 8); - const result = await inspect(Buffer.concat([first, padding, second])); - if (!result.ok) throw result.error; - const bundles = result.value.objects.filter( - ({ kind }) => kind === "hip-bundle", - ); - expect(bundles).toHaveLength(2); - expect( - bundles.map(({ inspection, decoded_size }) => [inspection, decoded_size]), - ).toEqual([ - ["complete", expect.any(Number)], - ["complete", expect.any(Number)], - ]); - expect(bundles[0]!.sha256).toBe(bundles[1]!.sha256); - expect(result.value.kernels).toHaveLength(2); - expect(result.value.coverage.discovery).toBe("complete"); -}); +it.each(["zlib", "zstd"] as const)( + "uses each legacy CCOB v1 %s stream boundary across alignment padding", + async (codec) => { + const first = ccobFixture(1, codec), + second = ccobFixture(2, "zlib"), + padding = Buffer.alloc((8 - (first.length % 8)) % 8); + const result = await inspect(Buffer.concat([first, padding, second])); + if (!result.ok) throw result.error; + const bundles = result.value.objects.filter( + ({ kind }) => kind === "hip-bundle", + ); + expect(bundles).toHaveLength(2); + expect( + bundles.map(({ inspection, decoded_size }) => [inspection, decoded_size]), + ).toEqual([ + ["complete", expect.any(Number)], + ["complete", expect.any(Number)], + ]); + expect(bundles[0]!.sha256).toBe(bundles[1]!.sha256); + expect(bundles[0]!.encoded_size).toBe(first.length - 20); + expect(bundles[0]!.location.size).toBe(first.length - 20); + expect(bundles[1]!.metadata_location.offset).toBe( + first.length + padding.length, + ); + expect(result.value.kernels).toHaveLength(2); + expect(result.value.coverage.discovery).toBe("complete"); + }, +); it("keeps a later CCOB bundle after a truncated legacy v1 stream", async () => { const truncated = ccobFixture(1, "zlib").subarray(0, -1), later = ccobFixture(2, "zlib"); @@ -965,7 +1006,42 @@ it("keeps a later CCOB bundle after a truncated legacy v1 stream", async () => { ]); expect(result.value.kernels).toHaveLength(1); expect(result.value.coverage.discovery).toBe("partial"); + expect(bundles[0]!.limitations.join(" ")).toContain( + "uncertain recovery candidate", + ); }); +it.each(["checksum", "decoded-size"] as const)( + "preserves a completed v1 codec extent after a bad %s declaration", + async (declaration) => { + const bundle = hipFixture( + Buffer.from("ordinary CCOB marker"), + "host-other", + ), + compressed = deflateSync(bundle, { level: 0 }), + header = Buffer.from(ccobFixture(1, "zlib", bundle).subarray(0, 20)); + if (declaration === "checksum") header[12] = header[12]! ^ 1; + else header.writeUInt32LE(bundle.length + 1, 8); + const first = Buffer.concat([header, compressed]); + const result = await inspect( + Buffer.concat([first, ccobFixture(2, "zlib")]), + ); + if (!result.ok) throw result.error; + const wrappers = result.value.objects.filter( + ({ kind }) => kind === "hip-bundle", + ); + expect(wrappers.map(({ inspection }) => inspection)).toEqual([ + "malformed", + "complete", + ]); + expect(wrappers[0]).toMatchObject({ + encoded_size: compressed.length, + location: { size: compressed.length }, + }); + expect(wrappers[0]!.limitations.join(" ")).not.toContain("recovery"); + expect(wrappers[1]!.metadata_location.offset).toBe(first.length); + expect(result.value.coverage.discovery).toBe("partial"); + }, +); it("continues at the declared CCOB boundary after an unsupported codec", async () => { const unknown = ccobFixture(); unknown.writeUInt16LE(99, 6); diff --git a/tests/fixtures/gpu/codeObjects.ts b/tests/fixtures/gpu/codeObjects.ts index 7a39a4a562..49a3ca55b7 100644 --- a/tests/fixtures/gpu/codeObjects.ts +++ b/tests/fixtures/gpu/codeObjects.ts @@ -321,6 +321,16 @@ export const hipFixture = ( head.writeBigUInt64LE(2n, 24); return Buffer.concat([head, ...headers, payload]); }; +/** Representation recognition only; these bytes do not establish valid LLVM IR. */ +export const llvmBitcodeFixture = (wrapped = false): Buffer => { + const raw = Buffer.from([0x42, 0x43, 0xc0, 0xde, 0, 0, 0, 0]); + if (!wrapped) return raw; + const header = Buffer.alloc(20); + header.writeUInt32LE(0x0b17c0de, 0); + header.writeUInt32LE(20, 8); + header.writeUInt32LE(raw.length, 12); + return Buffer.concat([header, raw]); +}; export const ccobFixture = ( version: 1 | 2 | 3 = 2, method: "zlib" | "zstd" = "zstd",