mirror of
https://github.com/morluto/rea.git
synced 2026-11-16 19:53:50 +00:00
refactor(artifacts): break the artifacts-firmware and artifacts-gpu import cycles (#11331)
* refactor(domain): move BinaryContainer into src/domain BinaryContainer imports only domain modules and is shared by artifacts, firmware, gpu, go, il2cpp, python, rust and native. Living under src/native made artifacts and firmware import native for its failure types and BinaryReader. Pure move: git mv plus import specifier updates. * refactor(firmware): move the devicetree parser into src/domain/firmware The FIT artifact reader in src/artifacts parses embedded devicetree blobs with the parser that lived under src/firmware, while firmware's providers import src/artifacts: the two units imported each other. The parser and its fixture are pure and now sit beside domain/firmware/devicetree.ts. Pure move: git mv plus import specifier and baseline key updates. * refactor(artifacts): break the artifacts-gpu import cycle (#11335) * refactor(artifacts): move the LZ4 block decoder into src/artifacts Pure move: git mv plus import specifier updates. The next commit removes its GPU-specific budget. * refactor(artifacts): take the LZ4 output budget from the caller decodeLz4Block bounded every decode by the GPU object budget and threw through rejectGpu, so UnityFS decoding in src/artifacts imported src/gpu while gpu imports artifacts. Callers now pass their budget: GPU payloads keep GPU_OBJECT_BYTES; UnityFS passes its 64 MiB block bound, which its size checks already enforce before decoding. Malformed blocks still throw BinaryFormatFailure with the same messages. * test(architecture): drop the artifacts-firmware and artifacts-gpu cycles from the layer baseline * refactor(artifacts): move the PE resource reader into src/artifacts/pe (#11347) refactor(artifacts): move the PE resource layout and parser into src/artifacts/pe The Node SEA and Deno standalone readers in src/artifacts locate PE resources with the layout and parser that lived under src/native/pe, while those modules already read through src/artifacts (ArtifactReaderFailure, assertArtifactReadActive). Pure move: git mv plus import specifier, baseline key and spec path updates.
This commit is contained in:
1 parent
0c8d8cc879
commit
264dcd5fee
113 files changed
+191
-167
No files matched your search
@@ -3,7 +3,7 @@ import { decode } from "@msgpack/msgpack";
|
||||
import {
|
||||
BinaryResourceFailure,
|
||||
BinaryFormatFailure,
|
||||
} from "../native/metadata/BinaryContainer.js";
|
||||
} from "../domain/binaryContainer.js";
|
||||
import { rejectGpu } from "./GpuSource.js";
|
||||
|
||||
export const AMD_METADATA_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
@@ -4,7 +4,7 @@ import type { GpuObject } from "../domain/gpu/gpuCodeObjects.js";
|
||||
import {
|
||||
BinaryFormatFailure,
|
||||
BinaryResourceFailure,
|
||||
} from "../native/metadata/BinaryContainer.js";
|
||||
} from "../domain/binaryContainer.js";
|
||||
import { assertArtifactReadActive } from "../artifacts/ArtifactFileOperation.js";
|
||||
import {
|
||||
gpuCount,
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
import {
|
||||
BinaryReader,
|
||||
BinaryResourceFailure,
|
||||
} from "../native/metadata/BinaryContainer.js";
|
||||
} from "../domain/binaryContainer.js";
|
||||
import { GPU_OBJECT_BYTES, GPU_REPORT_BYTES, rejectGpu } from "./GpuSource.js";
|
||||
import { Budget } from "../domain/budget.js";
|
||||
import { jsonValueByteLength } from "../domain/jsonValueByteLength.js";
|
||||
|
||||
@@ -36,7 +36,7 @@ import {
|
||||
artifactFileCompletion,
|
||||
} from "../artifacts/ArtifactFileOperation.js";
|
||||
import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js";
|
||||
import { BinaryResourceFailure } from "../native/metadata/BinaryContainer.js";
|
||||
import { BinaryResourceFailure } from "../domain/binaryContainer.js";
|
||||
import {
|
||||
GpuFileSource,
|
||||
gpuCount,
|
||||
|
||||
@@ -17,7 +17,7 @@ import {
|
||||
BinaryReader,
|
||||
BinaryFormatFailure,
|
||||
BinaryResourceFailure,
|
||||
} from "../native/metadata/BinaryContainer.js";
|
||||
} from "../domain/binaryContainer.js";
|
||||
import {
|
||||
readElfSectionDirectory,
|
||||
type NativeSection,
|
||||
|
||||
@@ -23,7 +23,7 @@ import {
|
||||
BinaryFormatFailure,
|
||||
BinaryResourceFailure,
|
||||
isBinaryStringReadLimit,
|
||||
} from "../native/metadata/BinaryContainer.js";
|
||||
} from "../domain/binaryContainer.js";
|
||||
import {
|
||||
readElfSectionDirectory,
|
||||
readPeSectionDirectory,
|
||||
|
||||
@@ -2,8 +2,8 @@ import { nodeCodecResult } from "../compression/NodeCodecResult.js";
|
||||
import { constants, inflate, zstdDecompress } from "node:zlib";
|
||||
import { promisify } from "node:util";
|
||||
import { assertArtifactReadActive } from "../artifacts/ArtifactFileOperation.js";
|
||||
import { decodeLz4Block } from "./Lz4Block.js";
|
||||
import { rejectGpu } from "./GpuSource.js";
|
||||
import { decodeLz4Block } from "../artifacts/Lz4Block.js";
|
||||
import { GPU_OBJECT_BYTES, rejectGpu } from "./GpuSource.js";
|
||||
const unzip = promisify(inflate);
|
||||
const unzstd = promisify(zstdDecompress);
|
||||
/** Completed codec bytes and their encoded extent share one completion owner. */
|
||||
@@ -47,7 +47,8 @@ export const decodeGpuPayload = async (
|
||||
signal?: AbortSignal,
|
||||
): Promise<Buffer> => {
|
||||
assertArtifactReadActive(signal);
|
||||
if (compression === "lz4") return decodeLz4Block(bytes, expected, signal);
|
||||
if (compression === "lz4")
|
||||
return decodeLz4Block(bytes, expected, GPU_OBJECT_BYTES, signal);
|
||||
const decoded = await decodeGpuPayloadStream(
|
||||
bytes,
|
||||
compression,
|
||||
|
||||
@@ -5,7 +5,7 @@ import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
|
||||
import {
|
||||
BinaryFormatFailure,
|
||||
BinaryResourceFailure,
|
||||
} from "../native/metadata/BinaryContainer.js";
|
||||
} from "../domain/binaryContainer.js";
|
||||
import type {
|
||||
NativeReadRequest,
|
||||
NativeSectionDirectory,
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { decodeLz4Block } from "./Lz4Block.js";
|
||||
it("decodes overlapping matches by repeating the newly produced pattern", async () => {
|
||||
// The initial match is the same overlap shape observed in nvcc 12.9 speed-mode PTX.
|
||||
const input = Buffer.from("3c0a2f2f0300504142434445", "hex");
|
||||
expect((await decodeLz4Block(input, 24)).toString()).toBe(
|
||||
"\n//".repeat(6) + "\nABCDE",
|
||||
);
|
||||
});
|
||||
it.each(["10000000", "10000100", "f0ff", "f001", "00"])(
|
||||
"rejects malformed or mismatched block %s",
|
||||
async (hex) => {
|
||||
await expect(decodeLz4Block(Buffer.from(hex, "hex"), 8)).rejects.toThrow();
|
||||
},
|
||||
);
|
||||
it("preserves an already-aborted caller's cancellation", async () => {
|
||||
const signal = AbortSignal.abort("caller stopped");
|
||||
await expect(
|
||||
decodeLz4Block(Buffer.from("10aa", "hex"), 1, signal),
|
||||
).rejects.toMatchObject({ reason: "cancelled" });
|
||||
});
|
||||
@@ -1,75 +0,0 @@
|
||||
import { scheduler } from "node:timers/promises";
|
||||
import { assertArtifactReadActive } from "../artifacts/ArtifactFileOperation.js";
|
||||
import { GPU_OBJECT_BYTES, rejectGpu } from "./GpuSource.js";
|
||||
import { BinaryResourceFailure } from "../native/metadata/BinaryContainer.js";
|
||||
/** Raw LZ4 blocks used by CUDA's 0x2000 mode; not an LZ4 frame decoder. */
|
||||
export const decodeLz4Block = async (
|
||||
input: Buffer,
|
||||
expected: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Buffer> => {
|
||||
if (
|
||||
!Number.isSafeInteger(expected) ||
|
||||
expected < 0 ||
|
||||
expected > GPU_OBJECT_BYTES
|
||||
)
|
||||
throw new BinaryResourceFailure(
|
||||
"structure",
|
||||
GPU_OBJECT_BYTES,
|
||||
"LZ4 decoded length exceeds the object budget",
|
||||
);
|
||||
const output = Buffer.alloc(expected);
|
||||
let read = 0,
|
||||
write = 0,
|
||||
checkpoint = 0;
|
||||
const byte = (): number => {
|
||||
if (read >= input.length) return rejectGpu("Truncated LZ4 block");
|
||||
return input[read++]!;
|
||||
};
|
||||
const length = (nibble: number, remaining: number): number => {
|
||||
let total = nibble;
|
||||
if (nibble === 15) {
|
||||
let extra: number;
|
||||
do {
|
||||
extra = byte();
|
||||
if (extra > remaining - total)
|
||||
rejectGpu("LZ4 length exceeds its declared output");
|
||||
total += extra;
|
||||
} while (extra === 255);
|
||||
}
|
||||
if (total > remaining) rejectGpu("LZ4 length exceeds its declared output");
|
||||
return total;
|
||||
};
|
||||
while (read < input.length) {
|
||||
if (write - checkpoint >= 4 * 1024 * 1024) {
|
||||
await scheduler.yield();
|
||||
checkpoint = write;
|
||||
}
|
||||
assertArtifactReadActive(signal);
|
||||
const token = byte();
|
||||
const literals = length(token >>> 4, expected - write);
|
||||
if (literals > input.length - read)
|
||||
rejectGpu("LZ4 literals exceed their encoded input");
|
||||
input.copy(output, write, read, read + literals);
|
||||
read += literals;
|
||||
write += literals;
|
||||
if (read === input.length) break;
|
||||
const distance = byte() | (byte() << 8);
|
||||
if (distance === 0 || distance > write)
|
||||
rejectGpu("LZ4 match has an invalid backward distance");
|
||||
const match = length(token & 15, expected - write - 4) + 4;
|
||||
let copied = Math.min(distance, match);
|
||||
output.copy(output, write, write - distance, write - distance + copied);
|
||||
// Grow the already decoded pattern; Buffer.copy alone has memmove rather than LZ4 overlap semantics.
|
||||
while (copied < match) {
|
||||
const count = Math.min(copied, match - copied);
|
||||
output.copy(output, write + copied, write, write + count);
|
||||
copied += count;
|
||||
}
|
||||
write += match;
|
||||
}
|
||||
if (write !== expected)
|
||||
rejectGpu("LZ4 decoded length disagrees with its declaration");
|
||||
assertArtifactReadActive(signal);
|
||||
return output;
|
||||
};
|
||||
@@ -1,7 +1,7 @@
|
||||
import {
|
||||
BinaryFormatFailure,
|
||||
BinaryResourceFailure,
|
||||
} from "../native/metadata/BinaryContainer.js";
|
||||
} from "../domain/binaryContainer.js";
|
||||
import type { GpuKernel } from "../domain/gpu/gpuCodeObjects.js";
|
||||
import { GPU_REPORT_BYTES, rejectGpu } from "./GpuSource.js";
|
||||
interface Token {
|
||||
|
||||
Reference in new issue
Block a user