mirror of
https://github.com/morluto/rea.git
synced 2026-11-20 13:45:30 +00:00
fix(gguf): reserve directory framing before reading tensor dimensions (#12349)
Co-authored-by: ShuaixinHuang <197200901+ShuaixinHuang@users.noreply.github.com>
This commit is contained in:
1 parent
5a27508ae3
commit
48a5090bbb
2 files changed
+31
-1
No files matched your search
@@ -38,7 +38,9 @@ const NAMES = [
|
||||
const STRING_LENGTH_BYTES = 8;
|
||||
const WIDTHS = [1, 1, 2, 2, 4, 4, 4, 1, STRING_LENGTH_BYTES, 12, 8, 8, 8];
|
||||
const MIN_METADATA_ROW_BYTES = STRING_LENGTH_BYTES + 4 + 1;
|
||||
const MIN_TENSOR_DESCRIPTOR_BYTES = STRING_LENGTH_BYTES + 4 + 4 + 8;
|
||||
const TENSOR_STORAGE_FIELDS_BYTES = 4 + 8;
|
||||
const MIN_TENSOR_DESCRIPTOR_BYTES =
|
||||
STRING_LENGTH_BYTES + 4 + TENSOR_STORAGE_FIELDS_BYTES;
|
||||
|
||||
export interface GgufProgress {
|
||||
onHeader(info: {
|
||||
@@ -382,6 +384,9 @@ export class GgufHeader {
|
||||
const dimensions = Number(await this.#unsigned(4));
|
||||
if (dimensions > 4)
|
||||
rejectModel("GGUF tensor dimension count must be at most 4");
|
||||
this.reader.guarantee(
|
||||
dimensions * 8 + TENSOR_STORAGE_FIELDS_BYTES + remainingDirectoryBytes,
|
||||
);
|
||||
const shape: string[] = [];
|
||||
let parameters = 1n;
|
||||
for (let d = 0; d < dimensions; d++) {
|
||||
|
||||
@@ -1781,6 +1781,31 @@ it("rejects a tensor directory whose old 20-byte estimate fits but fixed fields
|
||||
});
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
"reserves later GGUF tensor framing before publishing a dimensioned row (big endian: %s)",
|
||||
async (bigEndian) => {
|
||||
const fixture = ggufFixture({
|
||||
bigEndian,
|
||||
metadata: [],
|
||||
tensors: [
|
||||
{ name: "first", type: 0, shape: [1n], offset: 0n, bytes: 4 },
|
||||
{ name: "next", type: 0, shape: [], offset: 32n, bytes: 4 },
|
||||
],
|
||||
});
|
||||
// Leave 23 bytes after the first descriptor, one short of the
|
||||
// second descriptor’s minimum framing even with an empty name.
|
||||
const { provider, path } = await providerAt(
|
||||
fixture.header.subarray(0, fixture.header.length - 5),
|
||||
);
|
||||
const result = await provider.inspect({ path, hash: "header" });
|
||||
if (result.ok) throw new Error("Truncated GGUF directory admitted");
|
||||
expect(result.error).toMatchObject(INVALID_FORMAT);
|
||||
expect(
|
||||
parseEvidence(result.error.partialObservation).normalized_result,
|
||||
).toMatchObject({ declared_tensor_count: 2, tensor_declarations: [] });
|
||||
},
|
||||
);
|
||||
|
||||
it("retains decoded counts when a physically fitting GGUF header exceeds the header budget", async () => {
|
||||
const count = 4_166_667;
|
||||
const header = Buffer.alloc(24);
|
||||
|
||||
Reference in new issue
Block a user