mirror of
https://github.com/morluto/rea.git
synced 2026-11-19 13:12:26 +00:00
refactor(model): read GGUF and safetensors fields via BinaryReader (#12318)
Route the 26 raw Buffer reads in GgufHeader and the 2 in SafetensorsHeader through BinaryReader. Add u8, i8, i16, i32, i64, f32 and f64 accessors to BinaryReader, with the same range checks as u16/u32/word, since GGUF metadata stores signed and floating scalars. Every migrated site reads an exact-size buffer from ModelHeaderReader.take(), so outputs and errors are unchanged. Shrink the raw-buffer-read baseline from 102 files / 711 sites to 100 / 683. Refs #11865
This commit is contained in:
1 parent
00e2346c2c
commit
5bdb7af43e
6 files changed
+160
-33
No files matched your search
@@ -68,8 +68,6 @@
|
||||
"src/il2cpp/Il2cppMetadataFormat.ts": 3,
|
||||
"src/il2cpp/Il2cppMetadataMembers.ts": 5,
|
||||
"src/il2cpp/Il2cppMetadataV108.ts": 3,
|
||||
"src/model/GgufHeader.ts": 26,
|
||||
"src/model/SafetensorsHeader.ts": 2,
|
||||
"src/native/AppleDispatchMetadata.ts": 4,
|
||||
"src/native/AppleObjcClassImportBindings.ts": 1,
|
||||
"src/native/AppleObjcDispatchFacets.ts": 1,
|
||||
|
||||
@@ -282,6 +282,16 @@ export class BinaryReader {
|
||||
return this.bytes.subarray(offset, offset + size);
|
||||
}
|
||||
|
||||
u8(offset: number): number {
|
||||
this.#checkRange(offset, 1, "8-bit field");
|
||||
return this.bytes.readUInt8(offset);
|
||||
}
|
||||
|
||||
i8(offset: number): number {
|
||||
this.#checkRange(offset, 1, "8-bit field");
|
||||
return this.bytes.readInt8(offset);
|
||||
}
|
||||
|
||||
u16(offset: number): number {
|
||||
this.#checkRange(offset, 2, "16-bit field");
|
||||
return this.little
|
||||
@@ -289,6 +299,13 @@ export class BinaryReader {
|
||||
: this.bytes.readUInt16BE(offset);
|
||||
}
|
||||
|
||||
i16(offset: number): number {
|
||||
this.#checkRange(offset, 2, "16-bit field");
|
||||
return this.little
|
||||
? this.bytes.readInt16LE(offset)
|
||||
: this.bytes.readInt16BE(offset);
|
||||
}
|
||||
|
||||
u32(offset: number): number {
|
||||
this.#checkRange(offset, 4, "32-bit field");
|
||||
return this.little
|
||||
@@ -296,6 +313,34 @@ export class BinaryReader {
|
||||
: this.bytes.readUInt32BE(offset);
|
||||
}
|
||||
|
||||
i32(offset: number): number {
|
||||
this.#checkRange(offset, 4, "32-bit field");
|
||||
return this.little
|
||||
? this.bytes.readInt32LE(offset)
|
||||
: this.bytes.readInt32BE(offset);
|
||||
}
|
||||
|
||||
i64(offset: number): bigint {
|
||||
this.#checkRange(offset, 8, "64-bit field");
|
||||
return this.little
|
||||
? this.bytes.readBigInt64LE(offset)
|
||||
: this.bytes.readBigInt64BE(offset);
|
||||
}
|
||||
|
||||
f32(offset: number): number {
|
||||
this.#checkRange(offset, 4, "32-bit field");
|
||||
return this.little
|
||||
? this.bytes.readFloatLE(offset)
|
||||
: this.bytes.readFloatBE(offset);
|
||||
}
|
||||
|
||||
f64(offset: number): number {
|
||||
this.#checkRange(offset, 8, "64-bit field");
|
||||
return this.little
|
||||
? this.bytes.readDoubleLE(offset)
|
||||
: this.bytes.readDoubleBE(offset);
|
||||
}
|
||||
|
||||
word(offset: number, bits: 32 | 64): bigint {
|
||||
if (bits === 32) return BigInt(this.u32(offset));
|
||||
this.#checkRange(offset, 8, "64-bit field");
|
||||
|
||||
+30
-30
@@ -1,3 +1,4 @@
|
||||
import { BinaryReader } from "../domain/binaryContainer.js";
|
||||
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
|
||||
import { isUtf8 } from "node:buffer";
|
||||
import { setImmediate } from "node:timers/promises";
|
||||
@@ -57,15 +58,17 @@ export class GgufHeader {
|
||||
readonly #budget = new ModelReportBudget();
|
||||
constructor(readonly reader: ModelHeaderReader) {}
|
||||
|
||||
// take() returns exactly `width` bytes, so field reads at 0 stay in range.
|
||||
async #field(width: number): Promise<BinaryReader> {
|
||||
return new BinaryReader(
|
||||
await this.reader.take(width),
|
||||
this.#little,
|
||||
"gguf",
|
||||
);
|
||||
}
|
||||
|
||||
async #unsigned(width: 4 | 8): Promise<bigint> {
|
||||
const b = await this.reader.take(width);
|
||||
const value =
|
||||
width === 4
|
||||
? BigInt(this.#little ? b.readUInt32LE() : b.readUInt32BE())
|
||||
: this.#little
|
||||
? b.readBigUInt64LE()
|
||||
: b.readBigUInt64BE();
|
||||
return value;
|
||||
return (await this.#field(width)).word(0, width === 4 ? 32 : 64);
|
||||
}
|
||||
|
||||
async #text(retain: boolean, maximum?: number): Promise<string> {
|
||||
@@ -183,47 +186,45 @@ export class GgufHeader {
|
||||
complete,
|
||||
};
|
||||
}
|
||||
const b = await this.reader.take(WIDTHS[type] ?? 0);
|
||||
const b = await this.#field(WIDTHS[type] ?? 0);
|
||||
let value: JsonValue;
|
||||
switch (type) {
|
||||
case 0:
|
||||
value = b.readUInt8();
|
||||
value = b.u8(0);
|
||||
break;
|
||||
case 1:
|
||||
value = b.readInt8();
|
||||
value = b.i8(0);
|
||||
break;
|
||||
case 2:
|
||||
value = this.#little ? b.readUInt16LE() : b.readUInt16BE();
|
||||
value = b.u16(0);
|
||||
break;
|
||||
case 3:
|
||||
value = this.#little ? b.readInt16LE() : b.readInt16BE();
|
||||
value = b.i16(0);
|
||||
break;
|
||||
case 4:
|
||||
value = this.#little ? b.readUInt32LE() : b.readUInt32BE();
|
||||
value = b.u32(0);
|
||||
break;
|
||||
case 5:
|
||||
value = this.#little ? b.readInt32LE() : b.readInt32BE();
|
||||
value = b.i32(0);
|
||||
break;
|
||||
case 6:
|
||||
value = this.#little ? b.readFloatLE() : b.readFloatBE();
|
||||
value = b.f32(0);
|
||||
break;
|
||||
case 7:
|
||||
if (b[0] !== 0 && b[0] !== 1)
|
||||
case 7: {
|
||||
const byte = b.u8(0);
|
||||
if (byte !== 0 && byte !== 1)
|
||||
rejectModel("GGUF boolean must be 0 or 1");
|
||||
value = b[0] === 1;
|
||||
value = byte === 1;
|
||||
break;
|
||||
}
|
||||
case 10:
|
||||
value = (
|
||||
this.#little ? b.readBigUInt64LE() : b.readBigUInt64BE()
|
||||
).toString();
|
||||
value = b.word(0, 64).toString();
|
||||
break;
|
||||
case 11:
|
||||
value = (
|
||||
this.#little ? b.readBigInt64LE() : b.readBigInt64BE()
|
||||
).toString();
|
||||
value = b.i64(0).toString();
|
||||
break;
|
||||
case 12:
|
||||
value = this.#little ? b.readDoubleLE() : b.readDoubleBE();
|
||||
value = b.f64(0);
|
||||
break;
|
||||
default:
|
||||
return rejectModel("Unsupported GGUF scalar");
|
||||
@@ -253,8 +254,8 @@ export class GgufHeader {
|
||||
limitations: string[];
|
||||
}> {
|
||||
const versionBytes = await this.reader.take(4);
|
||||
const littleVersion = versionBytes.readUInt32LE();
|
||||
const bigVersion = versionBytes.readUInt32BE();
|
||||
const littleVersion = new BinaryReader(versionBytes, true, "gguf").u32(0);
|
||||
const bigVersion = new BinaryReader(versionBytes, false, "gguf").u32(0);
|
||||
let version: number;
|
||||
if (littleVersion === 2 || littleVersion === 3) {
|
||||
this.#little = true;
|
||||
@@ -355,8 +356,7 @@ export class GgufHeader {
|
||||
const shape: string[] = [];
|
||||
let parameters = 1n;
|
||||
for (let d = 0; d < dimensions; d++) {
|
||||
const b = await this.reader.take(8);
|
||||
const value = this.#little ? b.readBigUInt64LE() : b.readBigUInt64BE();
|
||||
const value = await this.#unsigned(8);
|
||||
shape.push(value.toString());
|
||||
parameters *= value;
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { BinaryReader } from "../domain/binaryContainer.js";
|
||||
import { decodeStrictUtf8 } from "../domain/strictUtf8.js";
|
||||
import { parseJsonStringLiteral } from "../domain/jsonDuplicateMember.js";
|
||||
import { escapeJsonPointerToken } from "../domain/jsonPointer.js";
|
||||
@@ -218,8 +219,10 @@ export const readSafetensorsHeader = async (
|
||||
if ((pickle || zip) && reader.size < 9) rejectPickle();
|
||||
const second = await reader.take(4);
|
||||
reader.excludeSafetensorsLengthPrefix();
|
||||
// first and second are exact four-byte takes of the little-endian u64 prefix.
|
||||
const length =
|
||||
BigInt(first.readUInt32LE()) + (BigInt(second.readUInt32LE()) << 32n);
|
||||
new BinaryReader(first, true, "safetensors").word(0, 32) +
|
||||
(new BinaryReader(second, true, "safetensors").word(0, 32) << 32n);
|
||||
const brace = length === 0n ? undefined : (await reader.take(1))[0];
|
||||
// A brace after an admissible length is safetensors, even if truncated.
|
||||
if (
|
||||
|
||||
@@ -36,6 +36,28 @@ it("preserves representable offsets for invalid file ranges", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("decodes signed and floating fields in both byte orders", () => {
|
||||
const bytes = Buffer.alloc(18);
|
||||
bytes.writeInt16LE(-2, 0);
|
||||
bytes.writeInt32LE(-3, 2);
|
||||
bytes.writeFloatLE(1.5, 6);
|
||||
bytes.writeBigInt64LE(-4n, 10);
|
||||
const little = new BinaryReader(bytes, true, "zip");
|
||||
expect([little.u8(0), little.i8(0), little.i16(0), little.i32(2)]).toEqual([
|
||||
0xfe, -2, -2, -3,
|
||||
]);
|
||||
expect([little.f32(6), little.i64(10)]).toEqual([1.5, -4n]);
|
||||
bytes.writeDoubleBE(-0.25, 0);
|
||||
const big = new BinaryReader(bytes, false, "zip");
|
||||
expect([big.f64(0), big.i16(0), big.i32(0)]).toEqual([
|
||||
-0.25,
|
||||
bytes.readInt16BE(0),
|
||||
bytes.readInt32BE(0),
|
||||
]);
|
||||
expect(() => little.i64(11)).toThrow(BinaryFormatFailure);
|
||||
expect(() => big.f32(15)).toThrow(BinaryFormatFailure);
|
||||
});
|
||||
|
||||
it("decodes numeric fields at nonzero buffer offsets and preserves malformed and truncated range failures", () => {
|
||||
const backing = Buffer.from([
|
||||
0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0xfe,
|
||||
|
||||
@@ -759,6 +759,65 @@ it("keeps unknown safetensors storage validation separate from observed valid by
|
||||
});
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
"decodes every fixed-width GGUF scalar type, big-endian=%s",
|
||||
async (bigEndian) => {
|
||||
const scalar = (width: number, write: (b: Buffer) => void) => {
|
||||
const b = Buffer.alloc(width);
|
||||
write(b);
|
||||
return b;
|
||||
};
|
||||
const order = bigEndian ? "BE" : "LE";
|
||||
const rows = [
|
||||
[0, "UINT8", scalar(1, (b) => b.writeUInt8(0xfe)), 0xfe],
|
||||
[1, "INT8", scalar(1, (b) => b.writeInt8(-2)), -2],
|
||||
[2, "UINT16", scalar(2, (b) => b[`writeUInt16${order}`](0xfffe)), 0xfffe],
|
||||
[3, "INT16", scalar(2, (b) => b[`writeInt16${order}`](-300)), -300],
|
||||
[
|
||||
4,
|
||||
"UINT32",
|
||||
scalar(4, (b) => b[`writeUInt32${order}`](0xfffffffe)),
|
||||
0xfffffffe,
|
||||
],
|
||||
[5, "INT32", scalar(4, (b) => b[`writeInt32${order}`](-70000)), -70000],
|
||||
[6, "FLOAT32", scalar(4, (b) => b[`writeFloat${order}`](-1.5)), -1.5],
|
||||
[7, "BOOL", scalar(1, (b) => b.writeUInt8(1)), true],
|
||||
[
|
||||
10,
|
||||
"UINT64",
|
||||
scalar(8, (b) => b[`writeBigUInt64${order}`](2n ** 63n)),
|
||||
"9223372036854775808",
|
||||
],
|
||||
[
|
||||
11,
|
||||
"INT64",
|
||||
scalar(8, (b) => b[`writeBigInt64${order}`](-(2n ** 62n))),
|
||||
"-4611686018427387904",
|
||||
],
|
||||
[
|
||||
12,
|
||||
"FLOAT64",
|
||||
scalar(8, (b) => b[`writeDouble${order}`](-0.125)),
|
||||
-0.125,
|
||||
],
|
||||
] as const;
|
||||
const fixture = ggufFixture({
|
||||
bigEndian,
|
||||
metadata: rows.map(([type, name, value]) => ({
|
||||
key: `custom.${name.toLowerCase()}`,
|
||||
type,
|
||||
value,
|
||||
})),
|
||||
});
|
||||
const { path, provider } = await providerAt(fixture.bytes);
|
||||
const result = await provider.inspect({ path, hash: "header" });
|
||||
if (!result.ok) throw result.error;
|
||||
expect(result.value.metadata.map((row) => [row.type, row.value])).toEqual(
|
||||
rows.map(([, name, , expected]) => [name, expected]),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves nonfinite floating metadata and negative zero without JSON coercion", async () => {
|
||||
const values = [NaN, Infinity, -Infinity, -0];
|
||||
const fixture = ggufFixture({
|
||||
|
||||
Reference in new issue
Block a user