diff --git a/scripts/verify/baselines/raw-buffer-read.json b/scripts/verify/baselines/raw-buffer-read.json index 9483874f89..8b0970af28 100644 --- a/scripts/verify/baselines/raw-buffer-read.json +++ b/scripts/verify/baselines/raw-buffer-read.json @@ -68,8 +68,6 @@ "src/il2cpp/Il2cppMetadataFormat.ts": 3, "src/il2cpp/Il2cppMetadataMembers.ts": 5, "src/il2cpp/Il2cppMetadataV108.ts": 3, - "src/model/GgufHeader.ts": 26, - "src/model/SafetensorsHeader.ts": 2, "src/native/AppleDispatchMetadata.ts": 4, "src/native/AppleObjcClassImportBindings.ts": 1, "src/native/AppleObjcDispatchFacets.ts": 1, diff --git a/src/domain/binaryContainer.ts b/src/domain/binaryContainer.ts index a08726cd93..9290697ea0 100644 --- a/src/domain/binaryContainer.ts +++ b/src/domain/binaryContainer.ts @@ -282,6 +282,16 @@ export class BinaryReader { return this.bytes.subarray(offset, offset + size); } + u8(offset: number): number { + this.#checkRange(offset, 1, "8-bit field"); + return this.bytes.readUInt8(offset); + } + + i8(offset: number): number { + this.#checkRange(offset, 1, "8-bit field"); + return this.bytes.readInt8(offset); + } + u16(offset: number): number { this.#checkRange(offset, 2, "16-bit field"); return this.little @@ -289,6 +299,13 @@ export class BinaryReader { : this.bytes.readUInt16BE(offset); } + i16(offset: number): number { + this.#checkRange(offset, 2, "16-bit field"); + return this.little + ? this.bytes.readInt16LE(offset) + : this.bytes.readInt16BE(offset); + } + u32(offset: number): number { this.#checkRange(offset, 4, "32-bit field"); return this.little @@ -296,6 +313,34 @@ export class BinaryReader { : this.bytes.readUInt32BE(offset); } + i32(offset: number): number { + this.#checkRange(offset, 4, "32-bit field"); + return this.little + ? this.bytes.readInt32LE(offset) + : this.bytes.readInt32BE(offset); + } + + i64(offset: number): bigint { + this.#checkRange(offset, 8, "64-bit field"); + return this.little + ? this.bytes.readBigInt64LE(offset) + : this.bytes.readBigInt64BE(offset); + } + + f32(offset: number): number { + this.#checkRange(offset, 4, "32-bit field"); + return this.little + ? this.bytes.readFloatLE(offset) + : this.bytes.readFloatBE(offset); + } + + f64(offset: number): number { + this.#checkRange(offset, 8, "64-bit field"); + return this.little + ? this.bytes.readDoubleLE(offset) + : this.bytes.readDoubleBE(offset); + } + word(offset: number, bits: 32 | 64): bigint { if (bits === 32) return BigInt(this.u32(offset)); this.#checkRange(offset, 8, "64-bit field"); diff --git a/src/model/GgufHeader.ts b/src/model/GgufHeader.ts index 9e6078827e..dca81af9b6 100644 --- a/src/model/GgufHeader.ts +++ b/src/model/GgufHeader.ts @@ -1,3 +1,4 @@ +import { BinaryReader } from "../domain/binaryContainer.js"; import { strictUtf8Decoder } from "../domain/strictUtf8.js"; import { isUtf8 } from "node:buffer"; import { setImmediate } from "node:timers/promises"; @@ -57,15 +58,17 @@ export class GgufHeader { readonly #budget = new ModelReportBudget(); constructor(readonly reader: ModelHeaderReader) {} + // take() returns exactly `width` bytes, so field reads at 0 stay in range. + async #field(width: number): Promise { + return new BinaryReader( + await this.reader.take(width), + this.#little, + "gguf", + ); + } + async #unsigned(width: 4 | 8): Promise { - const b = await this.reader.take(width); - const value = - width === 4 - ? BigInt(this.#little ? b.readUInt32LE() : b.readUInt32BE()) - : this.#little - ? b.readBigUInt64LE() - : b.readBigUInt64BE(); - return value; + return (await this.#field(width)).word(0, width === 4 ? 32 : 64); } async #text(retain: boolean, maximum?: number): Promise { @@ -183,47 +186,45 @@ export class GgufHeader { complete, }; } - const b = await this.reader.take(WIDTHS[type] ?? 0); + const b = await this.#field(WIDTHS[type] ?? 0); let value: JsonValue; switch (type) { case 0: - value = b.readUInt8(); + value = b.u8(0); break; case 1: - value = b.readInt8(); + value = b.i8(0); break; case 2: - value = this.#little ? b.readUInt16LE() : b.readUInt16BE(); + value = b.u16(0); break; case 3: - value = this.#little ? b.readInt16LE() : b.readInt16BE(); + value = b.i16(0); break; case 4: - value = this.#little ? b.readUInt32LE() : b.readUInt32BE(); + value = b.u32(0); break; case 5: - value = this.#little ? b.readInt32LE() : b.readInt32BE(); + value = b.i32(0); break; case 6: - value = this.#little ? b.readFloatLE() : b.readFloatBE(); + value = b.f32(0); break; - case 7: - if (b[0] !== 0 && b[0] !== 1) + case 7: { + const byte = b.u8(0); + if (byte !== 0 && byte !== 1) rejectModel("GGUF boolean must be 0 or 1"); - value = b[0] === 1; + value = byte === 1; break; + } case 10: - value = ( - this.#little ? b.readBigUInt64LE() : b.readBigUInt64BE() - ).toString(); + value = b.word(0, 64).toString(); break; case 11: - value = ( - this.#little ? b.readBigInt64LE() : b.readBigInt64BE() - ).toString(); + value = b.i64(0).toString(); break; case 12: - value = this.#little ? b.readDoubleLE() : b.readDoubleBE(); + value = b.f64(0); break; default: return rejectModel("Unsupported GGUF scalar"); @@ -253,8 +254,8 @@ export class GgufHeader { limitations: string[]; }> { const versionBytes = await this.reader.take(4); - const littleVersion = versionBytes.readUInt32LE(); - const bigVersion = versionBytes.readUInt32BE(); + const littleVersion = new BinaryReader(versionBytes, true, "gguf").u32(0); + const bigVersion = new BinaryReader(versionBytes, false, "gguf").u32(0); let version: number; if (littleVersion === 2 || littleVersion === 3) { this.#little = true; @@ -355,8 +356,7 @@ export class GgufHeader { const shape: string[] = []; let parameters = 1n; for (let d = 0; d < dimensions; d++) { - const b = await this.reader.take(8); - const value = this.#little ? b.readBigUInt64LE() : b.readBigUInt64BE(); + const value = await this.#unsigned(8); shape.push(value.toString()); parameters *= value; } diff --git a/src/model/SafetensorsHeader.ts b/src/model/SafetensorsHeader.ts index deb7a2ea0a..7446334155 100644 --- a/src/model/SafetensorsHeader.ts +++ b/src/model/SafetensorsHeader.ts @@ -1,3 +1,4 @@ +import { BinaryReader } from "../domain/binaryContainer.js"; import { decodeStrictUtf8 } from "../domain/strictUtf8.js"; import { parseJsonStringLiteral } from "../domain/jsonDuplicateMember.js"; import { escapeJsonPointerToken } from "../domain/jsonPointer.js"; @@ -218,8 +219,10 @@ export const readSafetensorsHeader = async ( if ((pickle || zip) && reader.size < 9) rejectPickle(); const second = await reader.take(4); reader.excludeSafetensorsLengthPrefix(); + // first and second are exact four-byte takes of the little-endian u64 prefix. const length = - BigInt(first.readUInt32LE()) + (BigInt(second.readUInt32LE()) << 32n); + new BinaryReader(first, true, "safetensors").word(0, 32) + + (new BinaryReader(second, true, "safetensors").word(0, 32) << 32n); const brace = length === 0n ? undefined : (await reader.take(1))[0]; // A brace after an admissible length is safetensors, even if truncated. if ( diff --git a/src/native/metadata/BinaryContainer.test.ts b/src/native/metadata/BinaryContainer.test.ts index 724ba03534..bca3849232 100644 --- a/src/native/metadata/BinaryContainer.test.ts +++ b/src/native/metadata/BinaryContainer.test.ts @@ -36,6 +36,28 @@ it("preserves representable offsets for invalid file ranges", () => { ); }); +it("decodes signed and floating fields in both byte orders", () => { + const bytes = Buffer.alloc(18); + bytes.writeInt16LE(-2, 0); + bytes.writeInt32LE(-3, 2); + bytes.writeFloatLE(1.5, 6); + bytes.writeBigInt64LE(-4n, 10); + const little = new BinaryReader(bytes, true, "zip"); + expect([little.u8(0), little.i8(0), little.i16(0), little.i32(2)]).toEqual([ + 0xfe, -2, -2, -3, + ]); + expect([little.f32(6), little.i64(10)]).toEqual([1.5, -4n]); + bytes.writeDoubleBE(-0.25, 0); + const big = new BinaryReader(bytes, false, "zip"); + expect([big.f64(0), big.i16(0), big.i32(0)]).toEqual([ + -0.25, + bytes.readInt16BE(0), + bytes.readInt32BE(0), + ]); + expect(() => little.i64(11)).toThrow(BinaryFormatFailure); + expect(() => big.f32(15)).toThrow(BinaryFormatFailure); +}); + it("decodes numeric fields at nonzero buffer offsets and preserves malformed and truncated range failures", () => { const backing = Buffer.from([ 0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0xfe, diff --git a/tests/boundary/model/modelArtifact.test.ts b/tests/boundary/model/modelArtifact.test.ts index d3972232b6..1834d5352a 100644 --- a/tests/boundary/model/modelArtifact.test.ts +++ b/tests/boundary/model/modelArtifact.test.ts @@ -759,6 +759,65 @@ it("keeps unknown safetensors storage validation separate from observed valid by }); }); +it.each([false, true])( + "decodes every fixed-width GGUF scalar type, big-endian=%s", + async (bigEndian) => { + const scalar = (width: number, write: (b: Buffer) => void) => { + const b = Buffer.alloc(width); + write(b); + return b; + }; + const order = bigEndian ? "BE" : "LE"; + const rows = [ + [0, "UINT8", scalar(1, (b) => b.writeUInt8(0xfe)), 0xfe], + [1, "INT8", scalar(1, (b) => b.writeInt8(-2)), -2], + [2, "UINT16", scalar(2, (b) => b[`writeUInt16${order}`](0xfffe)), 0xfffe], + [3, "INT16", scalar(2, (b) => b[`writeInt16${order}`](-300)), -300], + [ + 4, + "UINT32", + scalar(4, (b) => b[`writeUInt32${order}`](0xfffffffe)), + 0xfffffffe, + ], + [5, "INT32", scalar(4, (b) => b[`writeInt32${order}`](-70000)), -70000], + [6, "FLOAT32", scalar(4, (b) => b[`writeFloat${order}`](-1.5)), -1.5], + [7, "BOOL", scalar(1, (b) => b.writeUInt8(1)), true], + [ + 10, + "UINT64", + scalar(8, (b) => b[`writeBigUInt64${order}`](2n ** 63n)), + "9223372036854775808", + ], + [ + 11, + "INT64", + scalar(8, (b) => b[`writeBigInt64${order}`](-(2n ** 62n))), + "-4611686018427387904", + ], + [ + 12, + "FLOAT64", + scalar(8, (b) => b[`writeDouble${order}`](-0.125)), + -0.125, + ], + ] as const; + const fixture = ggufFixture({ + bigEndian, + metadata: rows.map(([type, name, value]) => ({ + key: `custom.${name.toLowerCase()}`, + type, + value, + })), + }); + const { path, provider } = await providerAt(fixture.bytes); + const result = await provider.inspect({ path, hash: "header" }); + if (!result.ok) throw result.error; + expect(result.value.metadata.map((row) => [row.type, row.value])).toEqual( + rows.map(([, name, , expected]) => [name, expected]), + ); + }, +); + it("preserves nonfinite floating metadata and negative zero without JSON coercion", async () => { const values = [NaN, Infinity, -Infinity, -0]; const fixture = ggufFixture({