From 0d44afed7e3f19855a756ae7860ed1b3b00730ea Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:43:34 +0800 Subject: [PATCH] fix(model): retain exact safetensors shape dimensions Keep unsigned integer declarations exact until their owning field is interpreted. Bound actual offsets at conversion and validate shape products in declaration order, preserving large empty tensor shapes without hiding an earlier overflow. Fixes #6627 --- docs/model-artifacts.md | 8 +- src/model/SafetensorsHeader.ts | 54 ++++++++------ tests/acceptance/modelArtifactCli.test.ts | 86 ++++++++++++++++++++++ tests/boundary/model/modelArtifact.test.ts | 24 ++++++ 4 files changed, 148 insertions(+), 24 deletions(-) diff --git a/docs/model-artifacts.md b/docs/model-artifacts.md index 5c5269b0f2..63adc66c07 100644 --- a/docs/model-artifacts.md +++ b/docs/model-artifacts.md @@ -65,7 +65,13 @@ contradictions are rejected with a typed `file_format_rejected` constraint (publ A missing, non-regular or otherwise unusable selected path is also `target_unavailable`; host I/O or resource failures on a valid selection, such as `EMFILE` or `EIO`, are `execution_failure` with the path and reason in `details.diagnostics`. Empty -tensors and scalar shapes are supported; packed sub-byte types require whole +tensors and scalar shapes are supported. The reader retains shape dimensions as +exact unsigned 64-bit values, including values above JavaScript's safe-integer +range when a zero extent makes storage empty. Shape products are checked in +declaration order, as in the +[64-bit upstream reader](https://github.com/huggingface/safetensors/blob/e246a2560645b7525f5775669ed816eb57c5bcc8/safetensors/src/tensor.rs#L604-L609); +a later zero does not erase an earlier product overflow. Actual file and data +offsets must still fit exact host offset precision. Packed sub-byte types require whole byte storage. Unknown dtypes retain declared ranges and an explicit limitation that shape-to-storage validation is unavailable. Each tensor records `storage_validation: "consistent" | "unknown"`; `coverage.storage_sizes` remains diff --git a/src/model/SafetensorsHeader.ts b/src/model/SafetensorsHeader.ts index 9fc24b2113..04adfedd17 100644 --- a/src/model/SafetensorsHeader.ts +++ b/src/model/SafetensorsHeader.ts @@ -17,7 +17,7 @@ import { safetensorsBits } from "./ModelTensorTypes.js"; type HeaderValue = | string - | number + | bigint | HeaderValue[] | { [key: string]: HeaderValue }; @@ -100,20 +100,20 @@ class HeaderJson { } } const start = this.#position; - while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) this.#position++; + while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) { + this.#position++; + if (this.#position - start > 20) + rejectModel("Safetensors unsigned integer exceeds uint64"); + } const raw = this.text.slice(start, this.#position); if (!/^(?:0|[1-9][0-9]*)$/u.test(raw)) return rejectModel( "Safetensors header requires unsigned integer declarations", ); - const number = Number(raw); - if (!Number.isSafeInteger(number)) - throw new ModelFormatFailure( - "capacity", - "Safetensors integer exceeds exact host offset precision", - "exact-offset", - ); - return number; + const integer = BigInt(raw); + if (integer > 0xffffffffffffffffn) + rejectModel("Safetensors unsigned integer exceeds uint64"); + return integer; } *read(): Generator { if (this.text[0] !== "{") @@ -133,14 +133,23 @@ const objectOf = ( return rejectModel("Expected safetensors header object"); return value; }; -const integersOf = (value: HeaderValue | undefined): number[] => { +const integersOf = (value: HeaderValue | undefined): bigint[] => { if ( !Array.isArray(value) || - !value.every((item): item is number => typeof item === "number") + !value.every((item): item is bigint => typeof item === "bigint") ) return rejectModel("Expected safetensors integer array"); return value; }; +const exactOffset = (value: bigint): number => { + if (value > BigInt(Number.MAX_SAFE_INTEGER)) + throw new ModelFormatFailure( + "capacity", + "Safetensors data offset exceeds exact host offset precision", + "exact-offset", + ); + return Number(value); +}; const pointer = (value: string): string => value.replaceAll("~", "~0").replaceAll("/", "~1"); @@ -243,21 +252,20 @@ export const readSafetensorsHeader = async ( const offsets = integersOf(entry.data_offsets); if (offsets.length !== 2) rejectModel(`Safetensors tensor ${name} requires two data offsets`); - const start = offsets[0] ?? 0; - const end = offsets[1] ?? 0; + const start = exactOffset(offsets[0] ?? 0n); + const end = exactOffset(offsets[1] ?? 0n); if (end < start || end > reader.size - tensor_data_offset) rejectModel( `Safetensors tensor ${name} range [${String(start)},${String(end)}) is outside the data buffer`, ); - let parameters = shape.includes(0) ? 0n : 1n; - if (parameters !== 0n) - for (const dimension of shape) { - parameters *= BigInt(dimension); - if (parameters > 0xffffffffffffffffn) - rejectModel( - `Safetensors tensor ${name} parameter count overflows uint64`, - ); - } + let parameters = 1n; + for (const dimension of shape) { + parameters *= dimension; + if (parameters > 0xffffffffffffffffn) + rejectModel( + `Safetensors tensor ${name} parameter count overflows uint64`, + ); + } const bits = safetensorsBits(type); if (bits !== undefined) { const expected = parameters * BigInt(bits); diff --git a/tests/acceptance/modelArtifactCli.test.ts b/tests/acceptance/modelArtifactCli.test.ts index 0b807b73f7..f5b5d9d668 100644 --- a/tests/acceptance/modelArtifactCli.test.ts +++ b/tests/acceptance/modelArtifactCli.test.ts @@ -19,6 +19,92 @@ import { ggufFixture, safetensorsFixture } from "../fixtures/model/model.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { cliTest } from "../support/cli/cliFixture.js"; +cliTest( + "preserves exact large safetensors dimensions while bounding actual offsets", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-model-shape-precision-"); + const path = join(root, "empty.safetensors"); + const fixture = safetensorsFixture( + '{"a":{"dtype":"F32","shape":[0,9007199254740992],"data_offsets":[0,0]},"b":{"dtype":"F32","shape":[18446744073709551615,0],"data_offsets":[0,0]}}', + Buffer.alloc(0), + ); + await writeFile(path, fixture.bytes); + const result = await cli.run({ + arguments: [ + "inspect-model-artifact", + path, + "--detail", + "complete", + "--hash", + "full", + "--json", + ], + }); + expect(result.exitCode, result.stderr).toBe(0); + const report = modelArtifactSchema.parse( + toolContract("inspect_model_artifact").outputSchema.parse(result.json) + .normalized_result, + ); + expect( + report.tensors.map( + ({ shape, parameter_count, byte_length, storage_validation }) => ({ + shape, + parameter_count, + byte_length, + storage_validation, + }), + ), + ).toEqual([ + { + shape: ["0", "9007199254740992"], + parameter_count: "0", + byte_length: 0, + storage_validation: "consistent", + }, + { + shape: ["18446744073709551615", "0"], + parameter_count: "0", + byte_length: 0, + storage_validation: "consistent", + }, + ]); + expect(report.totals).toMatchObject({ + parameter_count: "0", + tensor_bytes: 0, + }); + expect(report.artifact.sha256).toBe( + createHash("sha256").update(fixture.bytes).digest("hex"), + ); + const { call } = await connectLocalToolsMcp(); + const response = await call("inspect_model_artifact", { + path, + detail: "complete", + hash: "full", + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(parseEvidence(response.structuredContent).normalized_result).toEqual( + report, + ); + + await writeFile( + path, + safetensorsFixture( + '{"x":{"dtype":"F32","shape":[0],"data_offsets":[9007199254740992,9007199254740992]}}', + Buffer.alloc(0), + ).bytes, + ); + const offsetFailure = parseMcpToolError( + await call("inspect_model_artifact", { path }), + ); + expect(offsetFailure.error).toMatchObject({ + code: "resource_constraint", + details: { + reported_limits: { maximum_exact_offset: Number.MAX_SAFE_INTEGER }, + }, + }); + }, +); + it.each(["named", "default", "inventory-only", "absent"] as const)( "reports actual chat-template presence in summaries and page context (%s)", async (kind) => { diff --git a/tests/boundary/model/modelArtifact.test.ts b/tests/boundary/model/modelArtifact.test.ts index 9fc4ae1574..35cb78959b 100644 --- a/tests/boundary/model/modelArtifact.test.ts +++ b/tests/boundary/model/modelArtifact.test.ts @@ -692,6 +692,30 @@ const malformed = [ Buffer.alloc(4), ).bytes, ], + [ + "safetensors dimension exceeding uint64", + () => + safetensorsFixture( + '{"x":{"dtype":"F32","shape":[0,18446744073709551616],"data_offsets":[0,0]}}', + Buffer.alloc(0), + ).bytes, + ], + [ + "overlong safetensors integer", + () => + safetensorsFixture( + '{"x":{"dtype":"F32","shape":[0,100000000000000000000],"data_offsets":[0,0]}}', + Buffer.alloc(0), + ).bytes, + ], + [ + "safetensors shape product overflowing before a zero extent", + () => + safetensorsFixture( + '{"x":{"dtype":"F32","shape":[18446744073709551615,18446744073709551615,0],"data_offsets":[0,0]}}', + Buffer.alloc(0), + ).bytes, + ], [ "safetensors shape mismatch", () =>