diff --git a/docs/model-artifacts.md b/docs/model-artifacts.md index 5c5269b0f2..63adc66c07 100644 --- a/docs/model-artifacts.md +++ b/docs/model-artifacts.md @@ -65,7 +65,13 @@ contradictions are rejected with a typed `file_format_rejected` constraint (publ A missing, non-regular or otherwise unusable selected path is also `target_unavailable`; host I/O or resource failures on a valid selection, such as `EMFILE` or `EIO`, are `execution_failure` with the path and reason in `details.diagnostics`. Empty -tensors and scalar shapes are supported; packed sub-byte types require whole +tensors and scalar shapes are supported. The reader retains shape dimensions as +exact unsigned 64-bit values, including values above JavaScript's safe-integer +range when a zero extent makes storage empty. Shape products are checked in +declaration order, as in the +[64-bit upstream reader](https://github.com/huggingface/safetensors/blob/e246a2560645b7525f5775669ed816eb57c5bcc8/safetensors/src/tensor.rs#L604-L609); +a later zero does not erase an earlier product overflow. Actual file and data +offsets must still fit exact host offset precision. Packed sub-byte types require whole byte storage. Unknown dtypes retain declared ranges and an explicit limitation that shape-to-storage validation is unavailable. Each tensor records `storage_validation: "consistent" | "unknown"`; `coverage.storage_sizes` remains diff --git a/src/model/SafetensorsHeader.ts b/src/model/SafetensorsHeader.ts index 9fc24b2113..04adfedd17 100644 --- a/src/model/SafetensorsHeader.ts +++ b/src/model/SafetensorsHeader.ts @@ -17,7 +17,7 @@ import { safetensorsBits } from "./ModelTensorTypes.js"; type HeaderValue = | string - | number + | bigint | HeaderValue[] | { [key: string]: HeaderValue }; @@ -100,20 +100,20 @@ class HeaderJson { } } const start = this.#position; - while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) this.#position++; + while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) { + this.#position++; + if (this.#position - start > 20) + rejectModel("Safetensors unsigned integer exceeds uint64"); + } const raw = this.text.slice(start, this.#position); if (!/^(?:0|[1-9][0-9]*)$/u.test(raw)) return rejectModel( "Safetensors header requires unsigned integer declarations", ); - const number = Number(raw); - if (!Number.isSafeInteger(number)) - throw new ModelFormatFailure( - "capacity", - "Safetensors integer exceeds exact host offset precision", - "exact-offset", - ); - return number; + const integer = BigInt(raw); + if (integer > 0xffffffffffffffffn) + rejectModel("Safetensors unsigned integer exceeds uint64"); + return integer; } *read(): Generator { if (this.text[0] !== "{") @@ -133,14 +133,23 @@ const objectOf = ( return rejectModel("Expected safetensors header object"); return value; }; -const integersOf = (value: HeaderValue | undefined): number[] => { +const integersOf = (value: HeaderValue | undefined): bigint[] => { if ( !Array.isArray(value) || - !value.every((item): item is number => typeof item === "number") + !value.every((item): item is bigint => typeof item === "bigint") ) return rejectModel("Expected safetensors integer array"); return value; }; +const exactOffset = (value: bigint): number => { + if (value > BigInt(Number.MAX_SAFE_INTEGER)) + throw new ModelFormatFailure( + "capacity", + "Safetensors data offset exceeds exact host offset precision", + "exact-offset", + ); + return Number(value); +}; const pointer = (value: string): string => value.replaceAll("~", "~0").replaceAll("/", "~1"); @@ -243,21 +252,20 @@ export const readSafetensorsHeader = async ( const offsets = integersOf(entry.data_offsets); if (offsets.length !== 2) rejectModel(`Safetensors tensor ${name} requires two data offsets`); - const start = offsets[0] ?? 0; - const end = offsets[1] ?? 0; + const start = exactOffset(offsets[0] ?? 0n); + const end = exactOffset(offsets[1] ?? 0n); if (end < start || end > reader.size - tensor_data_offset) rejectModel( `Safetensors tensor ${name} range [${String(start)},${String(end)}) is outside the data buffer`, ); - let parameters = shape.includes(0) ? 0n : 1n; - if (parameters !== 0n) - for (const dimension of shape) { - parameters *= BigInt(dimension); - if (parameters > 0xffffffffffffffffn) - rejectModel( - `Safetensors tensor ${name} parameter count overflows uint64`, - ); - } + let parameters = 1n; + for (const dimension of shape) { + parameters *= dimension; + if (parameters > 0xffffffffffffffffn) + rejectModel( + `Safetensors tensor ${name} parameter count overflows uint64`, + ); + } const bits = safetensorsBits(type); if (bits !== undefined) { const expected = parameters * BigInt(bits); diff --git a/tests/acceptance/modelArtifactCli.test.ts b/tests/acceptance/modelArtifactCli.test.ts index 0b807b73f7..f5b5d9d668 100644 --- a/tests/acceptance/modelArtifactCli.test.ts +++ b/tests/acceptance/modelArtifactCli.test.ts @@ -19,6 +19,92 @@ import { ggufFixture, safetensorsFixture } from "../fixtures/model/model.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { cliTest } from "../support/cli/cliFixture.js"; +cliTest( + "preserves exact large safetensors dimensions while bounding actual offsets", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-model-shape-precision-"); + const path = join(root, "empty.safetensors"); + const fixture = safetensorsFixture( + '{"a":{"dtype":"F32","shape":[0,9007199254740992],"data_offsets":[0,0]},"b":{"dtype":"F32","shape":[18446744073709551615,0],"data_offsets":[0,0]}}', + Buffer.alloc(0), + ); + await writeFile(path, fixture.bytes); + const result = await cli.run({ + arguments: [ + "inspect-model-artifact", + path, + "--detail", + "complete", + "--hash", + "full", + "--json", + ], + }); + expect(result.exitCode, result.stderr).toBe(0); + const report = modelArtifactSchema.parse( + toolContract("inspect_model_artifact").outputSchema.parse(result.json) + .normalized_result, + ); + expect( + report.tensors.map( + ({ shape, parameter_count, byte_length, storage_validation }) => ({ + shape, + parameter_count, + byte_length, + storage_validation, + }), + ), + ).toEqual([ + { + shape: ["0", "9007199254740992"], + parameter_count: "0", + byte_length: 0, + storage_validation: "consistent", + }, + { + shape: ["18446744073709551615", "0"], + parameter_count: "0", + byte_length: 0, + storage_validation: "consistent", + }, + ]); + expect(report.totals).toMatchObject({ + parameter_count: "0", + tensor_bytes: 0, + }); + expect(report.artifact.sha256).toBe( + createHash("sha256").update(fixture.bytes).digest("hex"), + ); + const { call } = await connectLocalToolsMcp(); + const response = await call("inspect_model_artifact", { + path, + detail: "complete", + hash: "full", + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(parseEvidence(response.structuredContent).normalized_result).toEqual( + report, + ); + + await writeFile( + path, + safetensorsFixture( + '{"x":{"dtype":"F32","shape":[0],"data_offsets":[9007199254740992,9007199254740992]}}', + Buffer.alloc(0), + ).bytes, + ); + const offsetFailure = parseMcpToolError( + await call("inspect_model_artifact", { path }), + ); + expect(offsetFailure.error).toMatchObject({ + code: "resource_constraint", + details: { + reported_limits: { maximum_exact_offset: Number.MAX_SAFE_INTEGER }, + }, + }); + }, +); + it.each(["named", "default", "inventory-only", "absent"] as const)( "reports actual chat-template presence in summaries and page context (%s)", async (kind) => { diff --git a/tests/boundary/model/modelArtifact.test.ts b/tests/boundary/model/modelArtifact.test.ts index 9fc4ae1574..35cb78959b 100644 --- a/tests/boundary/model/modelArtifact.test.ts +++ b/tests/boundary/model/modelArtifact.test.ts @@ -692,6 +692,30 @@ const malformed = [ Buffer.alloc(4), ).bytes, ], + [ + "safetensors dimension exceeding uint64", + () => + safetensorsFixture( + '{"x":{"dtype":"F32","shape":[0,18446744073709551616],"data_offsets":[0,0]}}', + Buffer.alloc(0), + ).bytes, + ], + [ + "overlong safetensors integer", + () => + safetensorsFixture( + '{"x":{"dtype":"F32","shape":[0,100000000000000000000],"data_offsets":[0,0]}}', + Buffer.alloc(0), + ).bytes, + ], + [ + "safetensors shape product overflowing before a zero extent", + () => + safetensorsFixture( + '{"x":{"dtype":"F32","shape":[18446744073709551615,18446744073709551615,0],"data_offsets":[0,0]}}', + Buffer.alloc(0), + ).bytes, + ], [ "safetensors shape mismatch", () =>