mirror of
https://github.com/morluto/rea.git
synced 2026-11-19 13:12:26 +00:00
test(model): verify safetensors ranges with Rust producer fixtures (#12030)
test(model): qualify safetensors storage with a Rust oracle Co-authored-by: ShuaixinHuang <197200901+ShuaixinHuang@users.noreply.github.com>
This commit is contained in:
1 parent
22a06e88c6
commit
6825ce707a
4 files changed
+474
No files matched your search
@@ -0,0 +1,139 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { expect } from "vitest";
|
||||
import { z } from "zod";
|
||||
|
||||
import { INSPECT_MODEL_ARTIFACT_CONTRACT } from "../../src/contracts/model/modelToolContracts.js";
|
||||
import { modelArtifactSchema } from "../../src/domain/model/modelArtifact.js";
|
||||
import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";
|
||||
import { cliTest } from "../support/cli/cliFixture.js";
|
||||
|
||||
const oracle = z
|
||||
.object({
|
||||
file_base64: z.string(),
|
||||
file_size: z.int(),
|
||||
sha256: z.string(),
|
||||
header_sha256: z.string(),
|
||||
tensor_data_offset: z.int(),
|
||||
metadata: z.record(z.string(), z.string()),
|
||||
tensors: z.array(
|
||||
z.object({
|
||||
name: z.string(),
|
||||
type: z.string(),
|
||||
shape: z.array(z.string()),
|
||||
parameter_count: z.string(),
|
||||
json_pointer: z.string(),
|
||||
data_offset: z.int(),
|
||||
file_offset: z.int(),
|
||||
byte_length: z.int(),
|
||||
}),
|
||||
),
|
||||
})
|
||||
.parse(
|
||||
JSON.parse(
|
||||
await readFile(
|
||||
new URL(
|
||||
"../fixtures/safetensors-producer/oracle.json",
|
||||
import.meta.url,
|
||||
),
|
||||
"utf8",
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
cliTest(
|
||||
"preserves the Rust safetensors producer's mixed tensor storage and provenance",
|
||||
async ({ cli }) => {
|
||||
const root = await createTestTempDirectory("rea-safetensors-producer-");
|
||||
const path = join(root, "producer.safetensors");
|
||||
// The complete bytes are emitted by Rust serialize and independently read by
|
||||
// Rust deserialize. Do not rebuild this carrier with REA's fixture writer.
|
||||
const bytes = Buffer.from(oracle.file_base64, "base64");
|
||||
expect(bytes.length).toBe(oracle.file_size);
|
||||
expect(createHash("sha256").update(bytes).digest("hex")).toBe(
|
||||
oracle.sha256,
|
||||
);
|
||||
expect(oracle.tensors).toHaveLength(20);
|
||||
expect(new Set(oracle.tensors.map(({ type }) => type)).size).toBe(18);
|
||||
await writeFile(path, bytes);
|
||||
for (const hash of ["header", "full"] as const) {
|
||||
const result = await cli.run({
|
||||
arguments: [
|
||||
"inspect-model-artifact",
|
||||
path,
|
||||
"--detail",
|
||||
"complete",
|
||||
"--hash",
|
||||
hash,
|
||||
"--json",
|
||||
],
|
||||
});
|
||||
expect(result.exitCode, result.stdout + result.stderr).toBe(0);
|
||||
const report = modelArtifactSchema.parse(
|
||||
INSPECT_MODEL_ARTIFACT_CONTRACT.outputSchema.parse(result.json)
|
||||
.normalized_result,
|
||||
);
|
||||
expect(report.artifact).toMatchObject({
|
||||
path,
|
||||
file_size: oracle.file_size,
|
||||
header_sha256: oracle.header_sha256,
|
||||
sha256: hash === "full" ? oracle.sha256 : null,
|
||||
});
|
||||
expect(report).toMatchObject({
|
||||
format: "safetensors",
|
||||
tensor_data_offset: oracle.tensor_data_offset,
|
||||
coverage: {
|
||||
header: "complete",
|
||||
tensor_data: "not-interpreted",
|
||||
tensor_ranges: "complete",
|
||||
storage_sizes: "complete",
|
||||
},
|
||||
});
|
||||
expect(report.metadata.map(({ key, value }) => [key, value])).toEqual(
|
||||
Object.entries(oracle.metadata),
|
||||
);
|
||||
expect(
|
||||
report.tensors.map(
|
||||
({
|
||||
name,
|
||||
type,
|
||||
shape,
|
||||
parameter_count,
|
||||
json_pointer,
|
||||
data_offset,
|
||||
file_offset,
|
||||
byte_length,
|
||||
}) => ({
|
||||
name,
|
||||
type,
|
||||
shape,
|
||||
parameter_count,
|
||||
json_pointer,
|
||||
data_offset,
|
||||
file_offset,
|
||||
byte_length,
|
||||
}),
|
||||
),
|
||||
).toEqual(oracle.tensors);
|
||||
for (const tensor of report.tensors) {
|
||||
expect(tensor).toMatchObject({
|
||||
source_offset: 8,
|
||||
source_length: oracle.tensor_data_offset - 8,
|
||||
byte_length_state: "declared",
|
||||
storage_validation: "consistent",
|
||||
});
|
||||
}
|
||||
expect(report.totals).toMatchObject({
|
||||
parameter_count: oracle.tensors
|
||||
.reduce((sum, tensor) => sum + BigInt(tensor.parameter_count), 0n)
|
||||
.toString(),
|
||||
tensor_bytes: oracle.tensors.reduce(
|
||||
(sum, tensor) => sum + tensor.byte_length,
|
||||
0,
|
||||
),
|
||||
});
|
||||
}
|
||||
expect((await readFile(path)).equals(bytes)).toBe(true);
|
||||
},
|
||||
);
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
# Safetensors producer storage oracle
|
||||
|
||||
`oracle.json` contains the complete inert bytes emitted by **safetensors 0.7.0**
|
||||
([released source](https://github.com/huggingface/safetensors/tree/v0.7.0)), encoded
|
||||
as base64 so no binary fixture is committed. Its expected declarations come from
|
||||
the package's Rust `deserialize`, independently of REA's JSON reader and dtype
|
||||
table. Python's JSON reader records the emitted wire offsets; each span is also
|
||||
checked against the bytes returned by Rust. The producer verifies every decoded
|
||||
payload against its self-authored input, without interpreting model behavior.
|
||||
|
||||
One 1,934-byte model contains 20 tensors and 18 emitted dtypes: integer, boolean,
|
||||
bfloat16, float16/32/64, complex64, three float8 forms and packed F4, plus a scalar
|
||||
and an empty tensor. The Python API's `float4_e2m1fn_x2` input shape `[2, 3]`
|
||||
becomes wire dtype `F4`, shape `[2, 6]` and six bytes. That result comes from the
|
||||
producer, not a transcription of REA's bit-width mapping. Literal metadata and
|
||||
names containing JSON Pointer delimiters qualify provenance alongside storage.
|
||||
|
||||
The compiled CLI regression checks both header-only and full-file hash modes,
|
||||
all tensor declarations and physical ranges, exact metadata, totals, and the
|
||||
existing distinction between consistent storage and uninterpreted tensor data.
|
||||
CI runs the recorded bytes without Python, NumPy, Torch or model weights.
|
||||
|
||||
Reproduce with a user-owned Python environment:
|
||||
|
||||
```sh
|
||||
python3 -m venv /tmp/rea-safetensors-oracle
|
||||
/tmp/rea-safetensors-oracle/bin/pip install safetensors==0.7.0
|
||||
/tmp/rea-safetensors-oracle/bin/python tests/fixtures/safetensors-producer/produce.py --check
|
||||
```
|
||||
|
||||
Omit `--check` to regenerate intentionally, then format the JSON with the
|
||||
repository formatter and review the changed bytes and facts. `--check` compares
|
||||
JSON values, including the exact encoded file bytes, rather than JSON whitespace.
|
||||
|
||||
Current production is correct for this sample. On main `da91bbd4e`, temporarily
|
||||
changing `F8_E8M0: 8` to `F8_E8M0: 16` in `ModelTensorTypes.ts` left all **161 tests
|
||||
in the four existing model boundary/CLI suites passing**. This new public CLI
|
||||
regression rejects the mutated behavior: the valid producer tensor is reported
|
||||
as having a mismatched byte length. The mutation was restored byte-for-byte
|
||||
before final checks. This qualifies a missing regression guard, not a production
|
||||
bug fix, model inference, tensor values, GPU execution or every future dtype.
|
||||
+213
@@ -0,0 +1,213 @@
|
||||
{
|
||||
"producer": "safetensors==0.7.0 (Rust serialize + deserialize)",
|
||||
"file_base64": "IAYAAAAAAAB7Il9fbWV0YWRhdGFfXyI6eyJwcm9kdWNlci9+77u/IjoiUnVzdCBzYWZldGVuc29ycyAwLjcuMDsgbGl0ZXJhbCDvu78gbWV0YWRhdGEifSwidGVuc29yL3VpbnQ2NH4xMiI6eyJkdHlwZSI6IlU2NCIsInNoYXBlIjpbMiwzXSwiZGF0YV9vZmZzZXRzIjpbMCw0OF19LCJ0ZW5zb3IvaW50NjR+MTEiOnsiZHR5cGUiOiJJNjQiLCJzaGFwZSI6WzIsM10sImRhdGFfb2Zmc2V0cyI6WzQ4LDk2XX0sInRlbnNvci9mbG9hdDY0fjgiOnsiZHR5cGUiOiJGNjQiLCJzaGFwZSI6WzIsM10sImRhdGFfb2Zmc2V0cyI6Wzk2LDE0NF19LCJ0ZW5zb3IvY29tcGxleDY0fjEzIjp7ImR0eXBlIjoiQzY0Iiwic2hhcGUiOlsyLDNdLCJkYXRhX29mZnNldHMiOlsxNDQsMTkyXX0sInNjYWxhciI6eyJkdHlwZSI6IkYzMiIsInNoYXBlIjpbXSwiZGF0YV9vZmZzZXRzIjpbMTkyLDE5Nl19LCJ0ZW5zb3IvZmxvYXQzMn43Ijp7ImR0eXBlIjoiRjMyIiwic2hhcGUiOlsyLDNdLCJkYXRhX29mZnNldHMiOlsxOTYsMjIwXX0sInRlbnNvci91aW50MzJ+MTAiOnsiZHR5cGUiOiJVMzIiLCJzaGFwZSI6WzIsM10sImRhdGFfb2Zmc2V0cyI6WzIyMCwyNDRdfSwidGVuc29yL2ludDMyfjkiOnsiZHR5cGUiOiJJMzIiLCJzaGFwZSI6WzIsM10sImRhdGFfb2Zmc2V0cyI6WzI0NCwyNjhdfSwiZW1wdHkiOnsiZHR5cGUiOiJCRjE2Iiwic2hhcGUiOlswLDddLCJkYXRhX29mZnNldHMiOlsyNjgsMjY4XX0sInRlbnNvci9iZmxvYXQxNn41Ijp7ImR0eXBlIjoiQkYxNiIsInNoYXBlIjpbMiwzXSwiZGF0YV9vZmZzZXRzIjpbMjY4LDI4MF19LCJ0ZW5zb3IvZmxvYXQxNn42Ijp7ImR0eXBlIjoiRjE2Iiwic2hhcGUiOlsyLDNdLCJkYXRhX29mZnNldHMiOlsyODAsMjkyXX0sInRlbnNvci91aW50MTZ+MyI6eyJkdHlwZSI6IlUxNiIsInNoYXBlIjpbMiwzXSwiZGF0YV9vZmZzZXRzIjpbMjkyLDMwNF19LCJ0ZW5zb3IvaW50MTZ+NCI6eyJkdHlwZSI6IkkxNiIsInNoYXBlIjpbMiwzXSwiZGF0YV9vZmZzZXRzIjpbMzA0LDMxNl19LCJ0ZW5zb3IvZmxvYXQ4X2U4bTBmbnV+MTYiOnsiZHR5cGUiOiJGOF9FOE0wIiwic2hhcGUiOlsyLDNdLCJkYXRhX29mZnNldHMiOlszMTYsMzIyXX0sInRlbnNvci9mbG9hdDhfZTRtM2ZufjE0Ijp7ImR0eXBlIjoiRjhfRTRNMyIsInNoYXBlIjpbMiwzXSwiZGF0YV9vZmZzZXRzIjpbMzIyLDMyOF19LCJ0ZW5zb3IvZmxvYXQ4X2U1bTJ+MTUiOnsiZHR5cGUiOiJGOF9FNU0yIiwic2hhcGUiOlsyLDNdLCJkYXRhX29mZnNldHMiOlszMjgsMzM0XX0sInRlbnNvci9pbnQ4fjIiOnsiZHR5cGUiOiJJOCIsInNoYXBlIjpbMiwzXSwiZGF0YV9vZmZzZXRzIjpbMzM0LDM0MF19LCJ0ZW5zb3IvdWludDh+MSI6eyJkdHlwZSI6IlU4Iiwic2hhcGUiOlsyLDNdLCJkYXRhX29mZnNldHMiOlszNDAsMzQ2XX0sInRlbnNvci9mbG9hdDRfZTJtMWZuX3gyfjE3Ijp7ImR0eXBlIjoiRjQiLCJzaGFwZSI6WzIsNl0sImRhdGFfb2Zmc2V0cyI6WzM0NiwzNTJdfSwidGVuc29yL2Jvb2x+MCI6eyJkdHlwZSI6IkJPT0wiLCJzaGFwZSI6WzIsM10sImRhdGFfb2Zmc2V0cyI6WzM1MiwzNThdfX0gICAgIMzZ5vMADRonNEFOW2h1go+cqbbD0N3q9wQRHis4RVJfbHmGk6CtusfU4e77CBUiL7vI1eLv/AkWIzA9SldkcX6LmKWyv8zZ5vMADRonNEFOW2h1go+cqbbD0N3q9wQRHoiVoq+8ydbj8P0KFyQxPktYZXJ/jJmms8DN2uf0AQ4bKDVCT1xpdoOQnaq3xNHe693q9wQRHis4RVJfbHmGk6CtusfU4e77CBUiLzxJVmNwfYqXpLG+y9jl8v8MGSYzQAAAgD93hJGeq7jF0t/s+QYTIC06R1RhbnuIlaKqt8TR3uv4BRIfLDlGU2BteoeUoa67yNWZprPAzdrn9AEOGyg1Qk9caXaDkJ2qt8RVYm98iZajsL3K1+Rmc4CNmqe0wc7b6PUzQE1aZ3SBjpuotcJEUV5reIWSn6y5xtMQHSo3RFHu+wgVIi//DBkmM0AiLzxJVmMRHis4RVIhLjtIVWIADRonNEE=",
|
||||
"file_size": 1934,
|
||||
"sha256": "5fb44d541cd95db65982fb252a957a0b4acc6910a964ec0157af5edc8947a6a4",
|
||||
"header_sha256": "14d0e34758aecc59cb692a79bd55032fbfd71acffbb3e63741ea943e8c7a7fd7",
|
||||
"tensor_data_offset": 1576,
|
||||
"metadata": {
|
||||
"producer/~\ufeff": "Rust safetensors 0.7.0; literal \ufeff metadata"
|
||||
},
|
||||
"tensors": [
|
||||
{
|
||||
"name": "empty",
|
||||
"type": "BF16",
|
||||
"shape": ["0", "7"],
|
||||
"json_pointer": "/empty",
|
||||
"parameter_count": "0",
|
||||
"data_offset": 268,
|
||||
"file_offset": 1844,
|
||||
"byte_length": 0
|
||||
},
|
||||
{
|
||||
"name": "scalar",
|
||||
"type": "F32",
|
||||
"shape": [],
|
||||
"json_pointer": "/scalar",
|
||||
"parameter_count": "1",
|
||||
"data_offset": 192,
|
||||
"file_offset": 1768,
|
||||
"byte_length": 4
|
||||
},
|
||||
{
|
||||
"name": "tensor/bfloat16~5",
|
||||
"type": "BF16",
|
||||
"shape": ["2", "3"],
|
||||
"json_pointer": "/tensor~1bfloat16~05",
|
||||
"parameter_count": "6",
|
||||
"data_offset": 268,
|
||||
"file_offset": 1844,
|
||||
"byte_length": 12
|
||||
},
|
||||
{
|
||||
"name": "tensor/bool~0",
|
||||
"type": "BOOL",
|
||||
"shape": ["2", "3"],
|
||||
"json_pointer": "/tensor~1bool~00",
|
||||
"parameter_count": "6",
|
||||
"data_offset": 352,
|
||||
"file_offset": 1928,
|
||||
"byte_length": 6
|
||||
},
|
||||
{
|
||||
"name": "tensor/complex64~13",
|
||||
"type": "C64",
|
||||
"shape": ["2", "3"],
|
||||
"json_pointer": "/tensor~1complex64~013",
|
||||
"parameter_count": "6",
|
||||
"data_offset": 144,
|
||||
"file_offset": 1720,
|
||||
"byte_length": 48
|
||||
},
|
||||
{
|
||||
"name": "tensor/float16~6",
|
||||
"type": "F16",
|
||||
"shape": ["2", "3"],
|
||||
"json_pointer": "/tensor~1float16~06",
|
||||
"parameter_count": "6",
|
||||
"data_offset": 280,
|
||||
"file_offset": 1856,
|
||||
"byte_length": 12
|
||||
},
|
||||
{
|
||||
"name": "tensor/float32~7",
|
||||
"type": "F32",
|
||||
"shape": ["2", "3"],
|
||||
"json_pointer": "/tensor~1float32~07",
|
||||
"parameter_count": "6",
|
||||
"data_offset": 196,
|
||||
"file_offset": 1772,
|
||||
"byte_length": 24
|
||||
},
|
||||
{
|
||||
"name": "tensor/float4_e2m1fn_x2~17",
|
||||
"type": "F4",
|
||||
"shape": ["2", "6"],
|
||||
"json_pointer": "/tensor~1float4_e2m1fn_x2~017",
|
||||
"parameter_count": "12",
|
||||
"data_offset": 346,
|
||||
"file_offset": 1922,
|
||||
"byte_length": 6
|
||||
},
|
||||
{
|
||||
"name": "tensor/float64~8",
|
||||
"type": "F64",
|
||||
"shape": ["2", "3"],
|
||||
"json_pointer": "/tensor~1float64~08",
|
||||
"parameter_count": "6",
|
||||
"data_offset": 96,
|
||||
"file_offset": 1672,
|
||||
"byte_length": 48
|
||||
},
|
||||
{
|
||||
"name": "tensor/float8_e4m3fn~14",
|
||||
"type": "F8_E4M3",
|
||||
"shape": ["2", "3"],
|
||||
"json_pointer": "/tensor~1float8_e4m3fn~014",
|
||||
"parameter_count": "6",
|
||||
"data_offset": 322,
|
||||
"file_offset": 1898,
|
||||
"byte_length": 6
|
||||
},
|
||||
{
|
||||
"name": "tensor/float8_e5m2~15",
|
||||
"type": "F8_E5M2",
|
||||
"shape": ["2", "3"],
|
||||
"json_pointer": "/tensor~1float8_e5m2~015",
|
||||
"parameter_count": "6",
|
||||
"data_offset": 328,
|
||||
"file_offset": 1904,
|
||||
"byte_length": 6
|
||||
},
|
||||
{
|
||||
"name": "tensor/float8_e8m0fnu~16",
|
||||
"type": "F8_E8M0",
|
||||
"shape": ["2", "3"],
|
||||
"json_pointer": "/tensor~1float8_e8m0fnu~016",
|
||||
"parameter_count": "6",
|
||||
"data_offset": 316,
|
||||
"file_offset": 1892,
|
||||
"byte_length": 6
|
||||
},
|
||||
{
|
||||
"name": "tensor/int16~4",
|
||||
"type": "I16",
|
||||
"shape": ["2", "3"],
|
||||
"json_pointer": "/tensor~1int16~04",
|
||||
"parameter_count": "6",
|
||||
"data_offset": 304,
|
||||
"file_offset": 1880,
|
||||
"byte_length": 12
|
||||
},
|
||||
{
|
||||
"name": "tensor/int32~9",
|
||||
"type": "I32",
|
||||
"shape": ["2", "3"],
|
||||
"json_pointer": "/tensor~1int32~09",
|
||||
"parameter_count": "6",
|
||||
"data_offset": 244,
|
||||
"file_offset": 1820,
|
||||
"byte_length": 24
|
||||
},
|
||||
{
|
||||
"name": "tensor/int64~11",
|
||||
"type": "I64",
|
||||
"shape": ["2", "3"],
|
||||
"json_pointer": "/tensor~1int64~011",
|
||||
"parameter_count": "6",
|
||||
"data_offset": 48,
|
||||
"file_offset": 1624,
|
||||
"byte_length": 48
|
||||
},
|
||||
{
|
||||
"name": "tensor/int8~2",
|
||||
"type": "I8",
|
||||
"shape": ["2", "3"],
|
||||
"json_pointer": "/tensor~1int8~02",
|
||||
"parameter_count": "6",
|
||||
"data_offset": 334,
|
||||
"file_offset": 1910,
|
||||
"byte_length": 6
|
||||
},
|
||||
{
|
||||
"name": "tensor/uint16~3",
|
||||
"type": "U16",
|
||||
"shape": ["2", "3"],
|
||||
"json_pointer": "/tensor~1uint16~03",
|
||||
"parameter_count": "6",
|
||||
"data_offset": 292,
|
||||
"file_offset": 1868,
|
||||
"byte_length": 12
|
||||
},
|
||||
{
|
||||
"name": "tensor/uint32~10",
|
||||
"type": "U32",
|
||||
"shape": ["2", "3"],
|
||||
"json_pointer": "/tensor~1uint32~010",
|
||||
"parameter_count": "6",
|
||||
"data_offset": 220,
|
||||
"file_offset": 1796,
|
||||
"byte_length": 24
|
||||
},
|
||||
{
|
||||
"name": "tensor/uint64~12",
|
||||
"type": "U64",
|
||||
"shape": ["2", "3"],
|
||||
"json_pointer": "/tensor~1uint64~012",
|
||||
"parameter_count": "6",
|
||||
"data_offset": 0,
|
||||
"file_offset": 1576,
|
||||
"byte_length": 48
|
||||
},
|
||||
{
|
||||
"name": "tensor/uint8~1",
|
||||
"type": "U8",
|
||||
"shape": ["2", "3"],
|
||||
"json_pointer": "/tensor~1uint8~01",
|
||||
"parameter_count": "6",
|
||||
"data_offset": 340,
|
||||
"file_offset": 1916,
|
||||
"byte_length": 6
|
||||
}
|
||||
]
|
||||
}
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
"""Produce an inert tensor-storage oracle with the pinned Rust implementation."""
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
import safetensors
|
||||
|
||||
VERSION = "0.7.0"
|
||||
DTYPES = {
|
||||
"bool": 1, "uint8": 1, "int8": 1,
|
||||
"uint16": 2, "int16": 2, "bfloat16": 2, "float16": 2,
|
||||
"float32": 4, "float64": 8, "int32": 4, "uint32": 4,
|
||||
"int64": 8, "uint64": 8, "complex64": 8,
|
||||
"float8_e4m3fn": 1, "float8_e5m2": 1, "float8_e8m0fnu": 1,
|
||||
"float4_e2m1fn_x2": 1,
|
||||
}
|
||||
|
||||
|
||||
def produce():
|
||||
if safetensors.__version__ != VERSION:
|
||||
raise RuntimeError(f"requires safetensors=={VERSION}")
|
||||
inputs = {}
|
||||
for index, (dtype, width) in enumerate(DTYPES.items()):
|
||||
# These are Python API dtype names, not REA's wire-type table. In
|
||||
# particular the Rust producer expands the packed F4 final dimension.
|
||||
inputs[f"tensor/{dtype}~{index}"] = {
|
||||
"dtype": dtype, "shape": [2, 3],
|
||||
"data": bytes((index * 17 + at * 13) % 256 for at in range(6 * width)),
|
||||
}
|
||||
inputs["scalar"] = {"dtype": "float32", "shape": [], "data": bytes([0, 0, 128, 63])}
|
||||
inputs["empty"] = {"dtype": "bfloat16", "shape": [0, 7], "data": b""}
|
||||
metadata = {"producer/~\ufeff": "Rust safetensors 0.7.0; literal \ufeff metadata"}
|
||||
encoded = safetensors.serialize(inputs, metadata=metadata)
|
||||
# Independently decode the exact emitted bytes with the official Rust
|
||||
# reader; no REA fixture builder, parser or dtype table participates.
|
||||
decoded = dict(safetensors.deserialize(encoded))
|
||||
assert decoded.keys() == inputs.keys()
|
||||
header_bytes = int.from_bytes(encoded[:8], "little")
|
||||
header = json.loads(encoded[8:8 + header_bytes])
|
||||
assert header.pop("__metadata__") == metadata
|
||||
tensors = []
|
||||
for name in sorted(decoded):
|
||||
row = decoded[name]
|
||||
assert bytes(row["data"]) == inputs[name]["data"]
|
||||
assert row["dtype"] == header[name]["dtype"]
|
||||
assert row["shape"] == header[name]["shape"]
|
||||
start, end = header[name]["data_offsets"]
|
||||
assert encoded[8 + header_bytes + start:8 + header_bytes + end] == row["data"]
|
||||
tensors.append({
|
||||
"name": name, "type": row["dtype"], "shape": [str(n) for n in row["shape"]],
|
||||
"json_pointer": "/" + name.replace("~", "~0").replace("/", "~1"),
|
||||
"parameter_count": str(math.prod(row["shape"])),
|
||||
"data_offset": start, "file_offset": 8 + header_bytes + start,
|
||||
"byte_length": len(row["data"]),
|
||||
})
|
||||
return {
|
||||
"producer": f"safetensors=={VERSION} (Rust serialize + deserialize)",
|
||||
"file_base64": base64.b64encode(encoded).decode(),
|
||||
"file_size": len(encoded), "sha256": hashlib.sha256(encoded).hexdigest(),
|
||||
"header_sha256": hashlib.sha256(encoded[:8 + header_bytes]).hexdigest(),
|
||||
"tensor_data_offset": 8 + header_bytes,
|
||||
"metadata": metadata, "tensors": tensors,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--check", action="store_true")
|
||||
args = parser.parse_args()
|
||||
target = Path(__file__).with_name("oracle.json")
|
||||
text = json.dumps(produce(), indent=2, ensure_ascii=True) + "\n"
|
||||
if args.check:
|
||||
if json.loads(target.read_text()) != json.loads(text):
|
||||
raise SystemExit("safetensors producer oracle differs")
|
||||
print("Pinned Rust producer and reader reproduce oracle.json exactly")
|
||||
else:
|
||||
target.write_text(text)
|
||||
print(target)
|
||||
Reference in new issue
Block a user