test(model): verify safetensors ranges with Rust producer fixtures (#12030)

test(model): qualify safetensors storage with a Rust oracle

Co-authored-by: ShuaixinHuang <197200901+ShuaixinHuang@users.noreply.github.com>
This commit is contained in:
ShuaixinHuangandShuaixinHuang authored and GitHub committed 2026-11-19 00:12:55 -05:00
1 parent 22a06e88c6
commit 6825ce707a
4 files changed
+474

No files matched your search

@@ -0,0 +1,139 @@
import { createHash } from "node:crypto";
import { readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect } from "vitest";
import { z } from "zod";
import { INSPECT_MODEL_ARTIFACT_CONTRACT } from "../../src/contracts/model/modelToolContracts.js";
import { modelArtifactSchema } from "../../src/domain/model/modelArtifact.js";
import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";
import { cliTest } from "../support/cli/cliFixture.js";
const oracle = z
.object({
file_base64: z.string(),
file_size: z.int(),
sha256: z.string(),
header_sha256: z.string(),
tensor_data_offset: z.int(),
metadata: z.record(z.string(), z.string()),
tensors: z.array(
z.object({
name: z.string(),
type: z.string(),
shape: z.array(z.string()),
parameter_count: z.string(),
json_pointer: z.string(),
data_offset: z.int(),
file_offset: z.int(),
byte_length: z.int(),
}),
),
})
.parse(
JSON.parse(
await readFile(
new URL(
"../fixtures/safetensors-producer/oracle.json",
import.meta.url,
),
"utf8",
),
),
);
cliTest(
"preserves the Rust safetensors producer's mixed tensor storage and provenance",
async ({ cli }) => {
const root = await createTestTempDirectory("rea-safetensors-producer-");
const path = join(root, "producer.safetensors");
// The complete bytes are emitted by Rust serialize and independently read by
// Rust deserialize. Do not rebuild this carrier with REA's fixture writer.
const bytes = Buffer.from(oracle.file_base64, "base64");
expect(bytes.length).toBe(oracle.file_size);
expect(createHash("sha256").update(bytes).digest("hex")).toBe(
oracle.sha256,
);
expect(oracle.tensors).toHaveLength(20);
expect(new Set(oracle.tensors.map(({ type }) => type)).size).toBe(18);
await writeFile(path, bytes);
for (const hash of ["header", "full"] as const) {
const result = await cli.run({
arguments: [
"inspect-model-artifact",
path,
"--detail",
"complete",
"--hash",
hash,
"--json",
],
});
expect(result.exitCode, result.stdout + result.stderr).toBe(0);
const report = modelArtifactSchema.parse(
INSPECT_MODEL_ARTIFACT_CONTRACT.outputSchema.parse(result.json)
.normalized_result,
);
expect(report.artifact).toMatchObject({
path,
file_size: oracle.file_size,
header_sha256: oracle.header_sha256,
sha256: hash === "full" ? oracle.sha256 : null,
});
expect(report).toMatchObject({
format: "safetensors",
tensor_data_offset: oracle.tensor_data_offset,
coverage: {
header: "complete",
tensor_data: "not-interpreted",
tensor_ranges: "complete",
storage_sizes: "complete",
},
});
expect(report.metadata.map(({ key, value }) => [key, value])).toEqual(
Object.entries(oracle.metadata),
);
expect(
report.tensors.map(
({
name,
type,
shape,
parameter_count,
json_pointer,
data_offset,
file_offset,
byte_length,
}) => ({
name,
type,
shape,
parameter_count,
json_pointer,
data_offset,
file_offset,
byte_length,
}),
),
).toEqual(oracle.tensors);
for (const tensor of report.tensors) {
expect(tensor).toMatchObject({
source_offset: 8,
source_length: oracle.tensor_data_offset - 8,
byte_length_state: "declared",
storage_validation: "consistent",
});
}
expect(report.totals).toMatchObject({
parameter_count: oracle.tensors
.reduce((sum, tensor) => sum + BigInt(tensor.parameter_count), 0n)
.toString(),
tensor_bytes: oracle.tensors.reduce(
(sum, tensor) => sum + tensor.byte_length,
0,
),
});
}
expect((await readFile(path)).equals(bytes)).toBe(true);
},
);
+41
View File
@@ -0,0 +1,41 @@
# Safetensors producer storage oracle
`oracle.json` contains the complete inert bytes emitted by **safetensors 0.7.0**
([released source](https://github.com/huggingface/safetensors/tree/v0.7.0)), encoded
as base64 so no binary fixture is committed. Its expected declarations come from
the package's Rust `deserialize`, independently of REA's JSON reader and dtype
table. Python's JSON reader records the emitted wire offsets; each span is also
checked against the bytes returned by Rust. The producer verifies every decoded
payload against its self-authored input, without interpreting model behavior.
One 1,934-byte model contains 20 tensors and 18 emitted dtypes: integer, boolean,
bfloat16, float16/32/64, complex64, three float8 forms and packed F4, plus a scalar
and an empty tensor. The Python API's `float4_e2m1fn_x2` input shape `[2, 3]`
becomes wire dtype `F4`, shape `[2, 6]` and six bytes. That result comes from the
producer, not a transcription of REA's bit-width mapping. Literal metadata and
names containing JSON Pointer delimiters qualify provenance alongside storage.
The compiled CLI regression checks both header-only and full-file hash modes,
all tensor declarations and physical ranges, exact metadata, totals, and the
existing distinction between consistent storage and uninterpreted tensor data.
CI runs the recorded bytes without Python, NumPy, Torch or model weights.
Reproduce with a user-owned Python environment:
```sh
python3 -m venv /tmp/rea-safetensors-oracle
/tmp/rea-safetensors-oracle/bin/pip install safetensors==0.7.0
/tmp/rea-safetensors-oracle/bin/python tests/fixtures/safetensors-producer/produce.py --check
```
Omit `--check` to regenerate intentionally, then format the JSON with the
repository formatter and review the changed bytes and facts. `--check` compares
JSON values, including the exact encoded file bytes, rather than JSON whitespace.
Current production is correct for this sample. On main `da91bbd4e`, temporarily
changing `F8_E8M0: 8` to `F8_E8M0: 16` in `ModelTensorTypes.ts` left all **161 tests
in the four existing model boundary/CLI suites passing**. This new public CLI
regression rejects the mutated behavior: the valid producer tensor is reported
as having a mismatched byte length. The mutation was restored byte-for-byte
before final checks. This qualifies a missing regression guard, not a production
bug fix, model inference, tensor values, GPU execution or every future dtype.
+213
View File
@@ -0,0 +1,213 @@
{
"producer": "safetensors==0.7.0 (Rust serialize + deserialize)",
"file_base64": "IAYAAAAAAAB7Il9fbWV0YWRhdGFfXyI6eyJwcm9kdWNlci9+77u/IjoiUnVzdCBzYWZldGVuc29ycyAwLjcuMDsgbGl0ZXJhbCDvu78gbWV0YWRhdGEifSwidGVuc29yL3VpbnQ2NH4xMiI6eyJkdHlwZSI6IlU2NCIsInNoYXBlIjpbMiwzXSwiZGF0YV9vZmZzZXRzIjpbMCw0OF19LCJ0ZW5zb3IvaW50NjR+MTEiOnsiZHR5cGUiOiJJNjQiLCJzaGFwZSI6WzIsM10sImRhdGFfb2Zmc2V0cyI6WzQ4LDk2XX0sInRlbnNvci9mbG9hdDY0fjgiOnsiZHR5cGUiOiJGNjQiLCJzaGFwZSI6WzIsM10sImRhdGFfb2Zmc2V0cyI6Wzk2LDE0NF19LCJ0ZW5zb3IvY29tcGxleDY0fjEzIjp7ImR0eXBlIjoiQzY0Iiwic2hhcGUiOlsyLDNdLCJkYXRhX29mZnNldHMiOlsxNDQsMTkyXX0sInNjYWxhciI6eyJkdHlwZSI6IkYzMiIsInNoYXBlIjpbXSwiZGF0YV9vZmZzZXRzIjpbMTkyLDE5Nl19LCJ0ZW5zb3IvZmxvYXQzMn43Ijp7ImR0eXBlIjoiRjMyIiwic2hhcGUiOlsyLDNdLCJkYXRhX29mZnNldHMiOlsxOTYsMjIwXX0sInRlbnNvci91aW50MzJ+MTAiOnsiZHR5cGUiOiJVMzIiLCJzaGFwZSI6WzIsM10sImRhdGFfb2Zmc2V0cyI6WzIyMCwyNDRdfSwidGVuc29yL2ludDMyfjkiOnsiZHR5cGUiOiJJMzIiLCJzaGFwZSI6WzIsM10sImRhdGFfb2Zmc2V0cyI6WzI0NCwyNjhdfSwiZW1wdHkiOnsiZHR5cGUiOiJCRjE2Iiwic2hhcGUiOlswLDddLCJkYXRhX29mZnNldHMiOlsyNjgsMjY4XX0sInRlbnNvci9iZmxvYXQxNn41Ijp7ImR0eXBlIjoiQkYxNiIsInNoYXBlIjpbMiwzXSwiZGF0YV9vZmZzZXRzIjpbMjY4LDI4MF19LCJ0ZW5zb3IvZmxvYXQxNn42Ijp7ImR0eXBlIjoiRjE2Iiwic2hhcGUiOlsyLDNdLCJkYXRhX29mZnNldHMiOlsyODAsMjkyXX0sInRlbnNvci91aW50MTZ+MyI6eyJkdHlwZSI6IlUxNiIsInNoYXBlIjpbMiwzXSwiZGF0YV9vZmZzZXRzIjpbMjkyLDMwNF19LCJ0ZW5zb3IvaW50MTZ+NCI6eyJkdHlwZSI6IkkxNiIsInNoYXBlIjpbMiwzXSwiZGF0YV9vZmZzZXRzIjpbMzA0LDMxNl19LCJ0ZW5zb3IvZmxvYXQ4X2U4bTBmbnV+MTYiOnsiZHR5cGUiOiJGOF9FOE0wIiwic2hhcGUiOlsyLDNdLCJkYXRhX29mZnNldHMiOlszMTYsMzIyXX0sInRlbnNvci9mbG9hdDhfZTRtM2ZufjE0Ijp7ImR0eXBlIjoiRjhfRTRNMyIsInNoYXBlIjpbMiwzXSwiZGF0YV9vZmZzZXRzIjpbMzIyLDMyOF19LCJ0ZW5zb3IvZmxvYXQ4X2U1bTJ+MTUiOnsiZHR5cGUiOiJGOF9FNU0yIiwic2hhcGUiOlsyLDNdLCJkYXRhX29mZnNldHMiOlszMjgsMzM0XX0sInRlbnNvci9pbnQ4fjIiOnsiZHR5cGUiOiJJOCIsInNoYXBlIjpbMiwzXSwiZGF0YV9vZmZzZXRzIjpbMzM0LDM0MF19LCJ0ZW5zb3IvdWludDh+MSI6eyJkdHlwZSI6IlU4Iiwic2hhcGUiOlsyLDNdLCJkYXRhX29mZnNldHMiOlszNDAsMzQ2XX0sInRlbnNvci9mbG9hdDRfZTJtMWZuX3gyfjE3Ijp7ImR0eXBlIjoiRjQiLCJzaGFwZSI6WzIsNl0sImRhdGFfb2Zmc2V0cyI6WzM0NiwzNTJdfSwidGVuc29yL2Jvb2x+MCI6eyJkdHlwZSI6IkJPT0wiLCJzaGFwZSI6WzIsM10sImRhdGFfb2Zmc2V0cyI6WzM1MiwzNThdfX0gICAgIMzZ5vMADRonNEFOW2h1go+cqbbD0N3q9wQRHis4RVJfbHmGk6CtusfU4e77CBUiL7vI1eLv/AkWIzA9SldkcX6LmKWyv8zZ5vMADRonNEFOW2h1go+cqbbD0N3q9wQRHoiVoq+8ydbj8P0KFyQxPktYZXJ/jJmms8DN2uf0AQ4bKDVCT1xpdoOQnaq3xNHe693q9wQRHis4RVJfbHmGk6CtusfU4e77CBUiLzxJVmNwfYqXpLG+y9jl8v8MGSYzQAAAgD93hJGeq7jF0t/s+QYTIC06R1RhbnuIlaKqt8TR3uv4BRIfLDlGU2BteoeUoa67yNWZprPAzdrn9AEOGyg1Qk9caXaDkJ2qt8RVYm98iZajsL3K1+Rmc4CNmqe0wc7b6PUzQE1aZ3SBjpuotcJEUV5reIWSn6y5xtMQHSo3RFHu+wgVIi//DBkmM0AiLzxJVmMRHis4RVIhLjtIVWIADRonNEE=",
"file_size": 1934,
"sha256": "5fb44d541cd95db65982fb252a957a0b4acc6910a964ec0157af5edc8947a6a4",
"header_sha256": "14d0e34758aecc59cb692a79bd55032fbfd71acffbb3e63741ea943e8c7a7fd7",
"tensor_data_offset": 1576,
"metadata": {
"producer/~\ufeff": "Rust safetensors 0.7.0; literal \ufeff metadata"
},
"tensors": [
{
"name": "empty",
"type": "BF16",
"shape": ["0", "7"],
"json_pointer": "/empty",
"parameter_count": "0",
"data_offset": 268,
"file_offset": 1844,
"byte_length": 0
},
{
"name": "scalar",
"type": "F32",
"shape": [],
"json_pointer": "/scalar",
"parameter_count": "1",
"data_offset": 192,
"file_offset": 1768,
"byte_length": 4
},
{
"name": "tensor/bfloat16~5",
"type": "BF16",
"shape": ["2", "3"],
"json_pointer": "/tensor~1bfloat16~05",
"parameter_count": "6",
"data_offset": 268,
"file_offset": 1844,
"byte_length": 12
},
{
"name": "tensor/bool~0",
"type": "BOOL",
"shape": ["2", "3"],
"json_pointer": "/tensor~1bool~00",
"parameter_count": "6",
"data_offset": 352,
"file_offset": 1928,
"byte_length": 6
},
{
"name": "tensor/complex64~13",
"type": "C64",
"shape": ["2", "3"],
"json_pointer": "/tensor~1complex64~013",
"parameter_count": "6",
"data_offset": 144,
"file_offset": 1720,
"byte_length": 48
},
{
"name": "tensor/float16~6",
"type": "F16",
"shape": ["2", "3"],
"json_pointer": "/tensor~1float16~06",
"parameter_count": "6",
"data_offset": 280,
"file_offset": 1856,
"byte_length": 12
},
{
"name": "tensor/float32~7",
"type": "F32",
"shape": ["2", "3"],
"json_pointer": "/tensor~1float32~07",
"parameter_count": "6",
"data_offset": 196,
"file_offset": 1772,
"byte_length": 24
},
{
"name": "tensor/float4_e2m1fn_x2~17",
"type": "F4",
"shape": ["2", "6"],
"json_pointer": "/tensor~1float4_e2m1fn_x2~017",
"parameter_count": "12",
"data_offset": 346,
"file_offset": 1922,
"byte_length": 6
},
{
"name": "tensor/float64~8",
"type": "F64",
"shape": ["2", "3"],
"json_pointer": "/tensor~1float64~08",
"parameter_count": "6",
"data_offset": 96,
"file_offset": 1672,
"byte_length": 48
},
{
"name": "tensor/float8_e4m3fn~14",
"type": "F8_E4M3",
"shape": ["2", "3"],
"json_pointer": "/tensor~1float8_e4m3fn~014",
"parameter_count": "6",
"data_offset": 322,
"file_offset": 1898,
"byte_length": 6
},
{
"name": "tensor/float8_e5m2~15",
"type": "F8_E5M2",
"shape": ["2", "3"],
"json_pointer": "/tensor~1float8_e5m2~015",
"parameter_count": "6",
"data_offset": 328,
"file_offset": 1904,
"byte_length": 6
},
{
"name": "tensor/float8_e8m0fnu~16",
"type": "F8_E8M0",
"shape": ["2", "3"],
"json_pointer": "/tensor~1float8_e8m0fnu~016",
"parameter_count": "6",
"data_offset": 316,
"file_offset": 1892,
"byte_length": 6
},
{
"name": "tensor/int16~4",
"type": "I16",
"shape": ["2", "3"],
"json_pointer": "/tensor~1int16~04",
"parameter_count": "6",
"data_offset": 304,
"file_offset": 1880,
"byte_length": 12
},
{
"name": "tensor/int32~9",
"type": "I32",
"shape": ["2", "3"],
"json_pointer": "/tensor~1int32~09",
"parameter_count": "6",
"data_offset": 244,
"file_offset": 1820,
"byte_length": 24
},
{
"name": "tensor/int64~11",
"type": "I64",
"shape": ["2", "3"],
"json_pointer": "/tensor~1int64~011",
"parameter_count": "6",
"data_offset": 48,
"file_offset": 1624,
"byte_length": 48
},
{
"name": "tensor/int8~2",
"type": "I8",
"shape": ["2", "3"],
"json_pointer": "/tensor~1int8~02",
"parameter_count": "6",
"data_offset": 334,
"file_offset": 1910,
"byte_length": 6
},
{
"name": "tensor/uint16~3",
"type": "U16",
"shape": ["2", "3"],
"json_pointer": "/tensor~1uint16~03",
"parameter_count": "6",
"data_offset": 292,
"file_offset": 1868,
"byte_length": 12
},
{
"name": "tensor/uint32~10",
"type": "U32",
"shape": ["2", "3"],
"json_pointer": "/tensor~1uint32~010",
"parameter_count": "6",
"data_offset": 220,
"file_offset": 1796,
"byte_length": 24
},
{
"name": "tensor/uint64~12",
"type": "U64",
"shape": ["2", "3"],
"json_pointer": "/tensor~1uint64~012",
"parameter_count": "6",
"data_offset": 0,
"file_offset": 1576,
"byte_length": 48
},
{
"name": "tensor/uint8~1",
"type": "U8",
"shape": ["2", "3"],
"json_pointer": "/tensor~1uint8~01",
"parameter_count": "6",
"data_offset": 340,
"file_offset": 1916,
"byte_length": 6
}
]
}
+81
View File
@@ -0,0 +1,81 @@
"""Produce an inert tensor-storage oracle with the pinned Rust implementation."""
import argparse
import base64
import hashlib
import json
import math
from pathlib import Path
import safetensors
VERSION = "0.7.0"
DTYPES = {
"bool": 1, "uint8": 1, "int8": 1,
"uint16": 2, "int16": 2, "bfloat16": 2, "float16": 2,
"float32": 4, "float64": 8, "int32": 4, "uint32": 4,
"int64": 8, "uint64": 8, "complex64": 8,
"float8_e4m3fn": 1, "float8_e5m2": 1, "float8_e8m0fnu": 1,
"float4_e2m1fn_x2": 1,
}
def produce():
if safetensors.__version__ != VERSION:
raise RuntimeError(f"requires safetensors=={VERSION}")
inputs = {}
for index, (dtype, width) in enumerate(DTYPES.items()):
# These are Python API dtype names, not REA's wire-type table. In
# particular the Rust producer expands the packed F4 final dimension.
inputs[f"tensor/{dtype}~{index}"] = {
"dtype": dtype, "shape": [2, 3],
"data": bytes((index * 17 + at * 13) % 256 for at in range(6 * width)),
}
inputs["scalar"] = {"dtype": "float32", "shape": [], "data": bytes([0, 0, 128, 63])}
inputs["empty"] = {"dtype": "bfloat16", "shape": [0, 7], "data": b""}
metadata = {"producer/~\ufeff": "Rust safetensors 0.7.0; literal \ufeff metadata"}
encoded = safetensors.serialize(inputs, metadata=metadata)
# Independently decode the exact emitted bytes with the official Rust
# reader; no REA fixture builder, parser or dtype table participates.
decoded = dict(safetensors.deserialize(encoded))
assert decoded.keys() == inputs.keys()
header_bytes = int.from_bytes(encoded[:8], "little")
header = json.loads(encoded[8:8 + header_bytes])
assert header.pop("__metadata__") == metadata
tensors = []
for name in sorted(decoded):
row = decoded[name]
assert bytes(row["data"]) == inputs[name]["data"]
assert row["dtype"] == header[name]["dtype"]
assert row["shape"] == header[name]["shape"]
start, end = header[name]["data_offsets"]
assert encoded[8 + header_bytes + start:8 + header_bytes + end] == row["data"]
tensors.append({
"name": name, "type": row["dtype"], "shape": [str(n) for n in row["shape"]],
"json_pointer": "/" + name.replace("~", "~0").replace("/", "~1"),
"parameter_count": str(math.prod(row["shape"])),
"data_offset": start, "file_offset": 8 + header_bytes + start,
"byte_length": len(row["data"]),
})
return {
"producer": f"safetensors=={VERSION} (Rust serialize + deserialize)",
"file_base64": base64.b64encode(encoded).decode(),
"file_size": len(encoded), "sha256": hashlib.sha256(encoded).hexdigest(),
"header_sha256": hashlib.sha256(encoded[:8 + header_bytes]).hexdigest(),
"tensor_data_offset": 8 + header_bytes,
"metadata": metadata, "tensors": tensors,
}
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--check", action="store_true")
args = parser.parse_args()
target = Path(__file__).with_name("oracle.json")
text = json.dumps(produce(), indent=2, ensure_ascii=True) + "\n"
if args.check:
if json.loads(target.read_text()) != json.loads(text):
raise SystemExit("safetensors producer oracle differs")
print("Pinned Rust producer and reader reproduce oracle.json exactly")
else:
target.write_text(text)
print(target)