From 6825ce707aebc436c3c5fdcb491fd84b25995ac1 Mon Sep 17 00:00:00 2001 From: ShuaixinHuang <1836656514@qq.com> Date: Thu, 19 Nov 2026 13:12:55 +0800 Subject: [PATCH] test(model): verify safetensors ranges with Rust producer fixtures (#12030) test(model): qualify safetensors storage with a Rust oracle Co-authored-by: ShuaixinHuang <197200901+ShuaixinHuang@users.noreply.github.com> --- .../modelSafetensorsProducer.test.ts | 139 ++++++++++++ tests/fixtures/safetensors-producer/README.md | 41 ++++ .../fixtures/safetensors-producer/oracle.json | 213 ++++++++++++++++++ .../fixtures/safetensors-producer/produce.py | 81 +++++++ 4 files changed, 474 insertions(+) create mode 100644 tests/acceptance/modelSafetensorsProducer.test.ts create mode 100644 tests/fixtures/safetensors-producer/README.md create mode 100644 tests/fixtures/safetensors-producer/oracle.json create mode 100644 tests/fixtures/safetensors-producer/produce.py diff --git a/tests/acceptance/modelSafetensorsProducer.test.ts b/tests/acceptance/modelSafetensorsProducer.test.ts new file mode 100644 index 0000000000..47a20e3378 --- /dev/null +++ b/tests/acceptance/modelSafetensorsProducer.test.ts @@ -0,0 +1,139 @@ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; +import { z } from "zod"; + +import { INSPECT_MODEL_ARTIFACT_CONTRACT } from "../../src/contracts/model/modelToolContracts.js"; +import { modelArtifactSchema } from "../../src/domain/model/modelArtifact.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +const oracle = z + .object({ + file_base64: z.string(), + file_size: z.int(), + sha256: z.string(), + header_sha256: z.string(), + tensor_data_offset: z.int(), + metadata: z.record(z.string(), z.string()), + tensors: z.array( + z.object({ + name: z.string(), + type: z.string(), + shape: z.array(z.string()), + parameter_count: z.string(), + json_pointer: z.string(), + data_offset: z.int(), + file_offset: z.int(), + byte_length: z.int(), + }), + ), + }) + .parse( + JSON.parse( + await readFile( + new URL( + "../fixtures/safetensors-producer/oracle.json", + import.meta.url, + ), + "utf8", + ), + ), + ); + +cliTest( + "preserves the Rust safetensors producer's mixed tensor storage and provenance", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-safetensors-producer-"); + const path = join(root, "producer.safetensors"); + // The complete bytes are emitted by Rust serialize and independently read by + // Rust deserialize. Do not rebuild this carrier with REA's fixture writer. + const bytes = Buffer.from(oracle.file_base64, "base64"); + expect(bytes.length).toBe(oracle.file_size); + expect(createHash("sha256").update(bytes).digest("hex")).toBe( + oracle.sha256, + ); + expect(oracle.tensors).toHaveLength(20); + expect(new Set(oracle.tensors.map(({ type }) => type)).size).toBe(18); + await writeFile(path, bytes); + for (const hash of ["header", "full"] as const) { + const result = await cli.run({ + arguments: [ + "inspect-model-artifact", + path, + "--detail", + "complete", + "--hash", + hash, + "--json", + ], + }); + expect(result.exitCode, result.stdout + result.stderr).toBe(0); + const report = modelArtifactSchema.parse( + INSPECT_MODEL_ARTIFACT_CONTRACT.outputSchema.parse(result.json) + .normalized_result, + ); + expect(report.artifact).toMatchObject({ + path, + file_size: oracle.file_size, + header_sha256: oracle.header_sha256, + sha256: hash === "full" ? oracle.sha256 : null, + }); + expect(report).toMatchObject({ + format: "safetensors", + tensor_data_offset: oracle.tensor_data_offset, + coverage: { + header: "complete", + tensor_data: "not-interpreted", + tensor_ranges: "complete", + storage_sizes: "complete", + }, + }); + expect(report.metadata.map(({ key, value }) => [key, value])).toEqual( + Object.entries(oracle.metadata), + ); + expect( + report.tensors.map( + ({ + name, + type, + shape, + parameter_count, + json_pointer, + data_offset, + file_offset, + byte_length, + }) => ({ + name, + type, + shape, + parameter_count, + json_pointer, + data_offset, + file_offset, + byte_length, + }), + ), + ).toEqual(oracle.tensors); + for (const tensor of report.tensors) { + expect(tensor).toMatchObject({ + source_offset: 8, + source_length: oracle.tensor_data_offset - 8, + byte_length_state: "declared", + storage_validation: "consistent", + }); + } + expect(report.totals).toMatchObject({ + parameter_count: oracle.tensors + .reduce((sum, tensor) => sum + BigInt(tensor.parameter_count), 0n) + .toString(), + tensor_bytes: oracle.tensors.reduce( + (sum, tensor) => sum + tensor.byte_length, + 0, + ), + }); + } + expect((await readFile(path)).equals(bytes)).toBe(true); + }, +); diff --git a/tests/fixtures/safetensors-producer/README.md b/tests/fixtures/safetensors-producer/README.md new file mode 100644 index 0000000000..07e507e130 --- /dev/null +++ b/tests/fixtures/safetensors-producer/README.md @@ -0,0 +1,41 @@ +# Safetensors producer storage oracle + +`oracle.json` contains the complete inert bytes emitted by **safetensors 0.7.0** +([released source](https://github.com/huggingface/safetensors/tree/v0.7.0)), encoded +as base64 so no binary fixture is committed. Its expected declarations come from +the package's Rust `deserialize`, independently of REA's JSON reader and dtype +table. Python's JSON reader records the emitted wire offsets; each span is also +checked against the bytes returned by Rust. The producer verifies every decoded +payload against its self-authored input, without interpreting model behavior. + +One 1,934-byte model contains 20 tensors and 18 emitted dtypes: integer, boolean, +bfloat16, float16/32/64, complex64, three float8 forms and packed F4, plus a scalar +and an empty tensor. The Python API's `float4_e2m1fn_x2` input shape `[2, 3]` +becomes wire dtype `F4`, shape `[2, 6]` and six bytes. That result comes from the +producer, not a transcription of REA's bit-width mapping. Literal metadata and +names containing JSON Pointer delimiters qualify provenance alongside storage. + +The compiled CLI regression checks both header-only and full-file hash modes, +all tensor declarations and physical ranges, exact metadata, totals, and the +existing distinction between consistent storage and uninterpreted tensor data. +CI runs the recorded bytes without Python, NumPy, Torch or model weights. + +Reproduce with a user-owned Python environment: + +```sh +python3 -m venv /tmp/rea-safetensors-oracle +/tmp/rea-safetensors-oracle/bin/pip install safetensors==0.7.0 +/tmp/rea-safetensors-oracle/bin/python tests/fixtures/safetensors-producer/produce.py --check +``` + +Omit `--check` to regenerate intentionally, then format the JSON with the +repository formatter and review the changed bytes and facts. `--check` compares +JSON values, including the exact encoded file bytes, rather than JSON whitespace. + +Current production is correct for this sample. On main `da91bbd4e`, temporarily +changing `F8_E8M0: 8` to `F8_E8M0: 16` in `ModelTensorTypes.ts` left all **161 tests +in the four existing model boundary/CLI suites passing**. This new public CLI +regression rejects the mutated behavior: the valid producer tensor is reported +as having a mismatched byte length. The mutation was restored byte-for-byte +before final checks. This qualifies a missing regression guard, not a production +bug fix, model inference, tensor values, GPU execution or every future dtype. diff --git a/tests/fixtures/safetensors-producer/oracle.json b/tests/fixtures/safetensors-producer/oracle.json new file mode 100644 index 0000000000..2f170db6af --- /dev/null +++ b/tests/fixtures/safetensors-producer/oracle.json @@ -0,0 +1,213 @@ +{ + "producer": "safetensors==0.7.0 (Rust serialize + deserialize)", + "file_base64": "IAYAAAAAAAB7Il9fbWV0YWRhdGFfXyI6eyJwcm9kdWNlci9+77u/IjoiUnVzdCBzYWZldGVuc29ycyAwLjcuMDsgbGl0ZXJhbCDvu78gbWV0YWRhdGEifSwidGVuc29yL3VpbnQ2NH4xMiI6eyJkdHlwZSI6IlU2NCIsInNoYXBlIjpbMiwzXSwiZGF0YV9vZmZzZXRzIjpbMCw0OF19LCJ0ZW5zb3IvaW50NjR+MTEiOnsiZHR5cGUiOiJJNjQiLCJzaGFwZSI6WzIsM10sImRhdGFfb2Zmc2V0cyI6WzQ4LDk2XX0sInRlbnNvci9mbG9hdDY0fjgiOnsiZHR5cGUiOiJGNjQiLCJzaGFwZSI6WzIsM10sImRhdGFfb2Zmc2V0cyI6Wzk2LDE0NF19LCJ0ZW5zb3IvY29tcGxleDY0fjEzIjp7ImR0eXBlIjoiQzY0Iiwic2hhcGUiOlsyLDNdLCJkYXRhX29mZnNldHMiOlsxNDQsMTkyXX0sInNjYWxhciI6eyJkdHlwZSI6IkYzMiIsInNoYXBlIjpbXSwiZGF0YV9vZmZzZXRzIjpbMTkyLDE5Nl19LCJ0ZW5zb3IvZmxvYXQzMn43Ijp7ImR0eXBlIjoiRjMyIiwic2hhcGUiOlsyLDNdLCJkYXRhX29mZnNldHMiOlsxOTYsMjIwXX0sInRlbnNvci91aW50MzJ+MTAiOnsiZHR5cGUiOiJVMzIiLCJzaGFwZSI6WzIsM10sImRhdGFfb2Zmc2V0cyI6WzIyMCwyNDRdfSwidGVuc29yL2ludDMyfjkiOnsiZHR5cGUiOiJJMzIiLCJzaGFwZSI6WzIsM10sImRhdGFfb2Zmc2V0cyI6WzI0NCwyNjhdfSwiZW1wdHkiOnsiZHR5cGUiOiJCRjE2Iiwic2hhcGUiOlswLDddLCJkYXRhX29mZnNldHMiOlsyNjgsMjY4XX0sInRlbnNvci9iZmxvYXQxNn41Ijp7ImR0eXBlIjoiQkYxNiIsInNoYXBlIjpbMiwzXSwiZGF0YV9vZmZzZXRzIjpbMjY4LDI4MF19LCJ0ZW5zb3IvZmxvYXQxNn42Ijp7ImR0eXBlIjoiRjE2Iiwic2hhcGUiOlsyLDNdLCJkYXRhX29mZnNldHMiOlsyODAsMjkyXX0sInRlbnNvci91aW50MTZ+MyI6eyJkdHlwZSI6IlUxNiIsInNoYXBlIjpbMiwzXSwiZGF0YV9vZmZzZXRzIjpbMjkyLDMwNF19LCJ0ZW5zb3IvaW50MTZ+NCI6eyJkdHlwZSI6IkkxNiIsInNoYXBlIjpbMiwzXSwiZGF0YV9vZmZzZXRzIjpbMzA0LDMxNl19LCJ0ZW5zb3IvZmxvYXQ4X2U4bTBmbnV+MTYiOnsiZHR5cGUiOiJGOF9FOE0wIiwic2hhcGUiOlsyLDNdLCJkYXRhX29mZnNldHMiOlszMTYsMzIyXX0sInRlbnNvci9mbG9hdDhfZTRtM2ZufjE0Ijp7ImR0eXBlIjoiRjhfRTRNMyIsInNoYXBlIjpbMiwzXSwiZGF0YV9vZmZzZXRzIjpbMzIyLDMyOF19LCJ0ZW5zb3IvZmxvYXQ4X2U1bTJ+MTUiOnsiZHR5cGUiOiJGOF9FNU0yIiwic2hhcGUiOlsyLDNdLCJkYXRhX29mZnNldHMiOlszMjgsMzM0XX0sInRlbnNvci9pbnQ4fjIiOnsiZHR5cGUiOiJJOCIsInNoYXBlIjpbMiwzXSwiZGF0YV9vZmZzZXRzIjpbMzM0LDM0MF19LCJ0ZW5zb3IvdWludDh+MSI6eyJkdHlwZSI6IlU4Iiwic2hhcGUiOlsyLDNdLCJkYXRhX29mZnNldHMiOlszNDAsMzQ2XX0sInRlbnNvci9mbG9hdDRfZTJtMWZuX3gyfjE3Ijp7ImR0eXBlIjoiRjQiLCJzaGFwZSI6WzIsNl0sImRhdGFfb2Zmc2V0cyI6WzM0NiwzNTJdfSwidGVuc29yL2Jvb2x+MCI6eyJkdHlwZSI6IkJPT0wiLCJzaGFwZSI6WzIsM10sImRhdGFfb2Zmc2V0cyI6WzM1MiwzNThdfX0gICAgIMzZ5vMADRonNEFOW2h1go+cqbbD0N3q9wQRHis4RVJfbHmGk6CtusfU4e77CBUiL7vI1eLv/AkWIzA9SldkcX6LmKWyv8zZ5vMADRonNEFOW2h1go+cqbbD0N3q9wQRHoiVoq+8ydbj8P0KFyQxPktYZXJ/jJmms8DN2uf0AQ4bKDVCT1xpdoOQnaq3xNHe693q9wQRHis4RVJfbHmGk6CtusfU4e77CBUiLzxJVmNwfYqXpLG+y9jl8v8MGSYzQAAAgD93hJGeq7jF0t/s+QYTIC06R1RhbnuIlaKqt8TR3uv4BRIfLDlGU2BteoeUoa67yNWZprPAzdrn9AEOGyg1Qk9caXaDkJ2qt8RVYm98iZajsL3K1+Rmc4CNmqe0wc7b6PUzQE1aZ3SBjpuotcJEUV5reIWSn6y5xtMQHSo3RFHu+wgVIi//DBkmM0AiLzxJVmMRHis4RVIhLjtIVWIADRonNEE=", + "file_size": 1934, + "sha256": "5fb44d541cd95db65982fb252a957a0b4acc6910a964ec0157af5edc8947a6a4", + "header_sha256": "14d0e34758aecc59cb692a79bd55032fbfd71acffbb3e63741ea943e8c7a7fd7", + "tensor_data_offset": 1576, + "metadata": { + "producer/~\ufeff": "Rust safetensors 0.7.0; literal \ufeff metadata" + }, + "tensors": [ + { + "name": "empty", + "type": "BF16", + "shape": ["0", "7"], + "json_pointer": "/empty", + "parameter_count": "0", + "data_offset": 268, + "file_offset": 1844, + "byte_length": 0 + }, + { + "name": "scalar", + "type": "F32", + "shape": [], + "json_pointer": "/scalar", + "parameter_count": "1", + "data_offset": 192, + "file_offset": 1768, + "byte_length": 4 + }, + { + "name": "tensor/bfloat16~5", + "type": "BF16", + "shape": ["2", "3"], + "json_pointer": "/tensor~1bfloat16~05", + "parameter_count": "6", + "data_offset": 268, + "file_offset": 1844, + "byte_length": 12 + }, + { + "name": "tensor/bool~0", + "type": "BOOL", + "shape": ["2", "3"], + "json_pointer": "/tensor~1bool~00", + "parameter_count": "6", + "data_offset": 352, + "file_offset": 1928, + "byte_length": 6 + }, + { + "name": "tensor/complex64~13", + "type": "C64", + "shape": ["2", "3"], + "json_pointer": "/tensor~1complex64~013", + "parameter_count": "6", + "data_offset": 144, + "file_offset": 1720, + "byte_length": 48 + }, + { + "name": "tensor/float16~6", + "type": "F16", + "shape": ["2", "3"], + "json_pointer": "/tensor~1float16~06", + "parameter_count": "6", + "data_offset": 280, + "file_offset": 1856, + "byte_length": 12 + }, + { + "name": "tensor/float32~7", + "type": "F32", + "shape": ["2", "3"], + "json_pointer": "/tensor~1float32~07", + "parameter_count": "6", + "data_offset": 196, + "file_offset": 1772, + "byte_length": 24 + }, + { + "name": "tensor/float4_e2m1fn_x2~17", + "type": "F4", + "shape": ["2", "6"], + "json_pointer": "/tensor~1float4_e2m1fn_x2~017", + "parameter_count": "12", + "data_offset": 346, + "file_offset": 1922, + "byte_length": 6 + }, + { + "name": "tensor/float64~8", + "type": "F64", + "shape": ["2", "3"], + "json_pointer": "/tensor~1float64~08", + "parameter_count": "6", + "data_offset": 96, + "file_offset": 1672, + "byte_length": 48 + }, + { + "name": "tensor/float8_e4m3fn~14", + "type": "F8_E4M3", + "shape": ["2", "3"], + "json_pointer": "/tensor~1float8_e4m3fn~014", + "parameter_count": "6", + "data_offset": 322, + "file_offset": 1898, + "byte_length": 6 + }, + { + "name": "tensor/float8_e5m2~15", + "type": "F8_E5M2", + "shape": ["2", "3"], + "json_pointer": "/tensor~1float8_e5m2~015", + "parameter_count": "6", + "data_offset": 328, + "file_offset": 1904, + "byte_length": 6 + }, + { + "name": "tensor/float8_e8m0fnu~16", + "type": "F8_E8M0", + "shape": ["2", "3"], + "json_pointer": "/tensor~1float8_e8m0fnu~016", + "parameter_count": "6", + "data_offset": 316, + "file_offset": 1892, + "byte_length": 6 + }, + { + "name": "tensor/int16~4", + "type": "I16", + "shape": ["2", "3"], + "json_pointer": "/tensor~1int16~04", + "parameter_count": "6", + "data_offset": 304, + "file_offset": 1880, + "byte_length": 12 + }, + { + "name": "tensor/int32~9", + "type": "I32", + "shape": ["2", "3"], + "json_pointer": "/tensor~1int32~09", + "parameter_count": "6", + "data_offset": 244, + "file_offset": 1820, + "byte_length": 24 + }, + { + "name": "tensor/int64~11", + "type": "I64", + "shape": ["2", "3"], + "json_pointer": "/tensor~1int64~011", + "parameter_count": "6", + "data_offset": 48, + "file_offset": 1624, + "byte_length": 48 + }, + { + "name": "tensor/int8~2", + "type": "I8", + "shape": ["2", "3"], + "json_pointer": "/tensor~1int8~02", + "parameter_count": "6", + "data_offset": 334, + "file_offset": 1910, + "byte_length": 6 + }, + { + "name": "tensor/uint16~3", + "type": "U16", + "shape": ["2", "3"], + "json_pointer": "/tensor~1uint16~03", + "parameter_count": "6", + "data_offset": 292, + "file_offset": 1868, + "byte_length": 12 + }, + { + "name": "tensor/uint32~10", + "type": "U32", + "shape": ["2", "3"], + "json_pointer": "/tensor~1uint32~010", + "parameter_count": "6", + "data_offset": 220, + "file_offset": 1796, + "byte_length": 24 + }, + { + "name": "tensor/uint64~12", + "type": "U64", + "shape": ["2", "3"], + "json_pointer": "/tensor~1uint64~012", + "parameter_count": "6", + "data_offset": 0, + "file_offset": 1576, + "byte_length": 48 + }, + { + "name": "tensor/uint8~1", + "type": "U8", + "shape": ["2", "3"], + "json_pointer": "/tensor~1uint8~01", + "parameter_count": "6", + "data_offset": 340, + "file_offset": 1916, + "byte_length": 6 + } + ] +} diff --git a/tests/fixtures/safetensors-producer/produce.py b/tests/fixtures/safetensors-producer/produce.py new file mode 100644 index 0000000000..f07a27ab32 --- /dev/null +++ b/tests/fixtures/safetensors-producer/produce.py @@ -0,0 +1,81 @@ +"""Produce an inert tensor-storage oracle with the pinned Rust implementation.""" +import argparse +import base64 +import hashlib +import json +import math +from pathlib import Path + +import safetensors + +VERSION = "0.7.0" +DTYPES = { + "bool": 1, "uint8": 1, "int8": 1, + "uint16": 2, "int16": 2, "bfloat16": 2, "float16": 2, + "float32": 4, "float64": 8, "int32": 4, "uint32": 4, + "int64": 8, "uint64": 8, "complex64": 8, + "float8_e4m3fn": 1, "float8_e5m2": 1, "float8_e8m0fnu": 1, + "float4_e2m1fn_x2": 1, +} + + +def produce(): + if safetensors.__version__ != VERSION: + raise RuntimeError(f"requires safetensors=={VERSION}") + inputs = {} + for index, (dtype, width) in enumerate(DTYPES.items()): + # These are Python API dtype names, not REA's wire-type table. In + # particular the Rust producer expands the packed F4 final dimension. + inputs[f"tensor/{dtype}~{index}"] = { + "dtype": dtype, "shape": [2, 3], + "data": bytes((index * 17 + at * 13) % 256 for at in range(6 * width)), + } + inputs["scalar"] = {"dtype": "float32", "shape": [], "data": bytes([0, 0, 128, 63])} + inputs["empty"] = {"dtype": "bfloat16", "shape": [0, 7], "data": b""} + metadata = {"producer/~\ufeff": "Rust safetensors 0.7.0; literal \ufeff metadata"} + encoded = safetensors.serialize(inputs, metadata=metadata) + # Independently decode the exact emitted bytes with the official Rust + # reader; no REA fixture builder, parser or dtype table participates. + decoded = dict(safetensors.deserialize(encoded)) + assert decoded.keys() == inputs.keys() + header_bytes = int.from_bytes(encoded[:8], "little") + header = json.loads(encoded[8:8 + header_bytes]) + assert header.pop("__metadata__") == metadata + tensors = [] + for name in sorted(decoded): + row = decoded[name] + assert bytes(row["data"]) == inputs[name]["data"] + assert row["dtype"] == header[name]["dtype"] + assert row["shape"] == header[name]["shape"] + start, end = header[name]["data_offsets"] + assert encoded[8 + header_bytes + start:8 + header_bytes + end] == row["data"] + tensors.append({ + "name": name, "type": row["dtype"], "shape": [str(n) for n in row["shape"]], + "json_pointer": "/" + name.replace("~", "~0").replace("/", "~1"), + "parameter_count": str(math.prod(row["shape"])), + "data_offset": start, "file_offset": 8 + header_bytes + start, + "byte_length": len(row["data"]), + }) + return { + "producer": f"safetensors=={VERSION} (Rust serialize + deserialize)", + "file_base64": base64.b64encode(encoded).decode(), + "file_size": len(encoded), "sha256": hashlib.sha256(encoded).hexdigest(), + "header_sha256": hashlib.sha256(encoded[:8 + header_bytes]).hexdigest(), + "tensor_data_offset": 8 + header_bytes, + "metadata": metadata, "tensors": tensors, + } + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + target = Path(__file__).with_name("oracle.json") + text = json.dumps(produce(), indent=2, ensure_ascii=True) + "\n" + if args.check: + if json.loads(target.read_text()) != json.loads(text): + raise SystemExit("safetensors producer oracle differs") + print("Pinned Rust producer and reader reproduce oracle.json exactly") + else: + target.write_text(text) + print(target)