refactor(model): let format parsers own tensor range validation

This commit is contained in:
morluto committed 2026-10-26 14:33:35 +08:00
1 parent f4768eb5d3
commit 2be39ef47a
3 files changed
+52 -10

No files matched your search

+9
View File
@@ -429,6 +429,15 @@ export class GgufHeader {
storage_validation:
tensor.byte_length === null ? "unknown" : "consistent",
}));
const intervals = completeTensors
.filter((row) => row.byte_length !== null)
.sort((a, b) => a.file_offset - b.file_offset);
let end = tensor_data_offset;
for (const row of intervals) {
if (row.file_offset < end && row.byte_length !== 0)
rejectModel(`Overlapping tensor byte ranges at ${row.name}`);
end = Math.max(end, row.file_offset + (row.byte_length ?? 0));
}
return {
version,
byte_order: this.#little ? "little" : "big",
-10
View File
@@ -40,7 +40,6 @@ import { GgufHeader } from "./GgufHeader.js";
import { readSafetensorsHeader } from "./SafetensorsHeader.js";
import {
ModelFormatFailure,
rejectModel,
rejectPickle,
MODEL_REPORT_BYTES,
modelCapacityRemediation,
@@ -311,15 +310,6 @@ export class ModelArtifactProvider implements ModelArtifactPort {
onLimitation: (message) => partial.limitations.push(message),
})
: await readSafetensorsHeader(reader, first);
const intervals = parsed.tensors
.filter((row) => row.byte_length !== null)
.sort((a, b) => a.file_offset - b.file_offset);
let end = parsed.tensor_data_offset;
for (const row of intervals) {
if (row.file_offset < end && row.byte_length !== 0)
rejectModel(`Overlapping tensor byte ranges at ${row.name}`);
end = Math.max(end, row.file_offset + (row.byte_length ?? 0));
}
partial.version = parsed.version;
partial.byte_order = parsed.byte_order;
partial.alignment = parsed.alignment;
@@ -15,6 +15,7 @@ import {
import { readSafetensorsHeader } from "../../../src/model/SafetensorsHeader.js";
import { GgufHeader } from "../../../src/model/GgufHeader.js";
import { createModelArtifactService } from "../../../src/composition/model.js";
import { parseEvidence } from "../../../src/domain/evidence.js";
import { modelArtifactSchema } from "../../../src/domain/model/modelArtifact.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import {
@@ -1063,6 +1064,48 @@ it.each(malformed)(
},
);
it("retains GGUF declarations and metadata when storage overlap is refused", async () => {
const seed = ggufFixture();
const fixture = ggufFixture({
metadata: [{ key: "general.name", type: 8, value: seed.text("Overlap") }],
tensors: [
{ name: "a", type: 0, shape: [1n], offset: 0n, bytes: 4 },
{ name: "b", type: 0, shape: [1n], offset: 0n, bytes: 4 },
],
});
const { provider, path } = await providerAt(fixture.bytes);
const result = await provider.inspect({ path, hash: "header" });
if (result.ok) throw new Error("Overlapping tensors admitted");
expect(result.error).toMatchObject({
_tag: "BinaryTargetError",
constraint: "file_format_rejected",
});
const evidence = parseEvidence(result.error.partialObservation);
expect(evidence.subject).toBeNull();
expect(evidence.normalized_result).toMatchObject({
artifact: {
header_sha256: null,
sha256: null,
file_identity: "unverified",
},
coverage: { header: "partial", metadata: "complete", tensors: "partial" },
metadata: [{ key: "general.name", value: "Overlap" }],
tensors: [],
tensor_declarations: [
{
name: "a",
file_offset: fixture.dataOffset,
storage_validation: "not-established",
},
{
name: "b",
file_offset: fixture.dataOffset,
storage_validation: "not-established",
},
],
});
});
it("accepts null safetensors metadata and ignores tensor extension fields", async () => {
const fixture = safetensorsFixture(
'{"__metadata__":null,"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":{"source":"local","flags":[null,false,-1,1.25]}}}',