mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
refactor(model): let format parsers own tensor range validation
This commit is contained in:
1 parent
f4768eb5d3
commit
2be39ef47a
3 files changed
+52
-10
No files matched your search
@@ -429,6 +429,15 @@ export class GgufHeader {
|
||||
storage_validation:
|
||||
tensor.byte_length === null ? "unknown" : "consistent",
|
||||
}));
|
||||
const intervals = completeTensors
|
||||
.filter((row) => row.byte_length !== null)
|
||||
.sort((a, b) => a.file_offset - b.file_offset);
|
||||
let end = tensor_data_offset;
|
||||
for (const row of intervals) {
|
||||
if (row.file_offset < end && row.byte_length !== 0)
|
||||
rejectModel(`Overlapping tensor byte ranges at ${row.name}`);
|
||||
end = Math.max(end, row.file_offset + (row.byte_length ?? 0));
|
||||
}
|
||||
return {
|
||||
version,
|
||||
byte_order: this.#little ? "little" : "big",
|
||||
|
||||
@@ -40,7 +40,6 @@ import { GgufHeader } from "./GgufHeader.js";
|
||||
import { readSafetensorsHeader } from "./SafetensorsHeader.js";
|
||||
import {
|
||||
ModelFormatFailure,
|
||||
rejectModel,
|
||||
rejectPickle,
|
||||
MODEL_REPORT_BYTES,
|
||||
modelCapacityRemediation,
|
||||
@@ -311,15 +310,6 @@ export class ModelArtifactProvider implements ModelArtifactPort {
|
||||
onLimitation: (message) => partial.limitations.push(message),
|
||||
})
|
||||
: await readSafetensorsHeader(reader, first);
|
||||
const intervals = parsed.tensors
|
||||
.filter((row) => row.byte_length !== null)
|
||||
.sort((a, b) => a.file_offset - b.file_offset);
|
||||
let end = parsed.tensor_data_offset;
|
||||
for (const row of intervals) {
|
||||
if (row.file_offset < end && row.byte_length !== 0)
|
||||
rejectModel(`Overlapping tensor byte ranges at ${row.name}`);
|
||||
end = Math.max(end, row.file_offset + (row.byte_length ?? 0));
|
||||
}
|
||||
partial.version = parsed.version;
|
||||
partial.byte_order = parsed.byte_order;
|
||||
partial.alignment = parsed.alignment;
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
import { readSafetensorsHeader } from "../../../src/model/SafetensorsHeader.js";
|
||||
import { GgufHeader } from "../../../src/model/GgufHeader.js";
|
||||
import { createModelArtifactService } from "../../../src/composition/model.js";
|
||||
import { parseEvidence } from "../../../src/domain/evidence.js";
|
||||
import { modelArtifactSchema } from "../../../src/domain/model/modelArtifact.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import {
|
||||
@@ -1063,6 +1064,48 @@ it.each(malformed)(
|
||||
},
|
||||
);
|
||||
|
||||
it("retains GGUF declarations and metadata when storage overlap is refused", async () => {
|
||||
const seed = ggufFixture();
|
||||
const fixture = ggufFixture({
|
||||
metadata: [{ key: "general.name", type: 8, value: seed.text("Overlap") }],
|
||||
tensors: [
|
||||
{ name: "a", type: 0, shape: [1n], offset: 0n, bytes: 4 },
|
||||
{ name: "b", type: 0, shape: [1n], offset: 0n, bytes: 4 },
|
||||
],
|
||||
});
|
||||
const { provider, path } = await providerAt(fixture.bytes);
|
||||
const result = await provider.inspect({ path, hash: "header" });
|
||||
if (result.ok) throw new Error("Overlapping tensors admitted");
|
||||
expect(result.error).toMatchObject({
|
||||
_tag: "BinaryTargetError",
|
||||
constraint: "file_format_rejected",
|
||||
});
|
||||
const evidence = parseEvidence(result.error.partialObservation);
|
||||
expect(evidence.subject).toBeNull();
|
||||
expect(evidence.normalized_result).toMatchObject({
|
||||
artifact: {
|
||||
header_sha256: null,
|
||||
sha256: null,
|
||||
file_identity: "unverified",
|
||||
},
|
||||
coverage: { header: "partial", metadata: "complete", tensors: "partial" },
|
||||
metadata: [{ key: "general.name", value: "Overlap" }],
|
||||
tensors: [],
|
||||
tensor_declarations: [
|
||||
{
|
||||
name: "a",
|
||||
file_offset: fixture.dataOffset,
|
||||
storage_validation: "not-established",
|
||||
},
|
||||
{
|
||||
name: "b",
|
||||
file_offset: fixture.dataOffset,
|
||||
storage_validation: "not-established",
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("accepts null safetensors metadata and ignores tensor extension fields", async () => {
|
||||
const fixture = safetensorsFixture(
|
||||
'{"__metadata__":null,"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":{"source":"local","flags":[null,false,-1,1.25]}}}',
|
||||
|
||||
Reference in new issue
Block a user