mirror of
https://github.com/morluto/rea.git
synced 2026-11-21 14:05:49 +00:00
refactor(model): let format parsers own tensor range validation
This commit is contained in:
1 parent
f4768eb5d3
commit
2be39ef47a
3 files changed
+52
-10
No files matched your search
@@ -429,6 +429,15 @@ export class GgufHeader {
|
||||
storage_validation:
|
||||
tensor.byte_length === null ? "unknown" : "consistent",
|
||||
}));
|
||||
const intervals = completeTensors
|
||||
.filter((row) => row.byte_length !== null)
|
||||
.sort((a, b) => a.file_offset - b.file_offset);
|
||||
let end = tensor_data_offset;
|
||||
for (const row of intervals) {
|
||||
if (row.file_offset < end && row.byte_length !== 0)
|
||||
rejectModel(`Overlapping tensor byte ranges at ${row.name}`);
|
||||
end = Math.max(end, row.file_offset + (row.byte_length ?? 0));
|
||||
}
|
||||
return {
|
||||
version,
|
||||
byte_order: this.#little ? "little" : "big",
|
||||
|
||||
@@ -40,7 +40,6 @@ import { GgufHeader } from "./GgufHeader.js";
|
||||
import { readSafetensorsHeader } from "./SafetensorsHeader.js";
|
||||
import {
|
||||
ModelFormatFailure,
|
||||
rejectModel,
|
||||
rejectPickle,
|
||||
MODEL_REPORT_BYTES,
|
||||
modelCapacityRemediation,
|
||||
@@ -311,15 +310,6 @@ export class ModelArtifactProvider implements ModelArtifactPort {
|
||||
onLimitation: (message) => partial.limitations.push(message),
|
||||
})
|
||||
: await readSafetensorsHeader(reader, first);
|
||||
const intervals = parsed.tensors
|
||||
.filter((row) => row.byte_length !== null)
|
||||
.sort((a, b) => a.file_offset - b.file_offset);
|
||||
let end = parsed.tensor_data_offset;
|
||||
for (const row of intervals) {
|
||||
if (row.file_offset < end && row.byte_length !== 0)
|
||||
rejectModel(`Overlapping tensor byte ranges at ${row.name}`);
|
||||
end = Math.max(end, row.file_offset + (row.byte_length ?? 0));
|
||||
}
|
||||
partial.version = parsed.version;
|
||||
partial.byte_order = parsed.byte_order;
|
||||
partial.alignment = parsed.alignment;
|
||||
|
||||
Reference in new issue
Block a user