mirror of
https://github.com/morluto/rea.git
synced 2026-11-20 13:45:30 +00:00
1845 lines
61 KiB
TypeScript
1845 lines
61 KiB
TypeScript
import { fileIdentity } from "../../fixtures/evidenceSubject.js";
|
||
import { createHash } from "node:crypto";
|
||
import { open, writeFile, symlink, truncate } from "node:fs/promises";
|
||
import { join } from "node:path";
|
||
import { expect, it, onTestFinished } from "vitest";
|
||
import { ModelArtifactProvider } from "../../../src/model/ModelArtifactProvider.js";
|
||
import { ModelHeaderReader } from "../../../src/model/ModelHeaderReader.js";
|
||
import {
|
||
MODEL_JSON_NESTING,
|
||
MODEL_METADATA_NESTING,
|
||
MODEL_REPORT_BYTES,
|
||
modelCapacityConstraint,
|
||
modelCapacityFailure,
|
||
} from "../../../src/model/ModelFormat.js";
|
||
import { readSafetensorsHeader } from "../../../src/model/SafetensorsHeader.js";
|
||
import { ParseFailureError } from "../../../src/domain/parseFailure.js";
|
||
import { GgufHeader } from "../../../src/model/GgufHeader.js";
|
||
import { createModelArtifactService } from "../../../src/composition/model.js";
|
||
import { parseEvidence } from "../../../src/domain/evidence.js";
|
||
import { modelArtifactSchema } from "../../../src/domain/model/modelArtifact.js";
|
||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||
import {
|
||
ggufFixture,
|
||
ggufKLayouts,
|
||
ggufShiftedTypeLayouts,
|
||
safetensorsFixture,
|
||
} from "../../fixtures/model/model.js";
|
||
|
||
/** A malformed model header is an invalid `path`, like every format failure. */
|
||
const INVALID_FORMAT = {
|
||
_tag: "AnalysisInputError",
|
||
issues: [
|
||
expect.objectContaining({ path: ["path"], reason: "invalid_format" }),
|
||
],
|
||
} as const;
|
||
|
||
const hash = (bytes: Buffer) =>
|
||
createHash("sha256").update(bytes).digest("hex");
|
||
const providerAt = async (bytes: Buffer) => {
|
||
const root = await createTestTempDirectory("rea-model-reader-");
|
||
const path = join(root, "content.without-format-extension");
|
||
await writeFile(path, bytes);
|
||
const provider = new ModelArtifactProvider();
|
||
onTestFinished(() => provider.close());
|
||
return { provider, path, root };
|
||
};
|
||
|
||
it.each([
|
||
["01000000", 1, 16_777_216],
|
||
["00000001", 16_777_216, 1],
|
||
["04000000", 4, 67_108_864],
|
||
["00000004", 67_108_864, 4],
|
||
["e7030000", 999, 3_875_733_504],
|
||
["000003e7", 3_875_733_504, 999],
|
||
["00000000", 0, 0],
|
||
["ffffffff", 4_294_967_295, 4_294_967_295],
|
||
] as const)(
|
||
"preserves unsupported GGUF version bytes %s without assuming byte order",
|
||
async (versionBytes, little, big) => {
|
||
// No counts are needed to explain a version refusal. Do not reinterpret
|
||
// an unknown version's subsequent layout or publish a header observation.
|
||
const { provider, path } = await providerAt(
|
||
Buffer.from(`47475546${versionBytes}`, "hex"),
|
||
);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (result.ok) throw new Error("Unknown version admitted");
|
||
expect(result.error).toMatchObject({
|
||
_tag: "AnalysisUnsupportedTargetError",
|
||
reason: `GGUF version bytes ${versionBytes} are unsupported (little-endian ${String(little)}, big-endian ${String(big)}; byte order unknown); supported versions are 2 and 3`,
|
||
});
|
||
},
|
||
);
|
||
|
||
it("continues reading little-endian GGUF v2 declarations", async () => {
|
||
const { provider, path } = await providerAt(
|
||
ggufFixture({ version: 2 }).bytes,
|
||
);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value).toMatchObject({ version: 2, byte_order: "little" });
|
||
expect(result.value.tensors.map(({ name }) => name)).toEqual([
|
||
"weight",
|
||
"quant",
|
||
]);
|
||
});
|
||
|
||
it("continues refusing the unsupported big-endian GGUF v2 layout", async () => {
|
||
const { provider, path } = await providerAt(
|
||
ggufFixture({ version: 2, bigEndian: true }).bytes,
|
||
);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (result.ok) throw new Error("Big-endian GGUF v2 admitted");
|
||
expect(result.error).toMatchObject({
|
||
...INVALID_FORMAT,
|
||
});
|
||
});
|
||
|
||
it.each([false, true])(
|
||
"reads exact GGUF declarations, endian=%s, without hashing tensor payload",
|
||
async (bigEndian) => {
|
||
const fixture = ggufFixture({ bigEndian });
|
||
const { provider, path } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value.artifact).toEqual({
|
||
path,
|
||
file_size: fixture.bytes.length,
|
||
header_sha256: hash(fixture.header),
|
||
header_length: fixture.header.length,
|
||
sha256: null,
|
||
bytes_read: fixture.header.length,
|
||
read_scope: "header",
|
||
});
|
||
expect(result.value.byte_order).toBe(bigEndian ? "big" : "little");
|
||
expect(
|
||
result.value.metadata.find((row) => row.key === "custom.exact")?.value,
|
||
).toBe("18446744073709551615");
|
||
expect(
|
||
result.value.metadata.find((row) => row.key === "tokenizer.ggml.tokens"),
|
||
).toMatchObject({
|
||
element_type: "STRING",
|
||
element_count: 10,
|
||
preview_complete: false,
|
||
value: Array.from({ length: 8 }, (_, n) => `token-${String(n)}`),
|
||
});
|
||
expect(
|
||
result.value.tensors.map(({ name, shape, byte_length, file_offset }) => ({
|
||
name,
|
||
shape,
|
||
byte_length,
|
||
file_offset,
|
||
})),
|
||
).toEqual([
|
||
{
|
||
name: "weight",
|
||
shape: ["2", "2"],
|
||
byte_length: 16,
|
||
file_offset: fixture.dataOffset,
|
||
},
|
||
{
|
||
name: "quant",
|
||
shape: ["32"],
|
||
byte_length: 18,
|
||
file_offset: fixture.dataOffset + 32,
|
||
},
|
||
]);
|
||
expect(result.value.totals).toMatchObject({
|
||
parameter_count: "36",
|
||
tensor_bytes: 34,
|
||
});
|
||
const full = await provider.inspect({ path, hash: "full" });
|
||
if (!full.ok) throw full.error;
|
||
expect(full.value.artifact.sha256).toBe(hash(fixture.bytes));
|
||
expect(full.value.artifact.header_sha256).toBe(
|
||
result.value.artifact.header_sha256,
|
||
);
|
||
},
|
||
);
|
||
|
||
it.each([false, true])(
|
||
"preserves GGUF empty extents, quantized empty rows and scalars, endian=%s",
|
||
async (bigEndian) => {
|
||
const declarations = [
|
||
{ name: "empty_first", type: 0, shape: [0n, 7n], offset: 0n, bytes: 0 },
|
||
{ name: "empty_later", type: 0, shape: [3n, 0n], offset: 0n, bytes: 0 },
|
||
{
|
||
name: "empty_quant_first",
|
||
type: 2,
|
||
shape: [0n, 32n],
|
||
offset: 0n,
|
||
bytes: 0,
|
||
},
|
||
{
|
||
name: "empty_quant_later",
|
||
type: 2,
|
||
shape: [32n, 0n],
|
||
offset: 0n,
|
||
bytes: 0,
|
||
},
|
||
{ name: "scalar", type: 0, shape: [], offset: 0n, bytes: 4 },
|
||
];
|
||
const fixture = ggufFixture({ bigEndian, tensors: declarations });
|
||
const { provider, path } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(
|
||
result.value.tensors.map(
|
||
({ name, shape, byte_length, parameter_count }) => ({
|
||
name,
|
||
shape,
|
||
byte_length,
|
||
parameter_count,
|
||
}),
|
||
),
|
||
).toEqual(
|
||
declarations.map(({ name, shape, bytes }) => ({
|
||
name,
|
||
shape: shape.map(String),
|
||
byte_length: bytes,
|
||
parameter_count: name === "scalar" ? "1" : "0",
|
||
})),
|
||
);
|
||
expect(result.value.totals).toMatchObject({
|
||
parameter_count: "1",
|
||
tensor_bytes: 4,
|
||
});
|
||
expect(result.value.artifact.bytes_read).toBe(fixture.header.length);
|
||
},
|
||
);
|
||
|
||
it.each([
|
||
{ name: "excess rank", type: 0, shape: [1n, 1n, 1n, 1n, 1n], bytes: 4 },
|
||
{ name: "quantized scalar", type: 2, shape: [], bytes: 0 },
|
||
{ name: "Q8_1 incomplete block", type: 9, shape: [31n], bytes: 40 },
|
||
])(
|
||
"rejects GGUF $name without relaxing representation constraints",
|
||
async (tensor) => {
|
||
const fixture = ggufFixture({ tensors: [{ ...tensor, offset: 0n }] });
|
||
const { provider, path } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (result.ok) throw new Error("Expected format refusal");
|
||
expect(result.error).toMatchObject({
|
||
...INVALID_FORMAT,
|
||
});
|
||
},
|
||
);
|
||
|
||
it.each([8, 24, 40])(
|
||
"accepts format-valid GGUF alignment %s and preserves exact offsets",
|
||
async (alignment) => {
|
||
const fixture = ggufFixture({ alignment });
|
||
const { path, provider } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value.alignment).toBe(alignment);
|
||
expect(result.value.tensor_data_offset).toBe(fixture.dataOffset);
|
||
expect(
|
||
result.value.tensors.every((row) => row.file_offset % alignment === 0),
|
||
).toBe(true);
|
||
expect(result.value.tensors[1]?.file_offset).toBe(
|
||
fixture.dataOffset + Math.ceil(16 / alignment) * alignment,
|
||
);
|
||
},
|
||
);
|
||
|
||
it("aligns the GGUF data section after an empty tensor directory", async () => {
|
||
const base = ggufFixture({ tensors: [], metadata: [] });
|
||
const dataOffset = Math.ceil(base.header.length / 32) * 32;
|
||
const bytes = Buffer.concat([
|
||
base.header,
|
||
Buffer.alloc(dataOffset - base.header.length),
|
||
]);
|
||
const { path, provider } = await providerAt(bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value.tensors).toEqual([]);
|
||
expect(result.value.tensor_data_offset).toBe(dataOffset);
|
||
expect(result.value.artifact.header_length).toBe(base.header.length);
|
||
});
|
||
|
||
it.each([0, 1, 2, 4, 12])(
|
||
"rejects GGUF alignment %s outside positive multiples of eight",
|
||
async (alignment) => {
|
||
const base = ggufFixture();
|
||
const fixture = ggufFixture({
|
||
metadata: [
|
||
{ key: "general.alignment", type: 4, value: base.u32(alignment) },
|
||
],
|
||
});
|
||
const { path, provider } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (result.ok) throw new Error("Invalid alignment accepted");
|
||
expect(result.error).toMatchObject({
|
||
...INVALID_FORMAT,
|
||
});
|
||
},
|
||
);
|
||
|
||
it.each([
|
||
"General.name",
|
||
"general.naïve",
|
||
"general..name",
|
||
"general/name",
|
||
"general.-name",
|
||
"general.name-",
|
||
"general.double--name",
|
||
"general.name_",
|
||
"general.__name",
|
||
])(
|
||
"rejects invalid GGUF metadata key %s before publishing facts",
|
||
async (key) => {
|
||
const fixture = ggufFixture({
|
||
metadata: [{ key, type: 7, value: Buffer.from([1]) }],
|
||
});
|
||
const { path, provider } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (result.ok) throw new Error("Invalid metadata key accepted");
|
||
expect(result.error).toMatchObject({
|
||
...INVALID_FORMAT,
|
||
});
|
||
},
|
||
);
|
||
|
||
it("accepts standardized GGUF metadata keys containing numeric hierarchy components", async () => {
|
||
const base = ggufFixture();
|
||
const fixture = ggufFixture({
|
||
metadata: [
|
||
{ key: "general.base_model.0.name", type: 8, value: base.text("parent") },
|
||
],
|
||
});
|
||
const { path, provider } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value.metadata[0]).toMatchObject({
|
||
key: "general.base_model.0.name",
|
||
value: "parent",
|
||
});
|
||
});
|
||
|
||
it.each([67_324_752, 67_324_760, 99_999_993, 100_000_000])(
|
||
"admits safetensors declared header length %s before its independent representation limit",
|
||
async (length) => {
|
||
const bytes = Buffer.alloc(9);
|
||
bytes.writeBigUInt64LE(BigInt(length));
|
||
bytes[8] = 0x7b;
|
||
const { path, provider } = await providerAt(bytes);
|
||
const file = await open(path, "r+");
|
||
try {
|
||
await file.truncate(8 + length);
|
||
} finally {
|
||
await file.close();
|
||
}
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (result.ok) throw new Error("Representation limit was bypassed");
|
||
expect(result.error).toMatchObject({
|
||
_tag: "AnalysisResourceConstraintError",
|
||
reason: expect.stringContaining("32 MiB"),
|
||
});
|
||
},
|
||
);
|
||
|
||
it("retains the absolute header budget for GGUF without allocating that extent", async () => {
|
||
const { path } = await providerAt(Buffer.alloc(0));
|
||
const file = await open(path, "r+");
|
||
onTestFinished(() => file.close());
|
||
await file.truncate(100_000_009);
|
||
const reader = new ModelHeaderReader(file, 100_000_009);
|
||
expect(() => reader.guarantee(100_000_000)).not.toThrow();
|
||
expect(() => reader.guarantee(100_000_001)).toThrow(ParseFailureError);
|
||
try {
|
||
reader.guarantee(100_000_001);
|
||
} catch (cause) {
|
||
expect(cause).toBeInstanceOf(ParseFailureError);
|
||
if (!(cause instanceof ParseFailureError))
|
||
throw new Error("Expected a model header capacity failure");
|
||
expect(cause.failure).toMatchObject({
|
||
kind: "limit",
|
||
format: "model",
|
||
limit: { name: "header-bytes", maximum: 100_000_000 },
|
||
});
|
||
expect(cause.message).toContain("100,000,000 bytes");
|
||
expect(cause.message).not.toContain("Safetensors");
|
||
}
|
||
});
|
||
|
||
it("separates impossible string lengths from nesting capacity", async () => {
|
||
const base = ggufFixture({ tensors: [] });
|
||
const oversized = ggufFixture({
|
||
metadata: [
|
||
{
|
||
key: "custom.length",
|
||
type: 8,
|
||
value: base.u64(BigInt(Number.MAX_SAFE_INTEGER) + 1n),
|
||
},
|
||
],
|
||
tensors: [],
|
||
});
|
||
let nested = Buffer.from([1]);
|
||
for (let frame = 0; frame < 33; frame += 1)
|
||
nested = Buffer.concat([
|
||
base.u32(frame === 0 ? 7 : 9),
|
||
base.u64(1n),
|
||
nested,
|
||
]);
|
||
const deep = ggufFixture({
|
||
metadata: [{ key: "custom.nested", type: 9, value: nested }],
|
||
tensors: [],
|
||
});
|
||
const invalid = await providerAt(oversized.bytes);
|
||
expect(
|
||
await invalid.provider.inspect({ path: invalid.path, hash: "header" }),
|
||
).toMatchObject({ ok: false, error: INVALID_FORMAT });
|
||
for (const [label, bytes, allowance] of [
|
||
["nesting", deep.bytes, "32 array frames"],
|
||
] as const) {
|
||
const { path, provider } = await providerAt(bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (result.ok) throw new Error(`${label} capacity was accepted`);
|
||
expect(result.error).toMatchObject({
|
||
_tag: "AnalysisResourceConstraintError",
|
||
retryable: false,
|
||
remediationAction: expect.stringContaining(allowance),
|
||
});
|
||
const remediation =
|
||
"remediationAction" in result.error ? result.error.remediationAction : "";
|
||
expect(remediation).not.toContain(String(MODEL_REPORT_BYTES));
|
||
expect(remediation).not.toContain("100,000,000");
|
||
expect(result.error).toMatchObject({
|
||
reportedLimits: expect.not.objectContaining({
|
||
maximum_report_bytes: MODEL_REPORT_BYTES,
|
||
}),
|
||
});
|
||
}
|
||
});
|
||
|
||
it.each([
|
||
[MODEL_METADATA_NESTING, "empty", true],
|
||
[MODEL_METADATA_NESTING, "scalar", true],
|
||
[MODEL_METADATA_NESTING + 1, "empty", false],
|
||
[MODEL_METADATA_NESTING + 1, "scalar", false],
|
||
[MODEL_METADATA_NESTING + 2, "empty", false],
|
||
] as const)(
|
||
"bounds GGUF metadata at %i array frames with a %s innermost array",
|
||
async (frames, innermost, admitted) => {
|
||
const base = ggufFixture({ tensors: [] });
|
||
// The advertised unit is the array frame, so an empty innermost array
|
||
// occupies one exactly like an array holding a scalar child.
|
||
let nested =
|
||
innermost === "scalar"
|
||
? Buffer.concat([base.u32(7), base.u64(1n), Buffer.from([1])])
|
||
: Buffer.concat([base.u32(7), base.u64(0n)]);
|
||
for (let frame = 1; frame < frames; frame += 1)
|
||
nested = Buffer.concat([base.u32(9), base.u64(1n), nested]);
|
||
const { provider, path } = await providerAt(
|
||
ggufFixture({
|
||
metadata: [{ key: "custom.nested", type: 9, value: nested }],
|
||
tensors: [],
|
||
}).bytes,
|
||
);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (admitted) {
|
||
if (!result.ok) throw result.error;
|
||
expect(
|
||
result.value.metadata.find((row) => row.key === "custom.nested"),
|
||
).toMatchObject({ type: "ARRAY", element_count: 1 });
|
||
return;
|
||
}
|
||
if (result.ok) throw new Error(`${String(frames)} frames were admitted`);
|
||
expect(result.error).toMatchObject({
|
||
_tag: "AnalysisResourceConstraintError",
|
||
retryable: false,
|
||
reportedLimits: { maximum_array_frames: MODEL_METADATA_NESTING },
|
||
});
|
||
},
|
||
);
|
||
|
||
it("preserves safetensors scalar, empty and packed tensor declarations and raw string metadata", async () => {
|
||
const fixture = safetensorsFixture();
|
||
const { provider, path } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value.artifact.header_sha256).toBe(hash(fixture.header));
|
||
expect(
|
||
result.value.tensors.map((row) => [row.name, row.shape, row.byte_length]),
|
||
).toEqual([
|
||
["empty", ["0", "7"], 0],
|
||
["weight", ["2", "2"], 16],
|
||
]);
|
||
expect(result.value.metadata.map((row) => [row.key, row.value])).toEqual([
|
||
["format", "pt"],
|
||
["license", "Apache-2.0"],
|
||
]);
|
||
const packed = safetensorsFixture(
|
||
JSON.stringify({
|
||
scalar: { dtype: "F4", shape: [2], data_offsets: [0, 1] },
|
||
empty: { dtype: "F32", shape: [0], data_offsets: [0, 0] },
|
||
}),
|
||
Buffer.alloc(1),
|
||
);
|
||
await writeFile(path, packed.bytes);
|
||
const checked = await provider.inspect({ path, hash: "full" });
|
||
if (!checked.ok) throw checked.error;
|
||
expect(checked.value.totals).toMatchObject({
|
||
parameter_count: "2",
|
||
tensor_bytes: 1,
|
||
});
|
||
});
|
||
|
||
it("returns unknown lengths for unknown GGUF types without inventing storage size", async () => {
|
||
const fixture = ggufFixture({
|
||
tensors: [{ name: "future", type: 999, shape: [1n], offset: 0n, bytes: 1 }],
|
||
});
|
||
const { provider, path } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value.tensors[0]).toMatchObject({
|
||
type_id: 999,
|
||
type: "UNKNOWN_999",
|
||
byte_length: null,
|
||
byte_length_state: "unknown",
|
||
});
|
||
expect(result.value.coverage.tensor_ranges).toBe("partial");
|
||
expect(result.value.totals.tensor_bytes).toBeNull();
|
||
});
|
||
|
||
it.each([
|
||
{ version: 2, bigEndian: false, blockBytes: 36 },
|
||
{ version: 2, bigEndian: false, blockBytes: 40 },
|
||
{ version: 3, bigEndian: false, blockBytes: 36 },
|
||
{ version: 3, bigEndian: false, blockBytes: 40 },
|
||
{ version: 3, bigEndian: true, blockBytes: 36 },
|
||
{ version: 3, bigEndian: true, blockBytes: 40 },
|
||
])(
|
||
"preserves unresolved Q8_1 storage for GGUF v$version, endian=$bigEndian, block bytes=$blockBytes",
|
||
async ({ version, bigEndian, blockBytes }) => {
|
||
const base = ggufFixture({ bigEndian });
|
||
const fixture = ggufFixture({
|
||
version,
|
||
bigEndian,
|
||
metadata: [
|
||
{ key: "general.quantization_version", type: 4, value: base.u32(2) },
|
||
],
|
||
tensors: [
|
||
{ name: "control", type: 0, shape: [4n], offset: 0n, bytes: 16 },
|
||
{
|
||
name: "weight",
|
||
type: 9,
|
||
shape: [320n],
|
||
offset: 32n,
|
||
bytes: 10 * blockBytes,
|
||
},
|
||
],
|
||
});
|
||
const { provider, path } = await providerAt(
|
||
Buffer.concat([fixture.bytes, Buffer.alloc(16)]),
|
||
);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value.tensors).toMatchObject([
|
||
{ name: "control", byte_length: 16, storage_validation: "consistent" },
|
||
{
|
||
name: "weight",
|
||
type: "Q8_1",
|
||
type_id: 9,
|
||
shape: ["320"],
|
||
data_offset: 32,
|
||
file_offset: fixture.dataOffset + 32,
|
||
parameter_count: "320",
|
||
byte_length: null,
|
||
byte_length_state: "unknown",
|
||
storage_validation: "unknown",
|
||
},
|
||
]);
|
||
expect(result.value.totals).toMatchObject({
|
||
parameter_count: "324",
|
||
tensor_bytes: null,
|
||
by_type: [
|
||
{ type: "F32", bytes: 16, parameter_count: "4" },
|
||
{ type: "Q8_1", bytes: null, parameter_count: "320" },
|
||
],
|
||
});
|
||
expect(result.value.coverage).toMatchObject({
|
||
header: "complete",
|
||
tensor_ranges: "partial",
|
||
storage_sizes: "partial",
|
||
});
|
||
expect(result.value.limitations).toContainEqual(
|
||
expect.stringContaining("Q8_1 has released 40-byte and 36-byte layouts"),
|
||
);
|
||
expect(result.value.limitations.join(" ")).not.toContain(
|
||
"Unknown tensor type Q8_1",
|
||
);
|
||
expect(result.value.artifact.bytes_read).toBe(fixture.header.length);
|
||
},
|
||
);
|
||
|
||
it("derives empty Q8_1 storage without choosing a historical block layout", async () => {
|
||
const fixture = ggufFixture({
|
||
metadata: [],
|
||
tensors: [
|
||
{ name: "empty", type: 9, shape: [32n, 0n], offset: 0n, bytes: 0 },
|
||
],
|
||
});
|
||
const { provider, path } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value.tensors[0]).toMatchObject({
|
||
type: "Q8_1",
|
||
parameter_count: "0",
|
||
byte_length: 0,
|
||
byte_length_state: "derived",
|
||
storage_validation: "consistent",
|
||
});
|
||
expect(result.value.totals.tensor_bytes).toBe(0);
|
||
expect(result.value.coverage).toMatchObject({
|
||
tensor_ranges: "complete",
|
||
storage_sizes: "complete",
|
||
});
|
||
});
|
||
|
||
it.each(ggufKLayouts)(
|
||
"preserves $name identity while leaving released K64/K256 storage unresolved",
|
||
async ({ id, name, bytes64, bytes256 }) => {
|
||
// The released GGUF writer pads each tensor to 32 bytes. For 256 elements,
|
||
// both layouts have the same padded extent, despite different byte lengths.
|
||
expect(Math.ceil((4 * bytes64) / 32)).toBe(Math.ceil(bytes256 / 32));
|
||
for (const [elements, bytes] of [
|
||
[64, bytes64],
|
||
[256, 4 * bytes64],
|
||
[256, bytes256],
|
||
] as const) {
|
||
const base = ggufFixture();
|
||
const fixture = ggufFixture({
|
||
metadata: [
|
||
{ key: "general.quantization_version", type: 4, value: base.u32(2) },
|
||
{ key: "general.file_type", type: 4, value: base.u32(15) },
|
||
],
|
||
tensors: [
|
||
{
|
||
name: "weight",
|
||
type: id,
|
||
shape: [BigInt(elements)],
|
||
offset: 0n,
|
||
bytes,
|
||
},
|
||
],
|
||
});
|
||
const { provider, path } = await providerAt(
|
||
Buffer.concat([
|
||
fixture.bytes,
|
||
Buffer.alloc(Math.ceil(bytes / 32) * 32 - bytes),
|
||
]),
|
||
);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value.tensors[0]).toMatchObject({
|
||
type: name,
|
||
type_id: id,
|
||
shape: [String(elements)],
|
||
parameter_count: String(elements),
|
||
byte_length: null,
|
||
storage_validation: "unknown",
|
||
});
|
||
expect(result.value.coverage).toMatchObject({
|
||
tensor_ranges: "partial",
|
||
storage_sizes: "partial",
|
||
});
|
||
expect(result.value.totals.tensor_bytes).toBeNull();
|
||
expect(result.value.limitations).toContainEqual(
|
||
expect.stringContaining("common 64-element row constraint"),
|
||
);
|
||
}
|
||
const { provider, path } = await providerAt(
|
||
ggufFixture({
|
||
tensors: [
|
||
{
|
||
name: "invalid",
|
||
type: id,
|
||
shape: [32n],
|
||
offset: 0n,
|
||
bytes: bytes64,
|
||
},
|
||
],
|
||
}).bytes,
|
||
);
|
||
const rejected = await provider.inspect({ path, hash: "header" });
|
||
if (rejected.ok) throw new Error("Incomplete common K row admitted");
|
||
expect(rejected.error).toMatchObject({
|
||
...INVALID_FORMAT,
|
||
issues: [
|
||
expect.objectContaining({
|
||
reason: "invalid_format",
|
||
message: expect.stringContaining("multiple of 64 elements"),
|
||
}),
|
||
],
|
||
});
|
||
},
|
||
);
|
||
|
||
it.each(ggufShiftedTypeLayouts)(
|
||
"retains ambiguous numeric type $id for released integer and $modernName bodies",
|
||
async ({ id, integerBytes, modernName, modernBytes256 }) => {
|
||
for (const [elements, bytes] of [
|
||
[3, 3 * integerBytes],
|
||
[256, 256 * integerBytes],
|
||
[256, modernBytes256],
|
||
[0, 0],
|
||
] as const) {
|
||
const fixture = ggufFixture({
|
||
tensors: [
|
||
{
|
||
name: "weight",
|
||
type: id,
|
||
shape: [BigInt(elements)],
|
||
offset: 0n,
|
||
bytes,
|
||
},
|
||
],
|
||
});
|
||
const { provider, path } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value.tensors[0]).toMatchObject({
|
||
type: `AMBIGUOUS_${String(id)}`,
|
||
type_id: id,
|
||
shape: [String(elements)],
|
||
parameter_count: String(elements),
|
||
byte_length: elements === 0 ? 0 : null,
|
||
storage_validation: elements === 0 ? "consistent" : "unknown",
|
||
});
|
||
expect(result.value.limitations).toContainEqual(
|
||
expect.stringContaining(`type ID ${String(id)} has released`),
|
||
);
|
||
expect(result.value.limitations.join(" ")).toContain(modernName);
|
||
expect(result.value.totals.tensor_bytes).toBe(elements === 0 ? 0 : null);
|
||
}
|
||
},
|
||
);
|
||
|
||
it("retains established storage for stable GGUF types and zero-byte K declarations", async () => {
|
||
const fixture = ggufFixture({
|
||
tensors: [
|
||
{ name: "integer", type: 26, shape: [3n], offset: 0n, bytes: 12 },
|
||
{ name: "empty-k", type: 10, shape: [64n, 0n], offset: 32n, bytes: 0 },
|
||
{ name: "quant", type: 2, shape: [32n], offset: 32n, bytes: 18 },
|
||
],
|
||
});
|
||
const { provider, path } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value.tensors).toMatchObject([
|
||
{ type: "I32", byte_length: 12, storage_validation: "consistent" },
|
||
{ type: "Q2_K", byte_length: 0, storage_validation: "consistent" },
|
||
{ type: "Q4_0", byte_length: 18, storage_validation: "consistent" },
|
||
]);
|
||
expect(result.value.totals.tensor_bytes).toBe(30);
|
||
expect(result.value.coverage).toMatchObject({
|
||
tensor_ranges: "complete",
|
||
storage_sizes: "complete",
|
||
});
|
||
});
|
||
|
||
it("keeps unknown safetensors storage validation separate from observed valid byte ranges", async () => {
|
||
const fixture = safetensorsFixture(
|
||
'{"future":{"dtype":"FUTURE4","shape":[2],"data_offsets":[0,4]}}',
|
||
Buffer.alloc(4),
|
||
);
|
||
const { path, provider } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value.tensors[0]).toMatchObject({
|
||
byte_length: 4,
|
||
byte_length_state: "declared",
|
||
storage_validation: "unknown",
|
||
});
|
||
expect(result.value.coverage).toMatchObject({
|
||
tensor_ranges: "complete",
|
||
storage_sizes: "partial",
|
||
});
|
||
});
|
||
|
||
it.each([false, true])(
|
||
"decodes every fixed-width GGUF scalar type, big-endian=%s",
|
||
async (bigEndian) => {
|
||
const scalar = (width: number, write: (b: Buffer) => void) => {
|
||
const b = Buffer.alloc(width);
|
||
write(b);
|
||
return b;
|
||
};
|
||
const order = bigEndian ? "BE" : "LE";
|
||
const rows = [
|
||
[0, "UINT8", scalar(1, (b) => b.writeUInt8(0xfe)), 0xfe],
|
||
[1, "INT8", scalar(1, (b) => b.writeInt8(-2)), -2],
|
||
[2, "UINT16", scalar(2, (b) => b[`writeUInt16${order}`](0xfffe)), 0xfffe],
|
||
[3, "INT16", scalar(2, (b) => b[`writeInt16${order}`](-300)), -300],
|
||
[
|
||
4,
|
||
"UINT32",
|
||
scalar(4, (b) => b[`writeUInt32${order}`](0xfffffffe)),
|
||
0xfffffffe,
|
||
],
|
||
[5, "INT32", scalar(4, (b) => b[`writeInt32${order}`](-70000)), -70000],
|
||
[6, "FLOAT32", scalar(4, (b) => b[`writeFloat${order}`](-1.5)), -1.5],
|
||
[7, "BOOL", scalar(1, (b) => b.writeUInt8(1)), true],
|
||
[
|
||
10,
|
||
"UINT64",
|
||
scalar(8, (b) => b[`writeBigUInt64${order}`](2n ** 63n)),
|
||
"9223372036854775808",
|
||
],
|
||
[
|
||
11,
|
||
"INT64",
|
||
scalar(8, (b) => b[`writeBigInt64${order}`](-(2n ** 62n))),
|
||
"-4611686018427387904",
|
||
],
|
||
[
|
||
12,
|
||
"FLOAT64",
|
||
scalar(8, (b) => b[`writeDouble${order}`](-0.125)),
|
||
-0.125,
|
||
],
|
||
] as const;
|
||
const fixture = ggufFixture({
|
||
bigEndian,
|
||
metadata: rows.map(([type, name, value]) => ({
|
||
key: `custom.${name.toLowerCase()}`,
|
||
type,
|
||
value,
|
||
})),
|
||
});
|
||
const { path, provider } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value.metadata.map((row) => [row.type, row.value])).toEqual(
|
||
rows.map(([, name, , expected]) => [name, expected]),
|
||
);
|
||
},
|
||
);
|
||
|
||
it("preserves nonfinite floating metadata and negative zero without JSON coercion", async () => {
|
||
const values = [NaN, Infinity, -Infinity, -0];
|
||
const fixture = ggufFixture({
|
||
metadata: values.map((value, index) => {
|
||
const b = Buffer.alloc(8);
|
||
b.writeDoubleLE(value);
|
||
return { key: `custom.float_${String(index)}`, type: 12, value: b };
|
||
}),
|
||
});
|
||
const { path, provider } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value.metadata.map((row) => [row.type, row.value])).toEqual(
|
||
["NaN", "+Infinity", "-Infinity", "-0"].map((value) => ["FLOAT64", value]),
|
||
);
|
||
});
|
||
|
||
it("validates booleans beyond an array preview without allocating the full value list", async () => {
|
||
const base = ggufFixture();
|
||
const values = Buffer.alloc(10000);
|
||
values[9999] = 2;
|
||
const fixture = ggufFixture({
|
||
metadata: [
|
||
{
|
||
key: "custom.flags",
|
||
type: 9,
|
||
value: Buffer.concat([base.u32(7), base.u64(10000n), values]),
|
||
},
|
||
],
|
||
});
|
||
const { path, provider } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (result.ok) throw new Error("Invalid omitted boolean was accepted");
|
||
expect(result.error).toMatchObject({
|
||
...INVALID_FORMAT,
|
||
});
|
||
});
|
||
|
||
it.each([false, true])(
|
||
"validates omitted UTF-8 strings across chunk boundaries, big endian=%s",
|
||
async (bigEndian) => {
|
||
const base = ggufFixture({ bigEndian });
|
||
const preview = Array.from({ length: 8 }, (_, n) => `\uFEFFtoken-${n} 🦙`);
|
||
const strings = [
|
||
...preview,
|
||
"",
|
||
"a".repeat(65533) + "€",
|
||
"a".repeat(65535) + "🦙",
|
||
"\0tail",
|
||
];
|
||
const fixture = ggufFixture({
|
||
bigEndian,
|
||
metadata: [
|
||
{
|
||
key: "custom.strings",
|
||
type: 9,
|
||
value: Buffer.concat([
|
||
base.u32(8),
|
||
base.u64(BigInt(strings.length)),
|
||
...strings.map(base.text),
|
||
]),
|
||
},
|
||
],
|
||
});
|
||
const { path, provider } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value.metadata[0]).toMatchObject({
|
||
value: preview,
|
||
element_count: strings.length,
|
||
preview_complete: false,
|
||
});
|
||
expect(result.value.coverage.array_values).toBe("partial");
|
||
expect(result.value.artifact).toMatchObject({
|
||
bytes_read: fixture.header.length,
|
||
header_sha256: createHash("sha256").update(fixture.header).digest("hex"),
|
||
});
|
||
expect(result.value.tensors).toHaveLength(2);
|
||
},
|
||
);
|
||
|
||
it.each([
|
||
["short", Buffer.from([0xc0, 0x80])],
|
||
[
|
||
"chunk-sized",
|
||
Buffer.concat([Buffer.alloc(65535, 0x61), Buffer.from([0xe2])]),
|
||
],
|
||
[
|
||
"cross-chunk",
|
||
Buffer.concat([
|
||
Buffer.alloc(65535, 0x61),
|
||
Buffer.from([0xf0, 0x9f, 0x61, 0x80]),
|
||
]),
|
||
],
|
||
] as const)(
|
||
"rejects malformed omitted %s UTF-8 strings",
|
||
async (_, invalid) => {
|
||
const base = ggufFixture();
|
||
const fixture = ggufFixture({
|
||
metadata: [
|
||
{
|
||
key: "custom.strings",
|
||
type: 9,
|
||
value: Buffer.concat([
|
||
base.u32(8),
|
||
base.u64(9n),
|
||
...Array.from({ length: 8 }, () => base.text("valid")),
|
||
base.u64(BigInt(invalid.length)),
|
||
invalid,
|
||
]),
|
||
},
|
||
],
|
||
});
|
||
const { path, provider } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (result.ok) throw new Error("Invalid omitted UTF-8 string was accepted");
|
||
expect(result.error).toMatchObject({
|
||
...INVALID_FORMAT,
|
||
issues: [
|
||
expect.objectContaining({
|
||
reason: "invalid_format",
|
||
message: expect.stringContaining("GGUF string is not valid UTF-8"),
|
||
}),
|
||
],
|
||
});
|
||
},
|
||
);
|
||
|
||
it.each([
|
||
[
|
||
"header-bytes",
|
||
{ maximum_header_bytes: 100_000_000 },
|
||
"header allowance is 100,000,000 bytes",
|
||
],
|
||
[
|
||
"report-bytes",
|
||
{ maximum_report_bytes: MODEL_REPORT_BYTES },
|
||
`report allowance is ${String(MODEL_REPORT_BYTES)} bytes`,
|
||
],
|
||
[
|
||
"representation-bytes",
|
||
{ maximum_representation_bytes: MODEL_REPORT_BYTES },
|
||
`representation allowance is ${String(MODEL_REPORT_BYTES)} bytes`,
|
||
],
|
||
[
|
||
"exact-offset",
|
||
{ maximum_exact_offset: Number.MAX_SAFE_INTEGER },
|
||
`offset precision stops at ${String(Number.MAX_SAFE_INTEGER)}`,
|
||
],
|
||
[
|
||
"metadata-nesting",
|
||
{ maximum_array_frames: MODEL_METADATA_NESTING },
|
||
`GGUF metadata nesting is limited to ${String(MODEL_METADATA_NESTING)} array frames`,
|
||
],
|
||
[
|
||
"json-nesting",
|
||
{ maximum_json_depth: MODEL_JSON_NESTING },
|
||
`Safetensors JSON nesting is limited to ${String(MODEL_JSON_NESTING)} levels`,
|
||
],
|
||
] as const)(
|
||
"projects legacy fields and recovery for %s",
|
||
(limit, reportedLimits, remediation) => {
|
||
const cause = modelCapacityFailure(limit, "Model allowance exhausted");
|
||
const error = modelCapacityConstraint("inspect_model_artifact", cause);
|
||
expect(error).toMatchObject({
|
||
reason: "Model allowance exhausted",
|
||
remediationAction: expect.stringContaining(remediation),
|
||
cause,
|
||
});
|
||
expect(error?.reportedLimits).toEqual(reportedLimits);
|
||
},
|
||
);
|
||
|
||
it("refuses an oversized declared header before reading or expanding it", async () => {
|
||
const bytes = Buffer.alloc(9);
|
||
bytes.writeBigUInt64LE(100_000_001n);
|
||
bytes[8] = 0x7b;
|
||
const { path, provider } = await providerAt(bytes);
|
||
const file = await open(path, "r+");
|
||
try {
|
||
await file.truncate(100_000_009);
|
||
} finally {
|
||
await file.close();
|
||
}
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (result.ok) throw new Error("Oversized header was accepted");
|
||
expect(result.error).toMatchObject({
|
||
_tag: "AnalysisResourceConstraintError",
|
||
retryable: false,
|
||
reportedLimits: { maximum_header_bytes: 100_000_000 },
|
||
remediationAction: expect.stringContaining("100,000,000"),
|
||
});
|
||
expect(result.error.message).not.toContain("Safetensors");
|
||
expect(
|
||
"remediationAction" in result.error ? result.error.remediationAction : "",
|
||
).not.toContain("Safetensors");
|
||
expect(
|
||
"reportedLimits" in result.error ? result.error.reportedLimits : {},
|
||
).not.toHaveProperty("maximum_report_bytes");
|
||
});
|
||
|
||
const malformed = [
|
||
["truncated GGUF", () => ggufFixture().bytes.subarray(0, 21)],
|
||
[
|
||
"GGUF string whose declared bytes exceed the file and report budget",
|
||
() => {
|
||
const base = ggufFixture({ tensors: [] });
|
||
return ggufFixture({
|
||
tensors: [],
|
||
metadata: [
|
||
{ key: "custom.value", type: 8, value: base.u64(6_000_000n) },
|
||
],
|
||
}).bytes;
|
||
},
|
||
],
|
||
[
|
||
"GGUF count whose minimum directory exceeds the file and report budget",
|
||
() => {
|
||
const header = Buffer.alloc(24);
|
||
header.write("GGUF");
|
||
header.writeUInt32LE(3, 4);
|
||
header.writeBigUInt64LE(65537n, 8);
|
||
return header;
|
||
},
|
||
],
|
||
[
|
||
"invalid UTF8",
|
||
() => {
|
||
const f = ggufFixture({
|
||
metadata: [
|
||
{
|
||
key: "x",
|
||
type: 8,
|
||
value: Buffer.concat([
|
||
Buffer.from([1, 0, 0, 0, 0, 0, 0, 0]),
|
||
Buffer.from([0xff]),
|
||
]),
|
||
},
|
||
],
|
||
});
|
||
return f.bytes;
|
||
},
|
||
],
|
||
[
|
||
"duplicate GGUF metadata",
|
||
() =>
|
||
ggufFixture({
|
||
metadata: [
|
||
{ key: "x", type: 7, value: Buffer.from([0]) },
|
||
{ key: "x", type: 7, value: Buffer.from([1]) },
|
||
],
|
||
}).bytes,
|
||
],
|
||
[
|
||
"invalid boolean",
|
||
() =>
|
||
ggufFixture({
|
||
metadata: [{ key: "x", type: 7, value: Buffer.from([2]) }],
|
||
}).bytes,
|
||
],
|
||
[
|
||
"misaligned GGUF tensor",
|
||
() =>
|
||
ggufFixture({
|
||
tensors: [{ name: "x", type: 0, shape: [1n], offset: 1n, bytes: 4 }],
|
||
}).bytes,
|
||
],
|
||
[
|
||
"invalid quantization row",
|
||
() =>
|
||
ggufFixture({
|
||
tensors: [
|
||
{ name: "x", type: 2, shape: [16n, 2n], offset: 0n, bytes: 18 },
|
||
],
|
||
}).bytes,
|
||
],
|
||
[
|
||
"overlapping GGUF tensors",
|
||
() =>
|
||
ggufFixture({
|
||
tensors: [
|
||
{ name: "a", type: 0, shape: [1n], offset: 0n, bytes: 4 },
|
||
{ name: "b", type: 0, shape: [1n], offset: 0n, bytes: 4 },
|
||
],
|
||
}).bytes,
|
||
],
|
||
[
|
||
"duplicate JSON keys",
|
||
() =>
|
||
safetensorsFixture(
|
||
'{"x":{"dtype":"F32","dtype":"F16","shape":[2],"data_offsets":[0,4]}}',
|
||
Buffer.alloc(4),
|
||
).bytes,
|
||
],
|
||
[
|
||
"safetensors dimension exceeding uint64",
|
||
() =>
|
||
safetensorsFixture(
|
||
'{"x":{"dtype":"F32","shape":[0,18446744073709551616],"data_offsets":[0,0]}}',
|
||
Buffer.alloc(0),
|
||
).bytes,
|
||
],
|
||
[
|
||
"overlong safetensors integer",
|
||
() =>
|
||
safetensorsFixture(
|
||
'{"x":{"dtype":"F32","shape":[0,100000000000000000000],"data_offsets":[0,0]}}',
|
||
Buffer.alloc(0),
|
||
).bytes,
|
||
],
|
||
[
|
||
"safetensors shape product overflowing before a zero extent",
|
||
() =>
|
||
safetensorsFixture(
|
||
'{"x":{"dtype":"F32","shape":[18446744073709551615,18446744073709551615,0],"data_offsets":[0,0]}}',
|
||
Buffer.alloc(0),
|
||
).bytes,
|
||
],
|
||
[
|
||
"safetensors shape mismatch",
|
||
() =>
|
||
safetensorsFixture(
|
||
'{"x":{"dtype":"F32","shape":[3],"data_offsets":[0,16]}}',
|
||
).bytes,
|
||
],
|
||
[
|
||
"safetensors gap",
|
||
() =>
|
||
safetensorsFixture(
|
||
'{"x":{"dtype":"F32","shape":[3],"data_offsets":[4,16]}}',
|
||
).bytes,
|
||
],
|
||
[
|
||
"safetensors overlap",
|
||
() =>
|
||
safetensorsFixture(
|
||
'{"a":{"dtype":"F32","shape":[4],"data_offsets":[0,16]},"b":{"dtype":"F32","shape":[1],"data_offsets":[0,4]}}',
|
||
).bytes,
|
||
],
|
||
["safetensors trailing data", () => safetensorsFixture("{}").bytes],
|
||
[
|
||
"safetensors non-string metadata",
|
||
() => safetensorsFixture('{"__metadata__":{"x":1}}', Buffer.alloc(0)).bytes,
|
||
],
|
||
[
|
||
"safetensors decimal metadata container",
|
||
() => safetensorsFixture('{"__metadata__":1.25}', Buffer.alloc(0)).bytes,
|
||
],
|
||
[
|
||
"safetensors negative metadata container",
|
||
() => safetensorsFixture('{"__metadata__":-1}', Buffer.alloc(0)).bytes,
|
||
],
|
||
[
|
||
"safetensors overflowing metadata container",
|
||
() =>
|
||
safetensorsFixture(
|
||
'{"__metadata__":18446744073709551616}',
|
||
Buffer.alloc(0),
|
||
).bytes,
|
||
],
|
||
[
|
||
"safetensors noninteger shape sentinel",
|
||
() =>
|
||
safetensorsFixture(
|
||
'{"x":{"dtype":"F32","shape":[1.25],"data_offsets":[0,0]}}',
|
||
Buffer.alloc(0),
|
||
).bytes,
|
||
],
|
||
[
|
||
"safetensors negative data-offset sentinel",
|
||
() =>
|
||
safetensorsFixture(
|
||
'{"x":{"dtype":"F32","shape":[1],"data_offsets":[-1,4]}}',
|
||
Buffer.alloc(0),
|
||
).bytes,
|
||
],
|
||
[
|
||
"safetensors tensor missing a required declaration",
|
||
() =>
|
||
safetensorsFixture('{"x":{"dtype":"F32","shape":[1]}}', Buffer.alloc(0))
|
||
.bytes,
|
||
],
|
||
[
|
||
"safetensors unpaired surrogate",
|
||
() =>
|
||
safetensorsFixture('{"__metadata__":{"x":"\\ud800"}}', Buffer.alloc(0))
|
||
.bytes,
|
||
],
|
||
] as const;
|
||
it.each(malformed)(
|
||
"rejects %s as an invalid_format path",
|
||
async (_name, make) => {
|
||
const { provider, path } = await providerAt(make());
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (result.ok) throw new Error("Malformed model was accepted");
|
||
expect(result.error).toMatchObject({
|
||
...INVALID_FORMAT,
|
||
});
|
||
},
|
||
);
|
||
|
||
it("retains GGUF representation capacity when declared string bytes can fit", async () => {
|
||
const base = ggufFixture({ tensors: [] });
|
||
const bytes = ggufFixture({
|
||
tensors: [],
|
||
metadata: [{ key: "custom.value", type: 8, value: base.u64(6_000_000n) }],
|
||
}).bytes;
|
||
const { provider, path } = await providerAt(bytes);
|
||
await truncate(path, bytes.length + 6_000_000);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (result.ok) throw new Error("Oversized representation was accepted");
|
||
expect(result.error).toMatchObject({
|
||
_tag: "AnalysisResourceConstraintError",
|
||
reportedLimits: { maximum_representation_bytes: MODEL_REPORT_BYTES },
|
||
});
|
||
});
|
||
|
||
it.each([
|
||
"metadata row",
|
||
"tensor descriptor",
|
||
"array element",
|
||
"outer array sibling",
|
||
] as const)(
|
||
"checks required GGUF %s framing before charging a large string",
|
||
async (following) => {
|
||
const seed = ggufFixture({ tensors: [] });
|
||
const largeStringBytes = 6_000_000;
|
||
const stringLength = seed.u64(BigInt(largeStringBytes));
|
||
const twoStrings = Buffer.concat([seed.u32(8), seed.u64(2n)]);
|
||
const array =
|
||
following === "array element" || following === "outer array sibling";
|
||
const outer = following === "outer array sibling";
|
||
const prefix = ggufFixture({
|
||
tensors: [],
|
||
metadata: [
|
||
{
|
||
key: "custom.value",
|
||
type: array ? 9 : 8,
|
||
value: Buffer.concat([
|
||
...(outer ? [seed.u32(9), seed.u64(2n)] : []),
|
||
...(array ? [twoStrings] : []),
|
||
stringLength,
|
||
]),
|
||
},
|
||
],
|
||
}).header;
|
||
if (following === "metadata row") prefix.writeBigUInt64LE(2n, 16);
|
||
if (following === "tensor descriptor") prefix.writeBigUInt64LE(1n, 8);
|
||
const stringEnd = prefix.length + largeStringBytes;
|
||
const emptyString = seed.text("");
|
||
const tail = {
|
||
"metadata row": Buffer.concat([
|
||
seed.text("custom.tail"),
|
||
seed.u32(0),
|
||
Buffer.from([1]),
|
||
]),
|
||
"tensor descriptor": Buffer.concat([
|
||
seed.text("weight"),
|
||
seed.u32(1),
|
||
seed.u64(1n),
|
||
seed.u32(0),
|
||
seed.u64(0n),
|
||
]),
|
||
"array element": emptyString,
|
||
"outer array sibling": Buffer.concat([
|
||
emptyString,
|
||
seed.u32(8),
|
||
seed.u64(0n),
|
||
]),
|
||
}[following];
|
||
const { provider, path } = await providerAt(prefix);
|
||
const handle = await open(path, "r+");
|
||
try {
|
||
// For nested arrays the immediate string sibling fits. Only the outer
|
||
// array's second element is missing, so its minimum must propagate.
|
||
if (outer)
|
||
await handle.write(emptyString, 0, emptyString.length, stringEnd);
|
||
else await handle.truncate(stringEnd);
|
||
const malformed = await provider.inspect({ path, hash: "header" });
|
||
if (malformed.ok) throw new Error("Missing GGUF framing was accepted");
|
||
expect.soft(malformed.error).toMatchObject(INVALID_FORMAT);
|
||
|
||
await handle.write(tail, 0, tail.length, stringEnd);
|
||
if (following === "tensor descriptor")
|
||
await handle.truncate(
|
||
Math.ceil((stringEnd + tail.length) / 32) * 32 + 4,
|
||
);
|
||
const limited = await provider.inspect({ path, hash: "header" });
|
||
if (limited.ok) throw new Error("Oversized representation was accepted");
|
||
expect(limited.error).toMatchObject({
|
||
_tag: "AnalysisResourceConstraintError",
|
||
reportedLimits: { maximum_representation_bytes: MODEL_REPORT_BYTES },
|
||
});
|
||
} finally {
|
||
await handle.close();
|
||
}
|
||
},
|
||
);
|
||
|
||
it("retains GGUF representation capacity when the declared directory can fit", async () => {
|
||
const header = Buffer.alloc(24);
|
||
header.write("GGUF");
|
||
header.writeUInt32LE(3, 4);
|
||
header.writeBigUInt64LE(65537n, 8);
|
||
const { provider, path } = await providerAt(header);
|
||
await truncate(path, 24 + 65537 * 24);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (result.ok) throw new Error("Oversized representation was accepted");
|
||
expect(result.error).toMatchObject({
|
||
_tag: "AnalysisResourceConstraintError",
|
||
reportedLimits: { maximum_representation_bytes: MODEL_REPORT_BYTES },
|
||
});
|
||
});
|
||
|
||
it("retains GGUF declarations and metadata when storage overlap is refused", async () => {
|
||
const seed = ggufFixture();
|
||
const fixture = ggufFixture({
|
||
metadata: [{ key: "general.name", type: 8, value: seed.text("Overlap") }],
|
||
tensors: [
|
||
{ name: "a", type: 0, shape: [1n], offset: 0n, bytes: 4 },
|
||
{ name: "b", type: 0, shape: [1n], offset: 0n, bytes: 4 },
|
||
],
|
||
});
|
||
const { provider, path } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (result.ok) throw new Error("Overlapping tensors admitted");
|
||
expect(result.error).toMatchObject({
|
||
...INVALID_FORMAT,
|
||
});
|
||
const evidence = parseEvidence(result.error.partialObservation);
|
||
expect(evidence.subject.identity.kind).toBe("unknown");
|
||
expect(evidence.normalized_result).toMatchObject({
|
||
artifact: {
|
||
header_sha256: null,
|
||
sha256: null,
|
||
file_identity: "unverified",
|
||
},
|
||
coverage: { header: "partial", metadata: "complete", tensors: "partial" },
|
||
metadata: [{ key: "general.name", value: "Overlap" }],
|
||
tensors: [],
|
||
tensor_declarations: [
|
||
{
|
||
name: "a",
|
||
file_offset: fixture.dataOffset,
|
||
storage_validation: "not-established",
|
||
},
|
||
{
|
||
name: "b",
|
||
file_offset: fixture.dataOffset,
|
||
storage_validation: "not-established",
|
||
},
|
||
],
|
||
});
|
||
});
|
||
|
||
it("accepts null safetensors metadata and ignores tensor extension fields", async () => {
|
||
const fixture = safetensorsFixture(
|
||
'{"__metadata__":null,"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":{"source":"local","flags":[null,false,-1,1.25]}}}',
|
||
Buffer.alloc(4),
|
||
);
|
||
const { provider, path } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value.metadata).toEqual([]);
|
||
expect(
|
||
result.value.tensors.map(({ name, type, shape }) => ({
|
||
name,
|
||
type,
|
||
shape,
|
||
})),
|
||
).toEqual([{ name: "weight", type: "F32", shape: ["1"] }]);
|
||
});
|
||
|
||
it("accepts nested JSON values in ignored Safetensors extension fields", async () => {
|
||
const nested = `${"[".repeat(12)}0${"]".repeat(12)}`;
|
||
const fixture = safetensorsFixture(
|
||
`{"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":${nested}}}`,
|
||
Buffer.alloc(4),
|
||
);
|
||
const { provider, path } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
expect(result.ok).toBe(true);
|
||
});
|
||
|
||
it("classifies over-depth Safetensors extension JSON as a resource limit", async () => {
|
||
const nested = `${"[".repeat(MODEL_JSON_NESTING + 1)}0${"]".repeat(MODEL_JSON_NESTING + 1)}`;
|
||
const fixture = safetensorsFixture(
|
||
`{"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":${nested}}}`,
|
||
Buffer.alloc(4),
|
||
);
|
||
const { provider, path } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (result.ok) throw new Error("Over-depth JSON was accepted");
|
||
expect(result.error).toMatchObject({
|
||
_tag: "AnalysisResourceConstraintError",
|
||
resource: "memory",
|
||
reason: expect.stringContaining("Safetensors JSON nesting"),
|
||
reportedLimits: { maximum_json_depth: MODEL_JSON_NESTING },
|
||
});
|
||
});
|
||
|
||
it.each([640, 896, 1152, 1408, 66_176])(
|
||
"admits safetensors header length %s whose prefix bytes resemble a pickle protocol marker",
|
||
async (length) => {
|
||
const header = JSON.stringify({
|
||
weight: { dtype: "F32", shape: [1], data_offsets: [0, 4] },
|
||
});
|
||
const fixture = safetensorsFixture(
|
||
header.padEnd(length, " "),
|
||
Buffer.alloc(4),
|
||
);
|
||
const { provider, path } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value).toMatchObject({ format: "safetensors" });
|
||
expect(result.value.tensors.map(({ name }) => name)).toEqual(["weight"]);
|
||
},
|
||
);
|
||
|
||
it.each([
|
||
// Protocol 2 pickle of {"a": 1}: PROTO 2, EMPTY_DICT, BINPUT 0, ...
|
||
["a ZIP local header", Buffer.from("504b0304140000000000", "hex")],
|
||
["a short ZIP signature", Buffer.from("504b0304", "hex")],
|
||
[
|
||
"a ZIP signature followed by a brace and an inadmissible length",
|
||
Buffer.from("504b0304140000007b", "hex"),
|
||
],
|
||
["a real pickle", Buffer.from("80027d710058010000006171014b01732e", "hex")],
|
||
// Protocol 2 pickle of {}: six bytes, shorter than a safetensors prefix.
|
||
["an empty dict", Buffer.from("80027d71002e", "hex")],
|
||
// Protocol 4 FRAME pickle of {"a": 1}.
|
||
[
|
||
"a protocol 4 frame",
|
||
Buffer.from("8004950a000000000000007d948c0161944b01732e", "hex"),
|
||
],
|
||
// The magic number torch.save writes first in its legacy format.
|
||
[
|
||
"the legacy torch.save magic",
|
||
Buffer.from("80028a0a6cfc9c46f9206aa850192e", "hex"),
|
||
],
|
||
// Protocol 2 pickle of "x{": its brace lands at offset 8.
|
||
[
|
||
"a brace after an inadmissible length",
|
||
Buffer.from("80025802000000787b71002e", "hex"),
|
||
],
|
||
[
|
||
"an in-file length without a JSON brace",
|
||
Buffer.concat([
|
||
Buffer.from("8002000000000000", "hex"),
|
||
Buffer.alloc(644, 0x2e),
|
||
]),
|
||
],
|
||
] as const)(
|
||
"refuses pickle protocol bytes followed by %s",
|
||
async (_case, bytes) => {
|
||
const { provider, path } = await providerAt(bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (result.ok) throw new Error("Pickle admitted");
|
||
expect(result.error).toMatchObject({
|
||
_tag: "AnalysisUnsupportedTargetError",
|
||
reason: expect.stringContaining("PyTorch pickle"),
|
||
});
|
||
},
|
||
);
|
||
|
||
it("reports a truncated safetensors header whose length resembles a pickle marker as malformed", async () => {
|
||
const fixture = safetensorsFixture(
|
||
JSON.stringify({
|
||
weight: { dtype: "F32", shape: [1], data_offsets: [0, 4] },
|
||
}).padEnd(896, " "),
|
||
Buffer.alloc(4),
|
||
);
|
||
const { provider, path } = await providerAt(fixture.bytes.subarray(0, 500));
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (result.ok) throw new Error("Truncated model was accepted");
|
||
expect(result.error).toMatchObject({
|
||
...INVALID_FORMAT,
|
||
});
|
||
});
|
||
|
||
it("bounds expansion before allocation and distinguishes unsupported pickle and paths", async () => {
|
||
const fixture = ggufFixture();
|
||
fixture.bytes.writeBigUInt64LE(0xffffffffffffffffn, 8);
|
||
const { provider, path, root } = await providerAt(fixture.bytes);
|
||
const huge = await provider.inspect({ path, hash: "header" });
|
||
if (huge.ok) throw new Error("Huge count admitted");
|
||
expect(huge.error).toMatchObject(INVALID_FORMAT);
|
||
await writeFile(path, Buffer.from([0x80, 4, 0, 0]));
|
||
const pickle = await provider.inspect({ path, hash: "header" });
|
||
if (pickle.ok) throw new Error("Pickle admitted");
|
||
expect(pickle.error._tag).toBe("AnalysisUnsupportedTargetError");
|
||
await writeFile(path, ggufFixture().bytes);
|
||
expect((await provider.inspect({ path, hash: "header" })).ok).toBe(true);
|
||
const alias = join(root, "link");
|
||
await symlink(path, alias);
|
||
const linked = await provider.inspect({ path: alias, hash: "header" });
|
||
if (linked.ok) throw new Error("Selected symlink admitted");
|
||
expect(linked.error.message).toMatch(/symlink|symbolic.link/iu);
|
||
const service = createModelArtifactService();
|
||
onTestFinished(() => service.close());
|
||
const invalid = await service.inspect({
|
||
path: "relative",
|
||
detail: "complete",
|
||
});
|
||
if (invalid.ok) throw new Error("Relative path admitted");
|
||
expect(invalid.error._tag).toBe("AnalysisInputError");
|
||
});
|
||
|
||
it("reads only the exact header, even when it ends in the middle of a normal buffer", async () => {
|
||
const fixture = safetensorsFixture();
|
||
const { path } = await providerAt(fixture.bytes);
|
||
const file = await open(path, "r");
|
||
onTestFinished(() => file.close());
|
||
const requests: [number, number][] = [];
|
||
const read = file.read.bind(file);
|
||
const boundary = {
|
||
read: async (
|
||
buffer: Buffer,
|
||
offset: number,
|
||
length: number,
|
||
position: number,
|
||
) => {
|
||
requests.push([position, length]);
|
||
expect(position + length).toBeLessThanOrEqual(fixture.header.length);
|
||
return read(buffer, offset, length, position);
|
||
},
|
||
} as Pick<typeof file, "read">;
|
||
const reader = new ModelHeaderReader(
|
||
boundary as typeof file,
|
||
fixture.bytes.length,
|
||
);
|
||
await readSafetensorsHeader(reader, await reader.take(4));
|
||
expect(await reader.finish(false)).toMatchObject({
|
||
bytes_read: fixture.header.length,
|
||
header_sha256: hash(fixture.header),
|
||
});
|
||
expect(requests.reduce((n, [, length]) => n + length, 0)).toBe(
|
||
fixture.header.length,
|
||
);
|
||
});
|
||
|
||
it.each(["header", "full"] as const)(
|
||
"classifies real truncation after admission during %s reading as an identity change",
|
||
async (mode) => {
|
||
const fixture = ggufFixture();
|
||
const { path } = await providerAt(fixture.bytes);
|
||
const file = await open(path, "r");
|
||
onTestFinished(() => file.close());
|
||
const admitted = await file.stat();
|
||
const reader = new ModelHeaderReader(file, admitted.size);
|
||
if (mode === "full") await reader.take(fixture.header.length);
|
||
await truncate(path, mode === "header" ? 0 : fixture.header.length);
|
||
const reading = mode === "header" ? reader.take(4) : reader.finish(true);
|
||
await expect(reading).rejects.toMatchObject({ reason: "integrity" });
|
||
},
|
||
);
|
||
|
||
it("keeps GGUF read-ahead inside the directory and hashes precisely that prefix", async () => {
|
||
const fixture = ggufFixture();
|
||
const { path } = await providerAt(fixture.bytes);
|
||
const file = await open(path, "r");
|
||
onTestFinished(() => file.close());
|
||
const read = file.read.bind(file);
|
||
let bytesRead = 0;
|
||
const boundary = {
|
||
read: async (
|
||
buffer: Buffer,
|
||
offset: number,
|
||
length: number,
|
||
position: number,
|
||
) => {
|
||
expect(position + length).toBeLessThanOrEqual(fixture.header.length);
|
||
const result = await read(buffer, offset, length, position);
|
||
bytesRead += result.bytesRead;
|
||
return result;
|
||
},
|
||
} as Pick<typeof file, "read">;
|
||
const reader = new ModelHeaderReader(
|
||
boundary as typeof file,
|
||
fixture.bytes.length,
|
||
);
|
||
await reader.take(4);
|
||
await new GgufHeader(reader).read();
|
||
await reader.finish(false);
|
||
expect(bytesRead).toBe(fixture.header.length);
|
||
});
|
||
|
||
it("inspects a 1 GiB sparse safetensors declaration using only its small header", async () => {
|
||
const dataBytes = 1024 ** 3;
|
||
const fixture = safetensorsFixture(
|
||
JSON.stringify({
|
||
weight: { dtype: "U8", shape: [dataBytes], data_offsets: [0, dataBytes] },
|
||
}),
|
||
Buffer.alloc(0),
|
||
);
|
||
const { path, provider } = await providerAt(fixture.header);
|
||
const file = await open(path, "r+");
|
||
try {
|
||
await file.truncate(fixture.header.length + dataBytes);
|
||
} finally {
|
||
await file.close();
|
||
}
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value.artifact.bytes_read).toBe(fixture.header.length);
|
||
expect(result.value.tensors[0]).toMatchObject({
|
||
byte_length: dataBytes,
|
||
parameter_count: String(dataBytes),
|
||
});
|
||
});
|
||
|
||
it("supports true scalar shapes and prototype-named metadata without changing their spelling", async () => {
|
||
const fixture = safetensorsFixture(
|
||
'{"__metadata__":{"__proto__":"raw","constructor":"kept"},"scalar":{"dtype":"F32","shape":[],"data_offsets":[0,4]}}',
|
||
Buffer.alloc(4),
|
||
);
|
||
const { path, provider } = await providerAt(fixture.bytes);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value.tensors[0]).toMatchObject({
|
||
shape: [],
|
||
parameter_count: "1",
|
||
byte_length: 4,
|
||
});
|
||
expect(result.value.metadata.map(({ key, value }) => [key, value])).toEqual([
|
||
["__proto__", "raw"],
|
||
["constructor", "kept"],
|
||
]);
|
||
});
|
||
|
||
it("yields while validating a large metadata array, preserves cancellation and releases admission", async () => {
|
||
const f = ggufFixture();
|
||
const count = 1_000_000;
|
||
const fixture = ggufFixture({
|
||
metadata: [
|
||
{
|
||
key: "custom.bools",
|
||
type: 9,
|
||
value: Buffer.concat([
|
||
f.u32(7),
|
||
f.u64(BigInt(count)),
|
||
Buffer.alloc(count),
|
||
]),
|
||
},
|
||
],
|
||
});
|
||
const { path, provider } = await providerAt(fixture.bytes);
|
||
const controller = new AbortController();
|
||
const reason = new Error("stop large array");
|
||
const pending = provider.inspect(
|
||
{ path, hash: "header" },
|
||
{ signal: controller.signal },
|
||
);
|
||
setTimeout(() => controller.abort(reason), 0);
|
||
const result = await pending;
|
||
if (result.ok) throw new Error("Cancellation ignored");
|
||
expect(result.error).toMatchObject({
|
||
_tag: "AnalysisCancelledError",
|
||
cause: reason,
|
||
});
|
||
expect(result.error.cause).toBe(reason);
|
||
await writeFile(path, ggufFixture().bytes);
|
||
const recovered = await provider.inspect({ path, hash: "header" });
|
||
if (!recovered.ok) throw recovered.error;
|
||
expect(recovered.value.tensors).toHaveLength(2);
|
||
});
|
||
|
||
it("retains explicit scoped identity through shared service and preserves caller abort reasons", async () => {
|
||
const fixture = ggufFixture();
|
||
const { path } = await providerAt(fixture.bytes);
|
||
const service = createModelArtifactService();
|
||
onTestFinished(() => service.close());
|
||
const result = await service.inspect({ path, detail: "complete" });
|
||
if (!result.ok) throw result.error;
|
||
expect(result.value.subject.identity.kind).toBe("unknown");
|
||
expect(
|
||
modelArtifactSchema.parse(result.value.normalized_result).artifact.sha256,
|
||
).toBeNull();
|
||
const full = await service.inspect({
|
||
path,
|
||
hash: "full",
|
||
detail: "complete",
|
||
});
|
||
if (!full.ok) throw full.error;
|
||
expect(fileIdentity(full.value.subject).sha256).toBe(hash(fixture.bytes));
|
||
const controller = new AbortController();
|
||
const reason = new Error("selected caller stop");
|
||
controller.abort(reason);
|
||
const stopped = await service.inspect(
|
||
{ path, detail: "complete" },
|
||
{ signal: controller.signal },
|
||
);
|
||
if (stopped.ok) throw new Error("Cancellation ignored");
|
||
expect(stopped.error.cause).toBe(reason);
|
||
});
|
||
|
||
it.each([false, true])(
|
||
"rejects raw uint64 extents before host conversion (big endian: %s)",
|
||
async (bigEndian) => {
|
||
const maximum = 0xffffffffffffffffn;
|
||
const base = ggufFixture({ bigEndian, tensors: [], metadata: [] });
|
||
const countCases = [8, 16].map((offset) => {
|
||
const bytes = Buffer.from(base.bytes);
|
||
(bigEndian
|
||
? bytes.writeBigUInt64BE.bind(bytes)
|
||
: bytes.writeBigUInt64LE.bind(bytes))(maximum, offset);
|
||
return bytes;
|
||
});
|
||
const tensor = ggufFixture({
|
||
bigEndian,
|
||
metadata: [],
|
||
tensors: [{ name: "x", type: 0, shape: [1n], offset: 0n, bytes: 4 }],
|
||
});
|
||
base.u64(maximum).copy(tensor.bytes, tensor.header.length - 8);
|
||
const cases = [
|
||
tensor.bytes,
|
||
...countCases,
|
||
ggufFixture({
|
||
bigEndian,
|
||
tensors: [],
|
||
metadata: [{ key: "value", type: 8, value: base.u64(maximum) }],
|
||
}).bytes,
|
||
ggufFixture({
|
||
bigEndian,
|
||
tensors: [],
|
||
metadata: [
|
||
{
|
||
key: "array",
|
||
type: 9,
|
||
value: Buffer.concat([base.u32(0), base.u64(maximum)]),
|
||
},
|
||
],
|
||
}).bytes,
|
||
];
|
||
for (const bytes of cases) {
|
||
const { provider, path } = await providerAt(bytes);
|
||
expect(await provider.inspect({ path, hash: "header" })).toMatchObject({
|
||
ok: false,
|
||
error: INVALID_FORMAT,
|
||
});
|
||
}
|
||
},
|
||
);
|
||
|
||
it("rejects impossible safetensors header and data extents before numeric conversion", async () => {
|
||
const prefix = Buffer.alloc(8);
|
||
prefix.writeBigUInt64LE(0xffffffffffffffffn);
|
||
const header = Buffer.from(
|
||
'{"x":{"dtype":"F32","shape":[1],"data_offsets":[0,18446744073709551615]}}',
|
||
);
|
||
const length = Buffer.alloc(8);
|
||
length.writeBigUInt64LE(BigInt(header.length));
|
||
for (const bytes of [
|
||
Buffer.concat([prefix, Buffer.from("{")]),
|
||
Buffer.concat([length, header, Buffer.alloc(4)]),
|
||
]) {
|
||
const { provider, path } = await providerAt(bytes);
|
||
expect(await provider.inspect({ path, hash: "header" })).toMatchObject({
|
||
ok: false,
|
||
error: INVALID_FORMAT,
|
||
});
|
||
}
|
||
});
|
||
|
||
it("rejects a tensor directory whose old 20-byte estimate fits but fixed fields do not", async () => {
|
||
const header = Buffer.alloc(24);
|
||
header.write("GGUF");
|
||
header.writeUInt32LE(3, 4);
|
||
header.writeBigUInt64LE(65537n, 8);
|
||
const { provider, path } = await providerAt(header);
|
||
await truncate(path, 24 + 65537 * 20);
|
||
expect(await provider.inspect({ path, hash: "header" })).toMatchObject({
|
||
ok: false,
|
||
error: INVALID_FORMAT,
|
||
});
|
||
});
|
||
|
||
it.each([false, true])(
|
||
"reserves later GGUF tensor framing before publishing a dimensioned row (big endian: %s)",
|
||
async (bigEndian) => {
|
||
const fixture = ggufFixture({
|
||
bigEndian,
|
||
metadata: [],
|
||
tensors: [
|
||
{ name: "first", type: 0, shape: [1n], offset: 0n, bytes: 4 },
|
||
{ name: "next", type: 0, shape: [], offset: 32n, bytes: 4 },
|
||
],
|
||
});
|
||
// Leave 23 bytes after the first descriptor, one short of the
|
||
// second descriptor’s minimum framing even with an empty name.
|
||
const { provider, path } = await providerAt(
|
||
fixture.header.subarray(0, fixture.header.length - 5),
|
||
);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
if (result.ok) throw new Error("Truncated GGUF directory admitted");
|
||
expect(result.error).toMatchObject(INVALID_FORMAT);
|
||
expect(
|
||
parseEvidence(result.error.partialObservation).normalized_result,
|
||
).toMatchObject({ declared_tensor_count: 2, tensor_declarations: [] });
|
||
},
|
||
);
|
||
|
||
it("retains decoded counts when a physically fitting GGUF header exceeds the header budget", async () => {
|
||
const count = 4_166_667;
|
||
const header = Buffer.alloc(24);
|
||
header.write("GGUF");
|
||
header.writeUInt32LE(3, 4);
|
||
header.writeBigUInt64LE(BigInt(count), 8);
|
||
const { provider, path } = await providerAt(header);
|
||
await truncate(path, header.length + count * 24);
|
||
const result = await provider.inspect({ path, hash: "header" });
|
||
expect(result).toMatchObject({
|
||
ok: false,
|
||
error: {
|
||
_tag: "AnalysisResourceConstraintError",
|
||
partialObservation: {
|
||
normalized_result: {
|
||
declared_tensor_count: count,
|
||
declared_metadata_count: 0,
|
||
},
|
||
},
|
||
},
|
||
});
|
||
});
|