mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-11-20 04:22:27 +00:00
Run DeepSeek-V4.1-Flash offline inference on SM120 GPUs (RTX PRO 6000 Blackwell) with the same entry points as SM100. SM100 and Ascend paths are unchanged. - Build: SM120 is detected from the first visible GPU (scripts/nvidia_arch.sh). It builds DeepEP and DeepGEMM from its nv_dev branch (vendor/DeepGEMM-sm120), and no FlashMLA or DeepSelect; FlashInfer is installed from a pinned commit (requirements-nvidia-sm120.txt). - Attention: V4.1 sparse MLA runs on FlashInfer's SM120 sparse MLA kernels, reading the inline-scale KV records as stored; KV blocks are aligned to 512 bytes. - Indexer: FlashInfer's fused DSA indexer scores the index keys and selects their top-k in one call, for decode on every indexer layer (candidate source and consumer layers included) and for prefill outside the candidate layers. The remaining selections (prefill of the candidate layers, the sampler) use FlashInfer's top-k behind DeepSelect's interface (model_executor/layers/topk.py), in deterministic mode. - MoE EP: DeepEP over NCCL GIN by default, or NCCL collectives (DEEPINFER_SM120_EP_BACKEND=nccl); NCCL settings known to break the EP paths are rejected. EPLB, including decode-only draft-layer EPLB, and per-layer YoCo MoE dispatch. - Engram tables live in page-locked host shared memory; killed workers leave nothing in /dev/shm. - Features: attention DP + MoE EP, DSpark/nextn speculative decoding, CUDA graphs, async scheduling, decoder SWA bounded replay, structured outputs, image inputs, multi-node deployments. Features without an SM120 implementation (CP > 1, MTP varlen, batch-invariant mode, GPU-resident Engram tables) are refused at engine start. - Eval: chat GSM8K and CV-Bench; tests for the SM120 kernels and their DeepInfer call sites. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
497 lines
24 KiB
Python
497 lines
24 KiB
Python
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
"""Bit-exact checks of the FP8/FP4 quantizers on the DeepSeek-V4.1 GPU path, on SM120.
|
|
|
|
Covers tile_kernels.quant.per_token_cast (activation e4m3, FP4 index/weight e2m1),
|
|
swiglu_forward_and_per_token_cast, batched_transpose_weight_sf, the tile_kernels.torch
|
|
cast/cast_back references, the fused RMSNorm + FP8 cast, and the KV-cache quantization
|
|
paths (store_kv_cache, load_paged_kv_cache, fused_norm_rope_hadamard_cast_store_kvcache).
|
|
|
|
Every quantized byte is compared with the independent references in _sm120_ref.py
|
|
(UE8M0 scale = 2^ceil(log2(max(amax, clamp) / max_q)), RNE narrowing), at row counts
|
|
on both sides of 512 and of tile_kernels' 4096-row small-batch switch.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Generator
|
|
|
|
import pytest
|
|
import tile_kernels
|
|
import torch
|
|
from tile_kernels.config import get_pdl, get_token_alignment
|
|
from tile_kernels.quant import batched_transpose_weight_sf, per_token_cast, swiglu_forward_and_per_token_cast
|
|
from tile_kernels.torch import batched_transpose_weight_sf as batched_transpose_weight_sf_ref
|
|
from tile_kernels.torch import cast as torch_cast
|
|
from tile_kernels.torch import cast_back as torch_cast_back
|
|
|
|
from deepinfer.model_executor.layers.linear import get_fp8_cast_params
|
|
from deepinfer.model_executor.layers.tilelang_ops.cache_common import (
|
|
FP4_INDEX_KV_FORMAT,
|
|
MXFP8_INDEX_KV_FORMAT,
|
|
V41_FORMAT,
|
|
V41_FP4_FORMAT,
|
|
KVCacheFormat,
|
|
split_kv_cache,
|
|
)
|
|
from deepinfer.model_executor.layers.tilelang_ops.fused_transform import (
|
|
CastArgs,
|
|
KVCacheOutArgs,
|
|
NormArgs,
|
|
RopeArgs,
|
|
fused_norm_rope_hadamard_cast_store_kvcache,
|
|
)
|
|
from deepinfer.model_executor.layers.tilelang_ops.load_paged_kv_cache import load_paged_kv_cache
|
|
from deepinfer.model_executor.layers.tilelang_ops.norm import norm
|
|
from deepinfer.model_executor.layers.tilelang_ops.store_kv_cache import store_kv_cache
|
|
from tests.kernels._sm120_ref import (
|
|
FP4_AMAX_CLAMP,
|
|
FP8_AMAX_CLAMP,
|
|
HEAD_DIM,
|
|
HIDDEN,
|
|
INDEX_HEAD_DIM,
|
|
INDEX_HEADS,
|
|
MOE_INTER,
|
|
NUM_HEADS,
|
|
O_GROUPS,
|
|
O_LORA_RANK,
|
|
Q_LORA_RANK,
|
|
QUANT_ROWS,
|
|
RMS_EPS,
|
|
ROPE_DIM,
|
|
SM120_MARKS,
|
|
SWIGLU_LIMIT,
|
|
align,
|
|
assert_bytes_equal,
|
|
assert_no_fp8_nan,
|
|
dequant_e2m1,
|
|
dequant_e4m3_ue8m0,
|
|
make_activations,
|
|
ref_e4m3_sf_fp4_quantize,
|
|
ref_ue8m0_quantize,
|
|
ue8m0_bytes_to_scale,
|
|
unpack_int32_ue8m0,
|
|
)
|
|
|
|
pytestmark = SM120_MARKS
|
|
|
|
|
|
@pytest.fixture(params=[False, True], ids=["nopdl", "pdl"])
|
|
def pdl(request: pytest.FixtureRequest) -> Generator[bool]:
|
|
"""DeepInfer workers enable PDL for tile_kernels (worker/runtime_config.py); test both."""
|
|
prev = get_pdl()
|
|
tile_kernels.set_pdl(request.param)
|
|
yield request.param
|
|
tile_kernels.set_pdl(prev)
|
|
|
|
|
|
def test_reference_constants_match_runtime() -> None:
|
|
"""The independent reference uses the same amax clamps and flags DeepInfer configures."""
|
|
assert tile_kernels.config.get_amax_clamp_for_quant("e4m3", False) == FP8_AMAX_CLAMP
|
|
assert tile_kernels.config.get_amax_clamp_for_quant("e2m1", False) == FP4_AMAX_CLAMP
|
|
assert get_fp8_cast_params() == {
|
|
"fmt": "e4m3",
|
|
"num_per_channels": 32,
|
|
"round_sf": True,
|
|
"use_packed_ue8m0": True,
|
|
"use_tma_aligned_col_major_sf": True,
|
|
}
|
|
|
|
|
|
def _check_col_major_packed_sf(sf: torch.Tensor, m: int, k: int, gran: int = 32) -> torch.Tensor:
|
|
"""Check the TMA-aligned column-major packed-UE8M0 layout; return the [m, k/gran] exponent bytes."""
|
|
packs = (k // gran + 3) // 4
|
|
assert sf.dtype == torch.int32 and sf.shape == (m, packs), f"{sf.dtype=} {sf.shape=}"
|
|
assert sf.stride() == (1, align(m, 4)), f"{sf.stride()=}"
|
|
return unpack_int32_ue8m0(sf)[:, : k // gran]
|
|
|
|
|
|
# (rows, hidden): hidden = V4.1 model width, wo_a/out_proj inputs, q_b input, and the
|
|
# 64-head x 512 attention context that attn.py casts with per_token_cast on SM120.
|
|
_ACT_CASES = (
|
|
[(m, HIDDEN) for m in QUANT_ROWS]
|
|
+ [(m, k) for k in (Q_LORA_RANK, O_GROUPS * O_LORA_RANK, MOE_INTER) for m in (511, 513, 4097)]
|
|
+ [(m, NUM_HEADS * HEAD_DIM) for m in (1, 511, 513, 4096, 4097, 8192)]
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("m,k", _ACT_CASES, ids=[f"m{m}-k{k}" for m, k in _ACT_CASES])
|
|
def test_per_token_cast_e4m3_activation(m: int, k: int, pdl: bool) -> None:
|
|
"""linear.py / attn.py / hc.py: per_token_cast(x, **get_fp8_cast_params())."""
|
|
x = make_activations(m, k, seed=m * 7 + k)
|
|
ref_data, ref_sf = ref_ue8m0_quantize(x, "e4m3", 32)
|
|
|
|
q, sf = per_token_cast(x, **get_fp8_cast_params())
|
|
assert q.dtype == torch.float8_e4m3fn and q.shape == (m, k)
|
|
sf_bytes = _check_col_major_packed_sf(sf, m, k)
|
|
assert_no_fp8_nan(q, "fp8 data")
|
|
assert_bytes_equal(sf_bytes, ref_sf, "ue8m0 sf")
|
|
assert_bytes_equal(q, ref_data, "fp8 data")
|
|
|
|
# Re-launch into the same buffers: a racy kernel would not be repeatable.
|
|
q2, sf2 = per_token_cast(x, **get_fp8_cast_params())
|
|
assert_bytes_equal(q2, q, "fp8 data (relaunch)")
|
|
assert torch.equal(sf2, sf)
|
|
|
|
|
|
@pytest.mark.parametrize("m", QUANT_ROWS)
|
|
@pytest.mark.parametrize("k", [INDEX_HEAD_DIM, INDEX_HEADS * INDEX_HEAD_DIM, HIDDEN])
|
|
def test_per_token_cast_e2m1_activation(m: int, k: int, pdl: bool) -> None:
|
|
"""FP4 (e2m1) per-32 UE8M0 cast with the flags cache_common uses for the FP4 index cache."""
|
|
x = make_activations(m, k, seed=m * 13 + k)
|
|
ref_data, ref_sf = ref_ue8m0_quantize(x, "e2m1", 32)
|
|
|
|
q, sf = per_token_cast(x, "e2m1", 32, round_sf=True, use_packed_ue8m0=True, use_tma_aligned_col_major_sf=True)
|
|
assert q.dtype == torch.int8 and q.shape == (m, k // 2)
|
|
sf_bytes = _check_col_major_packed_sf(sf, m, k)
|
|
assert_bytes_equal(sf_bytes, ref_sf, "ue8m0 sf")
|
|
assert_bytes_equal(q, ref_data, "fp4 data")
|
|
|
|
|
|
@pytest.mark.parametrize("n,k", [(2 * MOE_INTER, HIDDEN), (HIDDEN, MOE_INTER)], ids=["gate_up", "down"])
|
|
@pytest.mark.parametrize("n_experts", [1, 3])
|
|
def test_per_token_cast_e2m1_from_fp8_weight(n: int, k: int, n_experts: int, pdl: bool) -> None:
|
|
"""mlp.process_moe_weight(use_fp4=True): FP8 32x32-block weights recast to FP4 per-32 UE8M0."""
|
|
g = torch.Generator(device="cuda").manual_seed(n + k + n_experts)
|
|
# FP4-QAT checkpoint: the two low mantissa bits of every FP8 byte are zero.
|
|
w = (torch.randint(0, 256, (n_experts * n, k), generator=g, device="cuda", dtype=torch.int32) & 0xFC).to(torch.uint8)
|
|
w = w.view(torch.float8_e4m3fn)
|
|
w_sf = torch.ldexp(
|
|
torch.ones(n_experts * n // 32, k // 32, device="cuda"),
|
|
torch.randint(-12, 1, (n_experts * n // 32, k // 32), generator=g, device="cuda"),
|
|
)
|
|
w_sf[0, 0] = 2.0**-127 * 4 # tiny block scale
|
|
w.view(torch.uint8)[:32, 32:64] = 0 # an all-zero block
|
|
|
|
dequant = w.float() * w_sf.repeat_interleave(32, 0).repeat_interleave(32, 1)
|
|
ref_data, ref_sf = ref_ue8m0_quantize(dequant, "e2m1", 32)
|
|
|
|
q, sf = per_token_cast(
|
|
(w, w_sf), "e2m1", 32, x_block_size=(32, 32), round_sf=True, use_tma_aligned_col_major_sf=True, use_packed_ue8m0=True
|
|
)
|
|
sf_bytes = _check_col_major_packed_sf(sf, n_experts * n, k)
|
|
assert_bytes_equal(sf_bytes, ref_sf, "ue8m0 sf")
|
|
assert_bytes_equal(q, ref_data, "fp4 data")
|
|
|
|
|
|
@pytest.mark.parametrize("m", [1, 511, 513, 4096, 32768])
|
|
@pytest.mark.parametrize("routed", [True, False], ids=["routed", "shared"])
|
|
def test_swiglu_forward_and_per_token_cast(m: int, routed: bool, pdl: bool) -> None:
|
|
"""mlp.py: swiglu_forward_and_per_token_cast(x, **get_fp8_cast_params(), [psum, topk_weights], clamp_value=10)."""
|
|
g = torch.Generator(device="cuda").manual_seed(m + routed)
|
|
alignment = get_token_alignment()
|
|
if routed:
|
|
num_experts = 48 # 384 routed experts over EP 8
|
|
counts = torch.multinomial(torch.ones(num_experts), m, replacement=True, generator=torch.Generator().manual_seed(m)).bincount(
|
|
minlength=num_experts
|
|
)
|
|
counts[num_experts // 2] = 0 # an empty expert
|
|
ends, start = [], 0
|
|
for c in counts.tolist():
|
|
start = align(start, alignment) + c
|
|
ends.append(start)
|
|
psum = torch.tensor(ends, dtype=torch.int32, device="cuda")
|
|
total = ends[-1]
|
|
valid = torch.zeros(total, dtype=torch.bool, device="cuda")
|
|
begin = 0
|
|
for end in ends:
|
|
valid[align(begin, alignment) : end] = True
|
|
begin = end
|
|
weights = torch.rand(total, generator=g, device="cuda") * 0.5
|
|
kwargs = dict(psum_num_tokens_per_expert=psum, topk_weights=weights)
|
|
else:
|
|
total = m
|
|
valid = torch.ones(total, dtype=torch.bool, device="cuda")
|
|
weights = None
|
|
kwargs = {}
|
|
|
|
x = make_activations(total, 2 * MOE_INTER, seed=m * 3 + 1) * 4 # exercise the +/-10 clamp
|
|
out, sf = swiglu_forward_and_per_token_cast(x, **get_fp8_cast_params(), clamp_value=SWIGLU_LIMIT, **kwargs)
|
|
|
|
xl, xr = x.float().chunk(2, dim=-1)
|
|
xl = xl.clamp(max=SWIGLU_LIMIT)
|
|
xr = xr.clamp(-SWIGLU_LIMIT, SWIGLU_LIMIT)
|
|
y = xl / (1.0 + torch.exp(-xl)) * xr
|
|
if weights is not None:
|
|
y = y * weights.unsqueeze(1)
|
|
ref_data, ref_sf = ref_ue8m0_quantize(y[valid], "e4m3", 32)
|
|
|
|
sf_bytes = _check_col_major_packed_sf(sf, total, MOE_INTER)
|
|
out_bytes = out.view(torch.uint8)[valid]
|
|
assert_no_fp8_nan(out_bytes, "fp8 data")
|
|
assert_bytes_equal(sf_bytes[valid], ref_sf, "ue8m0 sf")
|
|
assert_bytes_equal(out_bytes, ref_data, "fp8 data")
|
|
|
|
|
|
@pytest.mark.parametrize("block_size", [32, 128])
|
|
def test_batched_transpose_weight_sf(block_size: int) -> None:
|
|
"""hugging_face/parameters.set_ue8m0_sf: FP32 power-of-two weight scales -> packed UE8M0 sf0/sf1."""
|
|
g = torch.Generator(device="cuda").manual_seed(block_size)
|
|
|
|
def pow2(*shape: int) -> torch.Tensor:
|
|
return torch.ldexp(torch.ones(shape, device="cuda"), torch.randint(-20, 4, shape, generator=g, device="cuda"))
|
|
|
|
sfs = [
|
|
pow2(HIDDEN // block_size, Q_LORA_RANK // block_size), # dense
|
|
pow2(4, 2 * MOE_INTER // block_size, HIDDEN // block_size), # grouped
|
|
pow2(O_GROUPS, O_LORA_RANK // block_size, (NUM_HEADS // O_GROUPS * HEAD_DIM) // block_size), # einsum wv
|
|
]
|
|
flags = [False, False, True]
|
|
outs = batched_transpose_weight_sf(sfs, block_size, flags, outs=[None] * len(sfs))
|
|
refs = batched_transpose_weight_sf_ref(sfs, block_size, flags)
|
|
for i, ((sf0, sf1), (rsf0, rsf1)) in enumerate(zip(outs, refs, strict=True)):
|
|
assert sf0.shape == rsf0.shape and sf0.stride() == rsf0.stride(), f"{i}: {sf0.shape=} {sf0.stride()=} {rsf0.stride()=}"
|
|
assert sf1.shape == rsf1.shape and sf1.stride() == rsf1.stride(), f"{i}: {sf1.shape=} {sf1.stride()=} {rsf1.stride()=}"
|
|
assert torch.equal(sf0, rsf0), f"weight {i}: sf0 mismatch"
|
|
assert torch.equal(sf1, rsf1), f"weight {i}: sf1 mismatch"
|
|
|
|
|
|
@pytest.mark.parametrize("m", [1, 513, 4096])
|
|
@pytest.mark.parametrize("fmt", ["e4m3", "e2m1"])
|
|
def test_torch_cast_references(m: int, fmt: str) -> None:
|
|
"""tile_kernels.torch.cast/cast_back (the references DeepInfer's cache/fused-transform tests rely on)."""
|
|
x = make_activations(m, HEAD_DIM, seed=m + len(fmt))
|
|
ref_data, ref_sf = ref_ue8m0_quantize(x, fmt, 32)
|
|
q, sf = torch_cast(x, fmt, (1, 32), round_sf=True, use_packed_ue8m0=True)
|
|
assert_bytes_equal(unpack_int32_ue8m0(sf), ref_sf, "ue8m0 sf")
|
|
assert_bytes_equal(q, ref_data, "data")
|
|
back = torch_cast_back((q, sf), "fp32", block_size=(1, 32))
|
|
expected = dequant_e4m3_ue8m0(ref_data, ref_sf) if fmt == "e4m3" else dequant_e2m1(ref_data, ue8m0_bytes_to_scale(ref_sf), 32)
|
|
assert torch.equal(back, expected)
|
|
|
|
|
|
@pytest.mark.parametrize("m", QUANT_ROWS)
|
|
@pytest.mark.parametrize("k", [HIDDEN, Q_LORA_RANK])
|
|
def test_norm_fp8_cast(m: int, k: int) -> None:
|
|
"""layernorm.RMSNorm(use_fp8=True) with return_bf16=True, eps=1e-20 (hc.py _hc_pre / attn q_a_norm)."""
|
|
x = make_activations(m, k, seed=m + 5 * k)
|
|
weight = (torch.randn(k, device="cuda") * 0.1 + 1.0).to(torch.bfloat16)
|
|
bf16_only = norm(x, weight, eps=RMS_EPS)
|
|
(q, sf), out_bf16 = norm(x, weight, eps=RMS_EPS, **get_fp8_cast_params(), return_bf16=True)
|
|
assert torch.equal(out_bf16, bf16_only), "bf16 side output differs from the bf16-only norm"
|
|
ref_data, ref_sf = ref_ue8m0_quantize(bf16_only, "e4m3", 32)
|
|
sf_bytes = _check_col_major_packed_sf(sf, m, k)
|
|
assert_no_fp8_nan(q, "fp8 data")
|
|
assert_bytes_equal(sf_bytes, ref_sf, "ue8m0 sf")
|
|
assert_bytes_equal(q, ref_data, "fp8 data")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# KV cache
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_V41_CACHE_FORMATS = {
|
|
"v41-fp8-main": V41_FORMAT,
|
|
"v41-fp4-main": V41_FP4_FORMAT,
|
|
"fp4-index": FP4_INDEX_KV_FORMAT,
|
|
"mxfp8-index": MXFP8_INDEX_KV_FORMAT,
|
|
}
|
|
# Main/SWA pages hold window + spill tokens (odd sizes occur); index pages hold block_size // ratio tokens.
|
|
_CACHE_CASES = [
|
|
(name, block_size)
|
|
for name, layout in _V41_CACHE_FORMATS.items()
|
|
for block_size in ((64, 133) if layout.token_interleaved else (32, 128))
|
|
]
|
|
|
|
|
|
def _ref_encode(x: torch.Tensor, layout: KVCacheFormat) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""Independent per-token (data bytes, sf bytes) for a cache format."""
|
|
if layout.quantized_dtype == "e2m1" and layout.sf_format == "e4m3":
|
|
return ref_e4m3_sf_fp4_quantize(x, layout.elements_per_sf)
|
|
assert layout.sf_format == "ue8m0"
|
|
return ref_ue8m0_quantize(x, layout.quantized_dtype, layout.elements_per_sf)
|
|
|
|
|
|
def _ref_decode(data: torch.Tensor, sf: torch.Tensor, layout: KVCacheFormat) -> torch.Tensor:
|
|
if layout.quantized_dtype == "e4m3":
|
|
return dequant_e4m3_ue8m0(data, sf, layout.elements_per_sf)
|
|
scale = sf.view(torch.float8_e4m3fn).float() if layout.sf_format == "e4m3" else ue8m0_bytes_to_scale(sf)
|
|
return dequant_e2m1(data, scale, layout.elements_per_sf)
|
|
|
|
|
|
def _guarded_cache(layout: KVCacheFormat, num_blocks: int, block_size: int, seed: int) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""Random-filled paged cache with page padding and a storage offset; returns (backing, cache view)."""
|
|
alignment = 512 if layout.token_interleaved else 32
|
|
page_bytes = block_size * layout.bytes_per_token
|
|
page_stride = align(page_bytes, alignment) + alignment
|
|
offset = alignment
|
|
g = torch.Generator(device="cuda").manual_seed(seed)
|
|
backing = torch.randint(0, 256, (offset + num_blocks * page_stride + alignment,), generator=g, device="cuda", dtype=torch.uint8)
|
|
bpt = layout.bytes_per_token
|
|
cache = backing.as_strided((num_blocks, block_size, 1, bpt), (page_stride, bpt, bpt, 1), offset)
|
|
return backing, cache
|
|
|
|
|
|
def _cache_bytes(cache: torch.Tensor, layout: KVCacheFormat, slots: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
|
block_size, views = split_kv_cache(cache, layout)
|
|
slots = slots.long()
|
|
data = views.quantized_data.view(torch.uint8)[slots // block_size, slots % block_size]
|
|
sf = views.sf_data.view(torch.uint8)[slots // block_size, slots % block_size]
|
|
return data, sf
|
|
|
|
|
|
def _slots(n: int, num_slots: int, seed: int, with_padding: bool = True) -> torch.Tensor:
|
|
g = torch.Generator(device="cuda").manual_seed(seed)
|
|
slots = torch.randperm(num_slots, generator=g, device="cuda")[:n].to(torch.int32)
|
|
if with_padding and n > 2:
|
|
slots[1::7] = -1
|
|
return slots
|
|
|
|
|
|
@pytest.mark.parametrize("n", [1, 511, 513, 4096, 32768])
|
|
@pytest.mark.parametrize("fmt_name,block_size", _CACHE_CASES)
|
|
def test_store_kv_cache(fmt_name: str, block_size: int, n: int) -> None:
|
|
"""attn.py: store_kv_cache(kv_swa, swa_cache, slot_mapping) and the index-cache formats."""
|
|
layout = _V41_CACHE_FORMATS[fmt_name]
|
|
num_blocks = (n + 2 * block_size - 1) // block_size + 1
|
|
backing, cache = _guarded_cache(layout, num_blocks, block_size, seed=n + block_size)
|
|
before = backing.clone()
|
|
slots = _slots(n, num_blocks * block_size, seed=n)
|
|
kv = make_activations(n, layout.num_elements, seed=n + 1, gran=layout.elements_per_sf).view(n, 1, layout.num_elements)
|
|
|
|
store_kv_cache(kv, cache, slots, layout=layout)
|
|
|
|
valid = slots >= 0
|
|
ref_data, ref_sf = _ref_encode(kv[valid, 0], layout)
|
|
data, sf = _cache_bytes(cache, layout, slots[valid])
|
|
assert_bytes_equal(sf, ref_sf, "cache sf bytes")
|
|
assert_bytes_equal(data, ref_data, "cache data bytes")
|
|
|
|
# Nothing outside the valid slots' records may change (padding slots, page gaps, canaries).
|
|
expected = before.clone()
|
|
exp_cache = expected.as_strided(cache.shape, cache.stride(), cache.storage_offset())
|
|
block, off = slots[valid].long() // block_size, slots[valid].long() % block_size
|
|
_, exp_views = split_kv_cache(exp_cache, layout)
|
|
_, act_views = split_kv_cache(cache, layout)
|
|
for exp_t, act_t in ((exp_views.quantized_data, act_views.quantized_data), (exp_views.sf_data, act_views.sf_data)):
|
|
exp_t.view(torch.uint8)[block, off] = act_t.view(torch.uint8)[block, off]
|
|
assert torch.equal(backing, expected), "store_kv_cache wrote outside the valid slots"
|
|
|
|
|
|
@pytest.mark.parametrize("total_k", [1, 511, 513, 4096, 32768])
|
|
@pytest.mark.parametrize("fmt_name,block_size", _CACHE_CASES)
|
|
def test_load_paged_kv_cache(fmt_name: str, block_size: int, total_k: int) -> None:
|
|
"""attn.py / index_attn.py: load_paged_kv_cache for prefill (dequant to bf16, or raw index K)."""
|
|
layout = _V41_CACHE_FORMATS[fmt_name]
|
|
g = torch.Generator().manual_seed(total_k + block_size)
|
|
# Segments [src_block, begin, end, dst]: random lengths, one zero-fill (-1) segment, shuffled blocks.
|
|
segs, dst = [], 0
|
|
while dst < total_k:
|
|
length = min(int(torch.randint(1, block_size + 1, (1,), generator=g)), total_k - dst)
|
|
begin = int(torch.randint(0, block_size - length + 1, (1,), generator=g))
|
|
segs.append([0, begin, begin + length, dst])
|
|
dst += length
|
|
num_blocks = len(segs) + 3
|
|
perm = torch.randperm(num_blocks, generator=g)
|
|
for i, s in enumerate(segs):
|
|
s[0] = int(perm[i])
|
|
if len(segs) > 2:
|
|
segs[len(segs) // 2][0] = -1
|
|
load_segments = torch.tensor(segs, dtype=torch.int32, device="cuda")
|
|
|
|
_, cache = _guarded_cache(layout, num_blocks, block_size, seed=total_k)
|
|
# Fill the cache with valid encodings of realistic activations.
|
|
src = make_activations(num_blocks * block_size, layout.num_elements, seed=total_k + 2, gran=layout.elements_per_sf)
|
|
enc_data, enc_sf = _ref_encode(src, layout)
|
|
_, views = split_kv_cache(cache, layout)
|
|
views.quantized_data.view(torch.uint8).copy_(enc_data.view(num_blocks, block_size, -1))
|
|
views.sf_data.view(torch.uint8).copy_(enc_sf.view(num_blocks, block_size, -1))
|
|
|
|
rows_src = []
|
|
for blk, begin, end, d in segs:
|
|
rows_src += [(-1 if blk < 0 else blk * block_size + t, d + t - begin) for t in range(begin, end)]
|
|
src_rows = torch.tensor([r for r, _ in rows_src], device="cuda")
|
|
dst_rows = torch.tensor([d for _, d in rows_src], device="cuda")
|
|
zero = src_rows < 0
|
|
src_rows = src_rows.clamp_min(0)
|
|
|
|
result = load_paged_kv_cache(cache, load_segments, total_k, layout=layout)
|
|
if layout.do_cast:
|
|
expected = _ref_decode(enc_data[src_rows], enc_sf[src_rows], layout).to(torch.bfloat16)
|
|
expected[zero] = 0
|
|
out = result.bf16_data.view(total_k, layout.num_elements)
|
|
assert torch.equal(out[dst_rows].view(torch.int16), expected.view(torch.int16)), (
|
|
f"dequantized load mismatch: max abs diff {(out[dst_rows].float() - expected.float()).abs().max().item()}"
|
|
)
|
|
else:
|
|
exp_data = torch.where(zero.unsqueeze(1), 0, enc_data[src_rows])
|
|
exp_sf = torch.where(zero.unsqueeze(1), 0, enc_sf[src_rows])
|
|
data = result.quantized_data.view(torch.uint8).view(total_k, -1)
|
|
sf = result.sf_data.contiguous().view(torch.uint8).view(total_k, -1)
|
|
assert_bytes_equal(data[dst_rows], exp_data, "loaded data")
|
|
assert_bytes_equal(sf[dst_rows], exp_sf, "loaded sf")
|
|
|
|
|
|
def _rope_cache(max_pos: int, rope_dim: int) -> torch.Tensor:
|
|
inv_freq = 1.0 / (10000 ** (torch.arange(0, rope_dim, 2, dtype=torch.float32) / rope_dim))
|
|
freqs = torch.outer(torch.arange(max_pos, dtype=torch.float32), inv_freq)
|
|
return torch.cat([freqs.cos(), freqs.sin()], dim=-1).cuda()
|
|
|
|
|
|
@pytest.mark.parametrize("n", [1, 511, 513, 4096, 32768])
|
|
@pytest.mark.parametrize(
|
|
"case",
|
|
["swa-decode-store", "swa-prefill-cast-back", "compress-store-prerope", "index-k-store", "index-q-fp4"],
|
|
)
|
|
def test_fused_transform_quant(case: str, n: int) -> None:
|
|
"""fused_norm_rope_hadamard_cast_store_kvcache quantized outputs, as attn.py / compress.py / index_attn.py call it.
|
|
|
|
The fused kernel's quantized bytes must equal the independent quantizer applied to the
|
|
same kernel's unquantized (bf16) transform output.
|
|
"""
|
|
g = torch.Generator(device="cuda").manual_seed(n)
|
|
cos_sin = _rope_cache(1 << 16, ROPE_DIM)
|
|
positions = torch.randint(0, 1 << 16, (n,), generator=g, device="cuda", dtype=torch.int32)
|
|
is_index = case.startswith("index")
|
|
head_dim = INDEX_HEAD_DIM if is_index else HEAD_DIM
|
|
num_heads = INDEX_HEADS if case == "index-q-fp4" else 1
|
|
x = make_activations(n * num_heads, head_dim, seed=n + 9).view(n, num_heads, head_dim)
|
|
if num_heads == 1 and case != "compress-store-prerope":
|
|
x = x[:, 0]
|
|
weight = (torch.randn(head_dim, generator=g, device="cuda") * 0.1 + 1.0).to(torch.bfloat16)
|
|
norm_args = None if case == "index-q-fp4" else NormArgs(weight, RMS_EPS)
|
|
rope_args = RopeArgs(positions, cos_sin, ROPE_DIM, False, offset=head_dim - ROPE_DIM)
|
|
|
|
transformed = fused_norm_rope_hadamard_cast_store_kvcache(x, norm_args=norm_args, rope_args=rope_args)
|
|
flat = transformed.reshape(n * num_heads, head_dim)
|
|
|
|
if case == "swa-prefill-cast-back":
|
|
cast = CastArgs("e4m3", 32, round_sf=True, sf_format="ue8m0", cast_back=True)
|
|
out = fused_norm_rope_hadamard_cast_store_kvcache(x, norm_args=norm_args, rope_args=rope_args, cast_args=cast)
|
|
ref_data, ref_sf = ref_ue8m0_quantize(flat, "e4m3", 32)
|
|
expected = dequant_e4m3_ue8m0(ref_data, ref_sf).to(torch.bfloat16)
|
|
assert torch.equal(out.reshape(n, head_dim).view(torch.int16), expected.view(torch.int16)), (
|
|
f"cast_back mismatch: max abs diff {(out.reshape(n, head_dim).float() - expected.float()).abs().max().item()}"
|
|
)
|
|
return
|
|
|
|
if case == "index-q-fp4":
|
|
cast = CastArgs.from_cache_format(FP4_INDEX_KV_FORMAT)
|
|
q, sf = fused_norm_rope_hadamard_cast_store_kvcache(x, rope_args=rope_args, cast_args=cast)
|
|
ref_data, ref_sf = ref_ue8m0_quantize(flat, "e2m1", 32)
|
|
assert_bytes_equal(sf.reshape(n * num_heads, -1).contiguous().view(torch.uint8), ref_sf, "index q sf")
|
|
assert_bytes_equal(q.reshape(n * num_heads, -1), ref_data, "index q fp4 data")
|
|
return
|
|
|
|
layout = FP4_INDEX_KV_FORMAT if is_index else V41_FORMAT
|
|
block_size = 64
|
|
num_blocks = (n + 2 * block_size - 1) // block_size + 1
|
|
backing, cache = _guarded_cache(layout, num_blocks, block_size, seed=n + 3)
|
|
slots = _slots(n, num_blocks * block_size, seed=n + 4)
|
|
pre_rope = torch.empty_like(x) if case == "compress-store-prerope" else None
|
|
out_args = KVCacheOutArgs(cache, slots, out=pre_rope, out_stage="norm", layout=layout)
|
|
result = fused_norm_rope_hadamard_cast_store_kvcache(
|
|
x, norm_args=norm_args, rope_args=rope_args, cast_args=CastArgs.from_cache_format(layout), out_args=out_args
|
|
)
|
|
if pre_rope is not None:
|
|
assert result is pre_rope or result.data_ptr() == pre_rope.data_ptr()
|
|
expected_norm = norm(x, weight, eps=RMS_EPS)
|
|
assert torch.equal(pre_rope, expected_norm), "out_stage='norm' output differs from the norm kernel"
|
|
|
|
valid = slots >= 0
|
|
ref_data, ref_sf = _ref_encode(flat[valid], layout)
|
|
data, sf = _cache_bytes(cache, layout, slots[valid])
|
|
assert_bytes_equal(sf, ref_sf, "cache sf bytes")
|
|
assert_bytes_equal(data, ref_data, "cache data bytes")
|