Files
DeepInfer/tests/kernels/_sm120_ref.py
T
Zihua WuandClaude Opus 5.5 efecbb5ba6 feat(sm120): offline inference on NVIDIA Blackwell SM120
Run DeepSeek-V4.1-Flash offline inference on SM120 GPUs (RTX PRO 6000
Blackwell) with the same entry points as SM100. SM100 and Ascend paths
are unchanged.

- Build: SM120 is detected from the first visible GPU
  (scripts/nvidia_arch.sh). It builds DeepEP and DeepGEMM from its nv_dev
  branch (vendor/DeepGEMM-sm120), and no FlashMLA or DeepSelect;
  FlashInfer is installed from a pinned commit
  (requirements-nvidia-sm120.txt).
- Attention: V4.1 sparse MLA runs on FlashInfer's SM120 sparse MLA
  kernels, reading the inline-scale KV records as stored; KV blocks are
  aligned to 512 bytes.
- Indexer: FlashInfer's fused DSA indexer scores the index keys and
  selects their top-k in one call, for decode on every indexer layer
  (candidate source and consumer layers included) and for prefill outside
  the candidate layers. The remaining selections (prefill of the
  candidate layers, the sampler) use FlashInfer's top-k behind
  DeepSelect's interface (model_executor/layers/topk.py), in
  deterministic mode.
- MoE EP: DeepEP over NCCL GIN by default, or NCCL collectives
  (DEEPINFER_SM120_EP_BACKEND=nccl); NCCL settings known to break the EP
  paths are rejected. EPLB, including decode-only draft-layer EPLB, and
  per-layer YoCo MoE dispatch.
- Engram tables live in page-locked host shared memory; killed workers
  leave nothing in /dev/shm.
- Features: attention DP + MoE EP, DSpark/nextn speculative decoding,
  CUDA graphs, async scheduling, decoder SWA bounded replay, structured
  outputs, image inputs, multi-node deployments. Features without an SM120
  implementation (CP > 1, MTP varlen, batch-invariant mode, GPU-resident
  Engram tables) are refused at engine start.
- Eval: chat GSM8K and CV-Bench; tests for the SM120 kernels and their
  DeepInfer call sites.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 18:32:01 -08:00

252 lines
11 KiB
Python

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Independent torch references, input generators and comparison helpers for the SM120 kernel tests.
The quantization references here deliberately do not call into tile_kernels or the
DeepInfer reference helpers (which themselves call tile_kernels.quant.per_token_cast),
so a kernel bug cannot be masked by sharing code with its reference.
"""
from __future__ import annotations
import pytest
import torch
IS_SM120 = torch.cuda.is_available() and torch.cuda.get_device_capability()[0] == 12
SM120_MARKS = [
pytest.mark.gpu,
pytest.mark.skipif(not IS_SM120, reason="requires an SM120 (compute capability 12.x) GPU"),
]
# DeepSeek-V4.1-Flash text config (DeepSeek-V4.1-Flash-1017/config.json).
HIDDEN = 5120
HEAD_DIM = 512
ROPE_DIM = 64
NUM_HEADS = 64
Q_LORA_RANK = 1280
O_LORA_RANK = 1024
O_GROUPS = 8
MOE_INTER = 2304
N_ROUTED_EXPERTS = 384
NUM_TOPK = 6
ROUTED_SCALING = 1.5
SWIGLU_LIMIT = 10.0
RMS_EPS = 1e-20
HC_MULT = 4
HC_SINKHORN_ITERS = 20
HC_EPS = 1e-6
HC_POST_MULT = 2.0
INDEX_HEADS = 32
INDEX_HEAD_DIM = 128
VOCAB = 129280
SLIDING_WINDOW = 128
ENGRAM_MAX_NGRAM = 4
ENGRAM_HEADS = 8
# Row counts on both sides of 512 rows, where block-32 activation quantizers on SM120 are prone
# to races, on both sides of tile_kernels' 4096-row small-batch switch, and a large prefill.
QUANT_ROWS = (1, 511, 512, 513, 4096, 4097, 32768)
FP8_MAX = 448.0
FP4_MAX = 6.0
# Minimum amax used before computing the scale (deepinfer tilelang_ops/common.py, tile_kernels config).
FP8_AMAX_CLAMP = 1e-4
FP4_AMAX_CLAMP = 6.0 * 2.0**-126
_E2M1_MIDPOINTS = (0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0)
def align(x: int, a: int) -> int:
return (x + a - 1) // a * a
def ceil_log2(v: torch.Tensor) -> torch.Tensor:
"""Exact ceil(log2(v)) for positive finite v, as int32."""
mant, exp = torch.frexp(v.double())
return (exp - (mant == 0.5).to(exp.dtype)).to(torch.int32)
def ue8m0_exponents(amax: torch.Tensor, max_q: float, amax_clamp: float) -> torch.Tensor:
"""Unbiased exponent e of the UE8M0 scale 2^e = 2^ceil(log2(max(amax, clamp) / max_q))."""
ratio = amax.double().clamp_min(amax_clamp) / max_q
return ceil_log2(ratio)
def e2m1_encode(q: torch.Tensor) -> torch.Tensor:
"""Round-to-nearest-even onto the e2m1 grid {0, .5, 1, 1.5, 2, 3, 4, 6} with saturation; 4-bit codes."""
mag = q.abs().float()
mids = torch.tensor(_E2M1_MIDPOINTS, dtype=torch.float32, device=q.device)
idx = torch.searchsorted(mids, mag.contiguous(), right=False)
tie = (idx < len(_E2M1_MIDPOINTS)) & (mag == mids[idx.clamp_max(len(_E2M1_MIDPOINTS) - 1)])
idx = torch.where(tie & (idx % 2 == 1), idx + 1, idx)
return idx.to(torch.uint8) | (torch.signbit(q).to(torch.uint8) << 3)
def e2m1_decode(codes: torch.Tensor) -> torch.Tensor:
grid = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0], dtype=torch.float32, device=codes.device)
c = codes.to(torch.int64)
val = grid[c & 7]
return torch.where((c & 8) != 0, -val, val)
def pack_e2m1(codes: torch.Tensor) -> torch.Tensor:
"""[..., 2n] 4-bit codes -> [..., n] uint8, element 2i in the low nibble."""
c = codes.view(*codes.shape[:-1], codes.shape[-1] // 2, 2)
return c[..., 0] | (c[..., 1] << 4)
def unpack_e2m1(packed: torch.Tensor) -> torch.Tensor:
p = packed.view(torch.uint8)
return torch.stack([p & 0xF, p >> 4], dim=-1).flatten(-2)
def ref_ue8m0_quantize(x: torch.Tensor, fmt: str, gran: int = 32) -> tuple[torch.Tensor, torch.Tensor]:
"""Per-``gran`` block quantization with power-of-two (UE8M0) scales.
Args:
x: [..., K] float tensor (bf16/fp32); values are taken exactly in fp32.
fmt: "e4m3" or "e2m1".
Returns:
(data, sf): data is the uint8 encoding [..., K] for e4m3 or packed [..., K/2] for e2m1;
sf is the biased UE8M0 exponent byte [..., K/gran] (scale = 2^(sf - 127)).
"""
assert fmt in ("e4m3", "e2m1")
*lead, k = x.shape
assert k % gran == 0
xf = x.float().reshape(*lead, k // gran, gran)
max_q, clamp = (FP8_MAX, FP8_AMAX_CLAMP) if fmt == "e4m3" else (FP4_MAX, FP4_AMAX_CLAMP)
e = ue8m0_exponents(xf.abs().amax(-1), max_q, clamp)
q = xf * torch.ldexp(torch.ones_like(xf[..., 0]), -e).unsqueeze(-1)
if fmt == "e4m3":
data = q.clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn).view(torch.uint8).reshape(*lead, k)
else:
data = pack_e2m1(e2m1_encode(q).reshape(*lead, k))
return data, (e + 127).to(torch.uint8)
def ref_e4m3_sf_fp4_quantize(x: torch.Tensor, gran: int = 16) -> tuple[torch.Tensor, torch.Tensor]:
"""e2m1 data with E4M3 per-``gran`` scales (V41 FP4 main KV cache).
sf = e4m3(max(amax, 6 * 2^-9) / 6) and data = e2m1(x * (1 / sf)), as documented by
cache_common.ref_cast_from_bf16. Returns (packed data uint8 [..., K/2], sf bytes uint8 [..., K/gran]).
"""
*lead, k = x.shape
xf = x.float().reshape(*lead, k // gran, gran)
amax = xf.abs().amax(-1).clamp_min(2.0**-9 * FP4_MAX)
sf = (amax * (1.0 / FP4_MAX)).clamp_max(FP8_MAX).to(torch.float8_e4m3fn)
q = xf * sf.float().reciprocal().unsqueeze(-1)
data = pack_e2m1(e2m1_encode(q).reshape(*lead, k))
return data, sf.view(torch.uint8)
def dequant_e4m3_ue8m0(data: torch.Tensor, sf: torch.Tensor, gran: int = 32) -> torch.Tensor:
"""Exact fp32 dequantization of e4m3 bytes with biased UE8M0 exponent bytes."""
*lead, k = data.shape
v = data.view(torch.float8_e4m3fn).float().reshape(*lead, k // gran, gran)
scale = torch.ldexp(torch.ones_like(v[..., 0]), sf.to(torch.int32) - 127)
return (v * scale.unsqueeze(-1)).reshape(*lead, k)
def dequant_e2m1(data: torch.Tensor, scale: torch.Tensor, gran: int) -> torch.Tensor:
"""Exact fp32 dequantization of packed e2m1 with fp32 per-block scale values."""
v = e2m1_decode(unpack_e2m1(data))
*lead, k = v.shape
return (v.reshape(*lead, k // gran, gran) * scale.float().unsqueeze(-1)).reshape(*lead, k)
def ue8m0_bytes_to_scale(sf: torch.Tensor) -> torch.Tensor:
return torch.ldexp(torch.ones(sf.shape, dtype=torch.float32, device=sf.device), sf.to(torch.int32) - 127)
def unpack_int32_ue8m0(sf: torch.Tensor) -> torch.Tensor:
"""[..., P] int32 packed UE8M0 (little-endian, 4 consecutive groups) -> [..., 4P] uint8, any strides."""
assert sf.dtype == torch.int32
return torch.stack([(sf >> (8 * j)) & 0xFF for j in range(4)], dim=-1).flatten(-2).to(torch.uint8)
def make_activations(
m: int,
k: int,
*,
seed: int,
dtype: torch.dtype = torch.bfloat16,
special_rows: bool = True,
gran: int = 32,
) -> torch.Tensor:
"""N(0, 1) activations with ~0.1% x64 outliers and, cycling over rows, special rows:
all-zero, tiny (below the FP8 amax clamp), very tiny (1e-30), large (1e4), signed tiny,
and blocks whose amax is an exact power-of-two multiple of 448 / 6 (UE8M0 bin edges)."""
g = torch.Generator(device="cuda").manual_seed(seed)
x = torch.randn(m, k, generator=g, device="cuda", dtype=torch.float32)
outliers = torch.rand(m, k, generator=g, device="cuda") < 1e-3
x = torch.where(outliers, x * 64.0, x)
if special_rows and m >= 3:
rows = torch.arange(m, device="cuda")
r = rows % 19
scale = torch.ones(m, device="cuda")
scale = torch.where(r == 3, torch.zeros_like(scale), scale)
scale = torch.where(r == 5, torch.full_like(scale, 1e-6), scale)
scale = torch.where(r == 7, torch.full_like(scale, 1e-30), scale)
scale = torch.where(r == 11, torch.full_like(scale, 1e4), scale)
scale = torch.where(r == 13, torch.full_like(scale, -3e-8), scale)
x = x * scale.unsqueeze(1)
edge_rows = (r == 17).nonzero().flatten()
if edge_rows.numel():
xe = x[edge_rows].clamp(-1.0, 1.0).view(edge_rows.numel(), k // gran, gran)
j = torch.arange(k // gran, device="cuda")
base = torch.where(j % 2 == 0, torch.full_like(j, 448, dtype=torch.float32), torch.full_like(j, 6, dtype=torch.float32))
edge = base * torch.pow(2.0, (j % 9 - 4).float())
xe[:, :, 0] = edge
xe[:, :, 1] = -edge * 0.75
x[edge_rows] = xe.view(edge_rows.numel(), k)
return x.to(dtype)
def assert_bytes_equal(actual: torch.Tensor, expected: torch.Tensor, name: str) -> None:
a = actual.contiguous().view(torch.uint8) if actual.dtype != torch.uint8 else actual
e = expected.contiguous().view(torch.uint8) if expected.dtype != torch.uint8 else expected
assert a.shape == e.shape, f"{name}: shape {tuple(a.shape)} != {tuple(e.shape)}"
diff = a != e
n_bad = int(diff.sum())
if n_bad:
idx = diff.nonzero()[:8].tolist()
rows = diff.reshape(diff.shape[0], -1).any(-1).nonzero().flatten() if diff.ndim > 1 else diff.nonzero().flatten()
samples = [(i, int(a[tuple(i)]), int(e[tuple(i)])) for i in idx]
raise AssertionError(
f"{name}: {n_bad}/{a.numel()} bytes differ ({n_bad / a.numel():.3e}); "
f"rows affected={rows.numel()} first rows={rows[:8].tolist()}; (index, actual, expected)={samples}"
)
def assert_no_fp8_nan(x: torch.Tensor, name: str) -> None:
b = x.contiguous().view(torch.uint8)
n_nan = int(((b & 0x7F) == 0x7F).sum())
assert n_nan == 0, f"{name}: {n_nan} FP8 NaN bytes"
def max_rel_err(actual: torch.Tensor, expected: torch.Tensor) -> float:
a, e = actual.double(), expected.double()
return float((a - e).abs().max() / e.abs().max().clamp_min(1e-300))
def assert_close_scaled(actual: torch.Tensor, expected: torch.Tensor, name: str, *, rtol: float, atol_frac: float = 0.0) -> None:
"""|a - e| <= rtol * |e| + atol_frac * rowmax|e| elementwise, rows taken along dim 0.
Scaling the absolute term by each row's magnitude keeps rows of tiny values meaningful.
"""
assert actual.shape == expected.shape, f"{name}: {actual.shape=} {expected.shape=}"
a = actual.double().reshape(actual.shape[0], -1)
e = expected.double().reshape(expected.shape[0], -1)
assert torch.isfinite(a).all(), f"{name}: non-finite values in actual"
row_max = e.abs().amax(-1, keepdim=True)
bound = rtol * e.abs() + atol_frac * row_max
bad = (a - e).abs() > bound
if bad.any():
err = ((a - e).abs() / (e.abs() + atol_frac * row_max).clamp_min(1e-300))[bad]
idx = bad.nonzero()[:5].tolist()
raise AssertionError(
f"{name}: {int(bad.sum())}/{a.numel()} elements out of tolerance (rtol={rtol}, atol_frac={atol_frac}); "
f"max scaled err={float(err.max()):.3e}; samples={[(i, float(a[tuple(i)]), float(e[tuple(i)])) for i in idx]}"
)