mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-11-20 04:22:27 +00:00
Run DeepSeek-V4.1-Flash offline inference on SM120 GPUs (RTX PRO 6000 Blackwell) with the same entry points as SM100. SM100 and Ascend paths are unchanged. - Build: SM120 is detected from the first visible GPU (scripts/nvidia_arch.sh). It builds DeepEP and DeepGEMM from its nv_dev branch (vendor/DeepGEMM-sm120), and no FlashMLA or DeepSelect; FlashInfer is installed from a pinned commit (requirements-nvidia-sm120.txt). - Attention: V4.1 sparse MLA runs on FlashInfer's SM120 sparse MLA kernels, reading the inline-scale KV records as stored; KV blocks are aligned to 512 bytes. - Indexer: FlashInfer's fused DSA indexer scores the index keys and selects their top-k in one call, for decode on every indexer layer (candidate source and consumer layers included) and for prefill outside the candidate layers. The remaining selections (prefill of the candidate layers, the sampler) use FlashInfer's top-k behind DeepSelect's interface (model_executor/layers/topk.py), in deterministic mode. - MoE EP: DeepEP over NCCL GIN by default, or NCCL collectives (DEEPINFER_SM120_EP_BACKEND=nccl); NCCL settings known to break the EP paths are rejected. EPLB, including decode-only draft-layer EPLB, and per-layer YoCo MoE dispatch. - Engram tables live in page-locked host shared memory; killed workers leave nothing in /dev/shm. - Features: attention DP + MoE EP, DSpark/nextn speculative decoding, CUDA graphs, async scheduling, decoder SWA bounded replay, structured outputs, image inputs, multi-node deployments. Features without an SM120 implementation (CP > 1, MTP varlen, batch-invariant mode, GPU-resident Engram tables) are refused at engine start. - Eval: chat GSM8K and CV-Bench; tests for the SM120 kernels and their DeepInfer call sites. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
252 lines
11 KiB
Python
252 lines
11 KiB
Python
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
"""Independent torch references, input generators and comparison helpers for the SM120 kernel tests.
|
|
|
|
The quantization references here deliberately do not call into tile_kernels or the
|
|
DeepInfer reference helpers (which themselves call tile_kernels.quant.per_token_cast),
|
|
so a kernel bug cannot be masked by sharing code with its reference.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
import torch
|
|
|
|
IS_SM120 = torch.cuda.is_available() and torch.cuda.get_device_capability()[0] == 12
|
|
SM120_MARKS = [
|
|
pytest.mark.gpu,
|
|
pytest.mark.skipif(not IS_SM120, reason="requires an SM120 (compute capability 12.x) GPU"),
|
|
]
|
|
|
|
# DeepSeek-V4.1-Flash text config (DeepSeek-V4.1-Flash-1017/config.json).
|
|
HIDDEN = 5120
|
|
HEAD_DIM = 512
|
|
ROPE_DIM = 64
|
|
NUM_HEADS = 64
|
|
Q_LORA_RANK = 1280
|
|
O_LORA_RANK = 1024
|
|
O_GROUPS = 8
|
|
MOE_INTER = 2304
|
|
N_ROUTED_EXPERTS = 384
|
|
NUM_TOPK = 6
|
|
ROUTED_SCALING = 1.5
|
|
SWIGLU_LIMIT = 10.0
|
|
RMS_EPS = 1e-20
|
|
HC_MULT = 4
|
|
HC_SINKHORN_ITERS = 20
|
|
HC_EPS = 1e-6
|
|
HC_POST_MULT = 2.0
|
|
INDEX_HEADS = 32
|
|
INDEX_HEAD_DIM = 128
|
|
VOCAB = 129280
|
|
SLIDING_WINDOW = 128
|
|
ENGRAM_MAX_NGRAM = 4
|
|
ENGRAM_HEADS = 8
|
|
|
|
# Row counts on both sides of 512 rows, where block-32 activation quantizers on SM120 are prone
|
|
# to races, on both sides of tile_kernels' 4096-row small-batch switch, and a large prefill.
|
|
QUANT_ROWS = (1, 511, 512, 513, 4096, 4097, 32768)
|
|
|
|
FP8_MAX = 448.0
|
|
FP4_MAX = 6.0
|
|
# Minimum amax used before computing the scale (deepinfer tilelang_ops/common.py, tile_kernels config).
|
|
FP8_AMAX_CLAMP = 1e-4
|
|
FP4_AMAX_CLAMP = 6.0 * 2.0**-126
|
|
|
|
_E2M1_MIDPOINTS = (0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0)
|
|
|
|
|
|
def align(x: int, a: int) -> int:
|
|
return (x + a - 1) // a * a
|
|
|
|
|
|
def ceil_log2(v: torch.Tensor) -> torch.Tensor:
|
|
"""Exact ceil(log2(v)) for positive finite v, as int32."""
|
|
mant, exp = torch.frexp(v.double())
|
|
return (exp - (mant == 0.5).to(exp.dtype)).to(torch.int32)
|
|
|
|
|
|
def ue8m0_exponents(amax: torch.Tensor, max_q: float, amax_clamp: float) -> torch.Tensor:
|
|
"""Unbiased exponent e of the UE8M0 scale 2^e = 2^ceil(log2(max(amax, clamp) / max_q))."""
|
|
ratio = amax.double().clamp_min(amax_clamp) / max_q
|
|
return ceil_log2(ratio)
|
|
|
|
|
|
def e2m1_encode(q: torch.Tensor) -> torch.Tensor:
|
|
"""Round-to-nearest-even onto the e2m1 grid {0, .5, 1, 1.5, 2, 3, 4, 6} with saturation; 4-bit codes."""
|
|
mag = q.abs().float()
|
|
mids = torch.tensor(_E2M1_MIDPOINTS, dtype=torch.float32, device=q.device)
|
|
idx = torch.searchsorted(mids, mag.contiguous(), right=False)
|
|
tie = (idx < len(_E2M1_MIDPOINTS)) & (mag == mids[idx.clamp_max(len(_E2M1_MIDPOINTS) - 1)])
|
|
idx = torch.where(tie & (idx % 2 == 1), idx + 1, idx)
|
|
return idx.to(torch.uint8) | (torch.signbit(q).to(torch.uint8) << 3)
|
|
|
|
|
|
def e2m1_decode(codes: torch.Tensor) -> torch.Tensor:
|
|
grid = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0], dtype=torch.float32, device=codes.device)
|
|
c = codes.to(torch.int64)
|
|
val = grid[c & 7]
|
|
return torch.where((c & 8) != 0, -val, val)
|
|
|
|
|
|
def pack_e2m1(codes: torch.Tensor) -> torch.Tensor:
|
|
"""[..., 2n] 4-bit codes -> [..., n] uint8, element 2i in the low nibble."""
|
|
c = codes.view(*codes.shape[:-1], codes.shape[-1] // 2, 2)
|
|
return c[..., 0] | (c[..., 1] << 4)
|
|
|
|
|
|
def unpack_e2m1(packed: torch.Tensor) -> torch.Tensor:
|
|
p = packed.view(torch.uint8)
|
|
return torch.stack([p & 0xF, p >> 4], dim=-1).flatten(-2)
|
|
|
|
|
|
def ref_ue8m0_quantize(x: torch.Tensor, fmt: str, gran: int = 32) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""Per-``gran`` block quantization with power-of-two (UE8M0) scales.
|
|
|
|
Args:
|
|
x: [..., K] float tensor (bf16/fp32); values are taken exactly in fp32.
|
|
fmt: "e4m3" or "e2m1".
|
|
Returns:
|
|
(data, sf): data is the uint8 encoding [..., K] for e4m3 or packed [..., K/2] for e2m1;
|
|
sf is the biased UE8M0 exponent byte [..., K/gran] (scale = 2^(sf - 127)).
|
|
"""
|
|
assert fmt in ("e4m3", "e2m1")
|
|
*lead, k = x.shape
|
|
assert k % gran == 0
|
|
xf = x.float().reshape(*lead, k // gran, gran)
|
|
max_q, clamp = (FP8_MAX, FP8_AMAX_CLAMP) if fmt == "e4m3" else (FP4_MAX, FP4_AMAX_CLAMP)
|
|
e = ue8m0_exponents(xf.abs().amax(-1), max_q, clamp)
|
|
q = xf * torch.ldexp(torch.ones_like(xf[..., 0]), -e).unsqueeze(-1)
|
|
if fmt == "e4m3":
|
|
data = q.clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn).view(torch.uint8).reshape(*lead, k)
|
|
else:
|
|
data = pack_e2m1(e2m1_encode(q).reshape(*lead, k))
|
|
return data, (e + 127).to(torch.uint8)
|
|
|
|
|
|
def ref_e4m3_sf_fp4_quantize(x: torch.Tensor, gran: int = 16) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""e2m1 data with E4M3 per-``gran`` scales (V41 FP4 main KV cache).
|
|
|
|
sf = e4m3(max(amax, 6 * 2^-9) / 6) and data = e2m1(x * (1 / sf)), as documented by
|
|
cache_common.ref_cast_from_bf16. Returns (packed data uint8 [..., K/2], sf bytes uint8 [..., K/gran]).
|
|
"""
|
|
*lead, k = x.shape
|
|
xf = x.float().reshape(*lead, k // gran, gran)
|
|
amax = xf.abs().amax(-1).clamp_min(2.0**-9 * FP4_MAX)
|
|
sf = (amax * (1.0 / FP4_MAX)).clamp_max(FP8_MAX).to(torch.float8_e4m3fn)
|
|
q = xf * sf.float().reciprocal().unsqueeze(-1)
|
|
data = pack_e2m1(e2m1_encode(q).reshape(*lead, k))
|
|
return data, sf.view(torch.uint8)
|
|
|
|
|
|
def dequant_e4m3_ue8m0(data: torch.Tensor, sf: torch.Tensor, gran: int = 32) -> torch.Tensor:
|
|
"""Exact fp32 dequantization of e4m3 bytes with biased UE8M0 exponent bytes."""
|
|
*lead, k = data.shape
|
|
v = data.view(torch.float8_e4m3fn).float().reshape(*lead, k // gran, gran)
|
|
scale = torch.ldexp(torch.ones_like(v[..., 0]), sf.to(torch.int32) - 127)
|
|
return (v * scale.unsqueeze(-1)).reshape(*lead, k)
|
|
|
|
|
|
def dequant_e2m1(data: torch.Tensor, scale: torch.Tensor, gran: int) -> torch.Tensor:
|
|
"""Exact fp32 dequantization of packed e2m1 with fp32 per-block scale values."""
|
|
v = e2m1_decode(unpack_e2m1(data))
|
|
*lead, k = v.shape
|
|
return (v.reshape(*lead, k // gran, gran) * scale.float().unsqueeze(-1)).reshape(*lead, k)
|
|
|
|
|
|
def ue8m0_bytes_to_scale(sf: torch.Tensor) -> torch.Tensor:
|
|
return torch.ldexp(torch.ones(sf.shape, dtype=torch.float32, device=sf.device), sf.to(torch.int32) - 127)
|
|
|
|
|
|
def unpack_int32_ue8m0(sf: torch.Tensor) -> torch.Tensor:
|
|
"""[..., P] int32 packed UE8M0 (little-endian, 4 consecutive groups) -> [..., 4P] uint8, any strides."""
|
|
assert sf.dtype == torch.int32
|
|
return torch.stack([(sf >> (8 * j)) & 0xFF for j in range(4)], dim=-1).flatten(-2).to(torch.uint8)
|
|
|
|
|
|
def make_activations(
|
|
m: int,
|
|
k: int,
|
|
*,
|
|
seed: int,
|
|
dtype: torch.dtype = torch.bfloat16,
|
|
special_rows: bool = True,
|
|
gran: int = 32,
|
|
) -> torch.Tensor:
|
|
"""N(0, 1) activations with ~0.1% x64 outliers and, cycling over rows, special rows:
|
|
all-zero, tiny (below the FP8 amax clamp), very tiny (1e-30), large (1e4), signed tiny,
|
|
and blocks whose amax is an exact power-of-two multiple of 448 / 6 (UE8M0 bin edges)."""
|
|
g = torch.Generator(device="cuda").manual_seed(seed)
|
|
x = torch.randn(m, k, generator=g, device="cuda", dtype=torch.float32)
|
|
outliers = torch.rand(m, k, generator=g, device="cuda") < 1e-3
|
|
x = torch.where(outliers, x * 64.0, x)
|
|
if special_rows and m >= 3:
|
|
rows = torch.arange(m, device="cuda")
|
|
r = rows % 19
|
|
scale = torch.ones(m, device="cuda")
|
|
scale = torch.where(r == 3, torch.zeros_like(scale), scale)
|
|
scale = torch.where(r == 5, torch.full_like(scale, 1e-6), scale)
|
|
scale = torch.where(r == 7, torch.full_like(scale, 1e-30), scale)
|
|
scale = torch.where(r == 11, torch.full_like(scale, 1e4), scale)
|
|
scale = torch.where(r == 13, torch.full_like(scale, -3e-8), scale)
|
|
x = x * scale.unsqueeze(1)
|
|
edge_rows = (r == 17).nonzero().flatten()
|
|
if edge_rows.numel():
|
|
xe = x[edge_rows].clamp(-1.0, 1.0).view(edge_rows.numel(), k // gran, gran)
|
|
j = torch.arange(k // gran, device="cuda")
|
|
base = torch.where(j % 2 == 0, torch.full_like(j, 448, dtype=torch.float32), torch.full_like(j, 6, dtype=torch.float32))
|
|
edge = base * torch.pow(2.0, (j % 9 - 4).float())
|
|
xe[:, :, 0] = edge
|
|
xe[:, :, 1] = -edge * 0.75
|
|
x[edge_rows] = xe.view(edge_rows.numel(), k)
|
|
return x.to(dtype)
|
|
|
|
|
|
def assert_bytes_equal(actual: torch.Tensor, expected: torch.Tensor, name: str) -> None:
|
|
a = actual.contiguous().view(torch.uint8) if actual.dtype != torch.uint8 else actual
|
|
e = expected.contiguous().view(torch.uint8) if expected.dtype != torch.uint8 else expected
|
|
assert a.shape == e.shape, f"{name}: shape {tuple(a.shape)} != {tuple(e.shape)}"
|
|
diff = a != e
|
|
n_bad = int(diff.sum())
|
|
if n_bad:
|
|
idx = diff.nonzero()[:8].tolist()
|
|
rows = diff.reshape(diff.shape[0], -1).any(-1).nonzero().flatten() if diff.ndim > 1 else diff.nonzero().flatten()
|
|
samples = [(i, int(a[tuple(i)]), int(e[tuple(i)])) for i in idx]
|
|
raise AssertionError(
|
|
f"{name}: {n_bad}/{a.numel()} bytes differ ({n_bad / a.numel():.3e}); "
|
|
f"rows affected={rows.numel()} first rows={rows[:8].tolist()}; (index, actual, expected)={samples}"
|
|
)
|
|
|
|
|
|
def assert_no_fp8_nan(x: torch.Tensor, name: str) -> None:
|
|
b = x.contiguous().view(torch.uint8)
|
|
n_nan = int(((b & 0x7F) == 0x7F).sum())
|
|
assert n_nan == 0, f"{name}: {n_nan} FP8 NaN bytes"
|
|
|
|
|
|
def max_rel_err(actual: torch.Tensor, expected: torch.Tensor) -> float:
|
|
a, e = actual.double(), expected.double()
|
|
return float((a - e).abs().max() / e.abs().max().clamp_min(1e-300))
|
|
|
|
|
|
def assert_close_scaled(actual: torch.Tensor, expected: torch.Tensor, name: str, *, rtol: float, atol_frac: float = 0.0) -> None:
|
|
"""|a - e| <= rtol * |e| + atol_frac * rowmax|e| elementwise, rows taken along dim 0.
|
|
|
|
Scaling the absolute term by each row's magnitude keeps rows of tiny values meaningful.
|
|
"""
|
|
assert actual.shape == expected.shape, f"{name}: {actual.shape=} {expected.shape=}"
|
|
a = actual.double().reshape(actual.shape[0], -1)
|
|
e = expected.double().reshape(expected.shape[0], -1)
|
|
assert torch.isfinite(a).all(), f"{name}: non-finite values in actual"
|
|
row_max = e.abs().amax(-1, keepdim=True)
|
|
bound = rtol * e.abs() + atol_frac * row_max
|
|
bad = (a - e).abs() > bound
|
|
if bad.any():
|
|
err = ((a - e).abs() / (e.abs() + atol_frac * row_max).clamp_min(1e-300))[bad]
|
|
idx = bad.nonzero()[:5].tolist()
|
|
raise AssertionError(
|
|
f"{name}: {int(bad.sum())}/{a.numel()} elements out of tolerance (rtol={rtol}, atol_frac={atol_frac}); "
|
|
f"max scaled err={float(err.max()):.3e}; samples={[(i, float(a[tuple(i)]), float(e[tuple(i)])) for i in idx]}"
|
|
)
|