mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-11-20 04:22:27 +00:00
Run DeepSeek-V4.1-Flash offline inference on SM120 GPUs (RTX PRO 6000 Blackwell) with the same entry points as SM100. SM100 and Ascend paths are unchanged. - Build: SM120 is detected from the first visible GPU (scripts/nvidia_arch.sh). It builds DeepEP and DeepGEMM from its nv_dev branch (vendor/DeepGEMM-sm120), and no FlashMLA or DeepSelect; FlashInfer is installed from a pinned commit (requirements-nvidia-sm120.txt). - Attention: V4.1 sparse MLA runs on FlashInfer's SM120 sparse MLA kernels, reading the inline-scale KV records as stored; KV blocks are aligned to 512 bytes. - Indexer: FlashInfer's fused DSA indexer scores the index keys and selects their top-k in one call, for decode on every indexer layer (candidate source and consumer layers included) and for prefill outside the candidate layers. The remaining selections (prefill of the candidate layers, the sampler) use FlashInfer's top-k behind DeepSelect's interface (model_executor/layers/topk.py), in deterministic mode. - MoE EP: DeepEP over NCCL GIN by default, or NCCL collectives (DEEPINFER_SM120_EP_BACKEND=nccl); NCCL settings known to break the EP paths are rejected. EPLB, including decode-only draft-layer EPLB, and per-layer YoCo MoE dispatch. - Engram tables live in page-locked host shared memory; killed workers leave nothing in /dev/shm. - Features: attention DP + MoE EP, DSpark/nextn speculative decoding, CUDA graphs, async scheduling, decoder SWA bounded replay, structured outputs, image inputs, multi-node deployments. Features without an SM120 implementation (CP > 1, MTP varlen, batch-invariant mode, GPU-resident Engram tables) are refused at engine start. - Eval: chat GSM8K and CV-Bench; tests for the SM120 kernels and their DeepInfer call sites. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
530 lines
26 KiB
Python
530 lines
26 KiB
Python
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
"""SM120 checks of the DeepSeek-V4.1 TileLang layer operators against torch references.
|
|
|
|
RMSNorm (tilelang_ops.norm), rotary (tile_kernels.transform.apply_rotary), the fused
|
|
norm/rope transform, the C1/C2 compressor kernels, the mHC operators used by
|
|
models/v4_modules/mhc_ops.py and mhc_modeling.py (including Sinkhorn), the MoE top-k gate
|
|
and the Engram kernels, all with V4.1 shapes.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import random
|
|
from collections.abc import Generator
|
|
|
|
import pytest
|
|
import tile_kernels
|
|
import torch
|
|
from tile_kernels.mhc.expand_kernel import expand_to_mhc_fwd
|
|
from tile_kernels.mhc.head_compute_mix_kernel import mhc_head_compute_mix_fwd
|
|
from tile_kernels.mhc.norm_fn_kernel import mhc_fn_normw_merge_fwd, mhc_reduce_partials_and_rmsnorm_fwd
|
|
from tile_kernels.mhc.post_kernel import mhc_post_fwd
|
|
from tile_kernels.mhc.pre_apply_mix_kernel import mhc_pre_apply_mix_fwd
|
|
from tile_kernels.mhc.pre_big_fuse_kernel import mhc_pre_big_fuse_fwd
|
|
from tile_kernels.mhc.pre_split_mixes_kernel import mhc_pre_split_mixes_fwd
|
|
from tile_kernels.mhc.sinkhorn_kernel import mhc_sinkhorn_fwd
|
|
from tile_kernels.moe import moe_topk_gate
|
|
from tile_kernels.torch import (
|
|
engram_gate_ref,
|
|
engram_hash_ref,
|
|
make_offsets,
|
|
mhc_head_compute_mix_ref,
|
|
mhc_post_ref,
|
|
mhc_pre_apply_mix_ref,
|
|
mhc_pre_norm_fn_partials_ref,
|
|
mhc_pre_norm_fn_ref,
|
|
mhc_pre_split_mixes_ref,
|
|
moe_topk_gate_forward,
|
|
rotary_embedding_ref,
|
|
sinkhorn_normalize_ref,
|
|
)
|
|
from tile_kernels.transform import apply_rotary
|
|
|
|
from deepinfer.model_executor.layers.tilelang_ops.compress import (
|
|
compress_and_update_state_decode,
|
|
compress_and_update_state_decode_ref,
|
|
compress_and_update_state_prefill,
|
|
compress_and_update_state_prefill_ref,
|
|
)
|
|
from deepinfer.model_executor.layers.tilelang_ops.fused_transform import (
|
|
NormArgs,
|
|
RopeArgs,
|
|
fused_norm_rope_hadamard_cast_store_kvcache,
|
|
)
|
|
from deepinfer.model_executor.layers.tilelang_ops.norm import norm
|
|
from tests.kernels._sm120_ref import (
|
|
ENGRAM_HEADS,
|
|
ENGRAM_MAX_NGRAM,
|
|
HC_EPS,
|
|
HC_MULT,
|
|
HC_POST_MULT,
|
|
HC_SINKHORN_ITERS,
|
|
HEAD_DIM,
|
|
HIDDEN,
|
|
INDEX_HEAD_DIM,
|
|
INDEX_HEADS,
|
|
N_ROUTED_EXPERTS,
|
|
NUM_HEADS,
|
|
NUM_TOPK,
|
|
Q_LORA_RANK,
|
|
RMS_EPS,
|
|
ROPE_DIM,
|
|
ROUTED_SCALING,
|
|
SM120_MARKS,
|
|
VOCAB,
|
|
assert_close_scaled,
|
|
make_activations,
|
|
max_rel_err,
|
|
)
|
|
|
|
pytestmark = SM120_MARKS
|
|
|
|
# bf16 results: allow a couple of ulps (2^-8 relative each) against an fp32 reference.
|
|
BF16_RTOL = 2.0**-6
|
|
|
|
|
|
@pytest.fixture(params=[False, True], ids=["nopdl", "pdl"])
|
|
def pdl(request: pytest.FixtureRequest) -> Generator[bool]:
|
|
prev = tile_kernels.config.get_pdl()
|
|
tile_kernels.set_pdl(request.param)
|
|
yield request.param
|
|
tile_kernels.set_pdl(prev)
|
|
|
|
|
|
def _rms_norm_ref(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
|
|
xf = x.float()
|
|
y = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps)
|
|
return y.to(x.dtype) * weight
|
|
|
|
|
|
def _rope_cache(max_pos: int, rope_dim: int, base: float = 10000.0) -> torch.Tensor:
|
|
inv_freq = 1.0 / (base ** (torch.arange(0, rope_dim, 2, dtype=torch.float32) / rope_dim))
|
|
freqs = torch.outer(torch.arange(max_pos, dtype=torch.float32), inv_freq)
|
|
return torch.cat([freqs.cos(), freqs.sin()], dim=-1).cuda()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# RMSNorm
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize("m", [1, 511, 513, 4096, 32768])
|
|
@pytest.mark.parametrize("k", [HIDDEN, Q_LORA_RANK, HEAD_DIM, INDEX_HEAD_DIM])
|
|
@pytest.mark.parametrize("has_residual", [False, True])
|
|
def test_norm(m: int, k: int, has_residual: bool) -> None:
|
|
"""tilelang_ops.norm (bf16 out) with eps=1e-20 on zero / tiny / outlier rows."""
|
|
x = make_activations(m, k, seed=m + k)
|
|
residual = make_activations(m, k, seed=m + k + 1, special_rows=False) if has_residual else None
|
|
if residual is not None:
|
|
residual[::5] = 0
|
|
weight = (torch.randn(k, device="cuda") * 0.1 + 1.0).to(torch.bfloat16)
|
|
|
|
result = norm(x, weight, residual=residual, eps=RMS_EPS)
|
|
if residual is not None:
|
|
out, res_out = result
|
|
summed = x + residual
|
|
assert torch.equal(res_out, summed), "residual_out != x + residual"
|
|
else:
|
|
out, summed = result, x
|
|
ref = _rms_norm_ref(summed, weight, RMS_EPS)
|
|
assert torch.isfinite(out.float()).all()
|
|
zero_rows = (summed == 0).all(-1)
|
|
assert (out[zero_rows] == 0).all(), "all-zero rows must normalize to zero"
|
|
assert_close_scaled(out, ref, "norm", rtol=BF16_RTOL, atol_frac=2.0**-8)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Rotary
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize("n", [1, 513, 4096])
|
|
@pytest.mark.parametrize("interleaved", [True, False], ids=["interleaved", "neox"])
|
|
@pytest.mark.parametrize("conjugate", [False, True])
|
|
@pytest.mark.parametrize("pos_dtype", [torch.int32, torch.int64])
|
|
def test_apply_rotary(n: int, interleaved: bool, conjugate: bool, pos_dtype: torch.dtype, pdl: bool) -> None:
|
|
"""rotary_embedding.RotaryEmbedding: in-place rope on the trailing 64 dims of the [n, 64, 512] context (conjugate on
|
|
the attention output), plus a separate single-head key."""
|
|
cos_sin = _rope_cache(1 << 17, ROPE_DIM)
|
|
positions = torch.randint(0, 1 << 17, (n,), device="cuda", dtype=pos_dtype)
|
|
context = make_activations(n * NUM_HEADS, HEAD_DIM, seed=n, special_rows=False).view(n, NUM_HEADS, HEAD_DIM)
|
|
key = make_activations(n, ROPE_DIM, seed=n + 1, special_rows=False).view(n, 1, ROPE_DIM)
|
|
half = ROPE_DIM // 2
|
|
cos, sin = cos_sin[:, :half], cos_sin[:, half:]
|
|
ref_q = rotary_embedding_ref(context[..., -ROPE_DIM:], cos, sin, positions, interleaved=interleaved, conjugate=conjugate)
|
|
ref_k = rotary_embedding_ref(key, cos, sin, positions, interleaved=interleaved, conjugate=conjugate)
|
|
untouched = context[..., :-ROPE_DIM].clone()
|
|
|
|
apply_rotary(context[..., -ROPE_DIM:], cos_sin, key, positions=positions, interleaved=interleaved, conjugate=conjugate)
|
|
|
|
assert torch.equal(context[..., :-ROPE_DIM], untouched), "rope wrote outside the rotary slice"
|
|
assert_close_scaled(context[..., -ROPE_DIM:].reshape(n, -1), ref_q.reshape(n, -1), "rope q", rtol=BF16_RTOL, atol_frac=2.0**-8)
|
|
assert_close_scaled(key.reshape(n, -1), ref_k.reshape(n, -1), "rope k", rtol=BF16_RTOL, atol_frac=2.0**-8)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fused norm + rope transform (bf16 output; the quantized paths are in test_sm120_quant.py)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize("n", [1, 511, 513, 4096])
|
|
@pytest.mark.parametrize(
|
|
"num_heads,head_dim,do_norm",
|
|
[(NUM_HEADS, HEAD_DIM, True), (1, HEAD_DIM, True), (INDEX_HEADS, INDEX_HEAD_DIM, False)],
|
|
ids=["q-64x512-norm", "kv-1x512-norm", "index-q-32x128"],
|
|
)
|
|
@pytest.mark.parametrize("is_neox", [False, True], ids=["interleaved", "neox"])
|
|
def test_fused_transform_bf16(n: int, num_heads: int, head_dim: int, do_norm: bool, is_neox: bool) -> None:
|
|
"""attn._q_b_norm_rope (q_b_norm + rope at offset head_dim - 64) against an unfused torch reference."""
|
|
g = torch.Generator(device="cuda").manual_seed(n + num_heads)
|
|
cos_sin = _rope_cache(1 << 16, ROPE_DIM)
|
|
positions = torch.randint(0, 1 << 16, (n,), generator=g, device="cuda", dtype=torch.int32)
|
|
x = make_activations(n * num_heads, head_dim, seed=n + 2).view(n, num_heads, head_dim)
|
|
weight = (torch.randn(head_dim, generator=g, device="cuda") * 0.1 + 1.0).to(torch.bfloat16)
|
|
rope = RopeArgs(positions, cos_sin, ROPE_DIM, is_neox, offset=head_dim - ROPE_DIM)
|
|
|
|
out = fused_norm_rope_hadamard_cast_store_kvcache(x, norm_args=NormArgs(weight, RMS_EPS) if do_norm else None, rope_args=rope)
|
|
|
|
ref = _rms_norm_ref(x, weight, RMS_EPS) if do_norm else x.clone()
|
|
half = ROPE_DIM // 2
|
|
ref[..., -ROPE_DIM:] = rotary_embedding_ref(
|
|
ref[..., -ROPE_DIM:], cos_sin[:, :half], cos_sin[:, half:], positions.long(), interleaved=not is_neox
|
|
)
|
|
assert out.shape == x.shape and out.dtype == torch.bfloat16
|
|
assert_close_scaled(
|
|
out.reshape(n * num_heads, -1), ref.reshape(n * num_heads, -1), "fused transform", rtol=2 * BF16_RTOL, atol_frac=2.0**-7
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Compressor (models/v4_modules/compress.py)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _cu(lens: torch.Tensor) -> torch.Tensor:
|
|
return torch.cumsum(torch.cat([torch.zeros(1, dtype=lens.dtype, device=lens.device), lens]), 0, dtype=torch.int32)
|
|
|
|
|
|
def _strided_state_cache(num_slots: int, size: int, dim: int) -> torch.Tensor:
|
|
"""fp32 [slots, size, 2, dim] carved out of a larger per-slot block (swa + kv regions precede it)."""
|
|
lead = 64 * 1024
|
|
base = torch.randn(num_slots, lead + size * 2 * dim, device="cuda")
|
|
return base[:, lead:].view(num_slots, size, 2, dim)
|
|
|
|
|
|
@pytest.mark.parametrize("compress_ratio", [1, 2])
|
|
@pytest.mark.parametrize("dim", [HEAD_DIM, INDEX_HEAD_DIM])
|
|
def test_compress_prefill(compress_ratio: int, dim: int) -> None:
|
|
"""compress_and_update_state_prefill with chunked prefill (cached prefix), incl. 1-token and odd-length chunks."""
|
|
torch.manual_seed(compress_ratio * 1000 + dim)
|
|
num_seq, num_slots, state_size = 12, 64, 136
|
|
seqlen_q = torch.randint(1, 700, (num_seq,), device="cuda")
|
|
seqlen_q[:3] = torch.tensor([1, 2, 3], device="cuda")
|
|
seqlen_k = seqlen_q + torch.randint(0, 3000, (num_seq,), device="cuda")
|
|
seqlen_k[3] = seqlen_q[3]
|
|
cu_q, cu_k = _cu(seqlen_q), _cu(seqlen_k)
|
|
n_comp = torch.clamp(seqlen_k // compress_ratio - (seqlen_k - seqlen_q) // compress_ratio, min=1)
|
|
cu_c = _cu(n_comp)
|
|
total_c = int(cu_c[-1])
|
|
mapping = torch.zeros(total_c, dtype=torch.int32, device="cuda")
|
|
for i in range(1, num_seq):
|
|
mapping[cu_c[i]] += 1
|
|
mapping = torch.cumsum(mapping, 0, dtype=torch.int32)
|
|
total_q = int(cu_q[-1])
|
|
score = torch.randn(total_q, dim, device="cuda") * 3
|
|
latent = torch.randn(total_q, dim, device="cuda")
|
|
state = _strided_state_cache(num_slots, state_size, dim)
|
|
state_idx = torch.randperm(num_slots, device="cuda")[:num_seq].int()
|
|
ref_state = state.clone()
|
|
|
|
out = compress_and_update_state_prefill(score, latent, state, state_idx, mapping, cu_q, cu_k, cu_c, compress_ratio)
|
|
ref = compress_and_update_state_prefill_ref(score, latent, ref_state, state_idx, mapping, cu_q, cu_k, cu_c, compress_ratio)
|
|
|
|
assert torch.equal(state, ref_state), "state cache update mismatch"
|
|
assert_close_scaled(out, ref.float(), "kv_compressed", rtol=BF16_RTOL, atol_frac=2.0**-8)
|
|
|
|
|
|
@pytest.mark.parametrize("compress_ratio", [1, 2])
|
|
@pytest.mark.parametrize("dim", [HEAD_DIM, INDEX_HEAD_DIM])
|
|
@pytest.mark.parametrize("nextn", [1, 4])
|
|
def test_compress_decode(compress_ratio: int, dim: int, nextn: int) -> None:
|
|
"""compress_and_update_state_decode with max_seqlen_q = nextn (DEEPINFER_USE_NEXTN_DECODE + 1)."""
|
|
torch.manual_seed(compress_ratio * 100 + dim + nextn)
|
|
random.seed(nextn)
|
|
num_seq, num_slots = 256, 300
|
|
state_size = max(136, compress_ratio + nextn - 1)
|
|
ctx_end = torch.randint(nextn, 5000, (num_seq,))
|
|
context_lens = (ctx_end.unsqueeze(1) - nextn + 1 + torch.arange(nextn)).flatten().int().cuda()
|
|
state_idx = torch.randperm(num_slots)[:num_seq].int().repeat_interleave(nextn).cuda()
|
|
total_q = num_seq * nextn
|
|
score = torch.randn(total_q, dim, device="cuda") * 3
|
|
latent = torch.randn(total_q, dim, device="cuda")
|
|
state = _strided_state_cache(num_slots, state_size, dim)
|
|
ref_state = state.clone()
|
|
|
|
out = compress_and_update_state_decode(score, latent, state, state_idx, context_lens, nextn, compress_ratio)
|
|
ref = compress_and_update_state_decode_ref(score, latent, ref_state, state_idx, context_lens, nextn, compress_ratio)
|
|
|
|
assert torch.equal(state, ref_state), "state cache update mismatch"
|
|
assert_close_scaled(out, ref.float(), "kv_compressed", rtol=BF16_RTOL, atol_frac=2.0**-8)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# mHC (models/v4_modules/mhc_ops.py, mhc_modeling.py)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
MHC3 = HC_MULT * (2 + HC_MULT)
|
|
|
|
|
|
@pytest.mark.parametrize("n", [1, 511, 513, 4096])
|
|
def test_mhc_expand_post_apply_mix(n: int, pdl: bool) -> None:
|
|
"""expand_to_mhc_fwd, mhc_post_fwd and mhc_pre_apply_mix_fwd with hc_mult=4, hidden=5120."""
|
|
hidden = make_activations(n, HIDDEN, seed=n)
|
|
expanded = torch.empty(n, HC_MULT, HIDDEN, dtype=torch.bfloat16, device="cuda")
|
|
expand_to_mhc_fwd(hidden, expanded)
|
|
assert torch.equal(expanded, hidden.unsqueeze(1).expand(n, HC_MULT, HIDDEN))
|
|
|
|
residual = make_activations(n * HC_MULT, HIDDEN, seed=n + 1).view(1, n, HC_MULT, HIDDEN)
|
|
x = make_activations(n, HIDDEN, seed=n + 2).view(1, n, HIDDEN)
|
|
post_mix = (torch.rand(1, n, HC_MULT, 1, device="cuda") * HC_POST_MULT).contiguous()
|
|
comb = torch.rand(1, n, HC_MULT, HC_MULT, device="cuda")
|
|
comb = comb / comb.sum(-1, keepdim=True)
|
|
out = mhc_post_fwd(x, residual, post_mix, comb)
|
|
assert_close_scaled(
|
|
out.view(n * HC_MULT, -1),
|
|
mhc_post_ref(x, residual, post_mix, comb).view(n * HC_MULT, -1),
|
|
"mhc_post",
|
|
rtol=BF16_RTOL,
|
|
atol_frac=2.0**-8,
|
|
)
|
|
|
|
mix = (torch.rand(n, HC_MULT, 1, device="cuda") + HC_EPS).contiguous()
|
|
o = torch.empty(n, HIDDEN, dtype=torch.bfloat16, device="cuda")
|
|
mhc_pre_apply_mix_fwd(residual.view(n, HC_MULT, HIDDEN), mix.view(n, HC_MULT), o)
|
|
ref = mhc_pre_apply_mix_ref(residual.view(n, HC_MULT, HIDDEN).float(), mix)
|
|
assert_close_scaled(o, ref, "mhc_pre_apply_mix", rtol=BF16_RTOL, atol_frac=2.0**-8)
|
|
|
|
|
|
def _mhc_inputs(n: int, seed: int) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
g = torch.Generator(device="cuda").manual_seed(seed)
|
|
residual = make_activations(n * HC_MULT, HIDDEN, seed=seed).view(1, n, HC_MULT, HIDDEN)
|
|
fn = torch.randn(MHC3, HC_MULT * HIDDEN, generator=g, device="cuda") * (HC_MULT * HIDDEN) ** -0.5
|
|
scale = torch.rand(3, generator=g, device="cuda") * 0.5 + 0.1
|
|
base = torch.randn(MHC3, generator=g, device="cuda") * 0.5
|
|
return residual, fn, scale, base
|
|
|
|
|
|
@pytest.mark.parametrize("n", [1, 511, 513, 4096])
|
|
@pytest.mark.parametrize("n_splits", [1, 16])
|
|
def test_mhc_reduce_partials_and_rmsnorm(n: int, n_splits: int, pdl: bool) -> None:
|
|
"""mhc_ops.mhc_pre_norm_fn epilogue: reduce split-K prenorm-GEMM partials + RMSNorm (eps = rms_norm_eps 1e-20)."""
|
|
residual, fn, _, _ = _mhc_inputs(n, seed=n + n_splits)
|
|
out_mul_s, sqrsum_s = mhc_pre_norm_fn_partials_ref(residual, fn, n_splits)
|
|
out_mul = torch.empty_like(out_mul_s[0])
|
|
sqrsum = torch.empty_like(sqrsum_s[0])
|
|
out = torch.empty(n, MHC3, dtype=torch.float32, device="cuda")
|
|
mhc_reduce_partials_and_rmsnorm_fwd(
|
|
out_mul_s, sqrsum_s, out_mul, sqrsum, out, rms_group_size=fn.shape[1], rms_eps=RMS_EPS, n_splits=n_splits
|
|
)
|
|
ref = mhc_pre_norm_fn_ref(residual, fn, None, RMS_EPS).view(n, MHC3)
|
|
zero_tokens = (residual.view(n, -1) == 0).all(-1)
|
|
assert (out[zero_tokens] == 0).all()
|
|
assert_close_scaled(out, ref, "mhc reduce+rmsnorm", rtol=1e-4, atol_frac=1e-5)
|
|
|
|
|
|
@pytest.mark.parametrize("n", [1, 513, 4096])
|
|
def test_mhc_fn_normw_merge(n: int) -> None:
|
|
"""mhc_fn_normw_merge_fwd (mhc_pre_norm_fn with a separate norm weight): fn * normw in fp32."""
|
|
_, fn, _, _ = _mhc_inputs(n, seed=n)
|
|
normw = (torch.randn(HC_MULT * HIDDEN, device="cuda") * 0.1 + 1.0).float()
|
|
out = torch.empty_like(fn)
|
|
mhc_fn_normw_merge_fwd(fn, normw, out)
|
|
assert torch.equal(out, fn * normw)
|
|
|
|
|
|
@pytest.mark.parametrize("n", [1, 511, 513, 4096, 32768])
|
|
def test_mhc_pre_split_mixes_and_sinkhorn(n: int, pdl: bool) -> None:
|
|
"""hc._hc_pre (hc_shifted): mhc_pre_split_mixes then sinkhorn_normalize(repeat=20, eps=1e-6)."""
|
|
g = torch.Generator(device="cuda").manual_seed(n)
|
|
mixes = torch.randn(1, n, MHC3, generator=g, device="cuda") * 3
|
|
_, _, scale, base = _mhc_inputs(1, seed=n)
|
|
pre = torch.empty(n, HC_MULT, device="cuda")
|
|
post = torch.empty(n, HC_MULT, device="cuda")
|
|
comb = torch.empty(n, HC_MULT * HC_MULT, device="cuda")
|
|
mhc_pre_split_mixes_fwd(mixes.view(n, MHC3), scale, base, pre, post, comb, HC_POST_MULT, HC_EPS)
|
|
ref_pre, ref_post, ref_comb = mhc_pre_split_mixes_ref(mixes, scale, base, HC_MULT, HC_POST_MULT, HC_EPS)
|
|
assert_close_scaled(pre, ref_pre.view(n, HC_MULT), "pre_layer_mix", rtol=1e-5, atol_frac=1e-6)
|
|
assert_close_scaled(post, ref_post.view(n, HC_MULT), "post_layer_mix", rtol=1e-5, atol_frac=1e-6)
|
|
assert_close_scaled(comb, ref_comb.reshape(n, -1), "comb_res_mix", rtol=1e-5, atol_frac=1e-6)
|
|
|
|
sk_in = ref_comb.view(n, HC_MULT, HC_MULT).contiguous()
|
|
sk_in[0] = -60.0 # saturated logits
|
|
sk_out = torch.empty_like(sk_in)
|
|
mhc_sinkhorn_fwd(sk_in, sk_out, HC_SINKHORN_ITERS, HC_EPS)
|
|
sk_ref = sinkhorn_normalize_ref(sk_in, HC_SINKHORN_ITERS, HC_EPS)
|
|
assert torch.isfinite(sk_out).all()
|
|
assert_close_scaled(sk_out.view(n, -1), sk_ref.view(n, -1), "sinkhorn", rtol=1e-4, atol_frac=1e-5)
|
|
# The last Sinkhorn step normalizes columns (row sums are only approximately 1, as in the reference).
|
|
assert (sk_out.sum(-2) - 1).abs().max() < 1e-4
|
|
|
|
|
|
@pytest.mark.parametrize("n", [1, 513, 4096])
|
|
@pytest.mark.parametrize("n_splits", [1, 16])
|
|
def test_mhc_pre_big_fuse(n: int, n_splits: int, pdl: bool) -> None:
|
|
"""mhc_modeling.mhc_pre_big_fuse epilogue (non-shifted HC): reduce + rmsnorm + split + sinkhorn + apply."""
|
|
residual, fn, scale, base = _mhc_inputs(n, seed=n + 7)
|
|
out_mul_s, sqrsum_s = mhc_pre_norm_fn_partials_ref(residual, fn, n_splits)
|
|
post = torch.empty(n, HC_MULT, device="cuda")
|
|
comb = torch.empty(n, HC_MULT * HC_MULT, device="cuda")
|
|
layer_input = torch.empty(n, HIDDEN, dtype=torch.bfloat16, device="cuda")
|
|
mhc_pre_big_fuse_fwd(
|
|
out_mul_s.view(n_splits, n, MHC3),
|
|
sqrsum_s.view(n_splits, n),
|
|
scale,
|
|
base,
|
|
residual.view(n, HC_MULT, HIDDEN),
|
|
post,
|
|
comb,
|
|
layer_input,
|
|
RMS_EPS,
|
|
HC_EPS,
|
|
HC_EPS,
|
|
HC_POST_MULT,
|
|
HC_SINKHORN_ITERS,
|
|
n_splits=n_splits,
|
|
)
|
|
mixes = mhc_pre_norm_fn_ref(residual, fn, None, RMS_EPS)
|
|
ref_pre, ref_post, ref_comb = mhc_pre_split_mixes_ref(mixes, scale, base, HC_MULT, HC_POST_MULT, HC_EPS)
|
|
ref_comb = sinkhorn_normalize_ref(ref_comb, HC_SINKHORN_ITERS, HC_EPS)
|
|
ref_input = mhc_pre_apply_mix_ref(residual.float(), ref_pre).view(n, HIDDEN)
|
|
assert_close_scaled(post, ref_post.view(n, HC_MULT), "post_mix", rtol=1e-4, atol_frac=1e-5)
|
|
assert_close_scaled(comb, ref_comb.view(n, -1), "comb_mix", rtol=1e-3, atol_frac=1e-4)
|
|
assert_close_scaled(layer_input, ref_input, "layer_input", rtol=BF16_RTOL, atol_frac=2.0**-8)
|
|
|
|
|
|
@pytest.mark.parametrize("n", [1, 513, 4096])
|
|
def test_mhc_head_compute_mix(n: int, pdl: bool) -> None:
|
|
g = torch.Generator(device="cuda").manual_seed(n)
|
|
mix = torch.randn(n, HC_MULT, generator=g, device="cuda") * 3
|
|
scale = torch.rand(1, generator=g, device="cuda") + 0.1
|
|
base = torch.randn(HC_MULT, generator=g, device="cuda")
|
|
out = torch.empty_like(mix)
|
|
mhc_head_compute_mix_fwd(mix, scale, base, out, HC_EPS)
|
|
assert_close_scaled(out, mhc_head_compute_mix_ref(mix, scale, base, HC_EPS), "head mix", rtol=1e-5, atol_frac=1e-6)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# MoE top-k gate
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize("n", [1, 511, 513, 4096, 32768])
|
|
@pytest.mark.parametrize("with_mask", [False, True])
|
|
@pytest.mark.parametrize("with_physical_map", [False, True])
|
|
@pytest.mark.parametrize("with_image", [False, True])
|
|
def test_moe_topk_gate(n: int, with_mask: bool, with_physical_map: bool, with_image: bool, pdl: bool) -> None:
|
|
"""mlp.DeepseekMoE.gate_forward on SM120 (no mega gate): sqrtsoftplus, noaux_tc bias, top-6 of 384, scale 1.5."""
|
|
g = torch.Generator(device="cuda").manual_seed(n)
|
|
logits = torch.randn(n, N_ROUTED_EXPERTS, generator=g, device="cuda") * 2
|
|
bias = torch.randn(N_ROUTED_EXPERTS, generator=g, device="cuda") * 0.1
|
|
mask = (torch.rand(n, generator=g, device="cuda") > 0.2) if with_mask else None
|
|
image_bias = torch.randn(N_ROUTED_EXPERTS, generator=g, device="cuda") * 0.1 if with_image else None
|
|
image_mask = (torch.rand(n, generator=g, device="cuda") < 0.3) if with_image else None
|
|
if with_physical_map:
|
|
# EPLB with 32 redundant experts: some logical experts have two physical replicas.
|
|
num_redundant = 32
|
|
to_physical = torch.full((N_ROUTED_EXPERTS, 2), -1, dtype=torch.int32, device="cuda")
|
|
to_physical[:, 0] = torch.arange(N_ROUTED_EXPERTS, dtype=torch.int32)
|
|
dup = torch.randperm(N_ROUTED_EXPERTS, generator=torch.Generator().manual_seed(n))[:num_redundant].cuda()
|
|
to_physical[dup, 1] = torch.arange(N_ROUTED_EXPERTS, N_ROUTED_EXPERTS + num_redundant, dtype=torch.int32, device="cuda")
|
|
logical_count = (to_physical >= 0).sum(-1).to(torch.int32)
|
|
else:
|
|
to_physical = logical_count = None
|
|
kwargs = dict(
|
|
num_topk=NUM_TOPK,
|
|
use_shared_as_routed=False,
|
|
num_shared_experts=0,
|
|
routed_scaling_factor=ROUTED_SCALING,
|
|
ep_rank=3,
|
|
mask=mask,
|
|
bias=bias,
|
|
image_bias=image_bias,
|
|
image_token_mask=image_mask,
|
|
to_physical_map=to_physical,
|
|
logical_count=logical_count,
|
|
scoring_func="sqrtsoftplus",
|
|
)
|
|
idx, w = moe_topk_gate(logits, **kwargs)
|
|
ref_idx, ref_w = moe_topk_gate_forward(logits, **kwargs)
|
|
active = mask if mask is not None else torch.ones(n, dtype=torch.bool, device="cuda")
|
|
# Kernel may emit the selected experts in a different order; compare as (expert -> weight) sets.
|
|
order = idx[active].argsort(-1)
|
|
ref_order = ref_idx[active].argsort(-1)
|
|
k_idx, k_w = idx[active].gather(1, order), w[active].gather(1, order)
|
|
r_idx, r_w = ref_idx[active].gather(1, ref_order), ref_w[active].gather(1, ref_order)
|
|
mismatch = (k_idx != r_idx).any(-1)
|
|
assert int(mismatch.sum()) == 0, f"{int(mismatch.sum())}/{int(active.sum())} tokens selected different experts"
|
|
assert_close_scaled(k_w, r_w, "topk weights", rtol=1e-5, atol_frac=1e-6)
|
|
torch.testing.assert_close(w[active].sum(-1), torch.full_like(w[active].sum(-1), ROUTED_SCALING), rtol=1e-5, atol=1e-5)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Engram
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _engram_tables(seed: int) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
g = torch.Generator().manual_seed(seed)
|
|
num_layers = 2
|
|
multipliers = torch.randint(0, 2**40, (num_layers, ENGRAM_MAX_NGRAM), generator=g, dtype=torch.int64) * 2 + 1
|
|
vocab_sizes = (16_000_000 + torch.randint(0, 50_000, (num_layers, ENGRAM_MAX_NGRAM - 1, ENGRAM_HEADS), generator=g)) | 1
|
|
vocab_sizes = vocab_sizes.to(torch.int32)
|
|
return multipliers.cuda(), vocab_sizes.cuda(), make_offsets(vocab_sizes.cuda())
|
|
|
|
|
|
@pytest.mark.parametrize("n", [1, 511, 513, 4096, 32768])
|
|
@pytest.mark.parametrize("with_image_mask", [False, True])
|
|
def test_engram_hash(n: int, with_image_mask: bool, pdl: bool) -> None:
|
|
"""deepseek.py: tile_kernels.engram.engram_hash(ngram_token_ids, multipliers, vocab_sizes, offsets, images_seq_mask)."""
|
|
multipliers, vocab_sizes, offsets = _engram_tables(n)
|
|
g = torch.Generator(device="cuda").manual_seed(n)
|
|
ids = torch.randint(0, VOCAB, (n, ENGRAM_MAX_NGRAM), generator=g, device="cuda", dtype=torch.int32)
|
|
ids[::9, 1:] = 2 # pad token in the n-gram history
|
|
mask = (torch.rand(n, generator=g, device="cuda") < 0.1) if with_image_mask else None
|
|
out = tile_kernels.engram.engram_hash(ids, multipliers, vocab_sizes, offsets, mask)
|
|
ref = engram_hash_ref(ids, multipliers, vocab_sizes, offsets, mask)
|
|
assert out.dtype == torch.int32 and torch.equal(out, ref)
|
|
|
|
|
|
def test_engram_fused_weight() -> None:
|
|
wh = (torch.randn(HC_MULT, HIDDEN, device="cuda") * 0.1 + 1).to(torch.bfloat16)
|
|
we = (torch.randn(HC_MULT, HIDDEN, device="cuda") * 0.1 + 1).to(torch.bfloat16)
|
|
out = tile_kernels.engram.fused_weight(wh, we)
|
|
assert out.dtype == torch.float32 and torch.equal(out, wh.float() * we.float())
|
|
|
|
|
|
@pytest.mark.parametrize("n", [1, 511, 513, 4096])
|
|
@pytest.mark.parametrize("with_image_mask", [False, True])
|
|
def test_engram_gate_fwd(n: int, with_image_mask: bool, pdl: bool) -> None:
|
|
"""engram.Engram.forward: engram_gate_fwd(hidden [n,4,5120], kv [n,5,5120], weight_fused, eps=1e-20, clamp=1e-6, inplace)."""
|
|
g = torch.Generator(device="cuda").manual_seed(n)
|
|
hidden = make_activations(n * HC_MULT, HIDDEN, seed=n).view(n, HC_MULT, HIDDEN)
|
|
kv = make_activations(n * (HC_MULT + 1), HIDDEN, seed=n + 1).view(n, HC_MULT + 1, HIDDEN)
|
|
wh = (torch.randn(HC_MULT, HIDDEN, generator=g, device="cuda") * 0.1 + 1).to(torch.bfloat16)
|
|
we = (torch.randn(HC_MULT, HIDDEN, generator=g, device="cuda") * 0.1 + 1).to(torch.bfloat16)
|
|
mask = (torch.rand(n, generator=g, device="cuda") < 0.2) if with_image_mask else None
|
|
ref = engram_gate_ref(hidden, kv, wh, we, clamp_value=1e-6, eps=RMS_EPS, image_token_mask=mask)
|
|
weight_fused = tile_kernels.engram.fused_weight(wh, we)
|
|
hidden_in = hidden.clone()
|
|
out, *rest = tile_kernels.engram.engram_gate_fwd(
|
|
hidden_in, kv, weight_fused, RMS_EPS, 1e-6, save_for_backward=False, image_token_mask=mask, inplace=True
|
|
)
|
|
assert out.data_ptr() == hidden_in.data_ptr() and all(r is None for r in rest)
|
|
if mask is not None:
|
|
assert torch.equal(out[mask], hidden[mask]), "image tokens must pass through unchanged"
|
|
assert torch.isfinite(out.float()).all()
|
|
assert_close_scaled(out.view(n * HC_MULT, -1), ref.view(n * HC_MULT, -1), "engram gate", rtol=BF16_RTOL, atol_frac=2.0**-8)
|
|
assert max_rel_err(out, ref) < 2e-2
|