feat(sm120): offline inference on NVIDIA Blackwell SM120

Run DeepSeek-V4.1-Flash offline inference on SM120 GPUs (RTX PRO 6000
Blackwell) with the same entry points as SM100. SM100 and Ascend paths
are unchanged.

- Build: SM120 is detected from the first visible GPU
  (scripts/nvidia_arch.sh). It builds DeepEP and DeepGEMM from its nv_dev
  branch (vendor/DeepGEMM-sm120), and no FlashMLA or DeepSelect;
  FlashInfer is installed from a pinned commit
  (requirements-nvidia-sm120.txt).
- Attention: V4.1 sparse MLA runs on FlashInfer's SM120 sparse MLA
  kernels, reading the inline-scale KV records as stored; KV blocks are
  aligned to 512 bytes.
- Indexer: FlashInfer's fused DSA indexer scores the index keys and
  selects their top-k in one call, for decode on every indexer layer
  (candidate source and consumer layers included) and for prefill outside
  the candidate layers. The remaining selections (prefill of the
  candidate layers, the sampler) use FlashInfer's top-k behind
  DeepSelect's interface (model_executor/layers/topk.py), in
  deterministic mode.
- MoE EP: DeepEP over NCCL GIN by default, or NCCL collectives
  (DEEPINFER_SM120_EP_BACKEND=nccl); NCCL settings known to break the EP
  paths are rejected. EPLB, including decode-only draft-layer EPLB, and
  per-layer YoCo MoE dispatch.
- Engram tables live in page-locked host shared memory; killed workers
  leave nothing in /dev/shm.
- Features: attention DP + MoE EP, DSpark/nextn speculative decoding,
  CUDA graphs, async scheduling, decoder SWA bounded replay, structured
  outputs, image inputs, multi-node deployments. Features without an SM120
  implementation (CP > 1, MTP varlen, batch-invariant mode, GPU-resident
  Engram tables) are refused at engine start.
- Eval: chat GSM8K and CV-Bench; tests for the SM120 kernels and their
  DeepInfer call sites.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Zihua WuandClaude Opus 5.5 committed 2026-11-17 18:32:01 -08:00
1 parent 553741557a
commit efecbb5ba6
68 files changed
+8049 -120

No files matched your search

@@ -1,5 +1,6 @@
from __future__ import annotations
import os
from collections.abc import Generator
from contextlib import contextmanager
from typing import NamedTuple
@@ -14,6 +15,8 @@ from deep_ep import EngramBuffer, EPBuffer
from deepinfer import PLATFORM, envs
from deepinfer.logger import init_logger
from deepinfer.model_executor.parallel_utils.cp import CPManager
from deepinfer.model_executor.parallel_utils.host_engram import HostEngramBuffer
from deepinfer.model_executor.parallel_utils.nccl_ep import NcclEPBuffer
logger = init_logger(__name__)
@@ -32,10 +35,48 @@ class _SymBufferSpec(NamedTuple):
activation: str
_SM120_EP_NUM_SMS = 12
_DEEPEP_NUM_NOTIFY_THREADS = 128
def _gib(num_bytes: int) -> str:
return f"{num_bytes / (1 << 30):.2f} GiB"
def _init_nccl_comm(group: dist.ProcessGroup) -> None:
# DeepEP reuses torch's NCCL communicator, which a lazily initialized group creates on its first collective.
dist.all_reduce(torch.zeros(1, device="cuda"), group=group)
def _create_sm120_ep_buffer(ep_kwargs: dict, num_experts: int) -> EPBuffer:
group, allow_hybrid_mode = ep_kwargs["group"], ep_kwargs["allow_hybrid_mode"]
_init_nccl_comm(group)
num_scaleout_ranks, num_scaleup_ranks = deep_ep.get_logical_domain_size(group, allow_hybrid_mode=allow_hybrid_mode)
num_local_experts = num_experts // group.size()
# Hybrid dispatch counts each scale-up rank and local expert with one of its 128 notify threads.
if num_scaleout_ranks > 1 and num_scaleup_ranks + num_local_experts > _DEEPEP_NUM_NOTIFY_THREADS:
raise ValueError(
f"DEEPINFER_SM120_EP_BACKEND=deepep supports at most {_DEEPEP_NUM_NOTIFY_THREADS - num_scaleup_ranks} experts per "
f"EP rank, got {num_local_experts}; use a larger EP group or DEEPINFER_SM120_EP_BACKEND=nccl"
)
try:
buffer = EPBuffer(**ep_kwargs, explicitly_destroy=True)
except RuntimeError as error:
raise RuntimeError(
"DEEPINFER_SM120_EP_BACKEND=deepep could not create DeepEP's EPBuffer, which needs NCCL GIN (GDAKI) for the "
f"EP communicator (NCCL_LSA_TEAM_SIZE={os.environ.get('NCCL_LSA_TEAM_SIZE')}); "
"set DEEPINFER_SM120_EP_BACKEND=nccl to use NCCL collectives instead"
) from error
num_nvlink_ranks = buffer.get_physical_domain_size()[1]
if num_nvlink_ranks > 1:
buffer.destroy()
raise RuntimeError(
f"DeepEP EPBuffer on SM120 has a {num_nvlink_ranks}-rank LSA team, on which its kernels fault; "
f"set NCCL_LSA_TEAM_SIZE=1 (got {os.environ.get('NCCL_LSA_TEAM_SIZE')}) or DEEPINFER_SM120_EP_BACKEND=nccl"
)
return buffer
@contextmanager
def limit_num_sms(num_sms: int, enabled: bool = True) -> Generator[None]:
if not enabled:
@@ -55,9 +96,9 @@ class _UserBufferManager:
"""Owns all process-wide communication buffers used by one loaded model."""
def __init__(self) -> None:
self.ep: EPBuffer | None = None
self.ep: EPBuffer | NcclEPBuffer | None = None
self.cp: CPManager | None = None
self.engram: EngramBuffer | None = None
self.engram: EngramBuffer | HostEngramBuffer | None = None
self._sym_buffers: dict[_SymBufferSpec, deep_gemm.SymmBuffer] = {}
def allocate_ep(
@@ -68,7 +109,7 @@ class _UserBufferManager:
num_topk: int,
num_experts: int,
use_fp8_dispatch: bool,
) -> EPBuffer:
) -> EPBuffer | NcclEPBuffer:
ep_kwargs = dict(
group=ep_group,
num_max_tokens_per_rank=max_num_tokens,
@@ -78,12 +119,25 @@ class _UserBufferManager:
allow_hybrid_mode=True,
allow_multiple_reduction=not (envs.DEEPINFER_BATCH_INVARIANT and envs.DEEPINFER_EXPERT_REARRANGEMENT_STEP_INTERVAL > 0),
)
if PLATFORM.is_sm120 and envs.DEEPINFER_SM120_EP_BACKEND == "nccl":
del ep_kwargs["allow_hybrid_mode"]
if self.ep is None:
self.ep = NcclEPBuffer(**ep_kwargs)
logger.info(f"Created NCCL EP buffer, (EP {ep_group.rank()}/{ep_group.size()}), {ep_kwargs=}")
assert isinstance(self.ep, NcclEPBuffer) and self.ep.group is ep_group
return self.ep
if self.ep is not None:
assert self.ep.num_bytes >= EPBuffer.get_buffer_size_hint(**ep_kwargs)
return self.ep
self.ep = EPBuffer(**ep_kwargs, explicitly_destroy=True)
if envs.DEEPINFER_USERBUFFER_COMM_SMS == 0:
if PLATFORM.is_sm120:
self.ep = _create_sm120_ep_buffer(ep_kwargs, num_experts)
else:
self.ep = EPBuffer(**ep_kwargs, explicitly_destroy=True)
if envs.DEEPINFER_USERBUFFER_COMM_SMS == 0 and PLATFORM.is_sm120:
# The theoretical SM count is derived from the NVLink bandwidth, which SM120 hosts lack.
envs.DEEPINFER_USERBUFFER_COMM_SMS = _SM120_EP_NUM_SMS
elif envs.DEEPINFER_USERBUFFER_COMM_SMS == 0:
num_sms = self.ep.get_theoretical_num_sms(num_experts=num_experts, num_topk=num_topk)
device_num_sms = torch.cuda.get_device_properties().multi_processor_count
compute_num_sms = min(32, device_num_sms // 4)
@@ -133,8 +187,23 @@ class _UserBufferManager:
num_entries_per_token: int,
dtype: torch.dtype,
num_sf_packs: int = 0,
) -> EngramBuffer:
) -> EngramBuffer | HostEngramBuffer:
assert self.engram is None, "allocate_engram can only be called once"
if PLATFORM.is_sm120:
self.engram = HostEngramBuffer(
engram_group,
num_entries_per_layer,
hidden_size,
max_num_tokens,
num_entries_per_token,
dtype,
num_sf_packs,
)
logger.info(
f"Created {_gib(self.engram.num_bytes)} host-resident EngramBuffer "
f"({len(num_entries_per_layer)} layers, Engram {engram_group.rank()}/{engram_group.size()})"
)
return self.engram
num_gpu_bytes, num_rdma_storage_size = EngramBuffer.get_storage_size_hint(
engram_group,
num_entries_per_layer,