Files
DeepInfer/deepinfer/model_executor/parallel_utils/nccl_ep.py
T
Zihua WuandClaude Opus 5.5 efecbb5ba6 feat(sm120): offline inference on NVIDIA Blackwell SM120
Run DeepSeek-V4.1-Flash offline inference on SM120 GPUs (RTX PRO 6000
Blackwell) with the same entry points as SM100. SM100 and Ascend paths
are unchanged.

- Build: SM120 is detected from the first visible GPU
  (scripts/nvidia_arch.sh). It builds DeepEP and DeepGEMM from its nv_dev
  branch (vendor/DeepGEMM-sm120), and no FlashMLA or DeepSelect;
  FlashInfer is installed from a pinned commit
  (requirements-nvidia-sm120.txt).
- Attention: V4.1 sparse MLA runs on FlashInfer's SM120 sparse MLA
  kernels, reading the inline-scale KV records as stored; KV blocks are
  aligned to 512 bytes.
- Indexer: FlashInfer's fused DSA indexer scores the index keys and
  selects their top-k in one call, for decode on every indexer layer
  (candidate source and consumer layers included) and for prefill outside
  the candidate layers. The remaining selections (prefill of the
  candidate layers, the sampler) use FlashInfer's top-k behind
  DeepSelect's interface (model_executor/layers/topk.py), in
  deterministic mode.
- MoE EP: DeepEP over NCCL GIN by default, or NCCL collectives
  (DEEPINFER_SM120_EP_BACKEND=nccl); NCCL settings known to break the EP
  paths are rejected. EPLB, including decode-only draft-layer EPLB, and
  per-layer YoCo MoE dispatch.
- Engram tables live in page-locked host shared memory; killed workers
  leave nothing in /dev/shm.
- Features: attention DP + MoE EP, DSpark/nextn speculative decoding,
  CUDA graphs, async scheduling, decoder SWA bounded replay, structured
  outputs, image inputs, multi-node deployments. Features without an SM120
  implementation (CP > 1, MTP varlen, batch-invariant mode, GPU-resident
  Engram tables) are refused at engine start.
- Eval: chat GSM8K and CV-Bench; tests for the SM120 kernels and their
  DeepInfer call sites.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 18:32:01 -08:00

334 lines
16 KiB
Python

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Expert-parallel dispatch and combine over plain NCCL collectives.
`NcclEPBuffer` mirrors the subset of `deep_ep.EPBuffer` used by `DeepseekMoE`: the expanded dispatch
layout (one row per token and local expert selection, grouped per local expert, each group padded to
`expert_alignment`) and the matching combine. Dispatch all-gathers every rank's tokens and selections
and gathers the rows of local experts; combine reduces each source token's rows on the expert rank
and returns the partial sums to the source ranks with an all-to-all.
Without CPU sync every shape depends only on `num_max_tokens_per_rank`, so the path is CUDA graph
capturable. With CPU sync, the received sizes are read back once per dispatch to allocate exactly.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
import torch
import torch.distributed as dist
from deepinfer.utils import allow_cuda_sync
_SEGMENT_ALIGNMENT = 16
_NUM_ALIGNED_SF_BYTES = 16
_MAX_COMBINE_GATHER_BYTES = 256 << 20
def _align(x: int, alignment: int) -> int:
return (x + alignment - 1) // alignment * alignment
class _RowLayout:
"""Per-row byte segments of a packed uint8 `[num_rows, row_bytes]` exchange buffer."""
def __init__(self, segments: dict[str, tuple[torch.dtype, int]]) -> None:
self.segments: dict[str, tuple[int, torch.dtype, int]] = {}
offset = 0
for name, (dtype, width) in segments.items():
self.segments[name] = (offset, dtype, width)
offset += _align(width * dtype.itemsize, _SEGMENT_ALIGNMENT)
self.row_bytes = offset
def empty(self, num_rows: int, device: torch.device) -> torch.Tensor:
return torch.empty((num_rows, self.row_bytes), dtype=torch.uint8, device=device)
def view(self, rows: torch.Tensor, name: str) -> torch.Tensor:
offset, dtype, width = self.segments[name]
return rows[:, offset : offset + width * dtype.itemsize].view(dtype)
class _Completed:
"""Stand-in for `deep_ep.EventOverlap`: the work is already ordered on the current stream."""
def __init__(self, result: Any) -> None:
self._result = result
def current_stream_wait(self, release_handle: bool = False) -> Any:
return self._result
@dataclass
class NcclEPHandle:
"""Dispatch layout consumed by `NcclEPBuffer.combine`.
Attributes:
num_tokens: the number of tokens this rank dispatched.
num_rows_per_rank: the per-rank token slots used by the exchange; combine returns this many rows per rank.
num_expanded_tokens: the number of allocated expanded rows.
psum_num_recv_tokens_per_expert: `[num_local_experts]` int32, the aligned start of each local expert's
group plus its actual count, as in `deep_ep.EPHandle` expand mode.
num_unaligned_recv_tokens_per_expert: `[num_local_experts]` int32, the actual count of each local expert.
slot_idx: `[num_ranks * num_rows_per_rank, num_topk]` int64, the expanded row of each local selection
of each source token slot, -1 for selections of other ranks' experts.
"""
num_tokens: int
num_rows_per_rank: int
num_expanded_tokens: int
psum_num_recv_tokens_per_expert: torch.Tensor
num_unaligned_recv_tokens_per_expert: torch.Tensor
slot_idx: torch.Tensor
class NcclEPBuffer:
"""Drop-in for the `deep_ep.EPBuffer` expanded dispatch/combine used by `DeepseekMoE`.
Within each local expert's group, rows are ordered by source rank, then source token. Alignment padding
rows and rows past the last group are unspecified (no zero padding), and their received weights are zero.
"""
num_bytes = 0
def __init__(
self,
group: dist.ProcessGroup,
num_max_tokens_per_rank: int,
hidden: int,
num_topk: int,
use_fp8_dispatch: bool = False,
allow_multiple_reduction: bool = True,
) -> None:
self.group = group
self.rank_idx = group.rank()
self.num_ranks = group.size()
self.num_max_tokens_per_rank = num_max_tokens_per_rank
self.hidden = hidden
self.num_topk = num_topk
self.use_fp8_dispatch = use_fp8_dispatch
self.allow_multiple_reduction = allow_multiple_reduction
def destroy(self) -> None:
self.group = None
@staticmethod
def capture() -> None:
return None
def _all_gather(self, rows: torch.Tensor) -> torch.Tensor:
if self.num_ranks == 1:
return rows
out = rows.new_empty((self.num_ranks * rows.shape[0], *rows.shape[1:]))
dist.all_gather_into_tensor(out, rows, group=self.group)
return out
def dispatch(
self,
x: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
topk_idx: torch.Tensor,
topk_weights: torch.Tensor | None = None,
cumulative_local_expert_recv_stats: torch.Tensor | None = None,
num_experts: int | None = None,
num_max_tokens_per_rank: int | None = None,
expert_alignment: int | None = None,
num_sms: int = 0,
previous_event: Any = None,
async_with_compute_stream: bool = False,
allocate_on_comm_stream: bool = False,
do_handle_copy: bool = False,
do_cpu_sync: bool | None = None,
do_expand: bool = False,
use_tma_aligned_col_major_sf: bool = False,
defer_epilogue: bool = False,
) -> _Completed:
"""Dispatch tokens to the ranks owning their selected experts, in the `deep_ep.EPBuffer` expanded layout.
Experts are split evenly over ranks; selections outside `[0, num_experts)` (e.g. -1) are dropped.
All ranks must pass the same `num_max_tokens_per_rank` and `do_cpu_sync`. `current_stream_wait()` on the
result returns `(recv_x, None, recv_topk_weights, handle)`, with `recv_x` a `(fp8, sf)` tuple when `x` is one,
and `recv_topk_weights` `[num_expanded_tokens]`.
"""
assert do_expand, "only the expanded layout is supported"
assert not do_handle_copy
assert num_experts is not None and num_experts % self.num_ranks == 0
x, sf = x if isinstance(x, tuple) else (x, None)
num_max_tokens_per_rank = num_max_tokens_per_rank or self.num_max_tokens_per_rank
expert_alignment = expert_alignment or 1
do_cpu_sync = True if do_cpu_sync is None else do_cpu_sync
device = x.device
num_tokens, hidden = x.shape
num_topk = topk_idx.shape[1]
assert num_tokens <= num_max_tokens_per_rank, f"{num_tokens=} > {num_max_tokens_per_rank=}"
assert topk_idx.shape[0] == num_tokens and (topk_weights is None or topk_weights.shape == topk_idx.shape)
num_ranks = self.num_ranks
num_local_experts = num_experts // num_ranks
expert_begin = self.rank_idx * num_local_experts
meta_segments = {"topk_idx": (torch.int32, num_topk), "topk_weights": (torch.float32, num_topk)}
data_segments = {"x": (x.dtype, hidden)}
if sf is not None:
assert sf.shape[0] == num_tokens
data_segments["sf"] = (sf.dtype, sf.shape[1])
def fill_meta(layout: _RowLayout, rows: torch.Tensor) -> None:
idx = layout.view(rows, "topk_idx")
idx[:num_tokens].copy_(topk_idx)
idx[num_tokens:].fill_(-1)
weights = layout.view(rows, "topk_weights")
if topk_weights is None:
weights.zero_()
else:
weights[:num_tokens].copy_(topk_weights)
def fill_data(layout: _RowLayout, rows: torch.Tensor) -> None:
layout.view(rows, "x")[:num_tokens].copy_(x)
if sf is not None:
layout.view(rows, "sf")[:num_tokens].copy_(sf)
if do_cpu_sync:
# Exchange routing first at the static per-rank capacity, then the data at the largest actual count.
meta_layout = _RowLayout(meta_segments)
meta_rows = meta_layout.empty(num_max_tokens_per_rank + 1, device)
fill_meta(meta_layout, meta_rows)
meta_layout.view(meta_rows, "topk_idx")[num_max_tokens_per_rank].fill_(num_tokens)
meta_all = self._all_gather(meta_rows).view(num_ranks, num_max_tokens_per_rank + 1, -1)
num_tokens_per_rank = meta_layout.view(meta_all[:, num_max_tokens_per_rank], "topk_idx")[:, 0]
meta_all = meta_all[:, :num_max_tokens_per_rank].flatten(0, 1)
idx_all = meta_layout.view(meta_all, "topk_idx")
weights_all = meta_layout.view(meta_all, "topk_weights")
else:
layout = _RowLayout(meta_segments | data_segments)
rows = layout.empty(num_max_tokens_per_rank, device)
fill_meta(layout, rows)
fill_data(layout, rows)
rows_all = self._all_gather(rows)
idx_all = layout.view(rows_all, "topk_idx")
weights_all = layout.view(rows_all, "topk_weights")
# Sort selections by local expert; a stable sort keeps (source rank, source token) order within each expert.
flat_idx = idx_all.reshape(-1)
is_local = (flat_idx >= expert_begin) & (flat_idx < expert_begin + num_local_experts)
local_expert = torch.where(is_local, flat_idx - expert_begin, num_local_experts)
sorted_expert, order = torch.sort(local_expert, stable=True)
expert_range = torch.arange(num_local_experts + 1, dtype=sorted_expert.dtype, device=device)
unaligned_begin = torch.searchsorted(sorted_expert, expert_range)
num_recv_per_expert = unaligned_begin.diff()
num_aligned_per_expert = (num_recv_per_expert + expert_alignment - 1) // expert_alignment * expert_alignment
aligned_end = num_aligned_per_expert.cumsum(0)
aligned_begin = aligned_end - num_aligned_per_expert
psum_num_recv_tokens_per_expert = (aligned_begin + num_recv_per_expert).int()
num_unaligned_recv_tokens_per_expert = num_recv_per_expert.int()
if cumulative_local_expert_recv_stats is not None:
cumulative_local_expert_recv_stats.add_(num_unaligned_recv_tokens_per_expert.to(cumulative_local_expert_recv_stats.dtype))
if do_cpu_sync:
with allow_cuda_sync():
host_values = torch.cat([num_tokens_per_rank.long(), aligned_end[-1:]]).tolist()
num_rows_per_rank = max(1, max(host_values[:-1]))
num_expanded_tokens = host_values[-1]
layout = _RowLayout(data_segments)
rows = layout.empty(num_rows_per_rank, device)
fill_data(layout, rows)
rows_all = self._all_gather(rows)
else:
num_rows_per_rank = num_max_tokens_per_rank
num_expanded_tokens = num_ranks * num_max_tokens_per_rank * min(num_topk, num_local_experts)
num_expanded_tokens += (expert_alignment - 1) * num_local_experts
num_expanded_tokens = _align(num_expanded_tokens, expert_alignment)
# Expanded row of each sorted selection; selections of other ranks' experts go to a sink row.
sorted_is_local = sorted_expert < num_local_experts
clamped_expert = sorted_expert.clamp(max=num_local_experts - 1).long()
position = torch.arange(sorted_expert.numel(), device=device)
dst_row = aligned_begin[clamped_expert] + position - unaligned_begin[clamped_expert]
dst_row = torch.where(sorted_is_local, dst_row, num_expanded_tokens)
src_meta_token = order // num_topk
src_token = src_meta_token // num_max_tokens_per_rank * num_rows_per_rank + src_meta_token % num_max_tokens_per_rank
row_src = torch.zeros(num_expanded_tokens + 1, dtype=torch.long, device=device)
row_src.scatter_(0, dst_row, src_token)
row_src = row_src[:num_expanded_tokens]
recv_x = layout.view(rows_all, "x").index_select(0, row_src)
recv_sf = None
if sf is not None:
gathered_sf = layout.view(rows_all, "sf").index_select(0, row_src)
if use_tma_aligned_col_major_sf:
num_aligned_rows = _align(num_expanded_tokens, max(1, _NUM_ALIGNED_SF_BYTES // sf.element_size()))
recv_sf = torch.empty_strided(gathered_sf.shape, (1, num_aligned_rows), dtype=sf.dtype, device=device)
recv_sf.copy_(gathered_sf)
else:
recv_sf = gathered_sf
recv_topk_weights = None
if topk_weights is not None:
recv_topk_weights = torch.zeros(num_expanded_tokens + 1, dtype=torch.float32, device=device)
recv_topk_weights.scatter_(0, dst_row, weights_all.reshape(-1)[order])
recv_topk_weights = recv_topk_weights[:num_expanded_tokens]
slot_idx = torch.empty_like(order)
slot_idx.scatter_(0, order, torch.where(sorted_is_local, dst_row, -1))
slot_idx = slot_idx.view(num_ranks, num_max_tokens_per_rank, num_topk)[:, :num_rows_per_rank].reshape(-1, num_topk)
handle = NcclEPHandle(
num_tokens=num_tokens,
num_rows_per_rank=num_rows_per_rank,
num_expanded_tokens=num_expanded_tokens,
psum_num_recv_tokens_per_expert=psum_num_recv_tokens_per_expert,
num_unaligned_recv_tokens_per_expert=num_unaligned_recv_tokens_per_expert,
slot_idx=slot_idx,
)
recv_x = (recv_x, recv_sf) if recv_sf is not None else recv_x
return _Completed((recv_x, None, recv_topk_weights, handle))
def combine(
self,
x: torch.Tensor,
handle: NcclEPHandle,
topk_weights: torch.Tensor | None = None,
bias: torch.Tensor | None = None,
num_sms: int = 0,
previous_event: Any = None,
async_with_compute_stream: bool = False,
allocate_on_comm_stream: bool = False,
defer_epilogue: bool = False,
) -> _Completed:
"""Sum each source token's expanded rows (weights already applied) back on its rank, plus `bias`.
`current_stream_wait()` on the result returns `(combined_x, None)` with `combined_x` bf16 `[num_tokens, hidden]`.
"""
assert topk_weights is None, "expanded combine expects weights already applied"
assert x.dim() == 2 and x.shape[0] == handle.num_expanded_tokens
slot_idx = handle.slot_idx
num_slots, num_topk = slot_idx.shape
hidden = x.shape[1]
# One local reduction in fp32, rounded to bf16 for the exchange unless only a single reduction is allowed.
partial_dtype = torch.bfloat16 if self.allow_multiple_reduction else torch.float32
partial = torch.empty((num_slots, hidden), dtype=partial_dtype, device=x.device)
if x.shape[0] == 0:
partial.zero_()
else:
chunk = max(1, _MAX_COMBINE_GATHER_BYTES // (num_topk * hidden * x.element_size()))
for begin in range(0, num_slots, chunk):
slots = slot_idx[begin : begin + chunk]
gathered = x.index_select(0, slots.clamp(min=0).flatten()).view(-1, num_topk, hidden)
gathered.masked_fill_((slots < 0).unsqueeze(-1), 0)
torch.sum(gathered, dim=1, dtype=partial_dtype, out=partial[begin : begin + chunk])
if self.num_ranks > 1:
received = torch.empty_like(partial)
dist.all_to_all_single(received, partial, group=self.group)
else:
received = partial
received = received.view(self.num_ranks, handle.num_rows_per_rank, hidden)[:, : handle.num_tokens]
combined = torch.sum(received, dim=0, dtype=torch.float32)
if bias is not None:
combined.add_(bias)
return _Completed((combined.to(torch.bfloat16), None))