mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-11-18 19:42:28 +00:00
Run DeepSeek-V4.1-Flash offline inference on SM120 GPUs (RTX PRO 6000 Blackwell) with the same entry points as SM100. SM100 and Ascend paths are unchanged. - Build: SM120 is detected from the first visible GPU (scripts/nvidia_arch.sh). It builds DeepEP and DeepGEMM from its nv_dev branch (vendor/DeepGEMM-sm120), and no FlashMLA or DeepSelect; FlashInfer is installed from a pinned commit (requirements-nvidia-sm120.txt). - Attention: V4.1 sparse MLA runs on FlashInfer's SM120 sparse MLA kernels, reading the inline-scale KV records as stored; KV blocks are aligned to 512 bytes. - Indexer: FlashInfer's fused DSA indexer scores the index keys and selects their top-k in one call, for decode on every indexer layer (candidate source and consumer layers included) and for prefill outside the candidate layers. The remaining selections (prefill of the candidate layers, the sampler) use FlashInfer's top-k behind DeepSelect's interface (model_executor/layers/topk.py), in deterministic mode. - MoE EP: DeepEP over NCCL GIN by default, or NCCL collectives (DEEPINFER_SM120_EP_BACKEND=nccl); NCCL settings known to break the EP paths are rejected. EPLB, including decode-only draft-layer EPLB, and per-layer YoCo MoE dispatch. - Engram tables live in page-locked host shared memory; killed workers leave nothing in /dev/shm. - Features: attention DP + MoE EP, DSpark/nextn speculative decoding, CUDA graphs, async scheduling, decoder SWA bounded replay, structured outputs, image inputs, multi-node deployments. Features without an SM120 implementation (CP > 1, MTP varlen, batch-invariant mode, GPU-resident Engram tables) are refused at engine start. - Eval: chat GSM8K and CV-Bench; tests for the SM120 kernels and their DeepInfer call sites. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
334 lines
16 KiB
Python
334 lines
16 KiB
Python
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
"""Expert-parallel dispatch and combine over plain NCCL collectives.
|
|
|
|
`NcclEPBuffer` mirrors the subset of `deep_ep.EPBuffer` used by `DeepseekMoE`: the expanded dispatch
|
|
layout (one row per token and local expert selection, grouped per local expert, each group padded to
|
|
`expert_alignment`) and the matching combine. Dispatch all-gathers every rank's tokens and selections
|
|
and gathers the rows of local experts; combine reduces each source token's rows on the expert rank
|
|
and returns the partial sums to the source ranks with an all-to-all.
|
|
|
|
Without CPU sync every shape depends only on `num_max_tokens_per_rank`, so the path is CUDA graph
|
|
capturable. With CPU sync, the received sizes are read back once per dispatch to allocate exactly.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
import torch
|
|
import torch.distributed as dist
|
|
|
|
from deepinfer.utils import allow_cuda_sync
|
|
|
|
_SEGMENT_ALIGNMENT = 16
|
|
_NUM_ALIGNED_SF_BYTES = 16
|
|
_MAX_COMBINE_GATHER_BYTES = 256 << 20
|
|
|
|
|
|
def _align(x: int, alignment: int) -> int:
|
|
return (x + alignment - 1) // alignment * alignment
|
|
|
|
|
|
class _RowLayout:
|
|
"""Per-row byte segments of a packed uint8 `[num_rows, row_bytes]` exchange buffer."""
|
|
|
|
def __init__(self, segments: dict[str, tuple[torch.dtype, int]]) -> None:
|
|
self.segments: dict[str, tuple[int, torch.dtype, int]] = {}
|
|
offset = 0
|
|
for name, (dtype, width) in segments.items():
|
|
self.segments[name] = (offset, dtype, width)
|
|
offset += _align(width * dtype.itemsize, _SEGMENT_ALIGNMENT)
|
|
self.row_bytes = offset
|
|
|
|
def empty(self, num_rows: int, device: torch.device) -> torch.Tensor:
|
|
return torch.empty((num_rows, self.row_bytes), dtype=torch.uint8, device=device)
|
|
|
|
def view(self, rows: torch.Tensor, name: str) -> torch.Tensor:
|
|
offset, dtype, width = self.segments[name]
|
|
return rows[:, offset : offset + width * dtype.itemsize].view(dtype)
|
|
|
|
|
|
class _Completed:
|
|
"""Stand-in for `deep_ep.EventOverlap`: the work is already ordered on the current stream."""
|
|
|
|
def __init__(self, result: Any) -> None:
|
|
self._result = result
|
|
|
|
def current_stream_wait(self, release_handle: bool = False) -> Any:
|
|
return self._result
|
|
|
|
|
|
@dataclass
|
|
class NcclEPHandle:
|
|
"""Dispatch layout consumed by `NcclEPBuffer.combine`.
|
|
|
|
Attributes:
|
|
num_tokens: the number of tokens this rank dispatched.
|
|
num_rows_per_rank: the per-rank token slots used by the exchange; combine returns this many rows per rank.
|
|
num_expanded_tokens: the number of allocated expanded rows.
|
|
psum_num_recv_tokens_per_expert: `[num_local_experts]` int32, the aligned start of each local expert's
|
|
group plus its actual count, as in `deep_ep.EPHandle` expand mode.
|
|
num_unaligned_recv_tokens_per_expert: `[num_local_experts]` int32, the actual count of each local expert.
|
|
slot_idx: `[num_ranks * num_rows_per_rank, num_topk]` int64, the expanded row of each local selection
|
|
of each source token slot, -1 for selections of other ranks' experts.
|
|
"""
|
|
|
|
num_tokens: int
|
|
num_rows_per_rank: int
|
|
num_expanded_tokens: int
|
|
psum_num_recv_tokens_per_expert: torch.Tensor
|
|
num_unaligned_recv_tokens_per_expert: torch.Tensor
|
|
slot_idx: torch.Tensor
|
|
|
|
|
|
class NcclEPBuffer:
|
|
"""Drop-in for the `deep_ep.EPBuffer` expanded dispatch/combine used by `DeepseekMoE`.
|
|
|
|
Within each local expert's group, rows are ordered by source rank, then source token. Alignment padding
|
|
rows and rows past the last group are unspecified (no zero padding), and their received weights are zero.
|
|
"""
|
|
|
|
num_bytes = 0
|
|
|
|
def __init__(
|
|
self,
|
|
group: dist.ProcessGroup,
|
|
num_max_tokens_per_rank: int,
|
|
hidden: int,
|
|
num_topk: int,
|
|
use_fp8_dispatch: bool = False,
|
|
allow_multiple_reduction: bool = True,
|
|
) -> None:
|
|
self.group = group
|
|
self.rank_idx = group.rank()
|
|
self.num_ranks = group.size()
|
|
self.num_max_tokens_per_rank = num_max_tokens_per_rank
|
|
self.hidden = hidden
|
|
self.num_topk = num_topk
|
|
self.use_fp8_dispatch = use_fp8_dispatch
|
|
self.allow_multiple_reduction = allow_multiple_reduction
|
|
|
|
def destroy(self) -> None:
|
|
self.group = None
|
|
|
|
@staticmethod
|
|
def capture() -> None:
|
|
return None
|
|
|
|
def _all_gather(self, rows: torch.Tensor) -> torch.Tensor:
|
|
if self.num_ranks == 1:
|
|
return rows
|
|
out = rows.new_empty((self.num_ranks * rows.shape[0], *rows.shape[1:]))
|
|
dist.all_gather_into_tensor(out, rows, group=self.group)
|
|
return out
|
|
|
|
def dispatch(
|
|
self,
|
|
x: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
|
topk_idx: torch.Tensor,
|
|
topk_weights: torch.Tensor | None = None,
|
|
cumulative_local_expert_recv_stats: torch.Tensor | None = None,
|
|
num_experts: int | None = None,
|
|
num_max_tokens_per_rank: int | None = None,
|
|
expert_alignment: int | None = None,
|
|
num_sms: int = 0,
|
|
previous_event: Any = None,
|
|
async_with_compute_stream: bool = False,
|
|
allocate_on_comm_stream: bool = False,
|
|
do_handle_copy: bool = False,
|
|
do_cpu_sync: bool | None = None,
|
|
do_expand: bool = False,
|
|
use_tma_aligned_col_major_sf: bool = False,
|
|
defer_epilogue: bool = False,
|
|
) -> _Completed:
|
|
"""Dispatch tokens to the ranks owning their selected experts, in the `deep_ep.EPBuffer` expanded layout.
|
|
|
|
Experts are split evenly over ranks; selections outside `[0, num_experts)` (e.g. -1) are dropped.
|
|
All ranks must pass the same `num_max_tokens_per_rank` and `do_cpu_sync`. `current_stream_wait()` on the
|
|
result returns `(recv_x, None, recv_topk_weights, handle)`, with `recv_x` a `(fp8, sf)` tuple when `x` is one,
|
|
and `recv_topk_weights` `[num_expanded_tokens]`.
|
|
"""
|
|
assert do_expand, "only the expanded layout is supported"
|
|
assert not do_handle_copy
|
|
assert num_experts is not None and num_experts % self.num_ranks == 0
|
|
x, sf = x if isinstance(x, tuple) else (x, None)
|
|
num_max_tokens_per_rank = num_max_tokens_per_rank or self.num_max_tokens_per_rank
|
|
expert_alignment = expert_alignment or 1
|
|
do_cpu_sync = True if do_cpu_sync is None else do_cpu_sync
|
|
|
|
device = x.device
|
|
num_tokens, hidden = x.shape
|
|
num_topk = topk_idx.shape[1]
|
|
assert num_tokens <= num_max_tokens_per_rank, f"{num_tokens=} > {num_max_tokens_per_rank=}"
|
|
assert topk_idx.shape[0] == num_tokens and (topk_weights is None or topk_weights.shape == topk_idx.shape)
|
|
num_ranks = self.num_ranks
|
|
num_local_experts = num_experts // num_ranks
|
|
expert_begin = self.rank_idx * num_local_experts
|
|
|
|
meta_segments = {"topk_idx": (torch.int32, num_topk), "topk_weights": (torch.float32, num_topk)}
|
|
data_segments = {"x": (x.dtype, hidden)}
|
|
if sf is not None:
|
|
assert sf.shape[0] == num_tokens
|
|
data_segments["sf"] = (sf.dtype, sf.shape[1])
|
|
|
|
def fill_meta(layout: _RowLayout, rows: torch.Tensor) -> None:
|
|
idx = layout.view(rows, "topk_idx")
|
|
idx[:num_tokens].copy_(topk_idx)
|
|
idx[num_tokens:].fill_(-1)
|
|
weights = layout.view(rows, "topk_weights")
|
|
if topk_weights is None:
|
|
weights.zero_()
|
|
else:
|
|
weights[:num_tokens].copy_(topk_weights)
|
|
|
|
def fill_data(layout: _RowLayout, rows: torch.Tensor) -> None:
|
|
layout.view(rows, "x")[:num_tokens].copy_(x)
|
|
if sf is not None:
|
|
layout.view(rows, "sf")[:num_tokens].copy_(sf)
|
|
|
|
if do_cpu_sync:
|
|
# Exchange routing first at the static per-rank capacity, then the data at the largest actual count.
|
|
meta_layout = _RowLayout(meta_segments)
|
|
meta_rows = meta_layout.empty(num_max_tokens_per_rank + 1, device)
|
|
fill_meta(meta_layout, meta_rows)
|
|
meta_layout.view(meta_rows, "topk_idx")[num_max_tokens_per_rank].fill_(num_tokens)
|
|
meta_all = self._all_gather(meta_rows).view(num_ranks, num_max_tokens_per_rank + 1, -1)
|
|
num_tokens_per_rank = meta_layout.view(meta_all[:, num_max_tokens_per_rank], "topk_idx")[:, 0]
|
|
meta_all = meta_all[:, :num_max_tokens_per_rank].flatten(0, 1)
|
|
idx_all = meta_layout.view(meta_all, "topk_idx")
|
|
weights_all = meta_layout.view(meta_all, "topk_weights")
|
|
else:
|
|
layout = _RowLayout(meta_segments | data_segments)
|
|
rows = layout.empty(num_max_tokens_per_rank, device)
|
|
fill_meta(layout, rows)
|
|
fill_data(layout, rows)
|
|
rows_all = self._all_gather(rows)
|
|
idx_all = layout.view(rows_all, "topk_idx")
|
|
weights_all = layout.view(rows_all, "topk_weights")
|
|
|
|
# Sort selections by local expert; a stable sort keeps (source rank, source token) order within each expert.
|
|
flat_idx = idx_all.reshape(-1)
|
|
is_local = (flat_idx >= expert_begin) & (flat_idx < expert_begin + num_local_experts)
|
|
local_expert = torch.where(is_local, flat_idx - expert_begin, num_local_experts)
|
|
sorted_expert, order = torch.sort(local_expert, stable=True)
|
|
expert_range = torch.arange(num_local_experts + 1, dtype=sorted_expert.dtype, device=device)
|
|
unaligned_begin = torch.searchsorted(sorted_expert, expert_range)
|
|
num_recv_per_expert = unaligned_begin.diff()
|
|
num_aligned_per_expert = (num_recv_per_expert + expert_alignment - 1) // expert_alignment * expert_alignment
|
|
aligned_end = num_aligned_per_expert.cumsum(0)
|
|
aligned_begin = aligned_end - num_aligned_per_expert
|
|
psum_num_recv_tokens_per_expert = (aligned_begin + num_recv_per_expert).int()
|
|
num_unaligned_recv_tokens_per_expert = num_recv_per_expert.int()
|
|
if cumulative_local_expert_recv_stats is not None:
|
|
cumulative_local_expert_recv_stats.add_(num_unaligned_recv_tokens_per_expert.to(cumulative_local_expert_recv_stats.dtype))
|
|
|
|
if do_cpu_sync:
|
|
with allow_cuda_sync():
|
|
host_values = torch.cat([num_tokens_per_rank.long(), aligned_end[-1:]]).tolist()
|
|
num_rows_per_rank = max(1, max(host_values[:-1]))
|
|
num_expanded_tokens = host_values[-1]
|
|
layout = _RowLayout(data_segments)
|
|
rows = layout.empty(num_rows_per_rank, device)
|
|
fill_data(layout, rows)
|
|
rows_all = self._all_gather(rows)
|
|
else:
|
|
num_rows_per_rank = num_max_tokens_per_rank
|
|
num_expanded_tokens = num_ranks * num_max_tokens_per_rank * min(num_topk, num_local_experts)
|
|
num_expanded_tokens += (expert_alignment - 1) * num_local_experts
|
|
num_expanded_tokens = _align(num_expanded_tokens, expert_alignment)
|
|
|
|
# Expanded row of each sorted selection; selections of other ranks' experts go to a sink row.
|
|
sorted_is_local = sorted_expert < num_local_experts
|
|
clamped_expert = sorted_expert.clamp(max=num_local_experts - 1).long()
|
|
position = torch.arange(sorted_expert.numel(), device=device)
|
|
dst_row = aligned_begin[clamped_expert] + position - unaligned_begin[clamped_expert]
|
|
dst_row = torch.where(sorted_is_local, dst_row, num_expanded_tokens)
|
|
|
|
src_meta_token = order // num_topk
|
|
src_token = src_meta_token // num_max_tokens_per_rank * num_rows_per_rank + src_meta_token % num_max_tokens_per_rank
|
|
|
|
row_src = torch.zeros(num_expanded_tokens + 1, dtype=torch.long, device=device)
|
|
row_src.scatter_(0, dst_row, src_token)
|
|
row_src = row_src[:num_expanded_tokens]
|
|
|
|
recv_x = layout.view(rows_all, "x").index_select(0, row_src)
|
|
recv_sf = None
|
|
if sf is not None:
|
|
gathered_sf = layout.view(rows_all, "sf").index_select(0, row_src)
|
|
if use_tma_aligned_col_major_sf:
|
|
num_aligned_rows = _align(num_expanded_tokens, max(1, _NUM_ALIGNED_SF_BYTES // sf.element_size()))
|
|
recv_sf = torch.empty_strided(gathered_sf.shape, (1, num_aligned_rows), dtype=sf.dtype, device=device)
|
|
recv_sf.copy_(gathered_sf)
|
|
else:
|
|
recv_sf = gathered_sf
|
|
|
|
recv_topk_weights = None
|
|
if topk_weights is not None:
|
|
recv_topk_weights = torch.zeros(num_expanded_tokens + 1, dtype=torch.float32, device=device)
|
|
recv_topk_weights.scatter_(0, dst_row, weights_all.reshape(-1)[order])
|
|
recv_topk_weights = recv_topk_weights[:num_expanded_tokens]
|
|
|
|
slot_idx = torch.empty_like(order)
|
|
slot_idx.scatter_(0, order, torch.where(sorted_is_local, dst_row, -1))
|
|
slot_idx = slot_idx.view(num_ranks, num_max_tokens_per_rank, num_topk)[:, :num_rows_per_rank].reshape(-1, num_topk)
|
|
|
|
handle = NcclEPHandle(
|
|
num_tokens=num_tokens,
|
|
num_rows_per_rank=num_rows_per_rank,
|
|
num_expanded_tokens=num_expanded_tokens,
|
|
psum_num_recv_tokens_per_expert=psum_num_recv_tokens_per_expert,
|
|
num_unaligned_recv_tokens_per_expert=num_unaligned_recv_tokens_per_expert,
|
|
slot_idx=slot_idx,
|
|
)
|
|
recv_x = (recv_x, recv_sf) if recv_sf is not None else recv_x
|
|
return _Completed((recv_x, None, recv_topk_weights, handle))
|
|
|
|
def combine(
|
|
self,
|
|
x: torch.Tensor,
|
|
handle: NcclEPHandle,
|
|
topk_weights: torch.Tensor | None = None,
|
|
bias: torch.Tensor | None = None,
|
|
num_sms: int = 0,
|
|
previous_event: Any = None,
|
|
async_with_compute_stream: bool = False,
|
|
allocate_on_comm_stream: bool = False,
|
|
defer_epilogue: bool = False,
|
|
) -> _Completed:
|
|
"""Sum each source token's expanded rows (weights already applied) back on its rank, plus `bias`.
|
|
|
|
`current_stream_wait()` on the result returns `(combined_x, None)` with `combined_x` bf16 `[num_tokens, hidden]`.
|
|
"""
|
|
assert topk_weights is None, "expanded combine expects weights already applied"
|
|
assert x.dim() == 2 and x.shape[0] == handle.num_expanded_tokens
|
|
slot_idx = handle.slot_idx
|
|
num_slots, num_topk = slot_idx.shape
|
|
hidden = x.shape[1]
|
|
|
|
# One local reduction in fp32, rounded to bf16 for the exchange unless only a single reduction is allowed.
|
|
partial_dtype = torch.bfloat16 if self.allow_multiple_reduction else torch.float32
|
|
partial = torch.empty((num_slots, hidden), dtype=partial_dtype, device=x.device)
|
|
if x.shape[0] == 0:
|
|
partial.zero_()
|
|
else:
|
|
chunk = max(1, _MAX_COMBINE_GATHER_BYTES // (num_topk * hidden * x.element_size()))
|
|
for begin in range(0, num_slots, chunk):
|
|
slots = slot_idx[begin : begin + chunk]
|
|
gathered = x.index_select(0, slots.clamp(min=0).flatten()).view(-1, num_topk, hidden)
|
|
gathered.masked_fill_((slots < 0).unsqueeze(-1), 0)
|
|
torch.sum(gathered, dim=1, dtype=partial_dtype, out=partial[begin : begin + chunk])
|
|
|
|
if self.num_ranks > 1:
|
|
received = torch.empty_like(partial)
|
|
dist.all_to_all_single(received, partial, group=self.group)
|
|
else:
|
|
received = partial
|
|
received = received.view(self.num_ranks, handle.num_rows_per_rank, hidden)[:, : handle.num_tokens]
|
|
combined = torch.sum(received, dim=0, dtype=torch.float32)
|
|
if bias is not None:
|
|
combined.add_(bias)
|
|
return _Completed((combined.to(torch.bfloat16), None))
|