mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-11-20 04:22:27 +00:00
nccl_ep.py and host_engram.py each defined a private _align identical to the existing tilelang_ops.common.align (an import-free constants module, so the communication modules stay lightweight). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
331 lines
15 KiB
Python
331 lines
15 KiB
Python
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
"""Expert-parallel dispatch and combine over plain NCCL collectives.
|
|
|
|
`NcclEPBuffer` mirrors the subset of `deep_ep.EPBuffer` used by `DeepseekMoE`: the expanded dispatch
|
|
layout (one row per token and local expert selection, grouped per local expert, each group padded to
|
|
`expert_alignment`) and the matching combine. Dispatch all-gathers every rank's tokens and selections
|
|
and gathers the rows of local experts; combine reduces each source token's rows on the expert rank
|
|
and returns the partial sums to the source ranks with an all-to-all.
|
|
|
|
Without CPU sync every shape depends only on `num_max_tokens_per_rank`, so the path is CUDA graph
|
|
capturable. With CPU sync, the received sizes are read back once per dispatch to allocate exactly.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
import torch
|
|
import torch.distributed as dist
|
|
|
|
from deepinfer.model_executor.layers.tilelang_ops.common import align
|
|
from deepinfer.utils import allow_cuda_sync
|
|
|
|
_SEGMENT_ALIGNMENT = 16
|
|
_NUM_ALIGNED_SF_BYTES = 16
|
|
_MAX_COMBINE_GATHER_BYTES = 256 << 20
|
|
|
|
|
|
class _RowLayout:
|
|
"""Per-row byte segments of a packed uint8 `[num_rows, row_bytes]` exchange buffer."""
|
|
|
|
def __init__(self, segments: dict[str, tuple[torch.dtype, int]]) -> None:
|
|
self.segments: dict[str, tuple[int, torch.dtype, int]] = {}
|
|
offset = 0
|
|
for name, (dtype, width) in segments.items():
|
|
self.segments[name] = (offset, dtype, width)
|
|
offset += align(width * dtype.itemsize, _SEGMENT_ALIGNMENT)
|
|
self.row_bytes = offset
|
|
|
|
def empty(self, num_rows: int, device: torch.device) -> torch.Tensor:
|
|
return torch.empty((num_rows, self.row_bytes), dtype=torch.uint8, device=device)
|
|
|
|
def view(self, rows: torch.Tensor, name: str) -> torch.Tensor:
|
|
offset, dtype, width = self.segments[name]
|
|
return rows[:, offset : offset + width * dtype.itemsize].view(dtype)
|
|
|
|
|
|
class _Completed:
|
|
"""Stand-in for `deep_ep.EventOverlap`: the work is already ordered on the current stream."""
|
|
|
|
def __init__(self, result: Any) -> None:
|
|
self._result = result
|
|
|
|
def current_stream_wait(self, release_handle: bool = False) -> Any:
|
|
return self._result
|
|
|
|
|
|
@dataclass
|
|
class NcclEPHandle:
|
|
"""Dispatch layout consumed by `NcclEPBuffer.combine`.
|
|
|
|
Attributes:
|
|
num_tokens: the number of tokens this rank dispatched.
|
|
num_rows_per_rank: the per-rank token slots used by the exchange; combine returns this many rows per rank.
|
|
num_expanded_tokens: the number of allocated expanded rows.
|
|
psum_num_recv_tokens_per_expert: `[num_local_experts]` int32, the aligned start of each local expert's
|
|
group plus its actual count, as in `deep_ep.EPHandle` expand mode.
|
|
num_unaligned_recv_tokens_per_expert: `[num_local_experts]` int32, the actual count of each local expert.
|
|
slot_idx: `[num_ranks * num_rows_per_rank, num_topk]` int64, the expanded row of each local selection
|
|
of each source token slot, -1 for selections of other ranks' experts.
|
|
"""
|
|
|
|
num_tokens: int
|
|
num_rows_per_rank: int
|
|
num_expanded_tokens: int
|
|
psum_num_recv_tokens_per_expert: torch.Tensor
|
|
num_unaligned_recv_tokens_per_expert: torch.Tensor
|
|
slot_idx: torch.Tensor
|
|
|
|
|
|
class NcclEPBuffer:
|
|
"""Drop-in for the `deep_ep.EPBuffer` expanded dispatch/combine used by `DeepseekMoE`.
|
|
|
|
Within each local expert's group, rows are ordered by source rank, then source token. Alignment padding
|
|
rows and rows past the last group are unspecified (no zero padding), and their received weights are zero.
|
|
"""
|
|
|
|
num_bytes = 0
|
|
|
|
def __init__(
|
|
self,
|
|
group: dist.ProcessGroup,
|
|
num_max_tokens_per_rank: int,
|
|
hidden: int,
|
|
num_topk: int,
|
|
use_fp8_dispatch: bool = False,
|
|
allow_multiple_reduction: bool = True,
|
|
) -> None:
|
|
self.group = group
|
|
self.rank_idx = group.rank()
|
|
self.num_ranks = group.size()
|
|
self.num_max_tokens_per_rank = num_max_tokens_per_rank
|
|
self.hidden = hidden
|
|
self.num_topk = num_topk
|
|
self.use_fp8_dispatch = use_fp8_dispatch
|
|
self.allow_multiple_reduction = allow_multiple_reduction
|
|
|
|
def destroy(self) -> None:
|
|
self.group = None
|
|
|
|
@staticmethod
|
|
def capture() -> None:
|
|
return None
|
|
|
|
def _all_gather(self, rows: torch.Tensor) -> torch.Tensor:
|
|
if self.num_ranks == 1:
|
|
return rows
|
|
out = rows.new_empty((self.num_ranks * rows.shape[0], *rows.shape[1:]))
|
|
dist.all_gather_into_tensor(out, rows, group=self.group)
|
|
return out
|
|
|
|
def dispatch(
|
|
self,
|
|
x: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
|
topk_idx: torch.Tensor,
|
|
topk_weights: torch.Tensor | None = None,
|
|
cumulative_local_expert_recv_stats: torch.Tensor | None = None,
|
|
num_experts: int | None = None,
|
|
num_max_tokens_per_rank: int | None = None,
|
|
expert_alignment: int | None = None,
|
|
num_sms: int = 0,
|
|
previous_event: Any = None,
|
|
async_with_compute_stream: bool = False,
|
|
allocate_on_comm_stream: bool = False,
|
|
do_handle_copy: bool = False,
|
|
do_cpu_sync: bool | None = None,
|
|
do_expand: bool = False,
|
|
use_tma_aligned_col_major_sf: bool = False,
|
|
defer_epilogue: bool = False,
|
|
) -> _Completed:
|
|
"""Dispatch tokens to the ranks owning their selected experts, in the `deep_ep.EPBuffer` expanded layout.
|
|
|
|
Experts are split evenly over ranks; selections outside `[0, num_experts)` (e.g. -1) are dropped.
|
|
All ranks must pass the same `num_max_tokens_per_rank` and `do_cpu_sync`. `current_stream_wait()` on the
|
|
result returns `(recv_x, None, recv_topk_weights, handle)`, with `recv_x` a `(fp8, sf)` tuple when `x` is one,
|
|
and `recv_topk_weights` `[num_expanded_tokens]`.
|
|
"""
|
|
assert do_expand, "only the expanded layout is supported"
|
|
assert not do_handle_copy
|
|
assert num_experts is not None and num_experts % self.num_ranks == 0
|
|
x, sf = x if isinstance(x, tuple) else (x, None)
|
|
num_max_tokens_per_rank = num_max_tokens_per_rank or self.num_max_tokens_per_rank
|
|
expert_alignment = expert_alignment or 1
|
|
do_cpu_sync = True if do_cpu_sync is None else do_cpu_sync
|
|
|
|
device = x.device
|
|
num_tokens, hidden = x.shape
|
|
num_topk = topk_idx.shape[1]
|
|
assert num_tokens <= num_max_tokens_per_rank, f"{num_tokens=} > {num_max_tokens_per_rank=}"
|
|
assert topk_idx.shape[0] == num_tokens and (topk_weights is None or topk_weights.shape == topk_idx.shape)
|
|
num_ranks = self.num_ranks
|
|
num_local_experts = num_experts // num_ranks
|
|
expert_begin = self.rank_idx * num_local_experts
|
|
|
|
meta_segments = {"topk_idx": (torch.int32, num_topk), "topk_weights": (torch.float32, num_topk)}
|
|
data_segments = {"x": (x.dtype, hidden)}
|
|
if sf is not None:
|
|
assert sf.shape[0] == num_tokens
|
|
data_segments["sf"] = (sf.dtype, sf.shape[1])
|
|
|
|
def fill_meta(layout: _RowLayout, rows: torch.Tensor) -> None:
|
|
idx = layout.view(rows, "topk_idx")
|
|
idx[:num_tokens].copy_(topk_idx)
|
|
idx[num_tokens:].fill_(-1)
|
|
weights = layout.view(rows, "topk_weights")
|
|
if topk_weights is None:
|
|
weights.zero_()
|
|
else:
|
|
weights[:num_tokens].copy_(topk_weights)
|
|
|
|
def fill_data(layout: _RowLayout, rows: torch.Tensor) -> None:
|
|
layout.view(rows, "x")[:num_tokens].copy_(x)
|
|
if sf is not None:
|
|
layout.view(rows, "sf")[:num_tokens].copy_(sf)
|
|
|
|
if do_cpu_sync:
|
|
# Exchange routing first at the static per-rank capacity, then the data at the largest actual count.
|
|
meta_layout = _RowLayout(meta_segments)
|
|
meta_rows = meta_layout.empty(num_max_tokens_per_rank + 1, device)
|
|
fill_meta(meta_layout, meta_rows)
|
|
meta_layout.view(meta_rows, "topk_idx")[num_max_tokens_per_rank].fill_(num_tokens)
|
|
meta_all = self._all_gather(meta_rows).view(num_ranks, num_max_tokens_per_rank + 1, -1)
|
|
num_tokens_per_rank = meta_layout.view(meta_all[:, num_max_tokens_per_rank], "topk_idx")[:, 0]
|
|
meta_all = meta_all[:, :num_max_tokens_per_rank].flatten(0, 1)
|
|
idx_all = meta_layout.view(meta_all, "topk_idx")
|
|
weights_all = meta_layout.view(meta_all, "topk_weights")
|
|
else:
|
|
layout = _RowLayout(meta_segments | data_segments)
|
|
rows = layout.empty(num_max_tokens_per_rank, device)
|
|
fill_meta(layout, rows)
|
|
fill_data(layout, rows)
|
|
rows_all = self._all_gather(rows)
|
|
idx_all = layout.view(rows_all, "topk_idx")
|
|
weights_all = layout.view(rows_all, "topk_weights")
|
|
|
|
# Sort selections by local expert; a stable sort keeps (source rank, source token) order within each expert.
|
|
flat_idx = idx_all.reshape(-1)
|
|
is_local = (flat_idx >= expert_begin) & (flat_idx < expert_begin + num_local_experts)
|
|
local_expert = torch.where(is_local, flat_idx - expert_begin, num_local_experts)
|
|
sorted_expert, order = torch.sort(local_expert, stable=True)
|
|
expert_range = torch.arange(num_local_experts + 1, dtype=sorted_expert.dtype, device=device)
|
|
unaligned_begin = torch.searchsorted(sorted_expert, expert_range)
|
|
num_recv_per_expert = unaligned_begin.diff()
|
|
num_aligned_per_expert = (num_recv_per_expert + expert_alignment - 1) // expert_alignment * expert_alignment
|
|
aligned_end = num_aligned_per_expert.cumsum(0)
|
|
aligned_begin = aligned_end - num_aligned_per_expert
|
|
psum_num_recv_tokens_per_expert = (aligned_begin + num_recv_per_expert).int()
|
|
num_unaligned_recv_tokens_per_expert = num_recv_per_expert.int()
|
|
if cumulative_local_expert_recv_stats is not None:
|
|
cumulative_local_expert_recv_stats.add_(num_unaligned_recv_tokens_per_expert.to(cumulative_local_expert_recv_stats.dtype))
|
|
|
|
if do_cpu_sync:
|
|
with allow_cuda_sync():
|
|
host_values = torch.cat([num_tokens_per_rank.long(), aligned_end[-1:]]).tolist()
|
|
num_rows_per_rank = max(1, max(host_values[:-1]))
|
|
num_expanded_tokens = host_values[-1]
|
|
layout = _RowLayout(data_segments)
|
|
rows = layout.empty(num_rows_per_rank, device)
|
|
fill_data(layout, rows)
|
|
rows_all = self._all_gather(rows)
|
|
else:
|
|
num_rows_per_rank = num_max_tokens_per_rank
|
|
num_expanded_tokens = num_ranks * num_max_tokens_per_rank * min(num_topk, num_local_experts)
|
|
num_expanded_tokens += (expert_alignment - 1) * num_local_experts
|
|
num_expanded_tokens = align(num_expanded_tokens, expert_alignment)
|
|
|
|
# Expanded row of each sorted selection; selections of other ranks' experts go to a sink row.
|
|
sorted_is_local = sorted_expert < num_local_experts
|
|
clamped_expert = sorted_expert.clamp(max=num_local_experts - 1).long()
|
|
position = torch.arange(sorted_expert.numel(), device=device)
|
|
dst_row = aligned_begin[clamped_expert] + position - unaligned_begin[clamped_expert]
|
|
dst_row = torch.where(sorted_is_local, dst_row, num_expanded_tokens)
|
|
|
|
src_meta_token = order // num_topk
|
|
src_token = src_meta_token // num_max_tokens_per_rank * num_rows_per_rank + src_meta_token % num_max_tokens_per_rank
|
|
|
|
row_src = torch.zeros(num_expanded_tokens + 1, dtype=torch.long, device=device)
|
|
row_src.scatter_(0, dst_row, src_token)
|
|
row_src = row_src[:num_expanded_tokens]
|
|
|
|
recv_x = layout.view(rows_all, "x").index_select(0, row_src)
|
|
recv_sf = None
|
|
if sf is not None:
|
|
gathered_sf = layout.view(rows_all, "sf").index_select(0, row_src)
|
|
if use_tma_aligned_col_major_sf:
|
|
num_aligned_rows = align(num_expanded_tokens, max(1, _NUM_ALIGNED_SF_BYTES // sf.element_size()))
|
|
recv_sf = torch.empty_strided(gathered_sf.shape, (1, num_aligned_rows), dtype=sf.dtype, device=device)
|
|
recv_sf.copy_(gathered_sf)
|
|
else:
|
|
recv_sf = gathered_sf
|
|
|
|
recv_topk_weights = None
|
|
if topk_weights is not None:
|
|
recv_topk_weights = torch.zeros(num_expanded_tokens + 1, dtype=torch.float32, device=device)
|
|
recv_topk_weights.scatter_(0, dst_row, weights_all.reshape(-1)[order])
|
|
recv_topk_weights = recv_topk_weights[:num_expanded_tokens]
|
|
|
|
slot_idx = torch.empty_like(order)
|
|
slot_idx.scatter_(0, order, torch.where(sorted_is_local, dst_row, -1))
|
|
slot_idx = slot_idx.view(num_ranks, num_max_tokens_per_rank, num_topk)[:, :num_rows_per_rank].reshape(-1, num_topk)
|
|
|
|
handle = NcclEPHandle(
|
|
num_tokens=num_tokens,
|
|
num_rows_per_rank=num_rows_per_rank,
|
|
num_expanded_tokens=num_expanded_tokens,
|
|
psum_num_recv_tokens_per_expert=psum_num_recv_tokens_per_expert,
|
|
num_unaligned_recv_tokens_per_expert=num_unaligned_recv_tokens_per_expert,
|
|
slot_idx=slot_idx,
|
|
)
|
|
recv_x = (recv_x, recv_sf) if recv_sf is not None else recv_x
|
|
return _Completed((recv_x, None, recv_topk_weights, handle))
|
|
|
|
def combine(
|
|
self,
|
|
x: torch.Tensor,
|
|
handle: NcclEPHandle,
|
|
topk_weights: torch.Tensor | None = None,
|
|
bias: torch.Tensor | None = None,
|
|
num_sms: int = 0,
|
|
previous_event: Any = None,
|
|
async_with_compute_stream: bool = False,
|
|
allocate_on_comm_stream: bool = False,
|
|
defer_epilogue: bool = False,
|
|
) -> _Completed:
|
|
"""Sum each source token's expanded rows (weights already applied) back on its rank, plus `bias`.
|
|
|
|
`current_stream_wait()` on the result returns `(combined_x, None)` with `combined_x` bf16 `[num_tokens, hidden]`.
|
|
"""
|
|
assert topk_weights is None, "expanded combine expects weights already applied"
|
|
assert x.dim() == 2 and x.shape[0] == handle.num_expanded_tokens
|
|
slot_idx = handle.slot_idx
|
|
num_slots, num_topk = slot_idx.shape
|
|
hidden = x.shape[1]
|
|
|
|
# One local reduction in fp32, rounded to bf16 for the exchange unless only a single reduction is allowed.
|
|
partial_dtype = torch.bfloat16 if self.allow_multiple_reduction else torch.float32
|
|
partial = torch.empty((num_slots, hidden), dtype=partial_dtype, device=x.device)
|
|
if x.shape[0] == 0:
|
|
partial.zero_()
|
|
else:
|
|
chunk = max(1, _MAX_COMBINE_GATHER_BYTES // (num_topk * hidden * x.element_size()))
|
|
for begin in range(0, num_slots, chunk):
|
|
slots = slot_idx[begin : begin + chunk]
|
|
gathered = x.index_select(0, slots.clamp(min=0).flatten()).view(-1, num_topk, hidden)
|
|
gathered.masked_fill_((slots < 0).unsqueeze(-1), 0)
|
|
torch.sum(gathered, dim=1, dtype=partial_dtype, out=partial[begin : begin + chunk])
|
|
|
|
if self.num_ranks > 1:
|
|
received = torch.empty_like(partial)
|
|
dist.all_to_all_single(received, partial, group=self.group)
|
|
else:
|
|
received = partial
|
|
received = received.view(self.num_ranks, handle.num_rows_per_rank, hidden)[:, : handle.num_tokens]
|
|
combined = torch.sum(received, dim=0, dtype=torch.float32)
|
|
if bias is not None:
|
|
combined.add_(bias)
|
|
return _Completed((combined.to(torch.bfloat16), None))
|