Files
DeepInfer/deepinfer/model_executor/parallel_utils/nccl_ep.py
T
shunyadandClaude Opus 5.5 c23e8c3a97 refactor(parallel_utils): reuse tilelang_ops.common.align instead of local _align copies
nccl_ep.py and host_engram.py each defined a private _align identical to the
existing tilelang_ops.common.align (an import-free constants module, so the
communication modules stay lightweight).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 22:28:43 -08:00

331 lines
15 KiB
Python

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Expert-parallel dispatch and combine over plain NCCL collectives.
`NcclEPBuffer` mirrors the subset of `deep_ep.EPBuffer` used by `DeepseekMoE`: the expanded dispatch
layout (one row per token and local expert selection, grouped per local expert, each group padded to
`expert_alignment`) and the matching combine. Dispatch all-gathers every rank's tokens and selections
and gathers the rows of local experts; combine reduces each source token's rows on the expert rank
and returns the partial sums to the source ranks with an all-to-all.
Without CPU sync every shape depends only on `num_max_tokens_per_rank`, so the path is CUDA graph
capturable. With CPU sync, the received sizes are read back once per dispatch to allocate exactly.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
import torch
import torch.distributed as dist
from deepinfer.model_executor.layers.tilelang_ops.common import align
from deepinfer.utils import allow_cuda_sync
_SEGMENT_ALIGNMENT = 16
_NUM_ALIGNED_SF_BYTES = 16
_MAX_COMBINE_GATHER_BYTES = 256 << 20
class _RowLayout:
"""Per-row byte segments of a packed uint8 `[num_rows, row_bytes]` exchange buffer."""
def __init__(self, segments: dict[str, tuple[torch.dtype, int]]) -> None:
self.segments: dict[str, tuple[int, torch.dtype, int]] = {}
offset = 0
for name, (dtype, width) in segments.items():
self.segments[name] = (offset, dtype, width)
offset += align(width * dtype.itemsize, _SEGMENT_ALIGNMENT)
self.row_bytes = offset
def empty(self, num_rows: int, device: torch.device) -> torch.Tensor:
return torch.empty((num_rows, self.row_bytes), dtype=torch.uint8, device=device)
def view(self, rows: torch.Tensor, name: str) -> torch.Tensor:
offset, dtype, width = self.segments[name]
return rows[:, offset : offset + width * dtype.itemsize].view(dtype)
class _Completed:
"""Stand-in for `deep_ep.EventOverlap`: the work is already ordered on the current stream."""
def __init__(self, result: Any) -> None:
self._result = result
def current_stream_wait(self, release_handle: bool = False) -> Any:
return self._result
@dataclass
class NcclEPHandle:
"""Dispatch layout consumed by `NcclEPBuffer.combine`.
Attributes:
num_tokens: the number of tokens this rank dispatched.
num_rows_per_rank: the per-rank token slots used by the exchange; combine returns this many rows per rank.
num_expanded_tokens: the number of allocated expanded rows.
psum_num_recv_tokens_per_expert: `[num_local_experts]` int32, the aligned start of each local expert's
group plus its actual count, as in `deep_ep.EPHandle` expand mode.
num_unaligned_recv_tokens_per_expert: `[num_local_experts]` int32, the actual count of each local expert.
slot_idx: `[num_ranks * num_rows_per_rank, num_topk]` int64, the expanded row of each local selection
of each source token slot, -1 for selections of other ranks' experts.
"""
num_tokens: int
num_rows_per_rank: int
num_expanded_tokens: int
psum_num_recv_tokens_per_expert: torch.Tensor
num_unaligned_recv_tokens_per_expert: torch.Tensor
slot_idx: torch.Tensor
class NcclEPBuffer:
"""Drop-in for the `deep_ep.EPBuffer` expanded dispatch/combine used by `DeepseekMoE`.
Within each local expert's group, rows are ordered by source rank, then source token. Alignment padding
rows and rows past the last group are unspecified (no zero padding), and their received weights are zero.
"""
num_bytes = 0
def __init__(
self,
group: dist.ProcessGroup,
num_max_tokens_per_rank: int,
hidden: int,
num_topk: int,
use_fp8_dispatch: bool = False,
allow_multiple_reduction: bool = True,
) -> None:
self.group = group
self.rank_idx = group.rank()
self.num_ranks = group.size()
self.num_max_tokens_per_rank = num_max_tokens_per_rank
self.hidden = hidden
self.num_topk = num_topk
self.use_fp8_dispatch = use_fp8_dispatch
self.allow_multiple_reduction = allow_multiple_reduction
def destroy(self) -> None:
self.group = None
@staticmethod
def capture() -> None:
return None
def _all_gather(self, rows: torch.Tensor) -> torch.Tensor:
if self.num_ranks == 1:
return rows
out = rows.new_empty((self.num_ranks * rows.shape[0], *rows.shape[1:]))
dist.all_gather_into_tensor(out, rows, group=self.group)
return out
def dispatch(
self,
x: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
topk_idx: torch.Tensor,
topk_weights: torch.Tensor | None = None,
cumulative_local_expert_recv_stats: torch.Tensor | None = None,
num_experts: int | None = None,
num_max_tokens_per_rank: int | None = None,
expert_alignment: int | None = None,
num_sms: int = 0,
previous_event: Any = None,
async_with_compute_stream: bool = False,
allocate_on_comm_stream: bool = False,
do_handle_copy: bool = False,
do_cpu_sync: bool | None = None,
do_expand: bool = False,
use_tma_aligned_col_major_sf: bool = False,
defer_epilogue: bool = False,
) -> _Completed:
"""Dispatch tokens to the ranks owning their selected experts, in the `deep_ep.EPBuffer` expanded layout.
Experts are split evenly over ranks; selections outside `[0, num_experts)` (e.g. -1) are dropped.
All ranks must pass the same `num_max_tokens_per_rank` and `do_cpu_sync`. `current_stream_wait()` on the
result returns `(recv_x, None, recv_topk_weights, handle)`, with `recv_x` a `(fp8, sf)` tuple when `x` is one,
and `recv_topk_weights` `[num_expanded_tokens]`.
"""
assert do_expand, "only the expanded layout is supported"
assert not do_handle_copy
assert num_experts is not None and num_experts % self.num_ranks == 0
x, sf = x if isinstance(x, tuple) else (x, None)
num_max_tokens_per_rank = num_max_tokens_per_rank or self.num_max_tokens_per_rank
expert_alignment = expert_alignment or 1
do_cpu_sync = True if do_cpu_sync is None else do_cpu_sync
device = x.device
num_tokens, hidden = x.shape
num_topk = topk_idx.shape[1]
assert num_tokens <= num_max_tokens_per_rank, f"{num_tokens=} > {num_max_tokens_per_rank=}"
assert topk_idx.shape[0] == num_tokens and (topk_weights is None or topk_weights.shape == topk_idx.shape)
num_ranks = self.num_ranks
num_local_experts = num_experts // num_ranks
expert_begin = self.rank_idx * num_local_experts
meta_segments = {"topk_idx": (torch.int32, num_topk), "topk_weights": (torch.float32, num_topk)}
data_segments = {"x": (x.dtype, hidden)}
if sf is not None:
assert sf.shape[0] == num_tokens
data_segments["sf"] = (sf.dtype, sf.shape[1])
def fill_meta(layout: _RowLayout, rows: torch.Tensor) -> None:
idx = layout.view(rows, "topk_idx")
idx[:num_tokens].copy_(topk_idx)
idx[num_tokens:].fill_(-1)
weights = layout.view(rows, "topk_weights")
if topk_weights is None:
weights.zero_()
else:
weights[:num_tokens].copy_(topk_weights)
def fill_data(layout: _RowLayout, rows: torch.Tensor) -> None:
layout.view(rows, "x")[:num_tokens].copy_(x)
if sf is not None:
layout.view(rows, "sf")[:num_tokens].copy_(sf)
if do_cpu_sync:
# Exchange routing first at the static per-rank capacity, then the data at the largest actual count.
meta_layout = _RowLayout(meta_segments)
meta_rows = meta_layout.empty(num_max_tokens_per_rank + 1, device)
fill_meta(meta_layout, meta_rows)
meta_layout.view(meta_rows, "topk_idx")[num_max_tokens_per_rank].fill_(num_tokens)
meta_all = self._all_gather(meta_rows).view(num_ranks, num_max_tokens_per_rank + 1, -1)
num_tokens_per_rank = meta_layout.view(meta_all[:, num_max_tokens_per_rank], "topk_idx")[:, 0]
meta_all = meta_all[:, :num_max_tokens_per_rank].flatten(0, 1)
idx_all = meta_layout.view(meta_all, "topk_idx")
weights_all = meta_layout.view(meta_all, "topk_weights")
else:
layout = _RowLayout(meta_segments | data_segments)
rows = layout.empty(num_max_tokens_per_rank, device)
fill_meta(layout, rows)
fill_data(layout, rows)
rows_all = self._all_gather(rows)
idx_all = layout.view(rows_all, "topk_idx")
weights_all = layout.view(rows_all, "topk_weights")
# Sort selections by local expert; a stable sort keeps (source rank, source token) order within each expert.
flat_idx = idx_all.reshape(-1)
is_local = (flat_idx >= expert_begin) & (flat_idx < expert_begin + num_local_experts)
local_expert = torch.where(is_local, flat_idx - expert_begin, num_local_experts)
sorted_expert, order = torch.sort(local_expert, stable=True)
expert_range = torch.arange(num_local_experts + 1, dtype=sorted_expert.dtype, device=device)
unaligned_begin = torch.searchsorted(sorted_expert, expert_range)
num_recv_per_expert = unaligned_begin.diff()
num_aligned_per_expert = (num_recv_per_expert + expert_alignment - 1) // expert_alignment * expert_alignment
aligned_end = num_aligned_per_expert.cumsum(0)
aligned_begin = aligned_end - num_aligned_per_expert
psum_num_recv_tokens_per_expert = (aligned_begin + num_recv_per_expert).int()
num_unaligned_recv_tokens_per_expert = num_recv_per_expert.int()
if cumulative_local_expert_recv_stats is not None:
cumulative_local_expert_recv_stats.add_(num_unaligned_recv_tokens_per_expert.to(cumulative_local_expert_recv_stats.dtype))
if do_cpu_sync:
with allow_cuda_sync():
host_values = torch.cat([num_tokens_per_rank.long(), aligned_end[-1:]]).tolist()
num_rows_per_rank = max(1, max(host_values[:-1]))
num_expanded_tokens = host_values[-1]
layout = _RowLayout(data_segments)
rows = layout.empty(num_rows_per_rank, device)
fill_data(layout, rows)
rows_all = self._all_gather(rows)
else:
num_rows_per_rank = num_max_tokens_per_rank
num_expanded_tokens = num_ranks * num_max_tokens_per_rank * min(num_topk, num_local_experts)
num_expanded_tokens += (expert_alignment - 1) * num_local_experts
num_expanded_tokens = align(num_expanded_tokens, expert_alignment)
# Expanded row of each sorted selection; selections of other ranks' experts go to a sink row.
sorted_is_local = sorted_expert < num_local_experts
clamped_expert = sorted_expert.clamp(max=num_local_experts - 1).long()
position = torch.arange(sorted_expert.numel(), device=device)
dst_row = aligned_begin[clamped_expert] + position - unaligned_begin[clamped_expert]
dst_row = torch.where(sorted_is_local, dst_row, num_expanded_tokens)
src_meta_token = order // num_topk
src_token = src_meta_token // num_max_tokens_per_rank * num_rows_per_rank + src_meta_token % num_max_tokens_per_rank
row_src = torch.zeros(num_expanded_tokens + 1, dtype=torch.long, device=device)
row_src.scatter_(0, dst_row, src_token)
row_src = row_src[:num_expanded_tokens]
recv_x = layout.view(rows_all, "x").index_select(0, row_src)
recv_sf = None
if sf is not None:
gathered_sf = layout.view(rows_all, "sf").index_select(0, row_src)
if use_tma_aligned_col_major_sf:
num_aligned_rows = align(num_expanded_tokens, max(1, _NUM_ALIGNED_SF_BYTES // sf.element_size()))
recv_sf = torch.empty_strided(gathered_sf.shape, (1, num_aligned_rows), dtype=sf.dtype, device=device)
recv_sf.copy_(gathered_sf)
else:
recv_sf = gathered_sf
recv_topk_weights = None
if topk_weights is not None:
recv_topk_weights = torch.zeros(num_expanded_tokens + 1, dtype=torch.float32, device=device)
recv_topk_weights.scatter_(0, dst_row, weights_all.reshape(-1)[order])
recv_topk_weights = recv_topk_weights[:num_expanded_tokens]
slot_idx = torch.empty_like(order)
slot_idx.scatter_(0, order, torch.where(sorted_is_local, dst_row, -1))
slot_idx = slot_idx.view(num_ranks, num_max_tokens_per_rank, num_topk)[:, :num_rows_per_rank].reshape(-1, num_topk)
handle = NcclEPHandle(
num_tokens=num_tokens,
num_rows_per_rank=num_rows_per_rank,
num_expanded_tokens=num_expanded_tokens,
psum_num_recv_tokens_per_expert=psum_num_recv_tokens_per_expert,
num_unaligned_recv_tokens_per_expert=num_unaligned_recv_tokens_per_expert,
slot_idx=slot_idx,
)
recv_x = (recv_x, recv_sf) if recv_sf is not None else recv_x
return _Completed((recv_x, None, recv_topk_weights, handle))
def combine(
self,
x: torch.Tensor,
handle: NcclEPHandle,
topk_weights: torch.Tensor | None = None,
bias: torch.Tensor | None = None,
num_sms: int = 0,
previous_event: Any = None,
async_with_compute_stream: bool = False,
allocate_on_comm_stream: bool = False,
defer_epilogue: bool = False,
) -> _Completed:
"""Sum each source token's expanded rows (weights already applied) back on its rank, plus `bias`.
`current_stream_wait()` on the result returns `(combined_x, None)` with `combined_x` bf16 `[num_tokens, hidden]`.
"""
assert topk_weights is None, "expanded combine expects weights already applied"
assert x.dim() == 2 and x.shape[0] == handle.num_expanded_tokens
slot_idx = handle.slot_idx
num_slots, num_topk = slot_idx.shape
hidden = x.shape[1]
# One local reduction in fp32, rounded to bf16 for the exchange unless only a single reduction is allowed.
partial_dtype = torch.bfloat16 if self.allow_multiple_reduction else torch.float32
partial = torch.empty((num_slots, hidden), dtype=partial_dtype, device=x.device)
if x.shape[0] == 0:
partial.zero_()
else:
chunk = max(1, _MAX_COMBINE_GATHER_BYTES // (num_topk * hidden * x.element_size()))
for begin in range(0, num_slots, chunk):
slots = slot_idx[begin : begin + chunk]
gathered = x.index_select(0, slots.clamp(min=0).flatten()).view(-1, num_topk, hidden)
gathered.masked_fill_((slots < 0).unsqueeze(-1), 0)
torch.sum(gathered, dim=1, dtype=partial_dtype, out=partial[begin : begin + chunk])
if self.num_ranks > 1:
received = torch.empty_like(partial)
dist.all_to_all_single(received, partial, group=self.group)
else:
received = partial
received = received.view(self.num_ranks, handle.num_rows_per_rank, hidden)[:, : handle.num_tokens]
combined = torch.sum(received, dim=0, dtype=torch.float32)
if bias is not None:
combined.add_(bias)
return _Completed((combined.to(torch.bfloat16), None))