mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-11-20 04:22:27 +00:00
nccl_ep.py and host_engram.py each defined a private _align identical to the existing tilelang_ops.common.align (an import-free constants module, so the communication modules stay lightweight). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
217 lines
11 KiB
Python
217 lines
11 KiB
Python
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
"""Host-resident Engram tables for GPUs without the RDMA Engram fetch path.
|
|
|
|
`HostEngramBuffer` mirrors the `deep_ep.EngramBuffer` interface used by `EngramRuntime`. All ranks of
|
|
the Engram group share one `/dev/shm` mapping that holds every layer's full table and scales; each
|
|
rank page-locks the mapping and gathers rows straight from host memory over UVA, so device memory
|
|
does not grow with the table size.
|
|
|
|
The table file never has a name: rank 0 creates it with O_TMPFILE and the other ranks open it through
|
|
`/proc/<rank 0 pid>/fd/<fd>`, so a killed group leaves nothing behind in `/dev/shm`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import mmap
|
|
import os
|
|
import socket
|
|
from collections.abc import Callable, Sequence
|
|
|
|
import torch
|
|
import torch.distributed as dist
|
|
|
|
from deepinfer import PLATFORM
|
|
from deepinfer.model_executor.layers.tilelang_ops.common import align
|
|
|
|
_SHM_DIR = "/dev/shm"
|
|
_NUM_ALIGNED_SF_PACKS = 4
|
|
_PAGE_BYTES = 1 << 21
|
|
|
|
|
|
class HostEngramBuffer:
|
|
def __init__(
|
|
self,
|
|
group: dist.ProcessGroup,
|
|
num_entries_per_layer: Sequence[int],
|
|
hidden: int,
|
|
max_num_tokens: int,
|
|
num_entries_per_token: int,
|
|
dtype: torch.dtype,
|
|
num_sf_packs: int = 0,
|
|
) -> None:
|
|
assert num_entries_per_layer and dtype.itemsize in (1, 2)
|
|
assert (num_sf_packs > 0) == (dtype.itemsize == 1), "FP8 tables carry scales, BF16 tables do not"
|
|
self.group = group
|
|
self.rank, self.num_ranks = group.rank(), group.size()
|
|
self.num_entries_per_layer = list(num_entries_per_layer)
|
|
self.hidden = hidden
|
|
self.max_num_tokens = max_num_tokens
|
|
self.num_entries_per_token = num_entries_per_token
|
|
self.dtype = dtype
|
|
self.num_sf_packs = num_sf_packs
|
|
self.sf_dtype = PLATFORM.fp8_sf_type
|
|
|
|
hosts: list[str | None] = [None] * self.num_ranks
|
|
dist.all_gather_object(hosts, socket.gethostname(), group=group)
|
|
if len(set(hosts)) != 1:
|
|
# Normally refused earlier by sm120_support.check_rank_placement; kept because /proc/<pid>/fd only
|
|
# resolves on the host that created the table file.
|
|
raise ValueError(
|
|
f"Host-resident Engram tables need the whole Engram group on one host; "
|
|
f"set engram_parallel_size to at most the ranks per host: {hosts}"
|
|
)
|
|
|
|
self._row_offsets: list[int] = []
|
|
self._sf_offsets: list[int] = []
|
|
offset = 0
|
|
for num_entries in self.num_entries_per_layer:
|
|
self._row_offsets.append(offset)
|
|
offset = align(offset + num_entries * self.num_ranks * hidden * dtype.itemsize, _PAGE_BYTES)
|
|
for num_entries in self.num_entries_per_layer:
|
|
self._sf_offsets.append(offset)
|
|
offset = align(offset + num_entries * self.num_ranks * num_sf_packs * self.sf_dtype.itemsize, _PAGE_BYTES)
|
|
self.num_bytes = offset
|
|
|
|
fd = -1
|
|
error: str | None = None
|
|
shared: list[str | None] = [None]
|
|
if self.rank == 0:
|
|
try:
|
|
fd = os.open(_SHM_DIR, os.O_TMPFILE | os.O_RDWR, 0o600)
|
|
os.ftruncate(fd, self.num_bytes)
|
|
shared[0] = f"/proc/{os.getpid()}/fd/{fd}"
|
|
except OSError as exc:
|
|
error = f"rank 0 cannot create it: {exc}"
|
|
dist.broadcast_object_list(shared, group_src=0, group=group)
|
|
if shared[0] is not None and self.rank != 0:
|
|
try:
|
|
fd = os.open(shared[0], os.O_RDWR)
|
|
except OSError as exc:
|
|
error = f"rank {self.rank} cannot open {shared[0]}: {exc}"
|
|
self._mmap: mmap.mmap | None = None
|
|
if error is None and fd >= 0:
|
|
try:
|
|
# Each rank reserves only its own shard so tmpfs places those pages on its NUMA node, and a full
|
|
# /dev/shm fails here on every rank instead of raising SIGBUS when write() touches the pages.
|
|
for begin, num_bytes in self._local_ranges():
|
|
os.posix_fallocate(fd, begin, num_bytes)
|
|
self._mmap = mmap.mmap(fd, self.num_bytes, mmap.MAP_SHARED, mmap.PROT_READ | mmap.PROT_WRITE)
|
|
except OSError as exc:
|
|
error = f"rank {self.rank}: {exc}"
|
|
# Rank 0 keeps its descriptor open until every rank has opened the file through it.
|
|
errors: list[str | None] = [None] * self.num_ranks
|
|
dist.all_gather_object(errors, error, group=group)
|
|
if fd >= 0:
|
|
os.close(fd)
|
|
if any(errors):
|
|
if self._mmap is not None:
|
|
self._mmap.close()
|
|
raise RuntimeError(
|
|
f"Failed to reserve the {self.num_bytes}-byte Engram table file in {_SHM_DIR}: " + "; ".join(e for e in errors if e)
|
|
)
|
|
|
|
self._host = torch.frombuffer(self._mmap, dtype=torch.uint8)
|
|
self._pinned = False
|
|
self._device_storage: torch.Tensor | None = None
|
|
self._rows: list[torch.Tensor] = []
|
|
self._sfs: list[torch.Tensor] = []
|
|
|
|
def _local_ranges(self) -> list[tuple[int, int]]:
|
|
"""Byte ranges of this rank's rows and scales in the shared file."""
|
|
ranges = []
|
|
for layer_idx, num_entries in enumerate(self.num_entries_per_layer):
|
|
row_bytes = num_entries * self.hidden * self.dtype.itemsize
|
|
ranges.append((self._row_offsets[layer_idx] + self.rank * row_bytes, row_bytes))
|
|
sf_bytes = num_entries * self.num_sf_packs * self.sf_dtype.itemsize
|
|
ranges.append((self._sf_offsets[layer_idx] + self.rank * sf_bytes, sf_bytes))
|
|
return [(begin, num_bytes) for begin, num_bytes in ranges if num_bytes > 0]
|
|
|
|
def _host_rows(self, layer_idx: int) -> torch.Tensor:
|
|
num_rows = self.num_entries_per_layer[layer_idx] * self.num_ranks
|
|
begin = self._row_offsets[layer_idx]
|
|
return self._host[begin : begin + num_rows * self.hidden * self.dtype.itemsize].view(self.dtype).view(num_rows, self.hidden)
|
|
|
|
def _host_sfs(self, layer_idx: int) -> torch.Tensor:
|
|
num_rows = self.num_entries_per_layer[layer_idx] * self.num_ranks
|
|
begin = self._sf_offsets[layer_idx]
|
|
num_bytes = num_rows * self.num_sf_packs * self.sf_dtype.itemsize
|
|
return self._host[begin : begin + num_bytes].view(self.sf_dtype).view(num_rows, self.num_sf_packs)
|
|
|
|
def write(self, storages: Sequence[torch.Tensor], sfs: Sequence[torch.Tensor] | None = None) -> None:
|
|
"""Copy this rank's shard of every layer; `sfs` are the local packed scales, not gathered ones."""
|
|
assert self._device_storage is None, "HostEngramBuffer.write can only be called once"
|
|
assert len(storages) == len(self.num_entries_per_layer)
|
|
assert (sfs is not None) == (self.num_sf_packs > 0)
|
|
for layer_idx, storage in enumerate(storages):
|
|
num_entries = self.num_entries_per_layer[layer_idx]
|
|
assert storage.shape == (num_entries, self.hidden) and storage.dtype == self.dtype, (storage.shape, storage.dtype)
|
|
begin = self.rank * num_entries
|
|
# The first touch places the pages on this rank's NUMA node.
|
|
self._host_rows(layer_idx)[begin : begin + num_entries].copy_(storage)
|
|
if sfs is not None:
|
|
sf = sfs[layer_idx]
|
|
assert sf.shape == (num_entries, self.num_sf_packs) and sf.dtype == self.sf_dtype, (sf.shape, sf.dtype)
|
|
self._host_sfs(layer_idx)[begin : begin + num_entries].copy_(sf)
|
|
dist.barrier(group=self.group)
|
|
|
|
PLATFORM.pin_host_memory(self._host, name="Engram tables")
|
|
self._pinned = True
|
|
device = torch.device("cuda", torch.cuda.current_device())
|
|
storage = torch._C._construct_storage_from_data_pointer(self._host.data_ptr(), device, self.num_bytes)
|
|
self._device_storage = torch.empty(0, dtype=torch.uint8, device=device).set_(storage, 0, (self.num_bytes,), (1,))
|
|
self._rows, self._sfs = [], []
|
|
for layer_idx, num_entries in enumerate(self.num_entries_per_layer):
|
|
num_rows = num_entries * self.num_ranks
|
|
begin = self._row_offsets[layer_idx]
|
|
rows = self._device_storage[begin : begin + num_rows * self.hidden * self.dtype.itemsize]
|
|
self._rows.append(rows.view(num_rows, self.hidden * self.dtype.itemsize))
|
|
if self.num_sf_packs:
|
|
begin = self._sf_offsets[layer_idx]
|
|
sf = self._device_storage[begin : begin + num_rows * self.num_sf_packs * self.sf_dtype.itemsize]
|
|
self._sfs.append(sf.view(self.sf_dtype).view(num_rows, self.num_sf_packs))
|
|
|
|
def _gather(self, layer_idx: int, indices: torch.Tensor, use_tma_aligned_col_major_sf: bool) -> object:
|
|
num_tokens = indices.size(0)
|
|
flat = indices.flatten()
|
|
# Image tokens hash to -1; their rows are masked by the gate, so return zeros.
|
|
invalid = (flat < 0).unsqueeze(1)
|
|
flat = flat.clamp_min(0)
|
|
rows = self._rows[layer_idx].index_select(0, flat).masked_fill_(invalid, 0)
|
|
rows = rows.view(self.dtype).view(num_tokens, self.num_entries_per_token * self.hidden)
|
|
if not self.num_sf_packs:
|
|
return rows
|
|
sf = self._sfs[layer_idx].index_select(0, flat).masked_fill_(invalid, 0)
|
|
sf = sf.view(num_tokens, self.num_entries_per_token * self.num_sf_packs)
|
|
if use_tma_aligned_col_major_sf:
|
|
aligned_tokens = align(num_tokens, _NUM_ALIGNED_SF_PACKS)
|
|
backing = sf.new_empty(sf.size(1), aligned_tokens)
|
|
backing[:, :num_tokens].copy_(sf.T)
|
|
sf = backing.as_strided((num_tokens, sf.size(1)), (1, aligned_tokens))
|
|
return rows, sf
|
|
|
|
def fetch(self, indices: torch.Tensor, num_qps: int = 0, use_tma_aligned_col_major_sf: bool = True) -> list[Callable]:
|
|
"""Return one hook per layer; each gathers `[num_tokens, num_entries_per_token * hidden]` rows (and scales)."""
|
|
assert self._device_storage is not None, "HostEngramBuffer.fetch called before write"
|
|
num_layers, num_tokens, num_entries_per_token = indices.shape
|
|
assert indices.dtype == torch.int32 and indices.is_cuda and indices.is_contiguous()
|
|
assert num_layers == len(self.num_entries_per_layer) and num_entries_per_token == self.num_entries_per_token
|
|
assert num_tokens <= self.max_num_tokens
|
|
|
|
def hook(layer_idx: int) -> Callable:
|
|
return lambda: self._gather(layer_idx, indices[layer_idx], use_tma_aligned_col_major_sf)
|
|
|
|
return [hook(layer_idx) for layer_idx in range(num_layers)]
|
|
|
|
def destroy(self) -> None:
|
|
self._rows, self._sfs = [], []
|
|
self._device_storage = None
|
|
if self._pinned:
|
|
PLATFORM.unpin_host_memory(self._host.data_ptr())
|
|
self._pinned = False
|
|
if self._mmap is not None:
|
|
del self._host
|
|
self._mmap.close()
|
|
self._mmap = None
|