Files
DeepInfer/deepinfer/model_executor/parallel_utils/host_engram.py
T
shunyadandClaude Opus 5.5 c23e8c3a97 refactor(parallel_utils): reuse tilelang_ops.common.align instead of local _align copies
nccl_ep.py and host_engram.py each defined a private _align identical to the
existing tilelang_ops.common.align (an import-free constants module, so the
communication modules stay lightweight).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 22:28:43 -08:00

217 lines
11 KiB
Python

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Host-resident Engram tables for GPUs without the RDMA Engram fetch path.
`HostEngramBuffer` mirrors the `deep_ep.EngramBuffer` interface used by `EngramRuntime`. All ranks of
the Engram group share one `/dev/shm` mapping that holds every layer's full table and scales; each
rank page-locks the mapping and gathers rows straight from host memory over UVA, so device memory
does not grow with the table size.
The table file never has a name: rank 0 creates it with O_TMPFILE and the other ranks open it through
`/proc/<rank 0 pid>/fd/<fd>`, so a killed group leaves nothing behind in `/dev/shm`.
"""
from __future__ import annotations
import mmap
import os
import socket
from collections.abc import Callable, Sequence
import torch
import torch.distributed as dist
from deepinfer import PLATFORM
from deepinfer.model_executor.layers.tilelang_ops.common import align
_SHM_DIR = "/dev/shm"
_NUM_ALIGNED_SF_PACKS = 4
_PAGE_BYTES = 1 << 21
class HostEngramBuffer:
def __init__(
self,
group: dist.ProcessGroup,
num_entries_per_layer: Sequence[int],
hidden: int,
max_num_tokens: int,
num_entries_per_token: int,
dtype: torch.dtype,
num_sf_packs: int = 0,
) -> None:
assert num_entries_per_layer and dtype.itemsize in (1, 2)
assert (num_sf_packs > 0) == (dtype.itemsize == 1), "FP8 tables carry scales, BF16 tables do not"
self.group = group
self.rank, self.num_ranks = group.rank(), group.size()
self.num_entries_per_layer = list(num_entries_per_layer)
self.hidden = hidden
self.max_num_tokens = max_num_tokens
self.num_entries_per_token = num_entries_per_token
self.dtype = dtype
self.num_sf_packs = num_sf_packs
self.sf_dtype = PLATFORM.fp8_sf_type
hosts: list[str | None] = [None] * self.num_ranks
dist.all_gather_object(hosts, socket.gethostname(), group=group)
if len(set(hosts)) != 1:
# Normally refused earlier by sm120_support.check_rank_placement; kept because /proc/<pid>/fd only
# resolves on the host that created the table file.
raise ValueError(
f"Host-resident Engram tables need the whole Engram group on one host; "
f"set engram_parallel_size to at most the ranks per host: {hosts}"
)
self._row_offsets: list[int] = []
self._sf_offsets: list[int] = []
offset = 0
for num_entries in self.num_entries_per_layer:
self._row_offsets.append(offset)
offset = align(offset + num_entries * self.num_ranks * hidden * dtype.itemsize, _PAGE_BYTES)
for num_entries in self.num_entries_per_layer:
self._sf_offsets.append(offset)
offset = align(offset + num_entries * self.num_ranks * num_sf_packs * self.sf_dtype.itemsize, _PAGE_BYTES)
self.num_bytes = offset
fd = -1
error: str | None = None
shared: list[str | None] = [None]
if self.rank == 0:
try:
fd = os.open(_SHM_DIR, os.O_TMPFILE | os.O_RDWR, 0o600)
os.ftruncate(fd, self.num_bytes)
shared[0] = f"/proc/{os.getpid()}/fd/{fd}"
except OSError as exc:
error = f"rank 0 cannot create it: {exc}"
dist.broadcast_object_list(shared, group_src=0, group=group)
if shared[0] is not None and self.rank != 0:
try:
fd = os.open(shared[0], os.O_RDWR)
except OSError as exc:
error = f"rank {self.rank} cannot open {shared[0]}: {exc}"
self._mmap: mmap.mmap | None = None
if error is None and fd >= 0:
try:
# Each rank reserves only its own shard so tmpfs places those pages on its NUMA node, and a full
# /dev/shm fails here on every rank instead of raising SIGBUS when write() touches the pages.
for begin, num_bytes in self._local_ranges():
os.posix_fallocate(fd, begin, num_bytes)
self._mmap = mmap.mmap(fd, self.num_bytes, mmap.MAP_SHARED, mmap.PROT_READ | mmap.PROT_WRITE)
except OSError as exc:
error = f"rank {self.rank}: {exc}"
# Rank 0 keeps its descriptor open until every rank has opened the file through it.
errors: list[str | None] = [None] * self.num_ranks
dist.all_gather_object(errors, error, group=group)
if fd >= 0:
os.close(fd)
if any(errors):
if self._mmap is not None:
self._mmap.close()
raise RuntimeError(
f"Failed to reserve the {self.num_bytes}-byte Engram table file in {_SHM_DIR}: " + "; ".join(e for e in errors if e)
)
self._host = torch.frombuffer(self._mmap, dtype=torch.uint8)
self._pinned = False
self._device_storage: torch.Tensor | None = None
self._rows: list[torch.Tensor] = []
self._sfs: list[torch.Tensor] = []
def _local_ranges(self) -> list[tuple[int, int]]:
"""Byte ranges of this rank's rows and scales in the shared file."""
ranges = []
for layer_idx, num_entries in enumerate(self.num_entries_per_layer):
row_bytes = num_entries * self.hidden * self.dtype.itemsize
ranges.append((self._row_offsets[layer_idx] + self.rank * row_bytes, row_bytes))
sf_bytes = num_entries * self.num_sf_packs * self.sf_dtype.itemsize
ranges.append((self._sf_offsets[layer_idx] + self.rank * sf_bytes, sf_bytes))
return [(begin, num_bytes) for begin, num_bytes in ranges if num_bytes > 0]
def _host_rows(self, layer_idx: int) -> torch.Tensor:
num_rows = self.num_entries_per_layer[layer_idx] * self.num_ranks
begin = self._row_offsets[layer_idx]
return self._host[begin : begin + num_rows * self.hidden * self.dtype.itemsize].view(self.dtype).view(num_rows, self.hidden)
def _host_sfs(self, layer_idx: int) -> torch.Tensor:
num_rows = self.num_entries_per_layer[layer_idx] * self.num_ranks
begin = self._sf_offsets[layer_idx]
num_bytes = num_rows * self.num_sf_packs * self.sf_dtype.itemsize
return self._host[begin : begin + num_bytes].view(self.sf_dtype).view(num_rows, self.num_sf_packs)
def write(self, storages: Sequence[torch.Tensor], sfs: Sequence[torch.Tensor] | None = None) -> None:
"""Copy this rank's shard of every layer; `sfs` are the local packed scales, not gathered ones."""
assert self._device_storage is None, "HostEngramBuffer.write can only be called once"
assert len(storages) == len(self.num_entries_per_layer)
assert (sfs is not None) == (self.num_sf_packs > 0)
for layer_idx, storage in enumerate(storages):
num_entries = self.num_entries_per_layer[layer_idx]
assert storage.shape == (num_entries, self.hidden) and storage.dtype == self.dtype, (storage.shape, storage.dtype)
begin = self.rank * num_entries
# The first touch places the pages on this rank's NUMA node.
self._host_rows(layer_idx)[begin : begin + num_entries].copy_(storage)
if sfs is not None:
sf = sfs[layer_idx]
assert sf.shape == (num_entries, self.num_sf_packs) and sf.dtype == self.sf_dtype, (sf.shape, sf.dtype)
self._host_sfs(layer_idx)[begin : begin + num_entries].copy_(sf)
dist.barrier(group=self.group)
PLATFORM.pin_host_memory(self._host, name="Engram tables")
self._pinned = True
device = torch.device("cuda", torch.cuda.current_device())
storage = torch._C._construct_storage_from_data_pointer(self._host.data_ptr(), device, self.num_bytes)
self._device_storage = torch.empty(0, dtype=torch.uint8, device=device).set_(storage, 0, (self.num_bytes,), (1,))
self._rows, self._sfs = [], []
for layer_idx, num_entries in enumerate(self.num_entries_per_layer):
num_rows = num_entries * self.num_ranks
begin = self._row_offsets[layer_idx]
rows = self._device_storage[begin : begin + num_rows * self.hidden * self.dtype.itemsize]
self._rows.append(rows.view(num_rows, self.hidden * self.dtype.itemsize))
if self.num_sf_packs:
begin = self._sf_offsets[layer_idx]
sf = self._device_storage[begin : begin + num_rows * self.num_sf_packs * self.sf_dtype.itemsize]
self._sfs.append(sf.view(self.sf_dtype).view(num_rows, self.num_sf_packs))
def _gather(self, layer_idx: int, indices: torch.Tensor, use_tma_aligned_col_major_sf: bool) -> object:
num_tokens = indices.size(0)
flat = indices.flatten()
# Image tokens hash to -1; their rows are masked by the gate, so return zeros.
invalid = (flat < 0).unsqueeze(1)
flat = flat.clamp_min(0)
rows = self._rows[layer_idx].index_select(0, flat).masked_fill_(invalid, 0)
rows = rows.view(self.dtype).view(num_tokens, self.num_entries_per_token * self.hidden)
if not self.num_sf_packs:
return rows
sf = self._sfs[layer_idx].index_select(0, flat).masked_fill_(invalid, 0)
sf = sf.view(num_tokens, self.num_entries_per_token * self.num_sf_packs)
if use_tma_aligned_col_major_sf:
aligned_tokens = align(num_tokens, _NUM_ALIGNED_SF_PACKS)
backing = sf.new_empty(sf.size(1), aligned_tokens)
backing[:, :num_tokens].copy_(sf.T)
sf = backing.as_strided((num_tokens, sf.size(1)), (1, aligned_tokens))
return rows, sf
def fetch(self, indices: torch.Tensor, num_qps: int = 0, use_tma_aligned_col_major_sf: bool = True) -> list[Callable]:
"""Return one hook per layer; each gathers `[num_tokens, num_entries_per_token * hidden]` rows (and scales)."""
assert self._device_storage is not None, "HostEngramBuffer.fetch called before write"
num_layers, num_tokens, num_entries_per_token = indices.shape
assert indices.dtype == torch.int32 and indices.is_cuda and indices.is_contiguous()
assert num_layers == len(self.num_entries_per_layer) and num_entries_per_token == self.num_entries_per_token
assert num_tokens <= self.max_num_tokens
def hook(layer_idx: int) -> Callable:
return lambda: self._gather(layer_idx, indices[layer_idx], use_tma_aligned_col_major_sf)
return [hook(layer_idx) for layer_idx in range(num_layers)]
def destroy(self) -> None:
self._rows, self._sfs = [], []
self._device_storage = None
if self._pinned:
PLATFORM.unpin_host_memory(self._host.data_ptr())
self._pinned = False
if self._mmap is not None:
del self._host
self._mmap.close()
self._mmap = None