Files
DeepInfer/deepinfer/model_executor/parallel_utils/host_engram.py
T
Zihua WuandClaude Opus 5.5 efecbb5ba6 feat(sm120): offline inference on NVIDIA Blackwell SM120
Run DeepSeek-V4.1-Flash offline inference on SM120 GPUs (RTX PRO 6000
Blackwell) with the same entry points as SM100. SM100 and Ascend paths
are unchanged.

- Build: SM120 is detected from the first visible GPU
  (scripts/nvidia_arch.sh). It builds DeepEP and DeepGEMM from its nv_dev
  branch (vendor/DeepGEMM-sm120), and no FlashMLA or DeepSelect;
  FlashInfer is installed from a pinned commit
  (requirements-nvidia-sm120.txt).
- Attention: V4.1 sparse MLA runs on FlashInfer's SM120 sparse MLA
  kernels, reading the inline-scale KV records as stored; KV blocks are
  aligned to 512 bytes.
- Indexer: FlashInfer's fused DSA indexer scores the index keys and
  selects their top-k in one call, for decode on every indexer layer
  (candidate source and consumer layers included) and for prefill outside
  the candidate layers. The remaining selections (prefill of the
  candidate layers, the sampler) use FlashInfer's top-k behind
  DeepSelect's interface (model_executor/layers/topk.py), in
  deterministic mode.
- MoE EP: DeepEP over NCCL GIN by default, or NCCL collectives
  (DEEPINFER_SM120_EP_BACKEND=nccl); NCCL settings known to break the EP
  paths are rejected. EPLB, including decode-only draft-layer EPLB, and
  per-layer YoCo MoE dispatch.
- Engram tables live in page-locked host shared memory; killed workers
  leave nothing in /dev/shm.
- Features: attention DP + MoE EP, DSpark/nextn speculative decoding,
  CUDA graphs, async scheduling, decoder SWA bounded replay, structured
  outputs, image inputs, multi-node deployments. Features without an SM120
  implementation (CP > 1, MTP varlen, batch-invariant mode, GPU-resident
  Engram tables) are refused at engine start.
- Eval: chat GSM8K and CV-Bench; tests for the SM120 kernels and their
  DeepInfer call sites.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 18:32:01 -08:00

217 lines
11 KiB
Python

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Host-resident Engram tables for GPUs without the RDMA Engram fetch path.
`HostEngramBuffer` mirrors the `deep_ep.EngramBuffer` interface used by `EngramRuntime`. All ranks of
the Engram group share one `/dev/shm` mapping that holds every layer's full table and scales; each
rank page-locks the mapping and gathers rows straight from host memory over UVA, so device memory
does not grow with the table size.
The table file never has a name: rank 0 creates it with O_TMPFILE and the other ranks open it through
`/proc/<rank 0 pid>/fd/<fd>`, so a killed group leaves nothing behind in `/dev/shm`.
"""
from __future__ import annotations
import mmap
import os
import socket
from collections.abc import Callable, Sequence
import torch
import torch.distributed as dist
from deepinfer import PLATFORM
_SHM_DIR = "/dev/shm"
_NUM_ALIGNED_SF_PACKS = 4
_PAGE_BYTES = 1 << 21
def _align(x: int, alignment: int) -> int:
return (x + alignment - 1) // alignment * alignment
class HostEngramBuffer:
def __init__(
self,
group: dist.ProcessGroup,
num_entries_per_layer: Sequence[int],
hidden: int,
max_num_tokens: int,
num_entries_per_token: int,
dtype: torch.dtype,
num_sf_packs: int = 0,
) -> None:
assert num_entries_per_layer and dtype.itemsize in (1, 2)
assert (num_sf_packs > 0) == (dtype.itemsize == 1), "FP8 tables carry scales, BF16 tables do not"
self.group = group
self.rank, self.num_ranks = group.rank(), group.size()
self.num_entries_per_layer = list(num_entries_per_layer)
self.hidden = hidden
self.max_num_tokens = max_num_tokens
self.num_entries_per_token = num_entries_per_token
self.dtype = dtype
self.num_sf_packs = num_sf_packs
self.sf_dtype = PLATFORM.fp8_sf_type
hosts: list[str | None] = [None] * self.num_ranks
dist.all_gather_object(hosts, socket.gethostname(), group=group)
assert len(set(hosts)) == 1, (
f"Host-resident Engram tables need the whole Engram group on one host; "
f"set engram_parallel_size to at most the ranks per host: {hosts}"
)
self._row_offsets: list[int] = []
self._sf_offsets: list[int] = []
offset = 0
for num_entries in self.num_entries_per_layer:
self._row_offsets.append(offset)
offset = _align(offset + num_entries * self.num_ranks * hidden * dtype.itemsize, _PAGE_BYTES)
for num_entries in self.num_entries_per_layer:
self._sf_offsets.append(offset)
offset = _align(offset + num_entries * self.num_ranks * num_sf_packs * self.sf_dtype.itemsize, _PAGE_BYTES)
self.num_bytes = offset
fd = -1
error: str | None = None
shared: list[str | None] = [None]
if self.rank == 0:
try:
fd = os.open(_SHM_DIR, os.O_TMPFILE | os.O_RDWR, 0o600)
os.ftruncate(fd, self.num_bytes)
shared[0] = f"/proc/{os.getpid()}/fd/{fd}"
except OSError as exc:
error = f"rank 0 cannot create it: {exc}"
dist.broadcast_object_list(shared, group_src=0, group=group)
if shared[0] is not None and self.rank != 0:
try:
fd = os.open(shared[0], os.O_RDWR)
except OSError as exc:
error = f"rank {self.rank} cannot open {shared[0]}: {exc}"
self._mmap: mmap.mmap | None = None
if error is None and fd >= 0:
try:
# Each rank reserves only its own shard so tmpfs places those pages on its NUMA node, and a full
# /dev/shm fails here on every rank instead of raising SIGBUS when write() touches the pages.
for begin, num_bytes in self._local_ranges():
os.posix_fallocate(fd, begin, num_bytes)
self._mmap = mmap.mmap(fd, self.num_bytes, mmap.MAP_SHARED, mmap.PROT_READ | mmap.PROT_WRITE)
except OSError as exc:
error = f"rank {self.rank}: {exc}"
# Rank 0 keeps its descriptor open until every rank has opened the file through it.
errors: list[str | None] = [None] * self.num_ranks
dist.all_gather_object(errors, error, group=group)
if fd >= 0:
os.close(fd)
if any(errors):
if self._mmap is not None:
self._mmap.close()
raise RuntimeError(
f"Failed to reserve the {self.num_bytes}-byte Engram table file in {_SHM_DIR}: " + "; ".join(e for e in errors if e)
)
self._host = torch.frombuffer(self._mmap, dtype=torch.uint8)
self._pinned = False
self._device_storage: torch.Tensor | None = None
self._rows: list[torch.Tensor] = []
self._sfs: list[torch.Tensor] = []
def _local_ranges(self) -> list[tuple[int, int]]:
"""Byte ranges of this rank's rows and scales in the shared file."""
ranges = []
for layer_idx, num_entries in enumerate(self.num_entries_per_layer):
row_bytes = num_entries * self.hidden * self.dtype.itemsize
ranges.append((self._row_offsets[layer_idx] + self.rank * row_bytes, row_bytes))
sf_bytes = num_entries * self.num_sf_packs * self.sf_dtype.itemsize
ranges.append((self._sf_offsets[layer_idx] + self.rank * sf_bytes, sf_bytes))
return [(begin, num_bytes) for begin, num_bytes in ranges if num_bytes > 0]
def _host_rows(self, layer_idx: int) -> torch.Tensor:
num_rows = self.num_entries_per_layer[layer_idx] * self.num_ranks
begin = self._row_offsets[layer_idx]
return self._host[begin : begin + num_rows * self.hidden * self.dtype.itemsize].view(self.dtype).view(num_rows, self.hidden)
def _host_sfs(self, layer_idx: int) -> torch.Tensor:
num_rows = self.num_entries_per_layer[layer_idx] * self.num_ranks
begin = self._sf_offsets[layer_idx]
num_bytes = num_rows * self.num_sf_packs * self.sf_dtype.itemsize
return self._host[begin : begin + num_bytes].view(self.sf_dtype).view(num_rows, self.num_sf_packs)
def write(self, storages: Sequence[torch.Tensor], sfs: Sequence[torch.Tensor] | None = None) -> None:
"""Copy this rank's shard of every layer; `sfs` are the local packed scales, not gathered ones."""
assert self._device_storage is None, "HostEngramBuffer.write can only be called once"
assert len(storages) == len(self.num_entries_per_layer)
assert (sfs is not None) == (self.num_sf_packs > 0)
for layer_idx, storage in enumerate(storages):
num_entries = self.num_entries_per_layer[layer_idx]
assert storage.shape == (num_entries, self.hidden) and storage.dtype == self.dtype, (storage.shape, storage.dtype)
begin = self.rank * num_entries
# The first touch places the pages on this rank's NUMA node.
self._host_rows(layer_idx)[begin : begin + num_entries].copy_(storage)
if sfs is not None:
sf = sfs[layer_idx]
assert sf.shape == (num_entries, self.num_sf_packs) and sf.dtype == self.sf_dtype, (sf.shape, sf.dtype)
self._host_sfs(layer_idx)[begin : begin + num_entries].copy_(sf)
dist.barrier(group=self.group)
PLATFORM.pin_host_memory(self._host, name="Engram tables")
self._pinned = True
device = torch.device("cuda", torch.cuda.current_device())
storage = torch._C._construct_storage_from_data_pointer(self._host.data_ptr(), device, self.num_bytes)
self._device_storage = torch.empty(0, dtype=torch.uint8, device=device).set_(storage, 0, (self.num_bytes,), (1,))
self._rows, self._sfs = [], []
for layer_idx, num_entries in enumerate(self.num_entries_per_layer):
num_rows = num_entries * self.num_ranks
begin = self._row_offsets[layer_idx]
rows = self._device_storage[begin : begin + num_rows * self.hidden * self.dtype.itemsize]
self._rows.append(rows.view(num_rows, self.hidden * self.dtype.itemsize))
if self.num_sf_packs:
begin = self._sf_offsets[layer_idx]
sf = self._device_storage[begin : begin + num_rows * self.num_sf_packs * self.sf_dtype.itemsize]
self._sfs.append(sf.view(self.sf_dtype).view(num_rows, self.num_sf_packs))
def _gather(self, layer_idx: int, indices: torch.Tensor, use_tma_aligned_col_major_sf: bool) -> object:
num_tokens = indices.size(0)
flat = indices.flatten()
# Image tokens hash to -1; their rows are masked by the gate, so return zeros.
invalid = (flat < 0).unsqueeze(1)
flat = flat.clamp_min(0)
rows = self._rows[layer_idx].index_select(0, flat).masked_fill_(invalid, 0)
rows = rows.view(self.dtype).view(num_tokens, self.num_entries_per_token * self.hidden)
if not self.num_sf_packs:
return rows
sf = self._sfs[layer_idx].index_select(0, flat).masked_fill_(invalid, 0)
sf = sf.view(num_tokens, self.num_entries_per_token * self.num_sf_packs)
if use_tma_aligned_col_major_sf:
aligned_tokens = _align(num_tokens, _NUM_ALIGNED_SF_PACKS)
backing = sf.new_empty(sf.size(1), aligned_tokens)
backing[:, :num_tokens].copy_(sf.T)
sf = backing.as_strided((num_tokens, sf.size(1)), (1, aligned_tokens))
return rows, sf
def fetch(self, indices: torch.Tensor, num_qps: int = 0, use_tma_aligned_col_major_sf: bool = True) -> list[Callable]:
"""Return one hook per layer; each gathers `[num_tokens, num_entries_per_token * hidden]` rows (and scales)."""
assert self._device_storage is not None, "HostEngramBuffer.fetch called before write"
num_layers, num_tokens, num_entries_per_token = indices.shape
assert indices.dtype == torch.int32 and indices.is_cuda and indices.is_contiguous()
assert num_layers == len(self.num_entries_per_layer) and num_entries_per_token == self.num_entries_per_token
assert num_tokens <= self.max_num_tokens
def hook(layer_idx: int) -> Callable:
return lambda: self._gather(layer_idx, indices[layer_idx], use_tma_aligned_col_major_sf)
return [hook(layer_idx) for layer_idx in range(num_layers)]
def destroy(self) -> None:
self._rows, self._sfs = [], []
self._device_storage = None
if self._pinned:
PLATFORM.unpin_host_memory(self._host.data_ptr())
self._pinned = False
if self._mmap is not None:
del self._host
self._mmap.close()
self._mmap = None