mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-11-18 19:42:28 +00:00
Run DeepSeek-V4.1-Flash offline inference on SM120 GPUs (RTX PRO 6000 Blackwell) with the same entry points as SM100. SM100 and Ascend paths are unchanged. - Build: SM120 is detected from the first visible GPU (scripts/nvidia_arch.sh). It builds DeepEP and DeepGEMM from its nv_dev branch (vendor/DeepGEMM-sm120), and no FlashMLA or DeepSelect; FlashInfer is installed from a pinned commit (requirements-nvidia-sm120.txt). - Attention: V4.1 sparse MLA runs on FlashInfer's SM120 sparse MLA kernels, reading the inline-scale KV records as stored; KV blocks are aligned to 512 bytes. - Indexer: FlashInfer's fused DSA indexer scores the index keys and selects their top-k in one call, for decode on every indexer layer (candidate source and consumer layers included) and for prefill outside the candidate layers. The remaining selections (prefill of the candidate layers, the sampler) use FlashInfer's top-k behind DeepSelect's interface (model_executor/layers/topk.py), in deterministic mode. - MoE EP: DeepEP over NCCL GIN by default, or NCCL collectives (DEEPINFER_SM120_EP_BACKEND=nccl); NCCL settings known to break the EP paths are rejected. EPLB, including decode-only draft-layer EPLB, and per-layer YoCo MoE dispatch. - Engram tables live in page-locked host shared memory; killed workers leave nothing in /dev/shm. - Features: attention DP + MoE EP, DSpark/nextn speculative decoding, CUDA graphs, async scheduling, decoder SWA bounded replay, structured outputs, image inputs, multi-node deployments. Features without an SM120 implementation (CP > 1, MTP varlen, batch-invariant mode, GPU-resident Engram tables) are refused at engine start. - Eval: chat GSM8K and CV-Bench; tests for the SM120 kernels and their DeepInfer call sites. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
217 lines
11 KiB
Python
217 lines
11 KiB
Python
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
"""Host-resident Engram tables for GPUs without the RDMA Engram fetch path.
|
|
|
|
`HostEngramBuffer` mirrors the `deep_ep.EngramBuffer` interface used by `EngramRuntime`. All ranks of
|
|
the Engram group share one `/dev/shm` mapping that holds every layer's full table and scales; each
|
|
rank page-locks the mapping and gathers rows straight from host memory over UVA, so device memory
|
|
does not grow with the table size.
|
|
|
|
The table file never has a name: rank 0 creates it with O_TMPFILE and the other ranks open it through
|
|
`/proc/<rank 0 pid>/fd/<fd>`, so a killed group leaves nothing behind in `/dev/shm`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import mmap
|
|
import os
|
|
import socket
|
|
from collections.abc import Callable, Sequence
|
|
|
|
import torch
|
|
import torch.distributed as dist
|
|
|
|
from deepinfer import PLATFORM
|
|
|
|
_SHM_DIR = "/dev/shm"
|
|
_NUM_ALIGNED_SF_PACKS = 4
|
|
_PAGE_BYTES = 1 << 21
|
|
|
|
|
|
def _align(x: int, alignment: int) -> int:
|
|
return (x + alignment - 1) // alignment * alignment
|
|
|
|
|
|
class HostEngramBuffer:
|
|
def __init__(
|
|
self,
|
|
group: dist.ProcessGroup,
|
|
num_entries_per_layer: Sequence[int],
|
|
hidden: int,
|
|
max_num_tokens: int,
|
|
num_entries_per_token: int,
|
|
dtype: torch.dtype,
|
|
num_sf_packs: int = 0,
|
|
) -> None:
|
|
assert num_entries_per_layer and dtype.itemsize in (1, 2)
|
|
assert (num_sf_packs > 0) == (dtype.itemsize == 1), "FP8 tables carry scales, BF16 tables do not"
|
|
self.group = group
|
|
self.rank, self.num_ranks = group.rank(), group.size()
|
|
self.num_entries_per_layer = list(num_entries_per_layer)
|
|
self.hidden = hidden
|
|
self.max_num_tokens = max_num_tokens
|
|
self.num_entries_per_token = num_entries_per_token
|
|
self.dtype = dtype
|
|
self.num_sf_packs = num_sf_packs
|
|
self.sf_dtype = PLATFORM.fp8_sf_type
|
|
|
|
hosts: list[str | None] = [None] * self.num_ranks
|
|
dist.all_gather_object(hosts, socket.gethostname(), group=group)
|
|
assert len(set(hosts)) == 1, (
|
|
f"Host-resident Engram tables need the whole Engram group on one host; "
|
|
f"set engram_parallel_size to at most the ranks per host: {hosts}"
|
|
)
|
|
|
|
self._row_offsets: list[int] = []
|
|
self._sf_offsets: list[int] = []
|
|
offset = 0
|
|
for num_entries in self.num_entries_per_layer:
|
|
self._row_offsets.append(offset)
|
|
offset = _align(offset + num_entries * self.num_ranks * hidden * dtype.itemsize, _PAGE_BYTES)
|
|
for num_entries in self.num_entries_per_layer:
|
|
self._sf_offsets.append(offset)
|
|
offset = _align(offset + num_entries * self.num_ranks * num_sf_packs * self.sf_dtype.itemsize, _PAGE_BYTES)
|
|
self.num_bytes = offset
|
|
|
|
fd = -1
|
|
error: str | None = None
|
|
shared: list[str | None] = [None]
|
|
if self.rank == 0:
|
|
try:
|
|
fd = os.open(_SHM_DIR, os.O_TMPFILE | os.O_RDWR, 0o600)
|
|
os.ftruncate(fd, self.num_bytes)
|
|
shared[0] = f"/proc/{os.getpid()}/fd/{fd}"
|
|
except OSError as exc:
|
|
error = f"rank 0 cannot create it: {exc}"
|
|
dist.broadcast_object_list(shared, group_src=0, group=group)
|
|
if shared[0] is not None and self.rank != 0:
|
|
try:
|
|
fd = os.open(shared[0], os.O_RDWR)
|
|
except OSError as exc:
|
|
error = f"rank {self.rank} cannot open {shared[0]}: {exc}"
|
|
self._mmap: mmap.mmap | None = None
|
|
if error is None and fd >= 0:
|
|
try:
|
|
# Each rank reserves only its own shard so tmpfs places those pages on its NUMA node, and a full
|
|
# /dev/shm fails here on every rank instead of raising SIGBUS when write() touches the pages.
|
|
for begin, num_bytes in self._local_ranges():
|
|
os.posix_fallocate(fd, begin, num_bytes)
|
|
self._mmap = mmap.mmap(fd, self.num_bytes, mmap.MAP_SHARED, mmap.PROT_READ | mmap.PROT_WRITE)
|
|
except OSError as exc:
|
|
error = f"rank {self.rank}: {exc}"
|
|
# Rank 0 keeps its descriptor open until every rank has opened the file through it.
|
|
errors: list[str | None] = [None] * self.num_ranks
|
|
dist.all_gather_object(errors, error, group=group)
|
|
if fd >= 0:
|
|
os.close(fd)
|
|
if any(errors):
|
|
if self._mmap is not None:
|
|
self._mmap.close()
|
|
raise RuntimeError(
|
|
f"Failed to reserve the {self.num_bytes}-byte Engram table file in {_SHM_DIR}: " + "; ".join(e for e in errors if e)
|
|
)
|
|
|
|
self._host = torch.frombuffer(self._mmap, dtype=torch.uint8)
|
|
self._pinned = False
|
|
self._device_storage: torch.Tensor | None = None
|
|
self._rows: list[torch.Tensor] = []
|
|
self._sfs: list[torch.Tensor] = []
|
|
|
|
def _local_ranges(self) -> list[tuple[int, int]]:
|
|
"""Byte ranges of this rank's rows and scales in the shared file."""
|
|
ranges = []
|
|
for layer_idx, num_entries in enumerate(self.num_entries_per_layer):
|
|
row_bytes = num_entries * self.hidden * self.dtype.itemsize
|
|
ranges.append((self._row_offsets[layer_idx] + self.rank * row_bytes, row_bytes))
|
|
sf_bytes = num_entries * self.num_sf_packs * self.sf_dtype.itemsize
|
|
ranges.append((self._sf_offsets[layer_idx] + self.rank * sf_bytes, sf_bytes))
|
|
return [(begin, num_bytes) for begin, num_bytes in ranges if num_bytes > 0]
|
|
|
|
def _host_rows(self, layer_idx: int) -> torch.Tensor:
|
|
num_rows = self.num_entries_per_layer[layer_idx] * self.num_ranks
|
|
begin = self._row_offsets[layer_idx]
|
|
return self._host[begin : begin + num_rows * self.hidden * self.dtype.itemsize].view(self.dtype).view(num_rows, self.hidden)
|
|
|
|
def _host_sfs(self, layer_idx: int) -> torch.Tensor:
|
|
num_rows = self.num_entries_per_layer[layer_idx] * self.num_ranks
|
|
begin = self._sf_offsets[layer_idx]
|
|
num_bytes = num_rows * self.num_sf_packs * self.sf_dtype.itemsize
|
|
return self._host[begin : begin + num_bytes].view(self.sf_dtype).view(num_rows, self.num_sf_packs)
|
|
|
|
def write(self, storages: Sequence[torch.Tensor], sfs: Sequence[torch.Tensor] | None = None) -> None:
|
|
"""Copy this rank's shard of every layer; `sfs` are the local packed scales, not gathered ones."""
|
|
assert self._device_storage is None, "HostEngramBuffer.write can only be called once"
|
|
assert len(storages) == len(self.num_entries_per_layer)
|
|
assert (sfs is not None) == (self.num_sf_packs > 0)
|
|
for layer_idx, storage in enumerate(storages):
|
|
num_entries = self.num_entries_per_layer[layer_idx]
|
|
assert storage.shape == (num_entries, self.hidden) and storage.dtype == self.dtype, (storage.shape, storage.dtype)
|
|
begin = self.rank * num_entries
|
|
# The first touch places the pages on this rank's NUMA node.
|
|
self._host_rows(layer_idx)[begin : begin + num_entries].copy_(storage)
|
|
if sfs is not None:
|
|
sf = sfs[layer_idx]
|
|
assert sf.shape == (num_entries, self.num_sf_packs) and sf.dtype == self.sf_dtype, (sf.shape, sf.dtype)
|
|
self._host_sfs(layer_idx)[begin : begin + num_entries].copy_(sf)
|
|
dist.barrier(group=self.group)
|
|
|
|
PLATFORM.pin_host_memory(self._host, name="Engram tables")
|
|
self._pinned = True
|
|
device = torch.device("cuda", torch.cuda.current_device())
|
|
storage = torch._C._construct_storage_from_data_pointer(self._host.data_ptr(), device, self.num_bytes)
|
|
self._device_storage = torch.empty(0, dtype=torch.uint8, device=device).set_(storage, 0, (self.num_bytes,), (1,))
|
|
self._rows, self._sfs = [], []
|
|
for layer_idx, num_entries in enumerate(self.num_entries_per_layer):
|
|
num_rows = num_entries * self.num_ranks
|
|
begin = self._row_offsets[layer_idx]
|
|
rows = self._device_storage[begin : begin + num_rows * self.hidden * self.dtype.itemsize]
|
|
self._rows.append(rows.view(num_rows, self.hidden * self.dtype.itemsize))
|
|
if self.num_sf_packs:
|
|
begin = self._sf_offsets[layer_idx]
|
|
sf = self._device_storage[begin : begin + num_rows * self.num_sf_packs * self.sf_dtype.itemsize]
|
|
self._sfs.append(sf.view(self.sf_dtype).view(num_rows, self.num_sf_packs))
|
|
|
|
def _gather(self, layer_idx: int, indices: torch.Tensor, use_tma_aligned_col_major_sf: bool) -> object:
|
|
num_tokens = indices.size(0)
|
|
flat = indices.flatten()
|
|
# Image tokens hash to -1; their rows are masked by the gate, so return zeros.
|
|
invalid = (flat < 0).unsqueeze(1)
|
|
flat = flat.clamp_min(0)
|
|
rows = self._rows[layer_idx].index_select(0, flat).masked_fill_(invalid, 0)
|
|
rows = rows.view(self.dtype).view(num_tokens, self.num_entries_per_token * self.hidden)
|
|
if not self.num_sf_packs:
|
|
return rows
|
|
sf = self._sfs[layer_idx].index_select(0, flat).masked_fill_(invalid, 0)
|
|
sf = sf.view(num_tokens, self.num_entries_per_token * self.num_sf_packs)
|
|
if use_tma_aligned_col_major_sf:
|
|
aligned_tokens = _align(num_tokens, _NUM_ALIGNED_SF_PACKS)
|
|
backing = sf.new_empty(sf.size(1), aligned_tokens)
|
|
backing[:, :num_tokens].copy_(sf.T)
|
|
sf = backing.as_strided((num_tokens, sf.size(1)), (1, aligned_tokens))
|
|
return rows, sf
|
|
|
|
def fetch(self, indices: torch.Tensor, num_qps: int = 0, use_tma_aligned_col_major_sf: bool = True) -> list[Callable]:
|
|
"""Return one hook per layer; each gathers `[num_tokens, num_entries_per_token * hidden]` rows (and scales)."""
|
|
assert self._device_storage is not None, "HostEngramBuffer.fetch called before write"
|
|
num_layers, num_tokens, num_entries_per_token = indices.shape
|
|
assert indices.dtype == torch.int32 and indices.is_cuda and indices.is_contiguous()
|
|
assert num_layers == len(self.num_entries_per_layer) and num_entries_per_token == self.num_entries_per_token
|
|
assert num_tokens <= self.max_num_tokens
|
|
|
|
def hook(layer_idx: int) -> Callable:
|
|
return lambda: self._gather(layer_idx, indices[layer_idx], use_tma_aligned_col_major_sf)
|
|
|
|
return [hook(layer_idx) for layer_idx in range(num_layers)]
|
|
|
|
def destroy(self) -> None:
|
|
self._rows, self._sfs = [], []
|
|
self._device_storage = None
|
|
if self._pinned:
|
|
PLATFORM.unpin_host_memory(self._host.data_ptr())
|
|
self._pinned = False
|
|
if self._mmap is not None:
|
|
del self._host
|
|
self._mmap.close()
|
|
self._mmap = None
|