# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """Host-resident Engram tables for GPUs without the RDMA Engram fetch path. `HostEngramBuffer` mirrors the `deep_ep.EngramBuffer` interface used by `EngramRuntime`. All ranks of the Engram group share one `/dev/shm` mapping that holds every layer's full table and scales; each rank page-locks the mapping and gathers rows straight from host memory over UVA, so device memory does not grow with the table size. The table file never has a name: rank 0 creates it with O_TMPFILE and the other ranks open it through `/proc//fd/`, so a killed group leaves nothing behind in `/dev/shm`. """ from __future__ import annotations import mmap import os import socket from collections.abc import Callable, Sequence import torch import torch.distributed as dist from deepinfer import PLATFORM from deepinfer.model_executor.layers.tilelang_ops.common import align _SHM_DIR = "/dev/shm" _NUM_ALIGNED_SF_PACKS = 4 _PAGE_BYTES = 1 << 21 class HostEngramBuffer: def __init__( self, group: dist.ProcessGroup, num_entries_per_layer: Sequence[int], hidden: int, max_num_tokens: int, num_entries_per_token: int, dtype: torch.dtype, num_sf_packs: int = 0, ) -> None: assert num_entries_per_layer and dtype.itemsize in (1, 2) assert (num_sf_packs > 0) == (dtype.itemsize == 1), "FP8 tables carry scales, BF16 tables do not" self.group = group self.rank, self.num_ranks = group.rank(), group.size() self.num_entries_per_layer = list(num_entries_per_layer) self.hidden = hidden self.max_num_tokens = max_num_tokens self.num_entries_per_token = num_entries_per_token self.dtype = dtype self.num_sf_packs = num_sf_packs self.sf_dtype = PLATFORM.fp8_sf_type hosts: list[str | None] = [None] * self.num_ranks dist.all_gather_object(hosts, socket.gethostname(), group=group) if len(set(hosts)) != 1: # Normally refused earlier by sm120_support.check_rank_placement; kept because /proc//fd only # resolves on the host that created the table file. raise ValueError( f"Host-resident Engram tables need the whole Engram group on one host; " f"set engram_parallel_size to at most the ranks per host: {hosts}" ) self._row_offsets: list[int] = [] self._sf_offsets: list[int] = [] offset = 0 for num_entries in self.num_entries_per_layer: self._row_offsets.append(offset) offset = align(offset + num_entries * self.num_ranks * hidden * dtype.itemsize, _PAGE_BYTES) for num_entries in self.num_entries_per_layer: self._sf_offsets.append(offset) offset = align(offset + num_entries * self.num_ranks * num_sf_packs * self.sf_dtype.itemsize, _PAGE_BYTES) self.num_bytes = offset fd = -1 error: str | None = None shared: list[str | None] = [None] if self.rank == 0: try: fd = os.open(_SHM_DIR, os.O_TMPFILE | os.O_RDWR, 0o600) os.ftruncate(fd, self.num_bytes) shared[0] = f"/proc/{os.getpid()}/fd/{fd}" except OSError as exc: error = f"rank 0 cannot create it: {exc}" dist.broadcast_object_list(shared, group_src=0, group=group) if shared[0] is not None and self.rank != 0: try: fd = os.open(shared[0], os.O_RDWR) except OSError as exc: error = f"rank {self.rank} cannot open {shared[0]}: {exc}" self._mmap: mmap.mmap | None = None if error is None and fd >= 0: try: # Each rank reserves only its own shard so tmpfs places those pages on its NUMA node, and a full # /dev/shm fails here on every rank instead of raising SIGBUS when write() touches the pages. for begin, num_bytes in self._local_ranges(): os.posix_fallocate(fd, begin, num_bytes) self._mmap = mmap.mmap(fd, self.num_bytes, mmap.MAP_SHARED, mmap.PROT_READ | mmap.PROT_WRITE) except OSError as exc: error = f"rank {self.rank}: {exc}" # Rank 0 keeps its descriptor open until every rank has opened the file through it. errors: list[str | None] = [None] * self.num_ranks dist.all_gather_object(errors, error, group=group) if fd >= 0: os.close(fd) if any(errors): if self._mmap is not None: self._mmap.close() raise RuntimeError( f"Failed to reserve the {self.num_bytes}-byte Engram table file in {_SHM_DIR}: " + "; ".join(e for e in errors if e) ) self._host = torch.frombuffer(self._mmap, dtype=torch.uint8) self._pinned = False self._device_storage: torch.Tensor | None = None self._rows: list[torch.Tensor] = [] self._sfs: list[torch.Tensor] = [] def _local_ranges(self) -> list[tuple[int, int]]: """Byte ranges of this rank's rows and scales in the shared file.""" ranges = [] for layer_idx, num_entries in enumerate(self.num_entries_per_layer): row_bytes = num_entries * self.hidden * self.dtype.itemsize ranges.append((self._row_offsets[layer_idx] + self.rank * row_bytes, row_bytes)) sf_bytes = num_entries * self.num_sf_packs * self.sf_dtype.itemsize ranges.append((self._sf_offsets[layer_idx] + self.rank * sf_bytes, sf_bytes)) return [(begin, num_bytes) for begin, num_bytes in ranges if num_bytes > 0] def _host_rows(self, layer_idx: int) -> torch.Tensor: num_rows = self.num_entries_per_layer[layer_idx] * self.num_ranks begin = self._row_offsets[layer_idx] return self._host[begin : begin + num_rows * self.hidden * self.dtype.itemsize].view(self.dtype).view(num_rows, self.hidden) def _host_sfs(self, layer_idx: int) -> torch.Tensor: num_rows = self.num_entries_per_layer[layer_idx] * self.num_ranks begin = self._sf_offsets[layer_idx] num_bytes = num_rows * self.num_sf_packs * self.sf_dtype.itemsize return self._host[begin : begin + num_bytes].view(self.sf_dtype).view(num_rows, self.num_sf_packs) def write(self, storages: Sequence[torch.Tensor], sfs: Sequence[torch.Tensor] | None = None) -> None: """Copy this rank's shard of every layer; `sfs` are the local packed scales, not gathered ones.""" assert self._device_storage is None, "HostEngramBuffer.write can only be called once" assert len(storages) == len(self.num_entries_per_layer) assert (sfs is not None) == (self.num_sf_packs > 0) for layer_idx, storage in enumerate(storages): num_entries = self.num_entries_per_layer[layer_idx] assert storage.shape == (num_entries, self.hidden) and storage.dtype == self.dtype, (storage.shape, storage.dtype) begin = self.rank * num_entries # The first touch places the pages on this rank's NUMA node. self._host_rows(layer_idx)[begin : begin + num_entries].copy_(storage) if sfs is not None: sf = sfs[layer_idx] assert sf.shape == (num_entries, self.num_sf_packs) and sf.dtype == self.sf_dtype, (sf.shape, sf.dtype) self._host_sfs(layer_idx)[begin : begin + num_entries].copy_(sf) dist.barrier(group=self.group) PLATFORM.pin_host_memory(self._host, name="Engram tables") self._pinned = True device = torch.device("cuda", torch.cuda.current_device()) storage = torch._C._construct_storage_from_data_pointer(self._host.data_ptr(), device, self.num_bytes) self._device_storage = torch.empty(0, dtype=torch.uint8, device=device).set_(storage, 0, (self.num_bytes,), (1,)) self._rows, self._sfs = [], [] for layer_idx, num_entries in enumerate(self.num_entries_per_layer): num_rows = num_entries * self.num_ranks begin = self._row_offsets[layer_idx] rows = self._device_storage[begin : begin + num_rows * self.hidden * self.dtype.itemsize] self._rows.append(rows.view(num_rows, self.hidden * self.dtype.itemsize)) if self.num_sf_packs: begin = self._sf_offsets[layer_idx] sf = self._device_storage[begin : begin + num_rows * self.num_sf_packs * self.sf_dtype.itemsize] self._sfs.append(sf.view(self.sf_dtype).view(num_rows, self.num_sf_packs)) def _gather(self, layer_idx: int, indices: torch.Tensor, use_tma_aligned_col_major_sf: bool) -> object: num_tokens = indices.size(0) flat = indices.flatten() # Image tokens hash to -1; their rows are masked by the gate, so return zeros. invalid = (flat < 0).unsqueeze(1) flat = flat.clamp_min(0) rows = self._rows[layer_idx].index_select(0, flat).masked_fill_(invalid, 0) rows = rows.view(self.dtype).view(num_tokens, self.num_entries_per_token * self.hidden) if not self.num_sf_packs: return rows sf = self._sfs[layer_idx].index_select(0, flat).masked_fill_(invalid, 0) sf = sf.view(num_tokens, self.num_entries_per_token * self.num_sf_packs) if use_tma_aligned_col_major_sf: aligned_tokens = align(num_tokens, _NUM_ALIGNED_SF_PACKS) backing = sf.new_empty(sf.size(1), aligned_tokens) backing[:, :num_tokens].copy_(sf.T) sf = backing.as_strided((num_tokens, sf.size(1)), (1, aligned_tokens)) return rows, sf def fetch(self, indices: torch.Tensor, num_qps: int = 0, use_tma_aligned_col_major_sf: bool = True) -> list[Callable]: """Return one hook per layer; each gathers `[num_tokens, num_entries_per_token * hidden]` rows (and scales).""" assert self._device_storage is not None, "HostEngramBuffer.fetch called before write" num_layers, num_tokens, num_entries_per_token = indices.shape assert indices.dtype == torch.int32 and indices.is_cuda and indices.is_contiguous() assert num_layers == len(self.num_entries_per_layer) and num_entries_per_token == self.num_entries_per_token assert num_tokens <= self.max_num_tokens def hook(layer_idx: int) -> Callable: return lambda: self._gather(layer_idx, indices[layer_idx], use_tma_aligned_col_major_sf) return [hook(layer_idx) for layer_idx in range(num_layers)] def destroy(self) -> None: self._rows, self._sfs = [], [] self._device_storage = None if self._pinned: PLATFORM.unpin_host_memory(self._host.data_ptr()) self._pinned = False if self._mmap is not None: del self._host self._mmap.close() self._mmap = None