Files
DeepInfer/tests/worker/test_host_engram.py
T
shunyadandClaude Opus 5.5 9da41ad18e refactor(platform): describe NVIDIA families with a capability table
Call sites tested `not PLATFORM.is_sm120` to mean "has the SM100 fused
kernels", hard-coded SM120 constants next to their use, and treated any GPU
that is not 12.x as SM100. A new family (or 12.x variant) would need every
site revisited, and an unsupported GPU silently took SM100 paths.

- deepinfer.platform.NVIDIA_ARCHS maps compute capability major to an
  NvidiaArch entry: sm100_fused_kernels, grouped_gemm_zero_padding,
  kv_block_alignment and default_comm_sms, with the same values SM100 and
  SM120 used before.
- Platform exposes them as capabilities (has_sm100_fused_kernels, ...), with
  the previous values on Ascend; the mega MoE / gate / mHC, fused
  norm-rope-attn-rope-cast, FP8 einsum output, grouped GEMM zero padding, KV
  block alignment and default EP comm SM sites ask for the capability.
- PlatformGPU.nvidia_arch selects the entry from the first visible GPU. No
  visible GPU still means SM100; a GPU of an unsupported family now raises
  instead of running SM100 kernels. is_sm120 stays for the SM120 backends.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 22:28:43 -08:00

274 lines
11 KiB
Python

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
import os
import signal
import time
from types import SimpleNamespace
import pytest
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
from deepinfer.platform import NVIDIA_ARCHS, PlatformGPU
from deepinfer.utils import get_open_port
_MIB = 1 << 20
class _FakeCudart:
cudaError = SimpleNamespace(success=0)
def __init__(self, fail_at: int | None = None) -> None:
self.registered: dict[int, int] = {}
self.calls: list[tuple[int, int]] = []
self.fail_at = fail_at
def cudaHostRegister(self, ptr: int, nbytes: int, flags: int) -> int:
self.calls.append((ptr, nbytes))
if self.fail_at is not None and len(self.calls) == self.fail_at:
return 2
self.registered[ptr] = nbytes
return 0
def cudaHostUnregister(self, ptr: int) -> int:
del self.registered[ptr]
return 0
def _platform(monkeypatch: pytest.MonkeyPatch, cudart: _FakeCudart, sm120: bool = True) -> PlatformGPU:
monkeypatch.setattr(torch.cuda, "cudart", lambda: cudart)
platform = PlatformGPU()
platform.__dict__["nvidia_arch"] = NVIDIA_ARCHS[12 if sm120 else 10]
return platform
def _fake_tensor(ptr: int, nbytes: int) -> SimpleNamespace:
return SimpleNamespace(data_ptr=lambda: ptr, nbytes=nbytes)
@pytest.mark.parametrize("offset", [0, 4096, 64 * _MIB - 4096])
def test_sm120_pins_in_aligned_chunks(monkeypatch: pytest.MonkeyPatch, offset: int) -> None:
cudart = _FakeCudart()
platform = _platform(monkeypatch, cudart)
ptr, nbytes = (1 << 40) + offset, 5 * 1024 * _MIB + 12345
platform.pin_host_memory(_fake_tensor(ptr, nbytes))
assert sum(size for _, size in cudart.calls) == nbytes
assert cudart.calls[0][0] == ptr
for (start, size), (next_start, _) in zip(cudart.calls, cudart.calls[1:], strict=False):
assert start + size == next_start and next_start % (64 * _MIB) == 0
assert max(size for _, size in cudart.calls) <= 64 * _MIB < 1 << 31
platform.unpin_host_memory(ptr)
assert not cudart.registered
def test_sm120_pin_failure_rolls_back_and_names_buffer(monkeypatch: pytest.MonkeyPatch) -> None:
cudart = _FakeCudart(fail_at=3)
platform = _platform(monkeypatch, cudart)
with pytest.raises(RuntimeError, match="Engram tables"):
platform.pin_host_memory(_fake_tensor(1 << 40, 256 * _MIB), name="Engram tables")
assert not cudart.registered
def test_non_sm120_pins_once(monkeypatch: pytest.MonkeyPatch) -> None:
cudart = _FakeCudart()
platform = _platform(monkeypatch, cudart, sm120=False)
platform.pin_host_memory(_fake_tensor(1 << 40, 3 << 30))
assert cudart.calls == [(1 << 40, 3 << 30)]
platform.unpin_host_memory(1 << 40)
assert not cudart.registered
_HIDDEN = 256
_ENTRIES_PER_TOKEN = 4
_SF_PACKS = 2
_SHARD_ROWS = [150_000, 70_001]
_NUM_TOKENS = 37
_TIMEOUT_S = 300
def _shard(layer_idx: int, rank: int) -> tuple[torch.Tensor, torch.Tensor]:
generator = torch.Generator().manual_seed(1000 * layer_idx + rank)
rows = torch.randint(0, 256, (_SHARD_ROWS[layer_idx], _HIDDEN), dtype=torch.uint8, generator=generator)
sf = torch.randint(0, 1 << 30, (_SHARD_ROWS[layer_idx], _SF_PACKS), dtype=torch.int32, generator=generator)
return rows.view(torch.float8_e4m3fn), sf
def _named_shm_mappings() -> list[str]:
"""Files mapped by this process that still have a name in /dev/shm (excluding multiprocessing's named semaphores)."""
with open("/proc/self/maps") as maps:
paths = {line.split(maxsplit=5)[5].strip() for line in maps if len(line.split(maxsplit=5)) == 6}
return sorted(
path for path in paths if path.startswith("/dev/shm/") and not path.startswith("/dev/shm/sem.") and not path.endswith("(deleted)")
)
def _worker(rank: int, world_size: int, port: int, failures: mp.Queue) -> None:
try:
torch.cuda.set_device(rank)
dist.init_process_group("nccl", init_method=f"tcp://127.0.0.1:{port}", rank=rank, world_size=world_size)
from deepinfer.model_executor.parallel_utils.host_engram import HostEngramBuffer
group = dist.group.WORLD
allocated_before = torch.cuda.memory_allocated()
buffer = HostEngramBuffer(group, _SHARD_ROWS, _HIDDEN, 64, _ENTRIES_PER_TOKEN, torch.float8_e4m3fn, _SF_PACKS)
shards = [_shard(layer_idx, rank) for layer_idx in range(len(_SHARD_ROWS))]
buffer.write([rows for rows, _ in shards], sfs=[sf for _, sf in shards])
full = [
[torch.cat(parts) for parts in zip(*(_shard(layer_idx, r) for r in range(world_size)), strict=True)]
for layer_idx in range(len(_SHARD_ROWS))
]
generator = torch.Generator().manual_seed(7)
indices = torch.stack(
[
torch.randint(0, n * world_size, (_NUM_TOKENS, _ENTRIES_PER_TOKEN), generator=generator, dtype=torch.int32)
for n in _SHARD_ROWS
]
)
indices[:, 3] = -1
hooks = buffer.fetch(indices.cuda().contiguous())
for layer_idx, hook in enumerate(hooks):
rows, sf = hook()
idx = indices[layer_idx].flatten().long()
invalid = idx < 0
ref_rows = full[layer_idx][0].view(torch.uint8)[idx.clamp_min(0)]
ref_rows[invalid] = 0
ref_sf = full[layer_idx][1][idx.clamp_min(0)]
ref_sf[invalid] = 0
assert rows.shape == (_NUM_TOKENS, _ENTRIES_PER_TOKEN * _HIDDEN) and rows.dtype == torch.float8_e4m3fn
assert torch.equal(rows.view(torch.uint8).cpu(), ref_rows.view(_NUM_TOKENS, -1))
aligned = (_NUM_TOKENS + 3) // 4 * 4
assert sf.shape == (_NUM_TOKENS, _ENTRIES_PER_TOKEN * _SF_PACKS) and sf.stride() == (1, aligned)
assert torch.equal(sf.cpu(), ref_sf.view(_NUM_TOKENS, -1))
torch.cuda.synchronize()
assert torch.cuda.memory_allocated() - allocated_before < 16 * _MIB
assert _named_shm_mappings() == []
buffer.destroy()
dist.barrier()
dist.destroy_process_group()
except BaseException as error:
failures.put(f"rank {rank}: {error!r}")
raise
def test_host_engram_fetch_matches_tables() -> None:
from deep_cluster.hardware import is_sm_major_12
if not torch.cuda.is_available() or torch.cuda.device_count() < 2 or not is_sm_major_12():
pytest.skip("needs two SM120 GPUs")
context = mp.get_context("spawn")
failures = context.Queue()
port = get_open_port()
processes = [context.Process(target=_worker, args=(rank, 2, port, failures)) for rank in range(2)]
for process in processes:
process.start()
try:
deadline = time.monotonic() + _TIMEOUT_S
for process in processes:
process.join(timeout=max(0.0, deadline - time.monotonic()))
assert not any(process.is_alive() for process in processes), f"timed out after {_TIMEOUT_S}s"
errors = [failures.get() for _ in range(failures.qsize())] if not failures.empty() else []
assert all(process.exitcode == 0 for process in processes), errors
finally:
for process in processes:
if process.is_alive():
process.kill()
process.join()
def _cpu_worker(rank: int, world_size: int, port: int, results: mp.Queue, hang_in_fallocate: bool) -> None:
"""Build a HostEngramBuffer over gloo: the table file never has a name and all ranks map the same file."""
try:
dist.init_process_group("gloo", init_method=f"tcp://127.0.0.1:{port}", rank=rank, world_size=world_size)
from deepinfer.model_executor.parallel_utils import host_engram
fallocate = os.posix_fallocate
def checked_fallocate(fd: int, offset: int, length: int) -> None:
results.put(("fallocate", rank, os.readlink(f"/proc/self/fd/{fd}")))
if hang_in_fallocate:
time.sleep(3600)
fallocate(fd, offset, length)
host_engram.os = SimpleNamespace(**{name: getattr(os, name) for name in dir(os) if not name.startswith("__")})
host_engram.os.posix_fallocate = checked_fallocate
buffer = host_engram.HostEngramBuffer(dist.group.WORLD, [1000, 300], 64, 16, 4, torch.float8_e4m3fn, 2)
results.put(("mapped", rank, _named_shm_mappings()))
rows = buffer._host_rows(1).view(torch.uint8)
rows[rank * 300 : (rank + 1) * 300].fill_(rank + 1)
dist.barrier()
results.put(("rows", rank, sorted(set(rows.flatten().tolist()))))
buffer.destroy()
dist.barrier()
dist.destroy_process_group()
results.put(("done", rank, None))
except BaseException as error:
results.put(("error", rank, repr(error)))
raise
def _run_cpu_group(hang_in_fallocate: bool) -> tuple[list[mp.Process], mp.Queue]:
context = mp.get_context("spawn")
results = context.Queue()
port = get_open_port()
processes = [context.Process(target=_cpu_worker, args=(rank, 2, port, results, hang_in_fallocate)) for rank in range(2)]
for process in processes:
process.start()
return processes, results
def _collect(results: mp.Queue, kinds: set[str], count: int) -> list[tuple]:
collected = []
while len([r for r in collected if r[0] in kinds]) < count:
item = results.get(timeout=_TIMEOUT_S)
assert item[0] != "error", item
collected.append(item)
return collected
def test_host_engram_table_file_never_has_a_name() -> None:
processes, results = _run_cpu_group(hang_in_fallocate=False)
try:
items = _collect(results, {"done"}, 2)
for kind, rank, value in items:
if kind == "fallocate":
assert value.startswith("/dev/shm/") and value.endswith("(deleted)"), (rank, value)
elif kind == "mapped":
assert value == [], (rank, value)
elif kind == "rows":
# Each of the two ranks writes its own shard and both see each other's
assert value == [1, 2], (rank, value)
assert {kind for kind, _, _ in items} == {"fallocate", "mapped", "rows", "done"}
for process in processes:
process.join(timeout=60)
assert process.exitcode == 0
finally:
for process in processes:
if process.is_alive():
process.kill()
process.join()
def test_a_group_killed_while_reserving_leaves_no_table_file() -> None:
before = set(os.listdir("/dev/shm"))
processes, results = _run_cpu_group(hang_in_fallocate=True)
try:
items = _collect(results, {"fallocate"}, 2)
assert all(value.endswith("(deleted)") for _, _, value in items), items
for process in processes:
os.kill(process.pid, signal.SIGKILL)
for process in processes:
process.join(timeout=60)
assert not any(name.startswith("deepinfer-engram-") for name in set(os.listdir("/dev/shm")) - before)
finally:
for process in processes:
if process.is_alive():
process.kill()
process.join()