mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-11-20 04:22:27 +00:00
Call sites tested `not PLATFORM.is_sm120` to mean "has the SM100 fused kernels", hard-coded SM120 constants next to their use, and treated any GPU that is not 12.x as SM100. A new family (or 12.x variant) would need every site revisited, and an unsupported GPU silently took SM100 paths. - deepinfer.platform.NVIDIA_ARCHS maps compute capability major to an NvidiaArch entry: sm100_fused_kernels, grouped_gemm_zero_padding, kv_block_alignment and default_comm_sms, with the same values SM100 and SM120 used before. - Platform exposes them as capabilities (has_sm100_fused_kernels, ...), with the previous values on Ascend; the mega MoE / gate / mHC, fused norm-rope-attn-rope-cast, FP8 einsum output, grouped GEMM zero padding, KV block alignment and default EP comm SM sites ask for the capability. - PlatformGPU.nvidia_arch selects the entry from the first visible GPU. No visible GPU still means SM100; a GPU of an unsupported family now raises instead of running SM100 kernels. is_sm120 stays for the SM120 backends. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
274 lines
11 KiB
Python
274 lines
11 KiB
Python
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import signal
|
|
import time
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
import torch
|
|
import torch.distributed as dist
|
|
import torch.multiprocessing as mp
|
|
|
|
from deepinfer.platform import NVIDIA_ARCHS, PlatformGPU
|
|
from deepinfer.utils import get_open_port
|
|
|
|
_MIB = 1 << 20
|
|
|
|
|
|
class _FakeCudart:
|
|
cudaError = SimpleNamespace(success=0)
|
|
|
|
def __init__(self, fail_at: int | None = None) -> None:
|
|
self.registered: dict[int, int] = {}
|
|
self.calls: list[tuple[int, int]] = []
|
|
self.fail_at = fail_at
|
|
|
|
def cudaHostRegister(self, ptr: int, nbytes: int, flags: int) -> int:
|
|
self.calls.append((ptr, nbytes))
|
|
if self.fail_at is not None and len(self.calls) == self.fail_at:
|
|
return 2
|
|
self.registered[ptr] = nbytes
|
|
return 0
|
|
|
|
def cudaHostUnregister(self, ptr: int) -> int:
|
|
del self.registered[ptr]
|
|
return 0
|
|
|
|
|
|
def _platform(monkeypatch: pytest.MonkeyPatch, cudart: _FakeCudart, sm120: bool = True) -> PlatformGPU:
|
|
monkeypatch.setattr(torch.cuda, "cudart", lambda: cudart)
|
|
platform = PlatformGPU()
|
|
platform.__dict__["nvidia_arch"] = NVIDIA_ARCHS[12 if sm120 else 10]
|
|
return platform
|
|
|
|
|
|
def _fake_tensor(ptr: int, nbytes: int) -> SimpleNamespace:
|
|
return SimpleNamespace(data_ptr=lambda: ptr, nbytes=nbytes)
|
|
|
|
|
|
@pytest.mark.parametrize("offset", [0, 4096, 64 * _MIB - 4096])
|
|
def test_sm120_pins_in_aligned_chunks(monkeypatch: pytest.MonkeyPatch, offset: int) -> None:
|
|
cudart = _FakeCudart()
|
|
platform = _platform(monkeypatch, cudart)
|
|
ptr, nbytes = (1 << 40) + offset, 5 * 1024 * _MIB + 12345
|
|
platform.pin_host_memory(_fake_tensor(ptr, nbytes))
|
|
|
|
assert sum(size for _, size in cudart.calls) == nbytes
|
|
assert cudart.calls[0][0] == ptr
|
|
for (start, size), (next_start, _) in zip(cudart.calls, cudart.calls[1:], strict=False):
|
|
assert start + size == next_start and next_start % (64 * _MIB) == 0
|
|
assert max(size for _, size in cudart.calls) <= 64 * _MIB < 1 << 31
|
|
|
|
platform.unpin_host_memory(ptr)
|
|
assert not cudart.registered
|
|
|
|
|
|
def test_sm120_pin_failure_rolls_back_and_names_buffer(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
cudart = _FakeCudart(fail_at=3)
|
|
platform = _platform(monkeypatch, cudart)
|
|
with pytest.raises(RuntimeError, match="Engram tables"):
|
|
platform.pin_host_memory(_fake_tensor(1 << 40, 256 * _MIB), name="Engram tables")
|
|
assert not cudart.registered
|
|
|
|
|
|
def test_non_sm120_pins_once(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
cudart = _FakeCudart()
|
|
platform = _platform(monkeypatch, cudart, sm120=False)
|
|
platform.pin_host_memory(_fake_tensor(1 << 40, 3 << 30))
|
|
assert cudart.calls == [(1 << 40, 3 << 30)]
|
|
platform.unpin_host_memory(1 << 40)
|
|
assert not cudart.registered
|
|
|
|
|
|
_HIDDEN = 256
|
|
_ENTRIES_PER_TOKEN = 4
|
|
_SF_PACKS = 2
|
|
_SHARD_ROWS = [150_000, 70_001]
|
|
_NUM_TOKENS = 37
|
|
_TIMEOUT_S = 300
|
|
|
|
|
|
def _shard(layer_idx: int, rank: int) -> tuple[torch.Tensor, torch.Tensor]:
|
|
generator = torch.Generator().manual_seed(1000 * layer_idx + rank)
|
|
rows = torch.randint(0, 256, (_SHARD_ROWS[layer_idx], _HIDDEN), dtype=torch.uint8, generator=generator)
|
|
sf = torch.randint(0, 1 << 30, (_SHARD_ROWS[layer_idx], _SF_PACKS), dtype=torch.int32, generator=generator)
|
|
return rows.view(torch.float8_e4m3fn), sf
|
|
|
|
|
|
def _named_shm_mappings() -> list[str]:
|
|
"""Files mapped by this process that still have a name in /dev/shm (excluding multiprocessing's named semaphores)."""
|
|
with open("/proc/self/maps") as maps:
|
|
paths = {line.split(maxsplit=5)[5].strip() for line in maps if len(line.split(maxsplit=5)) == 6}
|
|
return sorted(
|
|
path for path in paths if path.startswith("/dev/shm/") and not path.startswith("/dev/shm/sem.") and not path.endswith("(deleted)")
|
|
)
|
|
|
|
|
|
def _worker(rank: int, world_size: int, port: int, failures: mp.Queue) -> None:
|
|
try:
|
|
torch.cuda.set_device(rank)
|
|
dist.init_process_group("nccl", init_method=f"tcp://127.0.0.1:{port}", rank=rank, world_size=world_size)
|
|
from deepinfer.model_executor.parallel_utils.host_engram import HostEngramBuffer
|
|
|
|
group = dist.group.WORLD
|
|
allocated_before = torch.cuda.memory_allocated()
|
|
buffer = HostEngramBuffer(group, _SHARD_ROWS, _HIDDEN, 64, _ENTRIES_PER_TOKEN, torch.float8_e4m3fn, _SF_PACKS)
|
|
shards = [_shard(layer_idx, rank) for layer_idx in range(len(_SHARD_ROWS))]
|
|
buffer.write([rows for rows, _ in shards], sfs=[sf for _, sf in shards])
|
|
|
|
full = [
|
|
[torch.cat(parts) for parts in zip(*(_shard(layer_idx, r) for r in range(world_size)), strict=True)]
|
|
for layer_idx in range(len(_SHARD_ROWS))
|
|
]
|
|
generator = torch.Generator().manual_seed(7)
|
|
indices = torch.stack(
|
|
[
|
|
torch.randint(0, n * world_size, (_NUM_TOKENS, _ENTRIES_PER_TOKEN), generator=generator, dtype=torch.int32)
|
|
for n in _SHARD_ROWS
|
|
]
|
|
)
|
|
indices[:, 3] = -1
|
|
hooks = buffer.fetch(indices.cuda().contiguous())
|
|
for layer_idx, hook in enumerate(hooks):
|
|
rows, sf = hook()
|
|
idx = indices[layer_idx].flatten().long()
|
|
invalid = idx < 0
|
|
ref_rows = full[layer_idx][0].view(torch.uint8)[idx.clamp_min(0)]
|
|
ref_rows[invalid] = 0
|
|
ref_sf = full[layer_idx][1][idx.clamp_min(0)]
|
|
ref_sf[invalid] = 0
|
|
assert rows.shape == (_NUM_TOKENS, _ENTRIES_PER_TOKEN * _HIDDEN) and rows.dtype == torch.float8_e4m3fn
|
|
assert torch.equal(rows.view(torch.uint8).cpu(), ref_rows.view(_NUM_TOKENS, -1))
|
|
aligned = (_NUM_TOKENS + 3) // 4 * 4
|
|
assert sf.shape == (_NUM_TOKENS, _ENTRIES_PER_TOKEN * _SF_PACKS) and sf.stride() == (1, aligned)
|
|
assert torch.equal(sf.cpu(), ref_sf.view(_NUM_TOKENS, -1))
|
|
torch.cuda.synchronize()
|
|
assert torch.cuda.memory_allocated() - allocated_before < 16 * _MIB
|
|
assert _named_shm_mappings() == []
|
|
buffer.destroy()
|
|
dist.barrier()
|
|
dist.destroy_process_group()
|
|
except BaseException as error:
|
|
failures.put(f"rank {rank}: {error!r}")
|
|
raise
|
|
|
|
|
|
def test_host_engram_fetch_matches_tables() -> None:
|
|
from deep_cluster.hardware import is_sm_major_12
|
|
|
|
if not torch.cuda.is_available() or torch.cuda.device_count() < 2 or not is_sm_major_12():
|
|
pytest.skip("needs two SM120 GPUs")
|
|
context = mp.get_context("spawn")
|
|
failures = context.Queue()
|
|
port = get_open_port()
|
|
processes = [context.Process(target=_worker, args=(rank, 2, port, failures)) for rank in range(2)]
|
|
for process in processes:
|
|
process.start()
|
|
try:
|
|
deadline = time.monotonic() + _TIMEOUT_S
|
|
for process in processes:
|
|
process.join(timeout=max(0.0, deadline - time.monotonic()))
|
|
assert not any(process.is_alive() for process in processes), f"timed out after {_TIMEOUT_S}s"
|
|
errors = [failures.get() for _ in range(failures.qsize())] if not failures.empty() else []
|
|
assert all(process.exitcode == 0 for process in processes), errors
|
|
finally:
|
|
for process in processes:
|
|
if process.is_alive():
|
|
process.kill()
|
|
process.join()
|
|
|
|
|
|
def _cpu_worker(rank: int, world_size: int, port: int, results: mp.Queue, hang_in_fallocate: bool) -> None:
|
|
"""Build a HostEngramBuffer over gloo: the table file never has a name and all ranks map the same file."""
|
|
try:
|
|
dist.init_process_group("gloo", init_method=f"tcp://127.0.0.1:{port}", rank=rank, world_size=world_size)
|
|
from deepinfer.model_executor.parallel_utils import host_engram
|
|
|
|
fallocate = os.posix_fallocate
|
|
|
|
def checked_fallocate(fd: int, offset: int, length: int) -> None:
|
|
results.put(("fallocate", rank, os.readlink(f"/proc/self/fd/{fd}")))
|
|
if hang_in_fallocate:
|
|
time.sleep(3600)
|
|
fallocate(fd, offset, length)
|
|
|
|
host_engram.os = SimpleNamespace(**{name: getattr(os, name) for name in dir(os) if not name.startswith("__")})
|
|
host_engram.os.posix_fallocate = checked_fallocate
|
|
buffer = host_engram.HostEngramBuffer(dist.group.WORLD, [1000, 300], 64, 16, 4, torch.float8_e4m3fn, 2)
|
|
results.put(("mapped", rank, _named_shm_mappings()))
|
|
rows = buffer._host_rows(1).view(torch.uint8)
|
|
rows[rank * 300 : (rank + 1) * 300].fill_(rank + 1)
|
|
dist.barrier()
|
|
results.put(("rows", rank, sorted(set(rows.flatten().tolist()))))
|
|
buffer.destroy()
|
|
dist.barrier()
|
|
dist.destroy_process_group()
|
|
results.put(("done", rank, None))
|
|
except BaseException as error:
|
|
results.put(("error", rank, repr(error)))
|
|
raise
|
|
|
|
|
|
def _run_cpu_group(hang_in_fallocate: bool) -> tuple[list[mp.Process], mp.Queue]:
|
|
context = mp.get_context("spawn")
|
|
results = context.Queue()
|
|
port = get_open_port()
|
|
processes = [context.Process(target=_cpu_worker, args=(rank, 2, port, results, hang_in_fallocate)) for rank in range(2)]
|
|
for process in processes:
|
|
process.start()
|
|
return processes, results
|
|
|
|
|
|
def _collect(results: mp.Queue, kinds: set[str], count: int) -> list[tuple]:
|
|
collected = []
|
|
while len([r for r in collected if r[0] in kinds]) < count:
|
|
item = results.get(timeout=_TIMEOUT_S)
|
|
assert item[0] != "error", item
|
|
collected.append(item)
|
|
return collected
|
|
|
|
|
|
def test_host_engram_table_file_never_has_a_name() -> None:
|
|
processes, results = _run_cpu_group(hang_in_fallocate=False)
|
|
try:
|
|
items = _collect(results, {"done"}, 2)
|
|
for kind, rank, value in items:
|
|
if kind == "fallocate":
|
|
assert value.startswith("/dev/shm/") and value.endswith("(deleted)"), (rank, value)
|
|
elif kind == "mapped":
|
|
assert value == [], (rank, value)
|
|
elif kind == "rows":
|
|
# Each of the two ranks writes its own shard and both see each other's
|
|
assert value == [1, 2], (rank, value)
|
|
assert {kind for kind, _, _ in items} == {"fallocate", "mapped", "rows", "done"}
|
|
for process in processes:
|
|
process.join(timeout=60)
|
|
assert process.exitcode == 0
|
|
finally:
|
|
for process in processes:
|
|
if process.is_alive():
|
|
process.kill()
|
|
process.join()
|
|
|
|
|
|
def test_a_group_killed_while_reserving_leaves_no_table_file() -> None:
|
|
before = set(os.listdir("/dev/shm"))
|
|
processes, results = _run_cpu_group(hang_in_fallocate=True)
|
|
try:
|
|
items = _collect(results, {"fallocate"}, 2)
|
|
assert all(value.endswith("(deleted)") for _, _, value in items), items
|
|
for process in processes:
|
|
os.kill(process.pid, signal.SIGKILL)
|
|
for process in processes:
|
|
process.join(timeout=60)
|
|
assert not any(name.startswith("deepinfer-engram-") for name in set(os.listdir("/dev/shm")) - before)
|
|
finally:
|
|
for process in processes:
|
|
if process.is_alive():
|
|
process.kill()
|
|
process.join()
|