refactor: import deep_ep where it is used

SM120 has no DeepEP feature left (no deepep backend, Engram in host memory,
no context parallelism, no NVLink domain), yet five modules imported deep_ep
at the top, so every SM120 process needed a DeepEP build, one compiled
against an NCCL newer than the one the process loads.

- mlp, engram, cp, userbuffer: deep_ep names used only in annotations move
  under TYPE_CHECKING; EPBuffer, EngramBuffer, BucketBuffer and the CP
  alignment helpers are imported in the functions that build them.
- parallel_state: is_within_nvl_domain imports deep_ep after the SM120 and
  single-rank returns.

No behavior change where DeepEP is installed: the same code imports it,
later. tests/test_sm120_without_deep_ep.py imports each module in a fresh
interpreter that cannot import deep_ep, and runs the SM120 paths (nccl EP
buffer, no CP buffer, no NVLink domain) with deep_ep blocked; the nccl
backend allocation test blocks deep_ep too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
shunyadandClaude Opus 5.5 committed 2026-11-18 23:43:19 -08:00
1 parent a2d45e4f22
commit f009131253
7 files changed
+128 -9

No files matched your search

@@ -5,13 +5,15 @@
the fused KV projection and gate the result into the hidden states.
"""
from __future__ import annotations
from collections.abc import Callable, Sequence
from typing import TYPE_CHECKING
import numpy as np
import tile_kernels
import torch
import torch.distributed as dist
from deep_ep import EngramBuffer
from torch import nn
from deepinfer import PLATFORM, AttrDict, envs
@@ -24,6 +26,10 @@ from deepinfer.model_executor.parallel_utils.host_engram import HostEngramBuffer
from deepinfer.model_executor.parallel_utils.parallel_state import PG
from deepinfer.model_executor.parallel_utils.userbuffer import userbuffer
# deep_ep is imported where it is used: SM120 builds do not install DeepEP (see README.sm120.md)
if TYPE_CHECKING:
from deep_ep import EngramBuffer
def _pack_ue8m0_to_int(x: torch.Tensor) -> torch.Tensor:
"""Pack the ue8m0 scales represented in fp32 into int32 (CUDA) / int16 (ASC), 4/2 per group."""
@@ -9,13 +9,12 @@ HF-shaped ones.
from __future__ import annotations
import math
from typing import Any
from typing import TYPE_CHECKING, Any
import deep_gemm
import tile_kernels
import torch
import torch.distributed as dist
from deep_ep import EPBuffer, EPHandle, EventHandle, EventOverlap
from tile_kernels.moe import moe_topk_gate
from tile_kernels.quant import swiglu_forward_and_per_token_cast
from torch import nn
@@ -38,6 +37,10 @@ from deepinfer.model_executor.parallel_utils.userbuffer import limit_num_sms, us
from deepinfer.model_executor.utils import reconstruct_param
from deepinfer.timer import cuda_graph_timer
# deep_ep is imported where it is used: SM120 builds do not install DeepEP (see README.sm120.md)
if TYPE_CHECKING:
from deep_ep import EPBuffer, EPHandle, EventHandle, EventOverlap
logger = init_logger(__name__)
# Lower bound of num_max_tokens_per_rank for the consumer layers' no-sync dispatch. DeepEP JIT-instantiates the
@@ -1,9 +1,15 @@
from __future__ import annotations
from collections.abc import Callable, Generator
from contextlib import contextmanager
from typing import TYPE_CHECKING
import torch
import torch.distributed as dist
from deep_ep import BucketBuffer, BucketSession
# deep_ep is imported where it is used: SM120 builds do not install DeepEP (see README.sm120.md)
if TYPE_CHECKING:
from deep_ep import BucketSession
class CPManager:
@@ -14,6 +20,8 @@ class CPManager:
"""
def __init__(self, group: dist.ProcessGroup, num_bytes: int) -> None:
from deep_ep import BucketBuffer
self._buffer = BucketBuffer(group, num_bytes, explicitly_destroy=True)
self._rank = group.rank()
self._size = group.size()
@@ -1,7 +1,6 @@
import functools
from typing import Any
import deep_ep
import torch
import torch.distributed as dist
from einops import rearrange
@@ -134,7 +133,11 @@ class ParallelProcessGroup:
if PLATFORM.is_sm120:
# PCIe hosts have no NVLink domain; the LSA team NCCL reports may still span the host
return group.size() == 1
return group.size() == 1 or group.size() <= deep_ep.get_physical_domain_size(group)[1]
if group.size() == 1:
return True
import deep_ep # not installed on SM120, which returned above
return group.size() <= deep_ep.get_physical_domain_size(group)[1]
# --- Accessor methods ---
@@ -4,12 +4,10 @@ from collections.abc import Generator
from contextlib import contextmanager
from typing import TYPE_CHECKING, NamedTuple
import deep_ep
import deep_gemm
import tile_kernels
import torch
import torch.distributed as dist
from deep_ep import EngramBuffer, EPBuffer
from deepinfer import PLATFORM, envs, sm120_support
from deepinfer.logger import init_logger
@@ -17,7 +15,10 @@ from deepinfer.model_executor.parallel_utils.cp import CPManager
from deepinfer.model_executor.parallel_utils.host_engram import HostEngramBuffer
from deepinfer.model_executor.parallel_utils.nccl_ep import NcclEPBuffer
# deep_ep is imported where it is used: SM120 builds do not install DeepEP (see README.sm120.md)
if TYPE_CHECKING:
from deep_ep import EngramBuffer, EPBuffer
from deepinfer.model_executor.layers.cake_moe import CakeMoE
logger = init_logger(__name__)
@@ -129,6 +130,8 @@ class _UserBufferManager:
logger.info(f"Created NCCL EP buffer, (EP {ep_group.rank()}/{ep_group.size()}), {ep_kwargs=}")
assert isinstance(self.ep, NcclEPBuffer) and self.ep.group is ep_group
return self.ep
from deep_ep import EPBuffer
if self.ep is not None:
assert self.ep.num_bytes >= EPBuffer.get_buffer_size_hint(**ep_kwargs)
return self.ep
@@ -158,6 +161,8 @@ class _UserBufferManager:
if cp_group.size() <= 1:
assert self.cp is None, "cp buffer was already allocated for a multi-rank CP group"
return None
import deep_ep
max_ag_bytes_per_session = max_num_tokens * max_ag_bytes_per_token * cp_group.size()
max_ag_bytes_per_session += max_num_ags_per_session * deep_ep.get_num_rdma_alignment()
alignment = deep_ep.get_num_allocation_alignment()
@@ -204,6 +209,8 @@ class _UserBufferManager:
f"({len(num_entries_per_layer)} layers, Engram {engram_group.rank()}/{engram_group.size()})"
)
return self.engram
from deep_ep import EngramBuffer
num_gpu_bytes, num_rdma_storage_size = EngramBuffer.get_storage_size_hint(
engram_group,
num_entries_per_layer,
+91
View File
@@ -0,0 +1,91 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""SM120 builds do not install DeepEP: the modules import without deep_ep, and the SM120 paths never import it (CPU)."""
from __future__ import annotations
import os
import subprocess
import sys
import textwrap
from pathlib import Path
from types import SimpleNamespace
import pytest
from deepinfer import environ
from deepinfer.model_executor.parallel_utils import parallel_state, userbuffer
from deepinfer.model_executor.parallel_utils.nccl_ep import NcclEPBuffer
_ROOT = Path(__file__).resolve().parents[1]
_MODULES = (
"deepinfer.model_executor.parallel_utils.parallel_state",
"deepinfer.model_executor.parallel_utils.cp",
"deepinfer.model_executor.parallel_utils.userbuffer",
"deepinfer.model_executor.models.v4_modules.mlp",
"deepinfer.model_executor.models.v4_modules.engram",
"deepinfer.model_executor.models.deepseek",
)
@pytest.fixture
def no_deep_ep(monkeypatch: pytest.MonkeyPatch) -> None:
"""`import deep_ep` raises ImportError from here on, as on an SM120 build."""
monkeypatch.setitem(sys.modules, "deep_ep", None)
@pytest.mark.parametrize("module", _MODULES)
def test_module_imports_without_deep_ep(module: str) -> None:
"""In a fresh interpreter where deep_ep cannot be imported, the module must not need it at import time."""
script = textwrap.dedent(
f"""
import importlib, sys
class _NoDeepEP:
@staticmethod
def find_spec(name, *args):
if name == "deep_ep" or name.startswith("deep_ep."):
raise ImportError("NO_DEEP_EP")
sys.meta_path.insert(0, _NoDeepEP())
try:
importlib.import_module({module!r})
except ImportError as error:
print("DEEP_EP" if "NO_DEEP_EP" in str(error) else f"OTHER {{error}}")
else:
print("OK")
"""
)
result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, cwd=_ROOT, env=os.environ.copy())
output = result.stdout.strip().splitlines()[-1] if result.stdout.strip() else result.stderr
if output.startswith("OTHER"):
pytest.skip(f"{module} needs something else this environment lacks: {output}")
assert output == "OK", output
def test_sm120_has_no_nvlink_domain_without_deep_ep(monkeypatch: pytest.MonkeyPatch, no_deep_ep: None) -> None:
monkeypatch.setattr(parallel_state, "PLATFORM", SimpleNamespace(is_npu=False, is_sm120=True))
assert parallel_state.ParallelProcessGroup.is_within_nvl_domain(SimpleNamespace(size=lambda: 8)) is False
assert parallel_state.ParallelProcessGroup.is_within_nvl_domain(SimpleNamespace(size=lambda: 1)) is True
def test_sm120_nccl_ep_buffer_without_deep_ep(monkeypatch: pytest.MonkeyPatch, no_deep_ep: None) -> None:
monkeypatch.setenv("DEEPINFER_SM120_EP_BACKEND", "nccl")
environ.rearm_all()
monkeypatch.setattr(userbuffer, "PLATFORM", SimpleNamespace(is_sm120=True, is_gpu=True, default_comm_sms=None))
manager = userbuffer._UserBufferManager()
group = SimpleNamespace(size=lambda: 1, rank=lambda: 0)
buffer = manager.allocate_ep(group, max_num_tokens=64, hidden_size=256, num_topk=4, num_experts=16, use_fp8_dispatch=True)
assert isinstance(buffer, NcclEPBuffer)
# SM120 refuses CP, so the CP buffer is never built
assert manager.allocate_cp(group, max_num_tokens=64, max_ag_bytes_per_token=512) is None
manager.close()
environ.rearm_all()
def test_deep_ep_paths_still_import_it(no_deep_ep: None) -> None:
"""Off SM120 the lazy imports still reach deep_ep (here missing, so they fail loudly rather than silently)."""
manager = userbuffer._UserBufferManager()
with pytest.raises(ImportError):
manager.allocate_cp(SimpleNamespace(size=lambda: 2, rank=lambda: 0), max_num_tokens=64, max_ag_bytes_per_token=512)
+2 -1
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import math
import os
import sys
import traceback
from collections.abc import Callable, Generator
from contextlib import contextmanager
@@ -565,7 +566,7 @@ def _allocate(manager: _UserBufferManager, group: _Group, num_experts: int = 16,
def test_allocate_ep_sm120_nccl_backend(monkeypatch: pytest.MonkeyPatch) -> None:
manager = _sm120_manager(monkeypatch, "nccl")
monkeypatch.setattr(userbuffer_module, "EPBuffer", None)
monkeypatch.setitem(sys.modules, "deep_ep", None) # the nccl backend never needs DeepEP
group = _Group()
buffer = _allocate(manager, group)
assert isinstance(buffer, NcclEPBuffer)