From f009131253400a54328fa505577008ecadb0ff8d Mon Sep 17 00:00:00 2001 From: shunyad Date: Wed, 18 Nov 2026 23:43:19 -0800 Subject: [PATCH] refactor: import deep_ep where it is used SM120 has no DeepEP feature left (no deepep backend, Engram in host memory, no context parallelism, no NVLink domain), yet five modules imported deep_ep at the top, so every SM120 process needed a DeepEP build, one compiled against an NCCL newer than the one the process loads. - mlp, engram, cp, userbuffer: deep_ep names used only in annotations move under TYPE_CHECKING; EPBuffer, EngramBuffer, BucketBuffer and the CP alignment helpers are imported in the functions that build them. - parallel_state: is_within_nvl_domain imports deep_ep after the SM120 and single-rank returns. No behavior change where DeepEP is installed: the same code imports it, later. tests/test_sm120_without_deep_ep.py imports each module in a fresh interpreter that cannot import deep_ep, and runs the SM120 paths (nccl EP buffer, no CP buffer, no NVLink domain) with deep_ep blocked; the nccl backend allocation test blocks deep_ep too. Co-Authored-By: Claude Opus 5.5 --- .../models/v4_modules/engram.py | 8 +- .../model_executor/models/v4_modules/mlp.py | 7 +- deepinfer/model_executor/parallel_utils/cp.py | 10 +- .../parallel_utils/parallel_state.py | 7 +- .../parallel_utils/userbuffer.py | 11 ++- tests/test_sm120_without_deep_ep.py | 91 +++++++++++++++++++ tests/worker/test_nccl_ep.py | 3 +- 7 files changed, 128 insertions(+), 9 deletions(-) create mode 100644 tests/test_sm120_without_deep_ep.py diff --git a/deepinfer/model_executor/models/v4_modules/engram.py b/deepinfer/model_executor/models/v4_modules/engram.py index 47855ba5..01948555 100644 --- a/deepinfer/model_executor/models/v4_modules/engram.py +++ b/deepinfer/model_executor/models/v4_modules/engram.py @@ -5,13 +5,15 @@ the fused KV projection and gate the result into the hidden states. """ +from __future__ import annotations + from collections.abc import Callable, Sequence +from typing import TYPE_CHECKING import numpy as np import tile_kernels import torch import torch.distributed as dist -from deep_ep import EngramBuffer from torch import nn from deepinfer import PLATFORM, AttrDict, envs @@ -24,6 +26,10 @@ from deepinfer.model_executor.parallel_utils.host_engram import HostEngramBuffer from deepinfer.model_executor.parallel_utils.parallel_state import PG from deepinfer.model_executor.parallel_utils.userbuffer import userbuffer +# deep_ep is imported where it is used: SM120 builds do not install DeepEP (see README.sm120.md) +if TYPE_CHECKING: + from deep_ep import EngramBuffer + def _pack_ue8m0_to_int(x: torch.Tensor) -> torch.Tensor: """Pack the ue8m0 scales represented in fp32 into int32 (CUDA) / int16 (ASC), 4/2 per group.""" diff --git a/deepinfer/model_executor/models/v4_modules/mlp.py b/deepinfer/model_executor/models/v4_modules/mlp.py index db52c08a..da737e56 100644 --- a/deepinfer/model_executor/models/v4_modules/mlp.py +++ b/deepinfer/model_executor/models/v4_modules/mlp.py @@ -9,13 +9,12 @@ HF-shaped ones. from __future__ import annotations import math -from typing import Any +from typing import TYPE_CHECKING, Any import deep_gemm import tile_kernels import torch import torch.distributed as dist -from deep_ep import EPBuffer, EPHandle, EventHandle, EventOverlap from tile_kernels.moe import moe_topk_gate from tile_kernels.quant import swiglu_forward_and_per_token_cast from torch import nn @@ -38,6 +37,10 @@ from deepinfer.model_executor.parallel_utils.userbuffer import limit_num_sms, us from deepinfer.model_executor.utils import reconstruct_param from deepinfer.timer import cuda_graph_timer +# deep_ep is imported where it is used: SM120 builds do not install DeepEP (see README.sm120.md) +if TYPE_CHECKING: + from deep_ep import EPBuffer, EPHandle, EventHandle, EventOverlap + logger = init_logger(__name__) # Lower bound of num_max_tokens_per_rank for the consumer layers' no-sync dispatch. DeepEP JIT-instantiates the diff --git a/deepinfer/model_executor/parallel_utils/cp.py b/deepinfer/model_executor/parallel_utils/cp.py index 0e13d9eb..55614a2e 100644 --- a/deepinfer/model_executor/parallel_utils/cp.py +++ b/deepinfer/model_executor/parallel_utils/cp.py @@ -1,9 +1,15 @@ +from __future__ import annotations + from collections.abc import Callable, Generator from contextlib import contextmanager +from typing import TYPE_CHECKING import torch import torch.distributed as dist -from deep_ep import BucketBuffer, BucketSession + +# deep_ep is imported where it is used: SM120 builds do not install DeepEP (see README.sm120.md) +if TYPE_CHECKING: + from deep_ep import BucketSession class CPManager: @@ -14,6 +20,8 @@ class CPManager: """ def __init__(self, group: dist.ProcessGroup, num_bytes: int) -> None: + from deep_ep import BucketBuffer + self._buffer = BucketBuffer(group, num_bytes, explicitly_destroy=True) self._rank = group.rank() self._size = group.size() diff --git a/deepinfer/model_executor/parallel_utils/parallel_state.py b/deepinfer/model_executor/parallel_utils/parallel_state.py index c48d2bc1..5956cdce 100644 --- a/deepinfer/model_executor/parallel_utils/parallel_state.py +++ b/deepinfer/model_executor/parallel_utils/parallel_state.py @@ -1,7 +1,6 @@ import functools from typing import Any -import deep_ep import torch import torch.distributed as dist from einops import rearrange @@ -134,7 +133,11 @@ class ParallelProcessGroup: if PLATFORM.is_sm120: # PCIe hosts have no NVLink domain; the LSA team NCCL reports may still span the host return group.size() == 1 - return group.size() == 1 or group.size() <= deep_ep.get_physical_domain_size(group)[1] + if group.size() == 1: + return True + import deep_ep # not installed on SM120, which returned above + + return group.size() <= deep_ep.get_physical_domain_size(group)[1] # --- Accessor methods --- diff --git a/deepinfer/model_executor/parallel_utils/userbuffer.py b/deepinfer/model_executor/parallel_utils/userbuffer.py index cd9cc363..ec9b351e 100644 --- a/deepinfer/model_executor/parallel_utils/userbuffer.py +++ b/deepinfer/model_executor/parallel_utils/userbuffer.py @@ -4,12 +4,10 @@ from collections.abc import Generator from contextlib import contextmanager from typing import TYPE_CHECKING, NamedTuple -import deep_ep import deep_gemm import tile_kernels import torch import torch.distributed as dist -from deep_ep import EngramBuffer, EPBuffer from deepinfer import PLATFORM, envs, sm120_support from deepinfer.logger import init_logger @@ -17,7 +15,10 @@ from deepinfer.model_executor.parallel_utils.cp import CPManager from deepinfer.model_executor.parallel_utils.host_engram import HostEngramBuffer from deepinfer.model_executor.parallel_utils.nccl_ep import NcclEPBuffer +# deep_ep is imported where it is used: SM120 builds do not install DeepEP (see README.sm120.md) if TYPE_CHECKING: + from deep_ep import EngramBuffer, EPBuffer + from deepinfer.model_executor.layers.cake_moe import CakeMoE logger = init_logger(__name__) @@ -129,6 +130,8 @@ class _UserBufferManager: logger.info(f"Created NCCL EP buffer, (EP {ep_group.rank()}/{ep_group.size()}), {ep_kwargs=}") assert isinstance(self.ep, NcclEPBuffer) and self.ep.group is ep_group return self.ep + from deep_ep import EPBuffer + if self.ep is not None: assert self.ep.num_bytes >= EPBuffer.get_buffer_size_hint(**ep_kwargs) return self.ep @@ -158,6 +161,8 @@ class _UserBufferManager: if cp_group.size() <= 1: assert self.cp is None, "cp buffer was already allocated for a multi-rank CP group" return None + import deep_ep + max_ag_bytes_per_session = max_num_tokens * max_ag_bytes_per_token * cp_group.size() max_ag_bytes_per_session += max_num_ags_per_session * deep_ep.get_num_rdma_alignment() alignment = deep_ep.get_num_allocation_alignment() @@ -204,6 +209,8 @@ class _UserBufferManager: f"({len(num_entries_per_layer)} layers, Engram {engram_group.rank()}/{engram_group.size()})" ) return self.engram + from deep_ep import EngramBuffer + num_gpu_bytes, num_rdma_storage_size = EngramBuffer.get_storage_size_hint( engram_group, num_entries_per_layer, diff --git a/tests/test_sm120_without_deep_ep.py b/tests/test_sm120_without_deep_ep.py new file mode 100644 index 00000000..ed2ed9e7 --- /dev/null +++ b/tests/test_sm120_without_deep_ep.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SM120 builds do not install DeepEP: the modules import without deep_ep, and the SM120 paths never import it (CPU).""" + +from __future__ import annotations + +import os +import subprocess +import sys +import textwrap +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from deepinfer import environ +from deepinfer.model_executor.parallel_utils import parallel_state, userbuffer +from deepinfer.model_executor.parallel_utils.nccl_ep import NcclEPBuffer + +_ROOT = Path(__file__).resolve().parents[1] +_MODULES = ( + "deepinfer.model_executor.parallel_utils.parallel_state", + "deepinfer.model_executor.parallel_utils.cp", + "deepinfer.model_executor.parallel_utils.userbuffer", + "deepinfer.model_executor.models.v4_modules.mlp", + "deepinfer.model_executor.models.v4_modules.engram", + "deepinfer.model_executor.models.deepseek", +) + + +@pytest.fixture +def no_deep_ep(monkeypatch: pytest.MonkeyPatch) -> None: + """`import deep_ep` raises ImportError from here on, as on an SM120 build.""" + monkeypatch.setitem(sys.modules, "deep_ep", None) + + +@pytest.mark.parametrize("module", _MODULES) +def test_module_imports_without_deep_ep(module: str) -> None: + """In a fresh interpreter where deep_ep cannot be imported, the module must not need it at import time.""" + script = textwrap.dedent( + f""" + import importlib, sys + + class _NoDeepEP: + @staticmethod + def find_spec(name, *args): + if name == "deep_ep" or name.startswith("deep_ep."): + raise ImportError("NO_DEEP_EP") + + sys.meta_path.insert(0, _NoDeepEP()) + try: + importlib.import_module({module!r}) + except ImportError as error: + print("DEEP_EP" if "NO_DEEP_EP" in str(error) else f"OTHER {{error}}") + else: + print("OK") + """ + ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, cwd=_ROOT, env=os.environ.copy()) + output = result.stdout.strip().splitlines()[-1] if result.stdout.strip() else result.stderr + if output.startswith("OTHER"): + pytest.skip(f"{module} needs something else this environment lacks: {output}") + assert output == "OK", output + + +def test_sm120_has_no_nvlink_domain_without_deep_ep(monkeypatch: pytest.MonkeyPatch, no_deep_ep: None) -> None: + monkeypatch.setattr(parallel_state, "PLATFORM", SimpleNamespace(is_npu=False, is_sm120=True)) + assert parallel_state.ParallelProcessGroup.is_within_nvl_domain(SimpleNamespace(size=lambda: 8)) is False + assert parallel_state.ParallelProcessGroup.is_within_nvl_domain(SimpleNamespace(size=lambda: 1)) is True + + +def test_sm120_nccl_ep_buffer_without_deep_ep(monkeypatch: pytest.MonkeyPatch, no_deep_ep: None) -> None: + monkeypatch.setenv("DEEPINFER_SM120_EP_BACKEND", "nccl") + environ.rearm_all() + monkeypatch.setattr(userbuffer, "PLATFORM", SimpleNamespace(is_sm120=True, is_gpu=True, default_comm_sms=None)) + manager = userbuffer._UserBufferManager() + group = SimpleNamespace(size=lambda: 1, rank=lambda: 0) + buffer = manager.allocate_ep(group, max_num_tokens=64, hidden_size=256, num_topk=4, num_experts=16, use_fp8_dispatch=True) + assert isinstance(buffer, NcclEPBuffer) + # SM120 refuses CP, so the CP buffer is never built + assert manager.allocate_cp(group, max_num_tokens=64, max_ag_bytes_per_token=512) is None + manager.close() + environ.rearm_all() + + +def test_deep_ep_paths_still_import_it(no_deep_ep: None) -> None: + """Off SM120 the lazy imports still reach deep_ep (here missing, so they fail loudly rather than silently).""" + manager = userbuffer._UserBufferManager() + with pytest.raises(ImportError): + manager.allocate_cp(SimpleNamespace(size=lambda: 2, rank=lambda: 0), max_num_tokens=64, max_ag_bytes_per_token=512) diff --git a/tests/worker/test_nccl_ep.py b/tests/worker/test_nccl_ep.py index bc3f4c81..1ebd0d80 100644 --- a/tests/worker/test_nccl_ep.py +++ b/tests/worker/test_nccl_ep.py @@ -5,6 +5,7 @@ from __future__ import annotations import math import os +import sys import traceback from collections.abc import Callable, Generator from contextlib import contextmanager @@ -565,7 +566,7 @@ def _allocate(manager: _UserBufferManager, group: _Group, num_experts: int = 16, def test_allocate_ep_sm120_nccl_backend(monkeypatch: pytest.MonkeyPatch) -> None: manager = _sm120_manager(monkeypatch, "nccl") - monkeypatch.setattr(userbuffer_module, "EPBuffer", None) + monkeypatch.setitem(sys.modules, "deep_ep", None) # the nccl backend never needs DeepEP group = _Group() buffer = _allocate(manager, group) assert isinstance(buffer, NcclEPBuffer)