mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-11-20 04:22:27 +00:00
refactor: import deep_ep where it is used
SM120 has no DeepEP feature left (no deepep backend, Engram in host memory, no context parallelism, no NVLink domain), yet five modules imported deep_ep at the top, so every SM120 process needed a DeepEP build, one compiled against an NCCL newer than the one the process loads. - mlp, engram, cp, userbuffer: deep_ep names used only in annotations move under TYPE_CHECKING; EPBuffer, EngramBuffer, BucketBuffer and the CP alignment helpers are imported in the functions that build them. - parallel_state: is_within_nvl_domain imports deep_ep after the SM120 and single-rank returns. No behavior change where DeepEP is installed: the same code imports it, later. tests/test_sm120_without_deep_ep.py imports each module in a fresh interpreter that cannot import deep_ep, and runs the SM120 paths (nccl EP buffer, no CP buffer, no NVLink domain) with deep_ep blocked; the nccl backend allocation test blocks deep_ep too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a2d45e4f22
commit
f009131253
7 files changed
+128
-9
No files matched your search
@@ -5,13 +5,15 @@
|
||||
the fused KV projection and gate the result into the hidden states.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Sequence
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import tile_kernels
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from deep_ep import EngramBuffer
|
||||
from torch import nn
|
||||
|
||||
from deepinfer import PLATFORM, AttrDict, envs
|
||||
@@ -24,6 +26,10 @@ from deepinfer.model_executor.parallel_utils.host_engram import HostEngramBuffer
|
||||
from deepinfer.model_executor.parallel_utils.parallel_state import PG
|
||||
from deepinfer.model_executor.parallel_utils.userbuffer import userbuffer
|
||||
|
||||
# deep_ep is imported where it is used: SM120 builds do not install DeepEP (see README.sm120.md)
|
||||
if TYPE_CHECKING:
|
||||
from deep_ep import EngramBuffer
|
||||
|
||||
|
||||
def _pack_ue8m0_to_int(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Pack the ue8m0 scales represented in fp32 into int32 (CUDA) / int16 (ASC), 4/2 per group."""
|
||||
|
||||
@@ -9,13 +9,12 @@ HF-shaped ones.
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import deep_gemm
|
||||
import tile_kernels
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from deep_ep import EPBuffer, EPHandle, EventHandle, EventOverlap
|
||||
from tile_kernels.moe import moe_topk_gate
|
||||
from tile_kernels.quant import swiglu_forward_and_per_token_cast
|
||||
from torch import nn
|
||||
@@ -38,6 +37,10 @@ from deepinfer.model_executor.parallel_utils.userbuffer import limit_num_sms, us
|
||||
from deepinfer.model_executor.utils import reconstruct_param
|
||||
from deepinfer.timer import cuda_graph_timer
|
||||
|
||||
# deep_ep is imported where it is used: SM120 builds do not install DeepEP (see README.sm120.md)
|
||||
if TYPE_CHECKING:
|
||||
from deep_ep import EPBuffer, EPHandle, EventHandle, EventOverlap
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Lower bound of num_max_tokens_per_rank for the consumer layers' no-sync dispatch. DeepEP JIT-instantiates the
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Generator
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from deep_ep import BucketBuffer, BucketSession
|
||||
|
||||
# deep_ep is imported where it is used: SM120 builds do not install DeepEP (see README.sm120.md)
|
||||
if TYPE_CHECKING:
|
||||
from deep_ep import BucketSession
|
||||
|
||||
|
||||
class CPManager:
|
||||
@@ -14,6 +20,8 @@ class CPManager:
|
||||
"""
|
||||
|
||||
def __init__(self, group: dist.ProcessGroup, num_bytes: int) -> None:
|
||||
from deep_ep import BucketBuffer
|
||||
|
||||
self._buffer = BucketBuffer(group, num_bytes, explicitly_destroy=True)
|
||||
self._rank = group.rank()
|
||||
self._size = group.size()
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import functools
|
||||
from typing import Any
|
||||
|
||||
import deep_ep
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from einops import rearrange
|
||||
@@ -134,7 +133,11 @@ class ParallelProcessGroup:
|
||||
if PLATFORM.is_sm120:
|
||||
# PCIe hosts have no NVLink domain; the LSA team NCCL reports may still span the host
|
||||
return group.size() == 1
|
||||
return group.size() == 1 or group.size() <= deep_ep.get_physical_domain_size(group)[1]
|
||||
if group.size() == 1:
|
||||
return True
|
||||
import deep_ep # not installed on SM120, which returned above
|
||||
|
||||
return group.size() <= deep_ep.get_physical_domain_size(group)[1]
|
||||
|
||||
# --- Accessor methods ---
|
||||
|
||||
|
||||
@@ -4,12 +4,10 @@ from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING, NamedTuple
|
||||
|
||||
import deep_ep
|
||||
import deep_gemm
|
||||
import tile_kernels
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from deep_ep import EngramBuffer, EPBuffer
|
||||
|
||||
from deepinfer import PLATFORM, envs, sm120_support
|
||||
from deepinfer.logger import init_logger
|
||||
@@ -17,7 +15,10 @@ from deepinfer.model_executor.parallel_utils.cp import CPManager
|
||||
from deepinfer.model_executor.parallel_utils.host_engram import HostEngramBuffer
|
||||
from deepinfer.model_executor.parallel_utils.nccl_ep import NcclEPBuffer
|
||||
|
||||
# deep_ep is imported where it is used: SM120 builds do not install DeepEP (see README.sm120.md)
|
||||
if TYPE_CHECKING:
|
||||
from deep_ep import EngramBuffer, EPBuffer
|
||||
|
||||
from deepinfer.model_executor.layers.cake_moe import CakeMoE
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -129,6 +130,8 @@ class _UserBufferManager:
|
||||
logger.info(f"Created NCCL EP buffer, (EP {ep_group.rank()}/{ep_group.size()}), {ep_kwargs=}")
|
||||
assert isinstance(self.ep, NcclEPBuffer) and self.ep.group is ep_group
|
||||
return self.ep
|
||||
from deep_ep import EPBuffer
|
||||
|
||||
if self.ep is not None:
|
||||
assert self.ep.num_bytes >= EPBuffer.get_buffer_size_hint(**ep_kwargs)
|
||||
return self.ep
|
||||
@@ -158,6 +161,8 @@ class _UserBufferManager:
|
||||
if cp_group.size() <= 1:
|
||||
assert self.cp is None, "cp buffer was already allocated for a multi-rank CP group"
|
||||
return None
|
||||
import deep_ep
|
||||
|
||||
max_ag_bytes_per_session = max_num_tokens * max_ag_bytes_per_token * cp_group.size()
|
||||
max_ag_bytes_per_session += max_num_ags_per_session * deep_ep.get_num_rdma_alignment()
|
||||
alignment = deep_ep.get_num_allocation_alignment()
|
||||
@@ -204,6 +209,8 @@ class _UserBufferManager:
|
||||
f"({len(num_entries_per_layer)} layers, Engram {engram_group.rank()}/{engram_group.size()})"
|
||||
)
|
||||
return self.engram
|
||||
from deep_ep import EngramBuffer
|
||||
|
||||
num_gpu_bytes, num_rdma_storage_size = EngramBuffer.get_storage_size_hint(
|
||||
engram_group,
|
||||
num_entries_per_layer,
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""SM120 builds do not install DeepEP: the modules import without deep_ep, and the SM120 paths never import it (CPU)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from deepinfer import environ
|
||||
from deepinfer.model_executor.parallel_utils import parallel_state, userbuffer
|
||||
from deepinfer.model_executor.parallel_utils.nccl_ep import NcclEPBuffer
|
||||
|
||||
_ROOT = Path(__file__).resolve().parents[1]
|
||||
_MODULES = (
|
||||
"deepinfer.model_executor.parallel_utils.parallel_state",
|
||||
"deepinfer.model_executor.parallel_utils.cp",
|
||||
"deepinfer.model_executor.parallel_utils.userbuffer",
|
||||
"deepinfer.model_executor.models.v4_modules.mlp",
|
||||
"deepinfer.model_executor.models.v4_modules.engram",
|
||||
"deepinfer.model_executor.models.deepseek",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_deep_ep(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""`import deep_ep` raises ImportError from here on, as on an SM120 build."""
|
||||
monkeypatch.setitem(sys.modules, "deep_ep", None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("module", _MODULES)
|
||||
def test_module_imports_without_deep_ep(module: str) -> None:
|
||||
"""In a fresh interpreter where deep_ep cannot be imported, the module must not need it at import time."""
|
||||
script = textwrap.dedent(
|
||||
f"""
|
||||
import importlib, sys
|
||||
|
||||
class _NoDeepEP:
|
||||
@staticmethod
|
||||
def find_spec(name, *args):
|
||||
if name == "deep_ep" or name.startswith("deep_ep."):
|
||||
raise ImportError("NO_DEEP_EP")
|
||||
|
||||
sys.meta_path.insert(0, _NoDeepEP())
|
||||
try:
|
||||
importlib.import_module({module!r})
|
||||
except ImportError as error:
|
||||
print("DEEP_EP" if "NO_DEEP_EP" in str(error) else f"OTHER {{error}}")
|
||||
else:
|
||||
print("OK")
|
||||
"""
|
||||
)
|
||||
result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, cwd=_ROOT, env=os.environ.copy())
|
||||
output = result.stdout.strip().splitlines()[-1] if result.stdout.strip() else result.stderr
|
||||
if output.startswith("OTHER"):
|
||||
pytest.skip(f"{module} needs something else this environment lacks: {output}")
|
||||
assert output == "OK", output
|
||||
|
||||
|
||||
def test_sm120_has_no_nvlink_domain_without_deep_ep(monkeypatch: pytest.MonkeyPatch, no_deep_ep: None) -> None:
|
||||
monkeypatch.setattr(parallel_state, "PLATFORM", SimpleNamespace(is_npu=False, is_sm120=True))
|
||||
assert parallel_state.ParallelProcessGroup.is_within_nvl_domain(SimpleNamespace(size=lambda: 8)) is False
|
||||
assert parallel_state.ParallelProcessGroup.is_within_nvl_domain(SimpleNamespace(size=lambda: 1)) is True
|
||||
|
||||
|
||||
def test_sm120_nccl_ep_buffer_without_deep_ep(monkeypatch: pytest.MonkeyPatch, no_deep_ep: None) -> None:
|
||||
monkeypatch.setenv("DEEPINFER_SM120_EP_BACKEND", "nccl")
|
||||
environ.rearm_all()
|
||||
monkeypatch.setattr(userbuffer, "PLATFORM", SimpleNamespace(is_sm120=True, is_gpu=True, default_comm_sms=None))
|
||||
manager = userbuffer._UserBufferManager()
|
||||
group = SimpleNamespace(size=lambda: 1, rank=lambda: 0)
|
||||
buffer = manager.allocate_ep(group, max_num_tokens=64, hidden_size=256, num_topk=4, num_experts=16, use_fp8_dispatch=True)
|
||||
assert isinstance(buffer, NcclEPBuffer)
|
||||
# SM120 refuses CP, so the CP buffer is never built
|
||||
assert manager.allocate_cp(group, max_num_tokens=64, max_ag_bytes_per_token=512) is None
|
||||
manager.close()
|
||||
environ.rearm_all()
|
||||
|
||||
|
||||
def test_deep_ep_paths_still_import_it(no_deep_ep: None) -> None:
|
||||
"""Off SM120 the lazy imports still reach deep_ep (here missing, so they fail loudly rather than silently)."""
|
||||
manager = userbuffer._UserBufferManager()
|
||||
with pytest.raises(ImportError):
|
||||
manager.allocate_cp(SimpleNamespace(size=lambda: 2, rank=lambda: 0), max_num_tokens=64, max_ag_bytes_per_token=512)
|
||||
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
from collections.abc import Callable, Generator
|
||||
from contextlib import contextmanager
|
||||
@@ -565,7 +566,7 @@ def _allocate(manager: _UserBufferManager, group: _Group, num_experts: int = 16,
|
||||
|
||||
def test_allocate_ep_sm120_nccl_backend(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
manager = _sm120_manager(monkeypatch, "nccl")
|
||||
monkeypatch.setattr(userbuffer_module, "EPBuffer", None)
|
||||
monkeypatch.setitem(sys.modules, "deep_ep", None) # the nccl backend never needs DeepEP
|
||||
group = _Group()
|
||||
buffer = _allocate(manager, group)
|
||||
assert isinstance(buffer, NcclEPBuffer)
|
||||
|
||||
Reference in new issue
Block a user