mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-11-18 19:42:28 +00:00
Run DeepSeek-V4.1-Flash offline inference on SM120 GPUs (RTX PRO 6000 Blackwell) with the same entry points as SM100. SM100 and Ascend paths are unchanged. - Build: SM120 is detected from the first visible GPU (scripts/nvidia_arch.sh). It builds DeepEP and DeepGEMM from its nv_dev branch (vendor/DeepGEMM-sm120), and no FlashMLA or DeepSelect; FlashInfer is installed from a pinned commit (requirements-nvidia-sm120.txt). - Attention: V4.1 sparse MLA runs on FlashInfer's SM120 sparse MLA kernels, reading the inline-scale KV records as stored; KV blocks are aligned to 512 bytes. - Indexer: FlashInfer's fused DSA indexer scores the index keys and selects their top-k in one call, for decode on every indexer layer (candidate source and consumer layers included) and for prefill outside the candidate layers. The remaining selections (prefill of the candidate layers, the sampler) use FlashInfer's top-k behind DeepSelect's interface (model_executor/layers/topk.py), in deterministic mode. - MoE EP: DeepEP over NCCL GIN by default, or NCCL collectives (DEEPINFER_SM120_EP_BACKEND=nccl); NCCL settings known to break the EP paths are rejected. EPLB, including decode-only draft-layer EPLB, and per-layer YoCo MoE dispatch. - Engram tables live in page-locked host shared memory; killed workers leave nothing in /dev/shm. - Features: attention DP + MoE EP, DSpark/nextn speculative decoding, CUDA graphs, async scheduling, decoder SWA bounded replay, structured outputs, image inputs, multi-node deployments. Features without an SM120 implementation (CP > 1, MTP varlen, batch-invariant mode, GPU-resident Engram tables) are refused at engine start. - Eval: chat GSM8K and CV-Bench; tests for the SM120 kernels and their DeepInfer call sites. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
87 lines
3.5 KiB
Python
87 lines
3.5 KiB
Python
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from deepinfer import PLATFORM, environ
|
|
from deepinfer.model_executor.models.v4_modules import mlp
|
|
from deepinfer.model_executor.models.v4_modules.mlp import DeepseekMoE
|
|
|
|
_MAX_PREFILL = 8192
|
|
_MAX_DECODE = 384
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _rearm_env_after_test():
|
|
yield
|
|
environ.rearm_all()
|
|
|
|
|
|
@pytest.fixture
|
|
def ep8(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
group = SimpleNamespace(size=lambda: 8)
|
|
monkeypatch.setattr(mlp.PG, "cpu_expert_parallel_group", lambda: SimpleNamespace(size=lambda: 1), raising=False)
|
|
monkeypatch.setattr(mlp.PG, "expert_parallel_group", lambda: group, raising=False)
|
|
|
|
|
|
def _set_env(monkeypatch: pytest.MonkeyPatch, mode: str, policy: str, sm120: bool) -> None:
|
|
monkeypatch.setenv("DEEPINFER_YOCO_PREFILL_TRUNCATION_MODE", mode)
|
|
monkeypatch.setenv("DEEPINFER_YOCO_PREFILL_MOE_DISPATCH", policy)
|
|
monkeypatch.setattr(type(PLATFORM), "is_sm120", property(lambda self: sm120), raising=False)
|
|
environ.rearm_all()
|
|
|
|
|
|
def _decide(is_prompt: bool, bound: int | None = None, num_tokens: int = 100) -> tuple[int, bool]:
|
|
moe = SimpleNamespace(max_num_prefilling_tokens=_MAX_PREFILL, max_num_decoding_tokens=_MAX_DECODE)
|
|
metadata = SimpleNamespace(is_prompt=is_prompt, yoco_consumer_moe_max_tokens_per_rank=bound)
|
|
return DeepseekMoE._dispatch_size_and_cpu_sync(moe, metadata, num_tokens)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("rows", "expected"),
|
|
[(1, 128), (128, 128), (129, 256), (280, 512), (1000, 1024), (1024, 1024), (1025, None), (8192, None)],
|
|
)
|
|
def test_consumer_bound_buckets(ep8: None, rows: int, expected: int | None) -> None:
|
|
assert mlp.yoco_consumer_moe_max_tokens_per_rank(rows, _MAX_PREFILL) == expected
|
|
|
|
|
|
def test_consumer_bound_caps_non_power_of_two(ep8: None) -> None:
|
|
assert mlp.yoco_consumer_moe_max_tokens_per_rank(700, 6000) == 750
|
|
|
|
|
|
@pytest.mark.parametrize("mode", ["off", "mask"])
|
|
@pytest.mark.parametrize("sm120", [False, True])
|
|
def test_non_compact_unchanged(monkeypatch: pytest.MonkeyPatch, mode: str, sm120: bool) -> None:
|
|
_set_env(monkeypatch, mode, "auto", sm120)
|
|
assert not mlp.yoco_per_layer_moe_dispatch_enabled()
|
|
assert _decide(True) == (_MAX_PREFILL, True)
|
|
assert _decide(False) == (_MAX_DECODE, False)
|
|
|
|
|
|
def test_compact_auto_off_sm120_keeps_nosync(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_set_env(monkeypatch, "compact", "auto", sm120=False)
|
|
assert not mlp.yoco_per_layer_moe_dispatch_enabled()
|
|
assert _decide(True) == (_MAX_PREFILL, False)
|
|
assert _decide(False) == (_MAX_DECODE, False)
|
|
|
|
|
|
def test_compact_auto_sm120_per_layer(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_set_env(monkeypatch, "compact", "auto", sm120=True)
|
|
assert mlp.yoco_per_layer_moe_dispatch_enabled()
|
|
assert _decide(True, bound=None) == (_MAX_PREFILL, True)
|
|
assert _decide(True, bound=512, num_tokens=280) == (512, False)
|
|
assert _decide(False) == (_MAX_DECODE, False)
|
|
with pytest.raises(AssertionError):
|
|
_decide(True, bound=256, num_tokens=280)
|
|
|
|
|
|
@pytest.mark.parametrize(("policy", "sync"), [("sync", True), ("nosync", False)])
|
|
def test_compact_explicit_policy(monkeypatch: pytest.MonkeyPatch, policy: str, sync: bool) -> None:
|
|
_set_env(monkeypatch, "compact", policy, sm120=True)
|
|
assert not mlp.yoco_per_layer_moe_dispatch_enabled()
|
|
assert _decide(True, bound=None) == (_MAX_PREFILL, sync)
|