mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-11-20 04:22:27 +00:00
Draft-layer schedules went through the main-model validation, so a bad DEEPINFER_DRAFT_EXPERT_COUNT_WINDOW_SIZE was reported as "DEEPINFER_EXPERT_COUNT_WINDOW_SIZE=... must be between 1 and DEEPINFER_EXPERT_REARRANGEMENT_STEP_INTERVAL=...", naming variables the user never set. Draft settings could also be dropped without a word. - ExpertLoadBalancer takes setting_names for its error messages; the draft schedule passes the draft variables (or says the interval is inherited, or the window is the default). - Warn when DEEPINFER_DRAFT_EXPERT_COUNT_WINDOW_SIZE is set but no draft interval resolves, and when draft layers share an EPLB layout group with main-model layers and so cannot follow the draft schedule. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
113 lines
4.1 KiB
Python
113 lines
4.1 KiB
Python
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from deepinfer import environ
|
|
from deepinfer.model_executor import eplb
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _env(monkeypatch: pytest.MonkeyPatch):
|
|
monkeypatch.setenv("DEEPINFER_EXPERT_REARRANGEMENT_STEP_INTERVAL", "1000")
|
|
monkeypatch.setenv("DEEPINFER_EXPERT_COUNT_WINDOW_SIZE", "3")
|
|
monkeypatch.setenv("DEEPINFER_REARRANGE_AFTER_PREFILL_IN_NORMAL_ENGINE", "1")
|
|
environ.rearm_all()
|
|
yield
|
|
monkeypatch.undo()
|
|
environ.rearm_all()
|
|
|
|
|
|
def _balancer(monkeypatch: pytest.MonkeyPatch, **kwargs: Any) -> tuple[eplb.ExpertLoadBalancer, list[int]]:
|
|
balancer = eplb.ExpertLoadBalancer(
|
|
expert_weights=[],
|
|
num_moe_layers=2,
|
|
num_logical_experts=4,
|
|
num_physical_experts=6,
|
|
ep_group=SimpleNamespace(size=lambda: 2, rank=lambda: 0),
|
|
num_routed_experts=4,
|
|
num_extra_experts=2,
|
|
is_normal_engine=True,
|
|
**kwargs,
|
|
)
|
|
calls: list[int] = []
|
|
step = {"n": 0}
|
|
|
|
def fake_rearrange() -> None:
|
|
calls.append(step["n"])
|
|
|
|
monkeypatch.setattr(balancer, "_rearrange_experts", fake_rearrange)
|
|
monkeypatch.setattr(balancer, "_update_load_balance_stats", lambda: None)
|
|
|
|
def run(is_prompt: bool) -> None:
|
|
step["n"] += 1
|
|
balancer.expert_count.fill_(1 if not is_prompt else 7)
|
|
balancer.step(is_prompt)
|
|
|
|
balancer.run = run # type: ignore[attr-defined]
|
|
return balancer, calls
|
|
|
|
|
|
def test_default_rearranges_right_after_prefill(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
balancer, calls = _balancer(monkeypatch)
|
|
for _ in range(4):
|
|
balancer.run(True)
|
|
balancer.run(False)
|
|
assert calls == [5]
|
|
assert balancer._expert_count_window.shape[0] == 3
|
|
|
|
|
|
def test_decode_only_rearranges_after_a_window_of_decode_steps(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
balancer, calls = _balancer(monkeypatch, rearrangement_interval=40, expert_count_window_size=4, decode_only=True)
|
|
for _ in range(3):
|
|
balancer.run(True)
|
|
assert balancer._expert_count_window.sum() == 0
|
|
for _ in range(4):
|
|
balancer.run(False)
|
|
assert calls == [3 + 4]
|
|
assert int(balancer._expert_count_window.min()) == 1
|
|
# Then rearrange periodically; a prefill in between again triggers once a window has filled
|
|
for _ in range(10):
|
|
balancer.run(False)
|
|
balancer.run(True)
|
|
balancer.run(True)
|
|
for _ in range(4):
|
|
balancer.run(False)
|
|
assert calls == [7, 23]
|
|
for _ in range(40):
|
|
balancer.run(False)
|
|
assert calls == [7, 23, 63]
|
|
|
|
|
|
def test_decode_only_defers_periodic_until_window_is_full(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
# The first periodic rearrangement (after interval // 4 = 2 decode steps) lands before the window is full, so it
|
|
# is deferred until the window fills and rearranges only once
|
|
balancer, calls = _balancer(monkeypatch, rearrangement_interval=8, expert_count_window_size=3, decode_only=True)
|
|
balancer.run(True)
|
|
for _ in range(3):
|
|
balancer.run(False)
|
|
assert calls == [4]
|
|
for _ in range(8):
|
|
balancer.run(False)
|
|
assert calls == [4, 12]
|
|
|
|
|
|
def test_decode_only_validates_window(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
with pytest.raises(ValueError):
|
|
_balancer(monkeypatch, rearrangement_interval=4, expert_count_window_size=8, decode_only=True)
|
|
|
|
|
|
def test_validation_names_the_caller_settings(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
names = ("DEEPINFER_DRAFT_EXPERT_COUNT_WINDOW_SIZE", "DEEPINFER_DRAFT_EXPERT_REARRANGEMENT_STEP_INTERVAL")
|
|
with pytest.raises(
|
|
ValueError, match=r"^DEEPINFER_DRAFT_EXPERT_COUNT_WINDOW_SIZE=100 .*DEEPINFER_DRAFT_EXPERT_REARRANGEMENT_STEP_INTERVAL=30"
|
|
):
|
|
_balancer(monkeypatch, rearrangement_interval=30, expert_count_window_size=100, decode_only=True, setting_names=names)
|
|
with pytest.raises(ValueError, match=r"^DEEPINFER_EXPERT_COUNT_WINDOW_SIZE=100 "):
|
|
_balancer(monkeypatch, rearrangement_interval=30, expert_count_window_size=100)
|