Files
DeepInfer/tests/models/test_eplb_schedule.py
T
shunyadandClaude Opus 5.5 cb1d0b3ee2 fix(eplb): name the draft-layer settings in EPLB validation errors
Draft-layer schedules went through the main-model validation, so a bad
DEEPINFER_DRAFT_EXPERT_COUNT_WINDOW_SIZE was reported as
"DEEPINFER_EXPERT_COUNT_WINDOW_SIZE=... must be between 1 and
DEEPINFER_EXPERT_REARRANGEMENT_STEP_INTERVAL=...", naming variables the user
never set. Draft settings could also be dropped without a word.

- ExpertLoadBalancer takes setting_names for its error messages; the draft
  schedule passes the draft variables (or says the interval is inherited, or
  the window is the default).
- Warn when DEEPINFER_DRAFT_EXPERT_COUNT_WINDOW_SIZE is set but no draft
  interval resolves, and when draft layers share an EPLB layout group with
  main-model layers and so cannot follow the draft schedule.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 22:28:43 -08:00

113 lines
4.1 KiB
Python

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
from types import SimpleNamespace
from typing import Any
import pytest
from deepinfer import environ
from deepinfer.model_executor import eplb
@pytest.fixture(autouse=True)
def _env(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("DEEPINFER_EXPERT_REARRANGEMENT_STEP_INTERVAL", "1000")
monkeypatch.setenv("DEEPINFER_EXPERT_COUNT_WINDOW_SIZE", "3")
monkeypatch.setenv("DEEPINFER_REARRANGE_AFTER_PREFILL_IN_NORMAL_ENGINE", "1")
environ.rearm_all()
yield
monkeypatch.undo()
environ.rearm_all()
def _balancer(monkeypatch: pytest.MonkeyPatch, **kwargs: Any) -> tuple[eplb.ExpertLoadBalancer, list[int]]:
balancer = eplb.ExpertLoadBalancer(
expert_weights=[],
num_moe_layers=2,
num_logical_experts=4,
num_physical_experts=6,
ep_group=SimpleNamespace(size=lambda: 2, rank=lambda: 0),
num_routed_experts=4,
num_extra_experts=2,
is_normal_engine=True,
**kwargs,
)
calls: list[int] = []
step = {"n": 0}
def fake_rearrange() -> None:
calls.append(step["n"])
monkeypatch.setattr(balancer, "_rearrange_experts", fake_rearrange)
monkeypatch.setattr(balancer, "_update_load_balance_stats", lambda: None)
def run(is_prompt: bool) -> None:
step["n"] += 1
balancer.expert_count.fill_(1 if not is_prompt else 7)
balancer.step(is_prompt)
balancer.run = run # type: ignore[attr-defined]
return balancer, calls
def test_default_rearranges_right_after_prefill(monkeypatch: pytest.MonkeyPatch) -> None:
balancer, calls = _balancer(monkeypatch)
for _ in range(4):
balancer.run(True)
balancer.run(False)
assert calls == [5]
assert balancer._expert_count_window.shape[0] == 3
def test_decode_only_rearranges_after_a_window_of_decode_steps(monkeypatch: pytest.MonkeyPatch) -> None:
balancer, calls = _balancer(monkeypatch, rearrangement_interval=40, expert_count_window_size=4, decode_only=True)
for _ in range(3):
balancer.run(True)
assert balancer._expert_count_window.sum() == 0
for _ in range(4):
balancer.run(False)
assert calls == [3 + 4]
assert int(balancer._expert_count_window.min()) == 1
# Then rearrange periodically; a prefill in between again triggers once a window has filled
for _ in range(10):
balancer.run(False)
balancer.run(True)
balancer.run(True)
for _ in range(4):
balancer.run(False)
assert calls == [7, 23]
for _ in range(40):
balancer.run(False)
assert calls == [7, 23, 63]
def test_decode_only_defers_periodic_until_window_is_full(monkeypatch: pytest.MonkeyPatch) -> None:
# The first periodic rearrangement (after interval // 4 = 2 decode steps) lands before the window is full, so it
# is deferred until the window fills and rearranges only once
balancer, calls = _balancer(monkeypatch, rearrangement_interval=8, expert_count_window_size=3, decode_only=True)
balancer.run(True)
for _ in range(3):
balancer.run(False)
assert calls == [4]
for _ in range(8):
balancer.run(False)
assert calls == [4, 12]
def test_decode_only_validates_window(monkeypatch: pytest.MonkeyPatch) -> None:
with pytest.raises(ValueError):
_balancer(monkeypatch, rearrangement_interval=4, expert_count_window_size=8, decode_only=True)
def test_validation_names_the_caller_settings(monkeypatch: pytest.MonkeyPatch) -> None:
names = ("DEEPINFER_DRAFT_EXPERT_COUNT_WINDOW_SIZE", "DEEPINFER_DRAFT_EXPERT_REARRANGEMENT_STEP_INTERVAL")
with pytest.raises(
ValueError, match=r"^DEEPINFER_DRAFT_EXPERT_COUNT_WINDOW_SIZE=100 .*DEEPINFER_DRAFT_EXPERT_REARRANGEMENT_STEP_INTERVAL=30"
):
_balancer(monkeypatch, rearrangement_interval=30, expert_count_window_size=100, decode_only=True, setting_names=names)
with pytest.raises(ValueError, match=r"^DEEPINFER_EXPERT_COUNT_WINDOW_SIZE=100 "):
_balancer(monkeypatch, rearrangement_interval=30, expert_count_window_size=100)