Files
DeepInfer/tests/worker/test_cake_moe_errors.py
T
shunyadandClaude Opus 5.5 d2755d60a1 fix(sm120): check Cake MegaMoE protocol errors at runtime
The Cake kernel reports an invalid call (e.g. its lossless return codec ran
out of overflow blocks) in a fail-closed flag, but DeepInfer read it only
once at warmup, so such a call silently returned wrong MoE outputs. The flag
is per call (the next launch clears it) and lives in the workspace all
layers share, so it has to be captured right after each call:

- CakeMoE ORs the flag into its own protocol_errors tensor after every
  call, on the stream (no host sync, also inside CUDA graphs), and names
  itself after its MoE layer.
- userbuffer.check_cake_moe_errors(), called once per step from
  DeepseekForCausalLM.sample, copies the layers' flags to pinned memory
  behind the step and checks the previous step's copy, so an error stops
  the run one step later without a host synchronization.
- check_protocol reads the same flag tensor.

Tests: tests/worker/test_cake_moe_errors.py (CPU).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 22:46:30 -08:00

67 lines
2.2 KiB
Python

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""The per-step check of Cake MegaMoE protocol errors (userbuffer.check_cake_moe_errors), on CPU tensors."""
from __future__ import annotations
from types import SimpleNamespace
import pytest
import torch
from deepinfer.model_executor.parallel_utils.userbuffer import _UserBufferManager
def _manager(num_layers: int) -> tuple[_UserBufferManager, list[SimpleNamespace]]:
manager = _UserBufferManager()
layers = [
SimpleNamespace(name=f"MoE layer {i}", protocol_errors=torch.zeros(1, dtype=torch.int32), destroy=lambda: None)
for i in range(num_layers)
]
for layer in layers:
manager.register_cake_moe(layer)
return manager, layers
def test_no_cake_layers_is_a_no_op() -> None:
manager = _UserBufferManager()
manager.check_cake_moe_errors()
assert manager._cake_moe_errors_host is None
def test_clean_steps_pass() -> None:
manager, _ = _manager(3)
for _ in range(4):
manager.check_cake_moe_errors()
def test_error_surfaces_on_the_next_step() -> None:
manager, layers = _manager(3)
manager.check_cake_moe_errors()
layers[1].protocol_errors.bitwise_or_(torch.tensor([0x4], dtype=torch.int32))
# The step that set the flag only starts the copy ...
manager.check_cake_moe_errors()
# ... and the next one reports it, naming the layer and the flag
with pytest.raises(RuntimeError, match=r"MoE layer 1 \(0x4\)") as excinfo:
manager.check_cake_moe_errors()
assert "MoE layer 0" not in str(excinfo.value) and "MoE layer 2" not in str(excinfo.value)
def test_errors_are_sticky() -> None:
manager, layers = _manager(2)
layers[0].protocol_errors.fill_(1)
manager.check_cake_moe_errors()
for _ in range(2):
with pytest.raises(RuntimeError, match="MoE layer 0"):
manager.check_cake_moe_errors()
def test_close_resets_the_check() -> None:
manager, layers = _manager(1)
layers[0].protocol_errors.fill_(1)
manager.check_cake_moe_errors()
manager.close()
manager.check_cake_moe_errors()
assert manager._cake_moe_errors_host is None