mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-11-20 04:22:27 +00:00
The Cake kernel reports an invalid call (e.g. its lossless return codec ran out of overflow blocks) in a fail-closed flag, but DeepInfer read it only once at warmup, so such a call silently returned wrong MoE outputs. The flag is per call (the next launch clears it) and lives in the workspace all layers share, so it has to be captured right after each call: - CakeMoE ORs the flag into its own protocol_errors tensor after every call, on the stream (no host sync, also inside CUDA graphs), and names itself after its MoE layer. - userbuffer.check_cake_moe_errors(), called once per step from DeepseekForCausalLM.sample, copies the layers' flags to pinned memory behind the step and checks the previous step's copy, so an error stops the run one step later without a host synchronization. - check_protocol reads the same flag tensor. Tests: tests/worker/test_cake_moe_errors.py (CPU). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
67 lines
2.2 KiB
Python
67 lines
2.2 KiB
Python
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
"""The per-step check of Cake MegaMoE protocol errors (userbuffer.check_cake_moe_errors), on CPU tensors."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
import torch
|
|
|
|
from deepinfer.model_executor.parallel_utils.userbuffer import _UserBufferManager
|
|
|
|
|
|
def _manager(num_layers: int) -> tuple[_UserBufferManager, list[SimpleNamespace]]:
|
|
manager = _UserBufferManager()
|
|
layers = [
|
|
SimpleNamespace(name=f"MoE layer {i}", protocol_errors=torch.zeros(1, dtype=torch.int32), destroy=lambda: None)
|
|
for i in range(num_layers)
|
|
]
|
|
for layer in layers:
|
|
manager.register_cake_moe(layer)
|
|
return manager, layers
|
|
|
|
|
|
def test_no_cake_layers_is_a_no_op() -> None:
|
|
manager = _UserBufferManager()
|
|
manager.check_cake_moe_errors()
|
|
assert manager._cake_moe_errors_host is None
|
|
|
|
|
|
def test_clean_steps_pass() -> None:
|
|
manager, _ = _manager(3)
|
|
for _ in range(4):
|
|
manager.check_cake_moe_errors()
|
|
|
|
|
|
def test_error_surfaces_on_the_next_step() -> None:
|
|
manager, layers = _manager(3)
|
|
manager.check_cake_moe_errors()
|
|
layers[1].protocol_errors.bitwise_or_(torch.tensor([0x4], dtype=torch.int32))
|
|
# The step that set the flag only starts the copy ...
|
|
manager.check_cake_moe_errors()
|
|
# ... and the next one reports it, naming the layer and the flag
|
|
with pytest.raises(RuntimeError, match=r"MoE layer 1 \(0x4\)") as excinfo:
|
|
manager.check_cake_moe_errors()
|
|
assert "MoE layer 0" not in str(excinfo.value) and "MoE layer 2" not in str(excinfo.value)
|
|
|
|
|
|
def test_errors_are_sticky() -> None:
|
|
manager, layers = _manager(2)
|
|
layers[0].protocol_errors.fill_(1)
|
|
manager.check_cake_moe_errors()
|
|
for _ in range(2):
|
|
with pytest.raises(RuntimeError, match="MoE layer 0"):
|
|
manager.check_cake_moe_errors()
|
|
|
|
|
|
def test_close_resets_the_check() -> None:
|
|
manager, layers = _manager(1)
|
|
layers[0].protocol_errors.fill_(1)
|
|
manager.check_cake_moe_errors()
|
|
manager.close()
|
|
manager.check_cake_moe_errors()
|
|
assert manager._cake_moe_errors_host is None
|