mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-11-18 19:42:28 +00:00
Run DeepSeek-V4.1-Flash offline inference on SM120 GPUs (RTX PRO 6000 Blackwell) with the same entry points as SM100. SM100 and Ascend paths are unchanged. - Build: SM120 is detected from the first visible GPU (scripts/nvidia_arch.sh). It builds DeepEP and DeepGEMM from its nv_dev branch (vendor/DeepGEMM-sm120), and no FlashMLA or DeepSelect; FlashInfer is installed from a pinned commit (requirements-nvidia-sm120.txt). - Attention: V4.1 sparse MLA runs on FlashInfer's SM120 sparse MLA kernels, reading the inline-scale KV records as stored; KV blocks are aligned to 512 bytes. - Indexer: FlashInfer's fused DSA indexer scores the index keys and selects their top-k in one call, for decode on every indexer layer (candidate source and consumer layers included) and for prefill outside the candidate layers. The remaining selections (prefill of the candidate layers, the sampler) use FlashInfer's top-k behind DeepSelect's interface (model_executor/layers/topk.py), in deterministic mode. - MoE EP: DeepEP over NCCL GIN by default, or NCCL collectives (DEEPINFER_SM120_EP_BACKEND=nccl); NCCL settings known to break the EP paths are rejected. EPLB, including decode-only draft-layer EPLB, and per-layer YoCo MoE dispatch. - Engram tables live in page-locked host shared memory; killed workers leave nothing in /dev/shm. - Features: attention DP + MoE EP, DSpark/nextn speculative decoding, CUDA graphs, async scheduling, decoder SWA bounded replay, structured outputs, image inputs, multi-node deployments. Features without an SM120 implementation (CP > 1, MTP varlen, batch-invariant mode, GPU-resident Engram tables) are refused at engine start. - Eval: chat GSM8K and CV-Bench; tests for the SM120 kernels and their DeepInfer call sites. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
150 lines
6.3 KiB
Python
150 lines
6.3 KiB
Python
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
"""Teacher-forced greedy parity against a reference pin set.
|
|
|
|
Compare (8 GPUs, a local checkpoint and a pin file):
|
|
DEEPINFER_HF_MODEL_DIR=/path/to/checkpoint DEEPINFER_PARITY_PINS=/path/to/pins.json \
|
|
python -m pytest tests/models/test_numeric_parity.py -s -k compare
|
|
|
|
Each prompt is fed the reference's prompt and greedy continuation; at every generated position where the
|
|
reference's top-1 logit leads its top-2 by at least `MIN_MARGIN`, DeepInfer's argmax must equal the
|
|
reference's top-1 token.
|
|
|
|
Record a pin set on the reference platform from the prompts of an existing pin file:
|
|
DEEPINFER_HF_MODEL_DIR=... DEEPINFER_PARITY_PINS=/path/to/prompts.json DEEPINFER_PARITY_RECORD=/path/to/out.json \
|
|
python -m pytest tests/models/test_numeric_parity.py -s -k record
|
|
|
|
Pin file: {"prompts": {name: {"prompt_ids": [...], "greedy_ids": [...],
|
|
"positions": [{"top1_id": int, "margin": float}, ...]}}}
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
from functools import partial
|
|
|
|
import pytest
|
|
|
|
from examples._config import DEFAULT_MODEL_NAME, create_llm
|
|
from tests.models._runner import run_model_test
|
|
|
|
EP_SIZE = 8
|
|
NUM_POSITIONS = 64
|
|
MIN_MARGIN = 1.0
|
|
MAX_MODEL_LEN = 20480
|
|
|
|
|
|
def _load_pins(path: str) -> dict[str, dict]:
|
|
with open(path) as file:
|
|
prompts = json.load(file)["prompts"]
|
|
for name, pin in prompts.items():
|
|
assert len(pin["greedy_ids"]) >= NUM_POSITIONS and len(pin["positions"]) >= NUM_POSITIONS, name
|
|
assert len(pin["prompt_ids"]) + NUM_POSITIONS <= MAX_MODEL_LEN, name
|
|
return prompts
|
|
|
|
|
|
def _create_llm(dp_rank: int, max_num_seqs: int) -> object:
|
|
return create_llm(
|
|
DEFAULT_MODEL_NAME,
|
|
dp_rank,
|
|
ep_size=EP_SIZE,
|
|
env={
|
|
"DEEPINFER_USE_NEXTN_DECODE": "0",
|
|
"DEEPINFER_EXPERT_REARRANGEMENT_STEP_INTERVAL": "0",
|
|
"DEEPINFER_SAMPLER_FORCE_TOPK": "0",
|
|
"DEEPINFER_USE_MULTIMODAL_POOL": "0",
|
|
},
|
|
max_num_batched_tokens=8192,
|
|
max_model_len=MAX_MODEL_LEN,
|
|
max_num_seqs=max_num_seqs,
|
|
gpu_memory_utilization=0.85,
|
|
lookahead_tokens=0,
|
|
)
|
|
|
|
|
|
def _teacher_forced(llm: object, prompts: list[list[int]], continuations: list[list[int]]) -> list:
|
|
from deepinfer.sampling_params import SamplingParams
|
|
|
|
# Logprobs follow the temperature transform, so temperature 1 keeps logit margins on their own scale.
|
|
sampling_params = [
|
|
SamplingParams(
|
|
temperature=1.0,
|
|
top_p=1.0,
|
|
max_tokens=NUM_POSITIONS,
|
|
ignore_eos=True,
|
|
top_logprobs=2,
|
|
golden_token_ids=continuation[:NUM_POSITIONS],
|
|
)
|
|
for continuation in continuations
|
|
]
|
|
return llm.generate(prompts=None, sampling_params=sampling_params, prompt_token_ids=prompts, use_tqdm=False)
|
|
|
|
|
|
def _run_record(pins_path: str, out_path: str, dp_size: int, dp_rank: int) -> None:
|
|
with open(pins_path) as file:
|
|
prompts = {name: pin["prompt_ids"] for name, pin in json.load(file)["prompts"].items()}
|
|
llm = _create_llm(dp_rank, len(prompts))
|
|
from deepinfer.sampling_params import SamplingParams
|
|
|
|
greedy = llm.generate(
|
|
prompts=None,
|
|
sampling_params=SamplingParams(temperature=0.0, max_tokens=NUM_POSITIONS, ignore_eos=True),
|
|
prompt_token_ids=list(prompts.values()),
|
|
use_tqdm=False,
|
|
)
|
|
greedy_ids = [[step.token_id for step in output] for output in greedy]
|
|
outputs = _teacher_forced(llm, list(prompts.values()), greedy_ids)
|
|
pins = {}
|
|
for (name, prompt_ids), ids, output in zip(prompts.items(), greedy_ids, outputs, strict=True):
|
|
positions = [{"top1_id": step.top_token_ids[0], "margin": step.top_logprobs[0] - step.top_logprobs[1]} for step in output]
|
|
assert [p["top1_id"] for p in positions] == ids, f"{name}: teacher-forced argmax differs from greedy decoding"
|
|
pins[name] = {"prompt_ids": prompt_ids, "greedy_ids": ids, "positions": positions}
|
|
if dp_rank == 0:
|
|
with open(out_path, "w") as file:
|
|
json.dump({"prompts": pins}, file)
|
|
llm.close()
|
|
|
|
|
|
def _run_parity(pins_path: str, dp_size: int, dp_rank: int) -> None:
|
|
pins = _load_pins(pins_path)
|
|
llm = _create_llm(dp_rank, len(pins))
|
|
outputs = _teacher_forced(llm, [pin["prompt_ids"] for pin in pins.values()], [pin["greedy_ids"] for pin in pins.values()])
|
|
|
|
failures, report = [], {}
|
|
for (name, pin), output in zip(pins.items(), outputs, strict=True):
|
|
assert len(output) == NUM_POSITIONS, (name, len(output))
|
|
checked = mismatched = 0
|
|
rows = []
|
|
for position, (step, ref) in enumerate(zip(output, pin["positions"], strict=False)):
|
|
argmax = step.top_token_ids[0]
|
|
gated = ref["margin"] >= MIN_MARGIN
|
|
checked += gated
|
|
if gated and argmax != ref["top1_id"]:
|
|
mismatched += 1
|
|
failures.append(f"{name}@{position}: deepinfer {argmax} != reference {ref['top1_id']} (margin {ref['margin']:.3f})")
|
|
rows.append([argmax, ref["top1_id"], ref["margin"], step.top_logprobs[0] - step.top_logprobs[1]])
|
|
report[name] = {"checked": checked, "mismatched": mismatched, "positions": rows}
|
|
print(f"DP {dp_rank} {name}: {checked} gated positions, {mismatched} mismatched", flush=True)
|
|
|
|
if dp_rank == 0 and (report_path := os.environ.get("DEEPINFER_PARITY_REPORT")):
|
|
with open(report_path, "w") as file:
|
|
json.dump(report, file, indent=1)
|
|
llm.close()
|
|
assert not failures, f"DP {dp_rank}: " + "; ".join(failures)
|
|
|
|
|
|
def test_compare() -> None:
|
|
pins_path = os.environ.get("DEEPINFER_PARITY_PINS")
|
|
if not pins_path or os.environ.get("DEEPINFER_PARITY_RECORD"):
|
|
pytest.skip("DEEPINFER_PARITY_PINS is not set, or a pin set is being recorded")
|
|
run_model_test(partial(_run_parity, pins_path), EP_SIZE, timeout=3600)
|
|
|
|
|
|
def test_record() -> None:
|
|
pins_path, out_path = os.environ.get("DEEPINFER_PARITY_PINS"), os.environ.get("DEEPINFER_PARITY_RECORD")
|
|
if not pins_path or not out_path:
|
|
pytest.skip("DEEPINFER_PARITY_PINS and DEEPINFER_PARITY_RECORD are not both set")
|
|
run_model_test(partial(_run_record, pins_path, out_path), EP_SIZE, timeout=3600)
|