mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-11-18 19:42:28 +00:00
Run DeepSeek-V4.1-Flash offline inference on SM120 GPUs (RTX PRO 6000 Blackwell) with the same entry points as SM100. SM100 and Ascend paths are unchanged. - Build: SM120 is detected from the first visible GPU (scripts/nvidia_arch.sh). It builds DeepEP and DeepGEMM from its nv_dev branch (vendor/DeepGEMM-sm120), and no FlashMLA or DeepSelect; FlashInfer is installed from a pinned commit (requirements-nvidia-sm120.txt). - Attention: V4.1 sparse MLA runs on FlashInfer's SM120 sparse MLA kernels, reading the inline-scale KV records as stored; KV blocks are aligned to 512 bytes. - Indexer: FlashInfer's fused DSA indexer scores the index keys and selects their top-k in one call, for decode on every indexer layer (candidate source and consumer layers included) and for prefill outside the candidate layers. The remaining selections (prefill of the candidate layers, the sampler) use FlashInfer's top-k behind DeepSelect's interface (model_executor/layers/topk.py), in deterministic mode. - MoE EP: DeepEP over NCCL GIN by default, or NCCL collectives (DEEPINFER_SM120_EP_BACKEND=nccl); NCCL settings known to break the EP paths are rejected. EPLB, including decode-only draft-layer EPLB, and per-layer YoCo MoE dispatch. - Engram tables live in page-locked host shared memory; killed workers leave nothing in /dev/shm. - Features: attention DP + MoE EP, DSpark/nextn speculative decoding, CUDA graphs, async scheduling, decoder SWA bounded replay, structured outputs, image inputs, multi-node deployments. Features without an SM120 implementation (CP > 1, MTP varlen, batch-invariant mode, GPU-resident Engram tables) are refused at engine start. - Eval: chat GSM8K and CV-Bench; tests for the SM120 kernels and their DeepInfer call sites. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
333 lines
14 KiB
Python
333 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
from types import ModuleType
|
|
from typing import Any
|
|
|
|
EVAL_DIR = Path(__file__).resolve().parent
|
|
if str(EVAL_DIR) not in sys.path:
|
|
sys.path.insert(0, str(EVAL_DIR))
|
|
|
|
from common.io import write_jsonl # noqa: E402
|
|
from common.loader import SUPPORTED, load_bench # noqa: E402
|
|
from common.paths import ensure_sys_path, resolve_out_dir # noqa: E402
|
|
|
|
REPO = ensure_sys_path() # put repo root on sys.path so examples / deepinfer can be imported
|
|
|
|
from examples._config import DEFAULT_MODEL_NAME # noqa: E402
|
|
|
|
# reasoning_effort "default" passes None to the encoder, which then applies its own default effort.
|
|
BENCH_DEFAULTS = {
|
|
"aime": {"n_sample": 32, "temperature": 1.0, "top_p": 1.0, "reasoning_effort": 100, "max_len": 1024 * 144},
|
|
"hle": {"n_sample": 1, "temperature": 1.0, "top_p": 1.0, "reasoning_effort": 100, "max_len": 1024 * 144},
|
|
"gpqa": {"n_sample": 8, "temperature": 1.0, "top_p": 1.0, "reasoning_effort": 100, "max_len": 1024 * 144},
|
|
"gsm8k": {"n_sample": 1, "temperature": 0.0, "top_p": 1.0, "reasoning_effort": "default", "max_len": 1024 * 20, "max_tokens": 16384},
|
|
"cvbench": {"n_sample": 1, "temperature": 0.0, "top_p": 1.0, "reasoning_effort": "default", "max_len": 1024 * 16, "max_tokens": 32},
|
|
}
|
|
|
|
|
|
_REASONING_EFFORT_LEVELS = ("low", "high", "max", "default")
|
|
|
|
|
|
def _reasoning_effort(value: str) -> int | str:
|
|
"""Accept an integer 1-100, a named level low/high/max, or "default" (the encoder's own default)."""
|
|
if value in _REASONING_EFFORT_LEVELS:
|
|
return value
|
|
try:
|
|
effort = int(value)
|
|
except ValueError:
|
|
raise argparse.ArgumentTypeError(f"expected an integer 1-100 or one of {_REASONING_EFFORT_LEVELS}, got {value!r}") from None
|
|
if not 1 <= effort <= 100:
|
|
raise argparse.ArgumentTypeError(f"expected 1-100, got {effort}")
|
|
return effort
|
|
|
|
|
|
parser = argparse.ArgumentParser(description="eval benchmark")
|
|
# benchmark setting
|
|
parser.add_argument("--bench", default="aime", choices=list(SUPPORTED))
|
|
parser.add_argument("--model-name", default=DEFAULT_MODEL_NAME, help="HF model id or HF ckpt directory")
|
|
parser.add_argument("--data-file", required=True, help="problem jsonl")
|
|
parser.add_argument("--limit", type=int, default=None, help="run only the first N problems")
|
|
parser.add_argument("--out-dir", default=None)
|
|
parser.add_argument("--tag", default=None, help="output dir suffix, to compare settings of the same model")
|
|
# sample param
|
|
parser.add_argument(
|
|
"--n-sample", type=int, default=None, help="samples per problem (default: aime 32 / hle 1 / gpqa 8 / gsm8k 1 / cvbench 1)"
|
|
)
|
|
parser.add_argument("--temperature", type=float, default=None, help="default 1.0; gsm8k and cvbench 0.0 (greedy)")
|
|
parser.add_argument("--top-p", type=float, default=None, help="default 1.0")
|
|
parser.add_argument(
|
|
"--reasoning-effort",
|
|
type=_reasoning_effort,
|
|
default=None,
|
|
help="integer 1-100, low/high/max, or default (the encoder's default); higher means more thinking (default: 100; gsm8k default)",
|
|
)
|
|
parser.add_argument(
|
|
"--max-len", type=int, default=None, help="max prompt + output length per request (default: 147456; gsm8k 20480; cvbench 16384)"
|
|
)
|
|
parser.add_argument(
|
|
"--max-tokens",
|
|
type=int,
|
|
default=None,
|
|
help="max output tokens per request, within --max-len (default: no cap; gsm8k 16384; cvbench 32)",
|
|
)
|
|
# engine
|
|
parser.add_argument("--ep", type=int, default=8, help="expert parallel size = total GPUs in this job")
|
|
parser.add_argument("--cp", type=int, default=1)
|
|
parser.add_argument("--dp", type=int, default=None, help="defaults to ep // cp")
|
|
parser.add_argument("--max-num-seqs", type=int, default=384)
|
|
parser.add_argument("--max-num-batched-tokens", type=int, default=32768)
|
|
parser.add_argument("--gpu-memory-utilization", type=float, default=0.9)
|
|
parser.add_argument(
|
|
"--env", action="append", default=[], metavar="KEY=VALUE", help="extra env vars, repeatable; applied before the engine is built"
|
|
)
|
|
parser.add_argument("--decode-blacklist", default="0,2,129262..129264", help="tokens blocked during generation; empty string disables it")
|
|
parser.add_argument(
|
|
"--batch-invariant", action="store_true", help="DEEPINFER_BATCH_INVARIANT=1: results independent of batch composition (slower)"
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
# fill in per-benchmark defaults (only for args not passed explicitly)
|
|
for _key, _default in BENCH_DEFAULTS[args.bench].items():
|
|
if getattr(args, _key) is None:
|
|
setattr(args, _key, _default)
|
|
|
|
REASONING_EFFORT = None if args.reasoning_effort == "default" else args.reasoning_effort
|
|
|
|
OUT_DIR = resolve_out_dir(
|
|
args.out_dir,
|
|
bench=args.bench,
|
|
tag=args.tag or f"{args.model_name.replace('/', '_')}-n{args.n_sample}-j{args.reasoning_effort}",
|
|
)
|
|
META_PATH = os.path.join(OUT_DIR, "meta.json")
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# generation
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def _render_prompts(encoder: ModuleType, records: list[dict], thinking_mode: str) -> list[str]:
|
|
return [
|
|
encoder.encode_messages(
|
|
rec["messages"],
|
|
thinking_mode=thinking_mode,
|
|
reasoning_effort=REASONING_EFFORT,
|
|
)
|
|
for rec in records
|
|
]
|
|
|
|
|
|
def _multi_modal_entry(record: dict) -> dict:
|
|
"""One multi_modal_data entry: image bytes plus dimensions."""
|
|
return {
|
|
"image_bytes": [Path(path).read_bytes() for path in record.get("image_paths") or []],
|
|
"image_sizes": [tuple(size) for size in record.get("image_sizes") or []],
|
|
}
|
|
|
|
|
|
def _setup_env() -> None:
|
|
# MTP / DFlash speculative decoding, set before setup_common_env (which only setdefaults it).
|
|
os.environ["DEEPINFER_USE_NEXTN_DECODE"] = "5"
|
|
os.environ["DEEPINFER_ENABLE_MTP_VARLEN"] = "1"
|
|
|
|
from examples._config import setup_common_env
|
|
|
|
setup_common_env(model_name=args.model_name, sampler_force_topk=0, set_override_experts=True, enable_eplb=True)
|
|
for item in args.env:
|
|
key, _, value = item.partition("=")
|
|
assert key and "=" in item, f"--env expects KEY=VALUE, got {item!r}"
|
|
os.environ[key] = value
|
|
if args.batch_invariant:
|
|
os.environ["DEEPINFER_BATCH_INVARIANT"] = "1"
|
|
if args.decode_blacklist:
|
|
os.environ["DEEPINFER_DECODE_TOKEN_BLACKLIST"] = args.decode_blacklist
|
|
|
|
|
|
def _seed(global_index: int, sample_index: int) -> int:
|
|
"""Per-sample seed: identical seeds produce identical output, so every sample needs its own."""
|
|
return (global_index * args.n_sample + sample_index) % (2**64)
|
|
|
|
|
|
def _rows_from_outputs(tokenizer: Any, plan: list, records: list[dict], outputs: list) -> list[dict]:
|
|
rows = []
|
|
for (problem, sample), generated in zip(plan, outputs, strict=True):
|
|
assert generated, f"{records[problem]['task_id']}#{sample}: generated no tokens"
|
|
text = tokenizer.decode([token.token_id for token in generated], skip_special_tokens=False, clean_up_tokenization_spaces=False)
|
|
rows.append(
|
|
{
|
|
"task_id": records[problem]["task_id"],
|
|
"sample_idx": sample,
|
|
"seq_seed": _seed(problem, sample),
|
|
"prompt_len": generated[0].prompt_len,
|
|
"num_output_tokens": len(generated),
|
|
"finish_reason": generated[-1].finish_reason,
|
|
"raw_text": text,
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def run_rank(dp_size: int, dp_rank: int) -> None:
|
|
_setup_env()
|
|
|
|
from common.encoder import load_encoder
|
|
|
|
from examples._config import create_llm
|
|
|
|
encoder = load_encoder(args.model_name)
|
|
|
|
bench = load_bench(args.bench)
|
|
problems = bench.load(args.data_file)
|
|
records = bench.build_records(problems, limit=args.limit)
|
|
|
|
# Shard by sample so every rank gets an equal share and no rank inherits a problem's long tail.
|
|
plan = [(problem, sample) for problem in range(len(records)) for sample in range(args.n_sample)][dp_rank::dp_size]
|
|
|
|
llm = create_llm(
|
|
args.model_name,
|
|
dp_rank,
|
|
ep_size=args.ep,
|
|
cp_size=args.cp,
|
|
data_parallel_size=dp_size,
|
|
max_num_batched_tokens=args.max_num_batched_tokens,
|
|
# The engine's runtime_max_model_len is max_model_len + 1, and every request's
|
|
# prompt + max_tokens is already clamped to <= max_len, so a tiny margin is enough.
|
|
max_model_len=args.max_len + 64,
|
|
max_num_seqs=args.max_num_seqs,
|
|
gpu_memory_utilization=args.gpu_memory_utilization,
|
|
lookahead_tokens=min(1, int(os.environ.get("DEEPINFER_USE_NEXTN_DECODE", "0"))),
|
|
decode_sequence=False,
|
|
)
|
|
|
|
try:
|
|
# Import after create_llm: deepinfer imports torch, which must not happen before the engine
|
|
# has set up the accelerator devices.
|
|
from deepinfer.sampling_params import SamplingParams
|
|
|
|
tokenizer = llm.get_tokenizer()
|
|
|
|
if not plan:
|
|
# dp_size > number of requests leaves some ranks with an empty shard: write an empty
|
|
# file and stop, do not build a batch.
|
|
os.makedirs(OUT_DIR, exist_ok=True)
|
|
write_jsonl(os.path.join(OUT_DIR, f"raw_rank{dp_rank:03d}.jsonl"), [])
|
|
print(f"[rank {dp_rank}] empty shard (dp_size={dp_size} > request count), skipping", flush=True)
|
|
return
|
|
|
|
# Render and tokenize each needed prompt once: a rank may use the same prompt for several
|
|
# samples of the same problem.
|
|
needed = sorted({problem for problem, _ in plan})
|
|
# A benchmark renders in thinking mode unless its THINKING_MODE says otherwise (cvbench: "chat").
|
|
thinking_mode = getattr(bench, "THINKING_MODE", "thinking")
|
|
texts = dict(zip(needed, _render_prompts(encoder, [records[problem] for problem in needed], thinking_mode), strict=True))
|
|
has_images = any(records[problem].get("image_paths") for problem in needed)
|
|
multi_modal: dict[int, dict] = {}
|
|
if has_images:
|
|
# Replace the <image> placeholder with the image token the engine understands. Image
|
|
# questions then take the string path and the engine expands the token.
|
|
from examples._prompts import get_engine_image_token, replace_image_tokens
|
|
|
|
image_token_str, _ = get_engine_image_token(llm)
|
|
texts = dict(zip(needed, replace_image_tokens([texts[problem] for problem in needed], image_token_str), strict=True))
|
|
# The engine consumes multi_modal_data in place, so every request gets its own copy.
|
|
multi_modal = {problem: _multi_modal_entry(records[problem]) for problem in needed}
|
|
token_ids = {problem: tokenizer.encode(text, add_special_tokens=False) for problem, text in texts.items()}
|
|
|
|
# max_tokens is clamped to the remaining budget: max_len - prompt_len (and --max-tokens), at least 1.
|
|
sampling_params = [
|
|
SamplingParams(
|
|
temperature=args.temperature,
|
|
top_p=args.top_p,
|
|
max_tokens=max(min(args.max_len - len(token_ids[problem]), args.max_tokens or args.max_len), 1),
|
|
ignore_eos=False,
|
|
seq_seed=_seed(problem, sample),
|
|
)
|
|
for problem, sample in plan
|
|
]
|
|
|
|
lens = [len(token_ids[problem]) for problem, _ in plan]
|
|
print(
|
|
f"[rank {dp_rank}] problems={len(needed)} requests={len(plan)} "
|
|
f"prompt_tokens[min/avg/max]={min(lens)}/{sum(lens) // len(lens)}/{max(lens)}"
|
|
+ (f" multimodal={len(plan)}" if has_images else ""),
|
|
flush=True,
|
|
)
|
|
|
|
if has_images:
|
|
outputs = llm.generate(
|
|
prompts=[texts[problem] for problem, _ in plan],
|
|
multi_modal_data=[dict(multi_modal[problem]) for problem, _ in plan],
|
|
sampling_params=sampling_params,
|
|
use_tqdm=False,
|
|
)
|
|
else:
|
|
outputs = llm.generate(
|
|
prompts=None,
|
|
prompt_token_ids=[token_ids[problem] for problem, _ in plan],
|
|
sampling_params=sampling_params,
|
|
use_tqdm=False,
|
|
)
|
|
|
|
rows = _rows_from_outputs(tokenizer, plan, records, outputs)
|
|
|
|
os.makedirs(OUT_DIR, exist_ok=True)
|
|
path = os.path.join(OUT_DIR, f"raw_rank{dp_rank:03d}.jsonl")
|
|
write_jsonl(path, rows)
|
|
print(f"[rank {dp_rank}] wrote {path} rows={len(rows)}", flush=True)
|
|
finally:
|
|
llm.close()
|
|
|
|
|
|
def main() -> int:
|
|
dp_size = max(args.ep // args.cp, 1) if args.dp is None else args.dp
|
|
os.makedirs(OUT_DIR, exist_ok=True)
|
|
env_overrides = dict(item.partition("=")[::2] for item in args.env)
|
|
|
|
meta = {
|
|
"bench": args.bench,
|
|
"model_name": args.model_name,
|
|
"ep": args.ep,
|
|
"cp": args.cp,
|
|
"dp": dp_size,
|
|
"data_file": args.data_file,
|
|
"limit": args.limit,
|
|
"n_sample": args.n_sample,
|
|
"temperature": args.temperature,
|
|
"top_p": args.top_p,
|
|
"reasoning_effort": REASONING_EFFORT,
|
|
"max_len": args.max_len,
|
|
"max_tokens": args.max_tokens,
|
|
"mtp": int(env_overrides.get("DEEPINFER_USE_NEXTN_DECODE", 5)),
|
|
"mtp_varlen": env_overrides.get("DEEPINFER_ENABLE_MTP_VARLEN", "1") != "0",
|
|
"batch_invariant": args.batch_invariant,
|
|
"decode_blacklist": args.decode_blacklist,
|
|
"max_num_seqs": args.max_num_seqs,
|
|
"max_num_batched_tokens": args.max_num_batched_tokens,
|
|
"env": args.env,
|
|
}
|
|
run_metadata = getattr(load_bench(args.bench), "run_metadata", None)
|
|
if run_metadata is not None:
|
|
meta.update(run_metadata({**os.environ, **env_overrides}))
|
|
|
|
for old in Path(OUT_DIR).glob("raw_rank*.jsonl"):
|
|
old.unlink()
|
|
with open(META_PATH, "w") as f:
|
|
json.dump(meta, f, indent=2, ensure_ascii=False)
|
|
|
|
print(f"REPO={REPO}\nOUT_DIR={OUT_DIR}\nmeta={json.dumps(meta, ensure_ascii=False)}", flush=True)
|
|
|
|
from examples._launch import launch_dp
|
|
|
|
launch_dp(run_rank, dp_size)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|