feat(sm120): offline inference on NVIDIA Blackwell SM120

Run DeepSeek-V4.1-Flash offline inference on SM120 GPUs (RTX PRO 6000
Blackwell) with the same entry points as SM100. SM100 and Ascend paths
are unchanged.

- Build: SM120 is detected from the first visible GPU
  (scripts/nvidia_arch.sh). It builds DeepEP and DeepGEMM from its nv_dev
  branch (vendor/DeepGEMM-sm120), and no FlashMLA or DeepSelect;
  FlashInfer is installed from a pinned commit
  (requirements-nvidia-sm120.txt).
- Attention: V4.1 sparse MLA runs on FlashInfer's SM120 sparse MLA
  kernels, reading the inline-scale KV records as stored; KV blocks are
  aligned to 512 bytes.
- Indexer: FlashInfer's fused DSA indexer scores the index keys and
  selects their top-k in one call, for decode on every indexer layer
  (candidate source and consumer layers included) and for prefill outside
  the candidate layers. The remaining selections (prefill of the
  candidate layers, the sampler) use FlashInfer's top-k behind
  DeepSelect's interface (model_executor/layers/topk.py), in
  deterministic mode.
- MoE EP: DeepEP over NCCL GIN by default, or NCCL collectives
  (DEEPINFER_SM120_EP_BACKEND=nccl); NCCL settings known to break the EP
  paths are rejected. EPLB, including decode-only draft-layer EPLB, and
  per-layer YoCo MoE dispatch.
- Engram tables live in page-locked host shared memory; killed workers
  leave nothing in /dev/shm.
- Features: attention DP + MoE EP, DSpark/nextn speculative decoding,
  CUDA graphs, async scheduling, decoder SWA bounded replay, structured
  outputs, image inputs, multi-node deployments. Features without an SM120
  implementation (CP > 1, MTP varlen, batch-invariant mode, GPU-resident
  Engram tables) are refused at engine start.
- Eval: chat GSM8K and CV-Bench; tests for the SM120 kernels and their
  DeepInfer call sites.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Zihua WuandClaude Opus 5.5 committed 2026-11-17 18:32:01 -08:00
1 parent 553741557a
commit efecbb5ba6
68 files changed
+8049 -120

No files matched your search

+17 -4
View File
@@ -7,8 +7,15 @@ VENDOR_DIR="${REPO_ROOT}/vendor"
case "${DEEPINFER_ACCELERATOR:-nvidia}" in
nvidia)
BUILD_TARGET=CUDA
PACKAGES=(DeepSelect FlashMLA DeepEP DeepGEMM)
INSTALL=(deep_select flash_mla deep_ep deep_gemm)
NVIDIA_ARCH="$(bash "${REPO_ROOT}/scripts/nvidia_arch.sh")"
if [ "${NVIDIA_ARCH}" = sm120 ]; then
# SM120 selects top-k with FlashInfer (see deepinfer/model_executor/layers/topk.py), so it builds no DeepSelect.
PACKAGES=(DeepEP DeepGEMM-sm120)
INSTALL=(deep_ep deep_gemm)
else
PACKAGES=(DeepSelect FlashMLA DeepEP DeepGEMM)
INSTALL=(deep_select flash_mla deep_ep deep_gemm)
fi
;;
ascend)
BUILD_TARGET=ASCEND
@@ -22,6 +29,7 @@ case "${DEEPINFER_ACCELERATOR:-nvidia}" in
esac
PLATFORM_TAG="${DEEPINFER_ACCELERATOR:-nvidia}-$(uname -m)"
[ "${NVIDIA_ARCH:-}" = sm120 ] && PLATFORM_TAG="nvidia-sm120-$(uname -m)"
WHEEL_DIR="${WHEEL_DIR:-${REPO_ROOT}/build/${PLATFORM_TAG}/vendor-wheels}"
echo "📥 Initializing submodules ..."
@@ -36,12 +44,17 @@ target_var() {
}
mkdir -p "${WHEEL_DIR}"
for name in "${INSTALL[@]}"; do
rm -f "${WHEEL_DIR}/${name}-"*.whl
done
for pkg in "${PACKAGES[@]}"; do
echo "🔧 Building ${pkg} (${BUILD_TARGET}) ..."
envs=()
var="$(target_var "${pkg}")"
[ -n "${var}" ] && envs+=("${var}=${BUILD_TARGET}")
[ "${pkg}" = DeepGEMM ] && envs+=(DG_FORCE_BUILD=1)
case "${pkg}" in
DeepGEMM | DeepGEMM-sm120) envs+=(DG_FORCE_BUILD=1) ;;
esac
(
cd "${VENDOR_DIR}/${pkg}"
git submodule update --init --recursive
@@ -50,5 +63,5 @@ for pkg in "${PACKAGES[@]}"; do
done
echo "📦 Installing wheels from ${WHEEL_DIR} ..."
python -m pip install --no-deps --no-index --find-links "${WHEEL_DIR}" "${INSTALL[@]}"
python -m pip install --no-deps --no-index --force-reinstall --find-links "${WHEEL_DIR}" "${INSTALL[@]}"
echo "✅ vendor operators installed"
+38
View File
@@ -0,0 +1,38 @@
#!/bin/bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Prints the NVIDIA GPU family to build for: sm100 or sm120.
# DEEPINFER_NVIDIA_ARCH overrides detection; without it the first visible GPU decides (the first
# CUDA_VISIBLE_DEVICES entry when it is set), and hosts without nvidia-smi or a visible GPU build for sm100.
set -euo pipefail
arch="${DEEPINFER_NVIDIA_ARCH:-}"
if [ -z "${arch}" ]; then
capability=""
device="0"
if [ -n "${CUDA_VISIBLE_DEVICES+set}" ]; then
device="$(printf '%s' "${CUDA_VISIBLE_DEVICES%%,*}" | tr -d '[:space:]')"
fi
case "${device}" in
"" | -*) ;;
*)
if command -v nvidia-smi >/dev/null 2>&1; then
capability="$(nvidia-smi -i "${device}" --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -n 1 | tr -d '[:space:]' || true)"
fi
;;
esac
case "${capability}" in
12.*) arch=sm120 ;;
*) arch=sm100 ;;
esac
fi
case "${arch}" in
sm100|sm120) echo "${arch}" ;;
*)
echo "Unsupported DEEPINFER_NVIDIA_ARCH: '${arch}' (expect sm100 or sm120)" >&2
exit 1
;;
esac