model
source-documented, not executed source
#!/usr/bin/env bash
# DeepSeek-V4-Flash @ ~28 tok/s single-stream, 4-bit, coding quality intact.
# Box: 1x RTX 5090 (32GB, sm_120) + 2x Xeon Gold 6138 (Skylake-SP, 40 physical cores, AVX-512, NO AMX), 256GB DDR4-2400.
# Stack: stock kvcache-ai/ktransformers + its sglang (kt-kernel). See README for the one-time convert step.
set -euo pipefail
# --- edit these three paths ---
KT_DIR="${KT_DIR:-/path/to/ktransformers}" # built ktransformers checkout (contains .venv)
MODEL="${MODEL:-/path/to/dsv4_flash}" # source MIT weights: deepseek-ai/DeepSeek-V4-Flash
KT_WEIGHTS="${KT_WEIGHTS:-/path/to/dsv4_flash_rawint4}" # int4 numa-sharded output of convert_cpu_weights_ds4.py
# ------------------------------
cd "$KT_DIR"
export CUDA_HOME=/usr/local/cuda
export PATH="$CUDA_HOME/bin:$PWD/.venv/bin:$PATH"
export FLASHINFER_CUDA_ARCH_LIST=12.0a # RTX 5090 = sm_120
export TORCH_CUDA_ARCH_LIST="12.0+PTX"
export SGLANG_DSV4_MODE=2604
export SGLANG_DSV4_2604_SUBMODE=2604A # measured config (upstream docs default to 2604B; both work)
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
exec ./.venv/bin/python -m sglang.launch_server \
--host 127.0.0.1 --port 30000 \
--model "$MODEL" --kt-weight-path "$KT_WEIGHTS" --kt-method AMXINT4 \
--kt-num-gpu-experts 0 --kt-cpuinfer 40 --kt-threadpool-count 2 \
--kt-gpu-prefill-token-threshold 4096 --kt-enable-dynamic-expert-update \
--kt-max-deferred-experts-per-token 3 --kt-expert-placement-strategy frequency \
--tensor-parallel-size 1 --context-length 8192 --attention-backend flashinfer \
--mem-fraction-static 0.70 --chunked-prefill-size 2048 --max-prefill-tokens 2048 \
--max-running-requests 2 --watchdog-timeout 1200 --disable-shared-experts-fusion \
--trust-remote-code --json-model-override-args '{"num_experts_per_tok": 4}' \
--cuda-graph-bs 1 --cuda-graph-max-bs 1 --disable-radix-cache --skip-server-warmup \
2>&1 | tee /tmp/kt_server.log
# Notes:
# --kt-cpuinfer 40 = ONE thread per PHYSICAL core. Do NOT use hyperthreads (80 -> ~2.7 tok/s, 10x worse).
# --kt-num-gpu-experts 0 = CPU-only experts. Adding GPU experts under AMXINT4 was net-negative here (27.5 < 28).
# num_experts_per_tok: 4 = top-4 adaptive-k (from top-6). Coding-verified; top-3 and below break coding.
# No EAGLE/spec-decode: net-negative on this CPU-bandwidth-bound box (27.2 < 28).