diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh index 59b55c90c..fe76dedf6 100644 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh +++ b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh @@ -12,6 +12,20 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME +PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) +GMU=0.90 +PREFILL_SCHEDULE_ARGS=() +if [ "${DP_ATTENTION:-false}" = "true" ]; then + PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") + GMU=0.80 + PREFILL_SCHEDULE_ARGS=(--prefill-schedule-interval 4) +fi + +EP_ARGS=() +if [ "${EP_SIZE:-1}" -gt 1 ]; then + EP_ARGS=(--enable-expert-parallel) +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi @@ -22,6 +36,7 @@ nvidia-smi export TORCH_CUDA_ARCH_LIST="10.0" export PYTHONNOUSERSITE=1 +export VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS="" SERVER_LOG=/workspace/server.log @@ -41,17 +56,21 @@ export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 set -x vllm serve $MODEL --host 0.0.0.0 --port $PORT \ ---tensor-parallel-size=$TP \ ---gpu-memory-utilization 0.90 \ +"${PARALLEL_ARGS[@]}" \ +"${EP_ARGS[@]}" \ +"${PREFILL_SCHEDULE_ARGS[@]}" \ +--gpu-memory-utilization "$GMU" \ --max-model-len $MAX_MODEL_LEN \ --max-num-seqs $CONC \ --reasoning-parser kimi_k2 \ --tool-call-parser kimi_k2 \ --compilation_config.pass_config.fuse_allreduce_rms true \ --kv-cache-dtype fp8 \ ---max-cudagraph-capture-size 2048 \ +--max-cudagraph-capture-size "$CONC" \ --max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ +--stream-interval 32 \ +--attention-config '{"mla_prefill_backend":"FLASHINFER","use_prefill_query_quantization":true}' \ +--no-enable-prefix-caching \ --trust-remote-code > $SERVER_LOG 2>&1 & SERVER_PID=$! diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a30af5964..c3a343dfa 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1508,7 +1508,7 @@ kimik2.5-int4-h200-vllm-agentic: # Kimi-K2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available. kimik2.5-fp4-b200-vllm: - image: vllm/vllm-openai:v0.22.0 + image: vllm/vllm-openai:nightly-e2fa28594f7baad142a426b0b6a2cfe2c79201c7 model: nvidia/Kimi-K2.5-NVFP4 model-prefix: kimik2.5 runner: b200 @@ -1520,8 +1520,9 @@ kimik2.5-fp4-b200-vllm: - isl: 8192 osl: 1024 search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 4 } - - { tp: 4, ep: 1, conc-start: 1, conc-end: 128 } + - { tp: 8, ep: 1, conc-start: 1, conc-end: 512 } + - { tp: 4, ep: 1, conc-start: 1, conc-end: 512 } + - { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 512 } # NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html # does not have a B300-specific recipe, so this config reuses the existing diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6eebd9707..e5015d557 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5355,3 +5355,9 @@ - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path" - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213 + +- config-keys: + - kimik2.5-fp4-b200-vllm + description: + - "Kimi K2.5 NVFP4 B200 vLLM: nightly image, extend sweep space with TP/DEP arms, DEP gmu 0.80" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2443