diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml deleted file mode 100644 index 5cb6c798b6..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml +++ /dev/null @@ -1,112 +0,0 @@ -name: kimi-k2.6-vllm-disagg-b200-1p1d-dep4-dep8-c1024 -model: - path: kimi-k2.6-nvfp4 - container: vllm/vllm-openai:v0.25.1 - precision: fp4 -dynamo: - wheel: 1.3.0.dev20260721 - install: true -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 -infra: - etcd_nats_dedicated_node: true -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: null - dp_launch_mode: per_gpu - prefill_environment: - PYTHONUNBUFFERED: '1' - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_NCCL_SYMM_MEM: '0' - NCCL_CUMEM_ENABLE: '1' - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' - NCCL_WATCHDOG_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_CUDA_IPC_ENABLE_MNNVL: n - UCX_MAX_RMA_RAILS: '1' - UCX_MAX_RNDV_RAILS: '1' - UCX_RNDV_SCHEME: put_zcopy - decode_environment: - PYTHONUNBUFFERED: '1' - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_NCCL_SYMM_MEM: '0' - NCCL_CUMEM_ENABLE: '1' - NCCL_WATCHDOG_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_CUDA_IPC_ENABLE_MNNVL: n - UCX_MAX_RMA_RAILS: '1' - UCX_MAX_RNDV_RAILS: '1' - UCX_RNDV_SCHEME: put_zcopy - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}' - served-model-name: nvidia/Kimi-K2.6-NVFP4 - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13346 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 2048 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 8192 - safetensors-load-strategy: prefetch - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - attention-backend: FLASHINFER_MLA - block-size: 128 - attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' - gpu-memory-utilization: 0.94 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: nvidia/Kimi-K2.6-NVFP4 - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 2048 - max-num-batched-tokens: 8192 - safetensors-load-strategy: prefetch - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - async-scheduling: true - attention-backend: FLASHINFER_MLA - block-size: 128 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 1024 -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '1024' - req_rate: inf - use_chat_template: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a30af59641..58c94561a9 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4981,7 +4981,7 @@ kimik2.6-fp4-b200-dynamo-vllm: image: vllm/vllm-openai:v0.25.1 model: nvidia/Kimi-K2.6-NVFP4 model-prefix: kimik2.6 - runner: b200-multinode + runner: b200-new precision: fp4 framework: dynamo-vllm router: { name: dynamo-router, version: "1.3.0.dev20260721" } @@ -5045,19 +5045,6 @@ kimik2.6-fp4-b200-dynamo-vllm: tp: 4 ep: 1 dp-attn: false - - conc-list: [1024] - prefill: - num-worker: 1 - tp: 1 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml" - decode: - num-worker: 1 - tp: 1 - ep: 8 - dp-attn: true - conc-list: [2048] prefill: num-worker: 1 diff --git a/configs/runners.yaml b/configs/runners.yaml index 5e992eb4e5..e6014c3dbe 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -80,6 +80,16 @@ labels: - b200-dgxc-slurm_7 - b200-dgxc-slurm_8 - b200-dgxc-slurm_9 + b200-new: + - b200-nscale-slurm_0 + - b200-nscale-slurm_1 + - b200-nscale-slurm_2 + - b200-nscale-slurm_3 + - b200-nscale-slurm_4 + - b200-nscale-slurm_5 + - b200-nscale-slurm_6 + - b200-nscale-slurm_7 + - b200-nscale-slurm_8 mi300x: - mi300x-amds_00 - mi300x-amds_01 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6eebd97070..f68e4635d3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5355,3 +5355,11 @@ - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path" - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213 + +- config-keys: + - kimik2.6-fp4-b200-dynamo-vllm + description: + - "Run the Kimi K2.6 NVFP4 B200 Dynamo vLLM disaggregated configuration on the b200-new runner." + - "Remove the 1P/1D DEP4-to-DEP8 concurrency-1024 configuration." + - "Image: vllm/vllm-openai:v0.25.1" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2438 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index be68ce073b..28a0667f68 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -1,8 +1,8 @@ #!/usr/bin/bash # System-specific configuration for B200 DGXC Slurm cluster -SLURM_PARTITION="gpu-2" -SLURM_ACCOUNT="benchmark" +SLURM_PARTITION="${SLURM_PARTITION:-gpu-2}" +SLURM_ACCOUNT="${SLURM_ACCOUNT:-benchmark}" set -x @@ -53,7 +53,7 @@ elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/lustre/fsw/models/Kimi-K2.5-NVFP4" export SRT_SLURM_MODEL_PREFIX="kimik2.5-fp4" elif [[ $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/lustre/fsw/models/Kimi-K2.6-NVFP4" + export MODEL_PATH="${MODEL_PATH:-/lustre/fsw/models/Kimi-K2.6-NVFP4}" export SRT_SLURM_MODEL_PREFIX="kimi-k2.6-nvfp4" elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/lustre/fsw/models/MiniMax-M2.5" @@ -213,7 +213,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then local lock_file="${lock_dir}/${image_key}.lock" ( - flock -w 600 9 || { echo "Failed to acquire lock for $squash_file" >&2; exit 1; } + flock -w "${B200_SQUASH_LOCK_TIMEOUT:-600}" 9 || { echo "Failed to acquire lock for $squash_file" >&2; exit 1; } if unsquashfs -l "$squash_file" > /dev/null 2>&1; then echo "Squash file already exists and is valid, skipping import: $squash_file" else diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh new file mode 100644 index 0000000000..de54528e11 --- /dev/null +++ b/runners/launch_b200-nscale-slurm.sh @@ -0,0 +1,14 @@ +#!/usr/bin/bash + +export SLURM_PARTITION="batch_1" +export SLURM_ACCOUNT="benchmark" +export MODEL_PATH="/scratch/models/Kimi-K2.6-NVFP4" + +export B200_SQUASH_DIR="/data/home/sa-shared/containers" +export B200_SQUASH_LOCK_TIMEOUT="3600" +mkdir -p "$B200_SQUASH_DIR" || { + echo "Unable to prepare nscale runner storage" >&2 + exit 1 +} + +exec bash "$(dirname "${BASH_SOURCE[0]}")/launch_b200-dgxc.sh" "$@"