From 2567e886ef5de41240704d77011cbaa23c76f872 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Thu, 30 Jul 2026 16:25:31 -0400 Subject: [PATCH 01/13] feat: add MiniMax M3 MXFP8 GB200 Dynamo-vLLM EAGLE3 MTP recipes and fix non-MTP comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add minimaxm3-fp8-gb200-dynamo-vllm-mtp: two EAGLE3 SLURM recipes (1p2d-dep4-tep8 conc 16/64/128, 2p1d-dep4-dep16 conc 512/1024) under minimax-m3-gb200-fp8/8k1k/mtp/; launcher already cp-rT's the whole minimax-m3-gb200-fp8 tree so no runner changes needed - Fix minimaxm3-fp8-gb200-dynamo-vllm: correct 4 wrong comments (Xp2d DEP8 -> Xp1d DEP16), remove phantom Marlin entry from header, bump router version 1.3.0.dev20260614 -> 1.3.0.dev20260710 中文：新增 MiniMax M3 MXFP8 GB200 Dynamo-vLLM EAGLE3 投机解码配方并修正 非 MTP 配置注释 - 新增 minimaxm3-fp8-gb200-dynamo-vllm-mtp：两个 EAGLE3 SLURM 配方 （1p2d-dep4-tep8 并发 16/64/128，2p1d-dep4-dep16 并发 512/1024）， 位于 minimax-m3-gb200-fp8/8k1k/mtp/；启动器已通过 cp-rT 复制整个 minimax-m3-gb200-fp8 目录，无需修改 runner 脚本 - 修正 minimaxm3-fp8-gb200-dynamo-vllm：修正 4 处错误注释 （Xp2d DEP8 -> Xp1d DEP16），移除头注释中不存在的 Marlin 条目， 将 router 版本从 1.3.0.dev20260614 升级至 1.3.0.dev20260710 Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml | 104 +++++++++++++++++ .../8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml | 106 ++++++++++++++++++ configs/nvidia-master.yaml | 59 +++++++++- perf-changelog.yaml | 7 ++ 4 files changed, 270 insertions(+), 6 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml new file mode 100644 index 0000000000..3cc83e86ae --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml @@ -0,0 +1,104 @@ +name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep8-fp8-8k1k-eagle3" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + +dynamo: + install: true + version: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 4 + prefill_workers: 1 + decode_workers: 2 + gpus_per_prefill: 4 + gpus_per_decode: 8 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "16x64x128" + req_rate: "inf" + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml new file mode 100644 index 0000000000..1546180747 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml @@ -0,0 +1,106 @@ +name: "minimax-m3-vllm-disagg-gb200-2p1d-dep4-dep16-fp8-8k1k-eagle3" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + +dynamo: + install: true + version: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 4 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 16 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512x1024" + req_rate: "inf" + random_range_ratio: 0.8 + use_chat_template: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a30af59641..d974d3fb62 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8060,7 +8060,7 @@ qwen3.5-fp4-b200-trt: - { tp: 8, ep: 8, dp-attn: true, conc-list: [256, 512, 1024] } # MiniMax-M3 GB200 disagg sweep — adapted from NV B300 PR #1863. -# All prefill DEP4 (TP1 DP4 EP, 4 GPU/worker). Decode: Marlin, TEP8, DEP8, TEP4. +# All prefill DEP4 (TP1 DP4 EP, 4 GPU/worker). Decode: TEP8, DEP16, TEP4. # 4 GPU/node (GB200 NVL72). FLASHINFER attention with FP8 KV cache. minimaxm3-fp8-gb200-dynamo-vllm: image: vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 @@ -8069,7 +8069,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: runner: gb200 precision: fp8 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260614" } + router: { name: dynamo-router, version: "1.3.0.dev20260710" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -8078,7 +8078,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: - isl: 8192 osl: 1024 search-space: - # 1p2d DEP4+DEP8, 5n: conc 512 + # 1p1d DEP4+DEP16, 5n: conc 512 - conc-list: [512] prefill: num-worker: 1 @@ -8123,7 +8123,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 8 dp-attn: false - # 2p2d DEP4+DEP8, 6n: conc 512,1024 + # 2p1d DEP4+DEP16, 6n: conc 512,1024 - conc-list: [512, 1024] prefill: num-worker: 2 @@ -8138,7 +8138,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 16 dp-attn: true - # 3p2d DEP4+DEP8, 7n: conc 4096 + # 3p1d DEP4+DEP16, 7n: conc 4096 - conc-list: [4096] prefill: num-worker: 3 @@ -8153,7 +8153,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 16 dp-attn: true - # 5p2d DEP4+DEP8, 9n: conc 4096 + # 5p1d DEP4+DEP16, 9n: conc 4096 - conc-list: [4096] prefill: num-worker: 5 @@ -8168,6 +8168,53 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 16 dp-attn: true +minimaxm3-fp8-gb200-dynamo-vllm-mtp: + image: vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 + model: MiniMaxAI/MiniMax-M3-MXFP8 + model-prefix: minimaxm3 + runner: gb200 + precision: fp8 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260710" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # 1p2d DEP4+TEP8, 5n: conc 16,64,128 MTP + - spec-decoding: "mtp" + conc-list: [16, 64, 128] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml" + decode: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: false + # 2p1d DEP4+DEP16, 6n: conc 512,1024 MTP + - spec-decoding: "mtp" + conc-list: [512, 1024] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + qwen3.5-fp4-b200-trt-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18 model: nvidia/Qwen3.5-397B-A17B-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6eebd97070..c5fead94a4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5355,3 +5355,10 @@ - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path" - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213 + +- config-keys: + - minimaxm3-fp8-gb200-dynamo-vllm-mtp + description: + - "Add MiniMax M3 MXFP8 GB200 Dynamo-vLLM disaggregated EAGLE3 MTP recipes" + - "Fix comment errors in minimaxm3-fp8-gb200-dynamo-vllm (Xp2d DEP8 -> Xp1d DEP16, remove phantom Marlin entry, bump router to 1.3.0.dev20260710)" + pr-link: TBD From 08509712f93bc8d0e801018e8f36dede1b7670cf Mon Sep 17 00:00:00 2001 From: Xin Li Date: Thu, 30 Jul 2026 16:26:16 -0400 Subject: [PATCH 02/13] chore: fill perf-changelog pr-link for minimaxm3-fp8-gb200-dynamo-vllm-mtp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：补全 minimaxm3-fp8-gb200-dynamo-vllm-mtp 的 perf-changelog pr-link Co-Authored-By: Claude Sonnet 4.6 (1M context) --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c5fead94a4..d3544cb162 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5361,4 +5361,4 @@ description: - "Add MiniMax M3 MXFP8 GB200 Dynamo-vLLM disaggregated EAGLE3 MTP recipes" - "Fix comment errors in minimaxm3-fp8-gb200-dynamo-vllm (Xp2d DEP8 -> Xp1d DEP16, remove phantom Marlin entry, bump router to 1.3.0.dev20260710)" - pr-link: TBD + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2432 From c1ef64f760e9838af03e5a326ffd04117dea7480 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Thu, 30 Jul 2026 18:39:29 -0400 Subject: [PATCH 03/13] fix: change GB200 fp8 MTP prefill from TP1 DP4 to TP2 DP2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both EAGLE3 MTP recipe YAMLs and master-config tp/ep updated. 中文：将 GB200 fp8 MTP 预填充并行从 TP1 DP4 改为 TP2 DP2， 同步更新两个 EAGLE3 MTP 配方 YAML 及主配置 tp/ep 字段 Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml | 4 ++-- .../8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml | 4 ++-- configs/nvidia-master.yaml | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml index 3cc83e86ae..7498395b18 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml @@ -60,8 +60,8 @@ backend: prefill: kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 + tensor-parallel-size: 2 + data-parallel-size: 2 data-parallel-rpc-port: 13345 enable-expert-parallel: true trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml index 1546180747..55ee65a05d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml @@ -60,8 +60,8 @@ backend: prefill: kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 + tensor-parallel-size: 2 + data-parallel-size: 2 data-parallel-rpc-port: 13345 enable-expert-parallel: true trust-remote-code: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index d974d3fb62..2bcc939d8d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8184,13 +8184,13 @@ minimaxm3-fp8-gb200-dynamo-vllm-mtp: - isl: 8192 osl: 1024 search-space: - # 1p2d DEP4+TEP8, 5n: conc 16,64,128 MTP + # 1p2d TP2DP2+TEP8, 5n: conc 16,64,128 MTP - spec-decoding: "mtp" conc-list: [16, 64, 128] prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 2 + ep: 2 dp-attn: true additional-settings: - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml" @@ -8199,13 +8199,13 @@ minimaxm3-fp8-gb200-dynamo-vllm-mtp: tp: 8 ep: 8 dp-attn: false - # 2p1d DEP4+DEP16, 6n: conc 512,1024 MTP + # 2p1d TP2DP2+DEP16, 6n: conc 512,1024 MTP - spec-decoding: "mtp" conc-list: [512, 1024] prefill: num-worker: 2 - tp: 4 - ep: 4 + tp: 2 + ep: 2 dp-attn: true additional-settings: - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml" From a36159fc0519e2b06fab2602081b8017a9b96ad4 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Thu, 30 Jul 2026 19:25:56 -0400 Subject: [PATCH 04/13] fix: update GB200 fp8 MTP image to vllm/vllm-openai:v0.26.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：将 GB200 fp8 MTP 镜像更新为 vllm/vllm-openai:v0.26.0 Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml | 2 +- .../8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml | 2 +- configs/nvidia-master.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml index 7498395b18..fb3b40fd53 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml @@ -2,7 +2,7 @@ name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep8-fp8-8k1k-eagle3" model: path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + container: "vllm/vllm-openai:v0.26.0" precision: "fp8" dynamo: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml index 55ee65a05d..97e85cf969 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml @@ -2,7 +2,7 @@ name: "minimax-m3-vllm-disagg-gb200-2p1d-dep4-dep16-fp8-8k1k-eagle3" model: path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + container: "vllm/vllm-openai:v0.26.0" precision: "fp8" dynamo: diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 2bcc939d8d..a839325f2e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8169,7 +8169,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: dp-attn: true minimaxm3-fp8-gb200-dynamo-vllm-mtp: - image: vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 + image: vllm/vllm-openai:v0.26.0 model: MiniMaxAI/MiniMax-M3-MXFP8 model-prefix: minimaxm3 runner: gb200 From 34fec4305d2d2e86b1d26396b050cd42017cb595 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Fri, 31 Jul 2026 00:05:15 -0400 Subject: [PATCH 05/13] fix: update GB200 fp8 MTP image to nightly-6f91edf --- .../8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml | 2 +- .../8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml | 2 +- configs/nvidia-master.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml index fb3b40fd53..186e9e3c63 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml @@ -2,7 +2,7 @@ name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep8-fp8-8k1k-eagle3" model: path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:v0.26.0" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" precision: "fp8" dynamo: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml index 97e85cf969..7be9165961 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml @@ -2,7 +2,7 @@ name: "minimax-m3-vllm-disagg-gb200-2p1d-dep4-dep16-fp8-8k1k-eagle3" model: path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:v0.26.0" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" precision: "fp8" dynamo: diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a839325f2e..a2e05234b7 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8169,7 +8169,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: dp-attn: true minimaxm3-fp8-gb200-dynamo-vllm-mtp: - image: vllm/vllm-openai:v0.26.0 + image: vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de model: MiniMaxAI/MiniMax-M3-MXFP8 model-prefix: minimaxm3 runner: gb200 From 7db4551259a0ff65a6838ea3cf30531f4d1d9f46 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sat, 1 Aug 2026 01:27:31 -0400 Subject: [PATCH 06/13] redesign GB200 MTP sweep: 1P DEP4 prefill, 1D x TP4/TP8/DEP4/DEP8/DEP16/DEP32 decode, conc 1-1024 --- ....yaml => 1p1d-dep4-dep16-eagle3-8k1k.yaml} | 14 +-- .../8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml | 106 ++++++++++++++++++ .../8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml | 106 ++++++++++++++++++ .../8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml | 106 ++++++++++++++++++ .../8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml | 104 +++++++++++++++++ ...1k.yaml => 1p1d-dep4-tp8-eagle3-8k1k.yaml} | 14 +-- configs/nvidia-master.yaml | 79 ++++++++++--- 7 files changed, 502 insertions(+), 27 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/{2p1d-dep4-dep16-eagle3-8k1k.yaml => 1p1d-dep4-dep16-eagle3-8k1k.yaml} (91%) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml rename benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/{1p2d-dep4-tep8-eagle3-8k1k.yaml => 1p1d-dep4-tp8-eagle3-8k1k.yaml} (91%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml similarity index 91% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml index 7be9165961..a6e771274e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml @@ -1,4 +1,4 @@ -name: "minimax-m3-vllm-disagg-gb200-2p1d-dep4-dep16-fp8-8k1k-eagle3" +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep16-fp8-8k1k-eagle3" model: path: "minimax-m3-mxfp8" @@ -23,9 +23,9 @@ srun_options: resources: gpu_type: "gb200" gpus_per_node: 4 - prefill_nodes: 2 + prefill_nodes: 1 decode_nodes: 4 - prefill_workers: 2 + prefill_workers: 1 decode_workers: 1 gpus_per_prefill: 4 gpus_per_decode: 16 @@ -60,8 +60,8 @@ backend: prefill: kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 2 - data-parallel-size: 2 + tensor-parallel-size: 1 + data-parallel-size: 4 data-parallel-rpc-port: 13345 enable-expert-parallel: true trust-remote-code: true @@ -72,7 +72,7 @@ backend: language-model-only: true kv-cache-dtype: fp8 speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - stream-interval: 100 + stream-interval: 32 max-cudagraph-capture-size: 2048 max-num-batched-tokens: 16384 @@ -100,7 +100,7 @@ benchmark: type: "sa-bench" isl: 8192 osl: 1024 - concurrencies: "512x1024" + concurrencies: "1x2x4x8x16x32x64x128x256x512x1024" req_rate: "inf" random_range_ratio: 0.8 use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml new file mode 100644 index 0000000000..78eab6394e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml @@ -0,0 +1,106 @@ +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep32-fp8-8k1k-eagle3" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" + precision: "fp8" + +dynamo: + install: true + version: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 8 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 32 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64x128x256x512x1024" + req_rate: "inf" + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml new file mode 100644 index 0000000000..849eed6b75 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml @@ -0,0 +1,106 @@ +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep4-fp8-8k1k-eagle3" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" + precision: "fp8" + +dynamo: + install: true + version: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64x128x256x512x1024" + req_rate: "inf" + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml new file mode 100644 index 0000000000..8f41b6b052 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml @@ -0,0 +1,106 @@ +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep8-fp8-8k1k-eagle3" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" + precision: "fp8" + +dynamo: + install: true + version: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64x128x256x512x1024" + req_rate: "inf" + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml new file mode 100644 index 0000000000..a582ac07f4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml @@ -0,0 +1,104 @@ +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-tp4-fp8-8k1k-eagle3" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-6f91edf96d3f3272945809c04702380053bff4de" + precision: "fp8" + +dynamo: + install: true + version: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 4 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64x128x256x512x1024" + req_rate: "inf" + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml similarity index 91% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml index 186e9e3c63..bc57a8afd1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml @@ -1,4 +1,4 @@ -name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep8-fp8-8k1k-eagle3" +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-tp8-fp8-8k1k-eagle3" model: path: "minimax-m3-mxfp8" @@ -24,9 +24,9 @@ resources: gpu_type: "gb200" gpus_per_node: 4 prefill_nodes: 1 - decode_nodes: 4 + decode_nodes: 2 prefill_workers: 1 - decode_workers: 2 + decode_workers: 1 gpus_per_prefill: 4 gpus_per_decode: 8 @@ -60,8 +60,8 @@ backend: prefill: kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 2 - data-parallel-size: 2 + tensor-parallel-size: 1 + data-parallel-size: 4 data-parallel-rpc-port: 13345 enable-expert-parallel: true trust-remote-code: true @@ -92,13 +92,13 @@ backend: stream-interval: 32 max-num-seqs: 1024 max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 + max-cudagraph-capture-size: 2048 benchmark: type: "sa-bench" isl: 8192 osl: 1024 - concurrencies: "16x64x128" + concurrencies: "1x2x4x8x16x32x64x128x256x512x1024" req_rate: "inf" random_range_ratio: 0.8 use_chat_template: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a2e05234b7..e9efa14b79 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8184,37 +8184,90 @@ minimaxm3-fp8-gb200-dynamo-vllm-mtp: - isl: 8192 osl: 1024 search-space: - # 1p2d TP2DP2+TEP8, 5n: conc 16,64,128 MTP - spec-decoding: "mtp" - conc-list: [16, 64, 128] + conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-tep8-eagle3-8k1k.yaml" + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml" decode: - num-worker: 2 + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml" + decode: + num-worker: 1 tp: 8 ep: 8 dp-attn: false - # 2p1d TP2DP2+DEP16, 6n: conc 512,1024 MTP - spec-decoding: "mtp" - conc-list: [512, 1024] + conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] prefill: - num-worker: 2 - tp: 2 - ep: 2 + num-worker: 1 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/2p1d-dep4-dep16-eagle3-8k1k.yaml" + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true qwen3.5-fp4-b200-trt-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18 model: nvidia/Qwen3.5-397B-A17B-NVFP4 From 76f1344569f56d19b8a0022a0d730aa4580090e4 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sat, 1 Aug 2026 22:52:54 -0400 Subject: [PATCH 07/13] bump dynamo to 1.3.0.dev20260713 --- .../8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml | 2 +- .../8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml | 2 +- .../8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml | 2 +- .../8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml | 2 +- .../8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml | 2 +- .../8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml | 2 +- configs/nvidia-master.yaml | 10 +++++----- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml index a6e771274e..5c8d33460e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml @@ -7,7 +7,7 @@ model: dynamo: install: true - version: 1.3.0.dev20260710 + version: 1.3.0.dev20260713 health_check: max_attempts: 720 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml index 78eab6394e..d3b28008bb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml @@ -7,7 +7,7 @@ model: dynamo: install: true - version: 1.3.0.dev20260710 + version: 1.3.0.dev20260713 health_check: max_attempts: 720 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml index 849eed6b75..ab196c7fb3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml @@ -7,7 +7,7 @@ model: dynamo: install: true - version: 1.3.0.dev20260710 + version: 1.3.0.dev20260713 health_check: max_attempts: 720 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml index 8f41b6b052..ae717008f1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml @@ -7,7 +7,7 @@ model: dynamo: install: true - version: 1.3.0.dev20260710 + version: 1.3.0.dev20260713 health_check: max_attempts: 720 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml index a582ac07f4..ed0014ce25 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml @@ -7,7 +7,7 @@ model: dynamo: install: true - version: 1.3.0.dev20260710 + version: 1.3.0.dev20260713 health_check: max_attempts: 720 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml index bc57a8afd1..7ba5ba25a6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml @@ -7,7 +7,7 @@ model: dynamo: install: true - version: 1.3.0.dev20260710 + version: 1.3.0.dev20260713 health_check: max_attempts: 720 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index e9efa14b79..ff91958ed1 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7393,7 +7393,7 @@ minimaxm3-fp4-b200-dynamo-vllm: runner: b200-multinode precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260710" } + router: { name: dynamo-router, version: "1.3.0.dev20260713" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -7774,7 +7774,7 @@ minimaxm3-fp4-b300-dynamo-vllm-mtp: runner: b300 precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260710" } + router: { name: dynamo-router, version: "1.3.0.dev20260713" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -7862,7 +7862,7 @@ minimaxm3-fp4-b300-dynamo-vllm-mtp-legacy-dep4: runner: b300 precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260710" } + router: { name: dynamo-router, version: "1.3.0.dev20260713" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -8069,7 +8069,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: runner: gb200 precision: fp8 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260710" } + router: { name: dynamo-router, version: "1.3.0.dev20260713" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -8175,7 +8175,7 @@ minimaxm3-fp8-gb200-dynamo-vllm-mtp: runner: gb200 precision: fp8 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260710" } + router: { name: dynamo-router, version: "1.3.0.dev20260713" } kv-p2p-transfer: nixl multinode: true disagg: true From 8f0159ada81ec5f66491591d6053abe082d4b482 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sat, 1 Aug 2026 23:29:20 -0400 Subject: [PATCH 08/13] bump prefill gpu-memory-utilization to 0.95 in GB200 MTP recipes --- .../8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml | 2 +- .../8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml | 2 +- .../8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml | 2 +- .../8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml | 2 +- .../8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml | 2 +- .../8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml index 5c8d33460e..6a13882ef5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml @@ -67,7 +67,7 @@ backend: trust-remote-code: true no-enable-prefix-caching: true block-size: 128 - gpu-memory-utilization: 0.90 + gpu-memory-utilization: 0.95 max-model-len: 9472 language-model-only: true kv-cache-dtype: fp8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml index d3b28008bb..948291a3f9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml @@ -67,7 +67,7 @@ backend: trust-remote-code: true no-enable-prefix-caching: true block-size: 128 - gpu-memory-utilization: 0.90 + gpu-memory-utilization: 0.95 max-model-len: 9472 language-model-only: true kv-cache-dtype: fp8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml index ab196c7fb3..73ef79d9cb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml @@ -67,7 +67,7 @@ backend: trust-remote-code: true no-enable-prefix-caching: true block-size: 128 - gpu-memory-utilization: 0.90 + gpu-memory-utilization: 0.95 max-model-len: 9472 language-model-only: true kv-cache-dtype: fp8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml index ae717008f1..eff77b14fa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml @@ -67,7 +67,7 @@ backend: trust-remote-code: true no-enable-prefix-caching: true block-size: 128 - gpu-memory-utilization: 0.90 + gpu-memory-utilization: 0.95 max-model-len: 9472 language-model-only: true kv-cache-dtype: fp8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml index ed0014ce25..f64f3ed48b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml @@ -67,7 +67,7 @@ backend: trust-remote-code: true no-enable-prefix-caching: true block-size: 128 - gpu-memory-utilization: 0.90 + gpu-memory-utilization: 0.95 max-model-len: 9472 language-model-only: true kv-cache-dtype: fp8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml index 7ba5ba25a6..083efa50f3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml @@ -67,7 +67,7 @@ backend: trust-remote-code: true no-enable-prefix-caching: true block-size: 128 - gpu-memory-utilization: 0.90 + gpu-memory-utilization: 0.95 max-model-len: 9472 language-model-only: true kv-cache-dtype: fp8 From fde999851ee0ea251bee449ab42a4ecb1f9671f6 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sat, 1 Aug 2026 23:47:18 -0400 Subject: [PATCH 09/13] add max-num-seqs: 256 to GB200 MTP prefill to free KV cache memory --- .../8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml | 1 + .../8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml | 1 + .../8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml | 1 + .../8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml | 1 + .../minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml | 1 + .../minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml | 1 + 6 files changed, 6 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml index 6a13882ef5..3d9adbb7e6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml @@ -75,6 +75,7 @@ backend: stream-interval: 32 max-cudagraph-capture-size: 2048 max-num-batched-tokens: 16384 + max-num-seqs: 256 decode: kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml index 948291a3f9..b8d801d711 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml @@ -75,6 +75,7 @@ backend: stream-interval: 32 max-cudagraph-capture-size: 2048 max-num-batched-tokens: 16384 + max-num-seqs: 256 decode: kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml index 73ef79d9cb..d36db794b7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml @@ -75,6 +75,7 @@ backend: stream-interval: 32 max-cudagraph-capture-size: 2048 max-num-batched-tokens: 16384 + max-num-seqs: 256 decode: kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml index eff77b14fa..51bb019d58 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml @@ -75,6 +75,7 @@ backend: stream-interval: 32 max-cudagraph-capture-size: 2048 max-num-batched-tokens: 16384 + max-num-seqs: 256 decode: kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml index f64f3ed48b..02bcca5a64 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml @@ -75,6 +75,7 @@ backend: stream-interval: 32 max-cudagraph-capture-size: 2048 max-num-batched-tokens: 16384 + max-num-seqs: 256 decode: kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml index 083efa50f3..f2c81d44cd 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml @@ -75,6 +75,7 @@ backend: stream-interval: 32 max-cudagraph-capture-size: 2048 max-num-batched-tokens: 16384 + max-num-seqs: 256 decode: kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' From 5419197e174f5ceae57f07cfa46a8b66c87ae95c Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sun, 2 Aug 2026 00:54:16 -0400 Subject: [PATCH 10/13] enforce-eager on prefill to free CUDA graph memory --- .../8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml | 1 + .../8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml | 1 + .../8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml | 1 + .../8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml | 1 + .../minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml | 1 + .../minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml | 1 + 6 files changed, 6 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml index 3d9adbb7e6..e198f930aa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml @@ -71,6 +71,7 @@ backend: max-model-len: 9472 language-model-only: true kv-cache-dtype: fp8 + enforce-eager: true speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 max-cudagraph-capture-size: 2048 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml index b8d801d711..4304016f6e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml @@ -71,6 +71,7 @@ backend: max-model-len: 9472 language-model-only: true kv-cache-dtype: fp8 + enforce-eager: true speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 max-cudagraph-capture-size: 2048 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml index d36db794b7..ba55613448 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml @@ -71,6 +71,7 @@ backend: max-model-len: 9472 language-model-only: true kv-cache-dtype: fp8 + enforce-eager: true speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 max-cudagraph-capture-size: 2048 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml index 51bb019d58..2ea133bc80 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml @@ -71,6 +71,7 @@ backend: max-model-len: 9472 language-model-only: true kv-cache-dtype: fp8 + enforce-eager: true speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 max-cudagraph-capture-size: 2048 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml index 02bcca5a64..38112658b3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml @@ -71,6 +71,7 @@ backend: max-model-len: 9472 language-model-only: true kv-cache-dtype: fp8 + enforce-eager: true speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 max-cudagraph-capture-size: 2048 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml index f2c81d44cd..4c13aa5221 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml @@ -71,6 +71,7 @@ backend: max-model-len: 9472 language-model-only: true kv-cache-dtype: fp8 + enforce-eager: true speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 max-cudagraph-capture-size: 2048 From d3a123203e213358d4b00b5ef95bb3300f68edf6 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sun, 2 Aug 2026 12:16:15 -0400 Subject: [PATCH 11/13] set decode max-num-seqs and max-cudagraph = conc/total_decode_gpus --- .../8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml | 6 +++--- .../8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml | 6 +++--- .../8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml | 6 +++--- .../8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml | 6 +++--- .../8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml | 6 +++--- .../8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml | 6 +++--- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml index e198f930aa..2e9ca5f698 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml @@ -74,7 +74,7 @@ backend: enforce-eager: true speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 - max-cudagraph-capture-size: 2048 + max-cudagraph-capture-size: 64 max-num-batched-tokens: 16384 max-num-seqs: 256 @@ -94,9 +94,9 @@ backend: kv-cache-dtype: fp8 speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 100 - max-num-seqs: 1024 + max-num-seqs: 64 max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 + max-cudagraph-capture-size: 64 benchmark: type: "sa-bench" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml index 4304016f6e..7d26772cd2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml @@ -74,7 +74,7 @@ backend: enforce-eager: true speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 - max-cudagraph-capture-size: 2048 + max-cudagraph-capture-size: 32 max-num-batched-tokens: 16384 max-num-seqs: 256 @@ -94,9 +94,9 @@ backend: kv-cache-dtype: fp8 speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 100 - max-num-seqs: 1024 + max-num-seqs: 32 max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 + max-cudagraph-capture-size: 32 benchmark: type: "sa-bench" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml index ba55613448..59ff9cc0c4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml @@ -74,7 +74,7 @@ backend: enforce-eager: true speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 - max-cudagraph-capture-size: 2048 + max-cudagraph-capture-size: 256 max-num-batched-tokens: 16384 max-num-seqs: 256 @@ -94,9 +94,9 @@ backend: kv-cache-dtype: fp8 speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 100 - max-num-seqs: 1024 + max-num-seqs: 256 max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 + max-cudagraph-capture-size: 256 benchmark: type: "sa-bench" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml index 2ea133bc80..e66ac1a100 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml @@ -74,7 +74,7 @@ backend: enforce-eager: true speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 - max-cudagraph-capture-size: 2048 + max-cudagraph-capture-size: 128 max-num-batched-tokens: 16384 max-num-seqs: 256 @@ -94,9 +94,9 @@ backend: kv-cache-dtype: fp8 speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 100 - max-num-seqs: 1024 + max-num-seqs: 128 max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 + max-cudagraph-capture-size: 128 benchmark: type: "sa-bench" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml index 38112658b3..afd2b2622b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml @@ -74,7 +74,7 @@ backend: enforce-eager: true speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 - max-cudagraph-capture-size: 2048 + max-cudagraph-capture-size: 256 max-num-batched-tokens: 16384 max-num-seqs: 256 @@ -92,9 +92,9 @@ backend: kv-cache-dtype: fp8 speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 - max-num-seqs: 1024 + max-num-seqs: 256 max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 + max-cudagraph-capture-size: 256 benchmark: type: "sa-bench" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml index 4c13aa5221..39d7585d1a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml @@ -74,7 +74,7 @@ backend: enforce-eager: true speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 - max-cudagraph-capture-size: 2048 + max-cudagraph-capture-size: 128 max-num-batched-tokens: 16384 max-num-seqs: 256 @@ -92,9 +92,9 @@ backend: kv-cache-dtype: fp8 speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 - max-num-seqs: 1024 + max-num-seqs: 128 max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 + max-cudagraph-capture-size: 128 benchmark: type: "sa-bench" From 40ab61b568b7d620c15228b85468b38a20c1fcef Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sun, 2 Aug 2026 13:57:55 -0400 Subject: [PATCH 12/13] remove max-cudagraph-capture-size from prefill (using enforce-eager) --- .../8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml | 1 - .../8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml | 1 - .../8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml | 1 - .../8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml | 1 - .../minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml | 1 - .../minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml | 1 - 6 files changed, 6 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml index 2e9ca5f698..16f8e28e75 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep16-eagle3-8k1k.yaml @@ -74,7 +74,6 @@ backend: enforce-eager: true speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 - max-cudagraph-capture-size: 64 max-num-batched-tokens: 16384 max-num-seqs: 256 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml index 7d26772cd2..77d8dddfc5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml @@ -74,7 +74,6 @@ backend: enforce-eager: true speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 - max-cudagraph-capture-size: 32 max-num-batched-tokens: 16384 max-num-seqs: 256 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml index 59ff9cc0c4..f23ae7c3bd 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep4-eagle3-8k1k.yaml @@ -74,7 +74,6 @@ backend: enforce-eager: true speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 - max-cudagraph-capture-size: 256 max-num-batched-tokens: 16384 max-num-seqs: 256 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml index e66ac1a100..83b43b0d08 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep8-eagle3-8k1k.yaml @@ -74,7 +74,6 @@ backend: enforce-eager: true speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 - max-cudagraph-capture-size: 128 max-num-batched-tokens: 16384 max-num-seqs: 256 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml index afd2b2622b..39b04ca7ef 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp4-eagle3-8k1k.yaml @@ -74,7 +74,6 @@ backend: enforce-eager: true speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 - max-cudagraph-capture-size: 256 max-num-batched-tokens: 16384 max-num-seqs: 256 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml index 39d7585d1a..35dcc3d2ba 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-tp8-eagle3-8k1k.yaml @@ -74,7 +74,6 @@ backend: enforce-eager: true speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' stream-interval: 32 - max-cudagraph-capture-size: 128 max-num-batched-tokens: 16384 max-num-seqs: 256 From 74337bdfab15576f30ae714ee0f0614c2a617447 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sun, 2 Aug 2026 14:00:13 -0400 Subject: [PATCH 13/13] =?UTF-8?q?GB200=20MTP:=20replace=201=C3=97DEP32=20d?= =?UTF-8?q?ecode=20with=202=C3=97DEP16?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...-eagle3-8k1k.yaml => 1p2d-dep4-dep16-eagle3-8k1k.yaml} | 8 ++++---- configs/nvidia-master.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/{1p1d-dep4-dep32-eagle3-8k1k.yaml => 1p2d-dep4-dep16-eagle3-8k1k.yaml} (95%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-dep16-eagle3-8k1k.yaml similarity index 95% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-dep16-eagle3-8k1k.yaml index 77d8dddfc5..9e67abf42c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-dep16-eagle3-8k1k.yaml @@ -1,4 +1,4 @@ -name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep32-fp8-8k1k-eagle3" +name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-dep16-fp8-8k1k-eagle3" model: path: "minimax-m3-mxfp8" @@ -26,9 +26,9 @@ resources: prefill_nodes: 1 decode_nodes: 8 prefill_workers: 1 - decode_workers: 1 + decode_workers: 2 gpus_per_prefill: 4 - gpus_per_decode: 32 + gpus_per_decode: 16 frontend: type: dynamo @@ -81,7 +81,7 @@ backend: kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' tensor-parallel-size: 1 - data-parallel-size: 32 + data-parallel-size: 16 data-parallel-rpc-port: 13345 enable-expert-parallel: true trust-remote-code: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index ff91958ed1..6066c49c10 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8262,11 +8262,11 @@ minimaxm3-fp8-gb200-dynamo-vllm-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p1d-dep4-dep32-eagle3-8k1k.yaml" + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/mtp/1p2d-dep4-dep16-eagle3-8k1k.yaml" decode: - num-worker: 1 - tp: 32 - ep: 32 + num-worker: 2 + tp: 16 + ep: 16 dp-attn: true qwen3.5-fp4-b200-trt-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18