diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh new file mode 100755 index 0000000000..d69f479c94 --- /dev/null +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# MiniMax-M3 NVFP4 B300 AgentX with EAGLE3-GQA and synthetic acceptance. +# DRAM KV offload uses vLLM's SimpleCPUOffloadConnector in lazy mode. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +export EVAL_FRAMEWORK="lm-eval" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION + +DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" +NUM_SPEC_TOKENS=3 +# Golden AL for the GQA draft head: golden_al_distribution/minimaxm3_eagle3_gqa.yaml +# minimax-m3.thinking_on[3]. The non-GQA curve (minimaxm3_eagle3.yaml) reads 2.83 +# at the same level -- that head is not what this script runs. +SYNTHETIC_ACCEPT_LEN=2.78 + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi + DRAFT_MODEL_PATH="/data/models/${DRAFT_MODEL##*/}" + if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" + hf download "$DRAFT_MODEL" + DRAFT_MODEL_PATH="$DRAFT_MODEL" +fi + +nvidia-smi +resolve_trace_source +install_agentic_deps + +OFFLOAD_ARGS=() +if require_agentic_kv_offload_backend vllm-simple; then + CPU_OFFLOAD_BYTES=$((TOTAL_CPU_DRAM_GB * 1024 * 1024 * 1024)) + export VLLM_USE_SIMPLE_KV_OFFLOAD=1 + OFFLOAD_CONFIG=$(printf \ + '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":%d,"lazy_offload":true}}' \ + "$CPU_OFFLOAD_BYTES") + OFFLOAD_ARGS=(--kv-transfer-config "$OFFLOAD_CONFIG") +fi + +export PYTHONNOUSERSITE=1 +export VLLM_ENGINE_READY_TIMEOUT_S=3600 +export VLLM_FLOAT32_MATMUL_PRECISION=high +export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm + +SERVER_LOG="$RESULT_DIR/server.log" +mkdir -p "$RESULT_DIR" + +SERVER_PID="" +cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$SERVER_PID" "vLLM server" 60 + exit "$exit_code" +} +trap cleanup_agentic_services EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +if [ "${EVAL_ONLY:-}" = "true" ]; then + SPEC_CONFIG=$(printf \ + '{"method":"eagle3","model":"%s","num_speculative_tokens":%d,"attention_backend":"FLASH_ATTN"}' \ + "$DRAFT_MODEL_PATH" "$NUM_SPEC_TOKENS") +else + SPEC_CONFIG=$(printf \ + '{"method":"eagle3","model":"%s","num_speculative_tokens":%d,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":%.2f}' \ + "$DRAFT_MODEL_PATH" "$NUM_SPEC_TOKENS" "$SYNTHETIC_ACCEPT_LEN") +fi + +{ set +x; } 2>/dev/null +VLLM_CMD=( + vllm serve "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$PORT" + --tensor-parallel-size "$TP" + --gpu-memory-utilization 0.9 + --block-size 128 + --language-model-only + --enable-prefix-caching + --no-enable-flashinfer-autotune + --reasoning-parser minimax_m3 + --tool-call-parser minimax_m3 + --enable-auto-tool-choice + --default-chat-template-kwargs '{"thinking_mode":"enabled"}' + --attention-config '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + --all2all-backend flashinfer_nvlink_one_sided + --max-cudagraph-capture-size 512 + --max-num-batched-tokens 16384 + --stream-interval 20 + --trust-remote-code + --speculative-config "$SPEC_CONFIG" + "${OFFLOAD_ARGS[@]}" +) +printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" +printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" +set -x + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" +if [ "${EVAL_ONLY}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a30af59641..baaf8e4f73 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8586,6 +8586,24 @@ minimaxm3-fp8-h200-vllm-agentic: - { tp: 8, ep: 8, kv-offloading: none, conc-list: [2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } +minimaxm3-fp4-b300-vllm-agentic-mtp: + image: vllm/vllm-openai:nightly-387189c42997b27e2c04b5d97ef8190ffa2bf909 + model: nvidia/MiniMax-M3-NVFP4 + model-prefix: minimaxm3 + runner: cluster:b300-nv + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + # 0.683 maps the B300 TP4 SimpleCPU point to a 1,024 GB + # engine-level CPU KV budget. GPU-resident points receive a zero budget. + - dram-utilization: 0.683 + search-space: + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 5, 10, 15, 20] } + - { tp: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 5] } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [30, 40, 50, 60, 80, 90, 100] } + dsv4-fp4-b200-sglang-agentic-hicache: image: lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6eebd97070..a9f5b18ec4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5355,3 +5355,13 @@ - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path" - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213 + +- config-keys: + - minimaxm3-fp4-b300-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Run five MiniMax-M3 NVFP4 B300 vLLM AgentX points on vllm/vllm-openai:nightly-387189c42997b27e2c04b5d97ef8190ffa2bf909: TP4 concurrency 1 and 5; TP2 concurrency 2 and 6; and TP4 concurrency 40 with lazy SimpleCPUOffloadConnector KV offload and a 1,024 GB engine-level CPU KV budget." + - "Use the Inferact/MiniMax-M3-EAGLE3-GQA drafter with three speculative tokens and synthetic acceptance length 2.78; enable prefix caching, FlashInfer TRT-LLM attention with FP8 indexer KV, and the EAGLE3 drafter on FLASH_ATTN." + - "Pin the AgentX client to SemiAnalysisAI/aiperf PR #31 commit deb5421c7ff3cd39f78228f469e23e25233ac6c6 for the revised trace timing, warmup handoff, and spawn/join replay behavior." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2328 diff --git a/utils/aiperf b/utils/aiperf index be758d6218..abf55f902c 160000 --- a/utils/aiperf +++ b/utils/aiperf @@ -1 +1 @@ -Subproject commit be758d6218268171e2957fbec9d4f557275bca2d +Subproject commit abf55f902cde0a3e8389c452a83ecbf5ba15dea9