diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh new file mode 100755 index 0000000000..d8cbd37e81 --- /dev/null +++ b/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# MiniMax-M3 MXFP8 MI325X (gfx942) AgentX (agentic-coding) recipe with EAGLE3 +# speculative decoding — the spec-decoding=mtp variant of +# agentic/minimaxm3_fp8_mi325x.sh. Everything outside the speculative block +# mirrors the non-MTP agentic sibling (--block-size 128, --language-model-only, TRITON_ATTN, FP8 KV on the nightly +# images, minimax_m3 parsers, VLLM_USE_BREAKABLE_CUDAGRAPH=0, TP4 gmu bump), so +# the spec-decode delta is readable at equal concurrency. +# +# Speculative config: Inferact/MiniMax-M3-EAGLE3 draft head, 3 speculative +# tokens. Unlike the CUDA recipes the drafter needs no attention_backend +# override — the FlashInfer "page size 128 requires GQA/MQA" limitation that +# forces FLASH_ATTN for the MHA EAGLE3 head is FlashInfer/CUDA-specific, and +# here the whole server runs TRITON_ATTN, which serves the MHA draft fine +# (same reasoning as fixed_seq_len/minimaxm3_fp8_mi325x_mtp.sh). +# +# No in-place SupportsEagle3 patch: the pinned ROCm nightly is built from +# upstream 2026-06-23, after vllm-project/vllm#45546 (EAGLE3 support on the AMD +# MiniMax-M3 model) merged on 2026-06-14. The fixed-seq-len MI325X MTP recipe +# still carries that runtime patch because it runs the older +# vllm-openai-rocm:minimax-m3 bring-up image. +# +# Throughput runs pin synthetic acceptance to the committed golden AL; the +# EVAL_ONLY accuracy run keeps real target verification. See SYNTHETIC_ACCEPT_LEN. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL IMAGE TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION + +DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + +# The EAGLE3 draft is never pre-staged next to the target checkpoint; fetch it +# into the shared HF cache, retrying because that cache is a network FS where +# concurrent downloads hit huggingface_hub's WeakFileLock stale-handle race. +for attempt in 1 2 3 4 5; do + hf download "$DRAFT_MODEL" && break + if [ "$attempt" = 5 ]; then echo "hf download of $DRAFT_MODEL failed after $attempt attempts" >&2; exit 1; fi + echo "hf download attempt $attempt failed; retrying in 60s" >&2 + sleep 60 +done +rocm-smi || true +amd-smi || true + +export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126 +resolve_trace_source +install_agentic_deps + +export VLLM_ENGINE_READY_TIMEOUT_S=3600 +export VLLM_USE_BREAKABLE_CUDAGRAPH=0 +export PYTHONNOUSERSITE=1 + +SERVER_LOG="$RESULT_DIR/server.log" +ROUTER_LOG="$RESULT_DIR/router.log" +mkdir -p "$RESULT_DIR" + +# GPU-resident only: no Mooncake on AMD. The HIP-built Mooncake transfer engine's +# TCP transport collapses mid-run on gfx942 -- the store spews +# "ServerSession::readBody failed ... Connection reset by peer" until the replay +# aborts (run 30517360090, MI300X TP8 mtp dram c8). This recipe therefore carries +# no host-DRAM KV offload arm, and fails loudly rather than silently serving a +# GPU-resident run under a dram label. +if [[ "${KV_OFFLOADING}" != "none" ]]; then + echo "Error: KV_OFFLOADING=$KV_OFFLOADING requested, but this recipe is GPU-resident only (Mooncake is not used on AMD)" >&2 + exit 1 +fi +OFFLOAD_ARGS=() + +PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) +if [[ "$DP_ATTENTION" == "true" ]]; then + PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") +fi + +EP_ARGS=() +if (( EP_SIZE > 1 )); then + EP_ARGS=(--enable-expert-parallel) +fi + +KV_CACHE_ARGS=() +if [[ "$IMAGE" == vllm/vllm-openai-rocm:nightly-* ]]; then + KV_CACHE_ARGS=(--kv-cache-dtype fp8) +fi + +VLLM_BACKEND_PORT="$PORT" +if [[ "$DP_ATTENTION" == "true" ]]; then + VLLM_BACKEND_PORT=$((PORT + 1)) + export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 + agentic_pip_install --quiet 'vllm-router==0.1.14' +fi + +# use 3 speculative tokens for all configs, matching the MiniMax-M3 MTP recipes +NUM_SPEC_TOKENS=3 + +# AgentX pins acceptance to the committed golden AL so submissions are compared +# on system performance at a fixed acceptance target rather than on draft-head +# quality (golden_al_distribution/README.md). 2.83 is the MiniMax-M3 EAGLE3 +# curve at num_speculative_tokens=3, thinking_on +# (golden_al_distribution/minimaxm3_eagle3.yaml). The separate +# minimaxm3_eagle3_gqa.yaml curve belongs to the Inferact/MiniMax-M3-EAGLE3-GQA +# draft and is not mixed in here. +# +# EVAL_ONLY switches back to real verification: synthetic acceptance commits +# drafted tokens regardless of the target logits, so generated text is wrong and +# the eval would score ~0 (same split as dsv4_fp4_b*_vllm_mtp.sh). +SYNTHETIC_ACCEPT_LEN=2.83 +if [ "${EVAL_ONLY:-false}" = "true" ]; then + SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" +else + SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" +fi + +# AgentX concurrency counts live session trees, not individual requests, so keep +# the non-MTP recipe's 2x scheduler headroom for subagent fan-out. Cudagraph +# capture is left at the sibling's default: the ROCm MiniMax-M3 MTP recipes run +# with VLLM_USE_BREAKABLE_CUDAGRAPH=0 and no explicit capture ceiling. +MAX_NUM_SEQS=$((2 * CONC)) +# 0.90, not the non-MTP sibling's 0.95/0.98: fixed_seq_len/minimaxm3_fp8_mi325x_mtp.sh +# passes no gmu flag at all (vLLM's 0.90 default), which is the proven MTP setting +# here. The H100 twin of this recipe died mid-warmup at 0.95 with +# torch.OutOfMemoryError on every rank once the EAGLE3 head and its KV were +# resident (run 30515793863), so the TP4 0.98 bump is dropped as well. +GPU_MEMORY_UTILIZATION=0.90 + +vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ + --host 0.0.0.0 \ + --port "$VLLM_BACKEND_PORT" \ + "${PARALLEL_ARGS[@]}" \ + "${EP_ARGS[@]}" \ + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \ + --block-size 128 \ + --language-model-only \ + --attention-backend TRITON_ATTN \ + --enable-prefix-caching \ + --max-num-seqs "$MAX_NUM_SEQS" \ + --speculative-config "$SPEC_CONFIG" \ + --tool-call-parser minimax_m3 \ + --reasoning-parser minimax_m3 \ + --enable-auto-tool-choice \ + --trust-remote-code \ + "${KV_CACHE_ARGS[@]}" \ + "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! + +wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [[ "$DP_ATTENTION" == "true" ]]; then + vllm-router \ + --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ + --policy consistent_hash \ + --intra-node-data-parallel-size "$TP" \ + --host 0.0.0.0 \ + --port "$PORT" \ + --prometheus-host 127.0.0.1 \ + --prometheus-port "$((PORT + 10000))" \ + --request-timeout-secs 14400 \ + --disable-retries > "$ROUTER_LOG" 2>&1 & + ROUTER_PID=$! + wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" +fi + +if [ "${EVAL_ONLY}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index b15ae5bccd..91903cf4ce 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -2230,6 +2230,38 @@ minimaxm3-fp8-mi325x-vllm-agentic: - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96], router: { name: vllm-router, version: "0.1.14" } } +# EAGLE3 speculative-decoding (spec-decoding: mtp) variant of +# minimaxm3-fp8-mi325x-vllm-agentic, pairing MiniMaxAI/MiniMax-M3-MXFP8 with the +# Inferact/MiniMax-M3-EAGLE3 draft head (3 speculative tokens) and pinning +# synthetic acceptance to the golden AL 2.83 +# (golden_al_distribution/minimaxm3_eagle3.yaml, thinking_on, K=3). The drafter +# needs no attention_backend override: the whole server runs TRITON_ATTN, which +# serves the MHA EAGLE3 head. The pinned ROCm nightly is built from upstream +# 2026-06-23, after vllm-project/vllm#45546 landed SupportsEagle3 on the AMD +# MiniMax-M3 model, so this recipe needs no in-place patch (unlike the +# fixed-seq-len MI325X MTP entry, which still runs the older minimax-m3 image). +# GPU-resident only: no Mooncake on AMD. The HIP-built Mooncake transfer engine's +# TCP transport collapses mid-run on gfx942 (run 30517360090), so the non-MTP +# entry's three host-DRAM arms are dropped rather than shipped broken. The +# remaining TP4/TP8/DEP arms run the AgentX MTP concurrency grid: every conc-list +# steps by at least 2 and stops hard at 16. +minimaxm3-fp8-mi325x-vllm-agentic-mtp: + image: vllm/vllm-openai-rocm:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914 + model: MiniMaxAI/MiniMax-M3-MXFP8 + model-prefix: minimaxm3 + runner: cluster:mi325x-amds + precision: fp8 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - search-space: + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] } + - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] } + - { tp: 8, ep: 8, dp-attn: true, spec-decoding: mtp, kv-offloading: none, conc-list: [8, 12, 16], router: { name: vllm-router, version: "0.1.14" } } + minimaxm3-fp4-mi355x-vllm-agentic: image: vllm/vllm-openai-rocm:nightly-dcfebf93f4eccf30f71872283331eee757915daf model: amd/MiniMax-M3-MXFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8f5831507e..cd8fe8cf3a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5339,3 +5339,14 @@ - "Pass --use-chat-template to the benchmark, as required for EAGLE-style speculative decoding" - "Validated on the node through the real launcher, every request successful: TEP4 concurrency 64 640/640 at 6895 tok/s (11% above the non-MTP arm's 6198 tok/s, mean TTFT 27.2s to 9.6s), and TEP4 concurrency 1 10/10 at 1233 tok/s (83% above the non-MTP arm's 672 tok/s, mean TPOT 12.6ms to 6.5ms)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2312 + +- config-keys: + - minimaxm3-fp8-mi325x-vllm-agentic-mtp + description: + - "Add the EAGLE3 speculative-decoding (spec-decoding: mtp) variant of minimaxm3-fp8-mi325x-vllm-agentic: MiniMax-M3 MXFP8 on MI325X (gfx942) with vLLM, agentic-coding scenario only, routed via spec-decoding=mtp to benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh." + - "Speculative config: method eagle3 on the Inferact/MiniMax-M3-EAGLE3 draft head, num_speculative_tokens 3, with no attention_backend override for the drafter -- the FlashInfer page-size-128-requires-GQA/MQA limitation that forces FLASH_ATTN on CUDA is FlashInfer-specific and this server runs TRITON_ATTN throughout (same reasoning as fixed_seq_len/minimaxm3_fp8_mi325x_mtp.sh)." + - "Throughput runs pin rejection_sample_method synthetic with synthetic_acceptance_length 2.83 -- the committed golden AL for MiniMax-M3 EAGLE3 at K=3, thinking_on (golden_al_distribution/minimaxm3_eagle3.yaml), per the AgentX policy in golden_al_distribution/README.md. The minimaxm3_eagle3_gqa.yaml curve belongs to the Inferact/MiniMax-M3-EAGLE3-GQA draft (run 29784780049) and is not mixed in. EVAL_ONLY runs drop synthetic acceptance and keep real target verification." + - "No in-place SupportsEagle3 patch, unlike fixed_seq_len/minimaxm3_fp8_mi325x_mtp.sh: this recipe runs the agentic sibling's vllm/vllm-openai-rocm:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914 (upstream 2026-06-23), which postdates vllm-project/vllm#45546 (merged 2026-06-14); the fixed-seq-len entry still needs the runtime patch because it pins the older vllm-openai-rocm:minimax-m3 bring-up image." + - "Serve shape is otherwise identical to the non-MTP agentic sibling (--block-size 128, --language-model-only, --attention-backend TRITON_ATTN, FP8 KV on nightly-* images, gpu-memory-utilization 0.95 with the 0.98 TP4 bump, minimax_m3 parsers, VLLM_USE_BREAKABLE_CUDAGRAPH=0, max-num-seqs 2*CONC, vllm-router for DP-attention). Cudagraph capture stays at the sibling default rather than the CUDA recipes' MAX_NUM_SEQS*(1+3) token ceiling, matching the merged ROCm MiniMax-M3 MTP recipes." + - "Search space is GPU-resident only -- TP4, TP4/EP4, TP8, TP8/EP8 on the AgentX MTP concurrency grid [1, 4, 8, 12, 16] and TP8/DEP8 dp-attn on [8, 12, 16], steps of at least 2 with a hard stop at conc 16. The non-MTP entry's three Mooncake host-DRAM arms are dropped: Mooncake is not used on AMD. Its HIP-built transfer engine runs over TCP here and the store collapses mid-replay (ServerSession::readBody failed ... Connection reset by peer until AIPerf aborts; observed on the MI300X sibling, run 30517360090), so a dram arm would measure the transport, not the engine." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2426