SemiAnalysisAI · yhyang201 · Aug 2, 2026 · Aug 2, 2026 · Aug 2, 2026 · Aug 2, 2026
diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh
@@ -0,0 +1,264 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+
+# Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B300 using SGLang
+# with EAGLE/MTP speculative decoding.
+#
+# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache.
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+INFERENCEX_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+export INFMAX_CONTAINER_WORKSPACE="${INFMAX_CONTAINER_WORKSPACE:-/workspace}"
+
+# The B200 DeepSeek-V4 Blackwell image installs SGLang editable under
+# /workspace, so its launcher mounts InferenceX at /ix instead. Resolve the
+# agentic tooling and results against the actual repository mount so the image
+# can keep its /workspace install and GitHub Actions can collect the outputs.
+if [[ ! -d "$INFMAX_CONTAINER_WORKSPACE/utils/aiperf" ]]; then
+    export INFMAX_CONTAINER_WORKSPACE="$INFERENCEX_ROOT"
+fi
+if [[ "${RESULT_DIR:-}" == /workspace/* && "$INFMAX_CONTAINER_WORKSPACE" != /workspace ]]; then
+    export RESULT_DIR="$INFMAX_CONTAINER_WORKSPACE/${RESULT_DIR#/workspace/}"
+fi
+
+source "$INFERENCEX_ROOT/benchmarks/benchmark_lib.sh"
+
+check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION
+
+if [[ -n "${SLURM_JOB_ID:-}" ]]; then
+    echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
+fi
+
+if [[ -n "${MODEL_PATH:-}" ]]; then
+    if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
+        hf download "$MODEL" --local-dir "$MODEL_PATH"
+    fi
+else
+    hf download "$MODEL"
+    export MODEL_PATH="$MODEL"
+fi
+nvidia-smi
+
+resolve_trace_source
+
+# Keep AIPerf's Transformers-main dependency from replacing the older
+# Transformers build pinned by the B200-specialized SGLang image. The server
+# always launches with the image's original interpreter; AIPerf and result
+# processing use the isolated environment when InferenceX is mounted at /ix.
+SGLANG_PYTHON="$(command -v python3)"
+if [[ "$INFMAX_CONTAINER_WORKSPACE" != /workspace ]]; then
+    AGENTIC_VENV="${AGENTIC_VENV:-/tmp/inferencex-agentic-venv}"
+    "$SGLANG_PYTHON" -m venv "$AGENTIC_VENV"
+    export PATH="$AGENTIC_VENV/bin:$PATH"
+fi
+install_agentic_deps
+
+SERVER_LOG="$RESULT_DIR/server.log"
+mkdir -p "$RESULT_DIR"
+
+export SGLANG_DEFAULT_THINKING=1
+export SGLANG_DSV4_REASONING_EFFORT=high
+export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1
+export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1
+
+CACHE_ARGS=()
+if require_agentic_kv_offload_backend hicache; then
+    # DeepSeek V4 HiCache currently rejects --hicache-size and supports
+    # capacity control only through a host/device token-capacity ratio.
+    # DSv4 exposes capacity as a host/device token ratio rather than bytes.
+    # Measurements put TP8 ratio=2 near 950 GB and TP4 ratio=8 near 1 TB,
+    # both below their configured capacities. The old TP4 ratio=16
+    # used roughly 2 TB and violated the half-node allocation rule.
+    if [ "$TP" -ge 8 ]; then
+        DEFAULT_HICACHE_RATIO=2
+    else
+        DEFAULT_HICACHE_RATIO=8
+    fi
+    HICACHE_RATIO="${HICACHE_RATIO:-$DEFAULT_HICACHE_RATIO}"
+    if [ "$HICACHE_RATIO" -gt "$DEFAULT_HICACHE_RATIO" ]; then
+        echo "Error: HICACHE_RATIO=$HICACHE_RATIO exceeds configured limit $DEFAULT_HICACHE_RATIO" >&2
+        exit 1
+    fi
+    HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_back}"
+    HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}"
+    HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}"
+    CACHE_ARGS=(
+        --enable-hierarchical-cache
+        --hicache-ratio "$HICACHE_RATIO"
+        --hicache-write-policy "$HICACHE_WRITE_POLICY"
+        --hicache-io-backend "$HICACHE_IO_BACKEND"
+        --hicache-mem-layout "$HICACHE_MEM_LAYOUT"
+    )
+    echo "HiCache DSv4 CPU tier: ratio=$HICACHE_RATIO, capacity=${TOTAL_CPU_DRAM_GB} GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT"
+fi
+
+USE_SGLANG_ROUTER=false
+SGLANG_BACKEND_PORT="$PORT"
+ROUTER_LOG="$RESULT_DIR/router.log"
+if [ "$DP_ATTENTION" = "true" ]; then
+    USE_SGLANG_ROUTER=true
+    export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true
+    SGLANG_BACKEND_PORT=$((PORT + 1))
+    SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000))
+    SGLANG_ROUTER_CMD=("$SGLANG_PYTHON" -m sglang_router.launch_router)
+fi
+
+PARALLEL_ARGS=(--tp "$TP")
+METRICS_ARGS=(--enable-metrics)
+MEM_FRACTION_STATIC=0.88
+CHUNKED_PREFILL_SIZE=8192
+if [ "$DP_ATTENTION" = "true" ]; then
+    PARALLEL_ARGS+=(
+        --dp "$TP"
+        --tokenizer-worker-num "$TP"
+        --enable-dp-attention
+        --enable-dp-attention-local-control-broadcast
+        --incremental-streaming-output
+        --stream-interval 20
+        --dist-init-addr "127.0.0.1:$((PORT + 2000))"
+        --ep-size "$EP_SIZE"
+        --moe-runner-backend flashinfer_mxfp4
+        --disable-flashinfer-autotune
+    )
+    MEM_FRACTION_STATIC=0.95
+    CHUNKED_PREFILL_SIZE=16384
+else
+    PARALLEL_ARGS+=(
+        --moe-runner-backend flashinfer_mxfp4
+        --disable-flashinfer-autotune
+        --enable-deepseek-v4-fp4-indexer
+    )
+fi
+
+MODEL_ARGS=(
+    --attention-backend compressed
+    --page-size 256
+    --disable-shared-experts-fusion
+)
+
+SPEC_ARGS=(
+    --speculative-algorithm EAGLE
+    --speculative-num-steps 3
+    --speculative-eagle-topk 1
+    --speculative-num-draft-tokens 4
+)
+
+# AgentX concurrency counts live session trees, not individual requests.
+# Allow subagent fan-out to exceed CONC without clipping request bursts.
+MAX_RUNNING_REQUESTS=$((2 * CONC))
+CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS
+[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64
+
+export PYTHONNOUSERSITE=1
+export TORCH_CUDA_ARCH_LIST=10.0
+# Agentic warmup dispatches hundreds of large prompts at once. SGLang's
+# tokenizer process can leave request bytes unacknowledged for longer than
+# AIPerf's 30-second TCP_USER_TIMEOUT while it admits that initial burst,
+# causing Linux to abort otherwise-live localhost connections. Keep the
+# six-hour request timeout unchanged, but allow up to 15 minutes for TCP
+# progress before declaring the connection dead.
+export AIPERF_HTTP_TCP_USER_TIMEOUT=900000
+export SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1
+export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1
+export SGLANG_OPT_USE_JIT_NORM=1
+export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1
+export SGLANG_OPT_USE_TOPK_V2=1
+export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1
+if [ "${EVAL_ONLY:-false}" != "true" ]; then
+    export SGLANG_SIMULATE_ACC_LEN=2.49
+    export SGLANG_SIMULATE_ACC_METHOD=match-expected
+    export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token
+fi
+TRITON_PTXAS_PATH=$(find \
+    /usr/local/cuda* \
+    /usr/local/lib/python*/dist-packages/nvidia \
+    /usr/local/lib/python*/site-packages/nvidia \
+    -type f -name ptxas -perm -u+x -print -quit 2>/dev/null || true)
+if [ -n "$TRITON_PTXAS_PATH" ]; then
+    export TRITON_PTXAS_PATH
+    echo "Using ptxas for Triton: $TRITON_PTXAS_PATH"
+fi
+SGLANG_CMD=(
+    "$SGLANG_PYTHON" -m sglang.launch_server
+    --model-path "$MODEL_PATH"
+    --served-model-name "$MODEL"
+    --host 0.0.0.0
+    --port "$SGLANG_BACKEND_PORT"
+    --trust-remote-code
+    "${PARALLEL_ARGS[@]}"
+    --mem-fraction-static "$MEM_FRACTION_STATIC"
+    --swa-full-tokens-ratio 0.1
+    --max-running-requests "$MAX_RUNNING_REQUESTS"
+    --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS"
+    --allow-auto-truncate
+    --enable-cache-report
+    --chunked-prefill-size "$CHUNKED_PREFILL_SIZE"
+    --tool-call-parser deepseekv4
+    --reasoning-parser deepseek-v4
+    --chat-template "$SCRIPT_DIR/../chat_templates/deepseek_v4_thinking.jinja"
+    --watchdog-timeout 1800
+    "${SPEC_ARGS[@]}"
+    "${MODEL_ARGS[@]}"
+    "${METRICS_ARGS[@]}"
+    "${CACHE_ARGS[@]}"
+)
+
+printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt"
+printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt"
+
+{
+    echo "=== SGLANG_* env vars at launch ==="
+    env | grep -E '^SGLANG_' | sort
+    echo "==================================="
+} | tee "$SERVER_LOG"
+
+echo "Starting SGLang server for B300..."
+"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 &
+SERVER_PID=$!
+echo "Server PID: $SERVER_PID"
+
+capture_cache_metrics() {
+    {
+        echo "=== SGLang cache metrics snapshot $(date --iso-8601=seconds) ==="
+        curl -fsS "http://localhost:$SGLANG_BACKEND_PORT/metrics" 2>/dev/null \
+            | grep -E '^(sglang:(cache_hit_rate|cached_tokens_total|prompt_tokens_total|hicache_host_used_tokens|hicache_host_total_tokens|token_usage|num_requests_running|num_requests_waiting))' \
+            || true
+        echo "============================================================"
+    } >> "$SERVER_LOG"
+}
+
+wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+
+if [ "$USE_SGLANG_ROUTER" = "true" ]; then
+    echo "Starting SGLang router on port $PORT for $TP DP ranks..."
+    "${SGLANG_ROUTER_CMD[@]}" \
+        --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \
+        --policy consistent_hashing \
+        --request-id-headers x-correlation-id \
+        --dp-aware \
+        --host 0.0.0.0 \
+        --port "$PORT" \
+        --prometheus-host 127.0.0.1 \
+        --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \
+        --connect-timeout-secs 900 \
+        --request-timeout-secs 14400 \
+        --disable-health-check \
+        --disable-retries > "$ROUTER_LOG" 2>&1 &
+    ROUTER_PID=$!
+    echo "Router PID: $ROUTER_PID"
+    wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID"
+fi
+
+if [ "${#METRICS_ARGS[@]}" -gt 0 ]; then
+    capture_cache_metrics
+    trap capture_cache_metrics EXIT
+fi
+
+if [ "${EVAL_ONLY}" = "true" ]; then
+    run_eval --port "$PORT"
+else
+    build_replay_cmd "$RESULT_DIR"
+    REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics"
+    run_agentic_replay_and_write_outputs "$RESULT_DIR"
+fi
@@ -8626,6 +8626,22 @@ dsv4-fp4-b300-sglang-agentic-hicache:
       - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 128], router: { name: sglang-router, version: "0.3.2" } }
       - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 144, 196, 512], router: { name: sglang-router, version: "0.3.2" } }
 
+dsv4-fp4-b300-sglang-agentic-mtp:
+  image: lmsysorg/sglang:nightly-dev-cu13-20260802-131bd51b
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4
+  runner: cluster:b300-nv
+  precision: fp4
+  framework: sglang
+  multinode: false
+  scenarios:
+    agentic-coding:
+    - dram-utilization: 0.80
+      search-space:
+      - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 8] }
+      - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1] }
+      - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [8] }
+
 # DEP8 prefill uses an 8K batch because 16K OOMs in the FP4 MoE intermediate;
 # decode uses FULL_DECODE_ONLY after the controlled graph test restored decode
 # throughput. Dynamo KV routing and AIPerf conversation-aware routing remain

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
@@ -5355,3 +5355,13 @@
     - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path"
     - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213
+
+- config-keys:
+    - dsv4-fp4-b300-sglang-agentic-mtp
+  description:
+    - "Add dsv4-fp4-b300-sglang-agentic-mtp: DeepSeek-V4-Pro FP4 B300 AgentX with SGLang EAGLE/MTP (--speculative-algorithm EAGLE, --speculative-num-steps 3, --speculative-eagle-topk 1, --speculative-num-draft-tokens 4), routed to benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh"
+    - "Pin simulated acceptance to the committed golden AL: SGLANG_SIMULATE_ACC_LEN=2.49 (golden_al_distribution/dsv4_mtp.yaml, thinking_on, K=3), SGLANG_SIMULATE_ACC_METHOD=match-expected, SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token; EVAL_ONLY runs keep real verification"
+    - "Bump image from lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233 to lmsysorg/sglang:nightly-dev-cu13-20260802-131bd51b, which reads SGLANG_SIMULATE_ACC_TOKEN_MODE (added in v0.5.16)"
+    - "Enable --enable-deepseek-v4-fp4-indexer on the plain-TP arm, --enable-cache-report, SGLANG_DEFAULT_THINKING=1 and SGLANG_DSV4_REASONING_EFFORT=high (the conditions the golden AL curve was measured under), and size the decode CUDA graph to max-running-requests instead of conc"
+    - "Search space is four points: TP8 and TP4 at conc 1 without KV offload, plus TP8 at conc 8 with and without HiCache host-DRAM offload. The conc 1 points match the topologies of the vLLM dsv4-fp4-b300-vllm-agentic-mtp arms; conc 8 raises max-running-requests above the observed AgentX subagent fan-out peak and is where the TP8 radix cache starts to outgrow the device pool"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2452