From 5a2208f1ea885b6774945abd6c3e77a9573059f5 Mon Sep 17 00:00:00 2001 From: yhyang201 Date: Sun, 2 Aug 2026 16:06:57 +0800 Subject: [PATCH 1/4] feat(agentx): add DSv4 B300 SGLang agentic MTP recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：新增 DSv4 B300 SGLang AgentX MTP 配方。 --- .../agentic/dsv4_fp4_b300_sglang_mtp.sh | 260 ++++++++++++++++++ configs/nvidia-master.yaml | 14 + perf-changelog.yaml | 9 + 3 files changed, 283 insertions(+) create mode 100755 benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh new file mode 100755 index 0000000000..571e477f4b --- /dev/null +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh @@ -0,0 +1,260 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B300 using SGLang +# with EAGLE/MTP speculative decoding. +# +# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +INFERENCEX_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +export INFMAX_CONTAINER_WORKSPACE="${INFMAX_CONTAINER_WORKSPACE:-/workspace}" + +# The B200 DeepSeek-V4 Blackwell image installs SGLang editable under +# /workspace, so its launcher mounts InferenceX at /ix instead. Resolve the +# agentic tooling and results against the actual repository mount so the image +# can keep its /workspace install and GitHub Actions can collect the outputs. +if [[ ! -d "$INFMAX_CONTAINER_WORKSPACE/utils/aiperf" ]]; then + export INFMAX_CONTAINER_WORKSPACE="$INFERENCEX_ROOT" +fi +if [[ "${RESULT_DIR:-}" == /workspace/* && "$INFMAX_CONTAINER_WORKSPACE" != /workspace ]]; then + export RESULT_DIR="$INFMAX_CONTAINER_WORKSPACE/${RESULT_DIR#/workspace/}" +fi + +source "$INFERENCEX_ROOT/benchmarks/benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi +nvidia-smi + +resolve_trace_source + +# Keep AIPerf's Transformers-main dependency from replacing the older +# Transformers build pinned by the B200-specialized SGLang image. The server +# always launches with the image's original interpreter; AIPerf and result +# processing use the isolated environment when InferenceX is mounted at /ix. +SGLANG_PYTHON="$(command -v python3)" +if [[ "$INFMAX_CONTAINER_WORKSPACE" != /workspace ]]; then + AGENTIC_VENV="${AGENTIC_VENV:-/tmp/inferencex-agentic-venv}" + "$SGLANG_PYTHON" -m venv "$AGENTIC_VENV" + export PATH="$AGENTIC_VENV/bin:$PATH" +fi +install_agentic_deps + +SERVER_LOG="$RESULT_DIR/server.log" +mkdir -p "$RESULT_DIR" + +export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 +export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 + +CACHE_ARGS=() +if require_agentic_kv_offload_backend hicache; then + # DeepSeek V4 HiCache currently rejects --hicache-size and supports + # capacity control only through a host/device token-capacity ratio. + # DSv4 exposes capacity as a host/device token ratio rather than bytes. + # Measurements put TP8 ratio=2 near 950 GB and TP4 ratio=8 near 1 TB, + # both below their configured capacities. The old TP4 ratio=16 + # used roughly 2 TB and violated the half-node allocation rule. + if [ "$TP" -ge 8 ]; then + DEFAULT_HICACHE_RATIO=2 + else + DEFAULT_HICACHE_RATIO=8 + fi + HICACHE_RATIO="${HICACHE_RATIO:-$DEFAULT_HICACHE_RATIO}" + if [ "$HICACHE_RATIO" -gt "$DEFAULT_HICACHE_RATIO" ]; then + echo "Error: HICACHE_RATIO=$HICACHE_RATIO exceeds configured limit $DEFAULT_HICACHE_RATIO" >&2 + exit 1 + fi + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_back}" + HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" + CACHE_ARGS=( + --enable-hierarchical-cache + --hicache-ratio "$HICACHE_RATIO" + --hicache-write-policy "$HICACHE_WRITE_POLICY" + --hicache-io-backend "$HICACHE_IO_BACKEND" + --hicache-mem-layout "$HICACHE_MEM_LAYOUT" + ) + echo "HiCache DSv4 CPU tier: ratio=$HICACHE_RATIO, capacity=${TOTAL_CPU_DRAM_GB} GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" +fi + +USE_SGLANG_ROUTER=false +SGLANG_BACKEND_PORT="$PORT" +ROUTER_LOG="$RESULT_DIR/router.log" +if [ "$DP_ATTENTION" = "true" ]; then + USE_SGLANG_ROUTER=true + export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true + SGLANG_BACKEND_PORT=$((PORT + 1)) + SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) + SGLANG_ROUTER_CMD=("$SGLANG_PYTHON" -m sglang_router.launch_router) +fi + +PARALLEL_ARGS=(--tp "$TP") +METRICS_ARGS=(--enable-metrics) +MEM_FRACTION_STATIC=0.88 +CHUNKED_PREFILL_SIZE=8192 +if [ "$DP_ATTENTION" = "true" ]; then + PARALLEL_ARGS+=( + --dp "$TP" + --tokenizer-worker-num "$TP" + --enable-dp-attention + --enable-dp-attention-local-control-broadcast + --incremental-streaming-output + --stream-interval 20 + --dist-init-addr "127.0.0.1:$((PORT + 2000))" + --ep-size "$EP_SIZE" + --moe-runner-backend flashinfer_mxfp4 + --disable-flashinfer-autotune + ) + MEM_FRACTION_STATIC=0.95 + CHUNKED_PREFILL_SIZE=16384 +else + PARALLEL_ARGS+=( + --moe-runner-backend flashinfer_mxfp4 + --disable-flashinfer-autotune + ) +fi + +MODEL_ARGS=( + --attention-backend compressed + --page-size 256 + --disable-shared-experts-fusion +) + +SPEC_ARGS=( + --speculative-algorithm EAGLE + --speculative-num-steps 3 + --speculative-eagle-topk 1 + --speculative-num-draft-tokens 4 +) + +# AgentX concurrency counts live session trees, not individual requests. +# Allow subagent fan-out to exceed CONC without clipping request bursts. +MAX_RUNNING_REQUESTS=$((2 * CONC)) +CUDA_GRAPH_MAX_BS=$CONC +[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 + +export PYTHONNOUSERSITE=1 +export TORCH_CUDA_ARCH_LIST=10.0 +# Agentic warmup dispatches hundreds of large prompts at once. SGLang's +# tokenizer process can leave request bytes unacknowledged for longer than +# AIPerf's 30-second TCP_USER_TIMEOUT while it admits that initial burst, +# causing Linux to abort otherwise-live localhost connections. Keep the +# six-hour request timeout unchanged, but allow up to 15 minutes for TCP +# progress before declaring the connection dead. +export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 +export SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 +export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1 +export SGLANG_OPT_USE_JIT_NORM=1 +export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1 +export SGLANG_OPT_USE_TOPK_V2=1 +export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1 +if [ "${EVAL_ONLY:-false}" != "true" ]; then + export SGLANG_SIMULATE_ACC_LEN=2.49 + export SGLANG_SIMULATE_ACC_METHOD=match-expected + export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token +fi +TRITON_PTXAS_PATH=$(find \ + /usr/local/cuda* \ + /usr/local/lib/python*/dist-packages/nvidia \ + /usr/local/lib/python*/site-packages/nvidia \ + -type f -name ptxas -perm -u+x -print -quit 2>/dev/null || true) +if [ -n "$TRITON_PTXAS_PATH" ]; then + export TRITON_PTXAS_PATH + echo "Using ptxas for Triton: $TRITON_PTXAS_PATH" +fi +SGLANG_CMD=( + "$SGLANG_PYTHON" -m sglang.launch_server + --model-path "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$SGLANG_BACKEND_PORT" + --trust-remote-code + "${PARALLEL_ARGS[@]}" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --swa-full-tokens-ratio 0.1 + --max-running-requests "$MAX_RUNNING_REQUESTS" + --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + --allow-auto-truncate + --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" + --tool-call-parser deepseekv4 + --reasoning-parser deepseek-v4 + --chat-template "$SCRIPT_DIR/../chat_templates/deepseek_v4_thinking.jinja" + --watchdog-timeout 1800 + "${SPEC_ARGS[@]}" + "${MODEL_ARGS[@]}" + "${METRICS_ARGS[@]}" + "${CACHE_ARGS[@]}" +) + +printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" +printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" + +{ + echo "=== SGLANG_* env vars at launch ===" + env | grep -E '^SGLANG_' | sort + echo "===================================" +} | tee "$SERVER_LOG" + +echo "Starting SGLang server for B300..." +"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +capture_cache_metrics() { + { + echo "=== SGLang cache metrics snapshot $(date --iso-8601=seconds) ===" + curl -fsS "http://localhost:$SGLANG_BACKEND_PORT/metrics" 2>/dev/null \ + | grep -E '^(sglang:(cache_hit_rate|cached_tokens_total|prompt_tokens_total|hicache_host_used_tokens|hicache_host_total_tokens|token_usage|num_requests_running|num_requests_waiting))' \ + || true + echo "============================================================" + } >> "$SERVER_LOG" +} + +wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "$USE_SGLANG_ROUTER" = "true" ]; then + echo "Starting SGLang router on port $PORT for $TP DP ranks..." + "${SGLANG_ROUTER_CMD[@]}" \ + --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \ + --policy consistent_hashing \ + --request-id-headers x-correlation-id \ + --dp-aware \ + --host 0.0.0.0 \ + --port "$PORT" \ + --prometheus-host 127.0.0.1 \ + --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \ + --connect-timeout-secs 900 \ + --request-timeout-secs 14400 \ + --disable-health-check \ + --disable-retries > "$ROUTER_LOG" 2>&1 & + ROUTER_PID=$! + echo "Router PID: $ROUTER_PID" + wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" +fi + +if [ "${#METRICS_ARGS[@]}" -gt 0 ]; then + capture_cache_metrics + trap capture_cache_metrics EXIT +fi + +if [ "${EVAL_ONLY}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a30af59641..5ccce2ef3d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8626,6 +8626,20 @@ dsv4-fp4-b300-sglang-agentic-hicache: - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 128], router: { name: sglang-router, version: "0.3.2" } } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 144, 196, 512], router: { name: sglang-router, version: "0.3.2" } } +dsv4-fp4-b300-sglang-agentic-mtp: + image: lmsysorg/sglang:nightly-dev-cu13-20260802-131bd51b + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:b300-nv + precision: fp4 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1] } + # DEP8 prefill uses an 8K batch because 16K OOMs in the FP4 MoE intermediate; # decode uses FULL_DECODE_ONLY after the controlled graph test restored decode # throughput. Dynamo KV routing and AIPerf conversation-aware routing remain diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6eebd97070..be9d568d7b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5355,3 +5355,12 @@ - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path" - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213 + +- config-keys: + - dsv4-fp4-b300-sglang-agentic-mtp + description: + - "Add dsv4-fp4-b300-sglang-agentic-mtp: DeepSeek-V4-Pro FP4 B300 AgentX with SGLang EAGLE/MTP (--speculative-algorithm EAGLE, --speculative-num-steps 3, --speculative-eagle-topk 1, --speculative-num-draft-tokens 4), routed to benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh" + - "Pin simulated acceptance to the committed golden AL: SGLANG_SIMULATE_ACC_LEN=2.49 (golden_al_distribution/dsv4_mtp.yaml, thinking_on, K=3), SGLANG_SIMULATE_ACC_METHOD=match-expected, SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token; EVAL_ONLY runs keep real verification" + - "Bump image from lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233 to lmsysorg/sglang:nightly-dev-cu13-20260802-131bd51b, which reads SGLANG_SIMULATE_ACC_TOKEN_MODE (added in v0.5.16)" + - "Search space is one point: TP8, kv-offloading none, conc 1 — same topology as the STP entry TP8 row and the vLLM dsv4-fp4-b300-vllm-agentic-mtp TP8 arm" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PENDING From 5ff1ccaa13c45ada5fee11397666e3ba6e2c003b Mon Sep 17 00:00:00 2001 From: yhyang201 Date: Sun, 2 Aug 2026 16:09:54 +0800 Subject: [PATCH 2/4] chore(agentx): fill in PR link for DSv4 B300 SGLang MTP changelog entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：回填 DSv4 B300 SGLang MTP changelog 条目的 PR 链接。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index be9d568d7b..04c33a6eb7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5363,4 +5363,4 @@ - "Pin simulated acceptance to the committed golden AL: SGLANG_SIMULATE_ACC_LEN=2.49 (golden_al_distribution/dsv4_mtp.yaml, thinking_on, K=3), SGLANG_SIMULATE_ACC_METHOD=match-expected, SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token; EVAL_ONLY runs keep real verification" - "Bump image from lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233 to lmsysorg/sglang:nightly-dev-cu13-20260802-131bd51b, which reads SGLANG_SIMULATE_ACC_TOKEN_MODE (added in v0.5.16)" - "Search space is one point: TP8, kv-offloading none, conc 1 — same topology as the STP entry TP8 row and the vLLM dsv4-fp4-b300-vllm-agentic-mtp TP8 arm" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PENDING + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2452 From 32071692f925c25bad974b4fb92bc075e1f6cd6d Mon Sep 17 00:00:00 2001 From: yhyang201 Date: Sun, 2 Aug 2026 17:10:03 +0800 Subject: [PATCH 3/4] feat(agentx): tune DSv4 B300 SGLang MTP recipe and add TP4 point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enable the FP4 indexer, cache reporting and the golden-AL thinking mode, size the decode CUDA graph to max-running-requests, and add a TP4 arm at conc 1 alongside TP8. 中文：为 DSv4 B300 SGLang MTP 配方启用 FP4 indexer、缓存上报和黄金 AL 对应的 thinking 模式，decode CUDA graph 按 max-running-requests 取值，并在 conc 1 上 新增 TP4 臂。 --- benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh | 6 +++++- configs/nvidia-master.yaml | 1 + perf-changelog.yaml | 3 ++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh index 571e477f4b..5dfbc0d7d5 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh @@ -57,6 +57,8 @@ install_agentic_deps SERVER_LOG="$RESULT_DIR/server.log" mkdir -p "$RESULT_DIR" +export SGLANG_DEFAULT_THINKING=1 +export SGLANG_DSV4_REASONING_EFFORT=high export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 @@ -125,6 +127,7 @@ else PARALLEL_ARGS+=( --moe-runner-backend flashinfer_mxfp4 --disable-flashinfer-autotune + --enable-deepseek-v4-fp4-indexer ) fi @@ -144,7 +147,7 @@ SPEC_ARGS=( # AgentX concurrency counts live session trees, not individual requests. # Allow subagent fan-out to exceed CONC without clipping request bursts. MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS=$CONC +CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 export PYTHONNOUSERSITE=1 @@ -189,6 +192,7 @@ SGLANG_CMD=( --max-running-requests "$MAX_RUNNING_REQUESTS" --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" --allow-auto-truncate + --enable-cache-report --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 5ccce2ef3d..27104a42e2 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8639,6 +8639,7 @@ dsv4-fp4-b300-sglang-agentic-mtp: - dram-utilization: 0.80 search-space: - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1] } + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1] } # DEP8 prefill uses an 8K batch because 16K OOMs in the FP4 MoE intermediate; # decode uses FULL_DECODE_ONLY after the controlled graph test restored decode diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 04c33a6eb7..b7082fb68a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5362,5 +5362,6 @@ - "Add dsv4-fp4-b300-sglang-agentic-mtp: DeepSeek-V4-Pro FP4 B300 AgentX with SGLang EAGLE/MTP (--speculative-algorithm EAGLE, --speculative-num-steps 3, --speculative-eagle-topk 1, --speculative-num-draft-tokens 4), routed to benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh" - "Pin simulated acceptance to the committed golden AL: SGLANG_SIMULATE_ACC_LEN=2.49 (golden_al_distribution/dsv4_mtp.yaml, thinking_on, K=3), SGLANG_SIMULATE_ACC_METHOD=match-expected, SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token; EVAL_ONLY runs keep real verification" - "Bump image from lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233 to lmsysorg/sglang:nightly-dev-cu13-20260802-131bd51b, which reads SGLANG_SIMULATE_ACC_TOKEN_MODE (added in v0.5.16)" - - "Search space is one point: TP8, kv-offloading none, conc 1 — same topology as the STP entry TP8 row and the vLLM dsv4-fp4-b300-vllm-agentic-mtp TP8 arm" + - "Enable --enable-deepseek-v4-fp4-indexer on the plain-TP arm, --enable-cache-report, SGLANG_DEFAULT_THINKING=1 and SGLANG_DSV4_REASONING_EFFORT=high (the conditions the golden AL curve was measured under), and size the decode CUDA graph to max-running-requests instead of conc" + - "Search space is two points at conc 1, kv-offloading none: TP8 and TP4, matching the topologies of the vLLM dsv4-fp4-b300-vllm-agentic-mtp arms" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2452 From d21cc34a4cf67ea43ff3c6b3e09fc4e1b8af17ec Mon Sep 17 00:00:00 2001 From: yhyang201 Date: Mon, 3 Aug 2026 00:19:01 +0800 Subject: [PATCH 4/4] feat(agentx): add conc 8 points with and without HiCache to DSv4 B300 SGLang MTP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：为 DSv4 B300 SGLang MTP 配方新增 conc 8 的有/无 HiCache 两个点。 --- configs/nvidia-master.yaml | 3 ++- perf-changelog.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 27104a42e2..eb5be56936 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8638,8 +8638,9 @@ dsv4-fp4-b300-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1] } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 8] } - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1] } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [8] } # DEP8 prefill uses an 8K batch because 16K OOMs in the FP4 MoE intermediate; # decode uses FULL_DECODE_ONLY after the controlled graph test restored decode diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b7082fb68a..b793ce5287 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5363,5 +5363,5 @@ - "Pin simulated acceptance to the committed golden AL: SGLANG_SIMULATE_ACC_LEN=2.49 (golden_al_distribution/dsv4_mtp.yaml, thinking_on, K=3), SGLANG_SIMULATE_ACC_METHOD=match-expected, SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token; EVAL_ONLY runs keep real verification" - "Bump image from lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233 to lmsysorg/sglang:nightly-dev-cu13-20260802-131bd51b, which reads SGLANG_SIMULATE_ACC_TOKEN_MODE (added in v0.5.16)" - "Enable --enable-deepseek-v4-fp4-indexer on the plain-TP arm, --enable-cache-report, SGLANG_DEFAULT_THINKING=1 and SGLANG_DSV4_REASONING_EFFORT=high (the conditions the golden AL curve was measured under), and size the decode CUDA graph to max-running-requests instead of conc" - - "Search space is two points at conc 1, kv-offloading none: TP8 and TP4, matching the topologies of the vLLM dsv4-fp4-b300-vllm-agentic-mtp arms" + - "Search space is four points: TP8 and TP4 at conc 1 without KV offload, plus TP8 at conc 8 with and without HiCache host-DRAM offload. The conc 1 points match the topologies of the vLLM dsv4-fp4-b300-vllm-agentic-mtp arms; conc 8 raises max-running-requests above the observed AgentX subagent fan-out peak and is where the TP8 radix cache starts to outgrow the device pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2452