feat(power): enable official dcgm energy lane for gb200/gb300 1p1d / 打开 gb200/gb300 官方 1P1D 能耗采集

.github/workflows/benchmark-multinode-tmpl.yml

@@ -370,6 +370,12 @@ jobs:
         env:
           RUNNER_TYPE: ${{ inputs.runner }}
         run: |
+          # Launcher stamp supplies the producer SHA unless the
+          # power-producer-sha input is set as a manual override
+          if [ -z "$POWER_PRODUCER_SHA" ] && [ -f power-producer-sha.txt ]; then
+            export POWER_PRODUCER_SHA="$(cat power-producer-sha.txt)"
+            echo "POWER_PRODUCER_SHA derived from launcher stamp: $POWER_PRODUCER_SHA"
+          fi
           # Process each result file
           for result_file in ${RESULT_FILENAME}_*.json; do
             if [ -f "$result_file" ]; then

.github/workflows/test-process-result.yml

@@ -8,12 +8,18 @@ on:
       - '.github/workflows/e2e-tests.yml'
       - '.github/workflows/test-process-result.yml'
       - 'benchmarks/benchmark_lib.sh'
+      - 'benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml'
+      - 'benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml'
+      - 'runners/launch_gb200-nv.sh'
+      - 'runners/launch_gb300-nv.sh'
       - 'utils/aggregate_power.py'
       - 'utils/aggregate_power_multinode.py'
       - 'utils/bench_serving/benchmark_serving.py'
       - 'utils/process_result.py'
       - 'utils/test_aggregate_power.py'
       - 'utils/test_aggregate_power_multinode.py'
+      - 'utils/test_gb200_power_official_contract.py'
+      - 'utils/test_gb300_power_official_contract.py'
       - 'utils/test_process_result.py'
 
 permissions:
@@ -37,9 +43,9 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pytest
+          pip install pytest pyyaml
 
       - name: Run pytest
         run: |
           cd utils
-          python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py test_process_result.py -v
+          python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py test_gb200_power_official_contract.py test_gb300_power_official_contract.py test_process_result.py -v

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml

@@ -119,3 +119,16 @@ benchmark:
   req_rate: "inf"
   random_range_ratio: 0.8
   concurrencies: "1x2x4x8x16x32x64x128"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: true
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 9401

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml

@@ -119,3 +119,18 @@ benchmark:
   req_rate: "inf"
   random_range_ratio: 0.8
   concurrencies: "1x2x4x8x16x32x64x128"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: true
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    # 9401 is already bound by the cluster-level exporter on im-gb300 nodes;
+    # use a port outside that range.
+    port: 19401

runners/launch_gb200-nv.sh

@@ -10,6 +10,12 @@ export SLURM_PARTITION="batch"
 export SLURM_ACCOUNT="benchmark"
 SQUASH_DIR="/mnt/lustre01/users-public/sa-shared"
 
+# dcgm-power producer pin — single source of truth for power lanes. Swap
+# URL+PIN here (and identically in launch_gb300-nv.sh) when the upstream
+# srt-slurm merge lands.
+POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git"
+POWER_SRT_SLURM_PIN="6609d46a4c74ed66fc2f7014d7e790efe1d23bde"
+
 if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then
     if [[ "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" ]]; then
         export MODEL_PATH="/mnt/numa1/models/DeepSeek-V4-Pro"
@@ -282,6 +288,27 @@ import_squash() {
 import_squash "$SQUASH_FILE" "$IMAGE"
 import_squash "$NGINX_SQUASH_FILE" "$NGINX_IMAGE"
 
+# Power lane is recipe-driven: on iff the recipe this run resolves carries an
+# enabled dcgm-power telemetry block. Grep the workspace mirror (it overlays
+# the srt-slurm clone later), since the pin decision precedes the clone.
+USES_DCGM_POWER=0
+_RECIPE_REL="${CONFIG_FILE%%:*}"
+_RECIPE_SRC="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${_RECIPE_REL#recipes/}"
+if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] \
+    && grep -q "provider: dcgm-power" "$_RECIPE_SRC" \
+    && grep -q "enabled: true" "$_RECIPE_SRC"; then
+    USES_DCGM_POWER=1
+fi
+
+if [[ "$USES_DCGM_POWER" == "1" ]]; then
+    DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless"
+    DCGM_EXPORTER_SQSH="${SQUASH_DIR}/$(echo "$DCGM_EXPORTER_IMAGE" | sed 's/[\/:@#]/_/g').sqsh"
+    import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_IMAGE"
+    test -r "$DCGM_EXPORTER_SQSH" || { echo "Error: DCGM exporter squash not readable: $DCGM_EXPORTER_SQSH" >&2; exit 1; }
+    unsquashfs -l "$DCGM_EXPORTER_SQSH" > /dev/null || { echo "Error: DCGM exporter squash invalid: $DCGM_EXPORTER_SQSH" >&2; exit 1; }
+    sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256"
+fi
+
 export EVAL_ONLY="${EVAL_ONLY:-false}"
 
 export ISL="$ISL"
@@ -401,8 +428,18 @@ elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.1" ]]; then
     mkdir -p recipes/sglang/glm5
     cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5" recipes/sglang/glm5
 elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then
-    git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR"
-    cd "$SRT_REPO_DIR"
+    if [[ "$USES_DCGM_POWER" == "1" ]]; then
+        # Power lanes run the exact pinned producer SHA, never a moving
+        # branch; CI derives POWER_PRODUCER_SHA from the stamp file.
+        git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR"
+        cd "$SRT_REPO_DIR"
+        git checkout "$POWER_SRT_SLURM_PIN" || exit 1
+        test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; }
+        git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt"
+    else
+        git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR"
+        cd "$SRT_REPO_DIR"
+    fi
     mkdir -p recipes/sglang/qwen3.5
     cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" recipes/sglang/qwen3.5
 elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.1" ]]; then
@@ -528,6 +565,11 @@ use_segment_sbatch_directive: false
 ${DEFAULT_MOUNTS_BLOCK}
 EOF
 
+# Appended via sed so non-power lanes' generated yaml stays byte-identical.
+if [[ "$USES_DCGM_POWER" == "1" ]]; then
+    sed -i "/^  nginx-sqsh:/a\\  dcgm-exporter: ${DCGM_EXPORTER_SQSH}" srtslurm.yaml
+fi
+
 echo "Generated srtslurm.yaml:"
 cat srtslurm.yaml
 
@@ -631,6 +673,13 @@ echo "Collecting results..."
 
 if [ -d "$LOGS_DIR" ]; then
     echo "Found logs directory: $LOGS_DIR"
+    # Provenance markers travel inside the server-logs bundle so the offline
+    # audit can tie artifacts to the exact producer SHA and exporter image.
+    if [[ "$USES_DCGM_POWER" == "1" ]]; then
+        mkdir -p "$LOGS_DIR/power"
+        cp "$GITHUB_WORKSPACE/exporter-image.sha256" "$LOGS_DIR/power/exporter-image.sha256"
+        cp "$GITHUB_WORKSPACE/power-producer-sha.txt" "$LOGS_DIR/power/power-producer-sha.txt"
+    fi
     cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS"
     bundle_server_logs "$LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"
 else

runners/launch_gb300-nv.sh

@@ -127,6 +127,33 @@ import_squash() {
 import_squash "$SQUASH_FILE" "$IMAGE"
 import_squash "$NGINX_SQUASH_FILE" "$NGINX_IMAGE"
 
+# Power lane detection: a recipe opts in via an enabled dcgm-power telemetry
+# block. CONFIG_FILE is srt-slurm-relative; resolve it against the workspace
+# recipe mirror (the same tree the clone step overlays), since the checkout
+# doesn't exist yet. Recipes that only exist upstream stay non-power.
+USES_DCGM_POWER=0
+_RECIPE_REL="${CONFIG_FILE%%:*}"
+_RECIPE_SRC="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${_RECIPE_REL#recipes/}"
+if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && grep -q "provider: dcgm-power" "$_RECIPE_SRC" && grep -q "enabled: true" "$_RECIPE_SRC"; then
+    USES_DCGM_POWER=1
+fi
+
+# dcgm-power producer pin — single source of truth for power lanes. Swap
+# URL+PIN here (and identically in launch_gb200-nv.sh) when the upstream
+# srt-slurm merge lands.
+POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git"
+POWER_SRT_SLURM_PIN="6609d46a4c74ed66fc2f7014d7e790efe1d23bde"
+
+if [[ "$USES_DCGM_POWER" == "1" ]]; then
+    DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless"
+    DCGM_EXPORTER_SQSH="/data/home/sa-shared/gharunners/squash/$(echo "$DCGM_EXPORTER_IMAGE" | sed 's/[\/:@#]/_/g').sqsh"
+    # import_squash validates the squash with unsquashfs inside the compute-node
+    # srun (the x86 login node cannot inspect the aarch64 image).
+    import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_IMAGE"
+    test -r "$DCGM_EXPORTER_SQSH" || { echo "Error: DCGM exporter squash not readable: $DCGM_EXPORTER_SQSH" >&2; exit 1; }
+    sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256"
+fi
+
 export EVAL_ONLY="${EVAL_ONLY:-false}"
 
 export ISL="$ISL"
@@ -214,12 +241,21 @@ elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then
     # Same branch the identical gb200-fp8 recipes run on. fp4 recipes pin
     # dynamo by version (pip install) and stay on the submission branch they
     # were validated against.
-    git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR"
-    cd "$SRT_REPO_DIR"
-    if [[ $PRECISION == "fp8" ]]; then
-        git checkout v1.0.25
+    if [[ "$USES_DCGM_POWER" == "1" ]]; then
+        git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR"
+        cd "$SRT_REPO_DIR"
+        git checkout "$POWER_SRT_SLURM_PIN" || exit 1
+        # The power lane must run the exact pinned producer SHA, never a moving branch.
+        test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; }
+        git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt"
     else
-        git checkout sa-submission-q2-2026
+        git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR"
+        cd "$SRT_REPO_DIR"
+        if [[ $PRECISION == "fp8" ]]; then
+            git checkout v1.0.25
+        else
+            git checkout sa-submission-q2-2026
+        fi
     fi
     mkdir -p recipes/sglang/qwen3.5
     cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" recipes/sglang/qwen3.5
@@ -312,6 +348,11 @@ containers:
 use_segment_sbatch_directive: false
 EOF
 
+# Appended via sed so non-power lanes' generated yaml stays byte-identical.
+if [[ "$USES_DCGM_POWER" == "1" ]]; then
+    sed -i "/^  nginx-sqsh:/a\\  dcgm-exporter: ${DCGM_EXPORTER_SQSH}" srtslurm.yaml
+fi
+
 echo "Generated srtslurm.yaml:"
 cat srtslurm.yaml
 
@@ -432,6 +473,13 @@ echo "Collecting results..."
 
 if [ -d "$LOGS_DIR" ]; then
     echo "Found logs directory: $LOGS_DIR"
+    # Power provenance markers travel inside the server-logs bundle so the
+    # offline audit can tie artifacts to the exact producer SHA and exporter.
+    if [[ "$USES_DCGM_POWER" == "1" ]]; then
+        mkdir -p "$LOGS_DIR/power"
+        cp "$GITHUB_WORKSPACE/exporter-image.sha256" "$LOGS_DIR/power/exporter-image.sha256"
+        cp "$GITHUB_WORKSPACE/power-producer-sha.txt" "$LOGS_DIR/power/power-producer-sha.txt"
+    fi
     # Tarball + LOGS copy are produced by the EXIT trap defined near
     # JOB_ID extraction (so cancel paths also get them); just log here.
     echo "multinode_server_logs.tar.gz will be (re)produced on script EXIT."