diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 80f9c7c167..6bd33ba7e7 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -370,6 +370,12 @@ jobs: env: RUNNER_TYPE: ${{ inputs.runner }} run: | + # Launcher stamp supplies the producer SHA unless the + # power-producer-sha input is set as a manual override + if [ -z "$POWER_PRODUCER_SHA" ] && [ -f power-producer-sha.txt ]; then + export POWER_PRODUCER_SHA="$(cat power-producer-sha.txt)" + echo "POWER_PRODUCER_SHA derived from launcher stamp: $POWER_PRODUCER_SHA" + fi # Process each result file for result_file in ${RESULT_FILENAME}_*.json; do if [ -f "$result_file" ]; then diff --git a/.github/workflows/test-process-result.yml b/.github/workflows/test-process-result.yml index f81ae79a0d..e6a380bd82 100644 --- a/.github/workflows/test-process-result.yml +++ b/.github/workflows/test-process-result.yml @@ -8,12 +8,18 @@ on: - '.github/workflows/e2e-tests.yml' - '.github/workflows/test-process-result.yml' - 'benchmarks/benchmark_lib.sh' + - 'benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml' + - 'benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml' + - 'runners/launch_gb200-nv.sh' + - 'runners/launch_gb300-nv.sh' - 'utils/aggregate_power.py' - 'utils/aggregate_power_multinode.py' - 'utils/bench_serving/benchmark_serving.py' - 'utils/process_result.py' - 'utils/test_aggregate_power.py' - 'utils/test_aggregate_power_multinode.py' + - 'utils/test_gb200_power_official_contract.py' + - 'utils/test_gb300_power_official_contract.py' - 'utils/test_process_result.py' permissions: @@ -37,9 +43,9 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pytest + pip install pytest pyyaml - name: Run pytest run: | cd utils - python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py test_process_result.py -v + python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py test_gb200_power_official_contract.py test_gb300_power_official_contract.py test_process_result.py -v diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml index d869b247a7..cc9a7335f5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml @@ -119,3 +119,16 @@ benchmark: req_rate: "inf" random_range_ratio: 0.8 concurrencies: "1x2x4x8x16x32x64x128" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml index 0a938fb48c..359eab8a66 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml @@ -119,3 +119,18 @@ benchmark: req_rate: "inf" random_range_ratio: 0.8 concurrencies: "1x2x4x8x16x32x64x128" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + # 9401 is already bound by the cluster-level exporter on im-gb300 nodes; + # use a port outside that range. + port: 19401 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 7bccc43e9c..46429e5482 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -10,6 +10,12 @@ export SLURM_PARTITION="batch" export SLURM_ACCOUNT="benchmark" SQUASH_DIR="/mnt/lustre01/users-public/sa-shared" +# dcgm-power producer pin — single source of truth for power lanes. Swap +# URL+PIN here (and identically in launch_gb300-nv.sh) when the upstream +# srt-slurm merge lands. +POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" +POWER_SRT_SLURM_PIN="6609d46a4c74ed66fc2f7014d7e790efe1d23bde" + if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then if [[ "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" ]]; then export MODEL_PATH="/mnt/numa1/models/DeepSeek-V4-Pro" @@ -282,6 +288,27 @@ import_squash() { import_squash "$SQUASH_FILE" "$IMAGE" import_squash "$NGINX_SQUASH_FILE" "$NGINX_IMAGE" +# Power lane is recipe-driven: on iff the recipe this run resolves carries an +# enabled dcgm-power telemetry block. Grep the workspace mirror (it overlays +# the srt-slurm clone later), since the pin decision precedes the clone. +USES_DCGM_POWER=0 +_RECIPE_REL="${CONFIG_FILE%%:*}" +_RECIPE_SRC="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${_RECIPE_REL#recipes/}" +if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] \ + && grep -q "provider: dcgm-power" "$_RECIPE_SRC" \ + && grep -q "enabled: true" "$_RECIPE_SRC"; then + USES_DCGM_POWER=1 +fi + +if [[ "$USES_DCGM_POWER" == "1" ]]; then + DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" + DCGM_EXPORTER_SQSH="${SQUASH_DIR}/$(echo "$DCGM_EXPORTER_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_IMAGE" + test -r "$DCGM_EXPORTER_SQSH" || { echo "Error: DCGM exporter squash not readable: $DCGM_EXPORTER_SQSH" >&2; exit 1; } + unsquashfs -l "$DCGM_EXPORTER_SQSH" > /dev/null || { echo "Error: DCGM exporter squash invalid: $DCGM_EXPORTER_SQSH" >&2; exit 1; } + sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256" +fi + export EVAL_ONLY="${EVAL_ONLY:-false}" export ISL="$ISL" @@ -401,8 +428,18 @@ elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.1" ]]; then mkdir -p recipes/sglang/glm5 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5" recipes/sglang/glm5 elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" + if [[ "$USES_DCGM_POWER" == "1" ]]; then + # Power lanes run the exact pinned producer SHA, never a moving + # branch; CI derives POWER_PRODUCER_SHA from the stamp file. + git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout "$POWER_SRT_SLURM_PIN" || exit 1 + test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; } + git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" + else + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + fi mkdir -p recipes/sglang/qwen3.5 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" recipes/sglang/qwen3.5 elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.1" ]]; then @@ -528,6 +565,11 @@ use_segment_sbatch_directive: false ${DEFAULT_MOUNTS_BLOCK} EOF +# Appended via sed so non-power lanes' generated yaml stays byte-identical. +if [[ "$USES_DCGM_POWER" == "1" ]]; then + sed -i "/^ nginx-sqsh:/a\\ dcgm-exporter: ${DCGM_EXPORTER_SQSH}" srtslurm.yaml +fi + echo "Generated srtslurm.yaml:" cat srtslurm.yaml @@ -631,6 +673,13 @@ echo "Collecting results..." if [ -d "$LOGS_DIR" ]; then echo "Found logs directory: $LOGS_DIR" + # Provenance markers travel inside the server-logs bundle so the offline + # audit can tie artifacts to the exact producer SHA and exporter image. + if [[ "$USES_DCGM_POWER" == "1" ]]; then + mkdir -p "$LOGS_DIR/power" + cp "$GITHUB_WORKSPACE/exporter-image.sha256" "$LOGS_DIR/power/exporter-image.sha256" + cp "$GITHUB_WORKSPACE/power-producer-sha.txt" "$LOGS_DIR/power/power-producer-sha.txt" + fi cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS" bundle_server_logs "$LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" else diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 7232174d92..5dbaaf60fd 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -127,6 +127,33 @@ import_squash() { import_squash "$SQUASH_FILE" "$IMAGE" import_squash "$NGINX_SQUASH_FILE" "$NGINX_IMAGE" +# Power lane detection: a recipe opts in via an enabled dcgm-power telemetry +# block. CONFIG_FILE is srt-slurm-relative; resolve it against the workspace +# recipe mirror (the same tree the clone step overlays), since the checkout +# doesn't exist yet. Recipes that only exist upstream stay non-power. +USES_DCGM_POWER=0 +_RECIPE_REL="${CONFIG_FILE%%:*}" +_RECIPE_SRC="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${_RECIPE_REL#recipes/}" +if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && grep -q "provider: dcgm-power" "$_RECIPE_SRC" && grep -q "enabled: true" "$_RECIPE_SRC"; then + USES_DCGM_POWER=1 +fi + +# dcgm-power producer pin — single source of truth for power lanes. Swap +# URL+PIN here (and identically in launch_gb200-nv.sh) when the upstream +# srt-slurm merge lands. +POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" +POWER_SRT_SLURM_PIN="6609d46a4c74ed66fc2f7014d7e790efe1d23bde" + +if [[ "$USES_DCGM_POWER" == "1" ]]; then + DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" + DCGM_EXPORTER_SQSH="/data/home/sa-shared/gharunners/squash/$(echo "$DCGM_EXPORTER_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + # import_squash validates the squash with unsquashfs inside the compute-node + # srun (the x86 login node cannot inspect the aarch64 image). + import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_IMAGE" + test -r "$DCGM_EXPORTER_SQSH" || { echo "Error: DCGM exporter squash not readable: $DCGM_EXPORTER_SQSH" >&2; exit 1; } + sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256" +fi + export EVAL_ONLY="${EVAL_ONLY:-false}" export ISL="$ISL" @@ -214,12 +241,21 @@ elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then # Same branch the identical gb200-fp8 recipes run on. fp4 recipes pin # dynamo by version (pip install) and stay on the submission branch they # were validated against. - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - if [[ $PRECISION == "fp8" ]]; then - git checkout v1.0.25 + if [[ "$USES_DCGM_POWER" == "1" ]]; then + git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout "$POWER_SRT_SLURM_PIN" || exit 1 + # The power lane must run the exact pinned producer SHA, never a moving branch. + test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; } + git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" else - git checkout sa-submission-q2-2026 + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + if [[ $PRECISION == "fp8" ]]; then + git checkout v1.0.25 + else + git checkout sa-submission-q2-2026 + fi fi mkdir -p recipes/sglang/qwen3.5 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" recipes/sglang/qwen3.5 @@ -312,6 +348,11 @@ containers: use_segment_sbatch_directive: false EOF +# Appended via sed so non-power lanes' generated yaml stays byte-identical. +if [[ "$USES_DCGM_POWER" == "1" ]]; then + sed -i "/^ nginx-sqsh:/a\\ dcgm-exporter: ${DCGM_EXPORTER_SQSH}" srtslurm.yaml +fi + echo "Generated srtslurm.yaml:" cat srtslurm.yaml @@ -432,6 +473,13 @@ echo "Collecting results..." if [ -d "$LOGS_DIR" ]; then echo "Found logs directory: $LOGS_DIR" + # Power provenance markers travel inside the server-logs bundle so the + # offline audit can tie artifacts to the exact producer SHA and exporter. + if [[ "$USES_DCGM_POWER" == "1" ]]; then + mkdir -p "$LOGS_DIR/power" + cp "$GITHUB_WORKSPACE/exporter-image.sha256" "$LOGS_DIR/power/exporter-image.sha256" + cp "$GITHUB_WORKSPACE/power-producer-sha.txt" "$LOGS_DIR/power/power-producer-sha.txt" + fi # Tarball + LOGS copy are produced by the EXIT trap defined near # JOB_ID extraction (so cancel paths also get them); just log here. echo "multinode_server_logs.tar.gz will be (re)produced on script EXIT." diff --git a/utils/test_gb200_power_official_contract.py b/utils/test_gb200_power_official_contract.py new file mode 100644 index 0000000000..21b652dacf --- /dev/null +++ b/utils/test_gb200_power_official_contract.py @@ -0,0 +1,197 @@ +"""Static contract for the official GB200 dcgm-power lane. + +Power behavior is recipe-gated: a run turns power on iff the recipe it +resolves carries an enabled dcgm-power telemetry block, and only then +provisions the exporter and clones the pinned producer fork. These tests read +launcher/recipe/workflow text; nothing is executed. Cross-launcher and +repo-wide invariants live here; the GB300 twin imports the shared helpers. +""" + +import re +import subprocess +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +RECIPE_PATH = REPO_ROOT / "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml" +GB200_LAUNCHER = REPO_ROOT / "runners/launch_gb200-nv.sh" +GB300_LAUNCHER = REPO_ROOT / "runners/launch_gb300-nv.sh" +TMPL_PATH = REPO_ROOT / ".github/workflows/benchmark-multinode-tmpl.yml" + +STAMP_NAME = "power-producer-sha.txt" +EXPORTER_IMAGE = "nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" + + +def launcher_power_constants(path): + """Derive URL+PIN from a launcher — their literals live nowhere else.""" + text = path.read_text() + url = re.search(r'^POWER_SRT_SLURM_URL="([^"]+)"$', text, re.M) + pin = re.search(r'^POWER_SRT_SLURM_PIN="([0-9a-f]{40})"$', text, re.M) + assert url and pin, path + return url.group(1), pin.group(1) + + +FORK_URL, PRODUCER_PIN = launcher_power_constants(GB200_LAUNCHER) + + +def git_grep_lines(*args): + proc = subprocess.run( + ["git", "grep", *args], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + # git grep exits 1 when nothing matches. + assert proc.returncode in (0, 1), proc.stderr + return proc.stdout.splitlines() + + +def assert_recipe_driven_detection(launcher): + assert "USES_DCGM_POWER=0" in launcher + assert '_RECIPE_REL="${CONFIG_FILE%%:*}"' in launcher + assert ( + '_RECIPE_SRC="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${_RECIPE_REL#recipes/}"' + in launcher + ) + # Upstream-only recipes have no workspace mirror and must stay non-power. + assert '-f "$_RECIPE_SRC"' in launcher + assert 'grep -q "provider: dcgm-power" "$_RECIPE_SRC"' in launcher + assert 'grep -q "enabled: true" "$_RECIPE_SRC"' in launcher + assert "USES_DCGM_POWER=1" in launcher + # Exporter provisioning, pinned clone, srtslurm.yaml injection, and audit + # copies are each gated so non-power lanes keep the pre-existing path. + assert launcher.count('if [[ "$USES_DCGM_POWER" == "1" ]]; then') >= 4 + + +def assert_pinned_clone_contract(launcher): + assert f'POWER_SRT_SLURM_URL="{FORK_URL}"' in launcher + assert f'POWER_SRT_SLURM_PIN="{PRODUCER_PIN}"' in launcher + assert 'git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR"' in launcher + assert 'git checkout "$POWER_SRT_SLURM_PIN" || exit 1' in launcher + assert 'test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN"' in launcher + assert f'git rev-parse HEAD > "$GITHUB_WORKSPACE/{STAMP_NAME}"' in launcher + + +def assert_exporter_provisioning(launcher): + assert f'DCGM_EXPORTER_IMAGE="{EXPORTER_IMAGE}"' in launcher + assert 'import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_IMAGE"' in launcher + assert 'test -r "$DCGM_EXPORTER_SQSH"' in launcher + assert ( + 'sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256"' + in launcher + ) + # Container line lands after the unique nginx-sqsh key so the heredoc + # output of non-power lanes stays byte-identical. + assert '"/^ nginx-sqsh:/a' in launcher + assert 'dcgm-exporter: ${DCGM_EXPORTER_SQSH}" srtslurm.yaml' in launcher + assert ( + 'cp "$GITHUB_WORKSPACE/exporter-image.sha256" "$LOGS_DIR/power/exporter-image.sha256"' + in launcher + ) + assert f'cp "$GITHUB_WORKSPACE/{STAMP_NAME}" "$LOGS_DIR/power/{STAMP_NAME}"' in launcher + + +def test_recipe_declares_enabled_dcgm_power_lane(): + recipe = yaml.safe_load(RECIPE_PATH.read_text()) + + telemetry = recipe["telemetry"] + assert telemetry["enabled"] is True + assert telemetry["provider"] == "dcgm-power" + assert telemetry["required"] is True + assert telemetry["dcgm_exporter"]["container_image"] == "dcgm-exporter" + assert telemetry["dcgm_exporter"]["port"] == 9401 + + # Official recipes keep the full sweep ladder; only the telemetry block + # was added. + assert recipe["benchmark"]["concurrencies"] == "1x2x4x8x16x32x64x128" + assert recipe["resources"]["prefill_nodes"] == 1 + assert recipe["resources"]["decode_nodes"] == 1 + assert recipe["resources"]["gpus_per_node"] == 4 + + +def test_launcher_detects_power_lane_from_recipe(): + assert_recipe_driven_detection(GB200_LAUNCHER.read_text()) + + +def test_launcher_provisions_exporter_through_squash_dir_cache(): + launcher = GB200_LAUNCHER.read_text() + assert_exporter_provisioning(launcher) + assert 'DCGM_EXPORTER_SQSH="${SQUASH_DIR}/' in launcher + assert 'unsquashfs -l "$DCGM_EXPORTER_SQSH"' in launcher + + +def test_launcher_pins_power_producer_and_keeps_nonpower_clone(): + launcher = GB200_LAUNCHER.read_text() + assert_pinned_clone_contract(launcher) + # Non-power qwen3.5 keeps the upstream clone with no ref pin. + assert 'git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR"' in launcher + assert "v1.0.25" not in launcher + + +def test_pin_constants_identical_across_launchers(): + assert launcher_power_constants(GB300_LAUNCHER) == (FORK_URL, PRODUCER_PIN) + assert re.fullmatch(r"[0-9a-f]{40}", PRODUCER_PIN) + assert FORK_URL.startswith("https://") and FORK_URL.endswith("/srt-slurm.git") + + +def test_pin_literal_lives_only_in_the_two_launchers(): + hits = git_grep_lines("-lF", PRODUCER_PIN) + assert sorted(hits) == [ + "runners/launch_gb200-nv.sh", + "runners/launch_gb300-nv.sh", + ] + + +def test_exactly_two_recipes_opt_into_dcgm_power(): + hits = git_grep_lines( + "-lF", "provider: dcgm-power", "--", "benchmarks/multi_node/srt-slurm-recipes" + ) + assert sorted(hits) == [ + "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml", + "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml", + ] + + +def test_lane_and_pin_have_no_env_override_backdoor(): + # All assignments are literals: no environment variable can flip the + # lane or redirect the pinned clone. + for path in (GB200_LAUNCHER, GB300_LAUNCHER): + text = path.read_text() + assert re.findall(r"^\s*USES_DCGM_POWER=(\S+)$", text, re.M) == ["0", "1"], path + for name in ("POWER_SRT_SLURM_URL", "POWER_SRT_SLURM_PIN"): + values = re.findall(rf"^\s*{name}=(.*)$", text, re.M) + assert len(values) == 1 and "$" not in values[0], (path, name) + + +def test_workflows_carry_no_producer_sha_literal(): + offenders = [] + for wf in sorted((REPO_ROOT / ".github/workflows").iterdir()): + if wf.suffix not in (".yml", ".yaml"): + continue + for lineno, line in enumerate(wf.read_text().splitlines(), 1): + # Action pins ("uses: ...@") are the only allowed 40-hex. + if "uses:" in line: + continue + if re.search(r"\b[0-9a-f]{40}\b", line): + offenders.append(f"{wf.name}:{lineno}") + assert offenders == [] + + +def test_stamp_filename_agrees_between_launchers_and_template(): + stamp_writes = set() + for path in (GB200_LAUNCHER, GB300_LAUNCHER): + match = re.search( + r'git rev-parse HEAD > "\$GITHUB_WORKSPACE/([^"]+)"', path.read_text() + ) + assert match, path + stamp_writes.add(match.group(1)) + + tmpl = TMPL_PATH.read_text() + read_match = re.search(r'export POWER_PRODUCER_SHA="\$\(cat ([^)]+)\)"', tmpl) + assert read_match + assert stamp_writes == {STAMP_NAME} == {read_match.group(1)} + + # The manual input stays the override: the stamp only fills an empty env. + assert f'if [ -z "$POWER_PRODUCER_SHA" ] && [ -f {STAMP_NAME} ]; then' in tmpl + assert "POWER_PRODUCER_SHA: ${{ inputs.power-producer-sha }}" in tmpl diff --git a/utils/test_gb300_power_official_contract.py b/utils/test_gb300_power_official_contract.py new file mode 100644 index 0000000000..42035225ab --- /dev/null +++ b/utils/test_gb300_power_official_contract.py @@ -0,0 +1,57 @@ +"""Static contract for the official GB300 dcgm-power lane. + +Shares helpers with the GB200 contract; GB300 specifics are the literal +shared squash cache path (no SQUASH_DIR var in this launcher), the 19401 +exporter port, and the preserved v1.0.25/sa-submission non-power refs. +""" + +import yaml + +from test_gb200_power_official_contract import ( + REPO_ROOT, + assert_exporter_provisioning, + assert_pinned_clone_contract, + assert_recipe_driven_detection, +) + +RECIPE_PATH = REPO_ROOT / "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml" +LAUNCHER_PATH = REPO_ROOT / "runners/launch_gb300-nv.sh" + + +def test_recipe_declares_enabled_dcgm_power_lane(): + recipe = yaml.safe_load(RECIPE_PATH.read_text()) + + telemetry = recipe["telemetry"] + assert telemetry["enabled"] is True + assert telemetry["provider"] == "dcgm-power" + assert telemetry["required"] is True + assert telemetry["dcgm_exporter"]["container_image"] == "dcgm-exporter" + # 9401 is already bound by the cluster-level exporter on im-gb300 nodes. + assert telemetry["dcgm_exporter"]["port"] == 19401 + + assert recipe["benchmark"]["concurrencies"] == "1x2x4x8x16x32x64x128" + assert recipe["resources"]["prefill_nodes"] == 1 + assert recipe["resources"]["decode_nodes"] == 1 + assert recipe["resources"]["gpus_per_node"] == 4 + + +def test_launcher_detects_power_lane_from_recipe(): + assert_recipe_driven_detection(LAUNCHER_PATH.read_text()) + + +def test_launcher_provisions_exporter_through_shared_squash_path(): + launcher = LAUNCHER_PATH.read_text() + assert_exporter_provisioning(launcher) + # No SQUASH_DIR var here; the /data/ mount avoids the /home NFS ELOOP bug. + assert 'DCGM_EXPORTER_SQSH="/data/home/sa-shared/gharunners/squash/' in launcher + + +def test_launcher_pins_power_producer(): + assert_pinned_clone_contract(LAUNCHER_PATH.read_text()) + + +def test_non_power_lane_keeps_existing_ref_logic(): + launcher = LAUNCHER_PATH.read_text() + assert 'git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR"' in launcher + assert "git checkout v1.0.25" in launcher + assert "git checkout sa-submission-q2-2026" in launcher