NVIDIA · j-rausch · Aug 24, 2026 · Aug 25, 2026 · Aug 25, 2026 · Aug 25, 2026
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -6,7 +6,7 @@ Changelog
 
 **New Features**
 
-- Add Puzzletron dynamic post-MIP downstream evaluation through ``lmms-eval`` with vLLM-backed checkpoint evaluation, setup-wizard topology/resource prompts, and an opt-in Nemotron-3 Nano 30B A3B BF16 example flow.
+- Add Puzzletron dynamic post-MIP downstream evaluation through ``lmms-eval`` with vLLM-backed checkpoint evaluation, setup-wizard topology/resource prompts, non-interactive setup automation, and an opt-in Nemotron-3 Nano 30B A3B BF16 example flow.
 - Add the ``day0-release`` agent skill (``.agents/skills/day0-release/``), a deterministic end-to-end driver that chains the PTQ → evaluation → comparison skills (the evaluation stage deploys the checkpoint itself) with an enforced gate after each stage and returns a publish decision (ACCEPT / REGRESSION / ANOMALOUS / INFEASIBLE). Ships three GPU-free, unit-tested gate scripts (``gate_ptq.py``, ``gate_run.py``, ``gate_compare.py``) that validate checkpoint coverage, evaluation-run completeness, and baseline-vs-candidate accuracy threshold. v1 reports and stops on regression; the recipe-search loop is deferred.
 - Add **streaming** speculative-decoding training (EAGLE3 / DFlash): the draft trains on base-model hidden states produced on the fly by a co-located ``vllm serve`` (no disk dump), moved trainer-side over NIXL RDMA, scaling to multi-node (dedicated serve replicas + DDP trainers). New launcher examples for NVFP4 Kimi-K2.5 / K2.6 on GB200/aarch64 under ``tools/launcher/examples/moonshotai/``.
 

@@ -85,6 +85,21 @@ The defaults file is loaded only when passed explicitly and takes precedence
 over the selected profile. To expose every per-section and nested setting, use
 the advanced flow explicitly:
 
+Automation can use the same setup entry point without answering prompts. The
+defaults file must provide every required value that has no resolved default:
+
+```bash
+python examples/puzzletron/puzzletron_setup_v2.py \
+  --defaults /path/to/setup-v2-defaults.yaml \
+  --campaign-dir /path/to/campaign \
+  --profile smoke \
+  --non-interactive
+```
+
+Non-interactive setup fails instead of guessing when a required answer has no
+resolved default. It generates and validates the same smoke and production
+bundles as the interactive wizard.
+
 ```bash
 python examples/puzzletron/puzzletron_setup_v2.py --full
 ```
@@ -269,13 +284,15 @@ git -C "${AUTOMODEL_ROOT}" rev-parse HEAD
 ```
 
 ```bash
-python - <<'PY'
+PYTHONPATH="${MODEL_OPT_ROOT}" python - <<'PY'
 import importlib.metadata as metadata
 import json
 import os
 
 from packaging.version import Version
 
+from examples.puzzletron.ci_environment import verify_installed_vcs_source
+
 import aiperf
 import lmms_eval
 import modelopt
@@ -307,10 +324,17 @@ assert Version(metadata.version("torchvision")).release == Version(
     ci_environment["torchvision"]
 ).release
 assert transformers.__version__ == ci_environment["transformers"]
-assert metadata.version("lmms-eval") == ci_environment["lmms_eval"]
+assert Version(metadata.version("lmms-eval")).base_version == (
+    ci_environment["lmms_eval"]["base_version"]
+)
 assert Version(metadata.version("nemo-automodel")).base_version == (
     ci_environment["nemo_automodel"]["base_version"]
 )
+for package, source in (
+    ("lmms-eval", ci_environment["lmms_eval"]),
+    ("nemo-automodel", ci_environment["nemo_automodel"]),
+):
+    verify_installed_vcs_source(package, source)
 assert torch.version.cuda == "12.9"
 assert torch.cuda.is_available()
 PY
@@ -492,6 +516,11 @@ The setup wizard can also add downstream evaluation for materialized campaign
 candidates. See [post-MIP pipelines](docs/post_mip_pipeline.md) to configure it
 or add it to an existing campaign.
 
+Remote model code and AIPerf v0.11 online tokenizer resolution are disabled by
+default. Enable remote code only for a trusted model source. The tokenizer
+compatibility option permits the AIPerf child process to resolve its tokenizer
+online even when the surrounding campaign is configured for offline loading.
+
 ### Legacy checked-in Nano campaign
 
 The checked-in Nano experiment uses the legacy `zero_shot_evaluation`,

@@ -1,11 +1,15 @@
 {
   "schema_version": 1,
-  "scope": "puzzletron_v2_cpu_ci",
+  "scope": "puzzletron_v2_ci",
   "python": "3.12",
   "torch": "2.11.0",
   "torchvision": "0.26.0",
   "transformers": "5.8.1",
-  "lmms_eval": "0.7.0",
+  "lmms_eval": {
+    "base_version": "0.7.0",
+    "repository": "https://github.com/EvolvingLMMs-Lab/lmms-eval.git",
+    "commit": "15c32bfec165df13c269ddd3cda03b2ed9137825"
+  },
   "nemo_automodel": {
     "base_version": "0.5.0",
     "repository": "https://github.com/Separius/Automodel.git",

@@ -0,0 +1,68 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Verification helpers for the pinned Puzzletron CI environment."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from importlib import metadata
+from typing import Any
+from urllib.parse import unquote, urlparse
+
+__all__ = ["verify_installed_vcs_source"]
+
+
+def _normalized_repository(url: object) -> str:
+    return str(url or "").removesuffix(".git").rstrip("/")
+
+
+def _installed_vcs_source(package: str) -> tuple[str | None, str | None]:
+    payload = json.loads(metadata.distribution(package).read_text("direct_url.json") or "{}")
+    vcs_info = payload.get("vcs_info") or {}
+    if vcs_info.get("commit_id"):
+        return payload.get("url"), vcs_info["commit_id"]
+    if (payload.get("dir_info") or {}).get("editable") and str(payload.get("url", "")).startswith(
+        "file:"
+    ):
+        root = unquote(urlparse(payload["url"]).path)
+        repository = subprocess.check_output(
+            ["git", "-C", root, "remote", "get-url", "origin"], text=True
+        ).strip()
+        commit = subprocess.check_output(
+            ["git", "-C", root, "rev-parse", "HEAD"], text=True
+        ).strip()
+        dirty = subprocess.check_output(
+            ["git", "-C", root, "status", "--porcelain", "--untracked-files=all"],
+            text=True,
+        ).strip()
+        if dirty:
+            raise RuntimeError(f"Pinned Puzzletron dependency {package!r} is dirty: {dirty}")
+        return repository, commit
+    return payload.get("url"), vcs_info.get("commit_id")
+
+
+def verify_installed_vcs_source(package: str, expected: dict[str, Any]) -> None:
+    """Require an installed VCS dependency to match its repository and commit."""
+
+    repository, commit = _installed_vcs_source(package)
+    expected_source = (_normalized_repository(expected["repository"]), expected["commit"])
+    actual_source = (_normalized_repository(repository), commit)
+    if actual_source != expected_source:
+        raise RuntimeError(
+            f"Pinned Puzzletron dependency {package!r} source mismatch: "
+            f"actual={actual_source!r}, expected={expected_source!r}"
+        )
@@ -1,6 +1,18 @@
 #!/usr/bin/env bash
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 set -Eeuo pipefail
 
@@ -10,10 +22,12 @@ set -Eeuo pipefail
 : "${WORKER_COUNT:?set WORKER_COUNT to the number of worker groups}"
 : "${PUZZLETRON_GROUP_INDEX:=${PUZZLETRON_TASK_INDEX:-${SLURM_PROCID:-}}}"
 : "${PUZZLETRON_GROUP_INDEX:?run this script as one orchestrator worker-group task}"
+: "${PUZZLETRON_GROUP_RANK:=0}"
 
 PYTHON_BIN="${PYTHON_BIN:-python}"
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
 GROUP_INDEX="${PUZZLETRON_GROUP_INDEX}"
+GROUP_RANK="${PUZZLETRON_GROUP_RANK}"
 JOB_ID="${SLURM_JOB_ID:-local}"
 WORKER_PREFIX="${JOB_ID}-depth-"
 MANIFEST_PATH="${CAMPAIGN_DIR}/manifest.json"
@@ -38,7 +52,7 @@ cleanup() {
   local rc=$?
   trap - EXIT INT TERM
   set +e
-  if [[ "${GROUP_INDEX}" == "0" ]]; then
+  if [[ "${GROUP_INDEX}" == "0" && "${GROUP_RANK}" == "0" ]]; then
     drain_workers
   fi
   if [[ -n "${worker_pid}" ]] && kill -0 "${worker_pid}" 2>/dev/null; then
@@ -50,7 +64,7 @@ cleanup() {
 trap cleanup EXIT INT TERM
 
 # Rank 0 creates the shared campaign before any worker attempts to open it.
-if [[ "${GROUP_INDEX}" == "0" && ! -f "${MANIFEST_PATH}" ]]; then
+if [[ "${GROUP_INDEX}" == "0" && "${GROUP_RANK}" == "0" && ! -f "${MANIFEST_PATH}" ]]; then
   CUDA_VISIBLE_DEVICES="" "${PYTHON_BIN}" \
     -m modelopt.torch.puzzletron.distributed_eval.cli init \
     --campaign-dir "${CAMPAIGN_DIR}" \
@@ -72,21 +86,17 @@ done
 
 # Every scheduler task owns one GPU slice and starts one worker group. Multiple
 # independent worker groups may share a node.
-export NNODES=1
-export NODE_RANK=0
 export NPROC_PER_NODE="${NPROC_PER_NODE:-${WORLD_SIZE}}"
 export WORKER_GROUP_INDEX="${GROUP_INDEX}"
 export WORKER_ID="${WORKER_PREFIX}${GROUP_INDEX}"
 export WORKER_HOST="${WORKER_HOST:-$(hostname -f)}"
 export WORKER_PORT="${WORKER_PORT:-$((5010 + GROUP_INDEX))}"
-export RDZV_ENDPOINT="127.0.0.1:$((29500 + GROUP_INDEX))"
-export RDZV_ID="depth-${JOB_ID}-${GROUP_INDEX}"
 
 bash "${SCRIPT_DIR}/run_worker.sh" &
 worker_pid=$!
 
 coordinator_rc=0
-if [[ "${GROUP_INDEX}" == "0" ]]; then
+if [[ "${GROUP_INDEX}" == "0" && "${GROUP_RANK}" == "0" ]]; then
   # Do not start depth iteration zero until every resident model is ready.
   CUDA_VISIBLE_DEVICES="" "${PYTHON_BIN}" - \
       "${CAMPAIGN_DIR}" \

@@ -1,6 +1,18 @@
 #!/usr/bin/env bash
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 set -Eeuo pipefail
 
@@ -10,10 +22,12 @@ set -Eeuo pipefail
 : "${WORKER_COUNT:?set WORKER_COUNT to the number of worker groups}"
 : "${PUZZLETRON_GROUP_INDEX:=${PUZZLETRON_TASK_INDEX:-${SLURM_PROCID:-}}}"
 : "${PUZZLETRON_GROUP_INDEX:?run this script as one orchestrator worker-group task}"
+: "${PUZZLETRON_GROUP_RANK:=0}"
 
 PYTHON_BIN="${PYTHON_BIN:-python}"
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
 GROUP_INDEX="${PUZZLETRON_GROUP_INDEX}"
+GROUP_RANK="${PUZZLETRON_GROUP_RANK}"
 JOB_ID="${SLURM_JOB_ID:-local}"
 WORKER_PREFIX="${JOB_ID}-replacement-"
 MANIFEST_PATH="${CAMPAIGN_DIR}/manifest.json"
@@ -38,7 +52,7 @@ cleanup() {
   local rc=$?
   trap - EXIT INT TERM
   set +e
-  if [[ "${GROUP_INDEX}" == "0" ]]; then
+  if [[ "${GROUP_INDEX}" == "0" && "${GROUP_RANK}" == "0" ]]; then
     drain_workers
   fi
   if [[ -n "${worker_pid}" ]] && kill -0 "${worker_pid}" 2>/dev/null; then
@@ -49,7 +63,7 @@ cleanup() {
 }
 trap cleanup EXIT INT TERM
 
-if [[ "${GROUP_INDEX}" == "0" && ! -f "${MANIFEST_PATH}" ]]; then
+if [[ "${GROUP_INDEX}" == "0" && "${GROUP_RANK}" == "0" && ! -f "${MANIFEST_PATH}" ]]; then
   CUDA_VISIBLE_DEVICES="" "${PYTHON_BIN}" \
     -m modelopt.torch.puzzletron.distributed_eval.cli init \
     --campaign-dir "${CAMPAIGN_DIR}" \
@@ -69,21 +83,17 @@ while [[ ! -f "${MANIFEST_PATH}" ]]; do
   sleep 1
 done
 
-export NNODES=1
-export NODE_RANK=0
 export NPROC_PER_NODE="${NPROC_PER_NODE:-${WORLD_SIZE}}"
 export WORKER_GROUP_INDEX="${GROUP_INDEX}"
 export WORKER_ID="${WORKER_PREFIX}${GROUP_INDEX}"
 export WORKER_HOST="${WORKER_HOST:-$(hostname -f)}"
 export WORKER_PORT="${WORKER_PORT:-$((5010 + GROUP_INDEX))}"
-export RDZV_ENDPOINT="127.0.0.1:$((29500 + GROUP_INDEX))"
-export RDZV_ID="replacement-${JOB_ID}-${GROUP_INDEX}"
 
 bash "${SCRIPT_DIR}/run_worker.sh" &
 worker_pid=$!
 
 coordinator_rc=0
-if [[ "${GROUP_INDEX}" == "0" ]]; then
+if [[ "${GROUP_INDEX}" == "0" && "${GROUP_RANK}" == "0" ]]; then
   CUDA_VISIBLE_DEVICES="" "${PYTHON_BIN}" - \
       "${CAMPAIGN_DIR}" \
       "${WORKER_COUNT}" \

@@ -1,16 +1,41 @@
 #!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 set -Eeuo pipefail
 
 : "${CAMPAIGN_DIR:?set CAMPAIGN_DIR}"
 : "${CONFIG_PATH:?set CONFIG_PATH}"
 
 PYTHON_BIN="${PYTHON_BIN:-python}"
 TORCHRUN="${TORCHRUN:-torchrun}"
-NNODES="${NNODES:-1}"
 NPROC_PER_NODE="${NPROC_PER_NODE:-8}"
-NODE_RANK="${NODE_RANK:-0}"
-RDZV_ID="${RDZV_ID:-distributed-eval-${SLURM_JOB_ID:-local}}"
-RDZV_ENDPOINT="${RDZV_ENDPOINT:-127.0.0.1:29500}"
+if [[ -n "${PUZZLETRON_GROUP_SIZE:-}" ]]; then
+  : "${PUZZLETRON_GROUP_RANK:?set PUZZLETRON_GROUP_RANK with task identity}"
+  : "${PUZZLETRON_RENDEZVOUS_ENDPOINT:?set PUZZLETRON_RENDEZVOUS_ENDPOINT with task identity}"
+  : "${PUZZLETRON_RENDEZVOUS_ID:?set PUZZLETRON_RENDEZVOUS_ID with task identity}"
+  NNODES="${PUZZLETRON_GROUP_SIZE}"
+  NODE_RANK="${PUZZLETRON_GROUP_RANK}"
+  RDZV_ENDPOINT="${PUZZLETRON_RENDEZVOUS_ENDPOINT}"
+  RDZV_ID="${PUZZLETRON_RENDEZVOUS_ID}"
+else
+  NNODES="${NNODES:-1}"
+  NODE_RANK="${NODE_RANK:-0}"
+  RDZV_ID="${RDZV_ID:-distributed-eval-${SLURM_JOB_ID:-local}}"
+  RDZV_ENDPOINT="${RDZV_ENDPOINT:-127.0.0.1:29500}"
+fi
 WORKER_HOST="${WORKER_HOST:-$(hostname -f)}"
 WORKER_PORT="${WORKER_PORT:-5010}"
 WORKER_ID="${WORKER_ID:-${SLURM_JOB_ID:-local}-group-${WORKER_GROUP_INDEX:-0}}"
@@ -25,18 +50,23 @@ fi
 export TORCH_NCCL_ASYNC_ERROR_HANDLING=1
 export PYTHONUNBUFFERED=1
 
-exec "${TORCHRUN}" \
-  --nnodes "${NNODES}" \
-  --nproc-per-node "${NPROC_PER_NODE}" \
-  --node-rank "${NODE_RANK}" \
-  --rdzv-backend c10d \
-  --rdzv-id "${RDZV_ID}" \
-  --rdzv-endpoint "${RDZV_ENDPOINT}" \
-  -m modelopt.torch.puzzletron.distributed_eval.cli worker \
-  --campaign-dir "${CAMPAIGN_DIR}" \
-  --config "${CONFIG_PATH}" \
-  --host "${WORKER_HOST}" \
-  --port "${WORKER_PORT}" \
-  --worker-id "${WORKER_ID}" \
-  --heartbeat-seconds "${HEARTBEAT_SECONDS:-10}" \
-  "${override_args[@]}"
+worker_command=(
+  "${TORCHRUN}"
+  --nnodes "${NNODES}"
+  --nproc-per-node "${NPROC_PER_NODE}"
+  --node-rank "${NODE_RANK}"
+  --rdzv-backend c10d
+  --rdzv-id "${RDZV_ID}"
+  --rdzv-endpoint "${RDZV_ENDPOINT}"
+  -m modelopt.torch.puzzletron.distributed_eval.cli worker
+  --campaign-dir "${CAMPAIGN_DIR}"
+  --config "${CONFIG_PATH}"
+  --host "${WORKER_HOST}"
+  --port "${WORKER_PORT}"
+  --worker-id "${WORKER_ID}"
+  --heartbeat-seconds "${HEARTBEAT_SECONDS:-10}"
+)
+if ((${#override_args[@]})); then
+  worker_command+=("${override_args[@]}")
+fi
+exec "${worker_command[@]}"