diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 4ab2cc138d..538a61acbb 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -182,6 +182,34 @@ on: required: false type: string default: "" + # Quality-eval inputs. Only consumed by quality-eval scenario types + # (runner: cluster:quality-eval); every other recipe leaves these + # empty and is unaffected. + benchmark-name: + description: "Quality benchmark to run (gpqa, mmlu_pro, hle, livecodebench, bfcl, scicode, swebench_pro, deepswe)" + required: false + type: string + default: "" + quality-endpoint: + description: "OpenAI-compatible endpoint URL for quality evals" + required: false + type: string + default: "" + quality-model-name: + description: "Model name for quality evals (e.g. openai/z-ai/glm-5.2)" + required: false + type: string + default: "" + smoke: + description: "Smoke-test mode: skip threshold validation, only verify artifact exists" + required: false + type: boolean + default: false + num-concurrent: + description: "API request concurrency for quality evals (passed as NUM_CONCURRENT env var)" + required: false + type: string + default: "" env: RANDOM_RANGE_RATIO: 0.8 HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} @@ -208,7 +236,7 @@ env: EVAL_ONLY: ${{ inputs.eval-only }} # Agentic-coding env. Fixed-seq-len jobs leave these empty. SCENARIO_TYPE: ${{ inputs.scenario-type }} - SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || 'fixed_seq_len/' }} + SCENARIO_SUBDIR: ${{ (inputs.scenario-type == 'agentic-coding' && 'agentic/') || (startsWith(inputs.scenario-type, 'quality-') && 'quality/') || 'fixed_seq_len/' }} IS_AGENTIC: ${{ inputs.scenario-type == 'agentic-coding' && '1' || '0' }} KV_OFFLOADING: ${{ inputs.kv-offloading }} KV_OFFLOAD_BACKEND: ${{ inputs.kv-offload-backend }} @@ -228,6 +256,13 @@ env: REMOTE_RESET_URL: ${{ inputs.remote-reset-url }} REMOTE_RUNNER_TYPE: ${{ inputs.remote-runner-type }} REMOTE_MAX_CONTEXT_LENGTH: ${{ inputs.remote-max-context-length }} + # Quality-eval env. Only consumed by quality-eval scenario types. + QUALITY_BENCHMARK_NAME: ${{ inputs.benchmark-name }} + QUALITY_ENDPOINT: ${{ inputs.quality-endpoint }} + QUALITY_API_KEY: ${{ secrets.GN_TOKEN_PLAN }} + QUALITY_MODEL_NAME: ${{ inputs.quality-model-name }} + SMOKE: ${{ inputs.smoke }} + NUM_CONCURRENT: ${{ inputs.num-concurrent }} AIPERF_FAILED_REQUEST_THRESHOLD: '0.10' RESULT_DIR: /workspace/results PYTHONDONTWRITEBYTECODE: '1' @@ -268,6 +303,7 @@ jobs: timeout-minutes: 500 name: >- p${{ inputs.priority }} | ${{ inputs.model-prefix }} ${{ inputs.precision }} ${{ inputs.runner }} ${{ inputs.framework == 'sglang' && 'sgl' || inputs.framework == 'dynamo-sglang' && 'dyn-sgl' || inputs.framework == 'sglang-disagg' && 'sgl-disagg' || inputs.framework }} + ${{ inputs.benchmark-name != '' && inputs.benchmark-name || '' }} TP${{ inputs.tp }}${{ inputs.pp != '' && inputs.pp != '1' && format('/PP{0}', inputs.pp) || '' }}${{ inputs.dcp-size != '' && inputs.dcp-size != '1' && format('/DCP{0}', inputs.dcp-size) || '' }}${{ inputs.pcp-size != '' && inputs.pcp-size != '1' && format('/PCP{0}', inputs.pcp-size) || '' }}${{ inputs.ep != '' && inputs.ep != '1' && format('/EP{0}', inputs.ep) || '' }}${{ inputs.dp-attn && '/DPA' || '' }} ${{ inputs.spec-decoding != 'none' && inputs.spec-decoding || '' }} ${{ inputs.kv-offloading != '' && inputs.kv-offloading != 'none' && format('{0} KV offload', inputs.kv-offloading) || '' }} @@ -321,7 +357,13 @@ jobs: # Export RESULT_FILENAME early so it's available for artifact uploads even if cancelled echo "RESULT_FILENAME=${RESULT_FILENAME}" >> $GITHUB_ENV - bash ./runners/launch_${RUNNER_NAME%%_*}.sh + # Quality-eval scenarios run on any runner with the quality-eval label; + # dispatch directly to launch_quality-eval.sh regardless of runner name. + if [ "${SCENARIO_SUBDIR}" = "quality/" ]; then + bash ./runners/launch_quality-eval.sh + else + bash ./runners/launch_${RUNNER_NAME%%_*}.sh + fi if [ "${{ inputs.eval-only }}" = "true" ]; then echo "Eval-only mode: skipping benchmark result file check" @@ -436,12 +478,21 @@ jobs: agent_preds.json predictions.jsonl swebench_report_*.json + eval_results*.json *.traj* + *.csv + lcb_results*.json + *.jsonl if-no-files-found: ${{ inputs.eval-only && 'error' || 'ignore' }} - name: Verify eval scores if: ${{ (success() || failure()) && inputs.eval-only }} - run: python3 utils/evals/validate_scores.py + run: | + SMOKE_FLAG="" + if [ "${{ inputs.smoke }}" = "true" ]; then + SMOKE_FLAG="--smoke" + fi + python3 utils/evals/validate_scores.py $SMOKE_FLAG - name: Cleanup eval outputs (post-upload) if: ${{ always() && (env.RUN_EVAL == 'true' || inputs.eval-only) }} @@ -450,7 +501,8 @@ jobs: # Remove any eval results JSONs that were moved into workspace rm -f results*.json || true rm -f sample*.jsonl || true - rm -f agent_preds.json predictions.jsonl swebench_report_*.json *.traj* || true + rm -f agent_preds.json predictions.jsonl swebench_report_*.json eval_results*.json *.traj* || true + rm -f *.csv lcb_results*.json || true - name: Resource cleanup (post-run) if: always() diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index b8e95f0806..565180eb26 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -925,6 +925,60 @@ jobs: eval-conc: ${{ matrix.config['eval-conc'] }} scenario-type: agentic-coding + # Quality-eval rows carry the quality-eval input shape (benchmark-name, + # quality-endpoint, quality-model-name, smoke), so they are dispatched + # with their own job rather than sweep-evals' fixed-seq-len inputs. + sweep-quality-evals: + needs: [setup, canary-select, canary-sweep] + if: >- + ${{ + !cancelled() && + needs.setup.result == 'success' && + needs.setup.outputs.reuse-enabled != 'true' && + (needs.canary-sweep.result == 'success' || needs.canary-sweep.result == 'skipped') && + toJson(fromJson(needs.setup.outputs.search-space-config).quality_evals) != '[]' && + toJson(fromJson(needs.setup.outputs.search-space-config).quality_evals) != 'null' + }} + uses: ./.github/workflows/benchmark-tmpl.yml + name: quality eval / + strategy: + fail-fast: ${{ contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') || contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast-no-canary') }} + matrix: + config: ${{ fromJson(needs.setup.outputs.search-space-config).quality_evals }} + secrets: inherit + with: + exp-name: ${{ matrix.config.exp-name }} + runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} + skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} + image: ${{ matrix.config.image }} + model: ${{ matrix.config.model }} + model-prefix: ${{ matrix.config.model-prefix }} + framework: ${{ matrix.config.framework }} + precision: ${{ matrix.config.precision }} + tp: '1' + pp: '1' + dcp-size: '1' + pcp-size: '1' + ep: '1' + dp-attn: false + conc: '1' + spec-decoding: 'none' + disagg: 'false' + isl: '0' + osl: '0' + max-model-len: '0' + run-eval: true + eval-only: true + scenario-type: ${{ matrix.config.scenario-type }} + benchmark-name: ${{ matrix.config.benchmark-name }} + quality-endpoint: ${{ matrix.config.quality-endpoint }} + quality-model-name: ${{ matrix.config.quality-model-name }} + smoke: ${{ matrix.config.smoke || false }} + num-concurrent: ${{ matrix.config.num-concurrent || '' }} + eval-limit: ${{ matrix.config.eval-limit || (contains(github.event.pull_request.labels.*.name, 'sweep-enabled') && '10' || '') }} + collect-results: needs: [ @@ -955,8 +1009,8 @@ jobs: result-prefix: "bmk" collect-evals: - needs: [sweep-evals, sweep-agentic-evals, sweep-multi-node-evals, sweep-multi-node-agentic-evals, setup] - if: ${{ always() && needs.setup.result != 'skipped' && (needs.sweep-evals.result != 'skipped' || needs.sweep-agentic-evals.result != 'skipped' || needs.sweep-multi-node-evals.result != 'skipped' || needs.sweep-multi-node-agentic-evals.result != 'skipped') }} + needs: [sweep-evals, sweep-agentic-evals, sweep-multi-node-evals, sweep-multi-node-agentic-evals, sweep-quality-evals, setup] + if: ${{ always() && needs.setup.result != 'skipped' && (needs.sweep-evals.result != 'skipped' || needs.sweep-agentic-evals.result != 'skipped' || needs.sweep-multi-node-evals.result != 'skipped' || needs.sweep-multi-node-agentic-evals.result != 'skipped' || needs.sweep-quality-evals.result != 'skipped') }} uses: ./.github/workflows/collect-evals.yml secrets: inherit diff --git a/benchmarks/single_node/quality/run_bfcl.sh b/benchmarks/single_node/quality/run_bfcl.sh new file mode 100755 index 0000000000..f3ae373c8e --- /dev/null +++ b/benchmarks/single_node/quality/run_bfcl.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Quality-eval script for BFCL v4, adapted for InferenceX CI. +# Env vars are set by runners/launch_quality-eval.sh, not sourced from .env. +# +# Required env: QUALITY_ENDPOINT, QUALITY_API_KEY, QUALITY_MODEL_NAME +# Optional env: RUN_ID, BFCL_MODE, TEST_CATEGORY, NUM_THREADS, TEMPERATURE, +# OPENAI_TIMEOUT, FULL_EVAL, OVERWRITE, TEST_CASE_IDS, LIMIT + +WORKSPACE_DIR="${QUALITY_WORKSPACE:-$(pwd)}" + +RAW_MODEL="${QUALITY_MODEL_NAME#openai/}" + +BFCL_DIR="${QUALITY_BFCL_DIR:-$WORKSPACE_DIR/BFCL/berkeley-function-call-leaderboard}" +PYTHON="${QUALITY_BFCL_VENV:-$BFCL_DIR/.venv-bfcl}/bin/python" +BFCL="${QUALITY_BFCL_VENV:-$BFCL_DIR/.venv-bfcl}/bin/bfcl" + +RUN_ID="${RUN_ID:-$(echo "$RAW_MODEL" | tr -c '[:alnum:]._-' '_')}" +OUT_DIR="$WORKSPACE_DIR/jobs/$RUN_ID/bfcl" +mkdir -p "$OUT_DIR" + +export BFCL_PROJECT_ROOT="$OUT_DIR" + +cat > "$OUT_DIR/.env" <&2; exit 1 ;; +esac + +TEST_CATEGORY="${TEST_CATEGORY:-simple_python,multiple,parallel,parallel_multiple,irrelevance}" +NUM_THREADS="${NUM_THREADS:-4}" +TEMPERATURE="${TEMPERATURE:-0.0}" +export OPENAI_TIMEOUT="${OPENAI_TIMEOUT:-90}" + +PARTIAL_EVAL_FLAG="--partial-eval" +if [[ "${FULL_EVAL:-0}" == "1" ]]; then + PARTIAL_EVAL_FLAG="" +fi + +RUN_IDS_ARG=() +if [[ -n "${TEST_CASE_IDS:-}" ]]; then + IDS_FILE="$OUT_DIR/test_case_ids_to_generate.json" + if [[ -f "$TEST_CASE_IDS" ]]; then + cp "$TEST_CASE_IDS" "$IDS_FILE" + else + printf '%s\n' "$TEST_CASE_IDS" > "$IDS_FILE" + fi + RUN_IDS_ARG=(--run-ids) +elif [[ -n "${LIMIT:-}" ]]; then + IDS_FILE="$OUT_DIR/test_case_ids_to_generate.json" + echo "=== LIMIT=$LIMIT set, generating subset ID file ===" + "$PYTHON" - "$BFCL_DIR" "$IDS_FILE" "$TEST_CATEGORY" "$LIMIT" <<'PY' +import json, pathlib, sys +bfcl_dir, out_file, categories_str, limit = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]) +from bfcl_eval.constants.category_mapping import ( + NON_LIVE_CATEGORY, LIVE_CATEGORY, MULTI_TURN_CATEGORY, +) +all_cats = NON_LIVE_CATEGORY + LIVE_CATEGORY + MULTI_TURN_CATEGORY +requested = [c.strip() for c in categories_str.split(",")] +# Map requested category names to data file names +cat_to_file = {} +for cat in all_cats: + cat_to_file[cat] = f"BFCL_v4_{cat}.json" +ids_map = {} +for cat in requested: + fname = cat_to_file.get(cat, f"BFCL_v4_{cat}.json") + fpath = pathlib.Path(bfcl_dir) / "bfcl_eval" / "data" / fname + if not fpath.exists(): + print(f" WARN: data file not found: {fpath}", file=sys.stderr) + ids_map[cat] = [] + continue + entries = [json.loads(line) for line in fpath.read_text().strip().split("\n")] + ids_map[cat] = [e["id"] for e in entries[:limit]] + print(f" {cat}: {len(ids_map[cat])} IDs (of {len(entries)} total)") +# Fill remaining categories with empty lists (BFCL expects all keys) +for cat in all_cats: + if cat not in ids_map: + ids_map[cat] = [] +pathlib.Path(out_file).write_text(json.dumps(ids_map, indent=2)) +print(f" Written: {out_file}") +PY + RUN_IDS_ARG=(--run-ids) +fi + +OVERWRITE_ARG=() +if [[ "${OVERWRITE:-0}" == "1" ]]; then + OVERWRITE_ARG=(--allow-overwrite) +fi + +echo "=== BFCL v4 run ===" +echo " RUN_ID : $RUN_ID" +echo " Mode : $BFCL_MODE (is_fc_model via OpenAICompletionsHandler)" +echo " BFCL model key : $BFCL_MODEL_KEY" +echo " Wire model id : $RAW_MODEL" +echo " Endpoint : $QUALITY_ENDPOINT" +echo " Test categories : $TEST_CATEGORY" +echo " Threads : $NUM_THREADS" +echo " Temperature : $TEMPERATURE" +echo " BFCL_PROJECT_ROOT : $OUT_DIR" +echo " Bin : $BFCL" +echo + +cd "$BFCL_DIR" + +"$BFCL" generate \ + --model "$BFCL_MODEL_KEY" \ + --test-category "$TEST_CATEGORY" \ + --num-threads "$NUM_THREADS" \ + --temperature "$TEMPERATURE" \ + "${RUN_IDS_ARG[@]}" \ + "${OVERWRITE_ARG[@]}" + +"$BFCL" evaluate \ + --model "$BFCL_MODEL_KEY" \ + --test-category "$TEST_CATEGORY" \ + $PARTIAL_EVAL_FLAG + +SCORE_FILE="$OUT_DIR/score/data_overall.csv" +if [[ -f "$SCORE_FILE" ]]; then + "$PYTHON" - "$SCORE_FILE" <<'PY' +import csv, sys, pathlib +path = pathlib.Path(sys.argv[1]) +with path.open(newline="") as f: + rows = list(csv.reader(f)) +if not rows: + print("(empty score file)") + sys.exit(0) +header, data = rows[0], rows[1:] +wanted = ["Rank", "Model", "Overall Acc", "Non-Live AST Acc", "Live Acc", + "Multi Turn Acc", "Relevance Detection", "Irrelevance Detection", + "Organization", "License"] +idx = [header.index(w) for w in wanted if w in header] +print(" | ".join(f"{header[i]}: {data[0][i]}" for i in idx) if data else "(no rows)") +print(f"\nFull table: {path}") +PY +else + echo "Score file not found: $SCORE_FILE" +fi + +echo +echo "=== BFCL artifacts ===" +echo " Results : $OUT_DIR/result/$(echo "$BFCL_MODEL_KEY" | tr '/' '_')/" +echo " Scores : $OUT_DIR/score/$(echo "$BFCL_MODEL_KEY" | tr '/' '_')/" +echo " Overall : $OUT_DIR/score/data_overall.csv" +echo " Non-live: $OUT_DIR/score/data_non_live.csv" + +# Write a results.json wrapper so benchmark-tmpl.yml's `ls results*.json` check passes +if [[ -f "$SCORE_FILE" ]]; then + "$PYTHON" - "$SCORE_FILE" "$OUT_DIR/results.json" <<'PY' +import csv, json, pathlib, sys +score_path, out_path = sys.argv[1], sys.argv[2] +with pathlib.Path(score_path).open(newline="") as f: + rows = list(csv.DictReader(f)) +result = { + "benchmark": "bfcl", + "scores": rows, + "score_file": str(score_path), +} +pathlib.Path(out_path).write_text(json.dumps(result, indent=2)) +print(f"Wrote results wrapper: {out_path}") +PY +fi diff --git a/benchmarks/single_node/quality/run_deepswe.sh b/benchmarks/single_node/quality/run_deepswe.sh new file mode 100755 index 0000000000..bbd169543f --- /dev/null +++ b/benchmarks/single_node/quality/run_deepswe.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Quality-eval script for DeepSWE, adapted for InferenceX CI. +# Env vars are set by runners/launch_quality-eval.sh, not sourced from .env. +# Requires Docker on the runner. +# +# Required env: QUALITY_ENDPOINT, QUALITY_API_KEY, QUALITY_MODEL_NAME +# Optional env: RUN_ID, N_TASKS, CCU, JOBS_DIR, JOB_NAME + +WORKSPACE_DIR="${QUALITY_WORKSPACE:-$(pwd)}" +export PATH="$HOME/.local/bin:$PATH" + +RAW_MODEL="${QUALITY_MODEL_NAME#openai/}" + +N_TASKS="${N_TASKS:-${LIMIT:-6}}" +CCU="${CCU:-$N_TASKS}" +RUN_ID="${RUN_ID:-$(echo "$RAW_MODEL" | tr -c '[:alnum:]._-' '_')}" +JOBS_DIR="${JOBS_DIR:-$WORKSPACE_DIR/jobs/$RUN_ID/deepswe}" +JOB_NAME="${JOB_NAME:-${N_TASKS}tasks-ccu${CCU}}" + +DEEPSWE_DIR="${QUALITY_DEEPSWE_DIR:-$WORKSPACE_DIR/deep-swe}" + +echo "=== DeepSWE run ===" +echo " RUN_ID : $RUN_ID" +echo " Model : $RAW_MODEL" +echo " N tasks : $N_TASKS" +echo " Concurrent : $CCU" +echo " Jobs dir : $JOBS_DIR" +echo + +cd "$DEEPSWE_DIR" + +uv tool run --from datacurve-pier pier run \ + -p tasks \ + --agent mini-swe-agent \ + --model "openai/${RAW_MODEL}" \ + --agent-kwarg model_class=litellm \ + --agent-env "MSWEA_API_KEY=$QUALITY_API_KEY" \ + --agent-env "OPENAI_API_KEY=$QUALITY_API_KEY" \ + --agent-env "OPENAI_BASE_URL=$QUALITY_ENDPOINT" \ + --agent-env "OPENAI_API_BASE=$QUALITY_ENDPOINT" \ + --n-tasks "$N_TASKS" \ + --sample-seed 0 \ + --n-concurrent "$CCU" \ + --agent-setup-timeout-multiplier 3 \ + --jobs-dir "$JOBS_DIR" \ + --job-name "$JOB_NAME" \ + --yes diff --git a/benchmarks/single_node/quality/run_gpqa.sh b/benchmarks/single_node/quality/run_gpqa.sh new file mode 100755 index 0000000000..fda2b66f9c --- /dev/null +++ b/benchmarks/single_node/quality/run_gpqa.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Quality-eval script for GPQA-Diamond, adapted for InferenceX CI. +# Env vars are set by runners/launch_quality-eval.sh (from benchmark-tmpl.yml +# inputs), not sourced from .env. +# +# Required env: QUALITY_ENDPOINT, QUALITY_API_KEY, QUALITY_MODEL_NAME +# Optional env: RUN_ID, LIMIT, HF_TOKEN, NUM_CONCURRENT, MAX_LENGTH, MAX_GEN_TOKS, +# TASK, NUM_FEWSHOT, BATCH_SIZE, SAMPLE_N, SAMPLE_SEED + +WORKSPACE_DIR="${QUALITY_WORKSPACE:-$(pwd)}" +export PATH="$HOME/.local/bin:$PATH" + +LM_EVAL="${QUALITY_VENV:-$WORKSPACE_DIR/.venv-lmeval}/bin/lm-eval" + +RAW_MODEL="${QUALITY_MODEL_NAME#openai/}" +ENDPOINT="${QUALITY_ENDPOINT%/}/chat/completions" +export OPENAI_API_KEY="$QUALITY_API_KEY" +export HF_TOKEN="${HF_TOKEN:-}" + +RUN_ID="${RUN_ID:-$(echo "$RAW_MODEL" | tr -c '[:alnum:]._-' '_')}" + +MAX_LENGTH="${MAX_LENGTH:-10240}" +MAX_GEN_TOKS="${MAX_GEN_TOKS:-10240}" +TASK="${TASK:-gpqa_diamond_cot_n_shot}" +NUM_FEWSHOT="${NUM_FEWSHOT:-5}" +BATCH_SIZE="${BATCH_SIZE:-1}" +NUM_CONCURRENT="${NUM_CONCURRENT:-4}" + +LIMIT="${LIMIT:-}" +SAMPLE_N="${SAMPLE_N:-}" +SAMPLE_SEED="${SAMPLE_SEED:-42}" + +OUT_DIR="$WORKSPACE_DIR/jobs/$RUN_ID/gpqa" +CACHE_DB="$OUT_DIR/cache.db" + +mkdir -p "$OUT_DIR" + +SUBSET_ARGS=() +if [[ -n "$SAMPLE_N" ]]; then + SAMPLE_JSON="$OUT_DIR/sample_indices.json" + "${QUALITY_VENV:-$WORKSPACE_DIR/.venv-lmeval}/bin/python" -c " +import json, random +random.seed(${SAMPLE_SEED}) +indices = sorted(random.sample(range(198), int('${SAMPLE_N}'))) +json.dump({'${TASK}': indices}, open('${SAMPLE_JSON}', 'w')) +" + SUBSET_ARGS=(--samples "$(cat "$SAMPLE_JSON")") + echo " Subset : random ${SAMPLE_N} (seed=${SAMPLE_SEED})" +elif [[ -n "$LIMIT" ]]; then + SUBSET_ARGS=(--limit "$LIMIT") + echo " Subset : first ${LIMIT}" +fi + +echo "=== GPQA run ===" +echo " RUN_ID : $RUN_ID" +echo " Model : $RAW_MODEL" +echo " Task : $TASK" +echo " Output dir : $OUT_DIR" +echo " Cache (resume): $CACHE_DB" +echo + +"$LM_EVAL" run \ + --model openai-chat-completions \ + --model_args "model=${RAW_MODEL},base_url=${ENDPOINT},tokenizer_backend=None,tokenized_requests=False,num_concurrent=${NUM_CONCURRENT},max_length=${MAX_LENGTH}" \ + --tasks "$TASK" \ + --num_fewshot "$NUM_FEWSHOT" \ + --apply_chat_template \ + --batch_size "$BATCH_SIZE" \ + --gen_kwargs "temperature=0,max_gen_toks=${MAX_GEN_TOKS}" \ + --log_samples \ + --use_cache "$CACHE_DB" \ + --output_path "$OUT_DIR" \ + "${SUBSET_ARGS[@]}" diff --git a/benchmarks/single_node/quality/run_hle.sh b/benchmarks/single_node/quality/run_hle.sh new file mode 100755 index 0000000000..b1fcbc313e --- /dev/null +++ b/benchmarks/single_node/quality/run_hle.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Quality-eval script for HLE (Humanity's Last Exam), adapted for InferenceX CI. +# Env vars are set by runners/launch_quality-eval.sh, not sourced from .env. +# +# Required env: QUALITY_ENDPOINT, QUALITY_API_KEY, QUALITY_MODEL_NAME +# Optional env: RUN_ID, LIMIT, NUM_CONCURRENT, MAX_LENGTH, MAX_GEN_TOKS, +# TASK, NUM_FEWSHOT, BATCH_SIZE, REQUEST_TIMEOUT + +WORKSPACE_DIR="${QUALITY_WORKSPACE:-$(pwd)}" +export PATH="$HOME/.local/bin:$PATH" + +LM_EVAL="${QUALITY_VENV:-$WORKSPACE_DIR/.venv-lmeval}/bin/lm-eval" + +RAW_MODEL="${QUALITY_MODEL_NAME#openai/}" +ENDPOINT="${QUALITY_ENDPOINT%/}/chat/completions" +export OPENAI_API_KEY="$QUALITY_API_KEY" +export HF_TOKEN="${HF_TOKEN:-}" + +RUN_ID="${RUN_ID:-$(echo "$RAW_MODEL" | tr -c '[:alnum:]._-' '_')}" + +MAX_LENGTH="${MAX_LENGTH:-32768}" +# GLM reasoning tokens count against this budget. The 8k smoke still exhausted +# the budget on exact-match questions before the model emitted its final line. +MAX_GEN_TOKS="${MAX_GEN_TOKS:-16384}" +TASK="${TASK:-hle}" +NUM_FEWSHOT="${NUM_FEWSHOT:-0}" +BATCH_SIZE="${BATCH_SIZE:-1}" +NUM_CONCURRENT="${NUM_CONCURRENT:-4}" +# lm-eval's default is a 300-second *total* aiohttp timeout. Streaming does not +# reset it when chunks arrive, and long HLE reasoning can legitimately exceed it. +REQUEST_TIMEOUT="${REQUEST_TIMEOUT:-1800}" + +LIMIT="${LIMIT:-}" + +OUT_DIR="$WORKSPACE_DIR/jobs/$RUN_ID/hle" +CACHE_DB="$OUT_DIR/cache.db" +TASKS_DIR="${QUALITY_TASKS_DIR:-$WORKSPACE_DIR/benchmarks/single_node/quality/tasks/hle}" + +mkdir -p "$OUT_DIR" + +SUBSET_ARGS=() +if [[ -n "$LIMIT" ]]; then + SUBSET_ARGS=(--limit "$LIMIT") +fi + +echo "=== HLE (Humanity's Last Exam) run ===" +echo " RUN_ID : $RUN_ID" +echo " Model : $RAW_MODEL" +echo " Task : $TASK" +echo " Output dir : $OUT_DIR" +echo " Cache (resume): $CACHE_DB" +echo " HTTP timeout : ${REQUEST_TIMEOUT}s" +if [[ -n "$LIMIT" ]]; then + echo " Subset : first ${LIMIT} per subtask" +fi +echo + +"$LM_EVAL" run \ + --model openai-chat-completions \ + --model_args "model=${RAW_MODEL},base_url=${ENDPOINT},tokenizer_backend=None,tokenized_requests=False,num_concurrent=${NUM_CONCURRENT},max_length=${MAX_LENGTH},timeout=${REQUEST_TIMEOUT}" \ + --tasks "$TASK" \ + --num_fewshot "$NUM_FEWSHOT" \ + --apply_chat_template \ + --batch_size "$BATCH_SIZE" \ + --gen_kwargs "temperature=0,max_gen_toks=${MAX_GEN_TOKS}" \ + --include_path "$TASKS_DIR" \ + --log_samples \ + --use_cache "$CACHE_DB" \ + --output_path "$OUT_DIR" \ + "${SUBSET_ARGS[@]}" + +# lm-eval accepts an empty API completion and scores it as incorrect. That is +# not a valid smoke test: it usually means the reasoning budget was exhausted +# or the endpoint response shape was incompatible. Fail loudly and preserve +# the sample logs so the reason is visible in the artifact. +"${QUALITY_VENV:-$WORKSPACE_DIR/.venv-lmeval}/bin/python" - "$OUT_DIR" <<'PY' +import json +import sys +from pathlib import Path + +sample_files = sorted(Path(sys.argv[1]).rglob("sample*.jsonl")) +if not sample_files: + raise SystemExit("HLE validation failed: lm-eval produced no sample log") + +empty = [] +total = 0 +for path in sample_files: + for line_number, line in enumerate(path.read_text().splitlines(), 1): + if not line.strip(): + continue + total += 1 + sample = json.loads(line) + responses = sample.get("resps", []) + strings = [] + stack = [responses] + while stack: + value = stack.pop() + if isinstance(value, str): + strings.append(value) + elif isinstance(value, (list, tuple)): + stack.extend(value) + if not any(value.strip() for value in strings): + empty.append(f"{path.name}:{line_number}") + +if total == 0: + raise SystemExit("HLE validation failed: sample logs contain no records") +if empty: + preview = ", ".join(empty[:10]) + raise SystemExit( + f"HLE validation failed: {len(empty)}/{total} completions are empty " + f"({preview}). The model may have exhausted max_gen_toks before emitting content." + ) +print(f"HLE response sanity check passed: {total}/{total} completions are non-empty") +PY diff --git a/benchmarks/single_node/quality/run_livecodebench.sh b/benchmarks/single_node/quality/run_livecodebench.sh new file mode 100755 index 0000000000..d10a5bd610 --- /dev/null +++ b/benchmarks/single_node/quality/run_livecodebench.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Quality-eval script for LiveCodeBench, adapted for InferenceX CI. +# Env vars are set by runners/launch_quality-eval.sh, not sourced from .env. +# +# Required env: QUALITY_ENDPOINT, QUALITY_API_KEY, QUALITY_MODEL_NAME +# Optional env: RUN_ID, SCENARIO, RELEASE_VERSION, N, TEMPERATURE, +# MAX_TOKENS, MULTIPROCESS, TIMEOUT, NUM_PROCESS_EVALUATE, OPENAI_TIMEOUT + +WORKSPACE_DIR="${QUALITY_WORKSPACE:-$(pwd)}" + +RAW_MODEL="${QUALITY_MODEL_NAME#openai/}" +export OPENAI_KEY="$QUALITY_API_KEY" +export OPENAI_BASE_URL="$QUALITY_ENDPOINT" + +LCB_DIR="${QUALITY_LCB_DIR:-$WORKSPACE_DIR/LiveCodeBench}" +PYTHON="${QUALITY_LCB_VENV:-$LCB_DIR/.venv-lcb}/bin/python" + +RUN_ID="${RUN_ID:-$(echo "$RAW_MODEL" | tr -c '[:alnum:]._-' '_')}" + +SCENARIO="${SCENARIO:-codegeneration}" +RELEASE_VERSION="${RELEASE_VERSION:-release_latest}" +N="${N:-1}" +TEMPERATURE="${TEMPERATURE:-0.0}" +MAX_TOKENS="${MAX_TOKENS:-8192}" +MULTIPROCESS="${MULTIPROCESS:-4}" +TIMEOUT="${TIMEOUT:-6}" +NUM_PROCESS_EVALUATE="${NUM_PROCESS_EVALUATE:-12}" +OPENAI_TIMEOUT="${OPENAI_TIMEOUT:-90}" + +OUT_DIR="$WORKSPACE_DIR/jobs/$RUN_ID/livecodebench" + +mkdir -p "$OUT_DIR" + +export LCB_OUTPUT_DIR="$OUT_DIR/" + +# LCB CLI has no --limit flag; pass LIMIT via LCB_LIMIT env var (patched main.py reads it) +export LCB_LIMIT="${LIMIT:-0}" + +echo "=== LiveCodeBench run ===" +echo " RUN_ID : $RUN_ID" +echo " Model : $RAW_MODEL" +echo " Scenario : $SCENARIO" +echo " Release version : $RELEASE_VERSION" +echo " N (samples) : $N" +echo " Temperature : $TEMPERATURE" +echo " Max tokens : $MAX_TOKENS" +echo " Multiprocess : $MULTIPROCESS" +echo " Eval timeout : ${TIMEOUT}s" +echo " Output dir : $OUT_DIR" +echo + +cd "$LCB_DIR" + +exec "$PYTHON" -m lcb_runner.runner.main \ + --model "$RAW_MODEL" \ + --scenario "$SCENARIO" \ + --release_version "$RELEASE_VERSION" \ + --n "$N" \ + --temperature "$TEMPERATURE" \ + --max_tokens "$MAX_TOKENS" \ + --multiprocess "$MULTIPROCESS" \ + --timeout "$TIMEOUT" \ + --num_process_evaluate "$NUM_PROCESS_EVALUATE" \ + --openai_timeout "$OPENAI_TIMEOUT" \ + --evaluate \ + --use_cache \ + --continue_existing diff --git a/benchmarks/single_node/quality/run_mmlu_pro.sh b/benchmarks/single_node/quality/run_mmlu_pro.sh new file mode 100755 index 0000000000..d265eb2ad8 --- /dev/null +++ b/benchmarks/single_node/quality/run_mmlu_pro.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Quality-eval script for MMLU-Pro, adapted for InferenceX CI. +# Env vars are set by runners/launch_quality-eval.sh, not sourced from .env. +# +# Required env: QUALITY_ENDPOINT, QUALITY_API_KEY, QUALITY_MODEL_NAME +# Optional env: RUN_ID, LIMIT, NUM_CONCURRENT, MAX_LENGTH, MAX_GEN_TOKS, +# TASK, NUM_FEWSHOT, BATCH_SIZE + +WORKSPACE_DIR="${QUALITY_WORKSPACE:-$(pwd)}" +export PATH="$HOME/.local/bin:$PATH" + +LM_EVAL="${QUALITY_VENV:-$WORKSPACE_DIR/.venv-lmeval}/bin/lm-eval" + +RAW_MODEL="${QUALITY_MODEL_NAME#openai/}" +ENDPOINT="${QUALITY_ENDPOINT%/}/chat/completions" +export OPENAI_API_KEY="$QUALITY_API_KEY" +export HF_TOKEN="${HF_TOKEN:-}" + +RUN_ID="${RUN_ID:-$(echo "$RAW_MODEL" | tr -c '[:alnum:]._-' '_')}" + +MAX_LENGTH="${MAX_LENGTH:-8192}" +# Reasoning-capable models can spend the old 2k budget before emitting a final +# choice. Keep this aligned with the context cap used by this task. +MAX_GEN_TOKS="${MAX_GEN_TOKS:-8192}" +TASK="${TASK:-mmlu_pro}" +NUM_FEWSHOT="${NUM_FEWSHOT:-5}" +BATCH_SIZE="${BATCH_SIZE:-1}" +NUM_CONCURRENT="${NUM_CONCURRENT:-4}" + +LIMIT="${LIMIT:-}" + +OUT_DIR="$WORKSPACE_DIR/jobs/$RUN_ID/mmlu_pro" +CACHE_DB="$OUT_DIR/cache.db" + +mkdir -p "$OUT_DIR" + +SUBSET_ARGS=() +if [[ -n "$LIMIT" ]]; then + SUBSET_ARGS=(--limit "$LIMIT") +fi + +echo "=== MMLU-Pro run ===" +echo " RUN_ID : $RUN_ID" +echo " Model : $RAW_MODEL" +echo " Task : $TASK" +echo " Output dir : $OUT_DIR" +echo " Cache (resume): $CACHE_DB" +if [[ -n "$LIMIT" ]]; then + echo " Subset : first ${LIMIT} per subtask" +fi +echo + +"$LM_EVAL" run \ + --model openai-chat-completions \ + --model_args "model=${RAW_MODEL},base_url=${ENDPOINT},tokenizer_backend=None,tokenized_requests=False,num_concurrent=${NUM_CONCURRENT},max_length=${MAX_LENGTH}" \ + --tasks "$TASK" \ + --num_fewshot "$NUM_FEWSHOT" \ + --apply_chat_template \ + --batch_size "$BATCH_SIZE" \ + --gen_kwargs "temperature=0,max_gen_toks=${MAX_GEN_TOKS}" \ + --log_samples \ + --use_cache "$CACHE_DB" \ + --output_path "$OUT_DIR" \ + "${SUBSET_ARGS[@]}" diff --git a/benchmarks/single_node/quality/run_scicode.sh b/benchmarks/single_node/quality/run_scicode.sh new file mode 100755 index 0000000000..9e8b89c658 --- /dev/null +++ b/benchmarks/single_node/quality/run_scicode.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Quality-eval script for SciCode, adapted for InferenceX CI. +# Env vars are set by runners/launch_quality-eval.sh, not sourced from .env. +# +# Required env: QUALITY_ENDPOINT, QUALITY_API_KEY, QUALITY_MODEL_NAME +# Optional env: RUN_ID, LIMIT, SPLIT, WITH_BACKGROUND, MAX_CONNECTIONS, MAX_TOKENS, +# SAMPLE_SHUFFLE, RETRY_ON_ERROR + +WORKSPACE_DIR="${QUALITY_WORKSPACE:-$(pwd)}" + +export PATH="${QUALITY_SCICODE_VENV:-$WORKSPACE_DIR/.venv-scicode}/bin:$HOME/.local/bin:$PATH" + +RAW_MODEL="${QUALITY_MODEL_NAME#openai/}" + +export OPENAI_API_KEY="$QUALITY_API_KEY" +export OPENAI_BASE_URL="$QUALITY_ENDPOINT" + +SCICODE_DIR="${QUALITY_SCICODE_DIR:-$WORKSPACE_DIR/SciCode}" +SCICODE_DATA_FILE="${QUALITY_SCICODE_DATA_FILE:-$SCICODE_DIR/eval/data/test_data.h5}" +INSPECT="${QUALITY_SCICODE_VENV:-$WORKSPACE_DIR/.venv-scicode}/bin/inspect" + +RUN_ID="${RUN_ID:-$(echo "$RAW_MODEL" | tr -c '[:alnum:]._-' '_')}" + +SPLIT="${SPLIT:-test}" +WITH_BACKGROUND="${WITH_BACKGROUND:-False}" +MAX_CONNECTIONS="${MAX_CONNECTIONS:-4}" +MAX_TOKENS="${MAX_TOKENS:-8192}" +LIMIT="${LIMIT:-}" +SAMPLE_SHUFFLE="${SAMPLE_SHUFFLE:-}" + +RETRY_ON_ERROR="${RETRY_ON_ERROR:-2}" + +OUT_DIR="$WORKSPACE_DIR/jobs/$RUN_ID/scicode" +LOG_DIR="$OUT_DIR/logs" + +mkdir -p "$OUT_DIR" "$LOG_DIR" + +# Do not spend tokens when the external numeric reference data was not +# provisioned. The launcher validates this more thoroughly during setup; this +# guard also protects direct invocations of this script. +if [[ ! -s "$SCICODE_DATA_FILE" ]]; then + echo "ERROR: SciCode numeric test data is missing or empty: $SCICODE_DATA_FILE" >&2 + echo "Run through runners/launch_quality-eval.sh to download and cache it." >&2 + exit 1 +fi +if ! "${QUALITY_SCICODE_VENV:-$WORKSPACE_DIR/.venv-scicode}/bin/python" \ + - "$SCICODE_DATA_FILE" <<'PY' +import sys + +import h5py + +try: + with h5py.File(sys.argv[1], "r") as data: + if len(data) == 0: + raise ValueError("empty HDF5 file") +except (OSError, ValueError) as exc: + print(f"ERROR: Invalid SciCode numeric test data: {exc}", file=sys.stderr) + raise SystemExit(1) +PY +then + echo "Run through runners/launch_quality-eval.sh to repair the cached download." >&2 + exit 1 +fi + +LIMIT_ARG=() +if [[ -n "$LIMIT" ]]; then + LIMIT_ARG=(--limit "$LIMIT") +fi + +SHUFFLE_ARG=() +if [[ -n "$SAMPLE_SHUFFLE" ]]; then + SHUFFLE_ARG=(--sample-shuffle "$SAMPLE_SHUFFLE") +fi + +cd "$SCICODE_DIR/eval/inspect_ai" + +echo "=== SciCode run ===" +echo " RUN_ID : $RUN_ID" +echo " Model : $RAW_MODEL" +echo " Split : $SPLIT" +echo " Output dir : $OUT_DIR" +echo " Log dir : $LOG_DIR" +echo " Test data : $SCICODE_DATA_FILE" +echo " Max tokens : $MAX_TOKENS" +echo " Retry on error: $RETRY_ON_ERROR" +if [[ -n "$LIMIT" ]]; then + echo " Limit : $LIMIT" +fi +if [[ -n "$SAMPLE_SHUFFLE" ]]; then + echo " Sample shuffle: seed=$SAMPLE_SHUFFLE" +fi +echo + +"$INSPECT" eval scicode.py \ + --model "openai/${RAW_MODEL}" \ + --temperature 0 \ + --max-connections "$MAX_CONNECTIONS" \ + --max-tokens "$MAX_TOKENS" \ + --log-dir "$LOG_DIR" \ + --log-format json \ + --max-retries "$RETRY_ON_ERROR" \ + --no-fail-on-error \ + "${LIMIT_ARG[@]}" \ + "${SHUFFLE_ARG[@]}" \ + -T split="$SPLIT" \ + -T output_dir="$OUT_DIR" \ + -T with_background="$WITH_BACKGROUND" \ + -T h5py_file="$SCICODE_DATA_FILE" \ + -T mode=normal diff --git a/benchmarks/single_node/quality/run_swebench_pro.sh b/benchmarks/single_node/quality/run_swebench_pro.sh new file mode 100755 index 0000000000..e11cd5a26b --- /dev/null +++ b/benchmarks/single_node/quality/run_swebench_pro.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Quality-eval script for SWE-bench Pro, adapted for InferenceX CI. +# Env vars are set by runners/launch_quality-eval.sh, not sourced from .env. +# Requires Docker on the runner for Phase 2 evaluation containers. +# +# Required env: QUALITY_ENDPOINT, QUALITY_API_KEY, QUALITY_MODEL_NAME +# Optional env: RUN_ID, LIMIT, WORKERS, EVAL_WORKERS, REDO_EXISTING, REDO_EVAL, +# DOCKERHUB_USERNAME, COST_LIMIT, STEP_LIMIT + +WORKSPACE_DIR="${QUALITY_WORKSPACE:-$(pwd)}" + +export OPENAI_API_KEY="$QUALITY_API_KEY" +export OPENAI_API_BASE="$QUALITY_ENDPOINT" +export MSWEA_MODEL_API_KEY="$QUALITY_API_KEY" +export MSWEA_SILENT_STARTUP=1 + +PYTHON="${QUALITY_SWEBENCHPRO_VENV:-$WORKSPACE_DIR/.venv-swebenchpro}/bin/python" + +RAW_MODEL="${QUALITY_MODEL_NAME#openai/}" + +RUN_ID="${RUN_ID:-$(echo "$RAW_MODEL" | tr -c '[:alnum:]._-' '_')}" + +SWEBENCH_DIR="${QUALITY_SWEBENCH_DIR:-$WORKSPACE_DIR/SWE-bench_Pro-os}" +INSTANCES_YAML="$SWEBENCH_DIR/SWE-agent/data/instances.yaml" +RAW_SAMPLE="${QUALITY_SWEBENCH_RAW_SAMPLE:-$SWEBENCH_DIR/helper_code/sweap_eval_full_v2.jsonl}" +CONFIG="${QUALITY_SWEBENCH_CONFIG:-$WORKSPACE_DIR/benchmarks/single_node/quality/tasks/swebench-pro/swebench_pro.yaml}" +RUN_SWEBENCH_PRO="${QUALITY_SWEBENCH_RUN_SCRIPT:-$WORKSPACE_DIR/benchmarks/single_node/quality/tasks/swebench-pro/run_swebench_pro.py}" + +OUT_DIR="$WORKSPACE_DIR/jobs/$RUN_ID/swebench_pro" +PRED_DIR="$OUT_DIR/preds" +PATCHES_JSON="$OUT_DIR/patches.json" +EVAL_DIR="$OUT_DIR/eval" + +mkdir -p "$PRED_DIR" "$EVAL_DIR" + +LIMIT="${LIMIT:-}" +WORKERS="${WORKERS:-4}" +EVAL_WORKERS="${EVAL_WORKERS:-4}" +REDO_EXISTING="${REDO_EXISTING:-0}" +REDO_EVAL="${REDO_EVAL:-0}" +DOCKERHUB_USERNAME="${DOCKERHUB_USERNAME:-jefzda}" +COST_LIMIT="${COST_LIMIT:-3.0}" +STEP_LIMIT="${STEP_LIMIT:-250}" + +SLICE_ARG=() +if [[ -n "$LIMIT" ]]; then + SLICE_ARG=(--slice "0:${LIMIT}") +fi + +REDO_ARG=() +if [[ "$REDO_EXISTING" == "1" ]]; then + REDO_ARG=(--redo-existing) +fi + +REDO_EVAL_ARG=() +if [[ "$REDO_EVAL" == "1" ]]; then + REDO_EVAL_ARG=(--redo) +fi + +echo "=== SWE-bench Pro run ===" +echo " RUN_ID : $RUN_ID" +echo " Model : $RAW_MODEL" +echo " Phase 1 (agent) : mini-swe-agent (litellm -> $OPENAI_API_BASE)" +echo " Phase 2 (eval) : swe_bench_pro_eval.py --use_local_docker" +echo " Instances yaml : $INSTANCES_YAML" +echo " Raw sample : $RAW_SAMPLE" +echo " Output dir : $OUT_DIR" +echo " Workers (agent) : $WORKERS" +echo " Workers (eval) : $EVAL_WORKERS" +echo " Cost/step limit : \$${COST_LIMIT} / ${STEP_LIMIT} steps per instance" +if [[ -n "$LIMIT" ]]; then + echo " Limit : first $LIMIT instances" +fi +echo + +RUN_CONFIG="$OUT_DIR/run_config.yaml" +sed -e "s/step_limit: [0-9]*/step_limit: ${STEP_LIMIT}/" \ + -e "s/cost_limit: [0-9.]*$/cost_limit: ${COST_LIMIT}/" \ + "$CONFIG" > "$RUN_CONFIG" + +echo "--- Phase 1: agent patch generation ---" +"$PYTHON" "$RUN_SWEBENCH_PRO" \ + --instances-path "$INSTANCES_YAML" \ + --output "$PRED_DIR" \ + --config "$RUN_CONFIG" \ + --model "openai/${RAW_MODEL}" \ + --workers "$WORKERS" \ + "${SLICE_ARG[@]}" \ + "${REDO_ARG[@]}" + +echo "--- Validating agent patches ---" +"$PYTHON" - "$PRED_DIR/preds.json" "${LIMIT:-0}" <<'PY' +import json +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +if not path.is_file(): + raise SystemExit(f"SWE-bench Pro generation failed: missing {path}") +predictions = json.loads(path.read_text()) +expected = int(sys.argv[2]) +if expected and len(predictions) != expected: + raise SystemExit( + f"SWE-bench Pro generation failed: expected {expected} predictions, got {len(predictions)}" + ) +empty = [ + key for key, value in predictions.items() + if not isinstance(value, dict) or not str(value.get("model_patch") or "").strip() +] +if empty: + raise SystemExit( + f"SWE-bench Pro generation failed: {len(empty)}/{len(predictions)} predictions " + f"have empty patches ({', '.join(empty[:10])})" + ) +print(f"SWE-bench Pro agent sanity check passed: {len(predictions)} non-empty patches") +PY + +echo "--- Gathering patches ---" +"$PYTHON" - "$PRED_DIR/preds.json" "$PATCHES_JSON" <<'PY' +import json, sys +src, dst = sys.argv[1], sys.argv[2] +data = json.load(open(src)) +patches = [ + {"instance_id": v["instance_id"], "patch": v.get("model_patch") or "", "prefix": "agent"} + for v in data.values() +] +json.dump(patches, open(dst, "w"), indent=2) +print(f"Wrote {len(patches)} patches to {dst}") +PY + +echo "--- Phase 2: patch evaluation ---" +cd "$SWEBENCH_DIR" +"$PYTHON" swe_bench_pro_eval.py \ + --raw_sample_path "$RAW_SAMPLE" \ + --patch_path "$PATCHES_JSON" \ + --output_dir "$EVAL_DIR" \ + --scripts_dir run_scripts \ + --num_workers "$EVAL_WORKERS" \ + --dockerhub_username "$DOCKERHUB_USERNAME" \ + --use_local_docker \ + "${REDO_EVAL_ARG[@]}" + +echo +echo "=== SWE-bench Pro complete ===" +echo " Predictions : $PRED_DIR/preds.json" +echo " Patches : $PATCHES_JSON" +echo " Eval results: $EVAL_DIR/eval_results.json" +echo +"$PYTHON" - "$EVAL_DIR/eval_results.json" "$OUT_DIR/results.json" "${LIMIT:-0}" <<'PY' +import json +import pathlib +import sys + +eval_path = pathlib.Path(sys.argv[1]) +result_path = pathlib.Path(sys.argv[2]) +expected = int(sys.argv[3]) + +if not eval_path.is_file(): + raise SystemExit(f"SWE-bench Pro evaluation failed: missing {eval_path}") +r = json.loads(eval_path.read_text()) +if not isinstance(r, dict) or not r: + raise SystemExit("SWE-bench Pro evaluation failed: no evaluated instances") +if not all(isinstance(value, bool) for value in r.values()): + raise SystemExit("SWE-bench Pro evaluation failed: invalid result values") +if expected and len(r) != expected: + raise SystemExit( + f"SWE-bench Pro evaluation failed: expected {expected} results, got {len(r)}" + ) +n = len(r) +p = sum(1 for v in r.values() if v) +score = p / n +result_path.write_text(json.dumps({ + "results": { + "swebench_pro": { + "exact_match,resolved": score, + }, + }, + "n-samples": { + "swebench_pro": { + "effective": n, + }, + }, +}, indent=2)) +print(f"Pass@1: {p}/{n} ({100 * score:.1f}%)") +print(f"Wrote normalized result: {result_path}") +PY diff --git a/benchmarks/single_node/quality/tasks/hle/_hle.yaml b/benchmarks/single_node/quality/tasks/hle/_hle.yaml new file mode 100644 index 0000000000..235775b00a --- /dev/null +++ b/benchmarks/single_node/quality/tasks/hle/_hle.yaml @@ -0,0 +1,10 @@ +group: hle +task: + - hle_exact_match + - hle_multiple_choice +aggregate_metric_list: + - aggregation: mean + metric: exact_match + weight_by_size: true +metadata: + version: 1.0 diff --git a/benchmarks/single_node/quality/tasks/hle/hle_exact_match.yaml b/benchmarks/single_node/quality/tasks/hle/hle_exact_match.yaml new file mode 100644 index 0000000000..e63bc6ae67 --- /dev/null +++ b/benchmarks/single_node/quality/tasks/hle/hle_exact_match.yaml @@ -0,0 +1,31 @@ +task: hle_exact_match +dataset_path: cais/hle +test_split: test +output_type: generate_until +process_docs: !function utils.process_docs_exact_match +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "exact-match" + filter: + - function: "regex" + regex_pattern: "#### (.*)" + group_select: -1 + - function: "take_first" + - name: "flexible-extract" + filter: + - function: "take_first" +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +generation_kwargs: + until: + - "Question:" + max_gen_toks: 4096 + do_sample: false + temperature: 0.0 +metadata: + version: 1.0 diff --git a/benchmarks/single_node/quality/tasks/hle/hle_multiple_choice.yaml b/benchmarks/single_node/quality/tasks/hle/hle_multiple_choice.yaml new file mode 100644 index 0000000000..2acce7e9f8 --- /dev/null +++ b/benchmarks/single_node/quality/tasks/hle/hle_multiple_choice.yaml @@ -0,0 +1,34 @@ +task: hle_multiple_choice +dataset_path: cais/hle +test_split: test +output_type: generate_until +process_docs: !function utils.process_docs_multiple_choice +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: '(?i)answer is\s*\(?([A-J])\)?' + group_select: -1 + - function: "take_first" + - name: "flexible-extract" + filter: + - function: "regex" + regex_pattern: "\\(([A-J])\\)" + group_select: -1 + - function: "take_first" +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +generation_kwargs: + until: + - "Question:" + max_gen_toks: 4096 + do_sample: false + temperature: 0.0 +metadata: + version: 1.0 diff --git a/benchmarks/single_node/quality/tasks/hle/utils.py b/benchmarks/single_node/quality/tasks/hle/utils.py new file mode 100644 index 0000000000..b0db697b14 --- /dev/null +++ b/benchmarks/single_node/quality/tasks/hle/utils.py @@ -0,0 +1,44 @@ +import re + + +def _is_text_only(doc): + """Return True if the question has no image content (text-only).""" + img = doc.get("image") + return not img or len(str(img)) <= 100 + + +def process_docs_exact_match(dataset): + """Filter to text-only exactMatch questions.""" + return dataset.filter( + lambda x: x["answer_type"] == "exactMatch" and _is_text_only(x) + ) + + +def process_docs_multiple_choice(dataset): + """Filter to text-only multipleChoice questions.""" + return dataset.filter( + lambda x: x["answer_type"] == "multipleChoice" and _is_text_only(x) + ) + + +def doc_to_text(doc): + """Format the question with an answer prompt. + + For exactMatch: ask the model to put the final answer after ####. + For multipleChoice: the choices are already in the question text; + ask for the letter answer. + """ + question = doc["question"] + if doc["answer_type"] == "exactMatch": + return ( + f"{question}\n\n" + f"Please solve this problem, then conclude with exactly one final line " + f"in the form \"#### \"." + ) + else: + return ( + f"{question}\n\n" + f"The answer is the letter of the correct choice. " + f"Conclude with exactly one final line in the form \"The answer is (X)\", " + f"where X is a letter from A through J." + ) diff --git a/benchmarks/single_node/quality/tasks/swebench-pro/run_swebench_pro.py b/benchmarks/single_node/quality/tasks/swebench-pro/run_swebench_pro.py new file mode 100644 index 0000000000..c8505039dd --- /dev/null +++ b/benchmarks/single_node/quality/tasks/swebench-pro/run_swebench_pro.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Run mini-SWE-agent on SWE-bench Pro instances loaded from our generated instances.yaml. + +This is a thin wrapper around minisweagent.run.extra.swebench that loads instances +from the YAML file produced by helper_code/generate_sweagent_instances.py (which +already contains the correct jefzda/sweap-images: image_name per instance) +instead of relying on the standard SWE-bench DATASET_MAPPING + default image-name +formula (which does not apply to SWE-bench Pro). + +Usage: + python tasks/swebench-pro/run_swebench_pro.py \ + --instances-path SWE-bench_Pro-os/SWE-agent/data/instances.yaml \ + --output jobs//swebench-pro/preds \ + --config tasks/swebench-pro/swebench_pro.yaml \ + --model openai/z-ai/glm-5.2 \ + --workers 4 \ + --slice 0:3 +""" + +import argparse +import concurrent.futures +import json +import random +import re +import time +import traceback +from pathlib import Path + +import yaml +from rich.live import Live + +from minisweagent.run.benchmarks.swebench import filter_instances, process_instance +from minisweagent.run.benchmarks.utils.batch_progress import RunBatchProgressManager +from minisweagent.utils.log import add_file_handler, logger + + +def load_instances_from_yaml(path: Path) -> list[dict]: + """Load instances from the YAML produced by generate_sweagent_instances.py. + + Each YAML entry has: image_name, problem_statement, instance_id, base_commit, repo_name. + We re-shape into the dict shape that minisweagent.run.extra.swebench expects: + instance_id, problem_statement, image_name, base_commit, repo. + """ + entries = yaml.safe_load(path.read_text()) + instances = [] + for e in entries: + instances.append( + { + "instance_id": e["instance_id"], + "problem_statement": e["problem_statement"], + "image_name": e["image_name"], + "base_commit": e.get("base_commit", ""), + "repo": e.get("repo_name", ""), + } + ) + return instances + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--instances-path", required=True, help="Path to instances.yaml") + parser.add_argument("--output", required=True, help="Output directory") + parser.add_argument("--config", required=True, help="Path to agent config yaml") + parser.add_argument("--model", default=None, help="Model name (e.g. openai/z-ai/glm-5.2)") + parser.add_argument("--model-class", default=None, help="Model class shortcut or import path") + parser.add_argument("--environment-class", default=None, help="Environment class (docker/singularity)") + parser.add_argument("--workers", type=int, default=1) + parser.add_argument("--slice", default="", help="Slice spec e.g. 0:3") + parser.add_argument("--filter", default="", help="Regex filter on instance_id") + parser.add_argument("--shuffle", action="store_true") + parser.add_argument("--redo-existing", action="store_true") + args = parser.parse_args() + + output_path = Path(args.output) + output_path.mkdir(parents=True, exist_ok=True) + logger.info(f"Results will be saved to {output_path}") + add_file_handler(output_path / "minisweagent.log") + + logger.info(f"Loading instances from {args.instances_path}") + instances = load_instances_from_yaml(Path(args.instances_path)) + logger.info(f"Loaded {len(instances)} instances") + + instances = filter_instances( + instances, filter_spec=args.filter, slice_spec=args.slice, shuffle=args.shuffle + ) + + if not args.redo_existing and (output_path / "preds.json").exists(): + existing = list(json.loads((output_path / "preds.json").read_text()).keys()) + logger.info(f"Skipping {len(existing)} existing instances") + instances = [i for i in instances if i["instance_id"] not in existing] + + logger.info(f"Running on {len(instances)} instances...") + + config = yaml.safe_load(Path(args.config).read_text()) + if args.environment_class is not None: + config.setdefault("environment", {})["environment_class"] = args.environment_class + if args.model is not None: + config.setdefault("model", {})["model_name"] = args.model + if args.model_class is not None: + config.setdefault("model", {})["model_class"] = args.model_class + + progress_manager = RunBatchProgressManager(len(instances), output_path / f"exit_statuses_{time.time()}.yaml") + + def process_futures(futures: dict[concurrent.futures.Future, str]): + for future in concurrent.futures.as_completed(futures): + try: + future.result() + except concurrent.futures.CancelledError: + pass + except Exception as e: + iid = futures[future] + logger.error(f"Error in future for instance {iid}: {e}", exc_info=True) + progress_manager.on_uncaught_exception(iid, e) + + with Live(progress_manager.render_group, refresh_per_second=4): + with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as executor: + futures = { + executor.submit(process_instance, inst, output_path, config, progress_manager): inst["instance_id"] + for inst in instances + } + try: + process_futures(futures) + except KeyboardInterrupt: + logger.info("Cancelling pending jobs. Press ^C again to exit immediately.") + for f in futures: + if not f.running() and not f.done(): + f.cancel() + process_futures(futures) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/single_node/quality/tasks/swebench-pro/swebench_pro.yaml b/benchmarks/single_node/quality/tasks/swebench-pro/swebench_pro.yaml new file mode 100644 index 0000000000..125eb098c7 --- /dev/null +++ b/benchmarks/single_node/quality/tasks/swebench-pro/swebench_pro.yaml @@ -0,0 +1,211 @@ +agent: + system_template: | + You are a helpful assistant that can interact multiple times with a computer shell to solve programming tasks. + Your response must contain exactly ONE bash code block with ONE command (or commands connected with && or ||). + + Include a THOUGHT section before your command where you explain your reasoning process. + Format your response as shown in . + + + THOUGHT: Your reasoning and analysis here + + ```bash + your_command_here + ``` + + + Failure to follow these rules will cause your response to be rejected. + instance_template: | + + Consider the following PR description: + {{task}} + + + + # Task Instructions + + ## Overview + You're a software engineer interacting continuously with a computer by submitting commands. + You'll be helping implement necessary changes to meet requirements in the PR description. + Your task is specifically to make changes to non-test files in the current directory in order to fix the issue described in the PR description in a way that is general and consistent with the codebase. + + IMPORTANT: This is an interactive process where you will think and issue ONE command, see its result, then think and issue your next command. + + For each response: + 1. Include a THOUGHT section explaining your reasoning and what you're trying to accomplish + 2. Provide exactly ONE bash command to execute + + ## Important Boundaries + - MODIFY: Regular source code files in /app (this is the working directory for all your subsequent commands) + - DO NOT MODIFY: Tests, configuration files (pyproject.toml, setup.cfg, etc.) + + ## Recommended Workflow + 1. Analyze the codebase by finding and reading relevant files + 2. Create a script to reproduce the issue + 3. Edit the source code to resolve the issue + 4. Verify your fix works by running your script again + 5. Test edge cases to ensure your fix is robust + + ## Command Execution Rules + You are operating in an environment where + 1. You write a single command + 2. The system executes that command in a subshell + 3. You see the result + 4. You write your next command + + Each response should include: + 1. A **THOUGHT** section where you explain your reasoning and plan + 2. A single bash code block with your command + + Format your responses like this: + + + THOUGHT: Here I explain my reasoning process, analysis of the current situation, + and what I'm trying to accomplish with the command below. + + ```bash + your_command_here + ``` + + + Commands must be specified in a single bash code block: + + ```bash + your_command_here + ``` + + **CRITICAL REQUIREMENTS:** + - Your response SHOULD include a THOUGHT section explaining your reasoning + - Your response MUST include EXACTLY ONE bash code block + - This bash block MUST contain EXACTLY ONE command (or a set of commands connected with && or ||) + - If you include zero or multiple bash blocks, or no command at all, YOUR RESPONSE WILL FAIL + - Do NOT try to run multiple independent commands in separate blocks in one response + - Directory or environment variable changes are not persistent. Every action is executed in a new subshell. + - However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files + + ## Environment Details + - You have a full Linux shell environment + - Always use non-interactive flags (-y, -f) for commands + - Avoid interactive tools like vi, nano, or any that require user input + - If a command isn't available, you can install it + + ## Useful Command Examples + + ### Create a new file: + ```bash + cat <<'EOF' > newfile.py + import numpy as np + hello = "world" + print(hello) + EOF + ``` + + ### Edit files with sed: + ```bash + # Replace all occurrences + sed -i 's/old_string/new_string/g' filename.py + + # Replace only first occurrence + sed -i 's/old_string/new_string/' filename.py + + # Replace first occurrence on line 1 + sed -i '1s/old_string/new_string/' filename.py + + # Replace all occurrences in lines 1-10 + sed -i '1,10s/old_string/new_string/g' filename.py + ``` + + ### View file content: + ```bash + # View specific lines with numbers + nl -ba filename.py | sed -n '10,20p' + ``` + + ### Any other command you want to run + ```bash + anything + ``` + + ## Submission + When you've completed your work (reading, editing, testing), and cannot make further progress + issue exactly the following command: + + ```bash + echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT && git add -A && git diff --cached + ``` + + This command will submit your work. + You cannot continue working (reading, editing, testing) in any way on this task after submitting. + + action_observation_template: | + {{output.returncode}} + {% if output.output | length < 10000 -%} + + {{ output.output -}} + + {%- else -%} + + The output of your last command was too long. + Please try a different command that produces less output. + If you're looking at a file you can try use head, tail or sed to view a smaller number of lines selectively. + If you're using grep or find and it produced too much output, you can use a more selective search pattern. + If you really need to see something from the full command's output, you can redirect output to a file and then search in that file. + + {%- set elided_chars = output.output | length - 10000 -%} + + {{ output.output[:5000] }} + + + {{ elided_chars }} characters elided + + + {{ output.output[-5000:] }} + + {%- endif -%} + format_error_template: | + Please always provide EXACTLY ONE action in triple backticks, found {{actions|length}} actions. + + Please format your action in triple backticks as shown in . + + + Here are some thoughts about why you want to perform the action. + + ```bash + + ``` + + + If you have completed your assignment, please consult the first message about how to + submit your solution (you will not be able to continue working on this task after that). + step_limit: 250 + cost_limit: 3. + +environment: + cwd: "/app" + timeout: 120 + env: + PAGER: cat + MANPAGER: cat + LESS: -R + PIP_PROGRESS_BAR: 'off' + TQDM_DISABLE: '1' + environment_class: docker + # sweap-images set ENTRYPOINT ["/bin/bash"], which swallows the `sleep 2h` + # arg passed by DockerEnvironment._start_container and causes the container + # to exit immediately. Clear the entrypoint so CMD ["sleep","2h"] runs as + # PID 1 and keeps the container alive for `docker exec`. + run_args: + - "--rm" + - "--entrypoint" + - "" + +model: + model_name: "openai/z-ai/glm-5.2" + # This agent prompt requests fenced bash commands, so use mini-swe-agent's + # text action parser rather than its native tool-call model. + model_class: "litellm_textbased" + action_regex: '```bash\s*\n(.*?)\n```' + model_kwargs: + drop_params: true + temperature: 0.0 + cost_tracking: "ignore_errors" diff --git a/configs/ci-priority.yaml b/configs/ci-priority.yaml index f6c9388623..667f05c043 100644 --- a/configs/ci-priority.yaml +++ b/configs/ci-priority.yaml @@ -14,6 +14,7 @@ adjustments: multi-node: 1.25 agentic: 1.0 eval-only: 0.0 + quality-eval: 0.0 precision: fp4: 0.75 spec-decoding: diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index caa0a7e082..d2d4131a37 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8511,3 +8511,62 @@ glm5.1-fp8-b200-tilert: dp-attn: false additional-settings: - "DECODE_NODES=1" + +# ============================================================================= +# Quality-eval configs: 8 benchmarks (GPQA, MMLU-Pro, HLE, LiveCodeBench, +# BFCL, SciCode, SWE-bench Pro, DeepSWE) run against an external OpenAI- +# compatible endpoint. No GPU server is launched — the runner only drives +# the benchmark script against the endpoint. Each scenario type maps to +# one benchmark; a single master config entry can include multiple scenario +# types to run several benchmarks in one sweep. +# ============================================================================= + +glm5.2-quality-eval: + image: quality-eval:latest + model: GLM-5.2 + model-prefix: glm5.2 + runner: cluster:quality-eval + precision: fp4 + framework: quality + multinode: false + scenarios: + # quality-gpqa: # smoke passed (CI run 34092392880) + # - quality-endpoint: "https://tokenplan.api.greennode.ai/v1" + # quality-model-name: "openai/z-ai/glm-5.2" + # search-space: + # - { benchmark-name: gpqa, smoke: true, num-concurrent: 4, eval-limit: 2 } + # quality-mmlu-pro: # smoke passed (CI run 34092392880) + # - quality-endpoint: "https://tokenplan.api.greennode.ai/v1" + # quality-model-name: "openai/z-ai/glm-5.2" + # search-space: + # - { benchmark-name: mmlu_pro, smoke: true, num-concurrent: 4, eval-limit: 2 } + # quality-hle: # smoke passed (CI run 34095182816) + # - quality-endpoint: "https://tokenplan.api.greennode.ai/v1" + # quality-model-name: "openai/z-ai/glm-5.2" + # search-space: + # - { benchmark-name: hle, smoke: true, num-concurrent: 4, eval-limit: 2 } + quality-livecodebench: # coding balanced tier + - quality-endpoint: "https://tokenplan.api.greennode.ai/v1" + quality-model-name: "openai/z-ai/glm-5.2" + search-space: + - { benchmark-name: livecodebench, smoke: false, num-concurrent: 4, eval-limit: 50 } + quality-bfcl: # agentic coding balanced tier + - quality-endpoint: "https://tokenplan.api.greennode.ai/v1" + quality-model-name: "openai/z-ai/glm-5.2" + search-space: + - { benchmark-name: bfcl, smoke: false, num-concurrent: 4, eval-limit: 100 } + quality-scicode: # coding balanced tier + - quality-endpoint: "https://tokenplan.api.greennode.ai/v1" + quality-model-name: "openai/z-ai/glm-5.2" + search-space: + - { benchmark-name: scicode, smoke: false, num-concurrent: 4, eval-limit: 8 } + quality-swebench-pro: # agentic coding balanced tier + - quality-endpoint: "https://tokenplan.api.greennode.ai/v1" + quality-model-name: "openai/z-ai/glm-5.2" + search-space: + - { benchmark-name: swebench_pro, smoke: false, num-concurrent: 4, eval-limit: 10 } + quality-deepswe: # agentic coding balanced tier + - quality-endpoint: "https://tokenplan.api.greennode.ai/v1" + quality-model-name: "openai/z-ai/glm-5.2" + search-space: + - { benchmark-name: deepswe, smoke: false, num-concurrent: 4, eval-limit: 10 } diff --git a/configs/runners.yaml b/configs/runners.yaml index 801180125b..f59b455841 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -314,6 +314,8 @@ labels: - mi355x-amds_08 cluster:remote-bench: - bench-client_01 + cluster:quality-eval: + - bench-client_00 hardware: cluster:h100-dgxc: available-cpu-dram-mib: 2_063_837 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 04ba9feb56..2aaff12bfc 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5953,3 +5953,114 @@ - "Use native EAGLE MTP (3 steps, top-k 1, 4 draft tokens) and golden synthetic acceptance length 2.49 for throughput; eval retains real verification." - "Follow the official SGLang DeepSeek-V4 Blackwell recipe, require nonempty SGLang server metrics, keep pooled AgentX connections alive, let AIPerf own HiCache warmup, and reserve transient MoE workspace at DEP8 c512." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2577 + +- config-keys: + - glm5.2-quality-eval + scenario-type: + - quality-gpqa + description: + - "Smoke run GPQA-Diamond with 10 samples against GreenNode endpoint for GLM-5.2 quality validation." + pr-link: https://github.com/vngcloud/InferenceX/pull/36 + +- config-keys: + - glm5.2-quality-eval + scenario-type: + - quality-mmlu-pro + description: + - "Smoke run MMLU-Pro with 20 questions, ccu4, against GreenNode endpoint." + pr-link: https://github.com/vngcloud/InferenceX/pull/36 + +- config-keys: + - glm5.2-quality-eval + scenario-type: + - quality-hle + description: + - "Smoke run HLE with 8 questions, ccu4, against GreenNode endpoint." + pr-link: https://github.com/vngcloud/InferenceX/pull/36 + +- config-keys: + - glm5.2-quality-eval + scenario-type: + - quality-bfcl + description: + - "Smoke run BFCL v4 with 4 questions, ccu1, against GreenNode endpoint." + pr-link: https://github.com/vngcloud/InferenceX/pull/36 + +- config-keys: + - glm5.2-quality-eval + scenario-type: + - quality-scicode + description: + - "Smoke run SciCode with 2 questions, ccu2, against GreenNode endpoint." + pr-link: https://github.com/vngcloud/InferenceX/pull/36 + +- config-keys: + - glm5.2-quality-eval + scenario-type: + - quality-deepswe + description: + - "Smoke run DeepSWE with 1 task against GreenNode endpoint." + pr-link: https://github.com/vngcloud/InferenceX/pull/36 + +- config-keys: + - glm5.2-quality-eval + scenario-type: + - quality-swebench-pro + description: + - "Re-run SWE-bench Pro smoke after aligning the mini-swe-agent text-based action parser and validating non-empty patches." + pr-link: https://github.com/vngcloud/InferenceX/pull/36 + +- config-keys: + - glm5.2-quality-eval + scenario-type: + - quality-hle + description: + - "Re-run HLE smoke after enabling streaming on lm-eval's OpenAI chat adapter and preserving GLM reasoning deltas." + pr-link: https://github.com/vngcloud/InferenceX/pull/36 + +- config-keys: + - glm5.2-quality-eval + scenario-type: + - quality-hle + description: + - "Dispatch HLE smoke only; disable the already-passed SWE-bench Pro smoke scenario." + pr-link: https://github.com/vngcloud/InferenceX/pull/36 + +- config-keys: + - glm5.2-quality-eval + scenario-type: + - quality-gpqa + - quality-mmlu-pro + - quality-hle + description: + - "Fix HLE A-J final-answer extraction, raise HLE and MMLU-Pro generation budgets for reasoning models, and dispatch a focused two-sample lm-eval smoke." + pr-link: https://github.com/vngcloud/InferenceX/pull/36 + +- config-keys: + - glm5.2-quality-eval + scenario-type: + - quality-hle + description: + - "Raise HLE's lm-eval HTTP total timeout to 1800 seconds for long streamed reasoning and re-run HLE smoke only." + pr-link: https://github.com/vngcloud/InferenceX/pull/36 + +- config-keys: + - glm5.2-quality-eval + scenario-type: + - quality-livecodebench + - quality-scicode + description: + - "Run the balanced coding recipe: 50 LiveCodeBench samples and 8 SciCode samples, both at ccu4." + pr-link: https://github.com/vngcloud/InferenceX/pull/36 + +- config-keys: + - glm5.2-quality-eval + scenario-type: + - quality-livecodebench + - quality-scicode + - quality-bfcl + - quality-swebench-pro + - quality-deepswe + description: + - "Run the combined balanced coding and agentic coding recipes at ccu4: LiveCodeBench 50, SciCode 8, BFCL 100, SWE-bench Pro 10, and DeepSWE 10." + pr-link: https://github.com/vngcloud/InferenceX/pull/36 diff --git a/runners/launch_bench-client.sh b/runners/launch_bench-client.sh index 4460b1ec09..c54abf8fc4 100755 --- a/runners/launch_bench-client.sh +++ b/runners/launch_bench-client.sh @@ -15,6 +15,14 @@ set -x export INFMAX_CONTAINER_WORKSPACE="${GITHUB_WORKSPACE:-$(pwd)}" export RESULT_DIR="${INFMAX_CONTAINER_WORKSPACE}/results" +# Quality-eval scenario types are dispatched to launch_quality-eval.sh, +# which sets up venvs and calls the appropriate benchmark script. +# Everything else is a remote-bench recipe. +if [[ "${SCENARIO_SUBDIR:-}" == "quality/" ]]; then + bash ./runners/launch_quality-eval.sh + exit $? +fi + BENCH_SCRIPT="benchmarks/single_node/${SCENARIO_SUBDIR}${EXP_NAME%%_*}_${PRECISION}_${FRAMEWORK}-remote-bench.sh" bash "$BENCH_SCRIPT" diff --git a/runners/launch_quality-eval.sh b/runners/launch_quality-eval.sh new file mode 100755 index 0000000000..b128f0f030 --- /dev/null +++ b/runners/launch_quality-eval.sh @@ -0,0 +1,1218 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# CPU runner for quality-eval benchmarks. No salloc, no Docker image, no +# squashfs — this box only drives benchmark scripts against an externally- +# managed inference endpoint (QUALITY_ENDPOINT). +# +# This script sets up the Python environments and clones the external +# benchmark repos (if not already present) before dispatching to the +# per-benchmark script under benchmarks/single_node/quality/. +# +# Venvs and cloned repos live under $QUALITY_CACHE_DIR (persistent across +# CI runs on a self-hosted runner). Job output (results, logs) lives under +# $QUALITY_WORKSPACE (= $GITHUB_WORKSPACE) and is uploaded as artifacts. +# +# Required env (set by benchmark-tmpl.yml): +# QUALITY_BENCHMARK_NAME e.g. gpqa, mmlu_pro, hle, livecodebench, bfcl, +# scicode, swebench_pro, deepswe +# QUALITY_ENDPOINT e.g. https://maas-llm-aiplatform-hcm.api.vngcloud.vn/v1 +# QUALITY_API_KEY API key for the endpoint +# QUALITY_MODEL_NAME e.g. openai/z-ai/glm-5.2 +# Optional env: +# RUN_ID, LIMIT (from EVAL_LIMIT), HF_TOKEN, SMOKE + +# --- Paths --------------------------------------------------------------- +# Job output: per-run workspace (cleaned by GitHub Actions each run) +export QUALITY_WORKSPACE="${GITHUB_WORKSPACE:-$(pwd)}" + +# Persistent cache: venvs + cloned repos survive across runs. +# On a self-hosted runner $HOME is stable. Fall back to /tmp for ephemeral CI. +export QUALITY_CACHE_DIR="${QUALITY_CACHE_DIR:-${HOME:-/tmp}/.quality-eval-cache}" +mkdir -p "$QUALITY_CACHE_DIR" + +export RESULT_DIR="${QUALITY_WORKSPACE}/results" + +BENCH_SCRIPT="benchmarks/single_node/quality/run_${QUALITY_BENCHMARK_NAME}.sh" + +if [[ ! -f "$BENCH_SCRIPT" ]]; then + echo "ERROR: Unknown quality benchmark '${QUALITY_BENCHMARK_NAME}'" >&2 + echo "Expected script: $BENCH_SCRIPT" >&2 + exit 1 +fi + +# Map InferenceX's EVAL_LIMIT to the LIMIT env var that all benchmark +# scripts read for subset/smoke runs. +export LIMIT="${EVAL_LIMIT:-${LIMIT:-}}" + +# Map NUM_CONCURRENT to per-benchmark concurrency env vars. +# Each benchmark script reads its own var; NUM_CONCURRENT is the unified knob. +if [[ -n "${NUM_CONCURRENT:-}" ]]; then + export NUM_CONCURRENT="$NUM_CONCURRENT" # gpqa, mmlu_pro, hle (lm-eval) + export MULTIPROCESS="$NUM_CONCURRENT" # livecodebench + export NUM_THREADS="$NUM_CONCURRENT" # bfcl + export MAX_CONNECTIONS="$NUM_CONCURRENT" # scicode + export WORKERS="$NUM_CONCURRENT" # swebench_pro + export CCU="$NUM_CONCURRENT" # deepswe +fi + +# Set RUN_ID from the experiment name if not already set. +export RUN_ID="${RUN_ID:-${EXP_NAME:-quality-eval}}" + +# Ensure uv is available (GitHub Actions runners may not have it pre-installed). +if ! command -v uv >/dev/null 2>&1; then + curl -LsSf https://astral.sh/uv/install.sh | sh + export PATH="$HOME/.local/bin:$PATH" +fi + +# --------------------------------------------------------------------------- +# Per-benchmark environment setup +# --------------------------------------------------------------------------- +# Each function ensures the venv and external repo are ready. +# Venvs/repos live under $QUALITY_CACHE_DIR and are reused across runs. +# First run: creates everything from scratch (~5-15 min depending on benchmark). +# Subsequent runs: skips setup entirely (just exports paths). + +setup_lmeval() { + local VENV="$QUALITY_CACHE_DIR/.venv-lmeval" + if [[ ! -x "$VENV/bin/lm-eval" ]] || ! "$VENV/bin/python" -c "import tenacity, PIL" 2>/dev/null; then + echo "=== Setting up lm-eval venv ===" + uv venv --clear --seed "$VENV" + uv pip install --python "$VENV/bin/python" \ + "lm-eval[api]>=0.4.5" "openai>=1.59.0" "Pillow>=10.0.0" + fi + # Install the repository-owned runtime hooks into the persistent venv on + # every run. This repairs already-cached environments and keeps reasoning + # delta assembly independent of the installed lm-eval source layout. + local SITE_PACKAGES + SITE_PACKAGES="$("$VENV/bin/python" -c 'import site; print(site.getsitepackages()[0])')" + cp "$QUALITY_WORKSPACE/utils/evals/patches/lm_eval_sitecustomize.py" \ + "$SITE_PACKAGES/sitecustomize.py" + # Patch openai_completions.py + api_models.py for streaming + local OAI_COMP="$VENV/lib/python3.12/site-packages/lm_eval/models/openai_completions.py" + if [[ -f "$OAI_COMP" ]] && ! grep -q '"stream": True' "$OAI_COMP" 2>/dev/null; then + echo "=== Patching lm-eval openai_completions.py for streaming ===" + python3 - "$OAI_COMP" <<'PY' +import pathlib, sys +p = pathlib.Path(sys.argv[1]) +src = p.read_text() +# Add stream=True to chat completions payload +old = ''' "seed": seed, + **gen_kwargs, + } + + def parse_generations(self, outputs: dict | list[dict], **kwargs) -> list[str]: + res = [] + if not isinstance(outputs, list): + outputs = [outputs] + for out in outputs: + try: + tmp = [None] * len(out["choices"]) + for choices in out["choices"]: + content = choices["message"]["content"]''' +new = ''' "seed": seed, + "stream": True, + **gen_kwargs, + } + + def parse_generations(self, outputs: dict | list[dict], **kwargs) -> list[str]: + res = [] + if not isinstance(outputs, list): + outputs = [outputs] + for out in outputs: + try: + tmp = [None] * len(out["choices"]) + for choices in out["choices"]: + content = choices["message"]["content"]''' +if old in src and '"stream": True' not in src: + src = src.replace(old, new, 1) + p.write_text(src) +PY + fi + local API_MODELS="$VENV/lib/python3.12/site-packages/lm_eval/models/api_models.py" + if [[ -f "$API_MODELS" ]] && ! grep -q '_parse_sse_stream' "$API_MODELS" 2>/dev/null; then + echo "=== Patching lm-eval api_models.py for streaming ===" + python3 - "$API_MODELS" <<'PY' +import pathlib, sys +p = pathlib.Path(sys.argv[1]) +src = p.read_text() +# Add SSE helpers after imports +old = '''from lm_eval.api.model import TemplateLM''' +new = '''from lm_eval.api.model import TemplateLM + + +def _parse_sse_stream(response): + """Parse SSE stream from requests.Response into a dict matching non-stream format.""" + import json as _json + content = "" + finish_reason = None + usage = None + model = None + for line in response.iter_lines(decode_unicode=True): + if not line or not line.startswith("data: "): + continue + data = line[6:] + if data.strip() == "[DONE]": + break + try: + chunk = _json.loads(data) + except _json.JSONDecodeError: + continue + if "usage" in chunk and chunk["usage"]: + usage = chunk["usage"] + if "model" in chunk and chunk["model"]: + model = chunk["model"] + if "choices" in chunk and chunk["choices"]: + delta = chunk["choices"][0].get("delta", {}) + if delta.get("content"): + content += delta["content"] + if chunk["choices"][0].get("finish_reason"): + finish_reason = chunk["choices"][0]["finish_reason"] + return { + "id": "stream-accumulated", + "object": "chat.completion", + "model": model or "", + "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": finish_reason or "stop"}], + "usage": usage or {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}, + } + + +async def _parse_sse_stream_async(response): + """Parse SSE stream from aiohttp response into a dict matching non-stream format.""" + import json as _json + content = "" + finish_reason = None + usage = None + model = None + async for raw_line in response.content: + line = raw_line.decode("utf-8").strip() + if not line or not line.startswith("data: "): + continue + data = line[6:] + if data.strip() == "[DONE]": + break + try: + chunk = _json.loads(data) + except _json.JSONDecodeError: + continue + if "usage" in chunk and chunk["usage"]: + usage = chunk["usage"] + if "model" in chunk and chunk["model"]: + model = chunk["model"] + if "choices" in chunk and chunk["choices"]: + delta = chunk["choices"][0].get("delta", {}) + if delta.get("content"): + content += delta["content"] + if chunk["choices"][0].get("finish_reason"): + finish_reason = chunk["choices"][0]["finish_reason"] + return { + "id": "stream-accumulated", + "object": "chat.completion", + "model": model or "", + "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": finish_reason or "stop"}], + "usage": usage or {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}, + }''' +if old in src and "_parse_sse_stream" not in src: + src = src.replace(old, new, 1) + # Patch model_call to use streaming + old2 = ''' response = requests.post( + self.base_url, + json=self._create_payload( + self.create_message(messages), + generate=generate, + gen_kwargs=gen_kwargs, + seed=self._seed, + eos=self.eos_string, + **kwargs, + ), + headers=self.header, + verify=self.verify_certificate, + timeout=self.timeout, + ) + if not response.ok: + eval_logger.warning( + f"API request failed with error message: {response.text}. Retrying..." + ) + response.raise_for_status() + return response.json()''' + new2 = ''' payload = self._create_payload( + self.create_message(messages), + generate=generate, + gen_kwargs=gen_kwargs, + seed=self._seed, + eos=self.eos_string, + **kwargs, + ) + is_stream = payload.get("stream", False) + response = requests.post( + self.base_url, + json=payload, + headers=self.header, + verify=self.verify_certificate, + timeout=self.timeout, + stream=is_stream, + ) + if not response.ok: + eval_logger.warning( + f"API request failed with error message: {response.text}. Retrying..." + ) + response.raise_for_status() + if is_stream: + return _parse_sse_stream(response) + return response.json()''' + if old2 in src: + src = src.replace(old2, new2, 1) + # Patch amodel_call to use streaming + old3 = ''' response.raise_for_status() + outputs = await response.json()''' + new3 = ''' response.raise_for_status() + if payload.get("stream", False): + outputs = await _parse_sse_stream_async(response) + else: + outputs = await response.json()''' + if old3 in src: + src = src.replace(old3, new3, 1) + p.write_text(src) +PY + fi + export QUALITY_VENV="$VENV" +} + +setup_livecodebench() { + local LCB_DIR="$QUALITY_CACHE_DIR/LiveCodeBench" + local VENV="$LCB_DIR/.venv-lcb" + if [[ ! -d "$LCB_DIR" ]]; then + echo "=== Cloning LiveCodeBench (first time) ===" + git clone --depth 1 https://github.com/LiveCodeBench/LiveCodeBench.git "$LCB_DIR" + fi + # Patch lcb_runner files that import HUMAN_PROMPT/AI_PROMPT without + # try/except fallback (anthropic>=0.42 removed them). code_generation.py + # already has a fallback; self_repair.py and test_output_prediction.py do not. + for f in lcb_runner/prompts/self_repair.py lcb_runner/prompts/test_output_prediction.py; do + if [[ -f "$LCB_DIR/$f" ]] && ! grep -q "HUMAN_PROMPT = None" "$LCB_DIR/$f" 2>/dev/null; then + echo "=== Patching $f for anthropic>=0.42 compatibility ===" + python3 - "$LCB_DIR/$f" <<'PY' +import pathlib, sys, re +p = pathlib.Path(sys.argv[1]) +src = p.read_text() +old = "from anthropic import HUMAN_PROMPT, AI_PROMPT" +new = """try: + from anthropic import HUMAN_PROMPT, AI_PROMPT +except ImportError: + HUMAN_PROMPT = None + AI_PROMPT = None""" +if old in src and "HUMAN_PROMPT = None" not in src: + src = src.replace(old, new, 1) + p.write_text(src) +PY + fi + done + # Patch code_generation.py: load_dataset needs config name = release_version, + # otherwise datasets looks for 'default' config which doesn't exist in cache. + local CG_FILE="$LCB_DIR/lcb_runner/benchmarks/code_generation.py" + if ! grep -q 'release_version, split=' "$CG_FILE" 2>/dev/null; then + echo "=== Patching code_generation.py load_dataset config name ===" + python3 - "$CG_FILE" <<'PY' +import pathlib, sys +p = pathlib.Path(sys.argv[1]) +src = p.read_text() +old = 'load_dataset("livecodebench/code_generation_lite", split="test", version_tag=release_version, trust_remote_code=True)' +new = 'load_dataset("livecodebench/code_generation_lite", release_version, split="test", version_tag=release_version, trust_remote_code=True)' +if old in src: + src = src.replace(old, new, 1) + p.write_text(src) +PY + fi + # Patch main.py: add LCB_LIMIT env var support to slice benchmark + local MAIN_FILE="$LCB_DIR/lcb_runner/runner/main.py" + if ! grep -q "LCB_LIMIT" "$MAIN_FILE" 2>/dev/null; then + echo "=== Patching main.py with LCB_LIMIT support ===" + python3 - "$MAIN_FILE" <<'PY' +import pathlib, sys +p = pathlib.Path(sys.argv[1]) +src = p.read_text() +old = ' if args.debug:' +new = ''' _lcb_limit = int(os.environ.get("LCB_LIMIT", "0")) + if _lcb_limit > 0: + print(f"LCB_LIMIT={_lcb_limit}: slicing benchmark from {len(benchmark)} to {_lcb_limit} instances") + benchmark = benchmark[:_lcb_limit] + if args.debug:''' +if old in src and "LCB_LIMIT" not in src: + src = src.replace(old, new, 1) + p.write_text(src) +PY + fi + # Patch lm_styles.py to add z-ai/glm-5.2 as an OpenAIChat model + # (LCB has a hardcoded LanguageModelStore dict; our model isn't in it) + local LM_STYLES="$LCB_DIR/lcb_runner/lm_styles.py" + if ! grep -q '"z-ai/glm-5.2"' "$LM_STYLES" 2>/dev/null; then + echo "=== Patching lm_styles.py with z-ai/glm-5.2 model ===" + python3 - "$LM_STYLES" <<'PY' +import pathlib, sys +p = pathlib.Path(sys.argv[1]) +src = p.read_text() +entry = ''' LanguageModel( + "z-ai/glm-5.2", + "GLM-5.2", + LMStyle.OpenAIChat, + datetime(2024, 12, 1), + "https://huggingface.co/z-ai", + ), +''' +marker = "\n]\n\nLanguageModelStore" +idx = src.find(marker) +if idx == -1: + raise SystemExit("marker not found") +src = src[:idx] + "\n" + entry + src[idx:] +p.write_text(src) +PY + fi + # Patch path_utils.py: use LCB_OUTPUT_DIR env var as base for output path + # so results land in $OUT_BASE (where collect_results looks), not $LCB_DIR/output/ + local PATH_UTILS="$LCB_DIR/lcb_runner/utils/path_utils.py" + if ! grep -q 'LCB_OUTPUT_DIR' "$PATH_UTILS" 2>/dev/null; then + echo "=== Patching path_utils.py with LCB_OUTPUT_DIR support ===" + python3 - "$PATH_UTILS" <<'PY' +import pathlib, sys +p = pathlib.Path(sys.argv[1]) +src = p.read_text() +old = ' path = f"output/{model_repr}/{scenario}_{n}_{temperature}{cot_suffix}.json"' +new = ''' _base = os.environ.get("LCB_OUTPUT_DIR", "") + path = f"{_base}output/{model_repr}/{scenario}_{n}_{temperature}{cot_suffix}.json" if _base else f"output/{model_repr}/{scenario}_{n}_{temperature}{cot_suffix}.json"''' +if old in src and "LCB_OUTPUT_DIR" not in src: + src = src.replace(old, new, 1) + if "import os" not in src: + src = "import os\n" + src + p.write_text(src) +PY + fi + # Patch oai_runner.py to use streaming (avoid proxy timeouts on long generations) + local OAI_RUNNER="$LCB_DIR/lcb_runner/runner/oai_runner.py" + if ! grep -q 'stream=True' "$OAI_RUNNER" 2>/dev/null; then + echo "=== Patching oai_runner.py for streaming ===" + python3 - "$OAI_RUNNER" <<'PY' +import pathlib, sys +p = pathlib.Path(sys.argv[1]) +src = p.read_text() +old = ''' response = OpenAIRunner.client.chat.completions.create( + messages=prompt, + **self.client_kwargs, + ) + except (''' +new = ''' response = OpenAIRunner.client.chat.completions.create( + messages=prompt, + stream=True, + **self.client_kwargs, + ) + contents = [""] * self.client_kwargs.get("n", 1) + for chunk in response: + if not chunk.choices: + continue + for choice in chunk.choices: + if choice.delta.content: + idx = choice.index if choice.index < len(contents) else 0 + contents[idx] += choice.delta.content + except (''' +if old in src and "stream=True" not in src: + src = src.replace(old, new, 1) + old2 = ' return [c.message.content for c in response.choices]' + new2 = ' return contents' + if old2 in src: + src = src.replace(old2, new2, 1) + p.write_text(src) +PY + fi + # Cache-bust: check livecodebench import works + if [[ ! -x "$VENV/bin/python" ]] || ! "$VENV/bin/python" -c "import lcb_runner" 2>/dev/null; then + echo "=== Setting up LiveCodeBench venv (first time) ===" + uv venv --clear --seed "$VENV" + uv pip install --python "$VENV/bin/python" -e "$LCB_DIR" + fi + export QUALITY_LCB_VENV="$VENV" + export QUALITY_LCB_DIR="$LCB_DIR" +} + +setup_bfcl() { + local BFCL_DIR="$QUALITY_CACHE_DIR/BFCL/berkeley-function-call-leaderboard" + local VENV="$BFCL_DIR/.venv-bfcl" + if [[ ! -d "$QUALITY_CACHE_DIR/BFCL" ]]; then + echo "=== Cloning BFCL (first time) ===" + git clone --depth 1 https://github.com/ShishirPatil/gorilla.git "$QUALITY_CACHE_DIR/BFCL" + fi + # Inject z-ai/glm-5.2 model config if not already present + local MC_FILE="$BFCL_DIR/bfcl_eval/constants/model_config.py" + if ! grep -q "z-ai/glm-5.2-FC" "$MC_FILE" 2>/dev/null; then + echo "=== Patching BFCL model_config.py with z-ai/glm-5.2 ===" + python3 - "$MC_FILE" <<'PY' +import pathlib, re, sys +p = pathlib.Path(sys.argv[1]) +src = p.read_text() +block = ''' "z-ai/glm-5.2-FC": ModelConfig( + model_name="z-ai/glm-5.2", + display_name="GLM-5.2 (FC, OpenAI-compatible)", + url="https://tokenplan.api.greennode.ai", + org="z-ai", + license="Proprietary", + model_handler=OpenAICompletionsHandler, + input_price=None, + output_price=None, + is_fc_model=True, + underscore_to_dot=False, + ), + "z-ai/glm-5.2-PROMPT": ModelConfig( + model_name="z-ai/glm-5.2", + display_name="GLM-5.2 (Prompt, OpenAI-compatible)", + url="https://tokenplan.api.greennode.ai", + org="z-ai", + license="Proprietary", + model_handler=OpenAICompletionsHandler, + input_price=None, + output_price=None, + is_fc_model=False, + underscore_to_dot=False, + ), +''' +marker = 'api_inference_model_map = {' +idx = src.find(marker) +if idx == -1: + print("ERROR: could not find api_inference_model_map marker", file=sys.stderr) + sys.exit(1) +insert_at = src.find('{', idx) + 1 +p.write_text(src[:insert_at] + '\n' + block + src[insert_at:]) +PY + fi + # Patch openai_completion.py to use streaming (avoid proxy timeouts) + local OAI_COMP="$BFCL_DIR/bfcl_eval/model_handler/api_inference/openai_completion.py" + if ! grep -q 'stream.*True' "$OAI_COMP" 2>/dev/null; then + echo "=== Patching BFCL openai_completion.py for streaming ===" + python3 - "$OAI_COMP" <<'PY' +import pathlib, sys +p = pathlib.Path(sys.argv[1]) +src = p.read_text() +old = ''' @retry_with_backoff(error_type=RateLimitError) + def generate_with_backoff(self, **kwargs): + start_time = time.time() + api_response = self.client.chat.completions.create(**kwargs) + end_time = time.time() + + return api_response, end_time - start_time''' +new = ''' @retry_with_backoff(error_type=RateLimitError) + def generate_with_backoff(self, **kwargs): + start_time = time.time() + kwargs["stream"] = True + api_response = self.client.chat.completions.create(**kwargs) + accumulated = self._accumulate_stream(api_response) + end_time = time.time() + + return accumulated, end_time - start_time + + @staticmethod + def _accumulate_stream(stream): + content = "" + reasoning_content = "" + tool_calls = {} + finish_reason = None + usage = None + for chunk in stream: + if hasattr(chunk, "usage") and chunk.usage is not None: + usage = chunk.usage + if not chunk.choices: + continue + delta = chunk.choices[0].delta + if delta.content: + content += delta.content + if hasattr(delta, "reasoning_content") and delta.reasoning_content: + reasoning_content += delta.reasoning_content + if delta.tool_calls: + for tc in delta.tool_calls: + idx = tc.index + if idx not in tool_calls: + tool_calls[idx] = {"id": "", "type": "function", "function": {"name": "", "arguments": ""}} + if tc.id: + tool_calls[idx]["id"] = tc.id + if tc.function: + if tc.function.name: + tool_calls[idx]["function"]["name"] += tc.function.name + if tc.function.arguments: + tool_calls[idx]["function"]["arguments"] += tc.function.arguments + if chunk.choices[0].finish_reason: + finish_reason = chunk.choices[0].finish_reason + + from types import SimpleNamespace + msg = SimpleNamespace(content=content if content else None, reasoning_content=reasoning_content if reasoning_content else None, tool_calls=None) + if tool_calls: + msg.tool_calls = [ + SimpleNamespace(id=tc["id"], type=tc["type"], function=SimpleNamespace(name=tc["function"]["name"], arguments=tc["function"]["arguments"])) + for tc in tool_calls.values() + ] + choice = SimpleNamespace(index=0, message=msg, finish_reason=finish_reason) + resp = SimpleNamespace(choices=[choice], usage=usage) + return resp''' +if old in src and "stream" not in src: + src = src.replace(old, new, 1) + p.write_text(src) +PY + fi + if [[ ! -x "$VENV/bin/bfcl" ]] || ! "$VENV/bin/python" -c "import soundfile" 2>/dev/null; then + echo "=== Setting up BFCL venv (first time) ===" + uv venv --clear --seed "$VENV" + uv pip install --python "$VENV/bin/python" -e "$BFCL_DIR" "soundfile>=0.12.0" + fi + export QUALITY_BFCL_VENV="$VENV" + export QUALITY_BFCL_DIR="$BFCL_DIR" +} + +setup_scicode() { + local SCICODE_DIR="$QUALITY_CACHE_DIR/SciCode" + local VENV="$QUALITY_CACHE_DIR/.venv-scicode" + if [[ ! -d "$SCICODE_DIR" ]]; then + echo "=== Cloning SciCode (first time) ===" + git clone --depth 1 https://github.com/scicode-bench/SciCode.git "$SCICODE_DIR" + fi + # Cache-bust: check scicode + inspect_ai import works + if [[ ! -x "$VENV/bin/inspect" ]] || ! "$VENV/bin/python" -c "import scicode; import inspect_ai" 2>/dev/null; then + echo "=== Setting up SciCode venv (first time) ===" + uv venv --clear --seed "$VENV" + # SciCode pyproject pins unpinned "datasets" → resolver picks 2.14.4, + # but inspect-ai requires datasets>=2.16. datasets 2.16.1 has a bug + # with SciCode1/SciCode dataset (TypeError in generate_from_dict). + # Pin datasets==5.0.1 + pyarrow==25.0.1 (known good, same as LCB/swebench). + uv pip install --python "$VENV/bin/python" \ + "datasets==5.0.1" "pyarrow==25.0.1" "openai>=3.1" "anthropic" "config" \ + "litellm" "inspect-ai" "rich" "pytest" "pytest-cov" \ + "matplotlib" "scipy" "sympy" "h5py" "jsonlines" \ + "google-generativeai" "gdown>=5.2,<6" + uv pip install --python "$VENV/bin/python" --no-deps -e "$SCICODE_DIR" + fi + + # SciCode keeps its numeric reference outputs outside the git repository. + # Cache them beside the persistent checkout so later CI runs can reuse the + # 1 GiB file instead of downloading it again. Validate before reuse and + # before atomically installing a new download; otherwise a missing or + # interrupted download is silently converted by SciCode into zero scores. + if ! "$VENV/bin/python" -c "import gdown" 2>/dev/null; then + echo "=== Installing SciCode data downloader ===" + uv pip install --python "$VENV/bin/python" "gdown>=5.2,<6" + fi + local SCICODE_DATA_DIR="$SCICODE_DIR/eval/data" + local SCICODE_DATA_FILE="$SCICODE_DATA_DIR/test_data.h5" + local SCICODE_DATA_URL="${SCICODE_DATA_URL:-https://drive.google.com/uc?id=17G_k65N_6yFFZ2O-jQH00Lh6iaw3z-AW}" + local SCICODE_DATA_SHA256="${SCICODE_DATA_SHA256:-48b0272a88b17dbd29777c217e1b4fb2b019b92e11cc2add847409db9541b890}" + mkdir -p "$SCICODE_DATA_DIR" + + validate_scicode_data() { + "$VENV/bin/python" - "$1" "$SCICODE_DATA_SHA256" <<'PY' +import hashlib +import pathlib +import sys + +import h5py + +path = pathlib.Path(sys.argv[1]) +expected_sha256 = sys.argv[2] +if not path.is_file() or path.stat().st_size == 0: + raise SystemExit(1) +try: + with h5py.File(path, "r") as data: + if len(data) == 0: + raise ValueError("HDF5 file contains no reference-data groups") +except (OSError, ValueError): + raise SystemExit(1) +digest = hashlib.sha256() +with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) +if digest.hexdigest() != expected_sha256: + raise SystemExit(1) +PY + } + + if validate_scicode_data "$SCICODE_DATA_FILE"; then + echo "=== Reusing cached SciCode numeric test data ===" + echo " Data file: $SCICODE_DATA_FILE" + else + echo "=== Downloading SciCode numeric test data (first time) ===" + local SCICODE_DATA_TMP + SCICODE_DATA_TMP="$(mktemp "$SCICODE_DATA_DIR/.test_data.h5.part.XXXXXX")" + if ! "$VENV/bin/python" -m gdown --no-cookies \ + "$SCICODE_DATA_URL" -O "$SCICODE_DATA_TMP"; then + rm -f "$SCICODE_DATA_TMP" + echo "ERROR: Failed to download SciCode numeric test data" >&2 + exit 1 + fi + if ! validate_scicode_data "$SCICODE_DATA_TMP"; then + rm -f "$SCICODE_DATA_TMP" + echo "ERROR: Downloaded SciCode numeric test data is not a valid, non-empty HDF5 file" >&2 + exit 1 + fi + mv -f "$SCICODE_DATA_TMP" "$SCICODE_DATA_FILE" + echo " Cached data file: $SCICODE_DATA_FILE" + fi + export QUALITY_SCICODE_DATA_FILE="$SCICODE_DATA_FILE" + + # Patch inspect_ai OpenAI provider for streaming (avoid proxy timeouts) + local OAI_PROVIDER="$VENV/lib/python3.12/site-packages/inspect_ai/model/_providers/openai.py" + if [[ -f "$OAI_PROVIDER" ]] && ! grep -q 'stream.*True' "$OAI_PROVIDER" 2>/dev/null; then + echo "=== Patching inspect_ai openai.py for streaming ===" + python3 - "$OAI_PROVIDER" <<'PY' +import pathlib, sys +p = pathlib.Path(sys.argv[1]) +src = p.read_text() +old = ''' # generate completion + completion: ChatCompletion = await self.client.chat.completions.create( + **request + ) + + # save response for model_call + response = completion.model_dump() + + # parse out choices + choices = self._chat_choices_from_response(completion, tools) + + # return output and call + return ModelOutput( + model=completion.model, + choices=choices, + usage=( + ModelUsage( + input_tokens=completion.usage.prompt_tokens, + output_tokens=completion.usage.completion_tokens, + total_tokens=completion.usage.total_tokens, + ) + if completion.usage + else None + ), + ), model_call()''' +new = ''' # generate completion (streaming to avoid proxy timeouts on long generations) + request["stream"] = True + stream = await self.client.chat.completions.create(**request) + + content = "" + tool_calls_map = {} + finish_reason = None + model_name = self.model_name + usage = None + async for chunk in stream: + if hasattr(chunk, "usage") and chunk.usage is not None: + usage = chunk.usage + if hasattr(chunk, "model") and chunk.model: + model_name = chunk.model + if not chunk.choices: + continue + delta = chunk.choices[0].delta + if delta and delta.content: + content += delta.content + if delta and hasattr(delta, "tool_calls") and delta.tool_calls: + for tc in delta.tool_calls: + idx = tc.index + if idx not in tool_calls_map: + tool_calls_map[idx] = {"id": "", "type": "function", "function": {"name": "", "arguments": ""}} + if tc.id: + tool_calls_map[idx]["id"] = tc.id + if tc.function: + if tc.function.name: + tool_calls_map[idx]["function"]["name"] += tc.function.name + if tc.function.arguments: + tool_calls_map[idx]["function"]["arguments"] += tc.function.arguments + if chunk.choices[0].finish_reason: + finish_reason = chunk.choices[0].finish_reason + + from openai.types.chat import ChatCompletion, ChatCompletionMessage, ChatCompletionMessageToolCall + from openai.types.chat.chat_completion import Choice + + msg = ChatCompletionMessage( + role="assistant", + content=content if content else None, + ) + if tool_calls_map: + msg.tool_calls = [ + ChatCompletionMessageToolCall(id=tc["id"], type=tc["type"], function=tc["function"]) + for tc in tool_calls_map.values() + ] + + choice = Choice(index=0, message=msg, finish_reason=finish_reason or "stop") + completion = ChatCompletion( + id="stream-accumulated", + model=model_name, + choices=[choice], + created=0, + object="chat.completion", + usage=usage, + ) + + # save response for model_call + response = completion.model_dump() + + # parse out choices + choices = self._chat_choices_from_response(completion, tools) + + # return output and call + return ModelOutput( + model=completion.model, + choices=choices, + usage=( + ModelUsage( + input_tokens=completion.usage.prompt_tokens, + output_tokens=completion.usage.completion_tokens, + total_tokens=completion.usage.total_tokens, + ) + if completion.usage + else None + ), + ), model_call()''' +if old in src and "stream" not in src: + src = src.replace(old, new, 1) + p.write_text(src) +PY + fi + # Repair environments cached with the original streaming patch. OpenAI SDK + # 2.x requires ChatCompletionMessage.role; without it response assembly + # raises after the full stream has already completed. + if [[ -f "$OAI_PROVIDER" ]] && grep -q 'ChatCompletionMessage(content=content if content else None)' "$OAI_PROVIDER" 2>/dev/null; then + echo "=== Repairing inspect_ai streaming response role ===" + python3 - "$OAI_PROVIDER" <<'PY' +import pathlib, sys +p = pathlib.Path(sys.argv[1]) +src = p.read_text() +old = 'msg = ChatCompletionMessage(content=content if content else None)' +new = 'msg = ChatCompletionMessage(role="assistant", content=content if content else None)' +if old not in src: + raise SystemExit(f"expected streaming response constructor not found in {p}") +p.write_text(src.replace(old, new, 1)) +PY + fi + + # SciCode otherwise hides all generation exceptions and silently writes a + # dummy response, which can make a zero-score smoke run appear healthy. + local SCICODE_TASK="$SCICODE_DIR/eval/inspect_ai/scicode.py" + if [[ -f "$SCICODE_TASK" ]] && grep -q '^ except:$' "$SCICODE_TASK" 2>/dev/null; then + echo "=== Patching SciCode to report generation exceptions ===" + python3 - "$SCICODE_TASK" <<'PY' +import pathlib, sys +p = pathlib.Path(sys.argv[1]) +src = p.read_text() +old = ''' except: + print(f"Failed to generate response for problem {prob_id} step {idx+1}.")''' +new = ''' except Exception as exc: + print( + f"Failed to generate response for problem {prob_id} step {idx+1}: " + f"{type(exc).__name__}: {exc}", + flush=True, + )''' +if old not in src: + raise SystemExit(f"expected SciCode exception handler not found in {p}") +p.write_text(src.replace(old, new, 1)) +PY + fi + export QUALITY_SCICODE_VENV="$VENV" + export QUALITY_SCICODE_DIR="$SCICODE_DIR" +} + +setup_swebench_pro() { + local SWEBENCH_DIR="$QUALITY_CACHE_DIR/SWE-bench_Pro-os" + local VENV="$QUALITY_CACHE_DIR/.venv-swebenchpro" + if [[ ! -d "$SWEBENCH_DIR" ]]; then + echo "=== Cloning SWE-bench Pro (first time, with submodules) ===" + git clone --recurse-submodules --depth 1 https://github.com/scaleapi/SWE-bench_Pro-os.git "$SWEBENCH_DIR" + fi + # Ensure SWE-agent submodule is present (cache may have shallow clone without it) + if [[ ! -d "$SWEBENCH_DIR/SWE-agent/.git" ]]; then + echo "=== Initializing SWE-agent submodule ===" + git -C "$SWEBENCH_DIR" submodule update --init --recursive + fi + # Generate instances.yaml if missing (required by run_swebench_pro.py) + local INSTANCES_YAML="$SWEBENCH_DIR/SWE-agent/data/instances.yaml" + if [[ ! -f "$INSTANCES_YAML" ]]; then + echo "=== Generating instances.yaml from HuggingFace dataset ===" + if [[ ! -x "$VENV/bin/python" ]]; then + uv venv --clear --seed "$VENV" + uv pip install --python "$VENV/bin/python" \ + -r "$SWEBENCH_DIR/requirements.txt" \ + "mini-swe-agent" "litellm" "rich" "pyyaml" "datasets" "tqdm" + fi + "$VENV/bin/python" "$SWEBENCH_DIR/helper_code/generate_sweagent_instances.py" \ + --dockerhub_username "${DOCKERHUB_USERNAME:-jefzda}" \ + --output_path "$INSTANCES_YAML" + fi + if [[ ! -x "$VENV/bin/python" ]] || ! "$VENV/bin/python" -c "import yaml" 2>/dev/null; then + echo "=== Setting up SWE-bench Pro venv (first time) ===" + uv venv --clear --seed "$VENV" + uv pip install --python "$VENV/bin/python" \ + -r "$SWEBENCH_DIR/requirements.txt" \ + "mini-swe-agent" "litellm" "rich" "pyyaml" + fi + # Patch litellm_model.py for streaming (shared by SWE-bench Pro + DeepSWE) + local LITELLM_MODEL="$VENV/lib/python3.12/site-packages/minisweagent/models/litellm_model.py" + if [[ -f "$LITELLM_MODEL" ]] && ! grep -q 'stream.*True' "$LITELLM_MODEL" 2>/dev/null; then + echo "=== Patching litellm_model.py for streaming ===" + python3 - "$LITELLM_MODEL" <<'PY' +import pathlib, sys +p = pathlib.Path(sys.argv[1]) +src = p.read_text() +# Add stream=True to _query +old1 = ' tools=[BASH_TOOL],\n **(self.config.model_kwargs | kwargs),' +new1 = ' tools=[BASH_TOOL],\n stream=True,\n **(self.config.model_kwargs | kwargs),' +if old1 in src: + src = src.replace(old1, new1, 1) +# Add accumulator function after imports +old2 = 'from minisweagent.exceptions import FormatError' +new2 = ''' + +def _accumulate_litellm_stream(stream): + """Accumulate a litellm streaming response into a single ModelResponse.""" + content = "" + tool_calls_map = {} + finish_reason = None + usage = None + model = None + for chunk in stream: + if hasattr(chunk, "usage") and chunk.usage is not None: + usage = chunk.usage + if hasattr(chunk, "model") and chunk.model: + model = chunk.model + if not chunk.choices: + continue + delta = chunk.choices[0].delta + if delta and getattr(delta, "content", None): + content += delta.content + if delta and getattr(delta, "tool_calls", None): + for tc in delta.tool_calls: + idx = tc.index + if idx not in tool_calls_map: + tool_calls_map[idx] = {"id": "", "type": "function", "function": {"name": "", "arguments": ""}} + if tc.id: + tool_calls_map[idx]["id"] = tc.id + if tc.function: + if tc.function.name: + tool_calls_map[idx]["function"]["name"] += tc.function.name + if tc.function.arguments: + tool_calls_map[idx]["function"]["arguments"] += tc.function.arguments + if chunk.choices[0].finish_reason: + finish_reason = chunk.choices[0].finish_reason + from types import SimpleNamespace + msg = SimpleNamespace(content=content if content else None, tool_calls=None) + if tool_calls_map: + msg.tool_calls = [ + SimpleNamespace(id=tc["id"], type=tc["type"], function=SimpleNamespace(name=tc["function"]["name"], arguments=tc["function"]["arguments"])) + for tc in tool_calls_map.values() + ] + def _message_dump(mode=None): + serialized_tool_calls = None + if msg.tool_calls: + serialized_tool_calls = [ + { + "id": tc.id, + "type": tc.type, + "function": { + "name": tc.function.name, + "arguments": tc.function.arguments, + }, + } + for tc in msg.tool_calls + ] + return {"role": "assistant", "content": msg.content, "tool_calls": serialized_tool_calls} + msg.model_dump = _message_dump + choice = SimpleNamespace(index=0, message=msg, finish_reason=finish_reason or "stop") + resp = SimpleNamespace(choices=[choice], usage=usage, model=model or "") + resp.model_dump = lambda mode=None: {"choices": [{"message": msg.model_dump(), "finish_reason": choice.finish_reason}], "usage": None, "model": resp.model} + return resp + + +from minisweagent.exceptions import FormatError''' +if old2 in src and "_accumulate_litellm_stream" not in src: + src = src.replace(old2, new2, 1) +# Add accumulation call in query() +old3 = ' response = self._query(self._prepare_messages_for_api(messages), **kwargs)\n cost_output = self._calculate_cost(response)' +new3 = ' response = self._query(self._prepare_messages_for_api(messages), **kwargs)\n response = _accumulate_litellm_stream(response)\n cost_output = self._calculate_cost(response)' +if old3 in src and "_accumulate_litellm_stream(response)" not in src: + src = src.replace(old3, new3, 1) +p.write_text(src) +PY + fi + # Repair cached environments created by the original accumulator, which + # discarded tool calls when serializing the reconstructed response. + if [[ -f "$LITELLM_MODEL" ]] && grep -q 'lambda mode=None: {"content": msg.content, "tool_calls": None}' "$LITELLM_MODEL" 2>/dev/null; then + echo "=== Repairing mini-swe-agent streaming tool-call serialization ===" + python3 - "$LITELLM_MODEL" <<'PY' +import pathlib, sys +p = pathlib.Path(sys.argv[1]) +src = p.read_text() +old = ' msg.model_dump = lambda mode=None: {"content": msg.content, "tool_calls": None}' +new = ''' def _message_dump(mode=None): + serialized_tool_calls = None + if msg.tool_calls: + serialized_tool_calls = [ + {"id": tc.id, "type": tc.type, "function": {"name": tc.function.name, "arguments": tc.function.arguments}} + for tc in msg.tool_calls + ] + return {"role": "assistant", "content": msg.content, "tool_calls": serialized_tool_calls} + msg.model_dump = _message_dump''' +if old not in src: + raise SystemExit(f"expected broken tool-call serializer not found in {p}") +p.write_text(src.replace(old, new, 1)) +PY + fi + # Also patch litellm_textbased_model.py + local LITELLM_TEXT="$VENV/lib/python3.12/site-packages/minisweagent/models/litellm_textbased_model.py" + if [[ -f "$LITELLM_TEXT" ]] && ! grep -q 'stream.*True' "$LITELLM_TEXT" 2>/dev/null; then + echo "=== Patching litellm_textbased_model.py for streaming ===" + python3 - "$LITELLM_TEXT" <<'PY' +import pathlib, sys +p = pathlib.Path(sys.argv[1]) +src = p.read_text() +old = 'model=self.config.model_name, messages=messages, **(self.config.model_kwargs | kwargs)' +new = 'model=self.config.model_name, messages=messages, stream=True, **(self.config.model_kwargs | kwargs)' +if old in src and "stream" not in src: + src = src.replace(old, new, 1) +# Import accumulator +old2 = 'from minisweagent.models.litellm_model import LitellmModel, LitellmModelConfig' +new2 = 'from minisweagent.models.litellm_model import LitellmModel, LitellmModelConfig, _accumulate_litellm_stream' +if old2 in src and "_accumulate_litellm_stream" not in src: + src = src.replace(old2, new2, 1) +p.write_text(src) +PY + fi + export QUALITY_SWEBENCHPRO_VENV="$VENV" + export QUALITY_SWEBENCH_DIR="$SWEBENCH_DIR" +} + +setup_deepswe() { + local DEEPSWE_DIR="$QUALITY_CACHE_DIR/deep-swe" + if [[ ! -d "$DEEPSWE_DIR" ]]; then + echo "=== Cloning DeepSWE (first time) ===" + git clone --depth 1 https://github.com/datacurve-ai/deep-swe.git "$DEEPSWE_DIR" + fi + export QUALITY_DEEPSWE_DIR="$DEEPSWE_DIR" +} + +# --------------------------------------------------------------------------- +# Result collection +# --------------------------------------------------------------------------- +# After the benchmark script runs, copy result files from the per-benchmark +# output directory to the workspace root so that benchmark-tmpl.yml's +# upload-artifact step (which globs for results*.json, *.traj*, etc. at +# workspace root) and validate_scores.py can find them. +# +# Also creates meta_env.json with the model prefix for threshold validation. + +collect_results() { + local BENCH="$1" + local OUT_BASE="$QUALITY_WORKSPACE/jobs/$RUN_ID/$BENCH" + local DEST="$QUALITY_WORKSPACE" + + echo "=== Collecting results from $OUT_BASE ===" + + # Create meta_env.json for validate_scores.py + local MODEL_PREFIX="${MODEL_PREFIX:-${EXP_NAME%%_*}}" + cat > "$DEST/meta_env.json" </dev/null || true) + + # result.json (singular) — DeepSWE / pier output; copy as results.json + # so benchmark-tmpl.yml's `ls results*.json` check passes. + while IFS= read -r -d '' f; do + cp -f "$f" "$DEST/results.json" + COPIED=$((COPIED + 1)) + done < <(find "$OUT_BASE" -maxdepth 2 -type f -name 'result.json' ! -path '*/ipython-session-bundle-*' -print0 2>/dev/null || true) + + # sample*.jsonl — lm-eval logged samples + while IFS= read -r -d '' f; do + cp -f "$f" "$DEST/" + COPIED=$((COPIED + 1)) + done < <(find "$OUT_BASE" -type f -name 'sample*.jsonl' -print0 2>/dev/null || true) + + # eval_results*.json — SWE-bench Pro, SciCode + # Also copy as results.json so benchmark-tmpl.yml's `ls results*.json` check passes. + while IFS= read -r -d '' f; do + cp -f "$f" "$DEST/" + if [[ ! -f "$DEST/results.json" ]]; then + cp -f "$f" "$DEST/results.json" + fi + COPIED=$((COPIED + 1)) + done < <(find "$OUT_BASE" -type f -name 'eval_results*.json' -print0 2>/dev/null || true) + + # inspect-ai log files (SciCode) — .json in logs/ subdir (with --log-format json) + # Convert inspect-ai eval log to validate_scores.py format: + # {"results": {"scicode": {"sub_problem_correctness": 0.0, "Problem Correctness/mean": 0.0}}} + # so validate_scores.py can check thresholds. + while IFS= read -r -d '' f; do + cp -f "$f" "$DEST/" + COPIED=$((COPIED + 1)) + # Convert inspect-ai log to results.json with validate_scores.py-compatible format + if [[ "$BENCH" == "scicode" ]]; then + python3 - "$f" "$DEST/results.json" <<'PY' || true +import json, sys +with open(sys.argv[1]) as f: + data = json.load(f) +results = {} +# Extract scores from reductions (inspect-ai summary) +for red in data.get("reductions", []): + scorer_name = red.get("scorer", "unknown") + for sample in red.get("samples", []): + pass # individual sample scores +# Extract from results.scores (aggregate metrics) +for score_entry in data.get("results", {}).get("scores", []): + scorer = score_entry.get("scorer", score_entry.get("name", "unknown")) + metrics = score_entry.get("metrics", {}) + task_key = f"scicode/{scorer}" + results[task_key] = {} + for metric_name, metric_val in metrics.items(): + val = metric_val.get("value") if isinstance(metric_val, dict) else metric_val + if isinstance(val, (int, float)): + results[task_key][metric_name] = val +# Also extract per-sample Problem Correctness from samples +for sample in data.get("samples", []): + sid = sample.get("id", "unknown") + for scorer_name, score_obj in sample.get("scores", {}).items(): + val = score_obj.get("value", {}) + if isinstance(val, dict) and "Problem Correctness" in val: + results[f"scicode/problem_{sid}"] = {"Problem Correctness": val["Problem Correctness"]} +if results: + out = {"results": results} + with open(sys.argv[2], "w") as f: + json.dump(out, f, indent=2) + print(f" Converted inspect-ai log to results.json with {len(results)} tasks") +PY + fi + done < <(find "$OUT_BASE" -type f -path '*/logs/*' -name '*.json' -print0 2>/dev/null || true) + + # predictions.jsonl, agent_preds.json — SWE-bench, agentic + while IFS= read -r -d '' f; do + cp -f "$f" "$DEST/" + COPIED=$((COPIED + 1)) + done < <(find "$OUT_BASE" -type f \( -name 'predictions.jsonl' -o -name 'agent_preds.json' \) -print0 2>/dev/null || true) + + # swebench_report_*.json + while IFS= read -r -d '' f; do + cp -f "$f" "$DEST/" + COPIED=$((COPIED + 1)) + done < <(find "$OUT_BASE" -type f -name 'swebench_report_*.json' -print0 2>/dev/null || true) + + # *.traj* — DeepSWE, SWE-bench trajectories + while IFS= read -r -d '' f; do + cp -f "$f" "$DEST/" + COPIED=$((COPIED + 1)) + done < <(find "$OUT_BASE" -type f -name '*.traj*' -print0 2>/dev/null || true) + + # BFCL score CSVs + while IFS= read -r -d '' f; do + cp -f "$f" "$DEST/" + COPIED=$((COPIED + 1)) + done < <(find "$OUT_BASE" -type f -name '*.csv' -print0 2>/dev/null || true) + + # LiveCodeBench result JSONs/JSONLs + # LCB writes to output//__.json and _eval.json + # Copy first .json as results.json so benchmark-tmpl.yml's glob matches. + local LCB_FIRST_JSON="" + while IFS= read -r -d '' f; do + cp -f "$f" "$DEST/" + COPIED=$((COPIED + 1)) + done < <(find "$OUT_BASE" -type f \( -name '*.jsonl' -o -name 'lcb_results*.json' \) -print0 2>/dev/null || true) + # Also pick up LCB's output/*.json files + while IFS= read -r -d '' f; do + cp -f "$f" "$DEST/" + if [[ -z "$LCB_FIRST_JSON" ]]; then + LCB_FIRST_JSON="$f" + fi + COPIED=$((COPIED + 1)) + done < <(find "$OUT_BASE" -type f -name '*.json' ! -name 'results*.json' ! -name 'eval_results*.json' -print0 2>/dev/null || true) + if [[ -n "$LCB_FIRST_JSON" && ! -f "$DEST/results.json" ]]; then + cp -f "$LCB_FIRST_JSON" "$DEST/results.json" + fi + + # Convert LCB _eval.json (list format) to validate_scores.py-compatible dict format: + # LCB output: [{"pass@1": 1.0, "detail": {...}}, ...] + # validate_scores.py expects: {"results": {"livecodebench": {"pass@1": 1.0}}} + if [[ "$BENCH" == "livecodebench" ]]; then + local LCB_EVAL_JSON="" + while IFS= read -r -d '' f; do + LCB_EVAL_JSON="$f" + break + done < <(find "$OUT_BASE" -type f -name '*_eval.json' -print0 2>/dev/null || true) + if [[ -n "$LCB_EVAL_JSON" ]]; then + python3 - "$LCB_EVAL_JSON" "$DEST/results.json" <<'PY' || true +import json, sys +with open(sys.argv[1]) as f: + data = json.load(f) +results = {} +# LCB _eval.json is a list; first element has pass@k scores +if isinstance(data, list) and data and isinstance(data[0], dict): + scores = data[0] + task_key = "livecodebench" + results[task_key] = {} + for k, v in scores.items(): + if isinstance(v, (int, float)): + results[task_key][k] = v +elif isinstance(data, dict): + # Already dict format — check for results key + if "results" in data: + results = data["results"] + else: + results["livecodebench"] = {k: v for k, v in data.items() if isinstance(v, (int, float))} +if results: + out = {"results": results} + with open(sys.argv[2], "w") as f: + json.dump(out, f, indent=2) + print(f" Converted LCB eval to results.json with {len(results)} tasks") +PY + fi + fi + + echo " Copied $COPIED result file(s) to $DEST" + if [[ "$COPIED" -eq 0 ]]; then + echo " WARNING: no result files found in $OUT_BASE" >&2 + # List what IS there for debugging + find "$OUT_BASE" -type f 2>/dev/null | head -20 || echo " (directory empty or missing)" + fi +} + +# --------------------------------------------------------------------------- +# Dispatch +# --------------------------------------------------------------------------- +echo "=== Quality-eval setup: ${QUALITY_BENCHMARK_NAME} ===" +echo " Workspace (output) : $QUALITY_WORKSPACE" +echo " Cache (venv/repos) : $QUALITY_CACHE_DIR" +echo " Run ID : $RUN_ID" +echo + +case "${QUALITY_BENCHMARK_NAME}" in + gpqa|mmlu_pro|hle) + setup_lmeval + ;; + livecodebench) + setup_livecodebench + ;; + bfcl) + setup_bfcl + ;; + scicode) + setup_scicode + ;; + swebench_pro) + setup_swebench_pro + ;; + deepswe) + setup_deepswe + ;; + *) + echo "ERROR: Unknown quality benchmark '${QUALITY_BENCHMARK_NAME}'" >&2 + exit 1 + ;; +esac + +echo "=== Dispatching to $BENCH_SCRIPT ===" +bash "$BENCH_SCRIPT" + +echo "=== Collecting results for artifact upload ===" +collect_results "${QUALITY_BENCHMARK_NAME}" diff --git a/utils/evals/patches/lm_eval_sitecustomize.py b/utils/evals/patches/lm_eval_sitecustomize.py index 9a71f36823..206357dd87 100644 --- a/utils/evals/patches/lm_eval_sitecustomize.py +++ b/utils/evals/patches/lm_eval_sitecustomize.py @@ -2,7 +2,102 @@ import json -from lm_eval.models.openai_completions import LocalChatCompletion +from lm_eval.models import api_models +from lm_eval.models.openai_completions import ( + LocalChatCompletion, + OpenAIChatCompletion, +) + + +def _stream_result(content, reasoning_content, finish_reason, usage, model): + return { + "id": "stream-accumulated", + "object": "chat.completion", + "model": model or "", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "".join(content), + "reasoning_content": "".join(reasoning_content), + }, + "finish_reason": finish_reason or "stop", + } + ], + "usage": usage + or {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}, + } + + +def _consume_sse_data(data, state): + if data.strip() == "[DONE]": + return True + try: + chunk = json.loads(data) + except json.JSONDecodeError: + return False + if chunk.get("usage"): + state["usage"] = chunk["usage"] + if chunk.get("model"): + state["model"] = chunk["model"] + for choice in chunk.get("choices") or []: + delta = choice.get("delta") or {} + if delta.get("reasoning_content"): + state["reasoning_content"].append(delta["reasoning_content"]) + if delta.get("content"): + state["content"].append(delta["content"]) + if choice.get("finish_reason"): + state["finish_reason"] = choice["finish_reason"] + return False + + +def _new_stream_state(): + return { + "content": [], + "reasoning_content": [], + "finish_reason": None, + "usage": None, + "model": None, + } + + +def _parse_sse_stream(response): + state = _new_stream_state() + for line in response.iter_lines(decode_unicode=True): + if line and line.startswith("data: "): + if _consume_sse_data(line[6:], state): + break + return _stream_result(**state) + + +async def _parse_sse_stream_async(response): + state = _new_stream_state() + buffer = "" + async for raw_chunk in response.content: + buffer += raw_chunk.decode("utf-8") + while "\n" in buffer: + line, buffer = buffer.split("\n", 1) + line = line.rstrip("\r") + if line.startswith("data: ") and _consume_sse_data(line[6:], state): + return _stream_result(**state) + if buffer.startswith("data: "): + _consume_sse_data(buffer[6:].rstrip("\r"), state) + return _stream_result(**state) + + +_openai_create_payload = OpenAIChatCompletion._create_payload + + +def _create_streaming_payload(self, *args, **kwargs): + payload = _openai_create_payload(self, *args, **kwargs) + payload["stream"] = True + return payload + + +OpenAIChatCompletion._create_payload = _create_streaming_payload +api_models._parse_sse_stream = _parse_sse_stream +api_models._parse_sse_stream_async = _parse_sse_stream_async def _parse_generations(outputs, **kwargs): diff --git a/utils/evals/test_batched_eval.py b/utils/evals/test_batched_eval.py index a5d6df0085..8072dd4584 100644 --- a/utils/evals/test_batched_eval.py +++ b/utils/evals/test_batched_eval.py @@ -6,8 +6,10 @@ import sys from pathlib import Path +import pytest + from validate_scores import main as validate_scores_main -from validate_scores import validate_batch_manifest +from validate_scores import validate_batch_manifest, validate_smoke_artifacts def _run_batched_eval( @@ -167,6 +169,30 @@ def test_validate_scores_fails_when_expected_batch_metadata_is_unreadable( assert "unavailable or invalid" in captured.err +def test_hle_smoke_rejects_empty_completions(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "meta_env.json").write_text( + json.dumps({"benchmark": "hle", "infmax_model_prefix": "glm5.2"}) + ) + (tmp_path / "samples_hle.jsonl").write_text( + json.dumps({"resps": [[""]]}) + "\n" + json.dumps({"resps": [["answer"]]}) + "\n" + ) + + assert validate_smoke_artifacts("meta_env.json") == [ + "HLE produced 1/2 empty completions" + ] + + +def test_hle_smoke_accepts_nonempty_nested_completions(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "meta_env.json").write_text(json.dumps({"benchmark": "hle"})) + (tmp_path / "samples_hle.jsonl").write_text( + json.dumps({"resps": [[" final answer "]]}) + "\n" + ) + + assert validate_smoke_artifacts("meta_env.json") == [] + + def test_workflow_concurrencies_are_independent_of_eval_metadata( tmp_path: Path, ) -> None: @@ -227,6 +253,147 @@ def test_validate_scores_checks_threshold_for_every_concurrency( assert "FAIL: [conc=4] gsm8k exact_match,strict-match" in captured.err +@pytest.mark.parametrize( + ("benchmark", "task", "metric", "score"), + [ + ("gpqa", "gpqa_diamond_cot_n_shot", "exact_match,strict-match", 0.40), + ("mmlu_pro", "mmlu_pro", "exact_match,custom-extract", 0.60), + ("hle", "hle", "exact_match,custom-extract", 0.20), + ], +) +def test_validate_scores_accepts_lm_eval_quality_benchmarks( + tmp_path: Path, + monkeypatch, + capsys, + benchmark: str, + task: str, + metric: str, + score: float, +) -> None: + (tmp_path / "meta_env.json").write_text(json.dumps({ + "benchmark": benchmark, + "infmax_model_prefix": "glm5.2", + })) + (tmp_path / "results.json").write_text(json.dumps({ + "results": { + task: { + metric: score, + # Auxiliary filters are useful in reports but are not release + # gates. A low auxiliary score must not create a false failure. + "exact_match,auxiliary-filter": 0.0, + }, + # Group benchmarks also publish per-category/per-format rows. + f"{task}_subtask": {metric: 0.0}, + }, + })) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(sys, "argv", ["validate_scores.py"]) + + assert validate_scores_main() == 0 + assert f"PASS: {task} {metric}" in capsys.readouterr().out + + +def test_validate_scores_accepts_livecodebench_pass_at_1( + tmp_path: Path, monkeypatch, capsys +) -> None: + (tmp_path / "meta_env.json").write_text(json.dumps({ + "benchmark": "livecodebench", + "infmax_model_prefix": "glm5.2", + })) + (tmp_path / "results.json").write_text(json.dumps({ + "results": {"livecodebench": {"pass@1": 0.62}}, + })) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(sys, "argv", ["validate_scores.py"]) + + assert validate_scores_main() == 0 + assert "PASS: livecodebench pass@1 = 0.6200" in capsys.readouterr().out + + +def test_validate_scores_checks_scicode_aggregate_against_benchmark_threshold( + tmp_path: Path, monkeypatch, capsys +) -> None: + (tmp_path / "meta_env.json").write_text(json.dumps({ + "benchmark": "scicode", + "infmax_model_prefix": "glm5.2", + })) + (tmp_path / "results.json").write_text(json.dumps({ + "results": { + "scicode/scicode_scorer": {"mean": 0.0}, + "scicode/problem_11": {"Problem Correctness": 0}, + }, + })) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(sys, "argv", ["validate_scores.py"]) + + assert validate_scores_main() == 1 + captured = capsys.readouterr() + assert "FAIL: scicode/scicode_scorer mean = 0.0000" in captured.err + assert "< 0.25 from models.glm5.2" in captured.err + + +def test_validate_scores_reads_swebench_pro_native_result( + tmp_path: Path, monkeypatch, capsys +) -> None: + (tmp_path / "meta_env.json").write_text(json.dumps({ + "benchmark": "swebench_pro", + "infmax_model_prefix": "glm5.2", + })) + (tmp_path / "results.json").write_text(json.dumps({ + "instance-1": True, + "instance-2": False, + "instance-3": True, + "instance-4": False, + })) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(sys, "argv", ["validate_scores.py"]) + + assert validate_scores_main() == 0 + assert ( + "PASS: swebench_pro exact_match,resolved = 0.5000" + in capsys.readouterr().out + ) + + +def test_validate_scores_reads_bfcl_native_result( + tmp_path: Path, monkeypatch, capsys +) -> None: + (tmp_path / "meta_env.json").write_text(json.dumps({ + "benchmark": "bfcl", + "infmax_model_prefix": "glm5.2", + })) + (tmp_path / "results.json").write_text(json.dumps({ + "benchmark": "bfcl", + "scores": [{"Model": "glm-5.2", "Overall Acc": "70%"}], + })) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(sys, "argv", ["validate_scores.py"]) + + assert validate_scores_main() == 0 + assert "PASS: bfcl overall_accuracy = 0.7000" in capsys.readouterr().out + + +def test_validate_scores_reads_deepswe_pier_result( + tmp_path: Path, monkeypatch, capsys +) -> None: + (tmp_path / "meta_env.json").write_text(json.dumps({ + "benchmark": "deepswe", + "infmax_model_prefix": "glm5.2", + })) + (tmp_path / "results.json").write_text(json.dumps({ + "stats": { + "evals": { + "agent__model__tasks": {"metrics": [{"reward": 0.2}]}, + }, + }, + })) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(sys, "argv", ["validate_scores.py"]) + + assert validate_scores_main() == 0 + assert "PASS: deepswe reward = 0.2000" in capsys.readouterr().out + + def test_amd_multinode_container_forwards_eval_concurrency_list() -> None: job_slurm = ( Path(__file__).resolve().parents[2] diff --git a/utils/evals/test_eval_patches.py b/utils/evals/test_eval_patches.py index da8a7e0f28..3408454944 100644 --- a/utils/evals/test_eval_patches.py +++ b/utils/evals/test_eval_patches.py @@ -1,4 +1,5 @@ import importlib.util +import asyncio import json import runpy import subprocess @@ -138,6 +139,10 @@ def test_lm_eval_sitecustomize_hooks(monkeypatch): class LocalChatCompletion: pass + class OpenAIChatCompletion: + def _create_payload(self, *args, **kwargs): + return {"model": "test"} + class JsonChatStr(str): pass @@ -146,6 +151,7 @@ class TemplateAPI: tokenized_requests = False completions.LocalChatCompletion = LocalChatCompletion + completions.OpenAIChatCompletion = OpenAIChatCompletion api_models.JsonChatStr = JsonChatStr api_models.TemplateAPI = TemplateAPI models.api_models = api_models @@ -169,6 +175,47 @@ class TemplateAPI: ] ) assert parsed == ["reason"] + assert OpenAIChatCompletion()._create_payload()["stream"] is True rendered = TemplateAPI().apply_chat_template([{"role": "user", "content": "hi"}]) assert isinstance(rendered, JsonChatStr) assert json.loads(rendered) == [{"role": "user", "content": "hi"}] + + class SyncResponse: + def iter_lines(self, decode_unicode=False): + assert decode_unicode + yield 'data: {"model":"glm","choices":[{"delta":{"reasoning_content":"think "}}]}' + yield 'data: {"choices":[{"delta":{"content":"answer"},"finish_reason":"stop"}]}' + yield "data: [DONE]" + + streamed = api_models._parse_sse_stream(SyncResponse()) + message = streamed["choices"][0]["message"] + assert message == { + "role": "assistant", + "content": "answer", + "reasoning_content": "think ", + } + + class AsyncContent: + def __aiter__(self): + chunks = iter( + [ + b'data: {"choices":[{"delta":{"reasoning_content":"rea', + b'son"}}]}\n\ndata: {"choices":[{"delta":{"content":"final"},', + b'"finish_reason":"length"}]}\n\ndata: [DONE]\n\n', + ] + ) + + async def next_chunk(): + try: + return next(chunks) + except StopIteration as exc: + raise StopAsyncIteration from exc + + return type("Iterator", (), {"__aiter__": lambda self: self, "__anext__": lambda self: next_chunk()})() + + async_streamed = asyncio.run( + api_models._parse_sse_stream_async(type("Response", (), {"content": AsyncContent()})()) + ) + assert async_streamed["choices"][0]["message"]["reasoning_content"] == "reason" + assert async_streamed["choices"][0]["message"]["content"] == "final" + assert async_streamed["choices"][0]["finish_reason"] == "length" diff --git a/utils/evals/test_hle_task.py b/utils/evals/test_hle_task.py new file mode 100644 index 0000000000..c8b857bfe0 --- /dev/null +++ b/utils/evals/test_hle_task.py @@ -0,0 +1,63 @@ +import importlib.util +import re +from pathlib import Path + +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] +TASK_DIR = REPO_ROOT / "benchmarks/single_node/quality/tasks/hle" +RUN_SCRIPT = REPO_ROOT / "benchmarks/single_node/quality/run_hle.sh" + + +def _load_hle_utils(): + spec = importlib.util.spec_from_file_location("hle_task_utils", TASK_DIR / "utils.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _filter_pattern(name: str) -> str: + config = yaml.load( + (TASK_DIR / "hle_multiple_choice.yaml").read_text(), + Loader=yaml.BaseLoader, + ) + filter_config = next(item for item in config["filter_list"] if item["name"] == name) + return filter_config["filter"][0]["regex_pattern"] + + +def test_hle_multiple_choice_extracts_last_final_answer_through_j(): + pattern = _filter_pattern("custom-extract") + response = "A and E are considered first. The answer is (C). The answer is (J)" + + assert re.findall(pattern, response)[-1] == "J" + + +def test_hle_flexible_extract_ignores_standalone_reasoning_letters(): + pattern = _filter_pattern("flexible-extract") + response = "A and E are considered first; the final choice is (I)." + + assert re.findall(pattern, response)[-1] == "I" + + +def test_hle_prompts_require_unambiguous_final_lines(): + task_utils = _load_hle_utils() + + exact_prompt = task_utils.doc_to_text( + {"question": "Q", "answer_type": "exactMatch"} + ) + choice_prompt = task_utils.doc_to_text( + {"question": "Q", "answer_type": "multipleChoice"} + ) + + assert 'exactly one final line in the form "#### "' in exact_prompt + assert 'exactly one final line in the form "The answer is (X)"' in choice_prompt + assert "A through J" in choice_prompt + + +def test_hle_uses_long_request_timeout_for_streamed_reasoning(): + script = RUN_SCRIPT.read_text() + + assert 'REQUEST_TIMEOUT="${REQUEST_TIMEOUT:-1800}"' in script + assert "timeout=${REQUEST_TIMEOUT}" in script diff --git a/utils/evals/thresholds.yaml b/utils/evals/thresholds.yaml index 6ff9731c45..4263518c83 100644 --- a/utils/evals/thresholds.yaml +++ b/utils/evals/thresholds.yaml @@ -3,6 +3,14 @@ default: gsm8k: 0.90 gpqa_diamond_cot_n_shot: 0.30 swebench_lite: 0.50 + # Quality-eval benchmarks + mmlu_pro: 0.50 + hle: 0.10 + livecodebench: 0.30 + bfcl: 0.60 + scicode: 0.20 + swebench_pro: 0.15 + deepswe: 0.10 models: dsr1: gsm8k: 0.91 @@ -12,6 +20,16 @@ models: gsm8k: 0.94 glm5.1: gsm8k: 0.93 + glm5.2: + gsm8k: 0.93 + gpqa_diamond_cot_n_shot: 0.35 + mmlu_pro: 0.55 + hle: 0.15 + livecodebench: 0.35 + bfcl: 0.65 + scicode: 0.25 + swebench_pro: 0.20 + deepswe: 0.15 gptoss: gsm8k: 0.91 kimik2.5: diff --git a/utils/evals/validate_scores.py b/utils/evals/validate_scores.py index ba7fc13962..9c2d837c5d 100644 --- a/utils/evals/validate_scores.py +++ b/utils/evals/validate_scores.py @@ -12,6 +12,33 @@ CONC_SUFFIX_RE = re.compile(r"_conc(\d+)(?:_\d+)?\.json$") +# Quality harnesses do not share lm-eval's ``exact_match,*`` metric naming. +# Keep the primary metric explicit so auxiliary/per-sample values are not +# accidentally treated as release gates. +QUALITY_PRIMARY_METRICS = { + "gpqa": {"exact_match,strict-match"}, + "mmlu_pro": {"exact_match,custom-extract"}, + "hle": {"exact_match,custom-extract"}, + "livecodebench": {"pass@1"}, + "scicode": {"mean", "sub_problem_correctness", "Problem Correctness/mean"}, + "swebench_pro": {"exact_match,resolved"}, +} + +QUALITY_PRIMARY_TASKS = { + "gpqa": "gpqa_diamond_cot_n_shot", + "mmlu_pro": "mmlu_pro", + "hle": "hle", + "livecodebench": "livecodebench", + "scicode": "scicode/scicode_scorer", + "swebench_pro": "swebench_pro", +} + +QUALITY_THRESHOLD_KEYS = { + # The workflow uses the short launcher name while thresholds retain the + # canonical lm-eval task name. + "gpqa": "gpqa_diamond_cot_n_shot", +} + def load_config(path: str) -> dict: """Load YAML or JSON thresholds, including legacy flat configs.""" @@ -58,6 +85,69 @@ def detect_model_prefix(meta_env_path: str, override: str | None) -> str | None: return None +def detect_benchmark(meta_env_path: str) -> str | None: + """Return the quality benchmark recorded by the launcher, if present.""" + try: + with open(meta_env_path) as f: + benchmark = json.load(f).get("benchmark") + return benchmark if isinstance(benchmark, str) and benchmark else None + except (json.JSONDecodeError, OSError, AttributeError): + return None + + +def _numeric_score(value) -> float | None: + """Parse numeric and percentage values emitted by external harnesses.""" + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return float(value) + if not isinstance(value, str): + return None + text = value.strip() + try: + if text.endswith("%"): + return float(text[:-1]) / 100.0 + return float(text) + except ValueError: + return None + + +def native_quality_scores(data: dict, benchmark: str | None): + """Yield primary scores from harness-native result structures.""" + if benchmark == "bfcl": + for row in data.get("scores", []): + if not isinstance(row, dict): + continue + value = _numeric_score(row.get("Overall Acc")) + if value is not None: + yield "bfcl", "overall_accuracy", value + return + elif benchmark == "deepswe": + evals = data.get("stats", {}).get("evals", {}) + if not isinstance(evals, dict): + return + for eval_data in evals.values(): + if not isinstance(eval_data, dict): + continue + for metrics in eval_data.get("metrics", []): + if not isinstance(metrics, dict): + continue + value = _numeric_score(metrics.get("reward")) + if value is not None: + yield "deepswe", "reward", value + elif benchmark == "swebench_pro": + # The upstream SWE-bench Pro evaluator writes eval_results.json as a + # bare {instance_id: resolved_bool} mapping. Accept that native format + # as well as the normalized lm-eval-shaped wrapper produced by our + # runner, so old/resumed artifacts validate correctly too. + if data and all(isinstance(value, bool) for value in data.values()): + yield ( + "swebench_pro", + "exact_match,resolved", + sum(data.values()) / len(data), + ) + + def resolve_threshold(config: dict, prefix: str | None, task: str, fallback: float): """Return (min_score, source) for a task, most-specific-first.""" models = config.get("models", {}) @@ -171,6 +261,49 @@ def validate_batch_manifest( return errors +def _nested_nonempty_strings(value) -> list[str]: + """Return non-empty strings from nested lm-eval response containers.""" + if isinstance(value, str): + return [value] if value.strip() else [] + if isinstance(value, (list, tuple)): + return [item for child in value for item in _nested_nonempty_strings(child)] + return [] + + +def validate_smoke_artifacts(meta_env_path: str) -> list[str]: + """Reject operationally invalid outputs that a score-only smoke test hides.""" + try: + with open(meta_env_path) as f: + benchmark = json.load(f).get("benchmark") + except (json.JSONDecodeError, OSError, AttributeError): + return [] + + errors = [] + if benchmark == "hle": + sample_files = sorted(glob.glob("sample*.jsonl")) + if not sample_files: + return ["HLE smoke test produced no sample logs"] + total = 0 + empty = 0 + for path in sample_files: + try: + with open(path) as fh: + for line in fh: + if not line.strip(): + continue + total += 1 + sample = json.loads(line) + if not _nested_nonempty_strings(sample.get("resps", [])): + empty += 1 + except (json.JSONDecodeError, OSError) as exc: + errors.append(f"could not inspect HLE sample log {path}: {exc}") + if total == 0: + errors.append("HLE sample logs contain no records") + elif empty: + errors.append(f"HLE produced {empty}/{total} empty completions") + return errors + + def main() -> int: # Keep merged CI logs ordered. for _stream in (sys.stdout, sys.stderr): @@ -209,6 +342,11 @@ def main() -> int: default=None, help="Space-separated concurrencies requested by the workflow", ) + parser.add_argument( + "--smoke", + action="store_true", + help="Smoke-test mode: verify result artifacts exist without checking thresholds", + ) args = parser.parse_args() expected_concs = None @@ -241,6 +379,7 @@ def main() -> int: # Identify the model so per-model thresholds can apply prefix = detect_model_prefix(args.meta_env, args.model_prefix) + benchmark = detect_benchmark(args.meta_env) if prefix and prefix in config.get("models", {}): print(f"Model prefix: {prefix} (per-model thresholds apply)") elif prefix: @@ -252,6 +391,11 @@ def main() -> int: checked = 0 result_files = sorted(glob.glob(args.results_glob)) + if args.smoke: + for error in validate_smoke_artifacts(args.meta_env): + print(f"FAIL: {error}", file=sys.stderr) + failed = True + manifest_errors = validate_batch_manifest( args.meta_env, result_files, @@ -278,14 +422,34 @@ def main() -> int: with open(f) as fh: data = json.load(fh) for task, metrics in data.get("results", {}).items(): + if not isinstance(metrics, dict): + continue min_score, source = resolve_threshold(config, prefix, task, args.min_score) for name, val in metrics.items(): - if not name.startswith(args.metric_prefix) or "stderr" in name: + primary_metrics = QUALITY_PRIMARY_METRICS.get(benchmark) + if primary_metrics is not None: + if task != QUALITY_PRIMARY_TASKS[benchmark]: + continue + if name not in primary_metrics: + continue + elif not name.startswith(args.metric_prefix) or "stderr" in name: continue if not isinstance(val, (int, float)): continue + if primary_metrics is not None: + min_score, source = resolve_threshold( + config, + prefix, + QUALITY_THRESHOLD_KEYS.get(benchmark, benchmark), + args.min_score, + ) checked += 1 - if val < min_score: + if args.smoke: + print( + f"PASS (smoke): {conc_label}{task} {name} = {val:.4f} " + f"(threshold check skipped)" + ) + elif val < min_score: print( f"FAIL: {conc_label}{task} {name} = {val:.4f} (< {min_score} from {source})", file=sys.stderr, @@ -296,7 +460,39 @@ def main() -> int: f"PASS: {conc_label}{task} {name} = {val:.4f} (>= {min_score} from {source})" ) + for task, name, val in native_quality_scores(data, benchmark): + min_score, source = resolve_threshold( + config, + prefix, + QUALITY_THRESHOLD_KEYS.get(benchmark, benchmark), + args.min_score, + ) + checked += 1 + if args.smoke: + print( + f"PASS (smoke): {conc_label}{task} {name} = {val:.4f} " + f"(threshold check skipped)" + ) + elif val < min_score: + print( + f"FAIL: {conc_label}{task} {name} = {val:.4f} " + f"(< {min_score} from {source})", + file=sys.stderr, + ) + failed = True + else: + print( + f"PASS: {conc_label}{task} {name} = {val:.4f} " + f"(>= {min_score} from {source})" + ) + if checked == 0: + if args.smoke: + if not result_files: + print("FAIL: smoke test produced no result artifacts", file=sys.stderr) + return 1 + print("PASS (smoke): no metrics matched prefix '{}' but result artifacts are parseable".format(args.metric_prefix)) + return 1 if failed else 0 print("WARN: no metrics matched prefix '{}'".format(args.metric_prefix), file=sys.stderr) return 1 if (failed or checked == 0) else 0 diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 5b2dee4108..3cacc14f46 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -11,10 +11,12 @@ from validation import ( validate_matrix_entry, validate_agentic_matrix_entry, + validate_quality_matrix_entry, load_config_files, load_runner_file, Fields, DEFAULT_AGENTIC_DURATION_SECONDS, + QUALITY_SCENARIO_TYPES, ) seq_len_stoi = { @@ -904,6 +906,55 @@ def generate_full_sweep(args, all_config_data, runner_data): validate_agentic_matrix_entry(entry) matrix_values.append(entry) + # ---- Quality-eval scenarios ---- + for scenario_type in QUALITY_SCENARIO_TYPES: + scenario_key = scenario_type + if scenario_filter is not None and scenario_type not in scenario_filter: + continue + quality_configs = scenarios.get(scenario_key, []) + if not quality_configs: + continue + if is_multinode and not args.multi_node: + continue + if not is_multinode and not args.single_node: + continue + + for quality_config in quality_configs: + quality_endpoint = quality_config[Fields.QUALITY_ENDPOINT.value] + quality_model_name = quality_config[Fields.QUALITY_MODEL_NAME.value] + bmk_space = quality_config[Fields.SEARCH_SPACE.value] + + for bmk in bmk_space: + benchmark_name = bmk[Fields.BENCHMARK_NAME.value] + smoke = bmk.get(Fields.SMOKE.value, False) + num_concurrent = bmk.get(Fields.NUM_CONCURRENT.value, None) + eval_limit = bmk.get(Fields.EVAL_LIMIT.value, None) + + runners_for_entry = runner_nodes_to_use if runner_nodes_to_use else [runner] + + for runner_value in runners_for_entry: + entry = { + Fields.IMAGE.value: image, + Fields.MODEL.value: model, + Fields.MODEL_PREFIX.value: model_code, + Fields.PRECISION.value: precision, + Fields.FRAMEWORK.value: framework, + Fields.RUNNER.value: runner_value, + Fields.BENCHMARK_NAME.value: benchmark_name, + Fields.QUALITY_ENDPOINT.value: quality_endpoint, + Fields.QUALITY_MODEL_NAME.value: quality_model_name, + Fields.EXP_NAME.value: f"{model_code}_quality_{benchmark_name}", + Fields.SCENARIO_TYPE.value: scenario_type, + Fields.SMOKE.value: smoke, + Fields.RUN_EVAL.value: False, + } + if num_concurrent is not None: + entry[Fields.NUM_CONCURRENT.value] = num_concurrent + if eval_limit is not None: + entry[Fields.EVAL_LIMIT.value] = eval_limit + validate_quality_matrix_entry(entry) + matrix_values.append(entry) + return matrix_values @@ -1197,6 +1248,44 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): entry.update(component_metadata(bmk, val)) matrix_values.append(validate_agentic_matrix_entry(entry)) + # ---- Quality-eval scenarios ---- + for scenario_type in QUALITY_SCENARIO_TYPES: + if scenario_filter is not None and scenario_type not in scenario_filter: + continue + quality_configs = val[Fields.SCENARIOS.value].get(scenario_type, []) + for quality_config in quality_configs: + quality_endpoint = quality_config[Fields.QUALITY_ENDPOINT.value] + quality_model_name = quality_config[Fields.QUALITY_MODEL_NAME.value] + bmk_space = quality_config[Fields.SEARCH_SPACE.value] + + for bmk in bmk_space: + benchmark_name = bmk[Fields.BENCHMARK_NAME.value] + smoke = bmk.get(Fields.SMOKE.value, False) + num_concurrent = bmk.get(Fields.NUM_CONCURRENT.value, None) + eval_limit = bmk.get(Fields.EVAL_LIMIT.value, None) + + for runner_value in runners_for_entry: + entry = { + Fields.IMAGE.value: image, + Fields.MODEL.value: model, + Fields.MODEL_PREFIX.value: model_code, + Fields.PRECISION.value: precision, + Fields.FRAMEWORK.value: framework, + Fields.RUNNER.value: runner_value, + Fields.BENCHMARK_NAME.value: benchmark_name, + Fields.QUALITY_ENDPOINT.value: quality_endpoint, + Fields.QUALITY_MODEL_NAME.value: quality_model_name, + Fields.EXP_NAME.value: f"{model_code}_quality_{benchmark_name}", + Fields.SCENARIO_TYPE.value: scenario_type, + Fields.SMOKE.value: smoke, + Fields.RUN_EVAL.value: False, + } + if num_concurrent is not None: + entry[Fields.NUM_CONCURRENT.value] = num_concurrent + if eval_limit is not None: + entry[Fields.EVAL_LIMIT.value] = eval_limit + matrix_values.append(validate_quality_matrix_entry(entry)) + return matrix_values @@ -1281,7 +1370,7 @@ def main(): parent_parser.add_argument( '--scenario-type', nargs='+', - choices=['fixed-seq-len', 'agentic-coding'], + choices=['fixed-seq-len', 'agentic-coding'] + QUALITY_SCENARIO_TYPES, required=False, help='Scenario type(s) to include. If not specified, all scenario types are generated.' ) diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index f95c1e63d7..ebdb90b29e 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -37,6 +37,14 @@ class Fields(Enum): # Scenario type keys FIXED_SEQ_LEN = 'fixed-seq-len' AGENTIC_CODING = 'agentic-coding' + QUALITY_GPQA = 'quality-gpqa' + QUALITY_MMLU_PRO = 'quality-mmlu-pro' + QUALITY_HLE = 'quality-hle' + QUALITY_LIVECODEBENCH = 'quality-livecodebench' + QUALITY_BFCL = 'quality-bfcl' + QUALITY_SCICODE = 'quality-scicode' + QUALITY_SWEBENCH_PRO = 'quality-swebench-pro' + QUALITY_DEEPSWE = 'quality-deepswe' # Seq-len-config fields ISL = 'isl' @@ -87,6 +95,14 @@ class Fields(Enum): EVAL_CONC = 'eval-conc' EVAL_ALL_CONCS = 'eval-all-concs' + # Quality-eval fields + BENCHMARK_NAME = 'benchmark-name' + QUALITY_ENDPOINT = 'quality-endpoint' + QUALITY_MODEL_NAME = 'quality-model-name' + SMOKE = 'smoke' + NUM_CONCURRENT = 'num-concurrent' + EVAL_LIMIT = 'eval-limit' + """ Below is the validation logic for the OUTPUT of utils/matrix_logic/generate_sweep_configs.py, i.e., @@ -363,6 +379,59 @@ def validate_disagg_transfer(self): AgenticMatrixEntry = Union[SingleNodeAgenticMatrixEntry, MultiNodeAgenticMatrixEntry] +QUALITY_SCENARIO_TYPES = [ + Fields.QUALITY_GPQA.value, + Fields.QUALITY_MMLU_PRO.value, + Fields.QUALITY_HLE.value, + Fields.QUALITY_LIVECODEBENCH.value, + Fields.QUALITY_BFCL.value, + Fields.QUALITY_SCICODE.value, + Fields.QUALITY_SWEBENCH_PRO.value, + Fields.QUALITY_DEEPSWE.value, +] + + +class QualityEvalMatrixEntry(BaseModel): + """Pydantic model for validating quality-eval matrix entries.""" + model_config = ConfigDict(extra='forbid', populate_by_name=True) + + image: str + model: str + model_prefix: str = Field(alias=Fields.MODEL_PREFIX.value) + precision: str + framework: str + runner: str + benchmark_name: str = Field(alias=Fields.BENCHMARK_NAME.value) + quality_endpoint: str = Field(alias=Fields.QUALITY_ENDPOINT.value) + quality_model_name: str = Field(alias=Fields.QUALITY_MODEL_NAME.value) + exp_name: str = Field(alias=Fields.EXP_NAME.value) + scenario_type: str = Field(alias=Fields.SCENARIO_TYPE.value) + smoke: bool = Field(default=False, alias=Fields.SMOKE.value) + num_concurrent: Optional[int] = Field(default=None, alias=Fields.NUM_CONCURRENT.value) + eval_limit: Optional[int] = Field(default=None, alias=Fields.EVAL_LIMIT.value) + run_eval: Optional[bool] = Field(default=None, alias=Fields.RUN_EVAL.value) + eval_only: Optional[bool] = Field(default=None, alias=Fields.EVAL_ONLY.value) + + @field_validator('scenario_type') + @classmethod + def validate_quality_scenario_type(cls, v: str) -> str: + if v not in QUALITY_SCENARIO_TYPES: + raise ValueError( + f"scenario-type must be one of {QUALITY_SCENARIO_TYPES}, got '{v}'" + ) + return v + + +def validate_quality_matrix_entry(entry: dict) -> dict: + """Validate that a quality-eval matrix entry matches the expected structure.""" + try: + QualityEvalMatrixEntry(**entry) + except ValidationError as e: + raise ValueError( + f"The following parsed quality-eval matrix entry failed validation:\n{pprint.pformat(entry)}\n{e}") + return entry + + def validate_agentic_matrix_entry(entry: dict) -> dict: """Validate that an agentic matrix entry matches the expected structure.""" try: @@ -659,6 +728,50 @@ def validate_dram_offload_capacity(self): return self +class QualityEvalSearchSpaceEntry(BaseModel): + """Quality-eval search space entry: which benchmarks to run.""" + model_config = ConfigDict(extra='forbid', populate_by_name=True) + + benchmark_name: str = Field( + alias=Fields.BENCHMARK_NAME.value, + description="One of: gpqa, mmlu_pro, hle, livecodebench, bfcl, " + "scicode, swebench_pro, deepswe", + ) + smoke: bool = Field(default=False, alias=Fields.SMOKE.value) + num_concurrent: Optional[int] = Field( + default=None, alias=Fields.NUM_CONCURRENT.value, + description="API request concurrency (num_concurrent / CCU / multiprocess). " + "Passed to benchmark scripts as NUM_CONCURRENT env var.", + ) + eval_limit: Optional[int] = Field( + default=None, alias=Fields.EVAL_LIMIT.value, + description="Number of questions/tasks for smoke runs. Overrides the workflow-level eval-limit.", + ) + + @field_validator('benchmark_name') + @classmethod + def validate_benchmark_name(cls, v: str) -> str: + valid = { + 'gpqa', 'mmlu_pro', 'hle', 'livecodebench', + 'bfcl', 'scicode', 'swebench_pro', 'deepswe', + } + if v not in valid: + raise ValueError( + f"benchmark-name must be one of {sorted(valid)}, got '{v}'" + ) + return v + + +class QualityEvalConfig(BaseModel): + """Quality-eval scenario configuration.""" + model_config = ConfigDict(extra='forbid', populate_by_name=True) + + search_space: List[QualityEvalSearchSpaceEntry] = Field( + alias=Fields.SEARCH_SPACE.value) + quality_endpoint: str = Field(alias=Fields.QUALITY_ENDPOINT.value) + quality_model_name: str = Field(alias=Fields.QUALITY_MODEL_NAME.value) + + class SingleNodeScenarios(BaseModel): """Scenarios wrapper for single-node configs.""" model_config = ConfigDict(extra='forbid', populate_by_name=True) @@ -667,10 +780,33 @@ class SingleNodeScenarios(BaseModel): default=None, alias=Fields.FIXED_SEQ_LEN.value) agentic_coding: Optional[List[AgenticCodingConfig]] = Field( default=None, alias=Fields.AGENTIC_CODING.value) + quality_gpqa: Optional[List[QualityEvalConfig]] = Field( + default=None, alias=Fields.QUALITY_GPQA.value) + quality_mmlu_pro: Optional[List[QualityEvalConfig]] = Field( + default=None, alias=Fields.QUALITY_MMLU_PRO.value) + quality_hle: Optional[List[QualityEvalConfig]] = Field( + default=None, alias=Fields.QUALITY_HLE.value) + quality_livecodebench: Optional[List[QualityEvalConfig]] = Field( + default=None, alias=Fields.QUALITY_LIVECODEBENCH.value) + quality_bfcl: Optional[List[QualityEvalConfig]] = Field( + default=None, alias=Fields.QUALITY_BFCL.value) + quality_scicode: Optional[List[QualityEvalConfig]] = Field( + default=None, alias=Fields.QUALITY_SCICODE.value) + quality_swebench_pro: Optional[List[QualityEvalConfig]] = Field( + default=None, alias=Fields.QUALITY_SWEBENCH_PRO.value) + quality_deepswe: Optional[List[QualityEvalConfig]] = Field( + default=None, alias=Fields.QUALITY_DEEPSWE.value) @model_validator(mode='after') def at_least_one_scenario(self): - if not self.fixed_seq_len and not self.agentic_coding: + has_any = any([ + self.fixed_seq_len, self.agentic_coding, + self.quality_gpqa, self.quality_mmlu_pro, self.quality_hle, + self.quality_livecodebench, self.quality_bfcl, + self.quality_scicode, self.quality_swebench_pro, + self.quality_deepswe, + ]) + if not has_any: raise ValueError("At least one scenario type must be specified") return self @@ -882,9 +1018,14 @@ class ChangelogEntry(BaseModel): "threshold are dropped after eval selection." ), ) - scenario_type: Optional[List[Literal["fixed-seq-len", "agentic-coding"]]] = Field( + scenario_type: Optional[List[Literal[ + "fixed-seq-len", "agentic-coding", + "quality-gpqa", "quality-mmlu-pro", "quality-hle", + "quality-livecodebench", "quality-bfcl", "quality-scicode", + "quality-swebench-pro", "quality-deepswe", + ]]] = Field( alias="scenario-type", default=None, min_length=1, - description="Restrict to specific scenario types (e.g., ['fixed-seq-len', 'agentic-coding'])" + description="Restrict to specific scenario types" ) @@ -923,6 +1064,12 @@ class ChangelogMatrixEntry(BaseModel): # the fixed-seq-len shape (isl/osl/max-model-len) multinode_evals rows do. multinode_agentic_evals: list[MultiNodeAgenticMatrixEntry] = Field( default_factory=list) + # Quality-eval rows live in their own bucket, dispatched by a dedicated + # run-sweep.yml job that passes quality-eval inputs (benchmark-name, + # quality-endpoint, quality-model-name, smoke) rather than fixed-seq-len + # or agentic inputs. + quality_evals: list[QualityEvalMatrixEntry] = Field( + default_factory=list) changelog_metadata: ChangelogMetadata diff --git a/utils/process_changelog.py b/utils/process_changelog.py index 91b276ba2f..e244280c84 100644 --- a/utils/process_changelog.py +++ b/utils/process_changelog.py @@ -10,10 +10,11 @@ from matrix_logic.validation import ( ChangelogEntry, ChangelogMatrixEntry, + QUALITY_SCENARIO_TYPES, load_config_files, ) -SCENARIO_TYPES = ("fixed-seq-len", "agentic-coding") +SCENARIO_TYPES = ("fixed-seq-len", "agentic-coding") + tuple(QUALITY_SCENARIO_TYPES) def _freeze_config_value(value): @@ -178,6 +179,7 @@ def main(): "agentic_evals": [], "multinode_evals": [], "multinode_agentic_evals": [], + "quality_evals": [], "changelog_metadata": { "base_ref": args.base_ref, "head_ref": args.head_ref, @@ -303,7 +305,9 @@ def main(): all_benchmark_results = trim_conc(all_benchmark_results) for result in all_benchmark_results: - if result.get("scenario-type") == "agentic-coding": + if result.get("scenario-type") in QUALITY_SCENARIO_TYPES: + final_results["quality_evals"].append(result) + elif result.get("scenario-type") == "agentic-coding": if result.get("prefill") is not None: final_results["multi_node"]["agentic"].append(result) else: @@ -320,11 +324,19 @@ def main(): # the fixed-seq-len inputs (isl/osl/max-model-len) they don't have. Same # split applies on the multi-node side (multinode_evals vs # multinode_agentic_evals). + # Quality-eval rows go to their own bucket for the same reason: they carry + # quality-eval inputs (benchmark-name, quality-endpoint, ...) rather than + # fixed-seq-len or agentic inputs. single_node_evals = [e for e in all_eval_results if e.get("prefill") is None] multi_node_evals = [e for e in all_eval_results if e.get("prefill") is not None] + final_results["quality_evals"].extend( + e for e in all_eval_results + if e.get("scenario-type") in QUALITY_SCENARIO_TYPES + ) final_results["evals"] = [ e for e in single_node_evals if e.get("scenario-type") != "agentic-coding" + and e.get("scenario-type") not in QUALITY_SCENARIO_TYPES ] final_results["agentic_evals"] = [ e for e in single_node_evals @@ -333,6 +345,7 @@ def main(): final_results["multinode_evals"] = [ e for e in multi_node_evals if e.get("scenario-type") != "agentic-coding" + and e.get("scenario-type") not in QUALITY_SCENARIO_TYPES ] final_results["multinode_agentic_evals"] = [ e for e in multi_node_evals diff --git a/utils/test_process_changelog.py b/utils/test_process_changelog.py index dfe677014e..fe01d2e567 100644 --- a/utils/test_process_changelog.py +++ b/utils/test_process_changelog.py @@ -8,6 +8,20 @@ import process_changelog +DEFAULT_EVAL_SCENARIOS = [ + "fixed-seq-len", + "agentic-coding", + "quality-gpqa", + "quality-mmlu-pro", + "quality-hle", + "quality-livecodebench", + "quality-bfcl", + "quality-scicode", + "quality-swebench-pro", + "quality-deepswe", +] + + def _scenario_values(command): if "--scenario-type" not in command: return [] @@ -102,7 +116,7 @@ def fake_run(command, **kwargs): assert "--all-evals" in commands[0] assert "--evals-only" in commands[0] assert "--no-evals" not in commands[0] - assert _scenario_values(commands[0]) == ["fixed-seq-len", "agentic-coding"] + assert _scenario_values(commands[0]) == DEFAULT_EVAL_SCENARIOS output = json.loads(capsys.readouterr().out) assert output["changelog_metadata"]["entries"][0]["all-evals"] is True @@ -150,7 +164,7 @@ def fake_run(command, **kwargs): assert "--no-evals" in commands[0] assert "--evals-only" in commands[1] assert "--all-evals" not in commands[1] - assert _scenario_values(commands[1]) == ["fixed-seq-len", "agentic-coding"] + assert _scenario_values(commands[1]) == DEFAULT_EVAL_SCENARIOS json.loads(capsys.readouterr().out) @@ -198,7 +212,7 @@ def fake_run(command, **kwargs): assert "--all-evals" not in commands[0] assert "--all-evals" in commands[1] assert "--evals-only" in commands[1] - assert _scenario_values(commands[1]) == ["fixed-seq-len", "agentic-coding"] + assert _scenario_values(commands[1]) == DEFAULT_EVAL_SCENARIOS json.loads(capsys.readouterr().out) @@ -292,7 +306,7 @@ def fake_run(command, **kwargs): assert "--evals-only" in commands[0] assert "--all-evals" not in commands[0] assert "--no-evals" not in commands[0] - assert _scenario_values(commands[0]) == ["fixed-seq-len", "agentic-coding"] + assert _scenario_values(commands[0]) == DEFAULT_EVAL_SCENARIOS json.loads(capsys.readouterr().out)