From 45ef2c57338c72d0a3f15755ffa3a1c2cde95030 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 24 Jul 2026 21:31:43 +0900 Subject: [PATCH 1/2] Kill hung runs early when a pinned pool model is delisted Extend the model-pool watchdog and fatal-failure classification with model-unavailable signatures (OpenRouter 'No endpoints found' / 'not a valid model ID', OpenAI-style model_not_found, ProviderModelNotFoundError) so a delisted pinned free model is killed within the poll interval and skipped instead of burning the candidate run budget. Sanitize ':' in per-candidate file names so ':free' candidates survive Windows and actions/upload-artifact. Review timing (timeouts, deadlines, context caps) is untouched. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- scripts/ci/run_opencode_review_model_pool.sh | 22 +++++++++++------ tests/test_opencode_model_pool_runner.py | 26 ++++++++++++++++++++ 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index 7b011d3e..f76ce804 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -159,7 +159,9 @@ write_prompt() { else intro="Review PR #\${PR_NUMBER} in \${OPENCODE_SOURCE_WORKDIR} with \${model_candidate}." fi - contract_file="$OPENCODE_REVIEW_WORKDIR/opencode-review-contract-${model_candidate//\//-}.md" + # Colon-safe: OpenRouter ":free" candidates would otherwise produce file + # names that Windows and actions/upload-artifact reject. + contract_file="$OPENCODE_REVIEW_WORKDIR/opencode-review-contract-${model_candidate//[\/:]/-}.md" evidence_excerpt_file="$OPENCODE_REVIEW_WORKDIR/bounded-review-evidence-excerpt.md" evidence_file_in_workdir="$OPENCODE_REVIEW_WORKDIR/bounded-review-evidence.md" cp "$GITHUB_WORKSPACE/scripts/ci/opencode_review_prompt_template.md" "$contract_file" @@ -243,7 +245,7 @@ is_fatal_provider_failure() { return 0 fi [ -s "$opencode_json_file" ] || return 1 - grep -Eiq 'budget limit|insufficient_quota' "$opencode_json_file" + grep -Eiq 'budget limit|insufficient_quota|model_not_found|model not found|ModelNotFoundError|not a valid model|no endpoints' "$opencode_json_file" } has_fatal_provider_error_event() { @@ -253,8 +255,12 @@ has_fatal_provider_error_event() { # Only structured "type":"error" events count while the process is still # running: model prose or tool output quoting these signatures is # JSON-escaped inside event strings, so a healthy streaming run is never - # killed for merely discussing context windows or quota errors. - awk 'tolower($0) ~ /"type"[[:space:]]*:[[:space:]]*"error"/ && tolower($0) ~ /contextoverflowerror|tokens_limit_reached|request body too large|context window|budget limit|insufficient_quota/ { found = 1; exit } END { exit !found }' "$opencode_json_file" + # killed for merely discussing context windows, quota errors, or missing + # models. Model-unavailable signatures (OpenRouter "No endpoints found" / + # "not a valid model ID", OpenAI-style model_not_found) matter because a + # delisted pinned free model would otherwise hang and burn the whole + # candidate run budget. + awk 'tolower($0) ~ /"type"[[:space:]]*:[[:space:]]*"error"/ && tolower($0) ~ /contextoverflowerror|tokens_limit_reached|request body too large|context window|budget limit|insufficient_quota|model_not_found|model not found|modelnotfounderror|not a valid model|no endpoints/ { found = 1; exit } END { exit !found }' "$opencode_json_file" } emit_sanitized_opencode_failure_detail() { @@ -276,6 +282,8 @@ emit_sanitized_opencode_failure_detail() { failure_class="context-window" elif grep -Eiq 'budget limit|insufficient_quota|quota exceeded' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then failure_class="quota-or-budget" + elif grep -Eiq 'model_not_found|model not found|ModelNotFoundError|not a valid model|no endpoints' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then + failure_class="model-unavailable" elif grep -Eiq 'rate.?limit|too many requests|(^|[^0-9])429([^0-9]|$)' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then failure_class="rate-limit" elif grep -Eiq 'permission denied|authentication|authorization|(^|[^0-9])(401|403)([^0-9]|$)' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then @@ -427,7 +435,7 @@ run_one_model_attempt() { printf 'OpenCode %s attempt %s/%s timed out after %ss; falling through within the remaining retry budget instead of blocking the org queue.\n' "$model_candidate" "$attempt" "$attempts" "$run_timeout_seconds" fi if is_fatal_provider_failure "$opencode_json_file"; then - printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, or quota); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts" + printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, quota, or model unavailable); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts" return 2 fi return 1 @@ -438,7 +446,7 @@ run_one_model_attempt() { printf 'OpenCode %s attempt %s/%s JSON output did not include a session id.\n' "$model_candidate" "$attempt" "$attempts" emit_rejected_opencode_artifact_metadata "sessionless-json" "$opencode_json_file" if is_fatal_provider_failure "$opencode_json_file"; then - printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, or quota); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts" + printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, quota, or model unavailable); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts" return 2 fi return 1 @@ -533,7 +541,7 @@ main() { continue fi assert_reasoning_effort_for_candidate "$model_candidate" - safe_model="${model_candidate//\//-}" + safe_model="${model_candidate//[\/:]/-}" prompt_file="${RUNNER_TEMP}/opencode-review-${safe_model}-prompt.md" candidate_output_file="${RUNNER_TEMP}/opencode-review-${safe_model}.md" opencode_json_file="${candidate_output_file}.jsonl" diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 71e96add..796496a0 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -576,6 +576,32 @@ def test_model_text_quoting_error_signatures_does_not_kill_run(tmp_path: Path) - assert "logged a fatal provider error while still running" not in result.stdout +def test_delisted_openrouter_model_error_kills_hung_run_early(tmp_path: Path) -> None: + """A delisted pinned OpenRouter model dies seconds after a model-unavailable error.""" + start = time.monotonic() + result = run_failed_model( + tmp_path, + json_line=( + '{"type":"error","error":{"name":"ProviderModelNotFoundError","data":' + '{"message":"No endpoints found for nvidia/nemotron-3-ultra-550b-a55b:free."}}}' + ), + model_candidates="openrouter/nvidia/nemotron-3-ultra-550b-a55b:free", + extra_env={ + "OPENROUTER_API_KEY": "fake-openrouter-key", + "FAKE_OPENCODE_HANG_SECONDS": "120", + "OPENCODE_RUN_TIMEOUT_SECONDS": "120", + "OPENCODE_TOTAL_RETRY_BUDGET_SECONDS": "240", + }, + ) + elapsed = time.monotonic() - start + + assert result.returncode == 1 + assert "logged a fatal provider error while still running" in result.stdout + assert "skipping remaining attempts for this model" in result.stdout + assert "class=model-unavailable" in result.stdout + assert elapsed < 25 + + def test_dynamic_review_cadence_uses_small_change_timeout(tmp_path: Path) -> None: """Small PRs fail through hung/unavailable providers quickly with a visible budget reason.""" result = run_failed_model( From c4cf0b90ffe4ff304166387383eb6e468e546876 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 25 Jul 2026 08:33:53 +0900 Subject: [PATCH 2/2] Bound paid-provider spend in the model pool Live run 30120972549 cycled the pool 102 times over ~3h re-sending the full review prompt to paid OpenRouter models and exhausted the org's entire credit; run 30122360773 kept cycling 700+ attempts into HTTP 402 until the retry budget elapsed. Three spend guards (timing untouched): - Credit exhaustion (HTTP 402 / 'Insufficient credits' / payment required) marks the candidate failed for the rest of the run via structured error events and stderr diagnostics, with a new class=credit-exhausted failure metadata label. - Control-rejected output now returns a distinct status and is capped per candidate (OPENCODE_INVALID_CONTROL_OUTPUT_CAP, default 3) so a model that cannot satisfy the control contract stops burning paid requests on identical rejections. - A per-run provider attempt ceiling (OPENCODE_POOL_MAX_TOTAL_ATTEMPTS, default 30 = ~3 cycles of the live 10-candidate pool) ends the pool as exhausted before unlimited cycles can accrue spend; when every candidate is marked failed the pool ends immediately instead of sleeping between empty cycles. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- scripts/ci/run_opencode_review_model_pool.sh | 77 ++++++++++++++++++- tests/test_opencode_model_pool_runner.py | 79 ++++++++++++++++++++ 2 files changed, 153 insertions(+), 3 deletions(-) diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index f76ce804..b709ca80 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -245,7 +245,7 @@ is_fatal_provider_failure() { return 0 fi [ -s "$opencode_json_file" ] || return 1 - grep -Eiq 'budget limit|insufficient_quota|model_not_found|model not found|ModelNotFoundError|not a valid model|no endpoints' "$opencode_json_file" + grep -Eiq 'budget limit|insufficient_quota|insufficient credits|payment required|model_not_found|model not found|ModelNotFoundError|not a valid model|no endpoints' "$opencode_json_file" } has_fatal_provider_error_event() { @@ -260,7 +260,24 @@ has_fatal_provider_error_event() { # "not a valid model ID", OpenAI-style model_not_found) matter because a # delisted pinned free model would otherwise hang and burn the whole # candidate run budget. - awk 'tolower($0) ~ /"type"[[:space:]]*:[[:space:]]*"error"/ && tolower($0) ~ /contextoverflowerror|tokens_limit_reached|request body too large|context window|budget limit|insufficient_quota|model_not_found|model not found|modelnotfounderror|not a valid model|no endpoints/ { found = 1; exit } END { exit !found }' "$opencode_json_file" + awk 'tolower($0) ~ /"type"[[:space:]]*:[[:space:]]*"error"/ && tolower($0) ~ /contextoverflowerror|tokens_limit_reached|request body too large|context window|budget limit|insufficient_quota|insufficient credits|payment required|model_not_found|model not found|modelnotfounderror|not a valid model|no endpoints/ { found = 1; exit } END { exit !found }' "$opencode_json_file" +} + +is_credit_exhausted_failure() { + local opencode_json_file="$1" + local opencode_stderr_file="$2" + + # Paid-provider credit exhaustion (OpenRouter HTTP 402 "Insufficient + # credits") can never recover within one run: every retry is a wasted + # paid request. Match structured "type":"error" events in the JSON + # stream (same trust model as has_fatal_provider_error_event) plus + # CLI diagnostics on stderr, which never contain model prose. + if [ -s "$opencode_json_file" ] && + awk 'tolower($0) ~ /"type"[[:space:]]*:[[:space:]]*"error"/ && tolower($0) ~ /insufficient credits|payment required|(^|[^0-9])402([^0-9]|$)/ { found = 1; exit } END { exit !found }' "$opencode_json_file"; then + return 0 + fi + [ -s "$opencode_stderr_file" ] || return 1 + grep -Eiq 'insufficient credits|payment required|"code"[[:space:]]*:[[:space:]]*402' "$opencode_stderr_file" } emit_sanitized_opencode_failure_detail() { @@ -280,6 +297,8 @@ emit_sanitized_opencode_failure_detail() { failure_class="unclassified" if grep -Eiq 'ContextOverflowError|tokens_limit_reached|Request body too large|context window' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then failure_class="context-window" + elif grep -Eiq 'insufficient credits|payment required|"code"[[:space:]]*:[[:space:]]*402' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then + failure_class="credit-exhausted" elif grep -Eiq 'budget limit|insufficient_quota|quota exceeded' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then failure_class="quota-or-budget" elif grep -Eiq 'model_not_found|model not found|ModelNotFoundError|not a valid model|no endpoints' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then @@ -467,7 +486,7 @@ run_one_model_attempt() { if ! normalize_opencode_output "$candidate_output_file"; then printf 'OpenCode %s attempt %s/%s output did not include a valid control conclusion.\n' "$model_candidate" "$attempt" "$attempts" emit_rejected_opencode_artifact_metadata "invalid-control-output" "$candidate_output_file" - return 1 + return 3 fi return 0 } @@ -477,8 +496,19 @@ main() { local opencode_json_file opencode_export_file agent retry_sleep original_run_timeout run_status cycle_sleep cycle max_cycles local uncapped_run_timeout local changed_file_count small_file_threshold medium_file_threshold + local invalid_control_cap max_total_attempts total_attempts alive_candidates + local -A dead_candidate_reasons invalid_control_counts local -a model_candidates + # Spend guards, not timing: a paid candidate that keeps producing + # control-rejected output or has exhausted provider credits must stop + # consuming paid requests instead of cycling until the retry budget + # elapses (run 30120972549 burned the org OpenRouter credit in ~102 + # cycles of re-sent full prompts). Timeouts/deadlines are untouched. + invalid_control_cap="$(env_integer_or_default OPENCODE_INVALID_CONTROL_OUTPUT_CAP 3)" + max_total_attempts="$(env_integer_or_default OPENCODE_POOL_MAX_TOTAL_ATTEMPTS 30)" + total_attempts=0 + attempts="${OPENCODE_MODEL_ATTEMPTS:-3}" original_run_timeout="${OPENCODE_RUN_TIMEOUT_SECONDS:-600}" budget_seconds="${OPENCODE_TOTAL_RETRY_BUDGET_SECONDS:-1500}" @@ -537,6 +567,11 @@ main() { while :; do printf 'Starting OpenCode model pool cycle %s.\n' "$cycle" for model_candidate in "${model_candidates[@]}"; do + if [ -n "${dead_candidate_reasons[$model_candidate]:-}" ]; then + printf 'Skipping OpenCode %s for the rest of this run: %s.\n' \ + "$model_candidate" "${dead_candidate_reasons[$model_candidate]}" + continue + fi if should_skip_model_candidate "$model_candidate"; then continue fi @@ -556,6 +591,14 @@ main() { fi exit 1 fi + if [ "$max_total_attempts" -gt 0 ] && [ "$total_attempts" -ge "$max_total_attempts" ]; then + printf 'OpenCode model pool reached the per-run provider attempt ceiling of %s attempts; ending the pool to bound provider spend. Set OPENCODE_POOL_MAX_TOTAL_ATTEMPTS=0 to disable.\n' "$max_total_attempts" + if finish_pool_without_model; then + exit 0 + fi + exit 1 + fi + total_attempts=$((total_attempts + 1)) remaining="$original_run_timeout" if [ "$deadline" -gt 0 ]; then remaining=$((deadline - now)) @@ -585,6 +628,20 @@ main() { else run_status=$? fi + if [ "$run_status" -ne 3 ] && is_credit_exhausted_failure "$opencode_json_file" "${opencode_json_file}.stderr"; then + dead_candidate_reasons[$model_candidate]="provider credits exhausted (HTTP 402 / payment required)" + printf 'OpenCode %s provider credits are exhausted; marking this candidate failed for the rest of the run so retries cannot accrue further spend.\n' "$model_candidate" + break + fi + if [ "$run_status" -eq 3 ]; then + invalid_control_counts[$model_candidate]=$((${invalid_control_counts[$model_candidate]:-0} + 1)) + if [ "$invalid_control_cap" -gt 0 ] && [ "${invalid_control_counts[$model_candidate]}" -ge "$invalid_control_cap" ]; then + dead_candidate_reasons[$model_candidate]="produced ${invalid_control_counts[$model_candidate]} control-rejected outputs" + printf 'OpenCode %s produced %s control-rejected outputs; marking this candidate failed for the rest of the run so paid retries cannot loop on rejected output. Set OPENCODE_INVALID_CONTROL_OUTPUT_CAP=0 to disable.\n' \ + "$model_candidate" "${invalid_control_counts[$model_candidate]}" + break + fi + fi if [ "$run_status" -eq 2 ]; then break fi @@ -601,6 +658,20 @@ main() { done done + alive_candidates=0 + for model_candidate in "${model_candidates[@]}"; do + if [ -z "${dead_candidate_reasons[$model_candidate]:-}" ]; then + alive_candidates=$((alive_candidates + 1)) + fi + done + if [ "$alive_candidates" -eq 0 ]; then + printf 'Every OpenCode model candidate is marked failed for this run; ending the pool without further provider spend.\n' + if finish_pool_without_model; then + exit 0 + fi + exit 1 + fi + printf 'OpenCode completed a full model-candidate cycle without a valid control conclusion; continuing until a model succeeds or the retry budget/GitHub Actions job timeout is reached.\n' if [ "$max_cycles" -gt 0 ] && [ "$cycle" -ge "$max_cycles" ]; then printf 'OpenCode model pool reached configured max cycle count %s without a valid control conclusion.\n' "$max_cycles" diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 796496a0..5e823348 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -602,6 +602,85 @@ def test_delisted_openrouter_model_error_kills_hung_run_early(tmp_path: Path) -> assert elapsed < 25 +def test_credit_exhausted_402_ends_pool_without_further_spend(tmp_path: Path) -> None: + """A paid candidate hitting HTTP 402 is dead for the run instead of cycling.""" + start = time.monotonic() + result = run_failed_model( + tmp_path, + json_line=( + '{"type":"error","error":{"name":"AI_APICallError","data":' + '{"message":"Insufficient credits. Add more using ' + 'https://openrouter.ai/settings/credits","statusCode":402}}}' + ), + model_candidates="openrouter/deepseek/deepseek-v3.2", + extra_env={ + "OPENROUTER_API_KEY": "fake-openrouter-key", + "OPENCODE_POOL_MAX_CYCLES": "0", + }, + ) + elapsed = time.monotonic() - start + + assert result.returncode == 1 + assert "provider credits are exhausted" in result.stdout + assert "marking this candidate failed for the rest of the run" in result.stdout + assert "Every OpenCode model candidate is marked failed for this run" in result.stdout + assert "class=credit-exhausted" in result.stdout + assert "Restarting OpenCode model pool" not in result.stdout + assert elapsed < 20 + + +def test_invalid_control_output_cap_marks_candidate_failed(tmp_path: Path) -> None: + """Repeated control-rejected output stops retrying at the cap, not the budget.""" + result = run_failed_model( + tmp_path, + json_line='{"type":"step_start","sessionID":"session-1"}', + extra_env={ + "FAKE_OPENCODE_RUN_EXIT": "0", + "FAKE_OPENCODE_EXPORT": json.dumps( + { + "messages": [ + { + "info": {"role": "assistant"}, + "parts": [ + {"type": "text", "text": "not a control conclusion"} + ], + } + ] + } + ), + "OPENCODE_MODEL_ATTEMPTS": "3", + "OPENCODE_INVALID_CONTROL_OUTPUT_CAP": "2", + "OPENCODE_BACKOFF_INITIAL_SECONDS": "1", + "OPENCODE_POOL_MAX_CYCLES": "0", + }, + ) + + assert result.returncode == 1 + assert "produced 2 control-rejected outputs" in result.stdout + assert "marking this candidate failed for the rest of the run" in result.stdout + assert "Every OpenCode model candidate is marked failed for this run" in result.stdout + assert "attempt 3/3" not in result.stdout + + +def test_attempt_ceiling_bounds_provider_spend(tmp_path: Path) -> None: + """The per-run attempt ceiling ends the pool before the retry budget elapses.""" + result = run_failed_model( + tmp_path, + extra_env={ + "OPENCODE_MODEL_ATTEMPTS": "3", + "OPENCODE_POOL_MAX_TOTAL_ATTEMPTS": "2", + "OPENCODE_BACKOFF_INITIAL_SECONDS": "1", + "OPENCODE_POOL_MAX_CYCLES": "0", + }, + ) + + assert result.returncode == 1 + assert ( + "reached the per-run provider attempt ceiling of 2 attempts" in result.stdout + ) + assert "attempt 3/3" not in result.stdout + + def test_dynamic_review_cadence_uses_small_change_timeout(tmp_path: Path) -> None: """Small PRs fail through hung/unavailable providers quickly with a visible budget reason.""" result = run_failed_model(