From d5f72c8903a81c61e28069a72e42f72dfe52d6dc Mon Sep 17 00:00:00 2001 From: Ling-Sen Peng Date: Fri, 24 Jul 2026 09:07:17 -0700 Subject: [PATCH 1/6] test: skip providers marked out of LLM budget --- .github/workflows/agent-e2e.yml | 1 + e2e/conftest.py | 3 ++ e2e/provider_budget.py | 39 +++++++++++++++++ e2e/test_suite1_basic_validation.py | 4 ++ e2e/test_suite25_media_input.py | 3 ++ tests/unit/ai/agents/test_provider_budget.py | 45 ++++++++++++++++++++ 6 files changed, 95 insertions(+) create mode 100644 e2e/provider_budget.py create mode 100644 tests/unit/ai/agents/test_provider_budget.py diff --git a/.github/workflows/agent-e2e.yml b/.github/workflows/agent-e2e.yml index ae4c1f50..fac006f1 100644 --- a/.github/workflows/agent-e2e.yml +++ b/.github/workflows/agent-e2e.yml @@ -36,6 +36,7 @@ jobs: env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + LLM_PROVIDERS_OUT_OF_BUDGET: ${{ vars.LLM_PROVIDERS_OUT_OF_BUDGET }} AGENTSPAN_SERVER_URL: http://localhost:8080/api AGENTSPAN_CLI_PATH: ${{ github.workspace }}/agentspan steps: diff --git a/e2e/conftest.py b/e2e/conftest.py index 442fa008..defdef55 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -6,6 +6,8 @@ import pytest import requests +from provider_budget import require_llm_provider + # ── Configuration from env (set by orchestrator) ──────────────────────── SERVER_URL = os.environ.get("AGENTSPAN_SERVER_URL", "http://localhost:8080/api") @@ -78,6 +80,7 @@ def runtime(): @pytest.fixture(scope="session") def model(): + require_llm_provider(MODEL) return MODEL diff --git a/e2e/provider_budget.py b/e2e/provider_budget.py new file mode 100644 index 00000000..9e9b3b20 --- /dev/null +++ b/e2e/provider_budget.py @@ -0,0 +1,39 @@ +"""Helpers for skipping E2E calls to providers with exhausted budgets.""" + +import os + +import pytest + + +OUT_OF_BUDGET_ENV = "LLM_PROVIDERS_OUT_OF_BUDGET" + + +def provider_from_model(model_or_provider: str) -> str: + """Return the provider name from either ``provider/model`` or a model name.""" + value = model_or_provider.strip().lower() + if "/" in value: + return value.split("/", 1)[0] + if value.startswith("claude"): + return "anthropic" + if value.startswith(("gpt-", "o1", "o3", "o4")): + return "openai" + return value + + +def out_of_budget_providers(raw_value: str | None = None) -> set[str]: + """Parse the comma-separated provider circuit breaker.""" + value = os.environ.get(OUT_OF_BUDGET_ENV, "") if raw_value is None else raw_value + return { + provider_from_model(provider) + for provider in value.split(",") + if provider.strip() + } + + +def require_llm_provider(model_or_provider: str) -> None: + """Skip the current test when its LLM provider is marked out of budget.""" + provider = provider_from_model(model_or_provider) + if provider in out_of_budget_providers(): + pytest.skip( + f"{provider} provider is marked out of budget by {OUT_OF_BUDGET_ENV}" + ) diff --git a/e2e/test_suite1_basic_validation.py b/e2e/test_suite1_basic_validation.py index b20ed4a5..4c0849d7 100644 --- a/e2e/test_suite1_basic_validation.py +++ b/e2e/test_suite1_basic_validation.py @@ -10,6 +10,8 @@ import pytest +from provider_budget import require_llm_provider + from conductor.ai.agents import ( Agent, Guardrail, @@ -351,6 +353,7 @@ def _build_judge_comparison(agent_spec: dict, result: dict) -> str: def _judge_call_anthropic(model: str, system: str, user: str) -> str: """Call Anthropic API. Returns raw text response.""" + require_llm_provider("anthropic") try: import anthropic except ImportError: @@ -377,6 +380,7 @@ def _judge_call_anthropic(model: str, system: str, user: str) -> str: def _judge_call_openai(model: str, system: str, user: str) -> str: """Call OpenAI API. Returns raw text response.""" + require_llm_provider("openai") try: import openai except ImportError: diff --git a/e2e/test_suite25_media_input.py b/e2e/test_suite25_media_input.py index e6a40459..900d0973 100644 --- a/e2e/test_suite25_media_input.py +++ b/e2e/test_suite25_media_input.py @@ -35,6 +35,8 @@ import pytest +from provider_budget import require_llm_provider + from conductor.ai.agents import Agent pytestmark = [ @@ -122,6 +124,7 @@ def _agent_slug(key_env: str) -> str: def _require_key(key_env: str): """Skip unless the provider key is set.""" + require_llm_provider(_agent_slug(key_env)) if not os.environ.get(key_env): pytest.skip(f"{key_env} not set — provider unavailable") diff --git a/tests/unit/ai/agents/test_provider_budget.py b/tests/unit/ai/agents/test_provider_budget.py new file mode 100644 index 00000000..2d230d1b --- /dev/null +++ b/tests/unit/ai/agents/test_provider_budget.py @@ -0,0 +1,45 @@ +import pytest + +from e2e.provider_budget import ( + OUT_OF_BUDGET_ENV, + out_of_budget_providers, + provider_from_model, + require_llm_provider, +) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("anthropic/claude-sonnet-4-6", "anthropic"), + ("claude-sonnet-4-6", "anthropic"), + ("openai/gpt-4o-mini", "openai"), + ("gpt-4o-mini", "openai"), + ], +) +def test_provider_from_model(value, expected): + assert provider_from_model(value) == expected + + +def test_out_of_budget_providers_parses_multiple_values(): + assert out_of_budget_providers(" Anthropic, OPENAI ,, ") == { + "anthropic", + "openai", + } + + +def test_out_of_budget_providers_allows_empty_value(): + assert out_of_budget_providers("") == set() + + +def test_require_llm_provider_skips_matching_provider(monkeypatch): + monkeypatch.setenv(OUT_OF_BUDGET_ENV, "anthropic") + + with pytest.raises(pytest.skip.Exception, match="anthropic provider"): + require_llm_provider("anthropic/claude-sonnet-4-6") + + +def test_require_llm_provider_allows_other_provider(monkeypatch): + monkeypatch.setenv(OUT_OF_BUDGET_ENV, "anthropic") + + require_llm_provider("openai/gpt-4o-mini") From b5b585d6711c5bf0189c506717922f8256b85bd5 Mon Sep 17 00:00:00 2001 From: Ling-Sen Peng Date: Fri, 24 Jul 2026 09:10:36 -0700 Subject: [PATCH 2/6] test: normalize LLM provider aliases --- e2e/provider_budget.py | 17 +++++++++++++++-- tests/unit/ai/agents/test_provider_budget.py | 16 ++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/e2e/provider_budget.py b/e2e/provider_budget.py index 9e9b3b20..6b2f539b 100644 --- a/e2e/provider_budget.py +++ b/e2e/provider_budget.py @@ -7,17 +7,30 @@ OUT_OF_BUDGET_ENV = "LLM_PROVIDERS_OUT_OF_BUDGET" +_PROVIDER_ALIASES = { + "azureopenai": "azure_openai", + "gemini": "google_gemini", + "vertex_ai": "google_vertex_ai", + "bedrock": "aws_bedrock", + "huggingface": "hugging_face", + "xai": "grok", +} + def provider_from_model(model_or_provider: str) -> str: """Return the provider name from either ``provider/model`` or a model name.""" value = model_or_provider.strip().lower() if "/" in value: - return value.split("/", 1)[0] + value = value.split("/", 1)[0] if value.startswith("claude"): return "anthropic" if value.startswith(("gpt-", "o1", "o3", "o4")): return "openai" - return value + if value.startswith("gemini-"): + return "google_gemini" + if value.startswith("grok-"): + return "grok" + return _PROVIDER_ALIASES.get(value, value) def out_of_budget_providers(raw_value: str | None = None) -> set[str]: diff --git a/tests/unit/ai/agents/test_provider_budget.py b/tests/unit/ai/agents/test_provider_budget.py index 2d230d1b..fe65584f 100644 --- a/tests/unit/ai/agents/test_provider_budget.py +++ b/tests/unit/ai/agents/test_provider_budget.py @@ -15,6 +15,22 @@ ("claude-sonnet-4-6", "anthropic"), ("openai/gpt-4o-mini", "openai"), ("gpt-4o-mini", "openai"), + ("azureopenai/gpt-4o-mini", "azure_openai"), + ("azure_openai/gpt-4o-mini", "azure_openai"), + ("gemini/gemini-2.5-flash", "google_gemini"), + ("google_gemini/gemini-2.5-flash", "google_gemini"), + ("vertex_ai/gemini-2.5-flash", "google_vertex_ai"), + ("google_vertex_ai/gemini-2.5-flash", "google_vertex_ai"), + ("bedrock/us.anthropic.claude-haiku-4-5", "aws_bedrock"), + ("aws_bedrock/us.anthropic.claude-haiku-4-5", "aws_bedrock"), + ("cohere/command-a-vision-07-2025", "cohere"), + ("grok/grok-4.5", "grok"), + ("xai/grok-4.5", "grok"), + ("perplexity/sonar", "perplexity"), + ("mistral/pixtral-large", "mistral"), + ("huggingface/model", "hugging_face"), + ("hugging_face/model", "hugging_face"), + ("stability/model", "stability"), ], ) def test_provider_from_model(value, expected): From dd1cfe048479dee7fbb710fcb6a78e64902ad269 Mon Sep 17 00:00:00 2001 From: Ling-Sen Peng Date: Fri, 24 Jul 2026 09:13:32 -0700 Subject: [PATCH 3/6] test: scope budget providers to Python E2E coverage --- e2e/provider_budget.py | 17 ++--------------- tests/unit/ai/agents/test_provider_budget.py | 16 ---------------- 2 files changed, 2 insertions(+), 31 deletions(-) diff --git a/e2e/provider_budget.py b/e2e/provider_budget.py index 6b2f539b..9e9b3b20 100644 --- a/e2e/provider_budget.py +++ b/e2e/provider_budget.py @@ -7,30 +7,17 @@ OUT_OF_BUDGET_ENV = "LLM_PROVIDERS_OUT_OF_BUDGET" -_PROVIDER_ALIASES = { - "azureopenai": "azure_openai", - "gemini": "google_gemini", - "vertex_ai": "google_vertex_ai", - "bedrock": "aws_bedrock", - "huggingface": "hugging_face", - "xai": "grok", -} - def provider_from_model(model_or_provider: str) -> str: """Return the provider name from either ``provider/model`` or a model name.""" value = model_or_provider.strip().lower() if "/" in value: - value = value.split("/", 1)[0] + return value.split("/", 1)[0] if value.startswith("claude"): return "anthropic" if value.startswith(("gpt-", "o1", "o3", "o4")): return "openai" - if value.startswith("gemini-"): - return "google_gemini" - if value.startswith("grok-"): - return "grok" - return _PROVIDER_ALIASES.get(value, value) + return value def out_of_budget_providers(raw_value: str | None = None) -> set[str]: diff --git a/tests/unit/ai/agents/test_provider_budget.py b/tests/unit/ai/agents/test_provider_budget.py index fe65584f..2d230d1b 100644 --- a/tests/unit/ai/agents/test_provider_budget.py +++ b/tests/unit/ai/agents/test_provider_budget.py @@ -15,22 +15,6 @@ ("claude-sonnet-4-6", "anthropic"), ("openai/gpt-4o-mini", "openai"), ("gpt-4o-mini", "openai"), - ("azureopenai/gpt-4o-mini", "azure_openai"), - ("azure_openai/gpt-4o-mini", "azure_openai"), - ("gemini/gemini-2.5-flash", "google_gemini"), - ("google_gemini/gemini-2.5-flash", "google_gemini"), - ("vertex_ai/gemini-2.5-flash", "google_vertex_ai"), - ("google_vertex_ai/gemini-2.5-flash", "google_vertex_ai"), - ("bedrock/us.anthropic.claude-haiku-4-5", "aws_bedrock"), - ("aws_bedrock/us.anthropic.claude-haiku-4-5", "aws_bedrock"), - ("cohere/command-a-vision-07-2025", "cohere"), - ("grok/grok-4.5", "grok"), - ("xai/grok-4.5", "grok"), - ("perplexity/sonar", "perplexity"), - ("mistral/pixtral-large", "mistral"), - ("huggingface/model", "hugging_face"), - ("hugging_face/model", "hugging_face"), - ("stability/model", "stability"), ], ) def test_provider_from_model(value, expected): From 05219225336551145f79b0b21c06af9a8c39929d Mon Sep 17 00:00:00 2001 From: Ling-Sen Peng Date: Fri, 24 Jul 2026 09:14:42 -0700 Subject: [PATCH 4/6] test: accept shared provider budget values --- e2e/provider_budget.py | 6 +++++- tests/unit/ai/agents/test_provider_budget.py | 13 ++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/e2e/provider_budget.py b/e2e/provider_budget.py index 9e9b3b20..774aea1c 100644 --- a/e2e/provider_budget.py +++ b/e2e/provider_budget.py @@ -21,7 +21,11 @@ def provider_from_model(model_or_provider: str) -> str: def out_of_budget_providers(raw_value: str | None = None) -> set[str]: - """Parse the comma-separated provider circuit breaker.""" + """Parse the comma-separated provider circuit breaker. + + Provider names unknown to this SDK are preserved so one shared variable can + safely contain providers used by other repositories. + """ value = os.environ.get(OUT_OF_BUDGET_ENV, "") if raw_value is None else raw_value return { provider_from_model(provider) diff --git a/tests/unit/ai/agents/test_provider_budget.py b/tests/unit/ai/agents/test_provider_budget.py index 2d230d1b..b22ad03a 100644 --- a/tests/unit/ai/agents/test_provider_budget.py +++ b/tests/unit/ai/agents/test_provider_budget.py @@ -22,9 +22,13 @@ def test_provider_from_model(value, expected): def test_out_of_budget_providers_parses_multiple_values(): - assert out_of_budget_providers(" Anthropic, OPENAI ,, ") == { + assert out_of_budget_providers( + " Anthropic, OPENAI, google_gemini, aws_bedrock ,, " + ) == { "anthropic", "openai", + "google_gemini", + "aws_bedrock", } @@ -43,3 +47,10 @@ def test_require_llm_provider_allows_other_provider(monkeypatch): monkeypatch.setenv(OUT_OF_BUDGET_ENV, "anthropic") require_llm_provider("openai/gpt-4o-mini") + + +def test_require_llm_provider_safely_matches_unknown_provider(monkeypatch): + monkeypatch.setenv(OUT_OF_BUDGET_ENV, "anthropic,aws_bedrock") + + with pytest.raises(pytest.skip.Exception, match="aws_bedrock provider"): + require_llm_provider("aws_bedrock/model-used-by-another-repository") From 46a32ea62383fa34b339924c0320cc9fb6a0a45f Mon Sep 17 00:00:00 2001 From: Ling-Sen Peng Date: Fri, 24 Jul 2026 09:19:00 -0700 Subject: [PATCH 5/6] test: remove hardcoded provider budget errors --- e2e/test_suite1_basic_validation.py | 19 +++++++------------ e2e/test_suite25_media_input.py | 6 ------ 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/e2e/test_suite1_basic_validation.py b/e2e/test_suite1_basic_validation.py index 4c0849d7..5cde0581 100644 --- a/e2e/test_suite1_basic_validation.py +++ b/e2e/test_suite1_basic_validation.py @@ -363,18 +363,13 @@ def _judge_call_anthropic(model: str, system: str, user: str) -> str: ) client = anthropic.Anthropic() # reads ANTHROPIC_API_KEY from env - try: - response = client.messages.create( - model=model, - max_tokens=1024, - system=system, - messages=[{"role": "user", "content": user}], - temperature=0, - ) - except anthropic.BadRequestError as exc: - if "reached your specified api usage limits" in str(exc).lower(): - pytest.skip("Anthropic API usage budget is exhausted") - raise + response = client.messages.create( + model=model, + max_tokens=1024, + system=system, + messages=[{"role": "user", "content": user}], + temperature=0, + ) return response.content[0].text.strip() diff --git a/e2e/test_suite25_media_input.py b/e2e/test_suite25_media_input.py index 900d0973..fe1d7ec1 100644 --- a/e2e/test_suite25_media_input.py +++ b/e2e/test_suite25_media_input.py @@ -216,12 +216,6 @@ def test_without_media_token_is_absent(self, runtime, key_env, model_id): result = runtime.run(agent, READ_PROMPT, timeout=TIMEOUT) - if ( - result.error - and "reached your specified api usage limits" in result.error.lower() - ): - pytest.skip("Anthropic API usage budget is exhausted") - assert result.status == "COMPLETED", ( f"no-media run did not complete: status={result.status} " f"execution_id={result.execution_id}" From b529a9d40c698ef2d42a8d94cd0483b2bfcc5378 Mon Sep 17 00:00:00 2001 From: Ling-Sen Peng Date: Fri, 24 Jul 2026 09:42:23 -0700 Subject: [PATCH 6/6] test: let LLM budget failures fail CI --- .github/workflows/agent-e2e.yml | 1 - e2e/conftest.py | 3 -- e2e/provider_budget.py | 43 --------------- e2e/test_suite1_basic_validation.py | 4 -- e2e/test_suite25_media_input.py | 3 -- tests/unit/ai/agents/test_provider_budget.py | 56 -------------------- 6 files changed, 110 deletions(-) delete mode 100644 e2e/provider_budget.py delete mode 100644 tests/unit/ai/agents/test_provider_budget.py diff --git a/.github/workflows/agent-e2e.yml b/.github/workflows/agent-e2e.yml index fac006f1..ae4c1f50 100644 --- a/.github/workflows/agent-e2e.yml +++ b/.github/workflows/agent-e2e.yml @@ -36,7 +36,6 @@ jobs: env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - LLM_PROVIDERS_OUT_OF_BUDGET: ${{ vars.LLM_PROVIDERS_OUT_OF_BUDGET }} AGENTSPAN_SERVER_URL: http://localhost:8080/api AGENTSPAN_CLI_PATH: ${{ github.workspace }}/agentspan steps: diff --git a/e2e/conftest.py b/e2e/conftest.py index defdef55..442fa008 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -6,8 +6,6 @@ import pytest import requests -from provider_budget import require_llm_provider - # ── Configuration from env (set by orchestrator) ──────────────────────── SERVER_URL = os.environ.get("AGENTSPAN_SERVER_URL", "http://localhost:8080/api") @@ -80,7 +78,6 @@ def runtime(): @pytest.fixture(scope="session") def model(): - require_llm_provider(MODEL) return MODEL diff --git a/e2e/provider_budget.py b/e2e/provider_budget.py deleted file mode 100644 index 774aea1c..00000000 --- a/e2e/provider_budget.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Helpers for skipping E2E calls to providers with exhausted budgets.""" - -import os - -import pytest - - -OUT_OF_BUDGET_ENV = "LLM_PROVIDERS_OUT_OF_BUDGET" - - -def provider_from_model(model_or_provider: str) -> str: - """Return the provider name from either ``provider/model`` or a model name.""" - value = model_or_provider.strip().lower() - if "/" in value: - return value.split("/", 1)[0] - if value.startswith("claude"): - return "anthropic" - if value.startswith(("gpt-", "o1", "o3", "o4")): - return "openai" - return value - - -def out_of_budget_providers(raw_value: str | None = None) -> set[str]: - """Parse the comma-separated provider circuit breaker. - - Provider names unknown to this SDK are preserved so one shared variable can - safely contain providers used by other repositories. - """ - value = os.environ.get(OUT_OF_BUDGET_ENV, "") if raw_value is None else raw_value - return { - provider_from_model(provider) - for provider in value.split(",") - if provider.strip() - } - - -def require_llm_provider(model_or_provider: str) -> None: - """Skip the current test when its LLM provider is marked out of budget.""" - provider = provider_from_model(model_or_provider) - if provider in out_of_budget_providers(): - pytest.skip( - f"{provider} provider is marked out of budget by {OUT_OF_BUDGET_ENV}" - ) diff --git a/e2e/test_suite1_basic_validation.py b/e2e/test_suite1_basic_validation.py index 5cde0581..3fd998d5 100644 --- a/e2e/test_suite1_basic_validation.py +++ b/e2e/test_suite1_basic_validation.py @@ -10,8 +10,6 @@ import pytest -from provider_budget import require_llm_provider - from conductor.ai.agents import ( Agent, Guardrail, @@ -353,7 +351,6 @@ def _build_judge_comparison(agent_spec: dict, result: dict) -> str: def _judge_call_anthropic(model: str, system: str, user: str) -> str: """Call Anthropic API. Returns raw text response.""" - require_llm_provider("anthropic") try: import anthropic except ImportError: @@ -375,7 +372,6 @@ def _judge_call_anthropic(model: str, system: str, user: str) -> str: def _judge_call_openai(model: str, system: str, user: str) -> str: """Call OpenAI API. Returns raw text response.""" - require_llm_provider("openai") try: import openai except ImportError: diff --git a/e2e/test_suite25_media_input.py b/e2e/test_suite25_media_input.py index fe1d7ec1..3512ff1b 100644 --- a/e2e/test_suite25_media_input.py +++ b/e2e/test_suite25_media_input.py @@ -35,8 +35,6 @@ import pytest -from provider_budget import require_llm_provider - from conductor.ai.agents import Agent pytestmark = [ @@ -124,7 +122,6 @@ def _agent_slug(key_env: str) -> str: def _require_key(key_env: str): """Skip unless the provider key is set.""" - require_llm_provider(_agent_slug(key_env)) if not os.environ.get(key_env): pytest.skip(f"{key_env} not set — provider unavailable") diff --git a/tests/unit/ai/agents/test_provider_budget.py b/tests/unit/ai/agents/test_provider_budget.py deleted file mode 100644 index b22ad03a..00000000 --- a/tests/unit/ai/agents/test_provider_budget.py +++ /dev/null @@ -1,56 +0,0 @@ -import pytest - -from e2e.provider_budget import ( - OUT_OF_BUDGET_ENV, - out_of_budget_providers, - provider_from_model, - require_llm_provider, -) - - -@pytest.mark.parametrize( - ("value", "expected"), - [ - ("anthropic/claude-sonnet-4-6", "anthropic"), - ("claude-sonnet-4-6", "anthropic"), - ("openai/gpt-4o-mini", "openai"), - ("gpt-4o-mini", "openai"), - ], -) -def test_provider_from_model(value, expected): - assert provider_from_model(value) == expected - - -def test_out_of_budget_providers_parses_multiple_values(): - assert out_of_budget_providers( - " Anthropic, OPENAI, google_gemini, aws_bedrock ,, " - ) == { - "anthropic", - "openai", - "google_gemini", - "aws_bedrock", - } - - -def test_out_of_budget_providers_allows_empty_value(): - assert out_of_budget_providers("") == set() - - -def test_require_llm_provider_skips_matching_provider(monkeypatch): - monkeypatch.setenv(OUT_OF_BUDGET_ENV, "anthropic") - - with pytest.raises(pytest.skip.Exception, match="anthropic provider"): - require_llm_provider("anthropic/claude-sonnet-4-6") - - -def test_require_llm_provider_allows_other_provider(monkeypatch): - monkeypatch.setenv(OUT_OF_BUDGET_ENV, "anthropic") - - require_llm_provider("openai/gpt-4o-mini") - - -def test_require_llm_provider_safely_matches_unknown_provider(monkeypatch): - monkeypatch.setenv(OUT_OF_BUDGET_ENV, "anthropic,aws_bedrock") - - with pytest.raises(pytest.skip.Exception, match="aws_bedrock provider"): - require_llm_provider("aws_bedrock/model-used-by-another-repository")