From 5151366c44c575b8b90aa29d5da185cd1fd6044a Mon Sep 17 00:00:00 2001 From: Ling-Sen Peng Date: Fri, 24 Jul 2026 09:48:52 -0700 Subject: [PATCH] Revert "Merge pull request #141 from conductor-oss/fix/agent-e2e-llm-budget" This reverts commit 3f873811ac9010d3cbcdc26e257529ec1622de65, reversing changes made to 9e335df4fcd214c0331910dde842a9cf3bad7c36. --- e2e/src/test/java/BaseTest.java | 2 - .../java/LlmBudgetExhaustionExtension.java | 29 ------------- .../LlmBudgetExhaustionExtensionTest.java | 42 ------------------- 3 files changed, 73 deletions(-) delete mode 100644 e2e/src/test/java/LlmBudgetExhaustionExtension.java delete mode 100644 e2e/src/test/java/LlmBudgetExhaustionExtensionTest.java diff --git a/e2e/src/test/java/BaseTest.java b/e2e/src/test/java/BaseTest.java index 4442d2436..5224165ff 100644 --- a/e2e/src/test/java/BaseTest.java +++ b/e2e/src/test/java/BaseTest.java @@ -19,7 +19,6 @@ import java.util.Map; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.extension.ExtendWith; import io.orkes.conductor.client.model.agent.CompileResponse; @@ -38,7 +37,6 @@ *
  • Helper to extract agentDef from a plan() result
  • * */ -@ExtendWith(LlmBudgetExhaustionExtension.class) public abstract class BaseTest { /** API URL for the Conductor server (includes /api suffix). */ diff --git a/e2e/src/test/java/LlmBudgetExhaustionExtension.java b/e2e/src/test/java/LlmBudgetExhaustionExtension.java deleted file mode 100644 index de86a22c2..000000000 --- a/e2e/src/test/java/LlmBudgetExhaustionExtension.java +++ /dev/null @@ -1,29 +0,0 @@ -import java.util.Locale; - -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.api.extension.TestExecutionExceptionHandler; -import org.opentest4j.TestAbortedException; - -/** Skips live LLM tests when a provider's configured API usage budget is exhausted. */ -final class LlmBudgetExhaustionExtension implements TestExecutionExceptionHandler { - - static final String USAGE_LIMIT_MESSAGE = "reached your specified api usage limits"; - - @Override - public void handleTestExecutionException(ExtensionContext context, Throwable throwable) throws Throwable { - if (isUsageLimitError(throwable)) { - throw new TestAbortedException("LLM API usage budget is exhausted", throwable); - } - throw throwable; - } - - static boolean isUsageLimitError(Throwable throwable) { - for (Throwable current = throwable; current != null; current = current.getCause()) { - String message = current.getMessage(); - if (message != null && message.toLowerCase(Locale.ROOT).contains(USAGE_LIMIT_MESSAGE)) { - return true; - } - } - return false; - } -} diff --git a/e2e/src/test/java/LlmBudgetExhaustionExtensionTest.java b/e2e/src/test/java/LlmBudgetExhaustionExtensionTest.java deleted file mode 100644 index 7eda81b43..000000000 --- a/e2e/src/test/java/LlmBudgetExhaustionExtensionTest.java +++ /dev/null @@ -1,42 +0,0 @@ -import org.junit.jupiter.api.Test; -import org.opentest4j.TestAbortedException; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class LlmBudgetExhaustionExtensionTest { - - @Test - void detectsUsageLimitMessageInCauseChain() { - Throwable failure = new AssertionError( - "agent failed", - new IllegalStateException("You have reached your specified API usage limits. Try again later.")); - - assertTrue(LlmBudgetExhaustionExtension.isUsageLimitError(failure)); - } - - @Test - void doesNotMaskOtherLlmFailures() { - Throwable failure = new AssertionError("Agent failed: invalid model"); - - assertFalse(LlmBudgetExhaustionExtension.isUsageLimitError(failure)); - } - - @Test - void abortsUsageLimitFailuresAndRethrowsOtherFailures() { - LlmBudgetExhaustionExtension extension = new LlmBudgetExhaustionExtension(); - Throwable budgetFailure = - new AssertionError("You have reached your specified API usage limits."); - Throwable otherFailure = new AssertionError("Agent failed: invalid model"); - - assertThrows( - TestAbortedException.class, - () -> extension.handleTestExecutionException(null, budgetFailure)); - Throwable rethrown = assertThrows( - Throwable.class, - () -> extension.handleTestExecutionException(null, otherFailure)); - assertSame(otherFailure, rethrown); - } -}