From a0d20d9accf3f5b28f26118fef113d2f786f66e5 Mon Sep 17 00:00:00 2001
From: Ling-Sen Peng <ling-sen.peng@orkes.io>
Date: Thu, 23 Jul 2026 14:57:44 -0700
Subject: [PATCH] test: skip agent e2e when LLM budget is exhausted

---
 e2e/src/test/java/BaseTest.java               |  2 +
 .../java/LlmBudgetExhaustionExtension.java    | 29 +++++++++++++
 .../LlmBudgetExhaustionExtensionTest.java     | 42 +++++++++++++++++++
 3 files changed, 73 insertions(+)
 create mode 100644 e2e/src/test/java/LlmBudgetExhaustionExtension.java
 create mode 100644 e2e/src/test/java/LlmBudgetExhaustionExtensionTest.java

diff --git a/e2e/src/test/java/BaseTest.java b/e2e/src/test/java/BaseTest.java
index 5224165ff..4442d2436 100644
--- a/e2e/src/test/java/BaseTest.java
+++ b/e2e/src/test/java/BaseTest.java
@@ -19,6 +19,7 @@
 import java.util.Map;
 
 import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.extension.ExtendWith;
 
 import io.orkes.conductor.client.model.agent.CompileResponse;
 
@@ -37,6 +38,7 @@
  *   <li>Helper to extract agentDef from a plan() result</li>
  * </ul>
  */
+@ExtendWith(LlmBudgetExhaustionExtension.class)
 public abstract class BaseTest {
 
     /** API URL for the Conductor server (includes /api suffix). */
diff --git a/e2e/src/test/java/LlmBudgetExhaustionExtension.java b/e2e/src/test/java/LlmBudgetExhaustionExtension.java
new file mode 100644
index 000000000..de86a22c2
--- /dev/null
+++ b/e2e/src/test/java/LlmBudgetExhaustionExtension.java
@@ -0,0 +1,29 @@
+import java.util.Locale;
+
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.TestExecutionExceptionHandler;
+import org.opentest4j.TestAbortedException;
+
+/** Skips live LLM tests when a provider's configured API usage budget is exhausted. */
+final class LlmBudgetExhaustionExtension implements TestExecutionExceptionHandler {
+
+    static final String USAGE_LIMIT_MESSAGE = "reached your specified api usage limits";
+
+    @Override
+    public void handleTestExecutionException(ExtensionContext context, Throwable throwable) throws Throwable {
+        if (isUsageLimitError(throwable)) {
+            throw new TestAbortedException("LLM API usage budget is exhausted", throwable);
+        }
+        throw throwable;
+    }
+
+    static boolean isUsageLimitError(Throwable throwable) {
+        for (Throwable current = throwable; current != null; current = current.getCause()) {
+            String message = current.getMessage();
+            if (message != null && message.toLowerCase(Locale.ROOT).contains(USAGE_LIMIT_MESSAGE)) {
+                return true;
+            }
+        }
+        return false;
+    }
+}
diff --git a/e2e/src/test/java/LlmBudgetExhaustionExtensionTest.java b/e2e/src/test/java/LlmBudgetExhaustionExtensionTest.java
new file mode 100644
index 000000000..7eda81b43
--- /dev/null
+++ b/e2e/src/test/java/LlmBudgetExhaustionExtensionTest.java
@@ -0,0 +1,42 @@
+import org.junit.jupiter.api.Test;
+import org.opentest4j.TestAbortedException;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class LlmBudgetExhaustionExtensionTest {
+
+    @Test
+    void detectsUsageLimitMessageInCauseChain() {
+        Throwable failure = new AssertionError(
+                "agent failed",
+                new IllegalStateException("You have reached your specified API usage limits. Try again later."));
+
+        assertTrue(LlmBudgetExhaustionExtension.isUsageLimitError(failure));
+    }
+
+    @Test
+    void doesNotMaskOtherLlmFailures() {
+        Throwable failure = new AssertionError("Agent failed: invalid model");
+
+        assertFalse(LlmBudgetExhaustionExtension.isUsageLimitError(failure));
+    }
+
+    @Test
+    void abortsUsageLimitFailuresAndRethrowsOtherFailures() {
+        LlmBudgetExhaustionExtension extension = new LlmBudgetExhaustionExtension();
+        Throwable budgetFailure =
+                new AssertionError("You have reached your specified API usage limits.");
+        Throwable otherFailure = new AssertionError("Agent failed: invalid model");
+
+        assertThrows(
+                TestAbortedException.class,
+                () -> extension.handleTestExecutionException(null, budgetFailure));
+        Throwable rethrown = assertThrows(
+                Throwable.class,
+                () -> extension.handleTestExecutionException(null, otherFailure));
+        assertSame(otherFailure, rethrown);
+    }
+}