test: skip agent e2e when LLM budget is exhausted

e2e/llm-budget-results-processor.cjs

@@ -0,0 +1,51 @@
+/* global module */
+
+const USAGE_LIMIT_MESSAGE = "reached your specified api usage limits";
+
+const isUsageLimitFailure = (failureMessages) =>
+  failureMessages.some((message) =>
+    String(message).toLowerCase().includes(USAGE_LIMIT_MESSAGE),
+  );
+
+const processResults = (results) => {
+  for (const suite of results.testResults) {
+    for (const assertion of suite.testResults) {
+      if (
+        assertion.status === "failed" &&
+        isUsageLimitFailure(assertion.failureMessages ?? [])
+      ) {
+        assertion.status = "pending";
+        assertion.failureMessages = [];
+        suite.numFailingTests -= 1;
+        suite.numPendingTests += 1;
+      }
+    }
+
+    if (suite.numFailingTests === 0 && !suite.testExecError) {
+      suite.failureMessage = null;
+    }
+  }
+
+  results.numFailedTests = results.testResults.reduce(
+    (count, suite) => count + suite.numFailingTests,
+    0,
+  );
+  results.numPendingTests = results.testResults.reduce(
+    (count, suite) => count + suite.numPendingTests,
+    0,
+  );
+  results.numFailedTestSuites = results.testResults.filter(
+    (suite) => suite.numFailingTests > 0 || suite.testExecError,
+  ).length;
+  results.numPassedTestSuites =
+    results.testResults.length - results.numFailedTestSuites;
+  results.success =
+    results.numFailedTests === 0 &&
+    results.numFailedTestSuites === 0 &&
+    results.numRuntimeErrorTestSuites === 0;
+
+  return results;
+};
+
+module.exports = processResults;
+module.exports.isUsageLimitFailure = isUsageLimitFailure;

jest.e2e.config.mjs

@@ -13,6 +13,7 @@ export default {
   // Upstream ran 3 vitest forks (credential names are unique per suite; 3
   // keeps server load manageable on the shared SQLite-backed Conductor).
   maxWorkers: 3,
+  testResultsProcessor: "<rootDir>/e2e/llm-budget-results-processor.cjs",
   // The package.json "jest-junit" block outranks these reporter options, so
   // the test:agent-e2e script pins JEST_JUNIT_OUTPUT_DIR/NAME env vars
   // (which outrank everything) to results/junit-e2e.xml.

src/agents/__tests__/llm-budget-results-processor.test.cjs

@@ -0,0 +1,55 @@
+/* eslint-disable @typescript-eslint/no-require-imports, no-undef */
+
+const { describe, expect, it } = require("@jest/globals");
+const processResults = require("../../../e2e/llm-budget-results-processor.cjs");
+
+const failure = (message) => ({
+  status: "failed",
+  failureMessages: [message],
+});
+
+const resultsWith = (...assertions) => ({
+  success: false,
+  numFailedTests: assertions.length,
+  numPendingTests: 0,
+  numFailedTestSuites: 1,
+  numPassedTestSuites: 0,
+  numRuntimeErrorTestSuites: 0,
+  testResults: [
+    {
+      failureMessage: "suite failed",
+      numFailingTests: assertions.length,
+      numPendingTests: 0,
+      testResults: assertions,
+    },
+  ],
+});
+
+describe("LLM budget results processor", () => {
+  it("converts the provider usage-limit failure to pending", () => {
+    const results = processResults(
+      resultsWith(
+        failure("You have reached your specified API usage limits. Try again later."),
+      ),
+    );
+
+    expect(results.success).toBe(true);
+    expect(results.numFailedTests).toBe(0);
+    expect(results.numPendingTests).toBe(1);
+    expect(results.testResults[0].testResults[0].status).toBe("pending");
+  });
+
+  it("preserves unrelated failures in a mixed run", () => {
+    const results = processResults(
+      resultsWith(
+        failure("You have reached your specified API usage limits."),
+        failure("Agent failed: invalid model"),
+      ),
+    );
+
+    expect(results.success).toBe(false);
+    expect(results.numFailedTests).toBe(1);
+    expect(results.numPendingTests).toBe(1);
+    expect(results.testResults[0].testResults[1].status).toBe("failed");
+  });
+});