diff --git a/e2e/llm-budget-results-processor.cjs b/e2e/llm-budget-results-processor.cjs new file mode 100644 index 00000000..c297dc6d --- /dev/null +++ b/e2e/llm-budget-results-processor.cjs @@ -0,0 +1,51 @@ +/* global module */ + +const USAGE_LIMIT_MESSAGE = "reached your specified api usage limits"; + +const isUsageLimitFailure = (failureMessages) => + failureMessages.some((message) => + String(message).toLowerCase().includes(USAGE_LIMIT_MESSAGE), + ); + +const processResults = (results) => { + for (const suite of results.testResults) { + for (const assertion of suite.testResults) { + if ( + assertion.status === "failed" && + isUsageLimitFailure(assertion.failureMessages ?? []) + ) { + assertion.status = "pending"; + assertion.failureMessages = []; + suite.numFailingTests -= 1; + suite.numPendingTests += 1; + } + } + + if (suite.numFailingTests === 0 && !suite.testExecError) { + suite.failureMessage = null; + } + } + + results.numFailedTests = results.testResults.reduce( + (count, suite) => count + suite.numFailingTests, + 0, + ); + results.numPendingTests = results.testResults.reduce( + (count, suite) => count + suite.numPendingTests, + 0, + ); + results.numFailedTestSuites = results.testResults.filter( + (suite) => suite.numFailingTests > 0 || suite.testExecError, + ).length; + results.numPassedTestSuites = + results.testResults.length - results.numFailedTestSuites; + results.success = + results.numFailedTests === 0 && + results.numFailedTestSuites === 0 && + results.numRuntimeErrorTestSuites === 0; + + return results; +}; + +module.exports = processResults; +module.exports.isUsageLimitFailure = isUsageLimitFailure; diff --git a/jest.e2e.config.mjs b/jest.e2e.config.mjs index 7ef48c0b..1207594a 100644 --- a/jest.e2e.config.mjs +++ b/jest.e2e.config.mjs @@ -13,6 +13,7 @@ export default { // Upstream ran 3 vitest forks (credential names are unique per suite; 3 // keeps server load manageable on the shared SQLite-backed Conductor). maxWorkers: 3, + testResultsProcessor: "/e2e/llm-budget-results-processor.cjs", // The package.json "jest-junit" block outranks these reporter options, so // the test:agent-e2e script pins JEST_JUNIT_OUTPUT_DIR/NAME env vars // (which outrank everything) to results/junit-e2e.xml. diff --git a/src/agents/__tests__/llm-budget-results-processor.test.cjs b/src/agents/__tests__/llm-budget-results-processor.test.cjs new file mode 100644 index 00000000..af26a7d3 --- /dev/null +++ b/src/agents/__tests__/llm-budget-results-processor.test.cjs @@ -0,0 +1,55 @@ +/* eslint-disable @typescript-eslint/no-require-imports, no-undef */ + +const { describe, expect, it } = require("@jest/globals"); +const processResults = require("../../../e2e/llm-budget-results-processor.cjs"); + +const failure = (message) => ({ + status: "failed", + failureMessages: [message], +}); + +const resultsWith = (...assertions) => ({ + success: false, + numFailedTests: assertions.length, + numPendingTests: 0, + numFailedTestSuites: 1, + numPassedTestSuites: 0, + numRuntimeErrorTestSuites: 0, + testResults: [ + { + failureMessage: "suite failed", + numFailingTests: assertions.length, + numPendingTests: 0, + testResults: assertions, + }, + ], +}); + +describe("LLM budget results processor", () => { + it("converts the provider usage-limit failure to pending", () => { + const results = processResults( + resultsWith( + failure("You have reached your specified API usage limits. Try again later."), + ), + ); + + expect(results.success).toBe(true); + expect(results.numFailedTests).toBe(0); + expect(results.numPendingTests).toBe(1); + expect(results.testResults[0].testResults[0].status).toBe("pending"); + }); + + it("preserves unrelated failures in a mixed run", () => { + const results = processResults( + resultsWith( + failure("You have reached your specified API usage limits."), + failure("Agent failed: invalid model"), + ), + ); + + expect(results.success).toBe(false); + expect(results.numFailedTests).toBe(1); + expect(results.numPendingTests).toBe(1); + expect(results.testResults[0].testResults[1].status).toBe("failed"); + }); +});