diff --git a/CHANGELOG.md b/CHANGELOG.md index 744a8d8..05a09f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/). --- +### Unreleased + +- **Failure classification** — every failed test is tagged with a category (`LOCATOR_NOT_FOUND`, `ELEMENT_NOT_INTERACTABLE`, `STALE_ELEMENT`, `TIMEOUT`, `ASSERTION_MISMATCH`, `NAVIGATION_ERROR`, `BROWSER_CRASH`, or `UNKNOWN`) and the evidence for it, from the exception class chain and message alone. No AI, no API key, no network. Shown as a badge in the HTML report's failure panel, as `failureCategory` / `failureEvidence` in the metrics JSON, and as the `type` of `` in the JUnit XML. The raw error and stack trace are unchanged. (#139) + +--- + ### v3.8.0 — 2026-10-11 - **`Locator.first()` and `Locator.last()`** — narrow matches to the first or last matching element (e.g. `getByRole(Role.LISTITEM).first()`). Both resolve at action time, re-evaluating candidate elements on every poll and retrying stale/not-yet-present elements up to `timeouts.explicit`. (#132) diff --git a/docs-site/docs/changelog.md b/docs-site/docs/changelog.md index ed4bf9a..bea6b37 100644 --- a/docs-site/docs/changelog.md +++ b/docs-site/docs/changelog.md @@ -11,6 +11,12 @@ All notable changes to Selenium Boot are documented here. --- +## [Unreleased] + +- **Failure classification** — every failed test is tagged with a category (`LOCATOR_NOT_FOUND`, `ELEMENT_NOT_INTERACTABLE`, `STALE_ELEMENT`, `TIMEOUT`, `ASSERTION_MISMATCH`, `NAVIGATION_ERROR`, `BROWSER_CRASH`, or `UNKNOWN`) and the evidence for it, from the exception class chain and message alone. No AI, no API key, no network. Shown as a badge in the HTML report's failure panel, as `failureCategory` / `failureEvidence` in the metrics JSON, and as the `type` of `` in the JUnit XML. The raw error and stack trace are unchanged. (#139) + +--- + ## [3.8.0] — 2026-10-11 - **`Locator.first()` and `Locator.last()`** — narrow matches to the first or last matching element (e.g. `getByRole(Role.LISTITEM).first()`). Both resolve at action time, re-evaluating candidate elements on every poll and retrying stale/not-yet-present elements up to `timeouts.explicit`. (#132) diff --git a/docs-site/docs/reporting/html-report.md b/docs-site/docs/reporting/html-report.md index 19bf3a4..6ba7821 100644 --- a/docs-site/docs/reporting/html-report.md +++ b/docs-site/docs/reporting/html-report.md @@ -59,6 +59,25 @@ Same detail panels as Test Cases tab, but only for failed tests. Detail panels a --- +## Failure classification + +Each failed test is tagged with a category and the evidence for it, worked out from the exception class chain and message the framework already records. It needs no AI, no API key and no network. The badge and evidence appear at the top of the failure's detail panel; the raw error and stack trace stay right below. + +| Category | Triggered by | +|---|---| +| `LOCATOR_NOT_FOUND` | `LocatorException`, `NoSuchElementException`, `NoSuchFrameException` or `NoSuchShadowRootException` anywhere in the cause chain, including a `LocatorException` caused by a `TimeoutException` | +| `ELEMENT_NOT_INTERACTABLE` | `ElementNotInteractableException`, `ElementClickInterceptedException`, `InvalidElementStateException` | +| `STALE_ELEMENT` | `StaleElementReferenceException` anywhere in the cause chain | +| `TIMEOUT` | a `TimeoutException` with no locator or navigation cause | +| `ASSERTION_MISMATCH` | the outermost error is an `AssertionError` (or TestNG/JUnit/`DbAssertException` equivalent), or soft assertions failed | +| `NAVIGATION_ERROR` | `net::ERR_*`, DNS failures, "Timed out receiving message from renderer", page load timeouts, HTTP 5xx in the message | +| `BROWSER_CRASH` | `NoSuchSessionException`, `UnreachableBrowserException`, "invalid session id", page crash | +| `UNKNOWN` | none of the above. The classifier never guesses | + +The category is also written to the metrics JSON as `failureCategory` and `failureEvidence`, and to the JUnit XML as the `type` of ``. + +--- + ## Self-contained format All screenshots are Base64-encoded and embedded inline. The report is a single file you can: diff --git a/docs-site/docs/reporting/junit-xml.md b/docs-site/docs/reporting/junit-xml.md index 7d2ffa7..3b5a836 100644 --- a/docs-site/docs/reporting/junit-xml.md +++ b/docs-site/docs/reporting/junit-xml.md @@ -32,7 +32,8 @@ target/ + type="LOCATOR_NOT_FOUND"> + [LOCATOR_NOT_FOUND] NoSuchElementException in cause chain org.openqa.selenium.NoSuchElementException: no such element at com.example.tests.CheckoutTest.checkout_withInvalidCard(CheckoutTest.java:42) ... @@ -52,6 +53,8 @@ target/ The `message` attribute on `` contains the actual assertion message or exception message from your test — not a generic "Test failed" placeholder. +The `type` attribute is the [failure category](html-report.md#failure-classification) (`LOCATOR_NOT_FOUND`, `TIMEOUT`, `UNKNOWN`, …), and the first line of the body names the evidence it was chosen from. The original stack trace follows unchanged. + This means CI tools that display failure summaries (GitHub Actions, Jenkins, Azure Pipelines) show meaningful error descriptions without needing to open the full report. --- diff --git a/src/main/java/com/seleniumboot/failure/FailureCategory.java b/src/main/java/com/seleniumboot/failure/FailureCategory.java new file mode 100644 index 0000000..cae5c3c --- /dev/null +++ b/src/main/java/com/seleniumboot/failure/FailureCategory.java @@ -0,0 +1,23 @@ +package com.seleniumboot.failure; + +/** Why a test failed, as decided by {@link FailureClassifier}. */ +public enum FailureCategory { + LOCATOR_NOT_FOUND("Locator not found"), + ELEMENT_NOT_INTERACTABLE("Element not interactable"), + STALE_ELEMENT("Stale element"), + TIMEOUT("Timeout"), + ASSERTION_MISMATCH("Assertion mismatch"), + NAVIGATION_ERROR("Navigation error"), + BROWSER_CRASH("Browser crash"), + UNKNOWN("Unknown"); + + private final String label; + + FailureCategory(String label) { + this.label = label; + } + + public String label() { + return label; + } +} diff --git a/src/main/java/com/seleniumboot/failure/FailureClassifier.java b/src/main/java/com/seleniumboot/failure/FailureClassifier.java new file mode 100644 index 0000000..64ae4cf --- /dev/null +++ b/src/main/java/com/seleniumboot/failure/FailureClassifier.java @@ -0,0 +1,149 @@ +package com.seleniumboot.failure; + +import com.seleniumboot.metrics.TestTiming; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Tags a failed test with a {@link FailureCategory} and the evidence for it, using only the + * error message and stack trace the framework already records. No AI, no network. + * + *

The exception class chain is read from the printed stack trace (the outer throwable on the + * first line, each cause on a {@code Caused by:} line), so a {@code LocatorException} whose cause + * is a {@code TimeoutException} is a locator problem, while a bare {@code TimeoutException} is not. + * Anything the rules do not recognise is {@link FailureCategory#UNKNOWN}; the classifier never guesses. + */ +public final class FailureClassifier { + + /** Category plus the one-line reason it was chosen. */ + public record Result(FailureCategory category, String evidence) { } + + private static final Pattern THROWABLE_LINE = Pattern.compile( + "^(?:Caused by: )?((?:[A-Za-z_$][\\w$]*\\.)+[A-Za-z_$][\\w$]*|[A-Za-z_$][\\w$]*(?:Exception|Error))" + + "(?::\\s?(.*))?$"); + + private static final int EVIDENCE_MAX = 140; + + private FailureClassifier() { } + + public static Result classify(TestTiming timing) { + if (timing == null) return unknown("no failure data"); + return classify(timing.getErrorMessage(), timing.getStackTrace()); + } + + public static Result classify(String errorMessage, String stackTrace) { + List chain = causeChain(stackTrace); + String outer = chain.isEmpty() ? null : chain.get(0); + String haystack = ((errorMessage != null ? errorMessage : "") + "\n" + + (stackTrace != null ? stackTrace : "")).toLowerCase(Locale.ROOT); + + if (chain.isEmpty() && (errorMessage == null || errorMessage.isBlank())) { + return unknown("no error message or stack trace recorded"); + } + + // A failed assertion is the test doing its job, whatever text its message happens to contain. + if (outer != null && isAssertion(outer)) { + return new Result(FailureCategory.ASSERTION_MISMATCH, "assertion failed (" + simple(outer) + ")"); + } + + String hit = firstMatch(chain, "NoSuchSessionException", "UnreachableBrowserException", + "SessionNotCreatedException"); + if (hit != null) return result(FailureCategory.BROWSER_CRASH, simple(hit) + " in cause chain"); + String crash = containsAny(haystack, "invalid session id", "session deleted because of page crash", + "chrome not reachable", "not connected to devtools", "target frame detached", + "browser has closed the connection", "tab crashed"); + if (crash != null) return result(FailureCategory.BROWSER_CRASH, "message contains \"" + crash + "\""); + + String nav = containsAny(haystack, "net::err_", "err_name_not_resolved", "dnsnotfound", + "reached error page", "unknownhostexception", "timed out receiving message from renderer", + "page load timeout", "pageloadstrategy"); + if (nav != null) return result(FailureCategory.NAVIGATION_ERROR, "message contains \"" + nav + "\""); + Matcher http = Pattern.compile("(?:http|status)[ :=]+(5\\d\\d)\\b").matcher(haystack); + if (http.find()) return result(FailureCategory.NAVIGATION_ERROR, "HTTP " + http.group(1) + " in message"); + + hit = firstMatch(chain, "StaleElementReferenceException"); + if (hit != null) return result(FailureCategory.STALE_ELEMENT, simple(hit) + " in cause chain"); + + hit = firstMatch(chain, "ElementNotInteractableException", "ElementClickInterceptedException", + "InvalidElementStateException", "ElementNotSelectableException"); + if (hit != null) return result(FailureCategory.ELEMENT_NOT_INTERACTABLE, simple(hit) + " in cause chain"); + + hit = firstMatch(chain, "LocatorException", "NoSuchElementException", "NoSuchFrameException", + "NoSuchShadowRootException"); + if (hit != null) return result(FailureCategory.LOCATOR_NOT_FOUND, simple(hit) + " in cause chain"); + + hit = firstMatch(chain, "TimeoutException"); + if (hit != null) return result(FailureCategory.TIMEOUT, simple(hit) + " with no locator or navigation cause"); + + for (String c : chain) { + if (isAssertion(c)) { + return result(FailureCategory.ASSERTION_MISMATCH, simple(c) + " in cause chain"); + } + } + if (haystack.contains("the following asserts failed")) { + return result(FailureCategory.ASSERTION_MISMATCH, "soft assertions failed"); + } + + return unknown(outer != null ? "unrecognised " + simple(outer) : "unrecognised error"); + } + + /** Fully-qualified class names of the outer throwable followed by each cause, in order. */ + static List causeChain(String stackTrace) { + List chain = new ArrayList<>(); + if (stackTrace == null) return chain; + boolean first = true; + for (String line : stackTrace.split("\\R")) { + if (line.isEmpty() || Character.isWhitespace(line.charAt(0))) continue; + Matcher m = THROWABLE_LINE.matcher(line); + boolean cause = line.startsWith("Caused by: "); + if (first) { + first = false; + if (m.matches()) chain.add(m.group(1)); + else return chain; + } else if (cause && m.matches()) { + chain.add(m.group(1)); + } + } + return chain; + } + + private static boolean isAssertion(String className) { + String s = simple(className); + return s.equals("AssertionError") || s.equals("AssertionFailedError") + || s.equals("ComparisonFailure") || s.equals("DbAssertException") + || s.equals("MultipleFailuresError"); + } + + private static String firstMatch(List chain, String... simpleNames) { + for (String c : chain) { + for (String n : simpleNames) { + if (simple(c).equals(n)) return c; + } + } + return null; + } + + private static String containsAny(String haystack, String... needles) { + for (String n : needles) { + if (haystack.contains(n)) return n; + } + return null; + } + + private static String simple(String className) { + int i = Math.max(className.lastIndexOf('.'), className.lastIndexOf('$')); + return i >= 0 ? className.substring(i + 1) : className; + } + + private static Result result(FailureCategory c, String evidence) { + return new Result(c, evidence.length() > EVIDENCE_MAX ? evidence.substring(0, EVIDENCE_MAX) + "…" : evidence); + } + + private static Result unknown(String evidence) { + return new Result(FailureCategory.UNKNOWN, evidence); + } +} diff --git a/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java b/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java index a42317a..4bac2b9 100644 --- a/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java +++ b/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java @@ -348,6 +348,12 @@ public static void exportToJson() { if (timing.getStackTrace() != null) { testEntry.put("stackTrace", timing.getStackTrace()); } + if ("FAILED".equals(timing.getStatus())) { + com.seleniumboot.failure.FailureClassifier.Result failure = + com.seleniumboot.failure.FailureClassifier.classify(timing); + testEntry.put("failureCategory", failure.category().name()); + testEntry.put("failureEvidence", failure.evidence()); + } if (timing.getBrowser() != null) { testEntry.put("browser", timing.getBrowser()); } diff --git a/src/main/java/com/seleniumboot/reporting/HtmlReportGenerator.java b/src/main/java/com/seleniumboot/reporting/HtmlReportGenerator.java index a458470..492ae07 100644 --- a/src/main/java/com/seleniumboot/reporting/HtmlReportGenerator.java +++ b/src/main/java/com/seleniumboot/reporting/HtmlReportGenerator.java @@ -336,11 +336,12 @@ private static String buildRow(JsonNode test, int rowIndex, String groupId, bool ? test.get("performanceMetrics") : null; String stepsHtml = buildStepTimeline(test); - boolean hasDetail = errorMsg != null || stackTrace != null || !stepsHtml.isEmpty() + boolean hasDetail = errorMsg != null || stackTrace != null || test.has("failureCategory") || !stepsHtml.isEmpty() || recordingPath != null || aiAnalysis != null || perfNode != null; String detailRow = ""; if (hasDetail) { - String errorHtml = errorMsg != null ? "

" + escapeHtml(errorMsg) + "
" : ""; + String whyHtml = test.has("failureCategory") ? buildFailureWhy(test) : ""; + String errorHtml = whyHtml + (errorMsg != null ? "
" + escapeHtml(errorMsg) + "
" : ""); String traceHtml = stackTrace != null ? "
"    + escapeHtml(stackTrace) + "
" : ""; String recordingHtml = recordingPath != null ? buildRecordingCell(recordingPath) : ""; String traceLink = tracePath != null ? buildTraceLink(tracePath) : ""; @@ -384,6 +385,19 @@ private static String buildRow(JsonNode test, int rowIndex, String groupId, bool + detailRow; } + private static String buildFailureWhy(JsonNode test) { + String cat = test.get("failureCategory").asText(); + String label; + try { + label = com.seleniumboot.failure.FailureCategory.valueOf(cat).label(); + } catch (IllegalArgumentException e) { + label = cat; + } + String evidence = test.has("failureEvidence") ? test.get("failureEvidence").asText() : ""; + return "
" + escapeHtml(label) + "" + + escapeHtml(evidence) + "
"; + } + private static String buildFailureRows(JsonNode tests) { return buildFailureRows(tests, false); } diff --git a/src/main/java/com/seleniumboot/reporting/JUnitXmlReporter.java b/src/main/java/com/seleniumboot/reporting/JUnitXmlReporter.java index b75a4e6..9002501 100644 --- a/src/main/java/com/seleniumboot/reporting/JUnitXmlReporter.java +++ b/src/main/java/com/seleniumboot/reporting/JUnitXmlReporter.java @@ -100,8 +100,11 @@ private static void writeXml(Collection timings, long totalDurationM String msg = t.getErrorMessage() != null ? t.getErrorMessage() : "Test failed"; String trace = t.getStackTrace() != null ? t.getStackTrace() : t.getTestId() + " FAILED"; xml.append(">\n"); - xml.append(" ") - .append(escapeXml(trace)) + com.seleniumboot.failure.FailureClassifier.Result why = + com.seleniumboot.failure.FailureClassifier.classify(t); + xml.append(" ") + .append(escapeXml("[" + why.category().name() + "] " + why.evidence() + "\n" + trace)) .append("\n"); xml.append(" \n"); break; diff --git a/src/main/resources/report-template.html b/src/main/resources/report-template.html index a3093ba..73f6ee8 100644 --- a/src/main/resources/report-template.html +++ b/src/main/resources/report-template.html @@ -254,6 +254,8 @@ .detail-row td { padding: 0; } .detail-panel { padding: 16px 24px; background: var(--surface2); border-top: 2px solid var(--bad); } .error-msg { font-size: 13px; font-weight: 600; color: var(--bad); margin-bottom: 10px; } + .failure-why { font-size: 12px; color: var(--muted, #6b7280); margin-bottom: 8px; } + .failure-badge { display: inline-block; padding: 1px 8px; margin-right: 6px; border-radius: 10px; font-size: 11px; font-weight: 700; letter-spacing: .03em; color: var(--bad); border: 1px solid var(--bad); } .stack-trace { font-family: 'Roboto Mono', monospace; font-size: 12px; color: var(--code-text); background: var(--code-bg); padding: 14px; border-radius: 6px; diff --git a/src/test/java/com/seleniumboot/unit/FailureClassifierTest.java b/src/test/java/com/seleniumboot/unit/FailureClassifierTest.java new file mode 100644 index 0000000..52b06a7 --- /dev/null +++ b/src/test/java/com/seleniumboot/unit/FailureClassifierTest.java @@ -0,0 +1,137 @@ +package com.seleniumboot.unit; + +import com.seleniumboot.failure.FailureCategory; +import com.seleniumboot.failure.FailureClassifier; +import com.seleniumboot.metrics.TestTiming; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.io.PrintWriter; +import java.io.StringWriter; + +/** Pure-function tests for {@link FailureClassifier}; no browser or I/O. */ +public class FailureClassifierTest { + + private static FailureCategory classify(Throwable t) { + return result(t).category(); + } + + private static FailureClassifier.Result result(Throwable t) { + StringWriter sw = new StringWriter(); + t.printStackTrace(new PrintWriter(sw)); + TestTiming timing = new TestTiming("T.t", "main"); + timing.setErrorMessage(t.getMessage()); + timing.setStackTrace(sw.toString()); + return FailureClassifier.classify(timing); + } + + @Test + public void locatorException_withTimeoutCause_isLocatorNotFound() { + Throwable t = new com.seleniumboot.locator.LocatorException("No element for css=#x", + new org.openqa.selenium.TimeoutException("Expected condition failed")); + Assert.assertEquals(classify(t), FailureCategory.LOCATOR_NOT_FOUND); + } + + @Test + public void bareTimeoutException_isTimeout() { + Assert.assertEquals(classify(new org.openqa.selenium.TimeoutException("waiting for visibility")), + FailureCategory.TIMEOUT); + } + + @Test + public void timeoutFromPageLoad_isNavigationError() { + Assert.assertEquals(classify(new org.openqa.selenium.TimeoutException( + "timeout: Timed out receiving message from renderer: 300.000")), FailureCategory.NAVIGATION_ERROR); + } + + @Test + public void dnsError_isNavigationError() { + Assert.assertEquals(classify(new org.openqa.selenium.WebDriverException( + "unknown error: net::ERR_NAME_NOT_RESOLVED")), FailureCategory.NAVIGATION_ERROR); + } + + @Test + public void http5xx_isNavigationError() { + Assert.assertEquals(classify(new RuntimeException("Unexpected HTTP 503 from /health")), + FailureCategory.NAVIGATION_ERROR); + } + + @Test + public void noSuchElement_isLocatorNotFound() { + Assert.assertEquals(classify(new org.openqa.selenium.NoSuchElementException("no such element")), + FailureCategory.LOCATOR_NOT_FOUND); + } + + @Test + public void staleElement_isStale() { + Assert.assertEquals(classify(new org.openqa.selenium.StaleElementReferenceException("stale")), + FailureCategory.STALE_ELEMENT); + } + + @Test + public void staleElement_nestedInCause_isStale() { + Throwable t = new RuntimeException("click failed", + new org.openqa.selenium.StaleElementReferenceException("stale")); + Assert.assertEquals(classify(t), FailureCategory.STALE_ELEMENT); + } + + @Test + public void elementNotInteractable_isNotInteractable() { + Assert.assertEquals(classify(new org.openqa.selenium.ElementNotInteractableException("hidden")), + FailureCategory.ELEMENT_NOT_INTERACTABLE); + } + + @Test + public void clickIntercepted_isNotInteractable() { + Assert.assertEquals(classify(new org.openqa.selenium.ElementClickInterceptedException("covered")), + FailureCategory.ELEMENT_NOT_INTERACTABLE); + } + + @Test + public void assertionError_isAssertionMismatch() { + Assert.assertEquals(classify(new AssertionError("expected [a] but found [b]")), + FailureCategory.ASSERTION_MISMATCH); + } + + @Test + public void assertionError_mentioningTimeout_staysAssertionMismatch() { + Assert.assertEquals(classify(new AssertionError("expected \"Timed out\" but found \"OK\"")), + FailureCategory.ASSERTION_MISMATCH); + } + + @Test + public void noSuchSession_isBrowserCrash() { + Assert.assertEquals(classify(new org.openqa.selenium.NoSuchSessionException("invalid session id")), + FailureCategory.BROWSER_CRASH); + } + + @Test + public void pageCrash_isBrowserCrash() { + Assert.assertEquals(classify(new org.openqa.selenium.WebDriverException( + "unknown error: session deleted because of page crash")), FailureCategory.BROWSER_CRASH); + } + + @Test + public void unrecognisedError_isUnknown() { + Assert.assertEquals(classify(new IllegalStateException("something odd")), FailureCategory.UNKNOWN); + } + + @Test + public void noData_isUnknown() { + Assert.assertEquals(FailureClassifier.classify((TestTiming) null).category(), FailureCategory.UNKNOWN); + Assert.assertEquals(FailureClassifier.classify(null, null).category(), FailureCategory.UNKNOWN); + } + + @Test + public void evidence_namesTheMatchedClass() { + Assert.assertTrue(result(new org.openqa.selenium.StaleElementReferenceException("stale")) + .evidence().contains("StaleElementReferenceException")); + } + + @Test + public void multilineSeleniumMessage_isNotMistakenForCause() { + Throwable t = new org.openqa.selenium.WebDriverException( + "boom\nCommand: [abc, get {url=x}]\nBuild info: version: '4'"); + Assert.assertEquals(classify(t), FailureCategory.UNKNOWN); + } +}