diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bb07f2..4147da7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Unreleased +- **Failure grouping** — the report's Failures tab now opens with a "N tests, same cause" panel: failed tests are grouped by a signature of category, exception type, normalised message (ids, numbers and quoted values stripped) and top application stack frame. The flat per-class list is unchanged below it. With `ai.failureAnalysis` on, one Claude call is made per group; the other tests in the group reuse its analysis. The metrics JSON gains `failureSignature`. (#140) + - **Failure classification** — every failed test is tagged with a category (`LOCATOR_NOT_FOUND`, `ELEMENT_NOT_INTERACTABLE`, `STALE_ELEMENT`, `TIMEOUT`, `ASSERTION_MISMATCH`, `NAVIGATION_ERROR`, `BROWSER_CRASH`, or `UNKNOWN`) and the evidence for it, from the exception class chain and message alone. No AI, no API key, no network. Shown as a badge in the HTML report's failure panel, as `failureCategory` / `failureEvidence` in the metrics JSON, and as the `type` of `` in the JUnit XML. The raw error and stack trace are unchanged. (#139) --- diff --git a/docs-site/docs/changelog.md b/docs-site/docs/changelog.md index 751a9f1..f940300 100644 --- a/docs-site/docs/changelog.md +++ b/docs-site/docs/changelog.md @@ -11,6 +11,8 @@ All notable changes to Selenium Boot are documented here. ## [Unreleased] +- **Failure grouping** — the report's Failures tab now opens with a "N tests, same cause" panel: failed tests are grouped by a signature of category, exception type, normalised message (ids, numbers and quoted values stripped) and top application stack frame. The flat per-class list is unchanged below it. With `ai.failureAnalysis` on, one Claude call is made per group; the other tests in the group reuse its analysis. The metrics JSON gains `failureSignature`. (#140) + - **`Locator` form actions: `clear()`, `check()`, `uncheck()`, `isChecked()`, `selectOption(text)`, `selectOptionByValue(value)`, `inputValue()`** — form controls can now be manipulated and inspected directly on `Locator` without falling back to `BasePage` helpers or raw `WebElement`s. `check()` and `uncheck()` are idempotent (no click if already in the target state), `isChecked()` queries checked state without waiting, and terminal actions (`clear`, `check`, `uncheck`, `selectOption`, `selectOptionByValue`, `inputValue`) auto-wait, re-resolving from scratch on every poll to survive stale or late-appearing elements. (#133) --- diff --git a/docs-site/docs/reporting/html-report.md b/docs-site/docs/reporting/html-report.md index 6ba7821..509c7d9 100644 --- a/docs-site/docs/reporting/html-report.md +++ b/docs-site/docs/reporting/html-report.md @@ -78,6 +78,12 @@ The category is also written to the metrics JSON as `failureCategory` and `failu --- +### Same-cause groups + +When two or more failures share a cause, the Failures tab lists them above the flat list as "N tests, same cause", largest group first, so one outage does not read as forty problems. Tests are grouped by a signature made of the failure category, the exception type, the first line of the message with ids, numbers and quoted values replaced by placeholders, and the top application stack frame (class and method, not line number). The flat list below still shows every failure. With `ai.failureAnalysis` on, one API call is made per group and the other tests in it reuse that analysis. + +--- + ## Self-contained format All screenshots are Base64-encoded and embedded inline. The report is a single file you can: diff --git a/src/main/java/com/seleniumboot/ai/AiFailureAnalyzer.java b/src/main/java/com/seleniumboot/ai/AiFailureAnalyzer.java index 4d03f01..eac5d37 100644 --- a/src/main/java/com/seleniumboot/ai/AiFailureAnalyzer.java +++ b/src/main/java/com/seleniumboot/ai/AiFailureAnalyzer.java @@ -13,6 +13,9 @@ import java.net.http.HttpResponse; import java.time.Duration; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -46,8 +49,17 @@ public final class AiFailureAnalyzer { private static final String API_URL = "https://api.anthropic.com/v1/messages"; private static final String API_VERSION = "2023-06-01"; + /** Analysis per failure signature, so one cause costs one API call. */ + private static final ConcurrentHashMap> BY_SIGNATURE = + new ConcurrentHashMap<>(); + private AiFailureAnalyzer() {} + /** Test hook: forget cached analyses. */ + static void resetCache() { + BY_SIGNATURE.clear(); + } + // ------------------------------------------------------------------ // Public API — called by TestExecutionListener // ------------------------------------------------------------------ @@ -74,8 +86,26 @@ public static void analyze(String testId, String pageUrl, String pageTitle) { TestTiming timing = ExecutionMetrics.getTiming(testId); if (timing == null) return; - String prompt = buildPrompt(timing, pageUrl, pageTitle); - String analysis = callApi(apiKey, aiCfg.getModel(), prompt, aiCfg.getTimeoutSeconds()); + // One call per distinct cause: tests failing for the same reason share the analysis of + // whichever failed first. Concurrent same-cause failures wait for that one call. + String signature = com.seleniumboot.failure.FailureSignature.of(timing); + CompletableFuture mine = new CompletableFuture<>(); + CompletableFuture existing = BY_SIGNATURE.putIfAbsent(signature, mine); + String analysis; + if (existing != null) { + analysis = existing.get(aiCfg.getTimeoutSeconds() + 5L, TimeUnit.SECONDS); + if (analysis != null) LOG.info("[AiFailureAnalyzer] Reusing analysis for same cause: " + testId); + } else { + String result = null; + try { + result = callApi(apiKey, aiCfg.getModel(), buildPrompt(timing, pageUrl, pageTitle), + aiCfg.getTimeoutSeconds()); + } finally { + if (result == null || result.isBlank()) BY_SIGNATURE.remove(signature, mine); + mine.complete(result); + } + analysis = result; + } if (analysis != null && !analysis.isBlank()) { ExecutionMetrics.recordAiAnalysis(testId, analysis.strip()); LOG.info("[AiFailureAnalyzer] Analysis recorded for: " + testId); diff --git a/src/main/java/com/seleniumboot/failure/FailureClassifier.java b/src/main/java/com/seleniumboot/failure/FailureClassifier.java index 64ae4cf..076b2ab 100644 --- a/src/main/java/com/seleniumboot/failure/FailureClassifier.java +++ b/src/main/java/com/seleniumboot/failure/FailureClassifier.java @@ -91,6 +91,10 @@ public static Result classify(String errorMessage, String stackTrace) { return unknown(outer != null ? "unrecognised " + simple(outer) : "unrecognised error"); } + static List causeChainOf(String stackTrace) { + return causeChain(stackTrace); + } + /** Fully-qualified class names of the outer throwable followed by each cause, in order. */ static List causeChain(String stackTrace) { List chain = new ArrayList<>(); diff --git a/src/main/java/com/seleniumboot/failure/FailureSignature.java b/src/main/java/com/seleniumboot/failure/FailureSignature.java new file mode 100644 index 0000000..27939ba --- /dev/null +++ b/src/main/java/com/seleniumboot/failure/FailureSignature.java @@ -0,0 +1,78 @@ +package com.seleniumboot.failure; + +import com.seleniumboot.metrics.TestTiming; + +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A stable fingerprint of why a test failed, so that many failures with one cause + * collapse into one group. + * + *

The signature joins the {@link FailureCategory}, the outer exception type, the first line of + * the message with volatile parts (UUIDs, hex ids, numbers, quoted values) replaced by + * placeholders, and the top application stack frame (class and method, no line number). + * Two tests share a signature when all four match. + */ +public final class FailureSignature { + + private static final Pattern UUID = Pattern.compile( + "\\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\\b"); + private static final Pattern HEX = Pattern.compile("\\b(?:0x)?[0-9a-fA-F]{12,}\\b"); + private static final Pattern QUOTED = Pattern.compile("'[^']*'|\"[^\"]*\""); + private static final Pattern NUMBER = Pattern.compile("\\d+"); + private static final Pattern SPACES = Pattern.compile("\\s+"); + private static final Pattern FRAME = Pattern.compile("^\\s+at ([^(\\s]+)\\("); + + private static final List FRAMEWORK_PREFIXES = List.of( + "java.", "javax.", "jdk.", "sun.", "org.openqa.", "org.testng.", "org.junit.", + "io.cucumber.", "com.seleniumboot."); + + private static final int MESSAGE_MAX = 160; + + private FailureSignature() { } + + public static String of(TestTiming timing) { + return timing == null ? of(null, null) : of(timing.getErrorMessage(), timing.getStackTrace()); + } + + public static String of(String errorMessage, String stackTrace) { + FailureCategory category = FailureClassifier.classify(errorMessage, stackTrace).category(); + List chain = FailureClassifier.causeChainOf(stackTrace); + String type = chain.isEmpty() ? "?" : chain.get(0); + return category.name() + " | " + type + " | " + normalise(errorMessage) + " | " + topAppFrame(stackTrace); + } + + /** First message line, with ids, numbers and quoted values replaced so recurring failures match. */ + static String normalise(String message) { + if (message == null) return ""; + String first = message.strip(); + int nl = first.indexOf('\n'); + if (nl >= 0) first = first.substring(0, nl); + first = UUID.matcher(first).replaceAll(""); + first = HEX.matcher(first).replaceAll(""); + first = QUOTED.matcher(first).replaceAll(""); + first = NUMBER.matcher(first).replaceAll("#"); + first = SPACES.matcher(first).replaceAll(" ").strip(); + return first.length() > MESSAGE_MAX ? first.substring(0, MESSAGE_MAX) : first; + } + + /** + * Class and method of the first stack frame outside the JDK, Selenium, the test runner and the + * framework: the page object or test code that was running. Falls back to the first frame. + */ + static String topAppFrame(String stackTrace) { + if (stackTrace == null) return ""; + String firstFrame = null; + for (String line : stackTrace.split("\\R")) { + if (line.startsWith("Caused by: ")) break; + Matcher m = FRAME.matcher(line); + if (!m.find()) continue; + String frame = m.group(1); + if (firstFrame == null) firstFrame = frame; + if (FRAMEWORK_PREFIXES.stream().noneMatch(frame::startsWith)) return frame; + } + return firstFrame != null ? firstFrame : ""; + } +} diff --git a/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java b/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java index 4bac2b9..b0831f1 100644 --- a/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java +++ b/src/main/java/com/seleniumboot/metrics/ExecutionMetrics.java @@ -353,6 +353,7 @@ public static void exportToJson() { com.seleniumboot.failure.FailureClassifier.classify(timing); testEntry.put("failureCategory", failure.category().name()); testEntry.put("failureEvidence", failure.evidence()); + testEntry.put("failureSignature", com.seleniumboot.failure.FailureSignature.of(timing)); } if (timing.getBrowser() != null) { testEntry.put("browser", timing.getBrowser()); diff --git a/src/main/java/com/seleniumboot/reporting/HtmlReportGenerator.java b/src/main/java/com/seleniumboot/reporting/HtmlReportGenerator.java index 492ae07..8a4277b 100644 --- a/src/main/java/com/seleniumboot/reporting/HtmlReportGenerator.java +++ b/src/main/java/com/seleniumboot/reporting/HtmlReportGenerator.java @@ -234,6 +234,7 @@ static String buildHtml(JsonNode root) { String slowestSection = tests != null ? buildSlowestTests(tests) : ""; String flakinessSection = buildFlakinessSection(); String failureRows = buildFailureRows(tests, isMatrixRun); + String failureGroups = buildFailureGroups(tests); String failureBadge = failedTests > 0 ? "" + failedTests + "" : ""; @@ -265,6 +266,7 @@ static String buildHtml(JsonNode root) { .replace("{{DONUT_DATA}}", donutData) .replace("{{ROWS}}", rows.toString()) .replace("{{FAILURE_ROWS}}", failureRows) + .replace("{{FAILURE_GROUPS}}", failureGroups) .replace("{{FAILURE_BADGE}}", failureBadge) .replace("{{FAILURES_SHORTCUT}}", failuresShortcut) .replace("{{EXECUTION_PERCENTILES}}", executionPercentiles.replace("'", "\\'")); @@ -436,6 +438,55 @@ private static String buildFailureRows(JsonNode tests, boolean showBrowser) { return rows.toString(); } + /** + * "N tests, same cause" panel shown above the flat failure list. Only causes shared by two or + * more tests are listed, largest first; the flat list below still shows every failure. + */ + private static String buildFailureGroups(JsonNode tests) { + if (tests == null) return ""; + java.util.Map> bySignature = new java.util.LinkedHashMap<>(); + for (JsonNode test : tests) { + if (!"FAILED".equals(test.has("status") ? test.get("status").asText() : "")) continue; + String signature = test.has("failureSignature") + ? test.get("failureSignature").asText() + : com.seleniumboot.failure.FailureSignature.of( + test.has("errorMessage") ? test.get("errorMessage").asText() : null, + test.has("stackTrace") ? test.get("stackTrace").asText() : null); + bySignature.computeIfAbsent(signature, k -> new java.util.ArrayList<>()).add(test); + } + java.util.List> groups = new java.util.ArrayList<>(); + for (java.util.List g : bySignature.values()) { + if (g.size() >= 2) groups.add(g); + } + if (groups.isEmpty()) return ""; + groups.sort((a, b) -> Integer.compare(b.size(), a.size())); + + StringBuilder sb = new StringBuilder("

") + .append("
Same cause (") + .append(groups.size()).append(groups.size() == 1 ? " group" : " groups").append(")
") + .append("
"); + for (java.util.List g : groups) { + JsonNode first = g.get(0); + sb.append("
"); + if (first.has("failureCategory")) { + sb.append(buildFailureWhy(first)).append(""); + } + sb.append("").append(g.size()).append(" tests, same cause
"); + String msg = first.has("errorMessage") ? first.get("errorMessage").asText() : ""; + int nl = msg.indexOf('\n'); + if (nl >= 0) msg = msg.substring(0, nl); + if (!msg.isEmpty()) { + sb.append("
").append(escapeHtml(msg)).append("
"); + } + sb.append("
    "); + for (JsonNode t : g) { + sb.append("
  • ").append(escapeHtml(t.has("testId") ? t.get("testId").asText() : "")).append("
    • "); + } + sb.append("
"); + } + return sb.append("
").toString(); + } + private static String noFailuresRow() { return "✓ No failures in this run"; } diff --git a/src/main/resources/report-template.html b/src/main/resources/report-template.html index 73f6ee8..150aed7 100644 --- a/src/main/resources/report-template.html +++ b/src/main/resources/report-template.html @@ -254,6 +254,11 @@ .detail-row td { padding: 0; } .detail-panel { padding: 16px 24px; background: var(--surface2); border-top: 2px solid var(--bad); } .error-msg { font-size: 13px; font-weight: 600; color: var(--bad); margin-bottom: 10px; } + .failure-group { padding: 10px 0; border-bottom: 1px solid var(--border); } + .failure-group:last-child { border-bottom: 0; } + .failure-group-title { font-size: 14px; margin-bottom: 4px; } + .failure-group-msg { font-size: 13px; color: var(--bad); margin-bottom: 6px; word-break: break-word; } + .failure-group-tests { margin: 0; padding-left: 20px; font-size: 12px; color: var(--text-secondary); } .failure-why { font-size: 12px; color: var(--muted, #6b7280); margin-bottom: 8px; } .failure-badge { display: inline-block; padding: 1px 8px; margin-right: 6px; border-radius: 10px; font-size: 11px; font-weight: 700; letter-spacing: .03em; color: var(--bad); border: 1px solid var(--bad); } .stack-trace { @@ -479,6 +484,7 @@

Selenium Boot Report

Failures ({{FAILED}} failed)
+ {{FAILURE_GROUPS}}
diff --git a/src/test/java/com/seleniumboot/unit/FailureGroupsReportTest.java b/src/test/java/com/seleniumboot/unit/FailureGroupsReportTest.java new file mode 100644 index 0000000..68c1113 --- /dev/null +++ b/src/test/java/com/seleniumboot/unit/FailureGroupsReportTest.java @@ -0,0 +1,43 @@ +package com.seleniumboot.unit; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.lang.reflect.Method; + +/** The failures tab lists shared causes above the flat list. */ +public class FailureGroupsReportTest { + + private static String groups(String json) throws Exception { + com.fasterxml.jackson.databind.JsonNode tests = new com.fasterxml.jackson.databind.ObjectMapper().readTree(json); + Method m = com.seleniumboot.reporting.HtmlReportGenerator.class + .getDeclaredMethod("buildFailureGroups", com.fasterxml.jackson.databind.JsonNode.class); + m.setAccessible(true); + return (String) m.invoke(null, tests); + } + + private static String failed(String id, String signature) { + return "{\"testId\":\"" + id + "\",\"status\":\"FAILED\",\"errorMessage\":\"gone\"," + + "\"failureCategory\":\"LOCATOR_NOT_FOUND\",\"failureEvidence\":\"x\"," + + "\"failureSignature\":\"" + signature + "\"}"; + } + + @Test + public void sharedCause_isListedWithCount() throws Exception { + String html = groups("[" + failed("A.a", "s1") + "," + failed("B.b", "s1") + "," + failed("C.c", "s2") + "]"); + Assert.assertTrue(html.contains("2 tests, same cause"), html); + Assert.assertTrue(html.contains("A.a") && html.contains("B.b"), html); + Assert.assertFalse(html.contains("C.c"), html); + } + + @Test + public void noSharedCause_rendersNothing() throws Exception { + Assert.assertEquals(groups("[" + failed("A.a", "s1") + "," + failed("C.c", "s2") + "]"), ""); + } + + @Test + public void passedTestsAreIgnored() throws Exception { + Assert.assertEquals(groups("[{\"testId\":\"A.a\",\"status\":\"PASSED\",\"failureSignature\":\"s1\"}," + + "{\"testId\":\"B.b\",\"status\":\"PASSED\",\"failureSignature\":\"s1\"}]"), ""); + } +} diff --git a/src/test/java/com/seleniumboot/unit/FailureSignatureTest.java b/src/test/java/com/seleniumboot/unit/FailureSignatureTest.java new file mode 100644 index 0000000..824fab1 --- /dev/null +++ b/src/test/java/com/seleniumboot/unit/FailureSignatureTest.java @@ -0,0 +1,81 @@ +package com.seleniumboot.unit; + +import com.seleniumboot.failure.FailureSignature; +import org.testng.Assert; +import org.testng.annotations.Test; + +/** Signature normalisation: same cause groups together, different cause does not. */ +public class FailureSignatureTest { + + private static String trace(String type, String msg, String frame) { + return type + ": " + msg + "\n" + + "\tat org.openqa.selenium.remote.RemoteWebDriver.findElement(RemoteWebDriver.java:300)\n" + + "\tat " + frame + "(Page.java:42)\n" + + "\tat com.example.tests.LoginTest.run(LoginTest.java:10)\n"; + } + + private static String sig(String type, String msg, String frame) { + return FailureSignature.of(msg, trace(type, msg, frame)); + } + + @Test + public void sameCause_differentNumbersAndIds_sameSignature() { + String a = sig("org.openqa.selenium.NoSuchElementException", + "no such element: #order-1042 after 5000ms (session 3f2a9c1e-1111-2222-3333-444455556666)", + "com.example.pages.OrdersPage.open"); + String b = sig("org.openqa.selenium.NoSuchElementException", + "no such element: #order-77 after 12000ms (session aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee)", + "com.example.pages.OrdersPage.open"); + Assert.assertEquals(a, b); + } + + @Test + public void quotedValuesAreIgnored() { + Assert.assertEquals( + sig("java.lang.AssertionError", "expected 'Alice' but found 'Bob'", "com.example.A.m"), + sig("java.lang.AssertionError", "expected 'Carol' but found 'Dan'", "com.example.A.m")); + } + + @Test + public void differentMessage_differentSignature() { + Assert.assertNotEquals( + sig("org.openqa.selenium.NoSuchElementException", "no such element: login button", "com.example.A.m"), + sig("org.openqa.selenium.NoSuchElementException", "no such element: logout link", "com.example.A.m")); + } + + @Test + public void differentCategory_differentSignature() { + Assert.assertNotEquals( + sig("org.openqa.selenium.StaleElementReferenceException", "boom", "com.example.A.m"), + sig("org.openqa.selenium.ElementNotInteractableException", "boom", "com.example.A.m")); + } + + @Test + public void differentApplicationFrame_differentSignature() { + Assert.assertNotEquals( + sig("org.openqa.selenium.NoSuchElementException", "gone", "com.example.pages.LoginPage.submit"), + sig("org.openqa.selenium.NoSuchElementException", "gone", "com.example.pages.CartPage.submit")); + } + + @Test + public void lineNumberShift_doesNotSplitGroup() { + String a = FailureSignature.of("gone", "org.openqa.selenium.NoSuchElementException: gone\n" + + "\tat com.example.pages.LoginPage.submit(LoginPage.java:10)\n"); + String b = FailureSignature.of("gone", "org.openqa.selenium.NoSuchElementException: gone\n" + + "\tat com.example.pages.LoginPage.submit(LoginPage.java:99)\n"); + Assert.assertEquals(a, b); + } + + @Test + public void frameworkFramesAreSkipped() { + String s = FailureSignature.of("gone", "org.openqa.selenium.NoSuchElementException: gone\n" + + "\tat com.seleniumboot.locator.Locator.click(Locator.java:1)\n" + + "\tat com.example.pages.LoginPage.submit(LoginPage.java:10)\n"); + Assert.assertTrue(s.endsWith("| com.example.pages.LoginPage.submit"), s); + } + + @Test + public void noData_doesNotThrow() { + Assert.assertNotNull(FailureSignature.of(null, null)); + } +}