From 0da941e6ff6465741fe2f6b8b2ae360f813bfa9c Mon Sep 17 00:00:00 2001 From: wenytang-ms Date: Thu, 3 Sep 2026 10:09:15 +0800 Subject: [PATCH 1/2] Skip flaky LLM checks in Maven resolve UI test --- test-plans/java-maven-resolve-type.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test-plans/java-maven-resolve-type.yaml b/test-plans/java-maven-resolve-type.yaml index 477bc100..f37112f6 100644 --- a/test-plans/java-maven-resolve-type.yaml +++ b/test-plans/java-maven-resolve-type.yaml @@ -75,6 +75,7 @@ steps: atLeast: true waitBefore: 8 timeout: 60 + skipLlmVerify: true # verifyEditor and verifyProblems are authoritative. # Close all editors before modifying pom.xml on disk. Having pom.xml # open in the editor while `insertLineInFile` writes to disk can leave @@ -122,6 +123,7 @@ steps: contains: "com.google.code.gson" waitBefore: 3 timeout: 10 + skipLlmVerify: true # verifyEditor is authoritative; screenshot rendering can lag. # Explicitly trigger a Maven re-import so the newly-added gson dependency is # picked up on the classpath. With `java.configuration.updateBuildConfiguration: From 4278c1e4ae5e8f7a6888b1b871a2c735536a2e66 Mon Sep 17 00:00:00 2001 From: wenytang-ms Date: Thu, 3 Sep 2026 10:30:03 +0800 Subject: [PATCH 2/2] Strengthen deterministic Maven resolve checks --- test-plans/java-maven-resolve-type.yaml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/test-plans/java-maven-resolve-type.yaml b/test-plans/java-maven-resolve-type.yaml index f37112f6..56715230 100644 --- a/test-plans/java-maven-resolve-type.yaml +++ b/test-plans/java-maven-resolve-type.yaml @@ -56,6 +56,8 @@ steps: - id: "open-app" action: "open file App.java" verify: "App.java file is open in the editor" + verifyProblems: + errors: 0 waitBefore: 5 timeout: 15 @@ -63,19 +65,22 @@ steps: # wiki: "type 'Gson gson;'" — line 4 places the field inside the class body. # The Problems panel is not auto-opened by autotest and the red squiggle # may take a moment to render, so verify text describes only the - # inserted code line. The deterministic verifyProblems.errors >= 1 - # polls the diagnostics API and is the ground truth for the LS error. + # inserted code line. The deterministic file assertion proves the edit + # landed inside the class, while the exact error count establishes the + # expected 0 → 1 diagnostic transition. - id: "insert-unknown-type" action: "insertLineInFile src/main/java/com/example/App.java 4 Gson gson;" verify: "App.java editor now shows the inserted 'Gson gson;' declaration inside the class body" + verifyFile: + path: "~/src/main/java/com/example/App.java" + matches: 'public class App \{\r?\n[ \t]*Gson gson;' verifyEditor: contains: "Gson gson;" verifyProblems: errors: 1 - atLeast: true waitBefore: 8 timeout: 60 - skipLlmVerify: true # verifyEditor and verifyProblems are authoritative. + skipLlmVerify: true # File, editor, and diagnostic checks are authoritative. # Close all editors before modifying pom.xml on disk. Having pom.xml # open in the editor while `insertLineInFile` writes to disk can leave @@ -121,9 +126,11 @@ steps: verify: "pom.xml is open in the editor and shows the inserted block referencing com.google.code.gson" verifyEditor: contains: "com.google.code.gson" + verifyEditorTab: + title: "pom.xml" waitBefore: 3 timeout: 10 - skipLlmVerify: true # verifyEditor is authoritative; screenshot rendering can lag. + skipLlmVerify: true # Editor content and visible tab checks are authoritative. # Explicitly trigger a Maven re-import so the newly-added gson dependency is # picked up on the classpath. With `java.configuration.updateBuildConfiguration: