Fail E2E runs when cache cleanup is incomplete

scripts/e2e_eval/run_eval.py

@@ -290,16 +290,31 @@
     return any(pat in combined for pat in _NO_SPACE_PATTERNS)
 
 
-def _clear_disk_caches() -> None:
-    """Delete HuggingFace, WinML, VitisAI EP caches and leaked temp files."""
+def _remove_cache_dir(cache_dir: Path) -> bool:
+    """Remove one cache directory and report whether it is gone."""
+    safe_print(f"  [cleanup] Removing cache: {cache_dir}")
+    try:
+        shutil.rmtree(cache_dir)
+    except FileNotFoundError:
+        pass
+    except OSError as exc:
+        safe_print(f"  [cleanup] Warning: could not remove {cache_dir}: {exc}")
+        return False
+
+    removed = not cache_dir.exists()
+    if removed:
+        safe_print(f"  [cleanup] Removed: {cache_dir}")
+    else:
+        safe_print(f"  [cleanup] Warning: cache still exists after removal: {cache_dir}")
+    return removed
+
+
+def _clear_disk_caches() -> bool:
+    """Delete caches and leaked temp files, returning whether all caches were removed."""
+    caches_removed = True
     for cache_dir in (_HF_CACHE, _WINML_CACHE, _VAIP_CACHE):
         if cache_dir is not None and cache_dir.exists():
-            safe_print(f"  [cleanup] Removing cache: {cache_dir}")
-            try:
-                shutil.rmtree(cache_dir)
-                safe_print(f"  [cleanup] Removed: {cache_dir}")
-            except OSError as exc:
-                safe_print(f"  [cleanup] Warning: could not remove {cache_dir}: {exc}")
+            caches_removed = _remove_cache_dir(cache_dir) and caches_removed
 
     # Clean leaked temp directories/files (winmlcli_*, winmlcli_compat_*, tmp*.onnx*)
     if _TEMP_DIR.is_dir():
@@ -328,6 +343,7 @@
                     pass  # Best-effort cleanup; ignore if file is locked or already removed
         if cleaned:
             safe_print(f"  [cleanup] Removed {cleaned} leaked temp entries from {_TEMP_DIR}")
+    return caches_removed
 
 
 # Stray scratch files that onnxruntime/QNN/EP libraries write into the *process
@@ -3000,6 +3016,7 @@
     skipped = 0
     run_start = time.perf_counter()
     interrupted = False
+    cache_cleanup_failed = False
     timeout_skip_set = _load_timeout_skip_set()
     if timeout_skip_set:
         safe_print(f"Timeout skip list: {len(timeout_skip_set)} models will be auto-skipped")
@@ -3221,7 +3238,7 @@
 
         if args.clean_cache:
             _clean_stray_cwd_artifacts(Path.cwd())
-            _clear_disk_caches()
+            cache_cleanup_failed = not _clear_disk_caches() or cache_cleanup_failed
 
     run_duration = time.perf_counter() - run_start
 
@@ -3259,11 +3276,17 @@
             safe_print(f"  ({skipped} cached from previous run)")
         if interrupted:
             safe_print(f"  (interrupted — {total_jobs - len(results)} jobs not evaluated)")
+        if cache_cleanup_failed:
+            safe_print("  (cache cleanup failed — see cleanup warnings above)")
 
     all_perf_pass = all((r.get("perf") or {}).get("passed", False) for r in perf_results)
     all_acc_pass = all(accuracy_status(r.get("accuracy")) == "PASS" for r in acc_results)
 
-    sys.exit(0 if not interrupted and all_perf_pass and all_acc_pass else 1)
+    sys.exit(
+        0
+        if not interrupted and not cache_cleanup_failed and all_perf_pass and all_acc_pass
+        else 1
+    )
 
 
 if __name__ == "__main__":

tests/unit/eval/test_run_eval_script.py

@@ -1737,7 +1737,7 @@ def test_removes_hf_winml_and_vaip_caches(self, run_eval, tmp_path, monkeypatch)
         # Keep the temp-file sweep away from the real TEMP directory.
         monkeypatch.setattr(run_eval, "_TEMP_DIR", tmp_path / "empty_temp")
 
-        run_eval._clear_disk_caches()
+        assert run_eval._clear_disk_caches() is True
 
         assert not hf.exists()
         assert not winml.exists()
@@ -1753,6 +1753,24 @@ def test_skips_vaip_cache_when_unset(self, run_eval, tmp_path, monkeypatch):
         monkeypatch.setattr(run_eval, "_VAIP_CACHE", None)
         monkeypatch.setattr(run_eval, "_TEMP_DIR", tmp_path / "empty_temp")
 
-        run_eval._clear_disk_caches()
+        assert run_eval._clear_disk_caches() is True
 
         assert not winml.exists()
+
+    def test_reports_persistent_cache_lock(self, run_eval, tmp_path, monkeypatch, capsys):
+        vaip = tmp_path / "vaip" / ".cache"
+        vaip.mkdir(parents=True)
+
+        monkeypatch.setattr(run_eval, "_HF_CACHE", tmp_path / "missing_hf")
+        monkeypatch.setattr(run_eval, "_WINML_CACHE", tmp_path / "missing_winml")
+        monkeypatch.setattr(run_eval, "_VAIP_CACHE", vaip)
+        monkeypatch.setattr(run_eval, "_TEMP_DIR", tmp_path / "empty_temp")
+
+        def persistent_lock(_path):
+            raise PermissionError("cache file is locked")
+
+        monkeypatch.setattr(run_eval.shutil, "rmtree", persistent_lock)
+
+        assert run_eval._clear_disk_caches() is False
+        assert vaip.exists()
+        assert "could not remove" in capsys.readouterr().out