Filter invalid filing destinations and allow review after fee errors

docs/docs/partners-courts/jurisdiction-config.md

@@ -246,7 +246,7 @@ A `courts:` query says which courts an answer leads to. Every rule is optional a
 | `exclude_code_pattern`, `exclude_name_pattern` | Drop the courts these match. |
 | `group` | The court belongs to this group — see `group_by` below. |
 
-Courts that are only a heading over the courts beneath them never reach the questions at all. "Cook County" is such a row: every Cook filing goes to one of the eighty locations whose code hangs off it, and choosing the county itself returns an empty case-category list with nothing to explain why. A court is dropped only when the e-filing service leaves it out of its fileable list **and** other courts hang off its code — Cook County - Chancery fails the second test, has locations under it, and takes filings of its own. A caption that names only such a county still routes: the questions it does settle are filled in, and `default_by` can start the rest somewhere sensible.
+Courts that are only a heading over the courts beneath them never reach the questions at all. "Cook County" is such a row: every Cook filing goes to one of the eighty locations whose code hangs off it, and choosing the county itself returns an empty case-category list with nothing to explain why. Cook County's divisions are headings too: a filing goes to a district location such as Cook County - Municipal Civil - District 1, and the e-filing service refuses one sent to Municipal Civil itself. A court is dropped only when the e-filing service leaves it out of its fileable list **and** a court that list does include hangs off its code. The fileable list alone is not enough, because it also leaves out courts that take filings, such as most Massachusetts District Courts. The same rule applies to the court lists, to filing code search, and to the check made when a search result is chosen. A caption that names only such a county still routes: the questions it does settle are filled in, and `default_by` can start the rest somewhere sensible.
 
 Prefer `name_pattern`. Court **names** are stable and readable; Tyler's codes differ from court to court and change without notice. `{value}`, and any earlier step's id, can be used as a placeholder inside a query.
 

efile_app/efile/api/dropdown_views.py

@@ -11,7 +11,7 @@
 from django.conf import settings
 from django.views.decorators.http import require_http_methods
 
-from efile.services.court_selection import is_non_filing_court
+from efile.services.court_selection import heading_court_codes, is_non_filing_court
 from efile.services.efsp_payload import parse_optional_services
 from efile.utils.jurisdiction_stuff import get_jurisdiction_from_request
 
@@ -286,12 +286,15 @@ def get_courts(request):
                     # Transform API data to our dropdown format
                     courts = []
                     if isinstance(api_data, list):
+                        headings = heading_court_codes(
+                            jurisdiction, [str(court.get("code")) for court in api_data if isinstance(court, dict)]
+                        )
                         for court in api_data:
                             if isinstance(court, dict) and "code" in court and "name" in court:
                                 # Rows that exist only inside Tyler -- test
                                 # fixtures, retired locations -- are not courts
                                 # anyone can file into.
-                                if is_non_filing_court(court["name"]):
+                                if is_non_filing_court(court["name"]) or str(court["code"]) in headings:
                                     continue
 
                                 courts.append({"value": court["code"], "text": court["name"]})

efile_app/efile/api/filing_code_search.py

@@ -52,7 +52,7 @@ def filing_code_search(request):
             path_id = int(request.GET["path_id"])
         except ValueError:
             return JsonResponse({"error": "Invalid filing path."}, status=400)
-        path = index.paths.filter(pk=path_id, initial=initial).select_related("index").first()
+        path = index.paths.filter(pk=path_id, initial=initial).first()
         if path is None:
             return JsonResponse({"error": "The code list has changed. Search again."}, status=409)
         try:

efile_app/efile/services/court_selection.py

@@ -64,6 +64,8 @@
 # How long the live court list is reused for. It changes a few times a year, and
 # the selector re-reads it on every answer the filer gives.
 COURT_LIST_TTL_SECONDS = 600
+# How long "nothing is dropped" stands after a court list could not be read.
+HEADING_COURTS_RETRY_SECONDS = 60
 
 _PLACEHOLDER = re.compile(r"\{(\w+)\}")
 
@@ -112,8 +114,10 @@ def fetch_courts(jurisdiction: str) -> list[dict[str, str]]:
         if isinstance(court, dict) and court.get("code") and court.get("name")
         if not is_non_filing_court(court["name"])
     ]
-    fileable = _fileable_codes(jurisdiction)
-    courts = [court for court in courts if not _is_heading_only(court, courts, fileable)]
+    headings = heading_court_codes(
+        jurisdiction, [str(court.get("code")) for court in payload if isinstance(court, dict)]
+    )
+    courts = [court for court in courts if court["value"] not in headings]
     rules = (selector_config(jurisdiction) or {}).get("court_names") or []
     for court in courts:
         court["text"] = _readable_name(court["text"], rules)
@@ -143,43 +147,74 @@ def _readable_name(name: str, rules: list[dict[str, str]]) -> str:
     return name
 
 
-def _fileable_codes(jurisdiction: str) -> set[str] | None:
-    """The courts the e-filing service says accept filings, or ``None``.
+def heading_court_codes(jurisdiction: str, codes: list[str] | None = None) -> frozenset[str]:
+    """Courts that are only headings over the courts that take filings.
+
+    The court picker, the dropdown API and code search all leave these out, so
+    a filer is never offered a court the e-filing service will refuse. Cached
+    briefly; if either list cannot be read, nothing is dropped, and that answer
+    is cached for a shorter time so search never waits on the service.
+    """
 
-    Taken as a hint rather than as the truth. It is demonstrably incomplete --
-    Cook County's Chancery division is missing from it and offers three case
-    categories -- so it is only ever used to confirm something already suspected
-    from the shape of the list, and a request that fails changes nothing.
+    cache_key = f"court-selector:heading-courts:{jurisdiction}"
+    cached = cache.get(cache_key)
+    if cached is not None:
+        return cached
+    if codes is None:
+        codes = _court_codes(jurisdiction, fileable_only=False)
+    fileable = None if codes is None else _court_codes(jurisdiction, fileable_only=True)
+    if codes is None or fileable is None:
+        cache.set(cache_key, frozenset(), HEADING_COURTS_RETRY_SECONDS)
+        return frozenset()
+    fileable_codes = set(fileable)
+    headings = frozenset(code for code in codes if _is_heading_only(code, codes, fileable_codes))
+    cache.set(cache_key, headings, COURT_LIST_TTL_SECONDS)
+    return headings
+
+
+def _court_codes(jurisdiction: str, *, fileable_only: bool) -> list[str] | None:
+    """One of the e-filing service's court code lists, or ``None``.
+
+    The fileable list is taken as a hint rather than as the truth. It leaves
+    out courts that publish case categories and take filings -- nearly every
+    Massachusetts District and Probate and Family Court, and Cook County's
+    Elder Law locations -- so it only ever confirms what the shape of the list
+    already suggests, and a request that fails changes nothing.
     """
 
     try:
         response = requests.get(
             f"{settings.EFSP_URL}/jurisdictions/{jurisdiction}/codes/courts/",
-            params={"fileable_only": True, "with_names": True},
-            timeout=15,
+            params={"fileable_only": fileable_only, "with_names": True},
+            timeout=5,
         )
         response.raise_for_status()
-        return {str(court["code"]) for court in response.json() if isinstance(court, dict) and court.get("code")}
-    except (requests.RequestException, ValueError, KeyError):
-        logger.warning("Could not read the fileable court list for %s", jurisdiction)
+        return [str(court["code"]) for court in response.json() if isinstance(court, dict) and court.get("code")]
+    except (requests.RequestException, ValueError, KeyError, TypeError):
+        logger.warning("Could not read the court list for %s", jurisdiction)
         return None
 
 
-def _is_heading_only(court: dict[str, str], courts: list[dict[str, str]], fileable: set[str] | None) -> bool:
-    """Whether a court is only a heading over the courts underneath it.
+def _is_heading_only(code: str, codes: list[str], fileable: set[str]) -> bool:
+    """Whether a court is only a heading over fileable courts underneath it.
 
-    "Cook County" is such a row: every Cook filing goes to one of the eighty
-    locations whose code hangs off it, and choosing the county itself is a dead
-    end -- the case category list comes back empty and the filer is stuck with
-    no way to see why. Both things have to be true to drop it, because a
-    division like Cook County - Chancery has locations under it and still takes
-    filings of its own.
+    "Cook County" is such a row, and so is each Cook division: filings go to a
+    district location such as "cook:cvd1", and choosing "cook:cvd" itself is
+    refused by the e-filing service even though it publishes filing codes.
+    Both things have to be true to drop a court -- the service leaves it out
+    of its fileable list, and a court it does list as fileable hangs off its
+    code -- because the fileable list alone leaves out real courts.
     """
 
-    if fileable is None or court["value"] in fileable:
+    if code in fileable:
         return False
-    prefix = f"{court['value']}:"
-    return any(other["value"].startswith(prefix) for other in courts)
+    return any(other != code and other in fileable and _hangs_off(other, code) for other in codes)
+
+
+def _hangs_off(other: str, code: str) -> bool:
+    # "cook" -> "cook:cvd1"; "cook:cvd" -> "cook:cvd1". A bare number is not a
+    # prefix: Massachusetts court "1" does not head court "1036:BA".
+    return other.startswith(f"{code}:") or (":" in code and other.startswith(code))
 
 
 def _fill(template: Any, answers: dict[str, str]) -> Any:

efile_app/efile/services/filing_code_search.py

@@ -48,7 +48,7 @@
     validate_case_filters,
 )
 from efile.services.case_type_guidance import case_topic as topic_of
-from efile.services.court_selection import is_non_filing_court
+from efile.services.court_selection import heading_court_codes, is_non_filing_court
 from efile.services.filing_availability import filing_unavailable_message
 from efile.services.glossary import glossary_for
 
@@ -468,11 +468,17 @@ def _refresh_index_legacy(jurisdiction, *, progress=None, cache_dir=None):
 
 
 def current_index(jurisdiction):
-    return FilingCodeIndex.objects.filter(
-        jurisdiction=jurisdiction,
-        source_url=source_url(),
-        rules_digest=rules()[1],
-    ).first()
+    # Search needs the vocabulary, but not the much larger import snapshots.
+    # Keep snapshots lazy for callers that inspect synchronization metadata.
+    return (
+        FilingCodeIndex.objects.filter(
+            jurisdiction=jurisdiction,
+            source_url=source_url(),
+            rules_digest=rules()[1],
+        )
+        .defer("court_snapshots")
+        .first()
+    )
 
 
 def case_description(path):
@@ -600,7 +606,9 @@ def matching_paths(index, query, initial):
     tokens = search_tokens(resolved_query, index.jurisdiction, query=True)
     if not tokens:
         return index.paths.none().annotate(score=Value(0)), []
-    paths = index.paths.filter(initial=initial)
+    # A court that only heads the locations under it is refused by the e-filing
+    # service, even when an older index imported filing codes for it.
+    paths = index.paths.filter(initial=initial).exclude(court__code__in=heading_court_codes(index.jurisdiction))
     if connection.vendor == "postgresql":
         search = SearchQuery(" ".join(sorted(tokens)), config="simple")
         paths = paths.annotate(document=SearchVector("search_text", config="simple")).filter(document=search)
@@ -648,7 +656,10 @@ def search_paths(index, query, *, initial=True, offset=0, limit=20, names=None,
     return {
         "results": [
             serialize_path(path, query=query, corrected=corrected, resolved_query=resolved_query)
-            for path in paths.select_related("index")[offset : offset + limit]
+            # index.paths already carries its index as a known related object.
+            # Joining it repeats megabytes of vocabulary/snapshots for every
+            # row, particularly on the court screen's 500-result pages.
+            for path in paths[offset : offset + limit]
         ],
         "total": total,
         "corrected_terms": corrected,
@@ -921,8 +932,11 @@ def filter_filing_groups(
 
 
 def filing_groups(index, query, initial, courts=()):
-    fingerprint = hashlib.sha256(f"{index.pk}:{index.refreshed_at}:{initial}:{query}:{courts}".encode()).hexdigest()
-    key = f"filing-groups-v8:{fingerprint}"
+    headings = sorted(heading_court_codes(index.jurisdiction))
+    fingerprint = hashlib.sha256(
+        f"{index.pk}:{index.refreshed_at}:{initial}:{query}:{courts}:{headings}".encode()
+    ).hexdigest()
+    key = f"filing-groups-v9:{fingerprint}"
     cached = cache.get(key)
     if cached is not None:
         return cached
@@ -1154,7 +1168,8 @@ def find(options, facet):
         return chosen
 
     try:
-        options = {"court": catalog.courts()}
+        headings = heading_court_codes(path.index.jurisdiction)
+        options = {"court": [court for court in catalog.courts() if court["code"] not in headings]}
         court = find(options["court"], "court")
         options["case_category"] = catalog.categories(court["code"], path.initial)
         category = find(options["case_category"], "case_category")

efile_app/efile/static/js/payment.js

@@ -23,8 +23,12 @@ const paymentMessages = {
         document.getElementById("errorMessage").hidden = true;
         document.getElementById("successMessage").hidden = true;
     },
-    showError(message, actions = []) {
+    // Only a fee quote error is one the filer can go on past to Review.
+    showError(message, actions = [], {
+        feeQuote = false
+    } = {}) {
         document.getElementById("errorText").textContent = message;
+        document.getElementById("feeQuoteHelp").hidden = !feeQuote;
         window.FilingErrorActions?.render("filing-error-actions", actions);
         const box = document.getElementById("errorMessage");
         box.hidden = false;
@@ -59,7 +63,9 @@ const PaymentPage = {
 
     setFeesState(loading) {
         document.getElementById("loadingSpinner").style.display = loading ? "block" : "none";
-        document.getElementById("submitButton").disabled = loading || this.removingAccount || this.waiverUploading || DocumentChecks.pending() || !this.feeQuoteReady || !document.getElementById("selected-payment-account").value;
+        // Review is also where filers correct answers that caused a fee error.
+        // A confirmed quote is required for submission, not for opening Review.
+        document.getElementById("submitButton").disabled = loading || this.removingAccount || this.waiverUploading || DocumentChecks.pending() || !document.getElementById("selected-payment-account").value;
     },
 
     // A copy added on this page is confirmed here before Review.
@@ -267,12 +273,16 @@ const PaymentPage = {
             if (result?.success && result.quote_superseded) {
                 paymentMessages.showError(gettext("This filing changed while we were calculating fees, perhaps in another window. Reload this page to calculate them again."));
             } else if (result?.success && !result.quote_recorded) {
-                paymentMessages.showError(gettext("The court did not return a fee total for this filing. Try again, or contact the court before you file."));
+                paymentMessages.showError(gettext("The court did not return a fee total for this filing. Try again, or contact the court before you file."), [], {
+                    feeQuote: true
+                });
             }
         } catch (error) {
             if (currentRequestId !== this.quoteRequestId) return;
             this.feeQuoteReady = false;
-            paymentMessages.showError(error?.serverMessage || gettext("We could not calculate fees. Please try again."), error?.errorActions);
+            paymentMessages.showError(error?.serverMessage || gettext("We could not calculate fees. Please try again."), error?.errorActions, {
+                feeQuote: true
+            });
             this.setFeesState(false);
         }
     },

efile_app/efile/templates/efile/payment.html

@@ -152,6 +152,7 @@ <h2>{% translate "Confirmed fees" %}</h2>
             </section>
             <div class="alert alert-danger" id="errorMessage" role="alert" hidden>
                 <i class="fas fa-exclamation-triangle me-2" aria-hidden="true"></i><span id="errorText"></span>
+                <p id="feeQuoteHelp" hidden>{% translate "If you have selected a payment method, you can continue to Review to check and change your responses. Fees must be confirmed before you submit your filing." %}</p>
                 <ul id="filing-error-actions" hidden>
                 </ul>
             </div>

efile_app/efile/tests/conftest.py

@@ -41,3 +41,13 @@ def appellate_code_lists(monkeypatch):
 def efsp_validation_metadata(monkeypatch):
     """Validation lookup tests opt into metadata; other tests use USPS fallback."""
     monkeypatch.setattr("efile.services.efsp_validation.RuleLookups.get", lambda *args: None)
+
+
+@pytest.fixture(autouse=True)
+def court_headings(monkeypatch):
+    """Search and the court dropdown drop no heading courts unless a test opts in.
+
+    Tests of the court lists themselves call court_selection directly.
+    """
+    monkeypatch.setattr("efile.services.filing_code_search.heading_court_codes", lambda *args: frozenset())
+    monkeypatch.setattr("efile.api.dropdown_views.heading_court_codes", lambda *args: frozenset())

efile_app/efile/tests/test_court_selection.py

@@ -410,6 +410,28 @@ def test_a_county_that_only_heads_its_divisions_is_not_offered(self):
         courts = self._fetch(payload, [{"code": "cook:cvd1"}, {"code": "will"}])
         assert [court["value"] for court in courts] == ["cook:chd", "cook:cvd1", "will"]
 
+    def test_a_division_heading_over_fileable_locations_is_not_offered(self):
+        payload = [
+            {"code": "cook:cvd", "name": "Cook County - Municipal Civil"},
+            {"code": "cook:cvd2", "name": "Cook County - Municipal Civil - District 2 - Skokie"},
+            {"code": "cook:eld1", "name": "Cook County - Elder Law - District 1 - Chicago"},
+        ]
+        # Elder Law takes filings although the fileable list leaves it out;
+        # only a row heading a location the service does list is dropped.
+        courts = self._fetch(payload, [payload[1]])
+        assert sorted(court["value"] for court in courts) == ["cook:cvd2", "cook:eld1"]
+
+    def test_courts_missing_from_the_fileable_list_are_still_offered(self):
+        # Massachusetts lists almost none of its District Courts as fileable,
+        # and a bare number is not a prefix of other courts' codes.
+        payload = [
+            {"code": "1", "name": "Supreme Judicial Court"},
+            {"code": "490", "name": "District Court - Cambridge"},
+            {"code": "1036:BA", "name": "Juvenile Court -- Barnstable County -- Barnstable"},
+        ]
+        courts = self._fetch(payload, [payload[2]])
+        assert sorted(court["value"] for court in courts) == ["1", "1036:BA", "490"]
+
     def test_nothing_is_dropped_when_the_fileable_list_cannot_be_read(self):
         payload = [
             {"code": "cook", "name": "Cook County"},