Add a bulk filing catalog export with per-court revisions

proxyserver/src/main/java/edu/suffolk/litlab/efsp/server/services/EcfCodesService.java

@@ -90,6 +90,28 @@ public Response getAll() {
     return cors(Response.ok(retMap));
   }
 
+  @GET
+  @Path("/filing_catalog")
+  public Response getFilingCatalogManifest() throws SQLException {
+    try (CodeDatabase cd = cdSupplier.get()) {
+      return cors(Response.ok(cd.getFilingCatalogManifest()));
+    }
+  }
+
+  @GET
+  @Path("/filing_catalog/courts/{court_id}")
+  public Response getFilingCatalogCourt(@PathParam("court_id") String courtId) throws SQLException {
+    try (CodeDatabase cd = cdSupplier.get()) {
+      return cd.getFilingCatalogCourt(courtId)
+          .map(catalog -> cors(Response.ok(catalog)))
+          .orElseGet(
+              () ->
+                  cors(
+                      Response.status(404)
+                          .entity("\"Court " + courtId + " has no complete filing catalog\"")));
+    }
+  }
+
   @Override
   public Response getCourts(
       HttpHeaders httpHeaders, boolean fileable, String fileableType, boolean withNames) {

proxyserver/src/main/java/edu/suffolk/litlab/efsp/tyler/ecfcodes/CodeDatabase.java

@@ -12,13 +12,15 @@
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.sql.Types;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -148,6 +150,117 @@ private String jurisStr() {
     return jurisdiction.getName();
   }
 
+  private static final String CATALOG_COURTS_SQL =
+      """
+      SELECT l.code, l.name, (l.initial ILIKE 'true' OR l.subsequent ILIKE 'true') AS fileable,
+        count(v.installedversion) AS versions,
+        md5(jsonb_build_array(l.name, jsonb_object_agg(v.codelist, v.installedversion)
+          FILTER (WHERE v.codelist IS NOT NULL))::text) AS revision
+      FROM location l LEFT JOIN installedversion v
+        ON v.jurisdiction=l.jurisdiction AND v.location=l.code
+        AND v.codelist IN ('casecategorycodes.zip', 'casetypecodes.zip', 'filingcodes.zip')
+        AND btrim(v.installedversion) <> ''
+      WHERE l.jurisdiction=?
+      GROUP BY l.code, l.name, l.initial, l.subsequent ORDER BY l.code
+      """;
+
+  private static final String CATALOG_CATEGORIES_SQL =
+      """
+      SELECT code, name FROM casecategory WHERE jurisdiction=? AND location=?
+        AND ecfcasetype != 'CriminalCase' ORDER BY code, name
+      """;
+
+  private static final String CATALOG_CASE_TYPES_SQL =
+      """
+      SELECT code, name, casecategory AS case_category, coalesce(lower(initial)='true', false) AS initial
+      FROM casetype WHERE jurisdiction=? AND location=? ORDER BY casecategory, code, name
+      """;
+
+  private static final String CATALOG_FILING_TYPES_SQL =
+      """
+      SELECT code, name, casecategory AS case_category, casetypeid AS case_type, filingtype AS timing
+      FROM filing WHERE jurisdiction=? AND location=? AND iscourtuseonly='False'
+        AND filingtype IN ('Initial', 'Subsequent', 'Both')
+      ORDER BY code, casecategory, casetypeid, filingtype, name
+      """;
+
+  /**
+   * Every court with a complete installed filing catalog. A court's revision changes when its name
+   * or the installed version of any of its three code lists changes. No Tyler requests.
+   */
+  public Map<String, Object> getFilingCatalogManifest() throws SQLException {
+    Map<String, Object> result = catalogHeader();
+    result.put("courts", catalogCourts());
+    return result;
+  }
+
+  /** One court's filing catalog, or empty if the court has no complete installed catalog. */
+  public Optional<Map<String, Object>> getFilingCatalogCourt(String courtCode) throws SQLException {
+    Optional<Map<String, Object>> court =
+        catalogCourts().stream().filter(c -> courtCode.equals(c.get("code"))).findFirst();
+    if (court.isEmpty()) {
+      return Optional.empty();
+    }
+    Map<String, Object> result = catalogHeader();
+    result.put("court", court.get());
+    result.put("categories", catalogRows(CATALOG_CATEGORIES_SQL, courtCode));
+    result.put("case_types", catalogRows(CATALOG_CASE_TYPES_SQL, courtCode));
+    result.put("filing_types", catalogRows(CATALOG_FILING_TYPES_SQL, courtCode));
+    return Optional.of(result);
+  }
+
+  private Map<String, Object> catalogHeader() {
+    Map<String, Object> result = new LinkedHashMap<>();
+    result.put("version", 1);
+    result.put("jurisdiction", jurisStr());
+    return result;
+  }
+
+  private List<Map<String, Object>> catalogCourts() throws SQLException {
+    List<Map<String, Object>> courts = new ArrayList<>();
+    try (PreparedStatement st = conn.prepareStatement(CATALOG_COURTS_SQL)) {
+      st.setString(1, jurisStr());
+      try (ResultSet rs = st.executeQuery()) {
+        while (rs.next()) {
+          if (rs.getInt("versions") == 3) {
+            courts.add(
+                Map.of(
+                    "code",
+                    rs.getString("code"),
+                    "name",
+                    rs.getString("name"),
+                    "revision",
+                    rs.getString("revision")));
+          } else if (rs.getBoolean("fileable")) {
+            // Fail loudly rather than let clients cache a filing court with missing codes.
+            throw new SQLException(
+                "Incomplete installed filing catalog for court " + rs.getString("code"));
+          }
+        }
+      }
+    }
+    return courts;
+  }
+
+  private List<Map<String, Object>> catalogRows(String sql, String courtCode) throws SQLException {
+    List<Map<String, Object>> rows = new ArrayList<>();
+    try (PreparedStatement st = conn.prepareStatement(sql)) {
+      st.setString(1, jurisStr());
+      st.setString(2, courtCode);
+      try (ResultSet rs = st.executeQuery()) {
+        ResultSetMetaData meta = rs.getMetaData();
+        while (rs.next()) {
+          Map<String, Object> row = new LinkedHashMap<>();
+          for (int column = 1; column <= meta.getColumnCount(); column++) {
+            row.put(meta.getColumnLabel(column), rs.getObject(column));
+          }
+          rows.add(row);
+        }
+      }
+    }
+    return rows;
+  }
+
   public void createTableIfAbsent(String tableName) throws SQLException {
     if (conn == null) {
       throw new SQLException();

proxyserver/src/test/java/edu/suffolk/litlab/efsp/server/services/FilingCatalogExportTest.java

@@ -0,0 +1,134 @@
+package edu.suffolk.litlab.efsp.server.services;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import edu.suffolk.litlab.efsp.Jurisdiction;
+import edu.suffolk.litlab.efsp.db.DatabaseVersionTest;
+import edu.suffolk.litlab.efsp.tyler.ecfcodes.CodeDatabase;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.List;
+import java.util.Optional;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.testcontainers.postgresql.PostgreSQLContainer;
+import org.testcontainers.utility.DockerImageName;
+
+@Tag("Docker")
+public class FilingCatalogExportTest {
+  private static final PostgreSQLContainer POSTGRES =
+      new PostgreSQLContainer(DockerImageName.parse(DatabaseVersionTest.POSTGRES_DOCKER_NAME));
+  private static final ObjectMapper JSON = new ObjectMapper();
+
+  @BeforeAll
+  static void start() {
+    POSTGRES.start();
+  }
+
+  @AfterAll
+  static void stop() {
+    POSTGRES.stop();
+  }
+
+  private Connection connect() throws SQLException {
+    return DriverManager.getConnection(
+        POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword());
+  }
+
+  private void execute(String sql) throws SQLException {
+    try (Connection conn = connect();
+        Statement st = conn.createStatement()) {
+      st.execute(sql);
+    }
+  }
+
+  private JsonNode manifest() throws SQLException {
+    try (CodeDatabase cd = new CodeDatabase(Jurisdiction.ILLINOIS, connect())) {
+      return JSON.valueToTree(cd.getFilingCatalogManifest());
+    }
+  }
+
+  private Optional<JsonNode> catalog(String court) throws SQLException {
+    try (CodeDatabase cd = new CodeDatabase(Jurisdiction.ILLINOIS, connect())) {
+      return cd.getFilingCatalogCourt(court).map(JSON::valueToTree);
+    }
+  }
+
+  @BeforeEach
+  void seed() throws SQLException {
+    try (CodeDatabase cd = new CodeDatabase(Jurisdiction.ILLINOIS, connect())) {
+      cd.createTablesIfAbsent();
+      for (String table : List.of("casecategory", "casetype", "filing")) {
+        cd.createTableIfAbsent(table);
+      }
+    }
+    execute(
+        """
+        TRUNCATE location, installedversion, casecategory, casetype, filing;
+        INSERT INTO location (code, name, initial, subsequent, jurisdiction) VALUES
+          ('court', 'Example County', 'True', 'True', 'illinois'),
+          ('other', 'Other County', 'True', 'True', 'illinois'),
+          ('parent', 'Parent grouping', 'False', 'False', 'illinois'),
+          ('foreign', 'Foreign court', 'True', 'True', 'vermont');
+        INSERT INTO installedversion (location, codelist, installedversion, jurisdiction)
+          SELECT location, codelist, '1', 'illinois'
+          FROM (VALUES ('court'), ('other')) AS loc(location)
+          CROSS JOIN (VALUES ('casecategorycodes.zip'), ('casetypecodes.zip'), ('filingcodes.zip')) AS lists(codelist);
+        INSERT INTO casecategory (code, name, ecfcasetype, location, jurisdiction) VALUES
+          ('civil', 'Civil', 'CivilCase', 'court', 'illinois'),
+          ('criminal', 'Criminal', 'CriminalCase', 'court', 'illinois');
+        INSERT INTO casetype (code, name, casecategory, initial, location, jurisdiction) VALUES
+          ('eviction', 'Eviction', 'civil', 'True', 'court', 'illinois'),
+          ('debt', 'Debt collection', 'civil', 'False', 'court', 'illinois');
+        INSERT INTO filing (code, name, casecategory, casetypeid, filingtype, iscourtuseonly, location, jurisdiction) VALUES
+          ('complaint', 'Complaint', '', '', 'Initial', 'False', 'court', 'illinois'),
+          ('answer', 'Answer', '', '', 'Subsequent', 'False', 'court', 'illinois'),
+          ('motion', 'Motion to dismiss', NULL, 'eviction', 'Subsequent', 'False', 'court', 'illinois'),
+          ('internal', 'Internal only', '', '', 'Both', 'True', 'court', 'illinois'),
+          ('foreign', 'Foreign filing', '', '', 'Both', 'False', 'court', 'vermont');
+        """);
+  }
+
+  @Test
+  void installedVersionsAndCourtNamesIdentifyOnlyChangedCourts() throws Exception {
+    JsonNode first = manifest();
+    assertEquals(first, manifest());
+    assertEquals(2, first.get("courts").size());
+    execute(
+        "UPDATE installedversion SET installedversion='2' WHERE location='court' AND codelist='filingcodes.zip'");
+    JsonNode changed = manifest();
+    assertNotEquals(first.at("/courts/0/revision"), changed.at("/courts/0/revision"));
+    assertEquals(first.at("/courts/1"), changed.at("/courts/1"));
+    execute("UPDATE location SET name='Renamed County' WHERE code='court'");
+    assertNotEquals(changed.at("/courts/0/revision"), manifest().at("/courts/0/revision"));
+  }
+
+  @Test
+  void incompleteActiveCourtRejectsTheManifest() throws Exception {
+    execute("DELETE FROM installedversion WHERE location='court' AND codelist='filingcodes.zip'");
+    assertThrows(SQLException.class, this::manifest);
+  }
+
+  @Test
+  void exportPreservesRelationshipsAndExcludesInternalAndForeignCodes() throws Exception {
+    JsonNode data = catalog("court").orElseThrow();
+    assertEquals(1, data.get("categories").size());
+    assertEquals(2, data.get("case_types").size());
+    assertEquals(3, data.get("filing_types").size());
+    assertEquals("eviction", data.at("/filing_types/2/case_type").asText());
+    assertTrue(data.at("/filing_types/2/case_category").isNull());
+    assertTrue(data.at("/case_types/1/initial").asBoolean());
+    assertEquals(manifest().at("/courts/0"), data.get("court"));
+    assertTrue(catalog("missing").isEmpty());
+  }
+}