fix(correlation): keep empty diagonals when building z_critical (#320)

chainladder/core/correlation.py

@@ -140,20 +140,23 @@ def __init__(self, triangle, p_critical: float = 0.5):
         numerator.values = numerator.values[..., :-1]
         numerator.ddims = numerator.ddims[:-1]
 
-        # I is the number of development periods in the triangle
-        I = len(triangle.development)
+        # num_dev_periods is the number of development periods in the
+        # triangle, denoted I in the Mack 97 paper
+        num_dev_periods = len(triangle.development)
 
         # k values are the column indexes for which we are calculating T_k
         k = xp.array(range(2, 2 + numerator.shape[3]))
 
         # denominator is the one in formula G4 of the Mack 97 paper
-        denominator = ((I - k) ** 3 - I + k)[None, None, None]
+        denominator = ((num_dev_periods - k) ** 3 - num_dev_periods + k)[
+            None, None, None
+        ]
 
         # complete formula G4, results in array of each T_k value
         self.t = 1 - 6 * xp.nan_to_num(numerator.values) / denominator
 
         # per Mack, weight is one less than the number of pairs for each T_k
-        weight = (I - k - 1)[None, None, None]
+        weight = (num_dev_periods - k - 1)[None, None, None]
 
         # Calculate big T, the weighted average of the T_k values
         t_expectation = (
@@ -163,7 +166,7 @@ def __init__(self, triangle, p_critical: float = 0.5):
         idx = triangle.index.set_index(triangle.key_labels).index
 
         # variance is result of formula G6
-        self.t_variance = 2 / ((I - 2) * (I - 3))
+        self.t_variance = 2 / ((num_dev_periods - 2) * (num_dev_periods - 3))
 
         # array of t values
         self.t = pd.DataFrame(self.t[0, 0, ...], columns=k, index=["T_k"])
@@ -317,19 +320,18 @@ def pZlower(z: int, n: int, p: float = 0.5) -> float:
         if not self.total:
             T = []
             for i in range(0, xp.max(m1large.shape[2:]) + 1):
-                T.append(
-                    [
-                        pZlower(i, j, 0.5)
-                        for j in range(0, xp.max(m1large.shape[2:]) + 1)
-                    ]
-                )
+                T.append([
+                    pZlower(i, j, 0.5) for j in range(0, xp.max(m1large.shape[2:]) + 1)
+                ])
             T = np.array(T)
             z_idx, n_idx = z.astype(int), n.astype(int)
             self.probs = T[z_idx, n_idx]
-            z_critical = triangle[triangle.valuation > triangle.valuation.min()]
-            # z_critical = z_critical[z_critical.development > z_critical.development.min()].dev_to_val().sum(
-            #     "origin") * 0
-            z_critical = z_critical.dev_to_val().dropna().sum("origin") * 0
+            # One column per link-ratio diagonal, labeled by ending valuation.
+            # Slicing by valuation (rather than dropna) keeps diagonals that
+            # are entirely NaN, so the columns stay aligned with self.probs
+            # when a triangle is missing its earliest diagonals (#320).
+            z_critical = triangle.dev_to_val().sum("origin")
+            z_critical = z_critical[z_critical.valuation > triangle.valuation.min()] * 0
             z_critical.values = np.array(self.probs) < p_critical
             z_critical.odims = triangle.odims[0:1]
             self.z_critical = z_critical

chainladder/core/tests/test_correlation.py

@@ -3,18 +3,34 @@
 
 raa = cl.load_sample("RAA")
 
+
 def test_val_corr_total_true():
     assert raa.valuation_correlation(p_critical=0.5, total=True)
 
+
 def test_val_corr_total_false():
     assert raa.valuation_correlation(p_critical=0.5, total=False)
 
+
 def test_dev_corr():
     assert raa.development_correlation(p_critical=0.5)
 
+
 def test_dev_corr_sparse():
-    assert raa.set_backend('sparse').development_correlation(p_critical=0.5)
+    assert raa.set_backend("sparse").development_correlation(p_critical=0.5)
+
 
 def test_validate_critical():
     with pytest.raises(ValueError):
-        raa.valuation_correlation(p_critical=1.5, total=True)
+        raa.valuation_correlation(p_critical=1.5, total=True)
+
+
+def test_val_corr_incomplete_triangle(xyz):
+    # GH #320: a triangle missing its earliest diagonals raised a
+    # ValueError on repr because z_critical dropped all-NaN diagonals
+    # while its values kept one entry per link-ratio diagonal.
+    z_critical = (
+        xyz["Paid"].valuation_correlation(p_critical=0.1, total=False).z_critical
+    )
+    assert z_critical.values.shape[-1] == len(z_critical.ddims)
+    assert repr(z_critical)

pyproject.toml

@@ -120,7 +120,6 @@ select = ["E2", "E4", "E7", "E9", "F"]
 "chainladder/adjustments/tests/test_bootstrap.py" = ["E231"]
 "chainladder/adjustments/tests/test_disposal.py" = ["E226", "E231", "E241", "E251", "E265", "F841"]
 "chainladder/adjustments/trend.py" = ["F401"]
-"chainladder/core/correlation.py" = ["E741"]
 "chainladder/core/display.py" = ["E203", "E252"]
 "chainladder/core/slice.py" = ["E225"]
 "chainladder/core/tests/rtest_correlation.py" = ["E266", "E722", "F821"]