From 22c01e5d8a713bdbf9f9b5c03251f084f5101046 Mon Sep 17 00:00:00 2001 From: "samie a." <90323643+abdulsamie10@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:06:09 +0500 Subject: [PATCH] fix(correlation): keep empty diagonals when building z_critical valuation_correlation() raised ValueError: Shape of passed values is (1, 10), indices imply (1, 9) on triangles missing their earliest diagonals. z_critical was built with .dropna(), which drops diagonals that are entirely NaN, while self.probs keeps one entry per link-ratio diagonal. Slicing by valuation instead keeps the empty diagonals so the columns stay aligned (#320). Output is byte-identical on complete triangles. Also renames the ambiguous I to num_dev_periods so the file no longer needs its E741 per-file lint suppression. --- chainladder/core/correlation.py | 32 ++++++++++++---------- chainladder/core/tests/test_correlation.py | 20 ++++++++++++-- pyproject.toml | 1 - 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/chainladder/core/correlation.py b/chainladder/core/correlation.py index c746904b9..f6baac0ba 100644 --- a/chainladder/core/correlation.py +++ b/chainladder/core/correlation.py @@ -140,20 +140,23 @@ def __init__(self, triangle, p_critical: float = 0.5): numerator.values = numerator.values[..., :-1] numerator.ddims = numerator.ddims[:-1] - # I is the number of development periods in the triangle - I = len(triangle.development) + # num_dev_periods is the number of development periods in the + # triangle, denoted I in the Mack 97 paper + num_dev_periods = len(triangle.development) # k values are the column indexes for which we are calculating T_k k = xp.array(range(2, 2 + numerator.shape[3])) # denominator is the one in formula G4 of the Mack 97 paper - denominator = ((I - k) ** 3 - I + k)[None, None, None] + denominator = ((num_dev_periods - k) ** 3 - num_dev_periods + k)[ + None, None, None + ] # complete formula G4, results in array of each T_k value self.t = 1 - 6 * xp.nan_to_num(numerator.values) / denominator # per Mack, weight is one less than the number of pairs for each T_k - weight = (I - k - 1)[None, None, None] + weight = (num_dev_periods - k - 1)[None, None, None] # Calculate big T, the weighted average of the T_k values t_expectation = ( @@ -163,7 +166,7 @@ def __init__(self, triangle, p_critical: float = 0.5): idx = triangle.index.set_index(triangle.key_labels).index # variance is result of formula G6 - self.t_variance = 2 / ((I - 2) * (I - 3)) + self.t_variance = 2 / ((num_dev_periods - 2) * (num_dev_periods - 3)) # array of t values self.t = pd.DataFrame(self.t[0, 0, ...], columns=k, index=["T_k"]) @@ -317,19 +320,18 @@ def pZlower(z: int, n: int, p: float = 0.5) -> float: if not self.total: T = [] for i in range(0, xp.max(m1large.shape[2:]) + 1): - T.append( - [ - pZlower(i, j, 0.5) - for j in range(0, xp.max(m1large.shape[2:]) + 1) - ] - ) + T.append([ + pZlower(i, j, 0.5) for j in range(0, xp.max(m1large.shape[2:]) + 1) + ]) T = np.array(T) z_idx, n_idx = z.astype(int), n.astype(int) self.probs = T[z_idx, n_idx] - z_critical = triangle[triangle.valuation > triangle.valuation.min()] - # z_critical = z_critical[z_critical.development > z_critical.development.min()].dev_to_val().sum( - # "origin") * 0 - z_critical = z_critical.dev_to_val().dropna().sum("origin") * 0 + # One column per link-ratio diagonal, labeled by ending valuation. + # Slicing by valuation (rather than dropna) keeps diagonals that + # are entirely NaN, so the columns stay aligned with self.probs + # when a triangle is missing its earliest diagonals (#320). + z_critical = triangle.dev_to_val().sum("origin") + z_critical = z_critical[z_critical.valuation > triangle.valuation.min()] * 0 z_critical.values = np.array(self.probs) < p_critical z_critical.odims = triangle.odims[0:1] self.z_critical = z_critical diff --git a/chainladder/core/tests/test_correlation.py b/chainladder/core/tests/test_correlation.py index 8b10a460f..03c9efe84 100644 --- a/chainladder/core/tests/test_correlation.py +++ b/chainladder/core/tests/test_correlation.py @@ -3,18 +3,34 @@ raa = cl.load_sample("RAA") + def test_val_corr_total_true(): assert raa.valuation_correlation(p_critical=0.5, total=True) + def test_val_corr_total_false(): assert raa.valuation_correlation(p_critical=0.5, total=False) + def test_dev_corr(): assert raa.development_correlation(p_critical=0.5) + def test_dev_corr_sparse(): - assert raa.set_backend('sparse').development_correlation(p_critical=0.5) + assert raa.set_backend("sparse").development_correlation(p_critical=0.5) + def test_validate_critical(): with pytest.raises(ValueError): - raa.valuation_correlation(p_critical=1.5, total=True) \ No newline at end of file + raa.valuation_correlation(p_critical=1.5, total=True) + + +def test_val_corr_incomplete_triangle(xyz): + # GH #320: a triangle missing its earliest diagonals raised a + # ValueError on repr because z_critical dropped all-NaN diagonals + # while its values kept one entry per link-ratio diagonal. + z_critical = ( + xyz["Paid"].valuation_correlation(p_critical=0.1, total=False).z_critical + ) + assert z_critical.values.shape[-1] == len(z_critical.ddims) + assert repr(z_critical) diff --git a/pyproject.toml b/pyproject.toml index 49e29acaa..116b2e571 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,7 +120,6 @@ select = ["E2", "E4", "E7", "E9", "F"] "chainladder/adjustments/tests/test_bootstrap.py" = ["E231"] "chainladder/adjustments/tests/test_disposal.py" = ["E226", "E231", "E241", "E251", "E265", "F841"] "chainladder/adjustments/trend.py" = ["F401"] -"chainladder/core/correlation.py" = ["E741"] "chainladder/core/display.py" = ["E203", "E252"] "chainladder/core/slice.py" = ["E225"] "chainladder/core/tests/rtest_correlation.py" = ["E266", "E722", "F821"]