diff --git a/chainladder/utils/cupy.py b/chainladder/utils/cupy.py index df49cbec..f7b77ace 100644 --- a/chainladder/utils/cupy.py +++ b/chainladder/utils/cupy.py @@ -4,14 +4,16 @@ import numpy as np from chainladder import options -from chainladder.utils.sparse import sp try: import cupy as cp cp.array([1]) module = "cupy" -except: +except (ImportError, RuntimeError): + # RuntimeError covers e.g. cupy.cuda.runtime.CUDARuntimeError, raised + # by cp.array([1]) when CuPy is installed but the GPU/CUDA runtime + # is unusable. if options.ARRAY_BACKEND == "cupy": import warnings @@ -22,29 +24,29 @@ def nansum(a, *args, **kwargs): - """ For cupy v0.6.0 compatibility """ + """For cupy v0.6.0 compatibility""" return cp.sum(cp.nan_to_num(a), *args, **kwargs) def nanmean(a, *args, **kwargs): - """ For cupy v0.6.0 compatibility """ + """For cupy v0.6.0 compatibility""" return cp.sum(cp.nan_to_num(a), *args, **kwargs) / cp.sum( ~cp.isnan(a), *args, **kwargs ) def nanmedian(a, *args, **kwargs): - """ For cupy v0.6.0 compatibility """ + """For cupy v0.6.0 compatibility""" return cp.array(np.nanmedian(cp.asnumpy(a), *args, **kwargs)) def nanquantile(a, *args, **kwargs): - """ For cupy v0.6.0 compatibility """ + """For cupy v0.6.0 compatibility""" return cp.array(np.nanquantile(cp.asnumpy(a), *args, **kwargs)) def unique(ar, axis=None, *args, **kwargs): - """ For cupy v0.6.0 compatibility """ + """For cupy v0.6.0 compatibility""" return cp.array(np.unique(cp.asnumpy(ar), axis=axis, *args, **kwargs)) diff --git a/chainladder/utils/dask.py b/chainladder/utils/dask.py index fad071f2..8614e46e 100644 --- a/chainladder/utils/dask.py +++ b/chainladder/utils/dask.py @@ -6,26 +6,31 @@ try: import dask.array as dp + dp.array([1]) module = "dask" -except: +except (ImportError, RuntimeError): + # RuntimeError covers the equivalent case of Dask being installed but + # its runtime being unusable when the dp.array([1]) probe runs. if options.ARRAY_BACKEND == "dask": import warnings warnings.warn("Unable to load Dask. Using numpy instead.") import numpy as dp + module = "numpy" dp.nan = np.nan def expand_dims(a, axis=0): - l = [] + slices = [] for i in range(len(a.shape)): if i == axis: - l.append(None) - l.append(slice(None)) - return a.__getitem__(tuple(l)) + slices.append(None) + slices.append(slice(None)) + return a.__getitem__(tuple(slices)) + if dp != np: dp.expand_dims = expand_dims diff --git a/chainladder/utils/sparse.py b/chainladder/utils/sparse.py index 51d21099..a02ffe37 100644 --- a/chainladder/utils/sparse.py +++ b/chainladder/utils/sparse.py @@ -6,7 +6,8 @@ from sparse import COO as COO from sparse import elemwise -def _setitem_not_supported(self, key, value) -> None: # noqa + +def _setitem_not_supported(self, key, value) -> None: # noqa raise TypeError( """ In-place item assignment (e.g. `triangle.values[...] = value`) is not @@ -18,20 +19,20 @@ def _setitem_not_supported(self, key, value) -> None: # noqa sp.isnan = np.isnan COO.nan = np.array([1.0, np.nan])[-1] COO.__setitem__ = _setitem_not_supported -setattr(sp, 'testing', np.testing) +setattr(sp, "testing", np.testing) sp.sqrt = np.sqrt sp.log = np.log sp.exp = np.exp sp.abs = np.abs -def nan_to_num(a, nan = 0.0): +def nan_to_num(a, nan=0.0): if type(a) in [int, float, np.int64, np.float64]: return np.nan_to_num(a) if hasattr(a, "fill_value"): a = a.copy() a.data[np.isnan(a.data)] = nan - return COO(coords=a.coords, data=a.data, fill_value = nan, shape = a.shape) + return COO(coords=a.coords, data=a.data, fill_value=nan, shape=a.shape) def ones(*args, **kwargs): @@ -43,6 +44,7 @@ def nansum(a, axis=None, keepdims=None, *args, **kwargs): axis=axis, keepdims=keepdims, *args, **kwargs ) + def nanquantile(a: COO, q: float, axis: int = 0, keepdims: bool = False): """ mimics np.nanquantile @@ -71,15 +73,13 @@ def nanquantile(a: COO, q: float, axis: int = 0, keepdims: bool = False): if not keep_axes: out = np.nanquantile(a.data, q) if keepdims: - out = np.asarray(out).reshape( - tuple(1 for _ in range(a.ndim)) - ) + out = np.asarray(out).reshape(tuple(1 for _ in range(a.ndim))) return COO(out) # map every stored value to an output location keep_coords = a.coords[list(keep_axes)] group_ids = np.ravel_multi_index(keep_coords, keep_shape) - + # sort by group order = np.argsort(group_ids) group_ids = group_ids[order] @@ -97,10 +97,11 @@ def nanquantile(a: COO, q: float, axis: int = 0, keepdims: bool = False): out = out.reshape(keep_shape) if keepdims: - out = np.expand_dims(out,axis) + out = np.expand_dims(out, axis) return COO(out) + def nanmedian(a: COO, axis: int = 0, keepdims: bool = False): """ mimics np.nanmean @@ -121,6 +122,7 @@ def nanmedian(a: COO, axis: int = 0, keepdims: bool = False): """ return nanquantile(a, 0.5, axis, keepdims) + def nanmean(a, axis=None, keepdims=None): n = nansum(a, axis=axis, keepdims=keepdims) d = nansum(nan_to_num(a) != 0, axis=axis, keepdims=keepdims).astype(n.dtype) @@ -129,12 +131,13 @@ def nanmean(a, axis=None, keepdims=None): out = n / d return COO(data=out.data, coords=out.coords, fill_value=0, shape=out.shape) + def array(a, *args, **kwargs): if kwargs.get("fill_value", None) is not None: fill_value = kwargs.pop("fill_value") else: fill_value = COO.nan - if type(a) == sp.COO: + if isinstance(a, sp.COO): return COO(a, *args, **kwargs, fill_value=fill_value) else: return COO(np.array(a, *args, **kwargs), fill_value=fill_value) @@ -172,4 +175,4 @@ def floor(x: COO) -> COO: sp.nanmean = nanmean sp.sum = COO.sum sp.nanquantile = nanquantile -sp.nanmedian = nanmedian \ No newline at end of file +sp.nanmedian = nanmedian diff --git a/chainladder/utils/tests/test_sparse.py b/chainladder/utils/tests/test_sparse.py index 344c8850..6e5a6f82 100644 --- a/chainladder/utils/tests/test_sparse.py +++ b/chainladder/utils/tests/test_sparse.py @@ -5,11 +5,12 @@ floor, COO, where, - nanquantile + nanquantile, ) from sparse import all as sparse_all + def test_array_from_list_default_fill_value() -> None: """ Tests chainladder.utils.sparse.array() when no fill value is provided. @@ -114,7 +115,8 @@ def test_floor_returns_copy() -> None: np.testing.assert_array_equal(result.todense(), [1.0, 2.0, -1.0]) np.testing.assert_array_equal(a.todense(), [1.2, 2.7, -0.3]) -def test_1D_nanquantile() -> None: + +def test_1d_nanquantile() -> None: """ Checks that nanquantile performs in 1D special case. @@ -122,9 +124,10 @@ def test_1D_nanquantile() -> None: ------- None """ - a = COO(np.array([1,2,3,4])) - assert nanquantile(a,0.5) == 2.5 - assert sparse_all(nanquantile(a,0.5,keepdims = True) == COO(np.array([2.5]))) + a = COO(np.array([1, 2, 3, 4])) + assert nanquantile(a, 0.5) == 2.5 + assert sparse_all(nanquantile(a, 0.5, keepdims=True) == COO(np.array([2.5]))) + def test_keepdims_nanquantile() -> None: """ @@ -134,5 +137,7 @@ def test_keepdims_nanquantile() -> None: ------- None """ - a = COO(np.array([[1,2,3,4],[3,4,5,6]])) - assert sparse_all(nanquantile(a,0.5,keepdims = True) == COO(np.array([[2,3,4,5]]))) \ No newline at end of file + a = COO(np.array([[1, 2, 3, 4], [3, 4, 5, 6]])) + assert sparse_all( + nanquantile(a, 0.5, keepdims=True) == COO(np.array([[2, 3, 4, 5]])) + ) diff --git a/chainladder/utils/tests/test_utilities.py b/chainladder/utils/tests/test_utilities.py index ed54981a..e3812790 100644 --- a/chainladder/utils/tests/test_utilities.py +++ b/chainladder/utils/tests/test_utilities.py @@ -10,9 +10,12 @@ import pandas as pd from chainladder import __dt64_unit__ - from chainladder.utils.data._manifest import SAMPLES -from chainladder.utils.utility_functions import date_delta_adjustment, maximum, minimum +from chainladder.utils.utility_functions import ( + date_delta_adjustment, + maximum, + minimum, +) from pathlib import Path from typing import TYPE_CHECKING diff --git a/chainladder/utils/triangle_weight.py b/chainladder/utils/triangle_weight.py index a0ebbfae..f62125aa 100644 --- a/chainladder/utils/triangle_weight.py +++ b/chainladder/utils/triangle_weight.py @@ -5,7 +5,6 @@ import numpy as np import pandas as pd -from chainladder.utils.sparse import sp from sklearn.base import BaseEstimator, TransformerMixin import warnings @@ -17,7 +16,8 @@ if TYPE_CHECKING: from chainladder.core.typing import TriangleLike -class TriangleWeight(BaseEstimator,TransformerMixin): + +class TriangleWeight(BaseEstimator, TransformerMixin): """ Helper class that produces a triangle of weights based on pattern selections @@ -55,11 +55,11 @@ class TriangleWeight(BaseEstimator,TransformerMixin): See order of operations below when combined with multiple drop parameters. .. note :: - + (Order of Drop Operations) - + When multiple drop parameters are used together, the weights are built in this order (steps 4 and 5 are reversed from `Development`): - + 1. ``n_periods`` — limit to the most recent origin periods. 2. ``drop`` — remove specific origin/development cells. 3. ``drop_valuation`` — remove entire valuation diagonal in the triangle. @@ -116,7 +116,7 @@ def fit(self, X: TriangleLike, y: None = None, sample_weight: None = None): Returns the instance itself. """ - self.w_ = self._set_weight_func(X=X,secondary_rank=sample_weight) + self.w_ = self._set_weight_func(X=X, secondary_rank=sample_weight) return self def transform(self, X: TriangleLike): @@ -138,10 +138,10 @@ def transform(self, X: TriangleLike): return X_new def _cascade_param( - self, - size:int, - param: bool | int | float | str | None | list[bool|int|float|str|None], - default_param: bool | int | float | str | None + self, + size: int, + param: bool | int | float | str | None | list[bool | int | float | str | None], + default_param: bool | int | float | str | None, ) -> np.ndarray: """ Internal helper function to explicitly cascade a parameter to a given triangle size @@ -151,7 +151,7 @@ def _cascade_param( size: integer the width of the triangle param: bool or int or float or str or None or list - the selected parameter, such as n_periods or drop_low, etc. + the selected parameter, such as n_periods or drop_low, etc. default_param: bool or int or float or str or None the default param to fill where unspecificied @@ -174,9 +174,7 @@ def _cascade_param( return out.astype(type(default_param)).to_numpy() def _set_weight_func( - self, - X: TriangleLike, - secondary_rank: TriangleLike | None = None + self, X: TriangleLike, secondary_rank: TriangleLike | None = None ) -> TriangleLike: """ Combines weights from all parameters @@ -230,7 +228,7 @@ def _assign_n_periods_weight_func(self, X: TriangleLike) -> TriangleLike: dev_len = X.shape[3] n_periods_param = self._cascade_param(dev_len, self.n_periods, -1) - #helper function that generates the weights for individual n_periods + # helper function that generates the weights for individual n_periods def _assign_n_periods_weight_int(X, n_periods): xp = X.get_array_module() val_offset = { @@ -253,8 +251,7 @@ def _assign_n_periods_weight_int(X, n_periods): # a dict of weights (val) by n_periods (key) dict_map = { - item: _assign_n_periods_weight_int(X, item) - for item in set(n_periods_param) + item: _assign_n_periods_weight_int(X, item) for item in set(n_periods_param) } # collection of development columns based on n_periods specified for that column conc = [ @@ -264,9 +261,7 @@ def _assign_n_periods_weight_int(X, n_periods): return xp.concatenate(tuple(conc), -1).astype(float) def _drop_n_func( - self, - X: TriangleLike, - secondary_rank: TriangleLike | None = None + self, X: TriangleLike, secondary_rank: TriangleLike | None = None ) -> TriangleLike: """ Generates weights for the `drop_high` and `drop_low` parameter @@ -282,7 +277,7 @@ def _drop_n_func( ------- A Triangle of weights - """ + """ # Preparing to set up 3D array for drop_n parameters X_val = X.values.copy() dev_len = X_val.shape[3] @@ -301,13 +296,13 @@ def _drop_n_func( # explicitly setting up 3D arrays for drop_n parameters to avoid broadcasting bugs drop_high_array = np.zeros((indices, columns, dev_len)) - drop_high_array[:, :, :] = self._cascade_param( - dev_len, self.drop_high, 0 - )[None, None] + drop_high_array[:, :, :] = self._cascade_param(dev_len, self.drop_high, 0)[ + None, None + ] drop_low_array = np.zeros((indices, columns, dev_len)) - drop_low_array[:, :, :] = self._cascade_param( - dev_len, self.drop_low, 0 - )[None, None] + drop_low_array[:, :, :] = self._cascade_param(dev_len, self.drop_low, 0)[ + None, None + ] preserve_array = np.zeros((indices, columns, dev_len)) preserve_array[:, :, :] = self._cascade_param( dev_len, self.preserve, self.preserve @@ -324,7 +319,7 @@ def _drop_n_func( # applying preserve preserve_trigger = (max_rank_unpreserve - drop_low_array) < preserve_array - + # setting up flag to produce warning warning_flag = np.any(preserve_trigger) @@ -333,9 +328,9 @@ def _drop_n_func( min_rank = np.where(preserve_trigger, 0, drop_low_array) # getting weights that are within the max and min ranks - w = ( - X_ranks < max_rank[:,:,None,:] - ) & (X_ranks > min_rank[:,:,None,:] - 1) + w = (X_ranks < max_rank[:, :, None, :]) & ( + X_ranks > min_rank[:, :, None, :] - 1 + ) # NOTE: The "Some exclusions have been ignored..." UserWarning below is # asserted by the test suite (see chainladder/development/tests/ @@ -361,7 +356,7 @@ def _drop_n_func( warnings.warn(warning) return w.astype(float) - + def _drop_func(self, X: TriangleLike) -> TriangleLike: """ Generates weights for the `drop` parameter @@ -375,7 +370,7 @@ def _drop_func(self, X: TriangleLike) -> TriangleLike: ------- A Triangle of weights - """ + """ # get the appropriate backend for nan_to_num xp = X.get_array_module() # turn single drop_valuation parameter to list if necessary @@ -391,9 +386,9 @@ def _drop_func(self, X: TriangleLike) -> TriangleLike: # create ndarray of drop_list for further operation in numpy drop_np = np.asarray(drop_list) # find indices of drop_np - origin_ind = np.where( - np.array([X.origin.astype("string")]) == drop_np[:, [0]] - )[1] + origin_ind = np.where(np.array([X.origin.astype("string")]) == drop_np[:, [0]])[ + 1 + ] dev_ind = np.where(np.array([dev_list]) == drop_np[:, [1]])[1] # set weight of dropped factors to 0 w[(origin_ind, dev_ind)] = 0 @@ -412,7 +407,7 @@ def _drop_valuation_func(self, X: TriangleLike) -> TriangleLike: ------- A Triangle of weights - """ + """ # get the appropriate backend for nan_to_num xp = X.get_array_module() # turn single drop_valuation parameter to list if necessary @@ -421,9 +416,9 @@ def _drop_valuation_func(self, X: TriangleLike) -> TriangleLike: else: drop_valuation_list = [self.drop_valuation] # turn drop_valuation to same valuation freq as X - v = pd.PeriodIndex( - drop_valuation_list, freq=X.development_grain - ).to_timestamp(how="e") + v = pd.PeriodIndex(drop_valuation_list, freq=X.development_grain).to_timestamp( + how="e" + ) # warn that some drop_valuation are outside of X if np.any(~v.isin(X.valuation)): warnings.warn("Some valuations could not be dropped.") @@ -433,7 +428,7 @@ def _drop_valuation_func(self, X: TriangleLike) -> TriangleLike: if w.sum() == 0: raise Exception("The entire triangle has been dropped via drop_valuation.") return w - + def _drop_x_func(self, X: TriangleLike) -> TriangleLike: """ Generates weights for the `drop_above` and `drop_below` parameters @@ -447,7 +442,7 @@ def _drop_x_func(self, X: TriangleLike) -> TriangleLike: ------- A Triangle of weights - """ + """ # Preparing to set up 3D array for drop_x parameters X_val = X.values.copy() dev_len = X_val.shape[3] @@ -460,9 +455,9 @@ def _drop_x_func(self, X: TriangleLike) -> TriangleLike: dev_len, self.drop_above, np.inf )[None, None] drop_below_array = np.zeros((indices, columns, dev_len)) - drop_below_array[:, :, :] = self._cascade_param( - dev_len, self.drop_below, 0.0 - )[None, None] + drop_below_array[:, :, :] = self._cascade_param(dev_len, self.drop_below, 0.0)[ + None, None + ] preserve_array = np.zeros((indices, columns, dev_len)) preserve_array[:, :, :] = self._cascade_param( dev_len, self.preserve, self.preserve @@ -472,8 +467,8 @@ def _drop_x_func(self, X: TriangleLike) -> TriangleLike: w = ~np.isnan(X_val) # weights without considering preserve - index_array_weights = (X_val < drop_above_array[:,:,None,:]) & ( - X_val > drop_below_array[:,:,None,:] + index_array_weights = (X_val < drop_above_array[:, :, None, :]) & ( + X_val > drop_below_array[:, :, None, :] ) # counting remaining factors @@ -482,7 +477,9 @@ def _drop_x_func(self, X: TriangleLike) -> TriangleLike: # applying preserve warning_flag = np.any(valid_count < preserve_array) w = np.where( - valid_count[:,:,None,:] < preserve_array[:,:,None,:], w, index_array_weights + valid_count[:, :, None, :] < preserve_array[:, :, None, :], + w, + index_array_weights, ) # NOTE: The "Some exclusions have been ignored..." UserWarning below is @@ -508,4 +505,4 @@ def _drop_x_func(self, X: TriangleLike) -> TriangleLike: ) warnings.warn(warning) - return w.astype(float) \ No newline at end of file + return w.astype(float) diff --git a/chainladder/utils/utility_functions.py b/chainladder/utils/utility_functions.py index 19536ec5..63834a54 100644 --- a/chainladder/utils/utility_functions.py +++ b/chainladder/utils/utility_functions.py @@ -942,7 +942,8 @@ class PatsyFormula(BaseEstimator, TransformerMixin): def __init__(self, formula=None): self.formula = formula - def _check_X(self, X): + @staticmethod + def _check_X(X): # noqa: N802 from chainladder.core import Triangle if isinstance(X, Triangle): @@ -1071,7 +1072,7 @@ def model_diagnostics( return concat(triangles, 0) -def PTF_formula( +def PTF_formula( # noqa: N802 alpha: list = None, gamma: list = None, iota: list = None, dgrain: int = 12 ): """Helper formula that builds a patsy formula string for the BarnettZehnwirth diff --git a/chainladder/utils/weighted_regression.py b/chainladder/utils/weighted_regression.py index 5add4f24..60f9d920 100644 --- a/chainladder/utils/weighted_regression.py +++ b/chainladder/utils/weighted_regression.py @@ -15,6 +15,7 @@ from typing import Literal from chainladder.core.typing import BackendArray + class WeightedRegression(BaseEstimator): """ Helper class that fits a system of regression equations @@ -24,8 +25,8 @@ class WeightedRegression(BaseEstimator): Parameters ---------- axis: integer (default = 2) - the axis along with the perform the regression; - axis of 2 is along the origin periods; + the axis along with the perform the regression; + axis of 2 is along the origin periods; axis of 3 is along the development periods; thru_orig: bool (default = False) whether the regression is forced to go through the origin @@ -43,10 +44,10 @@ class WeightedRegression(BaseEstimator): """ def __init__( - self, - axis: int = 2, - thru_orig: bool = False, - xp: ModuleType = np, + self, + axis: int = 2, + thru_orig: bool = False, + xp: ModuleType = np, ): self.axis = axis self.thru_orig = thru_orig @@ -64,11 +65,11 @@ def infer_x_w(self): return self def fit( - self, - X:BackendArray, - y:BackendArray|None=None, - sample_weight:BackendArray|None=None, - average: Literal["volume", "simple", "regression", "geometric"] | None = None + self, + X: BackendArray, + y: BackendArray | None = None, + sample_weight: BackendArray | None = None, + average: Literal["volume", "simple", "regression", "geometric"] | None = None, ): """ Fit the model with X. @@ -98,13 +99,13 @@ def fit( self.infer_x_w() if self.thru_orig: - self._fit_OLS_thru_orig() + self._fit_ols_thru_orig() else: - self._fit_OLS() + self._fit_ols() return self - def _fit_OLS_thru_orig(self): + def _fit_ols_thru_orig(self): """ Given a set of w, x, y, and an axis, this Function returns OLS slope and other statistics, while forcing an intercept of 0 @@ -138,7 +139,6 @@ def _fit_OLS_thru_orig(self): # but using the log link function and taking the differences is_geo = xp.array([a == "geometric" for a in average_[0, 0, 0]]) if is_geo.any(): - if xp.any((y == 0) & (x == 0)): warnings.warn( "Zero values present in triangle data used for geometric " @@ -165,7 +165,7 @@ def _fit_OLS_thru_orig(self): fitted_value = xp.repeat(xp.expand_dims(coef, axis), x.shape[axis], axis) fitted_value = fitted_value * x * (y * 0 + 1) - residual = (y - fitted_value) + residual = y - fitted_value wss_residual = xp.nansum(residual**2 * w, axis) mse_denom = xp.nansum((y * 0 + 1) * (xp.nan_to_num(w) != 0), axis) - 1 @@ -173,7 +173,7 @@ def _fit_OLS_thru_orig(self): mse = wss_residual / mse_denom sigma = xp.sqrt(mse) std_err = xp.sqrt(mse / denominator) - + self._w_reg = w self.slope_ = coef[..., None] @@ -182,7 +182,7 @@ def _fit_OLS_thru_orig(self): return self - def _fit_OLS(self): + def _fit_ols(self): """Given a set of w, x, y, and an axis, this Function returns OLS slope and intercept. TODO: diff --git a/pyproject.toml b/pyproject.toml index 8094fbe7..1b6622a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -147,17 +147,9 @@ select = ["E2", "E4", "E7", "E9", "F", "B018", "UP034", "N802"] "chainladder/tails/bondy.py" = ["F401"] "chainladder/tails/curve.py" = ["E226", "E721", "F401"] "chainladder/tails/tests/rtest_exponential.py" = ["E722", "F401"] -"chainladder/utils/cupy.py" = ["E722", "F401"] -"chainladder/utils/dask.py" = ["E722", "E741"] -"chainladder/utils/sparse.py" = ["E231", "E251", "E721"] -"chainladder/utils/tests/test_sparse.py" = ["E231", "E251", "N802"] -"chainladder/utils/triangle_weight.py" = ["E227", "E231", "E265", "F401"] -"chainladder/utils/utility_functions.py" = ["E226", "E227", "E231", "E251", "E252", "E721", "F401", "N802"] -"chainladder/utils/weighted_regression.py" = ["E227", "E231", "E252", "N802"] "chainladder/workflow/tests/test_voting.py" = ["E231", "E731", "UP034"] "chainladder/workflow/tests/test_workflow.py" = ["E203", "E241"] "chainladder/workflow/voting.py" = ["E231", "E252", "E265"] -".github/scripts/pytest_parallel.py" = ["E241", "E702"] "docs/friedland/chapter_10.ipynb" = ["E731", "F841"] "docs/friedland/chapter_7_part_2.ipynb" = ["N802"] "docs/friedland/chapter_9.ipynb" = ["E731"]