diff --git a/deeptab/models/_mixins/__init__.py b/deeptab/models/_mixins/__init__.py index 461d29d9..59bf00ab 100644 --- a/deeptab/models/_mixins/__init__.py +++ b/deeptab/models/_mixins/__init__.py @@ -16,6 +16,11 @@ BaseEstimator, # sklearn get_params / set_params / clone ) +``_PointEstimatorMixin`` is not part of ``SklearnBase``. The classifier and +regressor bases list it in front of ``SklearnBase`` for the public methods +they share but the LSS base does not have (``pretrain``, and +``optimize_hparams`` with external embeddings). + Note ---- These classes are internal implementation details. Import from @@ -26,6 +31,7 @@ from deeptab.models._mixins.fit import _FitMixin from deeptab.models._mixins.hpo import _HyperparameterMixin from deeptab.models._mixins.observability import _ObservabilityMixin +from deeptab.models._mixins.point import _PointEstimatorMixin from deeptab.models._mixins.predict import _PredictMixin from deeptab.models._mixins.serialization import _SerializationMixin @@ -33,6 +39,7 @@ "_FitMixin", "_HyperparameterMixin", "_ObservabilityMixin", + "_PointEstimatorMixin", "_PredictMixin", "_SerializationMixin", ] diff --git a/deeptab/models/_mixins/point.py b/deeptab/models/_mixins/point.py new file mode 100644 index 00000000..81bd68e6 --- /dev/null +++ b/deeptab/models/_mixins/point.py @@ -0,0 +1,172 @@ +"""Public methods shared by the point-prediction estimators.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, ClassVar + +from deeptab.core.exceptions import not_fitted_error + +if TYPE_CHECKING: + from deeptab.core.interfaces import IDataModule, ITaskModel + + +class _PointEstimatorMixin: + """Behaviour the classifier and regressor bases share but the LSS base does not. + + ``SklearnBaseClassifier`` and ``SklearnBaseRegressor`` list this mixin in + front of ``SklearnBase``. The task is read from the ``_task`` class + attribute, so one implementation serves both. + + Responsibilities + ---------------- + * ``pretrain`` — public contrastive pre-training of the embedding layer. + * ``optimize_hparams`` — hyperparameter search that also takes external + embeddings. + """ + + if TYPE_CHECKING: + # Attributes provided by SklearnBase when this mixin is composed. + # Declared here for static type-checkers only; never initialised in this class. + _task: ClassVar[str] + _built: bool + _is_pretrained: bool + _task_model: ITaskModel | None + _data_module: IDataModule | None + + def _pretrain(self, base_model: Any, train_dataloader: Any, **kwargs: Any) -> None: ... + + def pretrain( + self, + pretrain_epochs=15, + k_neighbors=10, + temperature=0.1, + save_path="pretrained_embeddings.pth", + lr=1e-3, + use_positive=True, + use_negative=False, + pool_sequence=True, + ): + """ + Pretrains the embedding layer of the model using a contrastive learning approach. + + This method performs pretraining by optimizing the embeddings with respect to + neighborhood structure in the feature space. The embeddings are saved after training. + + Parameters + ---------- + pretrain_epochs : int, default=15 + Number of epochs to run pretraining. + k_neighbors : int, default=10 + Number of neighbors used in the contrastive loss computation. + temperature : float, default=0.1 + Temperature parameter for contrastive loss scaling. + save_path : str, default="pretrained_embeddings.pth" + Path to save the pretrained embeddings. + lr : float, default=1e-3 + Learning rate for the pretraining optimizer. + use_positive : bool, default=True + Whether to include positive pairs in contrastive learning. + use_negative : bool, default=False + Whether to include negative pairs in contrastive learning. + pool_sequence : bool, default=True + Whether to apply sequence pooling before computing contrastive loss. + + Raises + ------ + ValueError + If the model has not been built before calling this method. + ValueError + If the model does not contain an embedding layer. + + Notes + ----- + - This function requires that `self.build_model()` has been called beforehand. + - The pretraining method uses `self.task_model.estimator.embedding_layer`. + - Classifiers pretrain with regression mode disabled, so class labels define the + positive and negative pairs. Regressors pretrain with regression mode enabled. + + """ + if not self._built: + raise ValueError("The model has not been built yet. Call model.build_model(**args) first.") + + if not hasattr(self._task_model.estimator, "embedding_layer"): # type: ignore[union-attr] + raise ValueError("The model does not have an embedding layer") + + if self._data_module is None: + raise not_fitted_error(type(self).__name__, "_pretrain") + self._data_module.setup("fit") + + self._pretrain( + self._task_model.estimator, # type: ignore[union-attr] + self._data_module, + pretrain_epochs=pretrain_epochs, + k_neighbors=k_neighbors, + temperature=temperature, + save_path=save_path, + regression=type(self)._task == "regression", + lr=lr, + use_positive=use_positive, + use_negative=use_negative, + pool_sequence=pool_sequence, + ) + self._is_pretrained = True + + def optimize_hparams( + self, + X, + y, + X_val=None, + y_val=None, + embeddings=None, + embeddings_val=None, + time=100, + max_epochs=200, + prune_by_epoch=True, + prune_epoch=5, + fixed_params=None, + custom_search_space=None, + **optimize_kwargs, + ): + """Optimizes hyperparameters using Bayesian optimization with optional pruning. + + Parameters + ---------- + X : array-like + Training data. + y : array-like + Training labels. + X_val, y_val : array-like, optional + Validation data and labels. + time : int + The number of optimization trials to run. + max_epochs : int + Maximum number of epochs for training. + prune_by_epoch : bool + Whether to prune based on a specific epoch (True) or the best validation loss (False). + prune_epoch : int + The specific epoch to prune by when prune_by_epoch is True. + **optimize_kwargs : dict + Additional keyword arguments passed to the fit method. + + Returns + ------- + best_hparams : list + Best hyperparameters found during optimization. + """ + + return super().optimize_hparams( # type: ignore[misc] + X, + y, + regression=type(self)._task == "regression", + X_val=X_val, + y_val=y_val, + embeddings=embeddings, + embeddings_val=embeddings_val, + time=time, + max_epochs=max_epochs, + prune_by_epoch=prune_by_epoch, + prune_epoch=prune_epoch, + fixed_params=fixed_params, + custom_search_space=custom_search_space, + **optimize_kwargs, + ) diff --git a/deeptab/models/classifier_base.py b/deeptab/models/classifier_base.py index 1c47a8bc..6a17c6e4 100644 --- a/deeptab/models/classifier_base.py +++ b/deeptab/models/classifier_base.py @@ -11,6 +11,7 @@ from deeptab.configs import TrainerConfig from deeptab.core.exceptions import NotFittedError, invalid_data_error, not_fitted_error from deeptab.metrics import get_default_metrics_dict +from deeptab.models._mixins import _PointEstimatorMixin from deeptab.models.base import SklearnBase, _validate_fit_inputs from deeptab.training.losses import build_classification_loss, compute_class_weights @@ -86,7 +87,7 @@ def _encode_labels(y, classes: np.ndarray, *, name: str = "y") -> np.ndarray: _PROBA_NAMES = frozenset({"auroc", "auprc", "log_loss", "brier", "ece"}) -class SklearnBaseClassifier(ClassifierMixin, SklearnBase): +class SklearnBaseClassifier(ClassifierMixin, _PointEstimatorMixin, SklearnBase): _task = "classification" def __init__( @@ -723,139 +724,3 @@ def _probability_metric_targets(self, y_true): return _encode_labels(np.asarray(y_true).ravel(), classes, name="y_true") except (TypeError, ValueError): return y_true - - def pretrain( - self, - pretrain_epochs=15, - k_neighbors=10, - temperature=0.1, - save_path="pretrained_embeddings.pth", - lr=1e-3, - use_positive=True, - use_negative=False, - pool_sequence=True, - ): - """ - Pretrains the embedding layer of the model using a contrastive learning approach. - - This method performs pretraining by optimizing the embeddings with respect to - neighborhood structure in the feature space. The embeddings are saved after training. - - Parameters - ---------- - pretrain_epochs : int, default=15 - Number of epochs to run pretraining. - k_neighbors : int, default=10 - Number of neighbors used in the contrastive loss computation. - temperature : float, default=0.1 - Temperature parameter for contrastive loss scaling. - save_path : str, default="pretrained_embeddings.pth" - Path to save the pretrained embeddings. - lr : float, default=1e-3 - Learning rate for the pretraining optimizer. - use_positive : bool, default=True - Whether to include positive pairs in contrastive learning. - use_negative : bool, default=False - Whether to include negative pairs in contrastive learning. - pool_sequence : bool, default=True - Whether to apply sequence pooling before computing contrastive loss. - - Raises - ------ - ValueError - If the model has not been built before calling this method. - ValueError - If the model does not contain an embedding layer. - - Notes - ----- - - This function requires that `self.build_model()` has been called beforehand. - - The pretraining method uses `self.task_model.estimator.embedding_layer`. - - The method invokes `super()._pretrain()` with regression mode disabled, so class labels - define the positive and negative pairs. - - """ - if not self._built: - raise ValueError("The model has not been built yet. Call model.build_model(**args) first.") - - if not hasattr(self._task_model.estimator, "embedding_layer"): # type: ignore[union-attr] - raise ValueError("The model does not have an embedding layer") - - if self._data_module is None: - raise not_fitted_error(type(self).__name__, "_pretrain") - self._data_module.setup("fit") - - super()._pretrain( - self._task_model.estimator, # type: ignore[union-attr] - self._data_module, - pretrain_epochs=pretrain_epochs, - k_neighbors=k_neighbors, - temperature=temperature, - save_path=save_path, - regression=False, - lr=lr, - use_positive=use_positive, - use_negative=use_negative, - pool_sequence=pool_sequence, - ) - self._is_pretrained = True - - def optimize_hparams( - self, - X, - y, - X_val=None, - y_val=None, - embeddings=None, - embeddings_val=None, - time=100, - max_epochs=200, - prune_by_epoch=True, - prune_epoch=5, - fixed_params=None, - custom_search_space=None, - **optimize_kwargs, - ): - """Optimizes hyperparameters using Bayesian optimization with optional pruning. - - Parameters - ---------- - X : array-like - Training data. - y : array-like - Training labels. - X_val, y_val : array-like, optional - Validation data and labels. - time : int - The number of optimization trials to run. - max_epochs : int - Maximum number of epochs for training. - prune_by_epoch : bool - Whether to prune based on a specific epoch (True) or the best validation loss (False). - prune_epoch : int - The specific epoch to prune by when prune_by_epoch is True. - **optimize_kwargs : dict - Additional keyword arguments passed to the fit method. - - Returns - ------- - best_hparams : list - Best hyperparameters found during optimization. - """ - - return super().optimize_hparams( - X, - y, - regression=False, - X_val=X_val, - y_val=y_val, - embeddings=embeddings, - embeddings_val=embeddings_val, - time=time, - max_epochs=max_epochs, - prune_by_epoch=prune_by_epoch, - prune_epoch=prune_epoch, - fixed_params=fixed_params, - custom_search_space=custom_search_space, - **optimize_kwargs, - ) diff --git a/deeptab/models/regressor_base.py b/deeptab/models/regressor_base.py index b5db64e9..799a5df1 100644 --- a/deeptab/models/regressor_base.py +++ b/deeptab/models/regressor_base.py @@ -6,10 +6,11 @@ from deeptab.core.exceptions import not_fitted_error from deeptab.metrics import get_default_metrics_dict +from deeptab.models._mixins import _PointEstimatorMixin from deeptab.models.base import SklearnBase -class SklearnBaseRegressor(RegressorMixin, SklearnBase): +class SklearnBaseRegressor(RegressorMixin, _PointEstimatorMixin, SklearnBase): def __init__( self, model_config=None, @@ -388,136 +389,3 @@ def score(self, X, y, embeddings=None, metric=r2_score): """ score = super()._score(X, y, embeddings, metric) return score - - def pretrain( - self, - pretrain_epochs=15, - k_neighbors=10, - temperature=0.1, - save_path="pretrained_embeddings.pth", - lr=1e-3, - use_positive=True, - use_negative=False, - pool_sequence=True, - ): - """ - Pretrains the embedding layer of the model using a contrastive learning approach. - - This method performs pretraining by optimizing the embeddings with respect to - neighborhood structure in the feature space. The embeddings are saved after training. - - Parameters - ---------- - pretrain_epochs : int, default=15 - Number of epochs to run pretraining. - k_neighbors : int, default=10 - Number of neighbors used in the contrastive loss computation. - temperature : float, default=0.1 - Temperature parameter for contrastive loss scaling. - save_path : str, default="pretrained_embeddings.pth" - Path to save the pretrained embeddings. - lr : float, default=1e-3 - Learning rate for the pretraining optimizer. - use_positive : bool, default=True - Whether to include positive pairs in contrastive learning. - use_negative : bool, default=False - Whether to include negative pairs in contrastive learning. - pool_sequence : bool, default=True - Whether to apply sequence pooling before computing contrastive loss. - - Raises - ------ - ValueError - If the model has not been built before calling this method. - ValueError - If the model does not contain an embedding layer. - - Notes - ----- - - This function requires that `self.build_model()` has been called beforehand. - - The pretraining method uses `self.task_model.estimator.embedding_layer`. - - The method invokes `super()._pretrain()` with regression mode enabled. - - """ - if not self._built: - raise ValueError("The model has not been built yet. Call model.build_model(**args) first.") - - if not hasattr(self._task_model.estimator, "embedding_layer"): # type: ignore[union-attr] - raise ValueError("The model does not have an embedding layer") - - self._data_module.setup("fit") # type: ignore[union-attr] - - super()._pretrain( - self._task_model.estimator, # type: ignore[union-attr] - self._data_module, # type: ignore[arg-type] - pretrain_epochs=pretrain_epochs, - k_neighbors=k_neighbors, - temperature=temperature, - save_path=save_path, - regression=True, - lr=lr, - use_positive=use_positive, - use_negative=use_negative, - pool_sequence=pool_sequence, - ) - self._is_pretrained = True - - def optimize_hparams( - self, - X, - y, - X_val=None, - y_val=None, - embeddings=None, - embeddings_val=None, - time=100, - max_epochs=200, - prune_by_epoch=True, - prune_epoch=5, - fixed_params=None, - custom_search_space=None, - **optimize_kwargs, - ): - """Optimizes hyperparameters using Bayesian optimization with optional pruning. - - Parameters - ---------- - X : array-like - Training data. - y : array-like - Training labels. - X_val, y_val : array-like, optional - Validation data and labels. - time : int - The number of optimization trials to run. - max_epochs : int - Maximum number of epochs for training. - prune_by_epoch : bool - Whether to prune based on a specific epoch (True) or the best validation loss (False). - prune_epoch : int - The specific epoch to prune by when prune_by_epoch is True. - **optimize_kwargs : dict - Additional keyword arguments passed to the fit method. - - Returns - ------- - best_hparams : list - Best hyperparameters found during optimization. - """ - - return super().optimize_hparams( - X, - y, - regression=True, - X_val=X_val, - y_val=y_val, - embeddings=embeddings, - embeddings_val=embeddings_val, - time=time, - max_epochs=max_epochs, - prune_by_epoch=prune_by_epoch, - prune_epoch=prune_epoch, - fixed_params=fixed_params, - custom_search_space=custom_search_space, - **optimize_kwargs, - ) diff --git a/tests/models/test_pretrain_contract.py b/tests/models/test_pretrain_contract.py new file mode 100644 index 00000000..8aeb7f84 --- /dev/null +++ b/tests/models/test_pretrain_contract.py @@ -0,0 +1,74 @@ +"""Contract of the public ``pretrain`` method of classifiers and regressors.""" + +import numpy as np +import pytest + +import deeptab.models._mixins.fit as fit_module +from deeptab.configs import MambularConfig, MLPConfig, PreprocessingConfig, TrainerConfig +from deeptab.models import MambularClassifier, MambularRegressor, MLPClassifier, MLPRegressor + + +def _data(classification): + features = np.random.default_rng(17).normal(size=(48, 3)).astype(np.float32) + targets = (features[:, 0] > 0).astype(int) if classification else features[:, 0].copy() + return features, targets + + +@pytest.mark.parametrize(("estimator", "regression"), [(MambularClassifier, False), (MambularRegressor, True)]) +def test_pretrain_forwards_its_options_and_the_task(monkeypatch, tmp_path, estimator, regression): + features, targets = _data(classification=not regression) + model = estimator( + model_config=MambularConfig(d_model=8, n_layers=1, d_state=4), + preprocessing_config=PreprocessingConfig(numerical_method="standardization", output_dim=7), + trainer_config=TrainerConfig(max_epochs=1, batch_size=16, checkpoint_path=str(tmp_path)), + random_state=17, + ) + model.build_model(features[:32], targets[:32], X_val=features[32:], y_val=targets[32:]) + assert model._task_model is not None + calls = [] + monkeypatch.setattr(fit_module, "pretrain_embeddings", lambda **kwargs: calls.append(kwargs)) + + model.pretrain( + pretrain_epochs=3, + k_neighbors=4, + temperature=0.5, + save_path="warm.pth", + lr=0.01, + use_positive=False, + use_negative=True, + pool_sequence=False, + ) + + assert len(calls) == 1 + assert calls[0].pop("base_model") is model._task_model.estimator + assert calls[0].pop("train_dataloader") is model._data_module + assert calls[0] == { + "pretrain_epochs": 3, + "k_neighbors": 4, + "temperature": 0.5, + "save_path": "warm.pth", + "regression": regression, + "lr": 0.01, + "use_positive": False, + "use_negative": True, + "pool_sequence": False, + } + assert model._is_pretrained is True + + +@pytest.mark.parametrize("estimator", [MLPClassifier, MLPRegressor]) +def test_pretrain_needs_a_built_model_with_an_embedding_layer(tmp_path, estimator): + features, targets = _data(classification=estimator is MLPClassifier) + model = estimator( + model_config=MLPConfig(layer_sizes=[8]), + preprocessing_config=PreprocessingConfig(numerical_method="standardization", output_dim=7), + trainer_config=TrainerConfig(max_epochs=1, batch_size=16, checkpoint_path=str(tmp_path)), + random_state=17, + ) + with pytest.raises(ValueError, match="has not been built"): + model.pretrain() + + model.build_model(features[:32], targets[:32], X_val=features[32:], y_val=targets[32:]) + with pytest.raises(ValueError, match="does not have an embedding layer"): + model.pretrain() + assert model._is_pretrained is False