Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions deeptab/models/_mixins/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@
BaseEstimator, # sklearn get_params / set_params / clone
)

``_PointEstimatorMixin`` is not part of ``SklearnBase``. The classifier and
regressor bases list it in front of ``SklearnBase`` for the public methods
they share but the LSS base does not have (``pretrain``, and
``optimize_hparams`` with external embeddings).

Note
----
These classes are internal implementation details. Import from
Expand All @@ -26,13 +31,15 @@
from deeptab.models._mixins.fit import _FitMixin
from deeptab.models._mixins.hpo import _HyperparameterMixin
from deeptab.models._mixins.observability import _ObservabilityMixin
from deeptab.models._mixins.point import _PointEstimatorMixin
from deeptab.models._mixins.predict import _PredictMixin
from deeptab.models._mixins.serialization import _SerializationMixin

__all__ = [
"_FitMixin",
"_HyperparameterMixin",
"_ObservabilityMixin",
"_PointEstimatorMixin",
"_PredictMixin",
"_SerializationMixin",
]
172 changes: 172 additions & 0 deletions deeptab/models/_mixins/point.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
"""Public methods shared by the point-prediction estimators."""

from __future__ import annotations

from typing import TYPE_CHECKING, Any, ClassVar

from deeptab.core.exceptions import not_fitted_error

if TYPE_CHECKING:
from deeptab.core.interfaces import IDataModule, ITaskModel


class _PointEstimatorMixin:
"""Behaviour the classifier and regressor bases share but the LSS base does not.

``SklearnBaseClassifier`` and ``SklearnBaseRegressor`` list this mixin in
front of ``SklearnBase``. The task is read from the ``_task`` class
attribute, so one implementation serves both.

Responsibilities
----------------
* ``pretrain`` — public contrastive pre-training of the embedding layer.
* ``optimize_hparams`` — hyperparameter search that also takes external
embeddings.
"""

if TYPE_CHECKING:
# Attributes provided by SklearnBase when this mixin is composed.
# Declared here for static type-checkers only; never initialised in this class.
_task: ClassVar[str]
_built: bool
_is_pretrained: bool
_task_model: ITaskModel | None
_data_module: IDataModule | None

def _pretrain(self, base_model: Any, train_dataloader: Any, **kwargs: Any) -> None: ...

def pretrain(
self,
pretrain_epochs=15,
k_neighbors=10,
temperature=0.1,
save_path="pretrained_embeddings.pth",
lr=1e-3,
use_positive=True,
use_negative=False,
pool_sequence=True,
):
"""
Pretrains the embedding layer of the model using a contrastive learning approach.

This method performs pretraining by optimizing the embeddings with respect to
neighborhood structure in the feature space. The embeddings are saved after training.

Parameters
----------
pretrain_epochs : int, default=15
Number of epochs to run pretraining.
k_neighbors : int, default=10
Number of neighbors used in the contrastive loss computation.
temperature : float, default=0.1
Temperature parameter for contrastive loss scaling.
save_path : str, default="pretrained_embeddings.pth"
Path to save the pretrained embeddings.
lr : float, default=1e-3
Learning rate for the pretraining optimizer.
use_positive : bool, default=True
Whether to include positive pairs in contrastive learning.
use_negative : bool, default=False
Whether to include negative pairs in contrastive learning.
pool_sequence : bool, default=True
Whether to apply sequence pooling before computing contrastive loss.

Raises
------
ValueError
If the model has not been built before calling this method.
ValueError
If the model does not contain an embedding layer.

Notes
-----
- This function requires that `self.build_model()` has been called beforehand.
- The pretraining method uses `self.task_model.estimator.embedding_layer`.
- Classifiers pretrain with regression mode disabled, so class labels define the
positive and negative pairs. Regressors pretrain with regression mode enabled.

"""
if not self._built:
raise ValueError("The model has not been built yet. Call model.build_model(**args) first.")

if not hasattr(self._task_model.estimator, "embedding_layer"): # type: ignore[union-attr]
raise ValueError("The model does not have an embedding layer")

if self._data_module is None:
raise not_fitted_error(type(self).__name__, "_pretrain")
self._data_module.setup("fit")

self._pretrain(
self._task_model.estimator, # type: ignore[union-attr]
self._data_module,
pretrain_epochs=pretrain_epochs,
k_neighbors=k_neighbors,
temperature=temperature,
save_path=save_path,
regression=type(self)._task == "regression",
lr=lr,
use_positive=use_positive,
use_negative=use_negative,
pool_sequence=pool_sequence,
)
self._is_pretrained = True

def optimize_hparams(
self,
X,
y,
X_val=None,
y_val=None,
embeddings=None,
embeddings_val=None,
time=100,
max_epochs=200,
prune_by_epoch=True,
prune_epoch=5,
fixed_params=None,
custom_search_space=None,
**optimize_kwargs,
):
"""Optimizes hyperparameters using Bayesian optimization with optional pruning.

Parameters
----------
X : array-like
Training data.
y : array-like
Training labels.
X_val, y_val : array-like, optional
Validation data and labels.
time : int
The number of optimization trials to run.
max_epochs : int
Maximum number of epochs for training.
prune_by_epoch : bool
Whether to prune based on a specific epoch (True) or the best validation loss (False).
prune_epoch : int
The specific epoch to prune by when prune_by_epoch is True.
**optimize_kwargs : dict
Additional keyword arguments passed to the fit method.

Returns
-------
best_hparams : list
Best hyperparameters found during optimization.
"""

return super().optimize_hparams( # type: ignore[misc]
X,
y,
regression=type(self)._task == "regression",
X_val=X_val,
y_val=y_val,
embeddings=embeddings,
embeddings_val=embeddings_val,
time=time,
max_epochs=max_epochs,
prune_by_epoch=prune_by_epoch,
prune_epoch=prune_epoch,
fixed_params=fixed_params,
custom_search_space=custom_search_space,
**optimize_kwargs,
)
139 changes: 2 additions & 137 deletions deeptab/models/classifier_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from deeptab.configs import TrainerConfig
from deeptab.core.exceptions import NotFittedError, invalid_data_error, not_fitted_error
from deeptab.metrics import get_default_metrics_dict
from deeptab.models._mixins import _PointEstimatorMixin
from deeptab.models.base import SklearnBase, _validate_fit_inputs
from deeptab.training.losses import build_classification_loss, compute_class_weights

Expand Down Expand Up @@ -86,7 +87,7 @@ def _encode_labels(y, classes: np.ndarray, *, name: str = "y") -> np.ndarray:
_PROBA_NAMES = frozenset({"auroc", "auprc", "log_loss", "brier", "ece"})


class SklearnBaseClassifier(ClassifierMixin, SklearnBase):
class SklearnBaseClassifier(ClassifierMixin, _PointEstimatorMixin, SklearnBase):
_task = "classification"

def __init__(
Expand Down Expand Up @@ -723,139 +724,3 @@ def _probability_metric_targets(self, y_true):
return _encode_labels(np.asarray(y_true).ravel(), classes, name="y_true")
except (TypeError, ValueError):
return y_true

def pretrain(
self,
pretrain_epochs=15,
k_neighbors=10,
temperature=0.1,
save_path="pretrained_embeddings.pth",
lr=1e-3,
use_positive=True,
use_negative=False,
pool_sequence=True,
):
"""
Pretrains the embedding layer of the model using a contrastive learning approach.

This method performs pretraining by optimizing the embeddings with respect to
neighborhood structure in the feature space. The embeddings are saved after training.

Parameters
----------
pretrain_epochs : int, default=15
Number of epochs to run pretraining.
k_neighbors : int, default=10
Number of neighbors used in the contrastive loss computation.
temperature : float, default=0.1
Temperature parameter for contrastive loss scaling.
save_path : str, default="pretrained_embeddings.pth"
Path to save the pretrained embeddings.
lr : float, default=1e-3
Learning rate for the pretraining optimizer.
use_positive : bool, default=True
Whether to include positive pairs in contrastive learning.
use_negative : bool, default=False
Whether to include negative pairs in contrastive learning.
pool_sequence : bool, default=True
Whether to apply sequence pooling before computing contrastive loss.

Raises
------
ValueError
If the model has not been built before calling this method.
ValueError
If the model does not contain an embedding layer.

Notes
-----
- This function requires that `self.build_model()` has been called beforehand.
- The pretraining method uses `self.task_model.estimator.embedding_layer`.
- The method invokes `super()._pretrain()` with regression mode disabled, so class labels
define the positive and negative pairs.

"""
if not self._built:
raise ValueError("The model has not been built yet. Call model.build_model(**args) first.")

if not hasattr(self._task_model.estimator, "embedding_layer"): # type: ignore[union-attr]
raise ValueError("The model does not have an embedding layer")

if self._data_module is None:
raise not_fitted_error(type(self).__name__, "_pretrain")
self._data_module.setup("fit")

super()._pretrain(
self._task_model.estimator, # type: ignore[union-attr]
self._data_module,
pretrain_epochs=pretrain_epochs,
k_neighbors=k_neighbors,
temperature=temperature,
save_path=save_path,
regression=False,
lr=lr,
use_positive=use_positive,
use_negative=use_negative,
pool_sequence=pool_sequence,
)
self._is_pretrained = True

def optimize_hparams(
self,
X,
y,
X_val=None,
y_val=None,
embeddings=None,
embeddings_val=None,
time=100,
max_epochs=200,
prune_by_epoch=True,
prune_epoch=5,
fixed_params=None,
custom_search_space=None,
**optimize_kwargs,
):
"""Optimizes hyperparameters using Bayesian optimization with optional pruning.

Parameters
----------
X : array-like
Training data.
y : array-like
Training labels.
X_val, y_val : array-like, optional
Validation data and labels.
time : int
The number of optimization trials to run.
max_epochs : int
Maximum number of epochs for training.
prune_by_epoch : bool
Whether to prune based on a specific epoch (True) or the best validation loss (False).
prune_epoch : int
The specific epoch to prune by when prune_by_epoch is True.
**optimize_kwargs : dict
Additional keyword arguments passed to the fit method.

Returns
-------
best_hparams : list
Best hyperparameters found during optimization.
"""

return super().optimize_hparams(
X,
y,
regression=False,
X_val=X_val,
y_val=y_val,
embeddings=embeddings,
embeddings_val=embeddings_val,
time=time,
max_epochs=max_epochs,
prune_by_epoch=prune_by_epoch,
prune_epoch=prune_epoch,
fixed_params=fixed_params,
custom_search_space=custom_search_space,
**optimize_kwargs,
)
Loading
Loading