Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion deeptab/models/_mixins/hpo.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from sys import float_info
from typing import TYPE_CHECKING, Any, cast

import numpy as np
from lightning.pytorch.callbacks import Callback
from skopt import gp_minimize

Expand Down Expand Up @@ -122,6 +123,15 @@ def optimize_hparams(
# LSS fit() signature does not accept them.
base_fit_kwargs = {"X_val": X_val, "y_val": y_val, **optimize_kwargs}
base_fit_kwargs.pop("rebuild", None)
# Without X_val, every fit() splits off its own validation rows. One seed
# for all fits of the search scores every trial on the same rows.
if X_val is None and base_fit_kwargs.get("random_state") is None and self.random_state is None:
base_fit_kwargs["random_state"] = int(np.random.randint(0, 2**31 - 1))
# A seeded fit() resets NumPy's global generator. Drawing the optimizer's
# seed before the first fit keeps gp_minimize from repeating its points.
search_seed = self.random_state
if search_seed is None:
search_seed = int(np.random.randint(0, 2**31 - 1))
if embeddings is not None:
base_fit_kwargs["embeddings"] = embeddings
if embeddings_val is not None:
Expand Down Expand Up @@ -212,7 +222,7 @@ def _objective(hyperparams):
print(f"Error encountered during fit with hyperparameters {hyperparams}: {e}")
return min(abs(best_val_loss) * 100 + 1_000_000, float_info.max)

gp_minimize(_objective, param_space, n_calls=time, random_state=self.random_state)
gp_minimize(_objective, param_space, n_calls=time, random_state=search_seed)
if best_hparams is None:
raise RuntimeError("No hyperparameter trial completed successfully")

Expand Down
81 changes: 81 additions & 0 deletions tests/integration/test_hpo_workflows.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,3 +179,84 @@ def minimize(objective, space, **kwargs):
assert np.isfinite(predictions).all()
if task == "lss":
assert model.family_name == "normal"


@pytest.mark.parametrize("task", ["regression", "classification", "lss"])
def test_search_without_validation_data_scores_every_trial_on_one_split(monkeypatch, tmp_path, task):
from skopt.space import Categorical

import deeptab.models._mixins.hpo as hpo_module
from deeptab.configs import MLPConfig, PreprocessingConfig, TrainerConfig
from deeptab.models import MLPLSS, MLPClassifier, MLPRegressor

estimator_class = {"regression": MLPRegressor, "classification": MLPClassifier, "lss": MLPLSS}[task]
model = estimator_class(
model_config=MLPConfig(d_model=16, dropout=0.0),
preprocessing_config=PreprocessingConfig(numerical_method="standardization", output_dim=7),
trainer_config=TrainerConfig(max_epochs=1, patience=1, lr_patience=1, batch_size=16),
)
features = np.random.default_rng(5).normal(size=(40, 4)).astype(np.float32)
targets = (features[:, 0] > 0).astype(int) if task == "classification" else features[:, 0]
monkeypatch.setattr(hpo_module, "get_search_space", lambda *args, **kwargs: (["d_model"], [Categorical([16, 32])]))
monkeypatch.setattr(
hpo_module,
"gp_minimize",
lambda objective, space, **kwargs: SimpleNamespace(x=min([[16], [32], [16]], key=objective)),
)
validation_sets = []
original_fit = model.fit

def fit(*args, **kwargs):
result = original_fit(*args, **kwargs)
validation_sets.append(tuple(sorted(map(tuple, np.asarray(model._data_module.X_val).tolist()))))
return result

monkeypatch.setattr(model, "fit", fit)
fit_options: dict[str, Any] = {
"accelerator": "cpu",
"enable_progress_bar": False,
"enable_model_summary": False,
"default_root_dir": str(tmp_path),
}
if task == "lss":
fit_options["family"] = "normal"
model.optimize_hparams(features, targets, time=3, max_epochs=1, **fit_options)
assert len(validation_sets) == 5
assert len(set(validation_sets)) == 1


@pytest.mark.parametrize("seed_source", ["none", "fit_argument"])
def test_seeded_search_still_draws_different_starting_points(monkeypatch, tmp_path, seed_source):
from skopt.space import Real

import deeptab.models._mixins.hpo as hpo_module
from deeptab.configs import MLPConfig, PreprocessingConfig, TrainerConfig
from deeptab.models import MLPRegressor

model = MLPRegressor(
model_config=MLPConfig(d_model=16),
preprocessing_config=PreprocessingConfig(numerical_method="standardization", output_dim=7),
trainer_config=TrainerConfig(max_epochs=1, patience=1, lr_patience=1, batch_size=16),
)
features = np.random.default_rng(5).normal(size=(40, 4)).astype(np.float32)
monkeypatch.setattr(hpo_module, "get_search_space", lambda *args, **kwargs: (["dropout"], [Real(0.0, 0.5)]))
tried = []
real_gp_minimize = hpo_module.gp_minimize

def recording_gp_minimize(*args, **kwargs):
result = real_gp_minimize(*args, **kwargs)
assert result is not None
tried.extend(point[0] for point in result.x_iters)
return result

monkeypatch.setattr(hpo_module, "gp_minimize", recording_gp_minimize)
fit_options: dict[str, Any] = {
"accelerator": "cpu",
"enable_progress_bar": False,
"enable_model_summary": False,
"default_root_dir": str(tmp_path),
}
if seed_source == "fit_argument":
fit_options["random_state"] = 7
model.optimize_hparams(features, features[:, 0], time=11, max_epochs=1, **fit_options)
assert len(set(tried[:10])) == 10
15 changes: 14 additions & 1 deletion tests/models/mixins/test_search_driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,20 @@ def test_search_applies_same_head_shape_and_refits_winner(search_driver, length)
def test_search_honors_estimator_seed(search_driver, seed):
model = _SearchEstimator({0.1: -3.0}, random_state=seed)
_, observed = search_driver(model, [[0.1]])
assert observed["random_state"] == seed
if seed is None:
# Without any seed, the optimizer and the fits each get one seed drawn up front.
assert isinstance(observed["random_state"], int)
assert len({fit["kwargs"]["random_state"] for fit in model.fits}) == 1
else:
assert observed["random_state"] == seed
assert all("random_state" not in fit["kwargs"] for fit in model.fits)


@pytest.mark.parametrize("seed", [None, 73])
def test_search_with_validation_data_passes_no_seed_to_fit(search_driver, seed):
model = _SearchEstimator({0.1: -3.0}, random_state=seed)
search_driver(model, [[0.1]], X_val=[[1.0]], y_val=[1.0])
assert all("random_state" not in fit["kwargs"] for fit in model.fits)


@pytest.mark.parametrize("container", ["single", "list", "tuple"])
Expand Down
Loading