From 06a477ebf316073fbdf43deb0a04b474cb3a2d3f Mon Sep 17 00:00:00 2001 From: jorisstrakeljahn Date: Mon, 16 Nov 2026 15:51:08 +0100 Subject: [PATCH] fix(hpo): score all trials of a search on one validation split --- deeptab/models/_mixins/hpo.py | 12 +++- tests/integration/test_hpo_workflows.py | 81 +++++++++++++++++++++++ tests/models/mixins/test_search_driver.py | 15 ++++- 3 files changed, 106 insertions(+), 2 deletions(-) diff --git a/deeptab/models/_mixins/hpo.py b/deeptab/models/_mixins/hpo.py index 1b2ee88d..b7948a4d 100644 --- a/deeptab/models/_mixins/hpo.py +++ b/deeptab/models/_mixins/hpo.py @@ -6,6 +6,7 @@ from sys import float_info from typing import TYPE_CHECKING, Any, cast +import numpy as np from lightning.pytorch.callbacks import Callback from skopt import gp_minimize @@ -122,6 +123,15 @@ def optimize_hparams( # LSS fit() signature does not accept them. base_fit_kwargs = {"X_val": X_val, "y_val": y_val, **optimize_kwargs} base_fit_kwargs.pop("rebuild", None) + # Without X_val, every fit() splits off its own validation rows. One seed + # for all fits of the search scores every trial on the same rows. + if X_val is None and base_fit_kwargs.get("random_state") is None and self.random_state is None: + base_fit_kwargs["random_state"] = int(np.random.randint(0, 2**31 - 1)) + # A seeded fit() resets NumPy's global generator. Drawing the optimizer's + # seed before the first fit keeps gp_minimize from repeating its points. + search_seed = self.random_state + if search_seed is None: + search_seed = int(np.random.randint(0, 2**31 - 1)) if embeddings is not None: base_fit_kwargs["embeddings"] = embeddings if embeddings_val is not None: @@ -212,7 +222,7 @@ def _objective(hyperparams): print(f"Error encountered during fit with hyperparameters {hyperparams}: {e}") return min(abs(best_val_loss) * 100 + 1_000_000, float_info.max) - gp_minimize(_objective, param_space, n_calls=time, random_state=self.random_state) + gp_minimize(_objective, param_space, n_calls=time, random_state=search_seed) if best_hparams is None: raise RuntimeError("No hyperparameter trial completed successfully") diff --git a/tests/integration/test_hpo_workflows.py b/tests/integration/test_hpo_workflows.py index 23b57518..2db2cb7c 100644 --- a/tests/integration/test_hpo_workflows.py +++ b/tests/integration/test_hpo_workflows.py @@ -179,3 +179,84 @@ def minimize(objective, space, **kwargs): assert np.isfinite(predictions).all() if task == "lss": assert model.family_name == "normal" + + +@pytest.mark.parametrize("task", ["regression", "classification", "lss"]) +def test_search_without_validation_data_scores_every_trial_on_one_split(monkeypatch, tmp_path, task): + from skopt.space import Categorical + + import deeptab.models._mixins.hpo as hpo_module + from deeptab.configs import MLPConfig, PreprocessingConfig, TrainerConfig + from deeptab.models import MLPLSS, MLPClassifier, MLPRegressor + + estimator_class = {"regression": MLPRegressor, "classification": MLPClassifier, "lss": MLPLSS}[task] + model = estimator_class( + model_config=MLPConfig(d_model=16, dropout=0.0), + preprocessing_config=PreprocessingConfig(numerical_method="standardization", output_dim=7), + trainer_config=TrainerConfig(max_epochs=1, patience=1, lr_patience=1, batch_size=16), + ) + features = np.random.default_rng(5).normal(size=(40, 4)).astype(np.float32) + targets = (features[:, 0] > 0).astype(int) if task == "classification" else features[:, 0] + monkeypatch.setattr(hpo_module, "get_search_space", lambda *args, **kwargs: (["d_model"], [Categorical([16, 32])])) + monkeypatch.setattr( + hpo_module, + "gp_minimize", + lambda objective, space, **kwargs: SimpleNamespace(x=min([[16], [32], [16]], key=objective)), + ) + validation_sets = [] + original_fit = model.fit + + def fit(*args, **kwargs): + result = original_fit(*args, **kwargs) + validation_sets.append(tuple(sorted(map(tuple, np.asarray(model._data_module.X_val).tolist())))) + return result + + monkeypatch.setattr(model, "fit", fit) + fit_options: dict[str, Any] = { + "accelerator": "cpu", + "enable_progress_bar": False, + "enable_model_summary": False, + "default_root_dir": str(tmp_path), + } + if task == "lss": + fit_options["family"] = "normal" + model.optimize_hparams(features, targets, time=3, max_epochs=1, **fit_options) + assert len(validation_sets) == 5 + assert len(set(validation_sets)) == 1 + + +@pytest.mark.parametrize("seed_source", ["none", "fit_argument"]) +def test_seeded_search_still_draws_different_starting_points(monkeypatch, tmp_path, seed_source): + from skopt.space import Real + + import deeptab.models._mixins.hpo as hpo_module + from deeptab.configs import MLPConfig, PreprocessingConfig, TrainerConfig + from deeptab.models import MLPRegressor + + model = MLPRegressor( + model_config=MLPConfig(d_model=16), + preprocessing_config=PreprocessingConfig(numerical_method="standardization", output_dim=7), + trainer_config=TrainerConfig(max_epochs=1, patience=1, lr_patience=1, batch_size=16), + ) + features = np.random.default_rng(5).normal(size=(40, 4)).astype(np.float32) + monkeypatch.setattr(hpo_module, "get_search_space", lambda *args, **kwargs: (["dropout"], [Real(0.0, 0.5)])) + tried = [] + real_gp_minimize = hpo_module.gp_minimize + + def recording_gp_minimize(*args, **kwargs): + result = real_gp_minimize(*args, **kwargs) + assert result is not None + tried.extend(point[0] for point in result.x_iters) + return result + + monkeypatch.setattr(hpo_module, "gp_minimize", recording_gp_minimize) + fit_options: dict[str, Any] = { + "accelerator": "cpu", + "enable_progress_bar": False, + "enable_model_summary": False, + "default_root_dir": str(tmp_path), + } + if seed_source == "fit_argument": + fit_options["random_state"] = 7 + model.optimize_hparams(features, features[:, 0], time=11, max_epochs=1, **fit_options) + assert len(set(tried[:10])) == 10 diff --git a/tests/models/mixins/test_search_driver.py b/tests/models/mixins/test_search_driver.py index edc986e4..630d8991 100644 --- a/tests/models/mixins/test_search_driver.py +++ b/tests/models/mixins/test_search_driver.py @@ -130,7 +130,20 @@ def test_search_applies_same_head_shape_and_refits_winner(search_driver, length) def test_search_honors_estimator_seed(search_driver, seed): model = _SearchEstimator({0.1: -3.0}, random_state=seed) _, observed = search_driver(model, [[0.1]]) - assert observed["random_state"] == seed + if seed is None: + # Without any seed, the optimizer and the fits each get one seed drawn up front. + assert isinstance(observed["random_state"], int) + assert len({fit["kwargs"]["random_state"] for fit in model.fits}) == 1 + else: + assert observed["random_state"] == seed + assert all("random_state" not in fit["kwargs"] for fit in model.fits) + + +@pytest.mark.parametrize("seed", [None, 73]) +def test_search_with_validation_data_passes_no_seed_to_fit(search_driver, seed): + model = _SearchEstimator({0.1: -3.0}, random_state=seed) + search_driver(model, [[0.1]], X_val=[[1.0]], y_val=[1.0]) + assert all("random_state" not in fit["kwargs"] for fit in model.fits) @pytest.mark.parametrize("container", ["single", "list", "tuple"])