From 60ee8a65cc65be2df34f5b7d63fb11757fcd7e9d Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Wed, 23 Sep 2026 23:25:53 +0800 Subject: [PATCH 1/2] fix: validate lifecycle backtest execution results Co-Authored-By: Codex --- src/evaluation.py | 28 +++++++++- src/strategy_lifecycle/orchestrator_runner.py | 32 +++++++++--- tests/test_orchestrator_runner.py | 51 +++++++++++++++++++ tests/test_performance_metrics.py | 40 +++++++++++++++ 4 files changed, 144 insertions(+), 7 deletions(-) diff --git a/src/evaluation.py b/src/evaluation.py index 9278661..6047cf0 100644 --- a/src/evaluation.py +++ b/src/evaluation.py @@ -8,13 +8,39 @@ from .utils import make_schedule +def _observed_returns(returns: pd.Series) -> pd.Series: + """Keep a leading warmup gap, and reject returns that cannot compound.""" + if returns.empty: + return returns + if isinstance(returns.index, pd.DatetimeIndex): + dates = returns.index.tz_convert("UTC").tz_localize(None) if returns.index.tz is not None else returns.index + if dates.normalize().has_duplicates: + raise ValueError("Performance metrics reject duplicate dates.") + numeric = pd.to_numeric(returns, errors="coerce") + values = numeric.to_numpy(dtype=float) + first_valid = 0 + while first_valid < len(values) and np.isnan(values[first_valid]): + first_valid += 1 + observed = values[first_valid:] + if observed.size == 0: + return numeric.iloc[0:0] + if not np.isfinite(observed).all(): + raise ValueError("Performance metrics reject non-finite or internal missing returns.") + if np.any(observed <= -1.0): + raise ValueError("Performance metrics reject returns at or below -100%.") + return numeric.iloc[first_valid:] + + def compute_performance_metrics( returns: pd.Series, turnover: Optional[pd.Series] = None, periods_per_year: int = 365, ) -> dict[str, float]: """Compute daily strategy performance metrics from a return series.""" - returns = returns.dropna() + original_count = len(returns) + returns = _observed_returns(returns) + if turnover is not None and len(returns) != original_count and len(turnover) == original_count: + turnover = turnover.iloc[-len(returns) :] if returns.empty: return { "CAGR": np.nan, diff --git a/src/strategy_lifecycle/orchestrator_runner.py b/src/strategy_lifecycle/orchestrator_runner.py index f37288d..6dfa295 100644 --- a/src/strategy_lifecycle/orchestrator_runner.py +++ b/src/strategy_lifecycle/orchestrator_runner.py @@ -53,6 +53,18 @@ def _synthetic_panel(*, days: int = 1500, symbols: tuple[str, ...] = ("BTCUSDT", return panel.sort_index() +def _execution_config(params: Mapping[str, Any]) -> dict[str, Any]: + supported = DEFAULT_BACKTEST_CONFIG["strategy"] + unsupported = sorted(str(key) for key in params if key not in supported) + if unsupported: + raise ValueError( + f"Unsupported execution params {unsupported}; supported={sorted(supported)}" + ) + strategy = dict(supported) + strategy.update(params) + return {"strategy": strategy} + + def _slice_panel(panel: pd.DataFrame, *, start_date: date | None, end_date: date | None) -> pd.DataFrame: frame = panel level_dates = frame.index.get_level_values("date") @@ -126,17 +138,25 @@ def run( raise ValueError("No panel rows for requested window") started = datetime.now(timezone.utc) - result = run_single_backtest(sliced, "final_score", DEFAULT_BACKTEST_CONFIG) + result = run_single_backtest(sliced, "final_score", _execution_config(params)) elapsed = (datetime.now(timezone.utc) - started).total_seconds() - eval_dates = sliced.index.get_level_values("date") + observed = result.returns.dropna() + if observed.empty: + raise ValueError("Insufficient return observations for the requested window") metrics = dict(result.metrics) - metrics["days"] = int(len(result.returns.dropna())) + metrics["days"] = int(len(observed)) + if isinstance(observed.index, pd.DatetimeIndex): + bound_start = pd.Timestamp(observed.index.min()).date() + bound_end = pd.Timestamp(observed.index.max()).date() + else: + bound_start = start_date + bound_end = end_date return _metrics_to_qpk_result( strategy_profile=strategy_profile, params=params, - metrics=result.metrics, - start_date=start_date or eval_dates.min().date(), - end_date=end_date or eval_dates.max().date(), + metrics=metrics, + start_date=bound_start, + end_date=bound_end, run_duration_seconds=elapsed, ) diff --git a/tests/test_orchestrator_runner.py b/tests/test_orchestrator_runner.py index a99ffb4..cb70bb1 100644 --- a/tests/test_orchestrator_runner.py +++ b/tests/test_orchestrator_runner.py @@ -10,7 +10,11 @@ if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) +import pandas as pd + +from src.backtest import run_single_backtest from src.strategy_lifecycle.orchestrator_runner import ( # noqa: E402 + DEFAULT_BACKTEST_CONFIG, PROFILE_NAME, SUPPORTED_PROFILES, CryptoLivePoolBacktestRunner, @@ -53,6 +57,53 @@ def test_walk_forward_produces_one_result_per_window(self) -> None: ) self.assertEqual(len(results), 2) + def test_supported_top_n_changes_executed_config(self) -> None: + dates = pd.date_range("2024-01-01", periods=6) + index = pd.MultiIndex.from_product([dates, ("A", "B")], names=["date", "symbol"]) + panel = pd.DataFrame(index=index) + panel["in_universe"] = True + panel["final_score"] = [1.0, 0.0] * len(dates) + panel["open"] = [100.0, 100.0, 100.0, 100.0, 150.0, 100.0, 150.0, 100.0, 150.0, 100.0, 150.0, 100.0] + top_one = {"strategy": {**DEFAULT_BACKTEST_CONFIG["strategy"], "top_n": 1}} + top_two = {"strategy": {**DEFAULT_BACKTEST_CONFIG["strategy"], "top_n": 2}} + direct_top_one = run_single_backtest(panel, "final_score", top_one) + direct_top_two = run_single_backtest(panel, "final_score", top_two) + self.assertNotAlmostEqual(direct_top_one.metrics["CAGR"], direct_top_two.metrics["CAGR"]) + + result = CryptoLivePoolBacktestRunner(panel=panel).run(PROFILE_NAME, {"top_n": 1}) + self.assertEqual(result.params["top_n"], 1) + self.assertAlmostEqual(result.cagr, direct_top_one.metrics["CAGR"]) + + def test_observation_count_uses_actual_return_window(self) -> None: + dates = pd.date_range("2024-01-01", periods=21) + index = pd.MultiIndex.from_product([dates, ("A", "B")], names=["date", "symbol"]) + panel = pd.DataFrame({"in_universe": True, "final_score": 1.0, "open": 100.0}, index=index) + panel["prediction_window_count"] = 1 + panel.loc[(dates[0], slice(None)), "final_score"] = pd.NA + panel.loc[(dates[0], slice(None)), "prediction_window_count"] = 0 + direct = run_single_backtest(panel, "final_score", DEFAULT_BACKTEST_CONFIG) + self.assertGreater(len(direct.returns), 0) + self.assertGreater(direct.returns.index.min(), dates[0]) + + result = CryptoLivePoolBacktestRunner(panel=panel).run(PROFILE_NAME, {}) + self.assertEqual(result.observation_count, len(direct.returns)) + self.assertEqual(result.start_date, direct.returns.index.min().date()) + self.assertEqual(result.end_date, direct.returns.index.max().date()) + + def test_insufficient_window_does_not_return_a_normal_result(self) -> None: + dates = pd.date_range("2024-01-01", periods=1) + index = pd.MultiIndex.from_product([dates, ("A",)], names=["date", "symbol"]) + panel = pd.DataFrame({"in_universe": True, "final_score": 1.0, "open": 100.0}, index=index) + with self.assertRaisesRegex(ValueError, "Insufficient"): + CryptoLivePoolBacktestRunner(panel=panel).run(PROFILE_NAME, {}) + + def test_unsupported_execution_params_are_rejected(self) -> None: + dates = pd.date_range("2024-01-01", periods=6) + index = pd.MultiIndex.from_product([dates, ("A",)], names=["date", "symbol"]) + panel = pd.DataFrame({"in_universe": True, "final_score": 1.0, "open": 100.0}, index=index) + with self.assertRaisesRegex(ValueError, "Unsupported execution params"): + CryptoLivePoolBacktestRunner(panel=panel).run(PROFILE_NAME, {"top_n": 99, "not_a_strategy_field": 1}) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_performance_metrics.py b/tests/test_performance_metrics.py index b776974..556a9d7 100644 --- a/tests/test_performance_metrics.py +++ b/tests/test_performance_metrics.py @@ -87,6 +87,46 @@ def test_simulator_metrics_reach_actual_qpk_consumer_without_sign_loss(self) -> self.assertAlmostEqual(persisted.calmar_ratio, result.calmar_ratio) self.assertAlmostEqual(persisted.max_drawdown, result.max_drawdown) + def test_internal_missing_returns_are_rejected(self) -> None: + with self.assertRaises(ValueError): + compute_performance_metrics(pd.Series([0.1, np.nan, -0.1])) + + def test_non_finite_returns_are_rejected(self) -> None: + for value in (np.inf, -np.inf): + with self.subTest(value=value): + with self.assertRaises(ValueError): + compute_performance_metrics(pd.Series([0.01, value, 0.02])) + + def test_duplicate_dates_are_rejected(self) -> None: + index = pd.to_datetime(["2024-01-01", "2024-01-01"]) + with self.assertRaisesRegex(ValueError, "duplicate dates"): + compute_performance_metrics(pd.Series([0.01, -0.02], index=index)) + + def test_returns_at_or_below_minus_100_percent_are_rejected(self) -> None: + for returns in ([-1.2, -1.2], [-1.0, 0.1], [-1.0]): + with self.subTest(returns=returns): + with self.assertRaises(ValueError): + compute_performance_metrics(pd.Series(returns)) + + def test_leading_warmup_nans_match_the_observed_window(self) -> None: + observed = pd.Series([0.01, -0.02, 0.03]) + warmed = pd.Series([np.nan, np.nan, 0.01, -0.02, 0.03]) + warmed_metrics = compute_performance_metrics(warmed) + observed_metrics = compute_performance_metrics(observed) + for key, value in observed_metrics.items(): + with self.subTest(metric=key): + if np.isnan(value): + self.assertTrue(np.isnan(warmed_metrics[key])) + else: + self.assertAlmostEqual(warmed_metrics[key], value) + + def test_ordinary_compound_returns_keep_defined_365_day_cagr(self) -> None: + returns = pd.Series([0.01, -0.02, 0.03]) + metrics = compute_performance_metrics(returns) + equity = float(np.prod(1.0 + returns.to_numpy())) + self.assertAlmostEqual(metrics["CAGR"], equity ** (365 / len(returns)) - 1.0) + self.assertTrue(np.isfinite(metrics["Sharpe"])) + if __name__ == "__main__": unittest.main() From edef94e7266ea48a7414c10b4077f81783e6728b Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Wed, 23 Sep 2026 23:29:20 +0800 Subject: [PATCH 2/2] chore: satisfy ruff imports in lifecycle test Co-Authored-By: Codex --- tests/test_orchestrator_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_orchestrator_runner.py b/tests/test_orchestrator_runner.py index cb70bb1..f9cf1cc 100644 --- a/tests/test_orchestrator_runner.py +++ b/tests/test_orchestrator_runner.py @@ -10,9 +10,9 @@ if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) -import pandas as pd +import pandas as pd # noqa: E402 -from src.backtest import run_single_backtest +from src.backtest import run_single_backtest # noqa: E402 from src.strategy_lifecycle.orchestrator_runner import ( # noqa: E402 DEFAULT_BACKTEST_CONFIG, PROFILE_NAME,