fix: validate lifecycle backtest execution results

src/evaluation.py

@@ -8,13 +8,39 @@
 from .utils import make_schedule
 
 
+def _observed_returns(returns: pd.Series) -> pd.Series:
+    """Keep a leading warmup gap, and reject returns that cannot compound."""
+    if returns.empty:
+        return returns
+    if isinstance(returns.index, pd.DatetimeIndex):
+        dates = returns.index.tz_convert("UTC").tz_localize(None) if returns.index.tz is not None else returns.index
+        if dates.normalize().has_duplicates:
+            raise ValueError("Performance metrics reject duplicate dates.")
+    numeric = pd.to_numeric(returns, errors="coerce")
+    values = numeric.to_numpy(dtype=float)
+    first_valid = 0
+    while first_valid < len(values) and np.isnan(values[first_valid]):
+        first_valid += 1
+    observed = values[first_valid:]
+    if observed.size == 0:
+        return numeric.iloc[0:0]
+    if not np.isfinite(observed).all():
+        raise ValueError("Performance metrics reject non-finite or internal missing returns.")
+    if np.any(observed <= -1.0):
+        raise ValueError("Performance metrics reject returns at or below -100%.")
+    return numeric.iloc[first_valid:]
+
+
 def compute_performance_metrics(
     returns: pd.Series,
     turnover: Optional[pd.Series] = None,
     periods_per_year: int = 365,
 ) -> dict[str, float]:
     """Compute daily strategy performance metrics from a return series."""
-    returns = returns.dropna()
+    original_count = len(returns)
+    returns = _observed_returns(returns)
+    if turnover is not None and len(returns) != original_count and len(turnover) == original_count:
+        turnover = turnover.iloc[-len(returns) :]
     if returns.empty:
         return {
             "CAGR": np.nan,

src/strategy_lifecycle/orchestrator_runner.py

@@ -53,6 +53,18 @@ def _synthetic_panel(*, days: int = 1500, symbols: tuple[str, ...] = ("BTCUSDT",
     return panel.sort_index()
 
 
+def _execution_config(params: Mapping[str, Any]) -> dict[str, Any]:
+    supported = DEFAULT_BACKTEST_CONFIG["strategy"]
+    unsupported = sorted(str(key) for key in params if key not in supported)
+    if unsupported:
+        raise ValueError(
+            f"Unsupported execution params {unsupported}; supported={sorted(supported)}"
+        )
+    strategy = dict(supported)
+    strategy.update(params)
+    return {"strategy": strategy}
+
+
 def _slice_panel(panel: pd.DataFrame, *, start_date: date | None, end_date: date | None) -> pd.DataFrame:
     frame = panel
     level_dates = frame.index.get_level_values("date")
@@ -126,17 +138,25 @@ def run(
             raise ValueError("No panel rows for requested window")
 
         started = datetime.now(timezone.utc)
-        result = run_single_backtest(sliced, "final_score", DEFAULT_BACKTEST_CONFIG)
+        result = run_single_backtest(sliced, "final_score", _execution_config(params))
         elapsed = (datetime.now(timezone.utc) - started).total_seconds()
-        eval_dates = sliced.index.get_level_values("date")
+        observed = result.returns.dropna()
+        if observed.empty:
+            raise ValueError("Insufficient return observations for the requested window")
         metrics = dict(result.metrics)
-        metrics["days"] = int(len(result.returns.dropna()))
+        metrics["days"] = int(len(observed))
+        if isinstance(observed.index, pd.DatetimeIndex):
+            bound_start = pd.Timestamp(observed.index.min()).date()
+            bound_end = pd.Timestamp(observed.index.max()).date()
+        else:
+            bound_start = start_date
+            bound_end = end_date
         return _metrics_to_qpk_result(
             strategy_profile=strategy_profile,
             params=params,
-            metrics=result.metrics,
-            start_date=start_date or eval_dates.min().date(),
-            end_date=end_date or eval_dates.max().date(),
+            metrics=metrics,
+            start_date=bound_start,
+            end_date=bound_end,
             run_duration_seconds=elapsed,
         )
 

tests/test_orchestrator_runner.py

@@ -10,7 +10,11 @@
 if str(PROJECT_ROOT) not in sys.path:
     sys.path.insert(0, str(PROJECT_ROOT))
 
+import pandas as pd  # noqa: E402
+
+from src.backtest import run_single_backtest  # noqa: E402
 from src.strategy_lifecycle.orchestrator_runner import (  # noqa: E402
+    DEFAULT_BACKTEST_CONFIG,
     PROFILE_NAME,
     SUPPORTED_PROFILES,
     CryptoLivePoolBacktestRunner,
@@ -53,6 +57,53 @@ def test_walk_forward_produces_one_result_per_window(self) -> None:
             )
             self.assertEqual(len(results), 2)
 
+    def test_supported_top_n_changes_executed_config(self) -> None:
+        dates = pd.date_range("2024-01-01", periods=6)
+        index = pd.MultiIndex.from_product([dates, ("A", "B")], names=["date", "symbol"])
+        panel = pd.DataFrame(index=index)
+        panel["in_universe"] = True
+        panel["final_score"] = [1.0, 0.0] * len(dates)
+        panel["open"] = [100.0, 100.0, 100.0, 100.0, 150.0, 100.0, 150.0, 100.0, 150.0, 100.0, 150.0, 100.0]
+        top_one = {"strategy": {**DEFAULT_BACKTEST_CONFIG["strategy"], "top_n": 1}}
+        top_two = {"strategy": {**DEFAULT_BACKTEST_CONFIG["strategy"], "top_n": 2}}
+        direct_top_one = run_single_backtest(panel, "final_score", top_one)
+        direct_top_two = run_single_backtest(panel, "final_score", top_two)
+        self.assertNotAlmostEqual(direct_top_one.metrics["CAGR"], direct_top_two.metrics["CAGR"])
+
+        result = CryptoLivePoolBacktestRunner(panel=panel).run(PROFILE_NAME, {"top_n": 1})
+        self.assertEqual(result.params["top_n"], 1)
+        self.assertAlmostEqual(result.cagr, direct_top_one.metrics["CAGR"])
+
+    def test_observation_count_uses_actual_return_window(self) -> None:
+        dates = pd.date_range("2024-01-01", periods=21)
+        index = pd.MultiIndex.from_product([dates, ("A", "B")], names=["date", "symbol"])
+        panel = pd.DataFrame({"in_universe": True, "final_score": 1.0, "open": 100.0}, index=index)
+        panel["prediction_window_count"] = 1
+        panel.loc[(dates[0], slice(None)), "final_score"] = pd.NA
+        panel.loc[(dates[0], slice(None)), "prediction_window_count"] = 0
+        direct = run_single_backtest(panel, "final_score", DEFAULT_BACKTEST_CONFIG)
+        self.assertGreater(len(direct.returns), 0)
+        self.assertGreater(direct.returns.index.min(), dates[0])
+
+        result = CryptoLivePoolBacktestRunner(panel=panel).run(PROFILE_NAME, {})
+        self.assertEqual(result.observation_count, len(direct.returns))
+        self.assertEqual(result.start_date, direct.returns.index.min().date())
+        self.assertEqual(result.end_date, direct.returns.index.max().date())
+
+    def test_insufficient_window_does_not_return_a_normal_result(self) -> None:
+        dates = pd.date_range("2024-01-01", periods=1)
+        index = pd.MultiIndex.from_product([dates, ("A",)], names=["date", "symbol"])
+        panel = pd.DataFrame({"in_universe": True, "final_score": 1.0, "open": 100.0}, index=index)
+        with self.assertRaisesRegex(ValueError, "Insufficient"):
+            CryptoLivePoolBacktestRunner(panel=panel).run(PROFILE_NAME, {})
+
+    def test_unsupported_execution_params_are_rejected(self) -> None:
+        dates = pd.date_range("2024-01-01", periods=6)
+        index = pd.MultiIndex.from_product([dates, ("A",)], names=["date", "symbol"])
+        panel = pd.DataFrame({"in_universe": True, "final_score": 1.0, "open": 100.0}, index=index)
+        with self.assertRaisesRegex(ValueError, "Unsupported execution params"):
+            CryptoLivePoolBacktestRunner(panel=panel).run(PROFILE_NAME, {"top_n": 99, "not_a_strategy_field": 1})
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_performance_metrics.py

@@ -87,6 +87,46 @@ def test_simulator_metrics_reach_actual_qpk_consumer_without_sign_loss(self) ->
             self.assertAlmostEqual(persisted.calmar_ratio, result.calmar_ratio)
             self.assertAlmostEqual(persisted.max_drawdown, result.max_drawdown)
 
+    def test_internal_missing_returns_are_rejected(self) -> None:
+        with self.assertRaises(ValueError):
+            compute_performance_metrics(pd.Series([0.1, np.nan, -0.1]))
+
+    def test_non_finite_returns_are_rejected(self) -> None:
+        for value in (np.inf, -np.inf):
+            with self.subTest(value=value):
+                with self.assertRaises(ValueError):
+                    compute_performance_metrics(pd.Series([0.01, value, 0.02]))
+
+    def test_duplicate_dates_are_rejected(self) -> None:
+        index = pd.to_datetime(["2024-01-01", "2024-01-01"])
+        with self.assertRaisesRegex(ValueError, "duplicate dates"):
+            compute_performance_metrics(pd.Series([0.01, -0.02], index=index))
+
+    def test_returns_at_or_below_minus_100_percent_are_rejected(self) -> None:
+        for returns in ([-1.2, -1.2], [-1.0, 0.1], [-1.0]):
+            with self.subTest(returns=returns):
+                with self.assertRaises(ValueError):
+                    compute_performance_metrics(pd.Series(returns))
+
+    def test_leading_warmup_nans_match_the_observed_window(self) -> None:
+        observed = pd.Series([0.01, -0.02, 0.03])
+        warmed = pd.Series([np.nan, np.nan, 0.01, -0.02, 0.03])
+        warmed_metrics = compute_performance_metrics(warmed)
+        observed_metrics = compute_performance_metrics(observed)
+        for key, value in observed_metrics.items():
+            with self.subTest(metric=key):
+                if np.isnan(value):
+                    self.assertTrue(np.isnan(warmed_metrics[key]))
+                else:
+                    self.assertAlmostEqual(warmed_metrics[key], value)
+
+    def test_ordinary_compound_returns_keep_defined_365_day_cagr(self) -> None:
+        returns = pd.Series([0.01, -0.02, 0.03])
+        metrics = compute_performance_metrics(returns)
+        equity = float(np.prod(1.0 + returns.to_numpy()))
+        self.assertAlmostEqual(metrics["CAGR"], equity ** (365 / len(returns)) - 1.0)
+        self.assertTrue(np.isfinite(metrics["Sharpe"]))
+
 
 if __name__ == "__main__":
     unittest.main()