From 9fa4ccf3c7ccf6b1c3e2bc1fc5d7f2f98aa09d3c Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Sat, 10 Oct 2026 22:54:15 +0900 Subject: [PATCH] feat(research_stats): RS-01 research-only PSR/DSR, stationary bootstrap, cost stress, backtest_report.v1 - New isolated package quant_platform_kit.research_stats (research-only). - PSR/DSR fail closed to UNCOMPUTABLE (value=None) on <2 trials, zero trial-Sharpe variance, short or invalid samples; never zero. - Stationary block bootstrap CIs with mandatory seed (numpy lazily imported). - Cost-stress recompute at 1x/2x/3x; invalid input is PARKED. - qsl.backtest_report.v1 dataclasses + packaged JSON Schema with live_ready=false / no_order=true invariants. - numpy/pandas only via new optional 'research' extra; core deps stay empty. - Import-boundary tests: no production module imports research_stats, and research_stats imports without numpy. - Synthetic-data tests only; no real backtest numbers. QPK_PIN untouched. --- docs/research_stats.zh-CN.md | 33 ++ pyproject.toml | 6 + .../research_stats/__init__.py | 77 +++ .../research_stats/_validation.py | 54 ++ .../research_stats/backtest_report.py | 400 ++++++++++++++ .../research_stats/bootstrap.py | 208 +++++++ .../research_stats/cost_stress.py | 143 +++++ .../research_stats/sharpe_inference.py | 234 ++++++++ .../schemas/backtest-report.v1.schema.json | 506 ++++++++++++++++++ tests/test_research_stats_backtest_report.py | 144 +++++ tests/test_research_stats_bootstrap.py | 98 ++++ tests/test_research_stats_cost_stress.py | 63 +++ tests/test_research_stats_import_boundary.py | 124 +++++ tests/test_research_stats_sharpe.py | 117 ++++ uv.lock | 11 +- 15 files changed, 2217 insertions(+), 1 deletion(-) create mode 100644 docs/research_stats.zh-CN.md create mode 100644 src/quant_platform_kit/research_stats/__init__.py create mode 100644 src/quant_platform_kit/research_stats/_validation.py create mode 100644 src/quant_platform_kit/research_stats/backtest_report.py create mode 100644 src/quant_platform_kit/research_stats/bootstrap.py create mode 100644 src/quant_platform_kit/research_stats/cost_stress.py create mode 100644 src/quant_platform_kit/research_stats/sharpe_inference.py create mode 100644 src/quant_platform_kit/schemas/backtest-report.v1.schema.json create mode 100644 tests/test_research_stats_backtest_report.py create mode 100644 tests/test_research_stats_bootstrap.py create mode 100644 tests/test_research_stats_cost_stress.py create mode 100644 tests/test_research_stats_import_boundary.py create mode 100644 tests/test_research_stats_sharpe.py diff --git a/docs/research_stats.zh-CN.md b/docs/research_stats.zh-CN.md new file mode 100644 index 00000000..cc6ceb9a --- /dev/null +++ b/docs/research_stats.zh-CN.md @@ -0,0 +1,33 @@ +# research_stats（RS-01，研究-only） + +`quant_platform_kit.research_stats` 是回测证据的研究统计原语。它**不**读取账户、不调用券商、不下单、不写风险政策，也不授予 paper/shadow/live 权限。运行时、执行内核和通知模块不得 import 它（`tests/test_research_stats_import_boundary.py` 强制）。 + +安装：`pip install 'quant-platform-kit[research]'`。numpy/pandas 只在 `research` extra（以及原有 `dev` extra）里，核心 `dependencies` 仍为空。PSR/DSR 与成本压力只用标准库；只有 bootstrap 延迟 import numpy。 + +本模块与测试只使用合成数据，不包含任何真实回测数字。 + +## 组件 + +| 组件 | 函数 | 失败语义 | +| --- | --- | --- | +| 概率 Sharpe（PSR，Bailey & López de Prado 2012） | `probabilistic_sharpe_ratio(returns, sr_benchmark=0, min_observations=30)` | 样本不足、零方差、非有限、≤−1、bool → `UNCOMPUTABLE`，`value=None` | +| 去膨胀 Sharpe（DSR，2014） | `deflated_sharpe_ratio(returns, trial_sharpes, n_effective_trials=None)` | 试验数 < 2、有效试验数 > 已登记数、试验 Sharpe 方差为 0 → `UNCOMPUTABLE` | +| 期望最大 Sharpe | `expected_max_sharpe(variance, n_trials)` | 同上 | +| 平稳块自助法（Politis & Romano 1994） | `stationary_bootstrap_ci(returns, *, seed, mean_block_length, n_resamples=1000, confidence=0.95)` | seed 必填；块长 > 样本、重抽样 < 100 → `UNCOMPUTABLE`；某个统计量无定义时只有该项 `UNCOMPUTABLE` | +| 成本压力 | `cost_stress_recompute(gross, turnover, *, cost_bps_per_side, multipliers=(1,2,3))` | 长度不等、负换手、非有限、重复倍数 → `PARKED` | +| 报告模板 | `BacktestReportV1` + `schemas/backtest-report.v1.schema.json` | `live_ready=False`、`no_order=True` 恒定；非 COMPUTED 指标必须 `value=None` + `reason_code`；development 数据不得标 `OOS_EVALUATED` | + +## 口径 + +- Sharpe 一律是**单期、未年化**（均值 / 样本标准差 ddof=1），rf/MAR 由调用方先扣。`trial_sharpes` 必须包括全部试验（含失败、拒绝），频率与 `returns` 一致。 +- 偏度/峰度用总体矩估计，峰度为非超额（正态 = 3）。 +- `min_observations=30` 只是沿用 Kelly 契约 v2 的结构下限，不是质量阈值。 +- bootstrap 统计量：单期对数增长 `mean(log1p r)`、单期 Sharpe、含初始净值的最大回撤；百分位区间，不做自相关以外的校正，不构成未来风险保证，也不替代多重试验校正。 +- 成本：`turnover[t]` = 期初买卖成交额合计 / 交易前净值，每单位成交额付 `cost_bps_per_side × 倍数`；`net = (1 − τ·c)(1 + gross) − 1`。半 L1 口径（现金腿不计费）须由调用方先换算。 +- 报告模板只验结构与诚实性约束，不判断经济有效性，不设任何 Sharpe/回撤及格线；否决项阈值必须由候选在读取留出结果前冻结（`gates.veto_items[].frozen_at/frozen_by`）。 + +## 非目标 + +- 不修改 `strategy_lifecycle/performance_metrics.py` 等现有共享指标函数的语义。 +- 不接入任何消费者，不升级任何下游 QPK pin，不改 `QPK_PIN`。 +- PBO（CSCV）、与 UES trial journal 的接线留给后续 RS-02。 diff --git a/pyproject.toml b/pyproject.toml index 017ce467..7bd52dac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,12 @@ dev = [ "numpy", "pandas", ] +# Research-only numerics for quant_platform_kit.research_stats. Never a core +# runtime dependency; production installs do not need this extra. +research = [ + "numpy", + "pandas", +] [project.scripts] quant-lifecycle = "quant_platform_kit.strategy_lifecycle.cli:main" diff --git a/src/quant_platform_kit/research_stats/__init__.py b/src/quant_platform_kit/research_stats/__init__.py new file mode 100644 index 00000000..5cfc3790 --- /dev/null +++ b/src/quant_platform_kit/research_stats/__init__.py @@ -0,0 +1,77 @@ +"""Research-only statistics for backtest evidence (RS-01). + +This package is **research-only**. It never reads accounts, calls brokers, +places orders, writes risk policy or grants paper/shadow/live authority. +Runtime, execution and notification modules must not import it (enforced by +``tests/test_research_stats_import_boundary.py``). + +``sharpe_inference`` and ``cost_stress`` use only the standard library. +``bootstrap`` lazily imports numpy from the optional ``research`` extra +(``pip install 'quant-platform-kit[research]'``). +""" + +from __future__ import annotations + +from quant_platform_kit.research_stats.backtest_report import ( + BACKTEST_REPORT_SCHEMA_VERSION, + BacktestReportV1, + BenchmarkSpec, + CostModelSpec, + DataSpec, + MetricEntry, + ReportGates, + TimingSpec, + TrialSummary, + UncertaintySpec, + VetoItem, + WalkForwardFold, + WalkForwardSpec, + load_backtest_report_schema, +) +from quant_platform_kit.research_stats.bootstrap import ( + BootstrapInterval, + BootstrapResult, + stationary_bootstrap_ci, +) +from quant_platform_kit.research_stats.cost_stress import ( + CostStressResult, + CostStressScenario, + cost_stress_recompute, +) +from quant_platform_kit.research_stats.sharpe_inference import ( + STATUS_COMPUTED, + STATUS_UNCOMPUTABLE, + StatResult, + deflated_sharpe_ratio, + expected_max_sharpe, + probabilistic_sharpe_ratio, +) + +__all__ = [ + "BACKTEST_REPORT_SCHEMA_VERSION", + "BacktestReportV1", + "BenchmarkSpec", + "BootstrapInterval", + "BootstrapResult", + "CostModelSpec", + "CostStressResult", + "CostStressScenario", + "DataSpec", + "MetricEntry", + "ReportGates", + "STATUS_COMPUTED", + "STATUS_UNCOMPUTABLE", + "StatResult", + "TimingSpec", + "TrialSummary", + "UncertaintySpec", + "VetoItem", + "WalkForwardFold", + "WalkForwardSpec", + "cost_stress_recompute", + "deflated_sharpe_ratio", + "expected_max_sharpe", + "load_backtest_report_schema", + "probabilistic_sharpe_ratio", + "stationary_bootstrap_ci", +] diff --git a/src/quant_platform_kit/research_stats/_validation.py b/src/quant_platform_kit/research_stats/_validation.py new file mode 100644 index 00000000..ac49451e --- /dev/null +++ b/src/quant_platform_kit/research_stats/_validation.py @@ -0,0 +1,54 @@ +"""Shared fail-closed input validation for research_stats (stdlib only).""" + +from __future__ import annotations + +import math +from typing import Iterable + + +class ResearchInputError(ValueError): + """Raised internally when an input cannot be used; callers map it to a status.""" + + +def finite_float(value: object, label: str) -> float: + """Accept int/float (and numpy scalars); reject bool, str and non-finite.""" + if isinstance(value, (bool, str, bytes)) or not hasattr(value, "__float__"): + raise ResearchInputError(f"{label} must be a finite number") + if type(value).__name__ in {"bool_", "bool"}: # numpy.bool_ + raise ResearchInputError(f"{label} must be a finite number") + try: + number = float(value) # type: ignore[arg-type] + except (TypeError, ValueError) as exc: + raise ResearchInputError(f"{label} must be a finite number") from exc + if not math.isfinite(number): + raise ResearchInputError(f"{label} must be finite") + return number + + +def simple_returns(values: Iterable[object], label: str = "returns") -> tuple[float, ...]: + """Validate ordinary compounding simple returns: finite and strictly > -1. + + Missing values are never filled with zero; any non-finite entry rejects the + whole series. + """ + if isinstance(values, (str, bytes)): + raise ResearchInputError(f"{label} must be a sequence of numbers") + try: + items = list(values) + except TypeError as exc: + raise ResearchInputError(f"{label} must be a sequence of numbers") from exc + out: list[float] = [] + for index, item in enumerate(items): + number = finite_float(item, f"{label}[{index}]") + if number <= -1.0: + raise ResearchInputError(f"{label}[{index}] must be greater than -1") + out.append(number) + return tuple(out) + + +def positive_int(value: object, label: str, *, minimum: int = 1) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ResearchInputError(f"{label} must be an integer") + if value < minimum: + raise ResearchInputError(f"{label} must be >= {minimum}") + return value diff --git a/src/quant_platform_kit/research_stats/backtest_report.py b/src/quant_platform_kit/research_stats/backtest_report.py new file mode 100644 index 00000000..97d03f07 --- /dev/null +++ b/src/quant_platform_kit/research_stats/backtest_report.py @@ -0,0 +1,400 @@ +"""``qsl.backtest_report.v1`` dataclasses (research-only). + +The dataclasses mirror ``schemas/backtest-report.v1.schema.json``. Python +validation enforces the invariants that matter most for honesty; the JSON +Schema is the full structural contract and is validated in tests. + +Hard invariants: ``live_ready`` is always ``False`` and ``no_order`` always +``True``. A report is never an order, allocation, promotion or live permit. +A ``COMPUTED`` metric must carry a finite value; anything else carries +``value=None`` plus a ``reason_code`` (missing is never zero). +""" + +from __future__ import annotations + +import json +import math +import re +from dataclasses import dataclass, field +from datetime import date +from importlib import resources +from typing import Any, Mapping + +BACKTEST_REPORT_SCHEMA_VERSION = "qsl.backtest_report.v1" +SCHEMA_RESOURCE = "backtest-report.v1.schema.json" + +DATA_IDENTITIES = ("development", "locked_oos", "forward") +EXECUTION_TIMINGS = ("next_open", "next_close", "same_close_research_only") +METRIC_STATUSES = ("COMPUTED", "UNCOMPUTABLE", "NOT_COMPUTED") +REPORT_STATUSES = ("UNCOMPUTABLE", "PARKED", "DEVELOPMENT_ONLY", "OOS_EVALUATED") +VETO_OPERATORS = ("<", "<=", ">", ">=", "==") + +_SHA256 = re.compile(r"[0-9a-f]{64}") +_REVISION = re.compile(r"[0-9a-f]{40}") + + +class BacktestReportError(ValueError): + """Raised when a report would violate the v1 contract.""" + + +def _text(value: object, label: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise BacktestReportError(f"{label} must be a non-empty string") + return value + + +def _iso(value: object, label: str) -> date: + if not isinstance(value, str): + raise BacktestReportError(f"{label} must be an ISO date string") + try: + return date.fromisoformat(value) + except ValueError as exc: + raise BacktestReportError(f"{label} must be an ISO date string") from exc + + +def _number(value: object, label: str, *, minimum: float | None = None, exclusive: bool = False) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(float(value)): + raise BacktestReportError(f"{label} must be a finite number") + number = float(value) + if minimum is not None and (number <= minimum if exclusive else number < minimum): + raise BacktestReportError(f"{label} out of range") + return number + + +def _count(value: object, label: str, minimum: int) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise BacktestReportError(f"{label} must be an integer >= {minimum}") + return value + + +def _choice(value: object, label: str, options: tuple[str, ...]) -> str: + if value not in options: + raise BacktestReportError(f"{label} must be one of {options}") + return str(value) + + +def _sha(value: object, label: str) -> str: + if not isinstance(value, str) or not _SHA256.fullmatch(value): + raise BacktestReportError(f"{label} must be a lowercase SHA-256 hex digest") + return value + + +@dataclass(frozen=True) +class DataSpec: + manifest_sha256: str + source: str + license: str + calendar: str + start_date: str + end_date: str + observations: int + data_identity: str + + def __post_init__(self) -> None: + _sha(self.manifest_sha256, "data.manifest_sha256") + for name in ("source", "license", "calendar"): + _text(getattr(self, name), f"data.{name}") + if _iso(self.start_date, "data.start_date") > _iso(self.end_date, "data.end_date"): + raise BacktestReportError("data window is reversed") + _count(self.observations, "data.observations", 1) + _choice(self.data_identity, "data.data_identity", DATA_IDENTITIES) + + +@dataclass(frozen=True) +class TimingSpec: + signal_cutoff: str + execution: str + rebalance_rule: str + + def __post_init__(self) -> None: + _text(self.signal_cutoff, "timing.signal_cutoff") + _choice(self.execution, "timing.execution", EXECUTION_TIMINGS) + _text(self.rebalance_rule, "timing.rebalance_rule") + + +@dataclass(frozen=True) +class CostModelSpec: + model_id: str + commission_bps: float + half_spread_bps: float + impact_bps: float + scenario_multipliers: tuple[float, ...] + settlement_rule: str + whole_shares: bool + + def __post_init__(self) -> None: + _text(self.model_id, "cost_model.model_id") + for name in ("commission_bps", "half_spread_bps", "impact_bps"): + _number(getattr(self, name), f"cost_model.{name}", minimum=0.0) + multipliers = tuple(self.scenario_multipliers) + if not multipliers or len(set(multipliers)) != len(multipliers): + raise BacktestReportError("cost_model.scenario_multipliers must be unique and non-empty") + for value in multipliers: + _number(value, "cost_model.scenario_multipliers[]", minimum=0.0, exclusive=True) + object.__setattr__(self, "scenario_multipliers", multipliers) + _text(self.settlement_rule, "cost_model.settlement_rule") + if not isinstance(self.whole_shares, bool): + raise BacktestReportError("cost_model.whole_shares must be a bool") + + +@dataclass(frozen=True) +class BenchmarkSpec: + name: str + rationale: str + same_dates: bool = True + + def __post_init__(self) -> None: + _text(self.name, "benchmark.name") + _text(self.rationale, "benchmark.rationale") + if self.same_dates is not True: + raise BacktestReportError("benchmark comparisons must use the same dates (no silent intersection)") + + +@dataclass(frozen=True) +class TrialSummary: + total: int + failed: int + rejected: int + selection_rule: str + trial_log_sha256: str + + def __post_init__(self) -> None: + _count(self.total, "trials.total", 1) + _count(self.failed, "trials.failed", 0) + _count(self.rejected, "trials.rejected", 0) + if self.failed + self.rejected > self.total: + raise BacktestReportError("failed + rejected trials cannot exceed total trials") + _text(self.selection_rule, "trials.selection_rule") + _sha(self.trial_log_sha256, "trials.trial_log_sha256") + + +@dataclass(frozen=True) +class MetricEntry: + name: str + scenario_multiplier: float + status: str + value: float | None = None + reason_code: str | None = None + + def __post_init__(self) -> None: + _text(self.name, "metrics[].name") + _number(self.scenario_multiplier, "metrics[].scenario_multiplier", minimum=0.0, exclusive=True) + _choice(self.status, "metrics[].status", METRIC_STATUSES) + if self.status == "COMPUTED": + _number(self.value, f"metrics[{self.name}].value") + if self.reason_code is not None: + raise BacktestReportError("computed metrics must not carry a reason_code") + else: + if self.value is not None: + raise BacktestReportError("non-computed metrics must have value=None (never a placeholder)") + _text(self.reason_code, f"metrics[{self.name}].reason_code") + + +@dataclass(frozen=True) +class UncertaintySpec: + mean_block_length: float + n_resamples: int + seed: int + confidence: float + method: str = "stationary_bootstrap" + + def __post_init__(self) -> None: + if self.method != "stationary_bootstrap": + raise BacktestReportError("uncertainty.method must be stationary_bootstrap") + _number(self.mean_block_length, "uncertainty.mean_block_length", minimum=1.0) + _count(self.n_resamples, "uncertainty.n_resamples", 100) + _count(self.seed, "uncertainty.seed", 0) + level = _number(self.confidence, "uncertainty.confidence") + if not 0.0 < level < 1.0: + raise BacktestReportError("uncertainty.confidence must be in (0, 1)") + + +@dataclass(frozen=True) +class WalkForwardFold: + train_start: str + train_end: str + test_start: str + test_end: str + + def __post_init__(self) -> None: + ts, te = _iso(self.train_start, "train_start"), _iso(self.train_end, "train_end") + vs, ve = _iso(self.test_start, "test_start"), _iso(self.test_end, "test_end") + if ts > te or vs > ve or te >= vs: + raise BacktestReportError("walk-forward fold must be ordered train < test") + + +@dataclass(frozen=True) +class WalkForwardSpec: + protocol: str + folds: tuple[WalkForwardFold, ...] = () + locked_oos: tuple[str, str] | None = None + + def __post_init__(self) -> None: + _text(self.protocol, "walk_forward.protocol") + object.__setattr__(self, "folds", tuple(self.folds)) + if self.locked_oos is not None: + start, end = self.locked_oos + if _iso(start, "locked_oos.start") > _iso(end, "locked_oos.end"): + raise BacktestReportError("locked OOS window is reversed") + object.__setattr__(self, "locked_oos", (start, end)) + + +@dataclass(frozen=True) +class VetoItem: + metric: str + op: str + threshold: float + frozen_at: str + frozen_by: str + + def __post_init__(self) -> None: + _text(self.metric, "veto.metric") + _choice(self.op, "veto.op", VETO_OPERATORS) + _number(self.threshold, "veto.threshold") + _text(self.frozen_at, "veto.frozen_at") + _text(self.frozen_by, "veto.frozen_by") + + +@dataclass(frozen=True) +class ReportGates: + report_items: tuple[str, ...] = () + ranking_items: tuple[str, ...] = () + veto_items: tuple[VetoItem, ...] = () + + def __post_init__(self) -> None: + for name in ("report_items", "ranking_items", "veto_items"): + object.__setattr__(self, name, tuple(getattr(self, name))) + for item in self.report_items + self.ranking_items: + _text(item, "gates item") + + +@dataclass(frozen=True) +class BacktestReportV1: + report_id: str + candidate_id: str + source_revision: str + created_at: str + data: DataSpec + timing: TimingSpec + cost_model: CostModelSpec + benchmark: BenchmarkSpec + trials: TrialSummary + walk_forward: WalkForwardSpec + status: str + metrics: tuple[MetricEntry, ...] = () + uncertainty: UncertaintySpec | None = None + gates: ReportGates = field(default_factory=ReportGates) + limitations: tuple[str, ...] = () + live_ready: bool = False + no_order: bool = True + schema_version: str = BACKTEST_REPORT_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.schema_version != BACKTEST_REPORT_SCHEMA_VERSION: + raise BacktestReportError("unsupported schema_version") + if self.live_ready is not False or self.no_order is not True: + raise BacktestReportError("research reports are always live_ready=False and no_order=True") + _text(self.report_id, "report_id") + _text(self.candidate_id, "candidate_id") + if not isinstance(self.source_revision, str) or not _REVISION.fullmatch(self.source_revision): + raise BacktestReportError("source_revision must be a lowercase 40-character Git revision") + _text(self.created_at, "created_at") + _choice(self.status, "status", REPORT_STATUSES) + if self.status == "OOS_EVALUATED" and self.data.data_identity == "development": + raise BacktestReportError("development data cannot be reported as OOS_EVALUATED") + object.__setattr__(self, "metrics", tuple(self.metrics)) + object.__setattr__(self, "limitations", tuple(self.limitations)) + for item in self.limitations: + _text(item, "limitations[]") + declared = set(self.cost_model.scenario_multipliers) + for metric in self.metrics: + if metric.scenario_multiplier not in declared: + raise BacktestReportError("metric scenario_multiplier is not declared in cost_model") + + def to_dict(self) -> dict[str, Any]: + d, t, c, b, tr, wf, g = ( + self.data, self.timing, self.cost_model, self.benchmark, self.trials, self.walk_forward, self.gates + ) + return { + "schema_version": self.schema_version, + "report_id": self.report_id, + "candidate_id": self.candidate_id, + "source_revision": self.source_revision, + "created_at": self.created_at, + "data": { + "manifest_sha256": d.manifest_sha256, + "source": d.source, + "license": d.license, + "calendar": d.calendar, + "start_date": d.start_date, + "end_date": d.end_date, + "observations": d.observations, + "data_identity": d.data_identity, + }, + "timing": {"signal_cutoff": t.signal_cutoff, "execution": t.execution, "rebalance_rule": t.rebalance_rule}, + "cost_model": { + "model_id": c.model_id, + "commission_bps": c.commission_bps, + "half_spread_bps": c.half_spread_bps, + "impact_bps": c.impact_bps, + "scenario_multipliers": list(c.scenario_multipliers), + "settlement_rule": c.settlement_rule, + "whole_shares": c.whole_shares, + }, + "benchmark": {"name": b.name, "rationale": b.rationale, "same_dates": b.same_dates}, + "trials": { + "total": tr.total, + "failed": tr.failed, + "rejected": tr.rejected, + "selection_rule": tr.selection_rule, + "trial_log_sha256": tr.trial_log_sha256, + }, + "metrics": [ + { + "name": m.name, + "scenario_multiplier": m.scenario_multiplier, + "status": m.status, + "value": m.value, + "reason_code": m.reason_code, + } + for m in self.metrics + ], + "uncertainty": None + if self.uncertainty is None + else { + "method": self.uncertainty.method, + "mean_block_length": self.uncertainty.mean_block_length, + "n_resamples": self.uncertainty.n_resamples, + "seed": self.uncertainty.seed, + "confidence": self.uncertainty.confidence, + }, + "walk_forward": { + "protocol": wf.protocol, + "folds": [ + {"train_start": f.train_start, "train_end": f.train_end, "test_start": f.test_start, "test_end": f.test_end} + for f in wf.folds + ], + "locked_oos": None if wf.locked_oos is None else {"start": wf.locked_oos[0], "end": wf.locked_oos[1]}, + }, + "gates": { + "report_items": list(g.report_items), + "ranking_items": list(g.ranking_items), + "veto_items": [ + {"metric": v.metric, "op": v.op, "threshold": v.threshold, "frozen_at": v.frozen_at, "frozen_by": v.frozen_by} + for v in g.veto_items + ], + }, + "status": self.status, + "live_ready": self.live_ready, + "no_order": self.no_order, + "limitations": list(self.limitations), + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True, indent=2) + + +def load_backtest_report_schema() -> Mapping[str, Any]: + """Load the packaged JSON Schema (no network).""" + text = (resources.files("quant_platform_kit") / "schemas" / SCHEMA_RESOURCE).read_text(encoding="utf-8") + return json.loads(text) diff --git a/src/quant_platform_kit/research_stats/bootstrap.py b/src/quant_platform_kit/research_stats/bootstrap.py new file mode 100644 index 00000000..bd943fe5 --- /dev/null +++ b/src/quant_platform_kit/research_stats/bootstrap.py @@ -0,0 +1,208 @@ +"""Stationary block bootstrap confidence intervals (research-only). + +Politis, D. N. and Romano, J. P. (1994), "The Stationary Bootstrap", JASA +89(428). Each resampled path is built from blocks with geometric lengths of +mean ``mean_block_length``; indices wrap around the sample end. This keeps +short-range serial dependence that an i.i.d. bootstrap would destroy, but it +is **not** a guarantee about future risk and does not correct for multiple +testing. + +Requires numpy from the optional ``research`` extra; numpy is imported lazily +so that importing ``quant_platform_kit.research_stats`` never needs it. + +The seed is mandatory: the same inputs, seed, block length and resample count +always reproduce the same interval. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from typing import Any, Iterable, Mapping + +from quant_platform_kit.research_stats._validation import ( + ResearchInputError, + finite_float, + positive_int, + simple_returns, +) +from quant_platform_kit.research_stats.sharpe_inference import ( + STATUS_COMPUTED, + STATUS_UNCOMPUTABLE, +) + +METHOD = "stationary_bootstrap" +SUPPORTED_STATISTICS = ("log_growth", "sharpe", "max_drawdown") +_MIN_OBSERVATIONS = 2 +_MIN_RESAMPLES = 100 + + +def _numpy() -> Any: + try: + import numpy as np # noqa: PLC0415 - optional research extra + except ImportError as exc: # pragma: no cover - exercised via subprocess test + raise ImportError( + "stationary_bootstrap_ci requires numpy; install 'quant-platform-kit[research]'" + ) from exc + return np + + +@dataclass(frozen=True) +class BootstrapInterval: + statistic: str + status: str + estimate: float | None + lower: float | None + upper: float | None + reason_code: str | None = None + + def to_dict(self) -> dict[str, object]: + return { + "statistic": self.statistic, + "status": self.status, + "estimate": self.estimate, + "lower": self.lower, + "upper": self.upper, + "reason_code": self.reason_code, + } + + +@dataclass(frozen=True) +class BootstrapResult: + status: str + method: str + observations: int + mean_block_length: float | None + n_resamples: int | None + seed: int | None + confidence: float | None + intervals: Mapping[str, BootstrapInterval] = field(default_factory=dict) + reason_code: str | None = None + + def to_dict(self) -> dict[str, object]: + return { + "status": self.status, + "method": self.method, + "observations": self.observations, + "mean_block_length": self.mean_block_length, + "n_resamples": self.n_resamples, + "seed": self.seed, + "confidence": self.confidence, + "intervals": {k: v.to_dict() for k, v in self.intervals.items()}, + "reason_code": self.reason_code, + } + + +def _stationary_indices(np: Any, rng: Any, n: int, n_resamples: int, mean_block_length: float) -> Any: + p_new_block = 1.0 / mean_block_length + starts = rng.integers(0, n, size=(n_resamples, n)) + new_block = rng.random((n_resamples, n)) < p_new_block + new_block[:, 0] = True + idx = np.empty((n_resamples, n), dtype=np.int64) + idx[:, 0] = starts[:, 0] + for t in range(1, n): + idx[:, t] = np.where(new_block[:, t], starts[:, t], (idx[:, t - 1] + 1) % n) + return idx + + +def _statistics(np: Any, paths: Any) -> dict[str, Any]: + """Row-wise statistics; returns NaN where a statistic is undefined.""" + log_growth = np.log1p(paths).mean(axis=1) + if paths.shape[1] >= 2: + std = paths.std(axis=1, ddof=1) + mean = paths.mean(axis=1) + with np.errstate(divide="ignore", invalid="ignore"): + sharpe = np.where(std > 0.0, mean / np.where(std > 0.0, std, 1.0), np.nan) + else: + sharpe = np.full(paths.shape[0], np.nan) + equity = np.cumprod(1.0 + paths, axis=1) + peak = np.maximum.accumulate(np.maximum(equity, 1.0), axis=1) + max_drawdown = np.minimum((equity / peak - 1.0).min(axis=1), 0.0) + return {"log_growth": log_growth, "sharpe": sharpe, "max_drawdown": max_drawdown} + + +def _failed(reason: str, observations: int = 0) -> BootstrapResult: + return BootstrapResult( + status=STATUS_UNCOMPUTABLE, + method=METHOD, + observations=observations, + mean_block_length=None, + n_resamples=None, + seed=None, + confidence=None, + reason_code=reason, + ) + + +def stationary_bootstrap_ci( + returns: Iterable[object], + *, + seed: int, + mean_block_length: object, + n_resamples: int = 1000, + confidence: object = 0.95, + statistics: Iterable[str] = SUPPORTED_STATISTICS, +) -> BootstrapResult: + """Percentile CIs for per-period log growth, per-period Sharpe and MDD. + + - ``log_growth``: mean of ``log(1 + r)`` per period (not annualized). + - ``sharpe``: mean / sample std (ddof=1) per period; rf/MAR removed by caller. + - ``max_drawdown``: most negative drawdown including initial equity 1. + + A statistic whose point estimate or any resample is undefined (for example + zero variance for Sharpe) is ``UNCOMPUTABLE`` for that statistic only. + """ + try: + if isinstance(seed, bool) or not isinstance(seed, int) or seed < 0: + raise ResearchInputError("seed must be an explicit non-negative integer") + series = simple_returns(returns) + block = finite_float(mean_block_length, "mean_block_length") + if block < 1.0: + raise ResearchInputError("mean_block_length must be >= 1") + resamples = positive_int(n_resamples, "n_resamples", minimum=_MIN_RESAMPLES) + level = finite_float(confidence, "confidence") + if not 0.0 < level < 1.0: + raise ResearchInputError("confidence must be in (0, 1)") + requested = tuple(statistics) + unknown = [s for s in requested if s not in SUPPORTED_STATISTICS] + if unknown or not requested: + raise ResearchInputError(f"unsupported statistics: {unknown or 'none'}") + except ResearchInputError: + return _failed("invalid_input") + n = len(series) + if n < _MIN_OBSERVATIONS: + return _failed("insufficient_observations", n) + if block > n: + return _failed("block_longer_than_sample", n) + + np = _numpy() + rng = np.random.default_rng(seed) + data = np.asarray(series, dtype=np.float64) + idx = _stationary_indices(np, rng, n, resamples, block) + resampled = _statistics(np, data[idx]) + point = _statistics(np, data[np.newaxis, :]) + alpha = (1.0 - level) / 2.0 + + intervals: dict[str, BootstrapInterval] = {} + for name in requested: + estimate = float(point[name][0]) + draws = resampled[name] + if not math.isfinite(estimate): + intervals[name] = BootstrapInterval(name, STATUS_UNCOMPUTABLE, None, None, None, "undefined_point_estimate") + continue + if not bool(np.all(np.isfinite(draws))): + intervals[name] = BootstrapInterval(name, STATUS_UNCOMPUTABLE, estimate, None, None, "degenerate_resample") + continue + lower, upper = np.quantile(draws, [alpha, 1.0 - alpha]) + intervals[name] = BootstrapInterval(name, STATUS_COMPUTED, estimate, float(lower), float(upper)) + + return BootstrapResult( + status=STATUS_COMPUTED, + method=METHOD, + observations=n, + mean_block_length=block, + n_resamples=resamples, + seed=seed, + confidence=level, + intervals=intervals, + ) diff --git a/src/quant_platform_kit/research_stats/cost_stress.py b/src/quant_platform_kit/research_stats/cost_stress.py new file mode 100644 index 00000000..5f461b2e --- /dev/null +++ b/src/quant_platform_kit/research_stats/cost_stress.py @@ -0,0 +1,143 @@ +"""Cost-stress recomputation of net returns (research-only, stdlib only). + +Convention (declared, not inferred): + +- ``turnover[t]`` is the **total traded notional** (buys + sells, each side + counted) at the start of period ``t``, as a fraction of pre-trade NAV. +- Every unit of traded notional pays ``cost_bps_per_side × multiplier``. +- The cost is deducted before the period return is earned: + ``net[t] = (1 − turnover[t]·c)·(1 + gross[t]) − 1``. + +Conventions that charge half of L1 turnover (e.g. a cash leg left unpriced) +must be converted by the caller before calling. Gross returns must already be +free of the costs being stressed; this function never deducts the same cost +twice and never fills missing values. + +Invalid or mismatched inputs return ``status="PARKED"`` with no numbers. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from typing import Iterable + +from quant_platform_kit.research_stats._validation import ( + ResearchInputError, + finite_float, + simple_returns, +) +from quant_platform_kit.research_stats.sharpe_inference import STATUS_COMPUTED + +STATUS_PARKED = "PARKED" +DEFAULT_MULTIPLIERS = (1.0, 2.0, 3.0) + + +@dataclass(frozen=True) +class CostStressScenario: + multiplier: float + cost_bps_per_side: float + net_returns: tuple[float, ...] + log_growth_per_period: float | None + gross_log_growth_per_period: float + total_cost_log_drag: float | None + ruined: bool = False + + def to_dict(self) -> dict[str, object]: + return { + "multiplier": self.multiplier, + "cost_bps_per_side": self.cost_bps_per_side, + "log_growth_per_period": self.log_growth_per_period, + "gross_log_growth_per_period": self.gross_log_growth_per_period, + "total_cost_log_drag": self.total_cost_log_drag, + "ruined": self.ruined, + "observations": len(self.net_returns), + } + + +@dataclass(frozen=True) +class CostStressResult: + status: str + base_cost_bps_per_side: float | None + scenarios: tuple[CostStressScenario, ...] = field(default_factory=tuple) + reason_code: str | None = None + + def scenario(self, multiplier: float) -> CostStressScenario: + for item in self.scenarios: + if item.multiplier == multiplier: + return item + raise KeyError(multiplier) + + def to_dict(self) -> dict[str, object]: + return { + "status": self.status, + "base_cost_bps_per_side": self.base_cost_bps_per_side, + "scenarios": [s.to_dict() for s in self.scenarios], + "reason_code": self.reason_code, + } + + +def _parked(reason: str) -> CostStressResult: + return CostStressResult(status=STATUS_PARKED, base_cost_bps_per_side=None, reason_code=reason) + + +def cost_stress_recompute( + gross_returns: Iterable[object], + turnover: Iterable[object], + *, + cost_bps_per_side: object, + multipliers: Iterable[object] = DEFAULT_MULTIPLIERS, +) -> CostStressResult: + try: + gross = simple_returns(gross_returns, "gross_returns") + if isinstance(turnover, (str, bytes)): + raise ResearchInputError("turnover must be a sequence of numbers") + traded = tuple(finite_float(v, f"turnover[{i}]") for i, v in enumerate(list(turnover))) + if any(v < 0.0 for v in traded): + raise ResearchInputError("turnover must be non-negative") + base = finite_float(cost_bps_per_side, "cost_bps_per_side") + if base < 0.0: + raise ResearchInputError("cost_bps_per_side must be non-negative") + if isinstance(multipliers, (str, bytes)): + raise ResearchInputError("multipliers must be a sequence of numbers") + factors = tuple(finite_float(v, f"multipliers[{i}]") for i, v in enumerate(list(multipliers))) + if not factors or any(f <= 0.0 for f in factors) or len(set(factors)) != len(factors): + raise ResearchInputError("multipliers must be unique positive numbers") + except (ResearchInputError, TypeError): + return _parked("invalid_input") + if not gross: + return _parked("empty_returns") + if len(traded) != len(gross): + return _parked("length_mismatch") + + gross_log = [math.log1p(g) for g in gross] + gross_g = math.fsum(gross_log) / len(gross) + scenarios: list[CostStressScenario] = [] + for factor in factors: + cost_rate = base * factor / 10_000.0 + net: list[float] = [] + ruined = False + for g, tau in zip(gross, traded): + retained = 1.0 - tau * cost_rate + if retained <= 0.0: + ruined = True + net.append(-1.0) + continue + net.append(retained * (1.0 + g) - 1.0) + if ruined: + scenarios.append( + CostStressScenario(factor, base * factor, tuple(net), None, gross_g, None, ruined=True) + ) + continue + net_log = [math.log1p(r) for r in net] + scenarios.append( + CostStressScenario( + multiplier=factor, + cost_bps_per_side=base * factor, + net_returns=tuple(net), + log_growth_per_period=math.fsum(net_log) / len(net), + gross_log_growth_per_period=gross_g, + total_cost_log_drag=math.fsum(gross_log) - math.fsum(net_log), + ) + ) + return CostStressResult(status=STATUS_COMPUTED, base_cost_bps_per_side=base, scenarios=tuple(scenarios)) diff --git a/src/quant_platform_kit/research_stats/sharpe_inference.py b/src/quant_platform_kit/research_stats/sharpe_inference.py new file mode 100644 index 00000000..00eed4e5 --- /dev/null +++ b/src/quant_platform_kit/research_stats/sharpe_inference.py @@ -0,0 +1,234 @@ +"""Probabilistic and Deflated Sharpe Ratio (research-only, stdlib only). + +References: +- Bailey, D. H. and López de Prado, M. (2012), "The Sharpe Ratio Efficient + Frontier", Journal of Risk 15(2) -- Probabilistic Sharpe Ratio (PSR). +- Bailey, D. H. and López de Prado, M. (2014), "The Deflated Sharpe Ratio: + Correcting for Selection Bias, Backtest Overfitting and Non-Normality", + Journal of Portfolio Management 40(5) -- DSR. + +All Sharpe ratios here are **per-period and non-annualized** (mean / sample +standard deviation of the same simple-return frequency, rf/MAR already +removed by the caller). Mixing frequencies or annualized trial Sharpes is a +caller error that this module cannot detect. + +Fail-closed: any condition under which the estimate is undefined returns +``status="UNCOMPUTABLE"`` with ``value=None`` and a ``reason_code``. A missing +estimate is never reported as zero. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from statistics import NormalDist +from typing import Iterable, Mapping, Sequence + +from quant_platform_kit.research_stats._validation import ( + ResearchInputError, + finite_float, + positive_int, + simple_returns, +) + +STATUS_COMPUTED = "COMPUTED" +STATUS_UNCOMPUTABLE = "UNCOMPUTABLE" + +EULER_MASCHERONI = 0.5772156649015329 +# Default observation floor mirrors the existing QPK Kelly contract v2 +# minimum sample (30). It is a structural floor, not a quality threshold. +DEFAULT_MIN_OBSERVATIONS = 30 +_MIN_TRIALS = 2 +_STD_NORMAL = NormalDist() + + +@dataclass(frozen=True) +class StatResult: + name: str + status: str + value: float | None + reason_code: str | None = None + details: Mapping[str, object] = field(default_factory=dict) + + @property + def computed(self) -> bool: + return self.status == STATUS_COMPUTED + + def to_dict(self) -> dict[str, object]: + return { + "name": self.name, + "status": self.status, + "value": self.value, + "reason_code": self.reason_code, + "details": dict(self.details), + } + + +def _uncomputable(name: str, reason: str, **details: object) -> StatResult: + return StatResult(name=name, status=STATUS_UNCOMPUTABLE, value=None, reason_code=reason, details=details) + + +def _moments(returns: Sequence[float]) -> tuple[float, float, float, float]: + """Return (per-period Sharpe, sample std, skewness, non-excess kurtosis). + + Sharpe uses the sample standard deviation (ddof=1). Skewness and kurtosis + are the population (biased) moment estimators used by Bailey & López de + Prado; kurtosis is non-excess (normal = 3). + """ + n = len(returns) + mean = math.fsum(returns) / n + deviations = [r - mean for r in returns] + m2 = math.fsum(d * d for d in deviations) / n + if m2 <= 0.0: + return float("nan"), 0.0, float("nan"), float("nan") + sample_std = math.sqrt(math.fsum(d * d for d in deviations) / (n - 1)) + m3 = math.fsum(d ** 3 for d in deviations) / n + m4 = math.fsum(d ** 4 for d in deviations) / n + skew = m3 / m2 ** 1.5 + kurt = m4 / m2 ** 2 + return mean / sample_std, sample_std, skew, kurt + + +def _psr_from_moments( + sharpe: float, benchmark: float, n: int, skew: float, kurt: float +) -> tuple[float | None, str | None, float | None]: + variance_term = 1.0 - skew * sharpe + (kurt - 1.0) / 4.0 * sharpe * sharpe + if not math.isfinite(variance_term) or variance_term <= 0.0: + return None, "non_positive_sharpe_variance", None + z = (sharpe - benchmark) * math.sqrt(n - 1) / math.sqrt(variance_term) + if not math.isfinite(z): + return None, "non_finite_z_score", None + return _STD_NORMAL.cdf(z), None, z + + +def probabilistic_sharpe_ratio( + returns: Iterable[object], + *, + sr_benchmark: object = 0.0, + min_observations: int = DEFAULT_MIN_OBSERVATIONS, +) -> StatResult: + """PSR: probability that the true per-period Sharpe exceeds ``sr_benchmark``. + + ``PSR = Φ((SR − SR*)·√(n−1) / √(1 − γ3·SR + (γ4−1)/4·SR²))``. + """ + name = "probabilistic_sharpe_ratio" + try: + floor = positive_int(min_observations, "min_observations", minimum=3) + series = simple_returns(returns) + benchmark = finite_float(sr_benchmark, "sr_benchmark") + except ResearchInputError as exc: + return _uncomputable(name, "invalid_input", message=str(exc)) + n = len(series) + if n < floor: + return _uncomputable(name, "insufficient_observations", observations=n, min_observations=floor) + sharpe, std, skew, kurt = _moments(series) + if std <= 0.0 or not math.isfinite(sharpe): + return _uncomputable(name, "zero_variance", observations=n) + value, reason, z = _psr_from_moments(sharpe, benchmark, n, skew, kurt) + details = { + "observations": n, + "sharpe_per_period": sharpe, + "sr_benchmark": benchmark, + "skewness": skew, + "kurtosis": kurt, + } + if value is None: + return _uncomputable(name, reason or "uncomputable", **details) + details["z_score"] = z + return StatResult(name=name, status=STATUS_COMPUTED, value=value, details=details) + + +def expected_max_sharpe(trial_sharpe_variance: object, n_trials: object) -> StatResult: + """Expected maximum Sharpe under the null of zero skill across ``n_trials``. + + ``SR0 = √V · ((1−γ)·Φ⁻¹(1 − 1/N) + γ·Φ⁻¹(1 − 1/(N·e)))``. + """ + name = "expected_max_sharpe" + try: + variance = finite_float(trial_sharpe_variance, "trial_sharpe_variance") + trials = positive_int(n_trials, "n_trials", minimum=1) + except ResearchInputError as exc: + return _uncomputable(name, "invalid_input", message=str(exc)) + if trials < _MIN_TRIALS: + return _uncomputable(name, "insufficient_trials", n_trials=trials, min_trials=_MIN_TRIALS) + if variance <= 0.0: + return _uncomputable(name, "zero_trial_sharpe_variance", n_trials=trials) + gamma = EULER_MASCHERONI + value = math.sqrt(variance) * ( + (1.0 - gamma) * _STD_NORMAL.inv_cdf(1.0 - 1.0 / trials) + + gamma * _STD_NORMAL.inv_cdf(1.0 - 1.0 / (trials * math.e)) + ) + return StatResult( + name=name, + status=STATUS_COMPUTED, + value=value, + details={"n_trials": trials, "trial_sharpe_variance": variance}, + ) + + +def deflated_sharpe_ratio( + returns: Iterable[object], + trial_sharpes: Iterable[object], + *, + n_effective_trials: int | None = None, + min_observations: int = DEFAULT_MIN_OBSERVATIONS, +) -> StatResult: + """DSR: PSR evaluated against the expected maximum Sharpe of all trials. + + ``trial_sharpes`` must contain the per-period Sharpe of **every** trial + (including failed/rejected ones), on the same frequency as ``returns``. + ``n_effective_trials`` may override the trial count (e.g. after + clustering correlated trials) but never below two. Fewer than two trials, + zero trial-Sharpe variance or too few observations are ``UNCOMPUTABLE``. + """ + name = "deflated_sharpe_ratio" + try: + floor = positive_int(min_observations, "min_observations", minimum=3) + series = simple_returns(returns) + if isinstance(trial_sharpes, (str, bytes)): + raise ResearchInputError("trial_sharpes must be a sequence of numbers") + trials = [finite_float(v, f"trial_sharpes[{i}]") for i, v in enumerate(list(trial_sharpes))] + if n_effective_trials is not None: + effective = positive_int(n_effective_trials, "n_effective_trials", minimum=1) + else: + effective = len(trials) + except (ResearchInputError, TypeError) as exc: + return _uncomputable(name, "invalid_input", message=str(exc)) + if len(trials) < _MIN_TRIALS or effective < _MIN_TRIALS: + return _uncomputable( + name, + "insufficient_trials", + recorded_trials=len(trials), + n_effective_trials=effective, + min_trials=_MIN_TRIALS, + ) + if effective > len(trials): + return _uncomputable( + name, "effective_trials_exceed_recorded", recorded_trials=len(trials), n_effective_trials=effective + ) + n = len(series) + if n < floor: + return _uncomputable(name, "insufficient_observations", observations=n, min_observations=floor) + trial_mean = math.fsum(trials) / len(trials) + trial_variance = math.fsum((t - trial_mean) ** 2 for t in trials) / (len(trials) - 1) + sr0 = expected_max_sharpe(trial_variance, effective) + if not sr0.computed or sr0.value is None: + return _uncomputable(name, sr0.reason_code or "uncomputable", **dict(sr0.details)) + sharpe, std, skew, kurt = _moments(series) + if std <= 0.0 or not math.isfinite(sharpe): + return _uncomputable(name, "zero_variance", observations=n) + value, reason, z = _psr_from_moments(sharpe, sr0.value, n, skew, kurt) + details = { + "observations": n, + "sharpe_per_period": sharpe, + "expected_max_sharpe": sr0.value, + "recorded_trials": len(trials), + "n_effective_trials": effective, + "trial_sharpe_variance": trial_variance, + "skewness": skew, + "kurtosis": kurt, + } + if value is None: + return _uncomputable(name, reason or "uncomputable", **details) + details["z_score"] = z + return StatResult(name=name, status=STATUS_COMPUTED, value=value, details=details) diff --git a/src/quant_platform_kit/schemas/backtest-report.v1.schema.json b/src/quant_platform_kit/schemas/backtest-report.v1.schema.json new file mode 100644 index 00000000..bf6c300f --- /dev/null +++ b/src/quant_platform_kit/schemas/backtest-report.v1.schema.json @@ -0,0 +1,506 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://quantstrategylab.github.io/schemas/backtest-report.v1.schema.json", + "title": "QSL research backtest report v1", + "description": "Research-only backtest report template (RS-01). Structure only: it does not certify economic validity and never grants paper/shadow/live authority.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "report_id", + "candidate_id", + "source_revision", + "created_at", + "data", + "timing", + "cost_model", + "benchmark", + "trials", + "metrics", + "uncertainty", + "walk_forward", + "gates", + "status", + "live_ready", + "no_order", + "limitations" + ], + "$defs": { + "nonEmpty": { + "type": "string", + "minLength": 1 + }, + "isoDate": { + "type": "string", + "pattern": "^\\d{4}-\\d{2}-\\d{2}$" + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "nonNegative": { + "type": "number", + "minimum": 0 + }, + "dateRange": { + "type": "object", + "additionalProperties": false, + "required": [ + "start", + "end" + ], + "properties": { + "start": { + "$ref": "#/$defs/isoDate" + }, + "end": { + "$ref": "#/$defs/isoDate" + } + } + } + }, + "properties": { + "schema_version": { + "const": "qsl.backtest_report.v1" + }, + "report_id": { + "$ref": "#/$defs/nonEmpty" + }, + "candidate_id": { + "$ref": "#/$defs/nonEmpty" + }, + "source_revision": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "created_at": { + "$ref": "#/$defs/nonEmpty" + }, + "data": { + "type": "object", + "additionalProperties": false, + "required": [ + "manifest_sha256", + "source", + "license", + "calendar", + "start_date", + "end_date", + "observations", + "data_identity" + ], + "properties": { + "manifest_sha256": { + "$ref": "#/$defs/sha256" + }, + "source": { + "$ref": "#/$defs/nonEmpty" + }, + "license": { + "$ref": "#/$defs/nonEmpty" + }, + "calendar": { + "$ref": "#/$defs/nonEmpty" + }, + "start_date": { + "$ref": "#/$defs/isoDate" + }, + "end_date": { + "$ref": "#/$defs/isoDate" + }, + "observations": { + "type": "integer", + "minimum": 1 + }, + "data_identity": { + "enum": [ + "development", + "locked_oos", + "forward" + ] + } + } + }, + "timing": { + "type": "object", + "additionalProperties": false, + "required": [ + "signal_cutoff", + "execution", + "rebalance_rule" + ], + "properties": { + "signal_cutoff": { + "$ref": "#/$defs/nonEmpty" + }, + "execution": { + "enum": [ + "next_open", + "next_close", + "same_close_research_only" + ] + }, + "rebalance_rule": { + "$ref": "#/$defs/nonEmpty" + } + } + }, + "cost_model": { + "type": "object", + "additionalProperties": false, + "required": [ + "model_id", + "commission_bps", + "half_spread_bps", + "impact_bps", + "scenario_multipliers", + "settlement_rule", + "whole_shares" + ], + "properties": { + "model_id": { + "$ref": "#/$defs/nonEmpty" + }, + "commission_bps": { + "$ref": "#/$defs/nonNegative" + }, + "half_spread_bps": { + "$ref": "#/$defs/nonNegative" + }, + "impact_bps": { + "$ref": "#/$defs/nonNegative" + }, + "scenario_multipliers": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "number", + "exclusiveMinimum": 0 + } + }, + "settlement_rule": { + "$ref": "#/$defs/nonEmpty" + }, + "whole_shares": { + "type": "boolean" + } + } + }, + "benchmark": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "rationale", + "same_dates" + ], + "properties": { + "name": { + "$ref": "#/$defs/nonEmpty" + }, + "rationale": { + "$ref": "#/$defs/nonEmpty" + }, + "same_dates": { + "const": true + } + } + }, + "trials": { + "type": "object", + "additionalProperties": false, + "required": [ + "total", + "failed", + "rejected", + "selection_rule", + "trial_log_sha256" + ], + "properties": { + "total": { + "type": "integer", + "minimum": 1 + }, + "failed": { + "type": "integer", + "minimum": 0 + }, + "rejected": { + "type": "integer", + "minimum": 0 + }, + "selection_rule": { + "$ref": "#/$defs/nonEmpty" + }, + "trial_log_sha256": { + "$ref": "#/$defs/sha256" + } + } + }, + "metrics": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "scenario_multiplier", + "status", + "value", + "reason_code" + ], + "properties": { + "name": { + "$ref": "#/$defs/nonEmpty" + }, + "scenario_multiplier": { + "type": "number", + "exclusiveMinimum": 0 + }, + "status": { + "enum": [ + "COMPUTED", + "UNCOMPUTABLE", + "NOT_COMPUTED" + ] + }, + "value": { + "type": [ + "number", + "null" + ] + }, + "reason_code": { + "type": [ + "string", + "null" + ] + } + }, + "if": { + "properties": { + "status": { + "const": "COMPUTED" + } + } + }, + "then": { + "properties": { + "value": { + "type": "number" + }, + "reason_code": { + "type": "null" + } + } + }, + "else": { + "properties": { + "value": { + "type": "null" + }, + "reason_code": { + "$ref": "#/$defs/nonEmpty" + } + } + } + } + }, + "uncertainty": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "method", + "mean_block_length", + "n_resamples", + "seed", + "confidence" + ], + "properties": { + "method": { + "const": "stationary_bootstrap" + }, + "mean_block_length": { + "type": "number", + "minimum": 1 + }, + "n_resamples": { + "type": "integer", + "minimum": 100 + }, + "seed": { + "type": "integer", + "minimum": 0 + }, + "confidence": { + "type": "number", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1 + } + } + } + ] + }, + "walk_forward": { + "type": "object", + "additionalProperties": false, + "required": [ + "protocol", + "folds", + "locked_oos" + ], + "properties": { + "protocol": { + "$ref": "#/$defs/nonEmpty" + }, + "folds": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "train_start", + "train_end", + "test_start", + "test_end" + ], + "properties": { + "train_start": { + "$ref": "#/$defs/isoDate" + }, + "train_end": { + "$ref": "#/$defs/isoDate" + }, + "test_start": { + "$ref": "#/$defs/isoDate" + }, + "test_end": { + "$ref": "#/$defs/isoDate" + } + } + } + }, + "locked_oos": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/$defs/dateRange" + } + ] + } + } + }, + "gates": { + "type": "object", + "additionalProperties": false, + "required": [ + "report_items", + "ranking_items", + "veto_items" + ], + "properties": { + "report_items": { + "type": "array", + "items": { + "$ref": "#/$defs/nonEmpty" + } + }, + "ranking_items": { + "type": "array", + "items": { + "$ref": "#/$defs/nonEmpty" + } + }, + "veto_items": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "metric", + "op", + "threshold", + "frozen_at", + "frozen_by" + ], + "properties": { + "metric": { + "$ref": "#/$defs/nonEmpty" + }, + "op": { + "enum": [ + "<", + "<=", + ">", + ">=", + "==" + ] + }, + "threshold": { + "type": "number" + }, + "frozen_at": { + "$ref": "#/$defs/nonEmpty" + }, + "frozen_by": { + "$ref": "#/$defs/nonEmpty" + } + } + } + } + } + }, + "status": { + "enum": [ + "UNCOMPUTABLE", + "PARKED", + "DEVELOPMENT_ONLY", + "OOS_EVALUATED" + ] + }, + "live_ready": { + "const": false + }, + "no_order": { + "const": true + }, + "limitations": { + "type": "array", + "items": { + "$ref": "#/$defs/nonEmpty" + } + } + }, + "if": { + "properties": { + "status": { + "const": "OOS_EVALUATED" + } + }, + "required": [ + "status" + ] + }, + "then": { + "properties": { + "data": { + "properties": { + "data_identity": { + "enum": [ + "locked_oos", + "forward" + ] + } + } + } + } + } +} diff --git a/tests/test_research_stats_backtest_report.py b/tests/test_research_stats_backtest_report.py new file mode 100644 index 00000000..7537ef60 --- /dev/null +++ b/tests/test_research_stats_backtest_report.py @@ -0,0 +1,144 @@ +"""Contract tests for qsl.backtest_report.v1. Synthetic placeholders only.""" + +from __future__ import annotations + +import copy +import dataclasses +import json +import random + +import pytest +from jsonschema import Draft202012Validator + +from quant_platform_kit.research_stats import ( + BacktestReportV1, + BenchmarkSpec, + CostModelSpec, + DataSpec, + MetricEntry, + ReportGates, + TimingSpec, + TrialSummary, + UncertaintySpec, + VetoItem, + WalkForwardFold, + WalkForwardSpec, + load_backtest_report_schema, + probabilistic_sharpe_ratio, +) +from quant_platform_kit.research_stats.backtest_report import BacktestReportError + +SYNTHETIC_SHA = "a" * 64 + + +def _validator() -> Draft202012Validator: + schema = load_backtest_report_schema() + Draft202012Validator.check_schema(schema) + return Draft202012Validator(schema) + + +def _report(**overrides) -> BacktestReportV1: + rng = random.Random(5) + synthetic = [rng.gauss(0.0, 0.01) for _ in range(60)] + psr = probabilistic_sharpe_ratio(synthetic) + fields = dict( + report_id="synthetic-report", + candidate_id="synthetic-candidate", + source_revision="0" * 40, + created_at="2026-10-10T00:00:00Z", + data=DataSpec(SYNTHETIC_SHA, "synthetic-fixture", "n/a-synthetic", "XNYS", "2020-01-02", "2020-03-31", 60, "development"), + timing=TimingSpec("as_of close", "next_open", "monthly"), + cost_model=CostModelSpec("synthetic-cost", 1.0, 2.0, 0.0, (1.0, 2.0, 3.0), "T+2 sale settlement", True), + benchmark=BenchmarkSpec("synthetic-benchmark", "unleveraged underlying"), + trials=TrialSummary(3, 1, 1, "pre-declared: max log growth", SYNTHETIC_SHA), + walk_forward=WalkForwardSpec( + "purged_walk_forward.v1", + (WalkForwardFold("2020-01-02", "2020-01-31", "2020-02-10", "2020-02-28"),), + None, + ), + status="DEVELOPMENT_ONLY", + metrics=( + MetricEntry("psr", 1.0, psr.status, psr.value, psr.reason_code), + MetricEntry("dsr", 1.0, "NOT_COMPUTED", None, "trial_sharpes_not_supplied"), + ), + uncertainty=UncertaintySpec(5.0, 1000, 42, 0.95), + gates=ReportGates(("log_growth",), ("log_growth",), (VetoItem("log_growth", ">", 0.0, "2026-10-10", "human"),)), + limitations=("synthetic fixture; not a backtest result",), + ) + fields.update(overrides) + return BacktestReportV1(**fields) + + +def test_valid_report_matches_schema_and_round_trips() -> None: + report = _report() + payload = report.to_dict() + _validator().validate(payload) + assert json.loads(report.to_json()) == payload + assert payload["live_ready"] is False and payload["no_order"] is True + assert payload["schema_version"] == "qsl.backtest_report.v1" + + +def test_live_flags_cannot_be_flipped() -> None: + with pytest.raises(BacktestReportError): + _report(live_ready=True) + with pytest.raises(BacktestReportError): + _report(no_order=False) + payload = _report().to_dict() + for key, value in (("live_ready", True), ("no_order", False)): + bad = copy.deepcopy(payload) + bad[key] = value + assert list(_validator().iter_errors(bad)) + + +def test_missing_metric_is_never_a_number() -> None: + with pytest.raises(BacktestReportError): + MetricEntry("dsr", 1.0, "UNCOMPUTABLE", 0.0, "insufficient_trials") + with pytest.raises(BacktestReportError): + MetricEntry("dsr", 1.0, "COMPUTED", None) + with pytest.raises(BacktestReportError): + MetricEntry("dsr", 1.0, "NOT_COMPUTED", None, None) + bad = _report().to_dict() + bad["metrics"][1]["value"] = 0.0 + assert list(_validator().iter_errors(bad)) + bad = _report().to_dict() + bad["metrics"][0]["value"] = None + assert list(_validator().iter_errors(bad)) + + +def test_development_data_cannot_be_labelled_oos() -> None: + with pytest.raises(BacktestReportError): + _report(status="OOS_EVALUATED") + bad = _report().to_dict() + bad["status"] = "OOS_EVALUATED" + assert list(_validator().iter_errors(bad)) + ok = _report().to_dict() + ok["status"] = "OOS_EVALUATED" + ok["data"]["data_identity"] = "locked_oos" + _validator().validate(ok) + + +def test_other_python_invariants() -> None: + with pytest.raises(BacktestReportError): + TrialSummary(2, 2, 1, "rule", SYNTHETIC_SHA) + with pytest.raises(BacktestReportError): + BenchmarkSpec("b", "r", same_dates=False) + with pytest.raises(BacktestReportError): + TimingSpec("as_of close", "same_close", "monthly") + with pytest.raises(BacktestReportError): + _report(metrics=(MetricEntry("psr", 4.0, "NOT_COMPUTED", None, "x"),)) + with pytest.raises(BacktestReportError): + _report(source_revision="abc") + with pytest.raises(BacktestReportError): + WalkForwardFold("2020-01-02", "2020-02-15", "2020-02-10", "2020-02-28") + + +def test_schema_rejects_unknown_fields() -> None: + bad = _report().to_dict() + bad["sharpe_pass"] = True + assert list(_validator().iter_errors(bad)) + + +def test_dataclass_is_frozen() -> None: + report = _report() + with pytest.raises(dataclasses.FrozenInstanceError): + report.live_ready = True # type: ignore[misc] diff --git a/tests/test_research_stats_bootstrap.py b/tests/test_research_stats_bootstrap.py new file mode 100644 index 00000000..014a9d04 --- /dev/null +++ b/tests/test_research_stats_bootstrap.py @@ -0,0 +1,98 @@ +"""Synthetic-only tests for the stationary block bootstrap. No real market data.""" + +from __future__ import annotations + +import random + +import numpy as np +import pytest + +from quant_platform_kit.research_stats import STATUS_COMPUTED, STATUS_UNCOMPUTABLE, stationary_bootstrap_ci +from quant_platform_kit.research_stats.bootstrap import _stationary_indices + + +def _synthetic(n: int = 300, seed: int = 11) -> list[float]: + rng = random.Random(seed) + return [0.0003 + rng.gauss(0.0, 0.012) for _ in range(n)] + + +def test_same_seed_reproduces_and_different_seed_differs() -> None: + xs = _synthetic() + a = stationary_bootstrap_ci(xs, seed=42, mean_block_length=5, n_resamples=300) + b = stationary_bootstrap_ci(xs, seed=42, mean_block_length=5, n_resamples=300) + c = stationary_bootstrap_ci(xs, seed=43, mean_block_length=5, n_resamples=300) + assert a.to_dict() == b.to_dict() + assert a.intervals["sharpe"].lower != c.intervals["sharpe"].lower + assert a.status == STATUS_COMPUTED and a.seed == 42 and a.method == "stationary_bootstrap" + + +def test_intervals_are_ordered_and_point_estimates_match_manual() -> None: + xs = _synthetic() + result = stationary_bootstrap_ci(xs, seed=1, mean_block_length=10, n_resamples=200, confidence=0.9) + for interval in result.intervals.values(): + assert interval.status == STATUS_COMPUTED + assert interval.lower <= interval.upper + arr = np.asarray(xs) + assert result.intervals["log_growth"].estimate == pytest.approx(float(np.log1p(arr).mean())) + assert result.intervals["sharpe"].estimate == pytest.approx(float(arr.mean() / arr.std(ddof=1))) + equity = np.cumprod(1 + arr) + peak = np.maximum.accumulate(np.maximum(equity, 1.0)) + assert result.intervals["max_drawdown"].estimate == pytest.approx(float((equity / peak - 1).min())) + + +def test_drawdown_counts_initial_equity() -> None: + result = stationary_bootstrap_ci([-0.1] + [0.01] * 5, seed=0, mean_block_length=1, n_resamples=100) + assert result.intervals["max_drawdown"].estimate == pytest.approx(-0.1) + + +def test_constant_series_sharpe_uncomputable_other_stats_degenerate() -> None: + result = stationary_bootstrap_ci([0.001] * 50, seed=3, mean_block_length=4, n_resamples=100) + sharpe = result.intervals["sharpe"] + assert sharpe.status == STATUS_UNCOMPUTABLE and sharpe.estimate is None and sharpe.lower is None + growth = result.intervals["log_growth"] + assert growth.status == STATUS_COMPUTED + assert growth.lower == pytest.approx(growth.estimate) and growth.upper == pytest.approx(growth.estimate) + assert result.intervals["max_drawdown"].estimate == 0.0 + + +@pytest.mark.parametrize( + ("kwargs", "reason"), + [ + ({"seed": True, "mean_block_length": 5}, "invalid_input"), + ({"seed": -1, "mean_block_length": 5}, "invalid_input"), + ({"seed": 1, "mean_block_length": 0.5}, "invalid_input"), + ({"seed": 1, "mean_block_length": 5, "n_resamples": 10}, "invalid_input"), + ({"seed": 1, "mean_block_length": 5, "confidence": 1.0}, "invalid_input"), + ({"seed": 1, "mean_block_length": 5, "statistics": ("cagr",)}, "invalid_input"), + ({"seed": 1, "mean_block_length": 500}, "block_longer_than_sample"), + ], +) +def test_invalid_inputs_fail_closed(kwargs: dict, reason: str) -> None: + result = stationary_bootstrap_ci(_synthetic(), **kwargs) + assert result.status == STATUS_UNCOMPUTABLE + assert result.reason_code == reason + assert result.intervals == {} + + +def test_seed_is_mandatory() -> None: + with pytest.raises(TypeError): + stationary_bootstrap_ci(_synthetic(), mean_block_length=5) # type: ignore[call-arg] + + +def test_missing_values_are_rejected_not_filled() -> None: + xs = _synthetic() + xs[10] = float("nan") + assert stationary_bootstrap_ci(xs, seed=1, mean_block_length=5).reason_code == "invalid_input" + + +def test_stationary_indices_follow_wrapped_blocks_with_expected_mean_length() -> None: + rng = np.random.default_rng(2026) + n, block = 200, 8.0 + idx = _stationary_indices(np, rng, n, 400, block) + steps = (idx[:, 1:] - idx[:, :-1]) % n + continuation = steps == 1 + # Every non-continuation is a fresh block start; observed mean block length + # is close to the requested one (deterministic under the fixed seed). + breaks = (~continuation).sum() + idx.shape[0] + observed_mean = idx.size / breaks + assert 0.85 * block < observed_mean < 1.15 * block diff --git a/tests/test_research_stats_cost_stress.py b/tests/test_research_stats_cost_stress.py new file mode 100644 index 00000000..df30ffa3 --- /dev/null +++ b/tests/test_research_stats_cost_stress.py @@ -0,0 +1,63 @@ +"""Synthetic-only tests for cost-stress recomputation. No real market data.""" + +from __future__ import annotations + +import math + +import pytest + +from quant_platform_kit.research_stats import cost_stress_recompute +from quant_platform_kit.research_stats.cost_stress import STATUS_PARKED + +GROSS = [0.01, -0.005, 0.002, 0.0, 0.004] +TURNOVER = [1.0, 0.0, 0.5, 0.0, 2.0] + + +def test_default_scenarios_are_1x_2x_3x_and_monotone() -> None: + result = cost_stress_recompute(GROSS, TURNOVER, cost_bps_per_side=10) + assert [s.multiplier for s in result.scenarios] == [1.0, 2.0, 3.0] + growth = [s.log_growth_per_period for s in result.scenarios] + assert growth[0] > growth[1] > growth[2] + drags = [s.total_cost_log_drag for s in result.scenarios] + assert 0 < drags[0] < drags[1] < drags[2] + + +def test_exact_formula() -> None: + result = cost_stress_recompute([0.02], [1.5], cost_bps_per_side=20, multipliers=[1]) + expected = (1 - 1.5 * 0.002) * 1.02 - 1 + assert result.scenario(1.0).net_returns[0] == pytest.approx(expected, rel=1e-15) + assert result.scenario(1.0).cost_bps_per_side == 20 + + +def test_zero_turnover_or_zero_cost_leaves_returns_unchanged() -> None: + for turnover, bps in (([0.0] * 5, 25), (TURNOVER, 0)): + result = cost_stress_recompute(GROSS, turnover, cost_bps_per_side=bps) + for scenario in result.scenarios: + assert scenario.net_returns == pytest.approx(tuple(GROSS)) + assert scenario.total_cost_log_drag == pytest.approx(0.0, abs=1e-15) + + +def test_ruin_is_flagged_without_log_growth() -> None: + result = cost_stress_recompute([0.01], [2.0], cost_bps_per_side=5000, multipliers=[1]) + scenario = result.scenario(1.0) + assert scenario.ruined is True + assert scenario.log_growth_per_period is None + + +@pytest.mark.parametrize( + ("gross", "turnover", "kwargs", "reason"), + [ + (GROSS, TURNOVER[:-1], {"cost_bps_per_side": 10}, "length_mismatch"), + (GROSS, [-0.1] + TURNOVER[1:], {"cost_bps_per_side": 10}, "invalid_input"), + ([math.nan] + GROSS[1:], TURNOVER, {"cost_bps_per_side": 10}, "invalid_input"), + (GROSS, TURNOVER, {"cost_bps_per_side": -1}, "invalid_input"), + (GROSS, TURNOVER, {"cost_bps_per_side": 10, "multipliers": [1, 1]}, "invalid_input"), + (GROSS, TURNOVER, {"cost_bps_per_side": 10, "multipliers": [0]}, "invalid_input"), + ([], [], {"cost_bps_per_side": 10}, "empty_returns"), + ], +) +def test_invalid_inputs_are_parked(gross, turnover, kwargs, reason) -> None: + result = cost_stress_recompute(gross, turnover, **kwargs) + assert result.status == STATUS_PARKED + assert result.reason_code == reason + assert result.scenarios == () diff --git a/tests/test_research_stats_import_boundary.py b/tests/test_research_stats_import_boundary.py new file mode 100644 index 00000000..433f0686 --- /dev/null +++ b/tests/test_research_stats_import_boundary.py @@ -0,0 +1,124 @@ +"""research_stats stays research-only. + +1. No module outside ``quant_platform_kit.research_stats`` imports it. +2. ``research_stats`` imports nothing from the rest of QPK (one-way boundary). +3. Importing core/production packages never loads it. +4. Importing it never requires numpy (optional ``research`` extra). +""" + +from __future__ import annotations + +import ast +import subprocess +import sys +import tomllib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +PKG = ROOT / "src" / "quant_platform_kit" +RESEARCH = PKG / "research_stats" +TARGET = "quant_platform_kit.research_stats" + + +def _imports(path: Path) -> list[str]: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + names: list[str] = [] + rel_parts = path.relative_to(PKG.parent).with_suffix("").parts + for node in ast.walk(tree): + if isinstance(node, ast.Import): + names.extend(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + if node.level: + base = list(rel_parts[: len(rel_parts) - node.level]) + module = ".".join(base + ([node.module] if node.module else [])) + else: + module = node.module or "" + names.append(module) + names.extend(f"{module}.{alias.name}" for alias in node.names) + return names + + +def test_no_production_module_imports_research_stats() -> None: + offenders = [] + for path in PKG.rglob("*.py"): + if RESEARCH in path.parents: + continue + for name in _imports(path): + if name == TARGET or name.startswith(TARGET + "."): + offenders.append(f"{path.relative_to(ROOT)} -> {name}") + assert offenders == [] + + +def test_research_stats_only_imports_itself_within_qpk() -> None: + offenders = [] + for path in RESEARCH.rglob("*.py"): + for name in _imports(path): + if name.startswith("quant_platform_kit") and not name.startswith(TARGET): + offenders.append(f"{path.relative_to(ROOT)} -> {name}") + assert offenders == [] + + +_PROBE = r""" +import importlib, pkgutil, sys +import quant_platform_kit +loaded = [] +for info in pkgutil.iter_modules(quant_platform_kit.__path__): + if info.name == "research_stats": + continue + try: + importlib.import_module(f"quant_platform_kit.{info.name}") + loaded.append(info.name) + except Exception: + pass # optional third-party SDKs may be absent in CI +bad = sorted(m for m in sys.modules if m == "quant_platform_kit.research_stats" or m.startswith("quant_platform_kit.research_stats.")) +print(len(loaded)) +sys.exit(1 if bad else 0) +""" + + +def test_importing_core_packages_does_not_load_research_stats() -> None: + proc = subprocess.run( + [sys.executable, "-c", _PROBE], + cwd=ROOT, + env={"PYTHONPATH": str(ROOT / "src"), "PATH": ""}, + capture_output=True, + text=True, + timeout=180, + ) + assert proc.returncode == 0, proc.stdout + proc.stderr + assert int(proc.stdout.strip().splitlines()[-1]) >= 5 + + +_NO_NUMPY = r""" +import sys +sys.modules["numpy"] = None +sys.modules["pandas"] = None +import quant_platform_kit.research_stats as rs +assert rs.probabilistic_sharpe_ratio([0.01, -0.02, 0.015] * 20).status == "COMPUTED" +assert rs.cost_stress_recompute([0.01], [1.0], cost_bps_per_side=5).status == "COMPUTED" +try: + rs.stationary_bootstrap_ci([0.01, -0.02] * 10, seed=1, mean_block_length=2) +except ImportError as exc: + assert "[research]" in str(exc) +else: + raise SystemExit("bootstrap should require numpy") +""" + + +def test_research_stats_imports_without_numpy() -> None: + proc = subprocess.run( + [sys.executable, "-c", _NO_NUMPY], + cwd=ROOT, + env={"PYTHONPATH": str(ROOT / "src"), "PATH": ""}, + capture_output=True, + text=True, + timeout=60, + ) + assert proc.returncode == 0, proc.stdout + proc.stderr + + +def test_numpy_pandas_are_only_an_optional_research_extra() -> None: + project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"] + assert project["dependencies"] == [] + research = project["optional-dependencies"]["research"] + assert sorted(research) == ["numpy", "pandas"] diff --git a/tests/test_research_stats_sharpe.py b/tests/test_research_stats_sharpe.py new file mode 100644 index 00000000..8991b2b7 --- /dev/null +++ b/tests/test_research_stats_sharpe.py @@ -0,0 +1,117 @@ +"""Synthetic-only tests for research_stats PSR/DSR. No real market data.""" + +from __future__ import annotations + +import math +import random +from statistics import NormalDist + +import pytest + +from quant_platform_kit.research_stats import ( + STATUS_COMPUTED, + STATUS_UNCOMPUTABLE, + deflated_sharpe_ratio, + expected_max_sharpe, + probabilistic_sharpe_ratio, +) + + +def _synthetic(n: int = 250, seed: int = 7, drift: float = 0.0005, scale: float = 0.01) -> list[float]: + rng = random.Random(seed) + return [drift + rng.gauss(0.0, scale) for _ in range(n)] + + +def _manual_moments(xs: list[float]) -> tuple[float, float, float]: + n = len(xs) + mean = sum(xs) / n + m2 = sum((x - mean) ** 2 for x in xs) / n + m3 = sum((x - mean) ** 3 for x in xs) / n + m4 = sum((x - mean) ** 4 for x in xs) / n + sd = math.sqrt(sum((x - mean) ** 2 for x in xs) / (n - 1)) + return mean / sd, m3 / m2 ** 1.5, m4 / m2 ** 2 + + +def test_psr_matches_independent_closed_form() -> None: + xs = _synthetic() + sr, skew, kurt = _manual_moments(xs) + z = sr * math.sqrt(len(xs) - 1) / math.sqrt(1 - skew * sr + (kurt - 1) / 4 * sr * sr) + result = probabilistic_sharpe_ratio(xs) + assert result.status == STATUS_COMPUTED + assert result.value == pytest.approx(NormalDist().cdf(z), rel=1e-12) + assert result.details["sharpe_per_period"] == pytest.approx(sr, rel=1e-12) + + +def test_psr_is_one_half_when_benchmark_equals_sample_sharpe() -> None: + xs = _synthetic() + sr = probabilistic_sharpe_ratio(xs).details["sharpe_per_period"] + assert probabilistic_sharpe_ratio(xs, sr_benchmark=sr).value == pytest.approx(0.5, abs=1e-12) + + +def test_psr_decreases_as_benchmark_rises() -> None: + xs = _synthetic() + values = [probabilistic_sharpe_ratio(xs, sr_benchmark=b).value for b in (-0.1, 0.0, 0.05, 0.1)] + assert values == sorted(values, reverse=True) + + +@pytest.mark.parametrize( + ("returns", "reason"), + [ + ([0.01] * 40, "zero_variance"), + (_synthetic(n=10), "insufficient_observations"), + ([0.01, float("nan")] * 20, "invalid_input"), + ([0.01, -1.0] * 20, "invalid_input"), + ([True, False] * 20, "invalid_input"), + ("0.01", "invalid_input"), + ], +) +def test_psr_fails_closed_without_numbers(returns: object, reason: str) -> None: + result = probabilistic_sharpe_ratio(returns) # type: ignore[arg-type] + assert result.status == STATUS_UNCOMPUTABLE + assert result.value is None + assert result.reason_code == reason + + +def test_expected_max_sharpe_requires_two_trials_and_grows_with_trials() -> None: + assert expected_max_sharpe(0.01, 1).status == STATUS_UNCOMPUTABLE + assert expected_max_sharpe(0.0, 10).reason_code == "zero_trial_sharpe_variance" + values = [expected_max_sharpe(0.01, n).value for n in (2, 5, 10, 100, 1000)] + assert all(v is not None for v in values) + assert values == sorted(values) + # Scales with the trial Sharpe standard deviation. + assert expected_max_sharpe(0.04, 10).value == pytest.approx(2 * expected_max_sharpe(0.01, 10).value) + + +def test_dsr_single_trial_is_uncomputable_not_zero() -> None: + result = deflated_sharpe_ratio(_synthetic(), [0.05]) + assert result.status == STATUS_UNCOMPUTABLE + assert result.value is None + assert result.reason_code == "insufficient_trials" + + +def test_dsr_effective_trial_guards() -> None: + xs, trials = _synthetic(), [0.01, 0.03, 0.05] + assert deflated_sharpe_ratio(xs, trials, n_effective_trials=1).reason_code == "insufficient_trials" + assert deflated_sharpe_ratio(xs, trials, n_effective_trials=4).reason_code == "effective_trials_exceed_recorded" + assert deflated_sharpe_ratio(xs, [0.02, 0.02]).reason_code == "zero_trial_sharpe_variance" + assert deflated_sharpe_ratio(_synthetic(n=10), trials).reason_code == "insufficient_observations" + + +def test_dsr_equals_psr_against_expected_max_sharpe() -> None: + xs = _synthetic() + trials = [0.01 * k for k in range(-3, 8)] + mean = sum(trials) / len(trials) + variance = sum((t - mean) ** 2 for t in trials) / (len(trials) - 1) + sr0 = expected_max_sharpe(variance, len(trials)).value + dsr = deflated_sharpe_ratio(xs, trials) + assert dsr.status == STATUS_COMPUTED + assert dsr.value == pytest.approx(probabilistic_sharpe_ratio(xs, sr_benchmark=sr0).value, rel=1e-12) + assert dsr.value < probabilistic_sharpe_ratio(xs).value + + +def test_dsr_non_increasing_in_effective_trials() -> None: + xs = _synthetic() + trials = [0.002 * k for k in range(200)] + values = [deflated_sharpe_ratio(xs, trials, n_effective_trials=n).value for n in (2, 5, 20, 100, 200)] + assert all(v is not None for v in values) + assert all(a >= b for a, b in zip(values, values[1:])) diff --git a/uv.lock b/uv.lock index 8fe55f91..c84a95f2 100644 --- a/uv.lock +++ b/uv.lock @@ -619,17 +619,26 @@ dev = [ { name = "pytest-cov" }, { name = "ruff" }, ] +research = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "pandas", version = "3.0.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, +] [package.metadata] requires-dist = [ { name = "jsonschema", marker = "extra == 'dev'", specifier = ">=4.18,<5" }, { name = "numpy", marker = "extra == 'dev'" }, + { name = "numpy", marker = "extra == 'research'" }, { name = "pandas", marker = "extra == 'dev'" }, + { name = "pandas", marker = "extra == 'research'" }, { name = "pytest", marker = "extra == 'dev'" }, { name = "pytest-cov", marker = "extra == 'dev'" }, { name = "ruff", marker = "extra == 'dev'" }, ] -provides-extras = ["dev"] +provides-extras = ["dev", "research"] [[package]] name = "referencing"