diff --git a/__init__.py b/__init__.py index 235a927..86989eb 100644 --- a/__init__.py +++ b/__init__.py @@ -61,6 +61,7 @@ benchmark_walkforward_kernels, logging_callback, score_strategy_output, + select_is_plateau_robust_record, select_flat_minima_record, stationary_bootstrap_sharpes, synthetic_walkforward_sharpes, diff --git a/core/results.py b/core/results.py index 4188598..041bb93 100644 --- a/core/results.py +++ b/core/results.py @@ -63,31 +63,32 @@ def daily_equity(self) -> pd.Series: def daily_returns(self) -> pd.Series: return self.daily_equity.pct_change().dropna() - def full_report(self, trading_days: int = 365) -> Dict: + def full_report(self, trading_days: int = 365, scope: str = "auto") -> Dict: """Return the standard QuantBT metrics dictionary for this result.""" + from .scopes import scoped_result from ..metrics.performance import full_report - return full_report(self, trading_days=trading_days) + return full_report(scoped_result(self, scope=scope), trading_days=trading_days) - def show_metrics(self, trading_days: int = 365) -> Dict: + def show_metrics(self, trading_days: int = 365, scope: str = "auto") -> Dict: """Print a legacy-style metrics report and return the metrics dict.""" from ..endpoint import format_metrics_report - report = self.full_report(trading_days=trading_days) + report = self.full_report(trading_days=trading_days, scope=scope) print(format_metrics_report(report)) return report - def quick_plot(self, theme: str = "dark", figsize: tuple = (14, 6)): + def quick_plot(self, theme: str = "dark", figsize: tuple = (14, 6), scope: str = "auto"): """Plot cumulative return and drawdown for this result.""" from ..viz import quick_plot - return quick_plot(self, theme=theme, figsize=figsize) + return quick_plot(self, theme=theme, figsize=figsize, scope=scope) - def tearsheet(self, theme: str = "dark", benchmark=None): + def tearsheet(self, theme: str = "dark", benchmark=None, scope: str = "auto"): """Render the QuantBT tearsheet for this result.""" from ..viz import tearsheet - return tearsheet(self, theme=theme, benchmark=benchmark) + return tearsheet(self, theme=theme, benchmark=benchmark, scope=scope) @classmethod def from_legacy(cls, result: BacktestResult) -> "BacktestResultV2": diff --git a/core/scopes.py b/core/scopes.py new file mode 100644 index 0000000..5fc618d --- /dev/null +++ b/core/scopes.py @@ -0,0 +1,96 @@ +""" +Reporting scope helpers. + +Walk-forward and train/test split runs store a full stitched timeline, but the +natural performance report is the OOS/test portion only. These helpers keep +endpoint-level and result-level metrics/plots consistent. +""" + +from __future__ import annotations + +import pandas as pd + +from .results import BacktestResultV2 +from .types import BacktestResult + + +def scoped_result(result, scope: str = "auto"): + """ + Return `result` or an OOS/test-sliced copy for reporting. + + `auto` means OOS/test for walk-forward artifacts and full result for normal + backtests. Use `full` to audit the complete stitched timeline. + """ + normalized = str(scope or "auto").lower().strip() + if normalized == "auto": + normalized = "oos" if "walk_forward" in result.metadata else "full" + if normalized == "full": + return result + if normalized in {"test", "oos"}: + return _slice_result_to_walk_forward_oos(result, scope=normalized) + raise ValueError("scope must be auto, full, test, or oos") + + +def _slice_result_to_walk_forward_oos(result, scope: str): + wf_meta = result.metadata.get("walk_forward") + if not wf_meta: + raise ValueError(f"scope={scope!r} is only available for walk_forward/train_test_split results") + fold_table = wf_meta.get("fold_table") + if fold_table is None or len(fold_table) == 0: + raise ValueError("walk-forward result does not contain a fold_table") + + idx = result.equity.index + mask = pd.Series(False, index=idx) + for _, row in fold_table.iterrows(): + start = pd.Timestamp(row["test_start"]) + end = pd.Timestamp(row["test_end"]) + mask |= (idx >= start) & (idx <= end) + if not bool(mask.any()): + raise ValueError("walk-forward OOS/test scope contains no bars in result index") + + sliced_metadata = dict(result.metadata) + sliced_wf_meta = dict(wf_meta) + sliced_wf_meta["report_scope"] = scope + sliced_metadata["walk_forward"] = sliced_wf_meta + + if isinstance(result, BacktestResultV2): + return BacktestResultV2( + equity=result.equity.loc[mask].copy(), + returns=result.returns.loc[mask].copy(), + positions=result.positions.loc[mask].copy(), + closes=result.closes.loc[mask].copy(), + symbols=list(result.symbols), + initial_capital=float(result.initial_capital), + leverage=float(result.leverage), + liquidated=bool(result.liquidated), + liquidation_bar=int(result.liquidation_bar), + orders=getattr(result, "orders", ()), + fills=getattr(result, "fills", ()), + trades=getattr(result, "trades", ()), + fees=_slice_indexed_like(result.fees, mask), + funding=_slice_indexed_like(result.funding, mask), + margin=_slice_indexed_like(result.margin, mask), + diagnostics=_slice_indexed_like(result.diagnostics, mask), + metadata=sliced_metadata, + ) + + return BacktestResult( + equity=result.equity.loc[mask].copy(), + returns=result.returns.loc[mask].copy(), + positions=result.positions.loc[mask].copy(), + closes=result.closes.loc[mask].copy(), + symbols=list(result.symbols), + initial_capital=float(result.initial_capital), + leverage=float(result.leverage), + liquidated=bool(result.liquidated), + liquidation_bar=int(result.liquidation_bar), + metadata=sliced_metadata, + ) + + +def _slice_indexed_like(obj, mask: pd.Series): + if obj is None: + return obj + if isinstance(obj, (pd.Series, pd.DataFrame)) and obj.index.equals(mask.index): + return obj.loc[mask].copy() + return obj diff --git a/core/types.py b/core/types.py index 3c785a2..fe5d961 100644 --- a/core/types.py +++ b/core/types.py @@ -59,28 +59,29 @@ def daily_equity(self) -> pd.Series: def daily_returns(self) -> pd.Series: return self.daily_equity.pct_change().dropna() - def full_report(self, trading_days: int = 365) -> Dict: + def full_report(self, trading_days: int = 365, scope: str = "auto") -> Dict: """Return the standard QuantBT metrics dictionary for this result.""" + from .scopes import scoped_result from ..metrics.performance import full_report - return full_report(self, trading_days=trading_days) + return full_report(scoped_result(self, scope=scope), trading_days=trading_days) - def show_metrics(self, trading_days: int = 365) -> Dict: + def show_metrics(self, trading_days: int = 365, scope: str = "auto") -> Dict: """Print a legacy-style metrics report and return the metrics dict.""" from ..endpoint import format_metrics_report - report = self.full_report(trading_days=trading_days) + report = self.full_report(trading_days=trading_days, scope=scope) print(format_metrics_report(report)) return report - def quick_plot(self, theme: str = "dark", figsize: tuple = (14, 6)): + def quick_plot(self, theme: str = "dark", figsize: tuple = (14, 6), scope: str = "auto"): """Plot cumulative return and drawdown for this result.""" from ..viz import quick_plot - return quick_plot(self, theme=theme, figsize=figsize) + return quick_plot(self, theme=theme, figsize=figsize, scope=scope) - def tearsheet(self, theme: str = "dark", benchmark=None): + def tearsheet(self, theme: str = "dark", benchmark=None, scope: str = "auto"): """Render the QuantBT tearsheet for this result.""" from ..viz import tearsheet - return tearsheet(self, theme=theme, benchmark=benchmark) + return tearsheet(self, theme=theme, benchmark=benchmark, scope=scope) diff --git a/docs/endpoint.md b/docs/endpoint.md index 69740f8..94fbae8 100644 --- a/docs/endpoint.md +++ b/docs/endpoint.md @@ -175,11 +175,11 @@ All time indexes are normalized to UTC and aligned to the run index. result = bt.backtest(...) result = bt.simulate(...) -rpt = bt.full_report(trading_days=365) -rpt = bt.show_metrics(trading_days=365) +rpt = bt.full_report(trading_days=365, scope="auto") +rpt = bt.show_metrics(trading_days=365, scope="auto") -bt.quick_plot(theme="dark", figsize=(14, 6)) -bt.tearsheet(theme="dark") +bt.quick_plot(theme="dark", figsize=(14, 6), scope="auto") +bt.tearsheet(theme="dark", scope="auto") bt.latest_orders bt.fills @@ -193,6 +193,12 @@ bt.export_fills("fills.csv") `show_metrics()` prints a stable legacy-style text report and returns the same dictionary as `full_report()`: +`scope="auto"` reports the natural tested window for endpoint and result helper +methods. Normal endpoints use the full result; `walk_forward()` and +`train_test_split()` report/plot OOS-test bars only so CAGR, Sharpe, Sortino, +and Calmar are annualized on the period actually traded. Pass `scope="full"` to +audit the complete stitched timeline with flat train bars included. + ```text Initial Capital $ 20,000 Final Equity $ 106,884.96 @@ -215,9 +221,9 @@ dictionary as `full_report()`: The latest result object also exposes the same convenience methods: ```python -result.full_report() -result.show_metrics() -result.quick_plot() +result.full_report(scope="auto") +result.show_metrics(scope="auto") +result.quick_plot(scope="auto") result.tearsheet() ``` @@ -784,8 +790,12 @@ Requirements: - `export_nautilus_report_bundle(...)` writes raw Nautilus account/order/fill/ position reports, normalized trade logs, a run manifest, equity/returns CSVs, `config.json`, and optional QuantStats daily HTML. `config.json` is filled - from endpoint/result metadata even when no explicit `config=` is passed. - QuantStats uses daily equity returns by default with + from endpoint/result metadata even when no explicit `config=` is passed. The + report config uses grouped fields such as `effective_account`, + `effective_sizing`, `effective_fees`, and `effective_execution` so there is + one clear effective view; extra `config={...}` values are saved under + `annotations` and do not override execution metadata. QuantStats uses daily + equity returns by default with `quantstats_periods_per_year=365` for crypto. Nautilus metadata: @@ -850,6 +860,13 @@ wfo = QuantBTEndpoint.walk_forward( "garch_vol_multiplier": 1.0, # mode_3_flat_minima: "flat_top_fraction", "flat_eps", # "flat_min_samples", "flat_selector" + # train-only selector for strict train/test split: + # "candidate_selection_metric": "is_plateau_robust", + # "scoring_backend": "endpoint", # endpoint | proxy + # "plateau_quantile": 0.25, + # "plateau_median_weight": 0.25, + # "plateau_std_penalty": 0.50, + # "plateau_size_bonus": 0.01, # crypto default annualization: 365; equities often use 252 "scoring_trading_days": 365, # optional under-trading penalty; None disables it @@ -964,9 +981,20 @@ Important rules: - for all optimization modes, Optuna receives only in-sample or synthetic in-sample objectives; OOS scoring is delayed until after the top IS candidate set is frozen, reducing indirect look-ahead bias; -- candidate selection is controlled by `top_is_fraction` or `top_is_k`; OOS - candidate ranking uses `candidate_selection_metric`, defaulting to - `robust_decay`; +- candidate selection is controlled by `top_is_fraction` or `top_is_k`; + `candidate_selection_metric` defaults to `robust_decay`. For strict + train/test split validation, use `candidate_selection_metric="is_plateau_robust"` + to select final params from the dense train-only plateau; OOS is then used + only for final reporting/audit, not for parameter selection; +- endpoint-created WFO for single-symbol routes defaults to + `scoring_backend="endpoint"` for `mode_1_decay` and `mode_3_flat_minima`. + This means Optuna scores the train fold with the same selected QuantBT route + (`pct_equity`, `signal_notional`, or `dca_ladder`) instead of the fast proxy. + Set `scoring_backend="proxy"` when you explicitly want approximate scoring; + `mode_2_sbb` always uses proxy return paths because it needs synthetic + bootstrap/GARCH simulations. Proxy scoring uses the signal exactly as emitted + by the strategy adapter; execution lag, if desired, belongs in the strategy + layer; - optimization-time scoring uses a transparent return proxy on strategy output; final accounting still comes from the stitched QuantBT backtest; - optimization Sharpe annualization uses `scoring_trading_days` from diff --git a/docs/nautilus_backend.md b/docs/nautilus_backend.md index 57a1a31..ac0aa5a 100644 --- a/docs/nautilus_backend.md +++ b/docs/nautilus_backend.md @@ -63,8 +63,10 @@ Report bundle: `fills_report.csv`, `positions_report.csv`, normalized `trade_log.csv`, `fill_log.txt`, `run_manifest.json`, `metrics_summary.json`, `config.json`, equity/returns CSVs, and optional `quantstats_daily.html`. - `config.json` is auto-filled from endpoint/result run metadata, including - capital, leverage, fees, slippage, sizing, funding, instrument, and timeframe. + `config.json` is auto-filled from endpoint/result run metadata using grouped + `effective_*` sections for capital, leverage, fees, slippage, sizing, + funding, instrument, and timeframe. Extra `config={...}` values are saved + under `annotations`, not mixed into the effective execution settings. - QuantStats input is daily-resampled from the equity curve by default, which is safer for multi-year intraday runs than treating raw 15m returns as daily returns. The default annualization is `quantstats_periods_per_year=365` for diff --git a/endpoint.py b/endpoint.py index 98d6844..1eced6e 100644 --- a/endpoint.py +++ b/endpoint.py @@ -382,6 +382,12 @@ def walk_forward( """ optimization_config = dict(optimization_config or {}) wf_config = kwargs.pop("walkforward_config", None) + scoring_backend = str( + optimization_config.get( + "scoring_backend", + _default_walkforward_scoring_backend(target_mode=target_mode, optimization_mode=optimization_mode), + ) + ) if wf_config is None: wf_config = WalkForwardConfig( split_mode=split_mode, @@ -417,6 +423,11 @@ def walk_forward( flat_eps=float(optimization_config.get("flat_eps", 0.15)), flat_min_samples=int(optimization_config.get("flat_min_samples", 3)), flat_selector=str(optimization_config.get("flat_selector", "medoid")), + plateau_quantile=float(optimization_config.get("plateau_quantile", 0.25)), + plateau_median_weight=float(optimization_config.get("plateau_median_weight", 0.25)), + plateau_std_penalty=float(optimization_config.get("plateau_std_penalty", 0.50)), + plateau_size_bonus=float(optimization_config.get("plateau_size_bonus", 0.01)), + scoring_backend=scoring_backend, scoring_trading_days=int(optimization_config.get("scoring_trading_days", 365)), min_trades_per_year=optimization_config.get("min_trades_per_year"), trade_penalty_factor=optimization_config.get("trade_penalty_factor"), @@ -597,39 +608,49 @@ def simulate( _print_order_logs(result, mode=order_log_mode, limit=order_log_limit) return result - def full_report(self, trading_days: int = 365) -> Dict: + def full_report(self, trading_days: int = 365, scope: str = "auto") -> Dict: """ Return the full QuantBT metrics dictionary for the latest result. + Parameters + ---------- + trading_days: + Annualization calendar. Use 365 for crypto and 252 for equities. + scope: + `auto` uses the natural reporting scope for the endpoint. For + walk-forward and train/test split runs this means OOS/test bars + only; other endpoints use the full result. Pass `full` to audit the + complete stitched timeline, or `test`/`oos` to force OOS reporting. + Raises ------ RuntimeError If no backtest has been run yet. """ - return _full_report(self._require_result(), trading_days=trading_days) + return _full_report(self._result_for_report_scope(scope), trading_days=trading_days) - def show_metrics(self, trading_days: int = 365) -> Dict: + def show_metrics(self, trading_days: int = 365, scope: str = "auto") -> Dict: """ Print key metrics and return the full metrics dictionary. This intentionally mirrors the convenience style of legacy `BacktestEngine.analyze()` without forcing a plot. """ - rpt = self.full_report(trading_days=trading_days) + rpt = self.full_report(trading_days=trading_days, scope=scope) print(format_metrics_report(rpt)) return rpt - def quick_plot(self, theme: str = "dark", figsize: tuple = (14, 6)): + def quick_plot(self, theme: str = "dark", figsize: tuple = (14, 6), scope: str = "auto"): """ Plot cumulative return and drawdown for the latest result. """ - return _quick_plot(self._require_result(), theme=theme, figsize=figsize) + return _quick_plot(self._require_result(), theme=theme, figsize=figsize, scope=scope) - def tearsheet(self, theme: str = "dark", benchmark=None): + def tearsheet(self, theme: str = "dark", benchmark=None, scope: str = "auto"): """ Render the full QuantBT tearsheet for the latest result. """ - return _tearsheet(self._require_result(), theme=theme, benchmark=benchmark) + return _tearsheet(self._require_result(), theme=theme, benchmark=benchmark, scope=scope) def export_orders(self, path: Union[str, Path]) -> None: """ @@ -912,14 +933,15 @@ def _run_walk_forward( if self.config.strategy_class is None: raise ValueError("walk_forward endpoint requires strategy_class") wf_config = self.config.walkforward_config or WalkForwardConfig(target_mode=self.config.walkforward_target_mode) - engine = WalkForwardEngine(strategy=self.config.strategy_class, config=wf_config) + target_mode = self.config.walkforward_target_mode.lower().strip() + scorer = _make_walkforward_endpoint_scorer(self.config, target_mode=target_mode, symbols=symbols) if wf_config.scoring_backend == "endpoint" else None + engine = WalkForwardEngine(strategy=self.config.strategy_class, config=wf_config, scorer=scorer) wf_result = engine.run( data=data if data is not None else closes, params=params, param_ranges=param_ranges, datetime_index=datetime_index, ) - target_mode = self.config.walkforward_target_mode.lower().strip() stitched = wf_result.oos_output if stitched is None: raise ValueError("walk-forward strategy produced no OOS output") @@ -980,6 +1002,7 @@ def _run_walk_forward( "engine": wf_result.metadata["engine"], "target_mode": target_mode, "n_folds": wf_result.metadata["n_folds"], + "report_scope": "test" if wf_result.metadata.get("split_frequency") == "single" else "oos", "split_frequency": wf_result.metadata.get("split_frequency"), "window_mode": wf_result.metadata.get("window_mode"), "params": wf_result.params, @@ -1008,6 +1031,11 @@ def _run_walk_forward( "garch_q": wf_result.metadata.get("garch_q"), "garch_dist": wf_result.metadata.get("garch_dist"), "garch_vol_multiplier": wf_result.metadata.get("garch_vol_multiplier"), + "plateau_quantile": wf_result.metadata.get("plateau_quantile"), + "plateau_median_weight": wf_result.metadata.get("plateau_median_weight"), + "plateau_std_penalty": wf_result.metadata.get("plateau_std_penalty"), + "plateau_size_bonus": wf_result.metadata.get("plateau_size_bonus"), + "scoring_backend": wf_result.metadata.get("scoring_backend"), "numba_enabled": wf_result.metadata.get("numba_enabled"), } result.metadata["walk_forward_result"] = wf_result @@ -1140,6 +1168,11 @@ def _require_result(self): raise RuntimeError("run backtest() or simulate() before requesting results") return self.result + def _result_for_report_scope(self, scope: str): + from .core.scopes import scoped_result + + return scoped_result(self._require_result(), scope=scope) + def _normalize_result_contract(result) -> None: """ @@ -1177,6 +1210,7 @@ def _normalize_result_contract(result) -> None: def _attach_endpoint_run_config(result, config: EndpointConfig) -> None: metadata = result.metadata payload = _endpoint_run_config_payload(config) + _sync_applied_nautilus_config(payload, metadata) metadata["run_config"] = payload metadata.setdefault("initial_capital", payload["account"]["initial_capital"]) metadata.setdefault("leverage", payload["account"]["leverage"]) @@ -1189,6 +1223,32 @@ def _attach_endpoint_run_config(result, config: EndpointConfig) -> None: metadata.setdefault("use_funding", payload["funding"]["use_funding"]) +def _sync_applied_nautilus_config(payload: Dict, metadata: Dict) -> None: + """Keep report run_config aligned with the adapter config that actually ran.""" + if payload.get("backend") != "nautilus": + return + nautilus = dict(payload.get("nautilus") or {}) + if not nautilus: + return + + for source_key, target_key in ( + ("instrument_id", "instrument_id"), + ("sizing_mode", "sizing_mode"), + ("trade_notional", "trade_notional"), + ("use_pyramiding", "use_pyramiding"), + ("close_positions_on_stop", "close_positions_on_stop"), + ): + if metadata.get(source_key) is not None: + nautilus[target_key] = _jsonable(metadata[source_key]) + + if metadata.get("timeframe") is not None: + nautilus["timeframe"] = _jsonable(metadata["timeframe"]) + if metadata.get("initial_capital") is not None: + nautilus["starting_balance"] = _jsonable(metadata["initial_capital"]) + + payload["nautilus"] = nautilus + + def _endpoint_run_config_payload(config: EndpointConfig) -> Dict: payload = { "mode": config.mode, @@ -1384,6 +1444,76 @@ def _resolve_backend(config: EndpointConfig) -> str: return "native_vectorized" +def _default_walkforward_scoring_backend(target_mode: str, optimization_mode: str) -> str: + mode = str(target_mode).lower().strip() + opt_mode = str(optimization_mode).lower().strip() + if opt_mode == "mode_2_sbb": + return "proxy" + if mode in {"pct_equity", "%_equity", "signal_notional", "single_signal", "dca_ladder"}: + return "endpoint" + return "proxy" + + +def _make_walkforward_endpoint_scorer(config: EndpointConfig, target_mode: str, symbols=None): + score_config = _walkforward_scoring_config(config, target_mode) + symbol_list = list(symbols or config.symbols or ["DEFAULT"]) + + def scorer(data, output, index, fold, params, context: str, trading_days: int) -> Dict[str, float]: + temp = QuantBTEndpoint(score_config) + sliced_data = _slice_wf_data_to_index(data, index) + try: + if score_config.mode == "portfolio": + result = temp.backtest(data=sliced_data, positions=output, symbols=symbol_list) + else: + result = temp.backtest(data=sliced_data, signal=output, symbols=symbol_list) + report = result.full_report(trading_days=trading_days, scope="full") + except Exception as exc: + raise RuntimeError( + "walk-forward endpoint scoring failed during " + f"{context} for fold_id={fold.fold_id}; target_mode={target_mode!r}; params={params}" + ) from exc + return { + "sharpe": float(report.get("sharpe", 0.0)), + "turnover": float(report.get("num_trades", 0.0)), + "trade_count": float(report.get("num_trades", 0.0)), + "mean_return": float(report.get("total_return_pct", 0.0)) / 100.0, + "volatility": 0.0, + "max_drawdown_pct": float(report.get("max_drawdown_pct", 0.0)), + "profit_factor": float(report.get("profit_factor", 0.0)), + } + + return scorer + + +def _walkforward_scoring_config(config: EndpointConfig, target_mode: str) -> EndpointConfig: + mode = str(target_mode).lower().strip() + if mode in {"pct_equity", "%_equity"}: + return replace(config, mode="pct_equity", backend="legacy", sizing="%_equity") + if mode == "dca_ladder": + return replace(config, mode="dca_ladder", backend="legacy", sizing="dca_ladder") + if mode in {"signal_notional", "single_signal"}: + return replace(config, mode="signal_notional", backend=config.backend, sizing="signal_notional") + if mode == "portfolio": + return replace(config, mode="portfolio", backend="legacy_portfolio") + raise NotImplementedError(f"endpoint scoring is not implemented for walk-forward target_mode={target_mode!r}") + + +def _slice_wf_data_to_index(data, index: pd.DatetimeIndex): + if isinstance(data, pd.DataFrame): + return data.reindex(index).copy() + if isinstance(data, pd.Series): + return data.reindex(index).copy() + if isinstance(data, dict): + out = {} + for key, value in data.items(): + if isinstance(value, (pd.DataFrame, pd.Series)): + out[key] = value.reindex(index).copy() + else: + out[key] = value + return out + return data + + def _normalize_single_data(data, signal, signal_col, datetime_index): if data is None: raise ValueError("single-symbol endpoint requires data DataFrame") diff --git a/reporting/nautilus_bundle.py b/reporting/nautilus_bundle.py index ae74860..ea4a86e 100644 --- a/reporting/nautilus_bundle.py +++ b/reporting/nautilus_bundle.py @@ -100,7 +100,7 @@ def export_nautilus_report_bundle( metadata = dict(result.metadata or {}) config_payload = _config_payload_from_result(result=result, metadata=metadata) if config: - config_payload.update(dict(config)) + config_payload["annotations"] = {**dict(config_payload.get("annotations", {})), **dict(config)} account_report = _report_frame(metadata.get("account_report")) orders_report = _report_frame(metadata.get("orders_report", metadata.get("order_report"))) @@ -397,30 +397,80 @@ def _run_manifest( def _config_payload_from_result(result: BacktestResultV2, metadata: Dict[str, Any]) -> Dict[str, Any]: - payload = dict(metadata.get("run_config") or {}) - payload.setdefault("backend", metadata.get("backend", "nautilus")) - payload.setdefault("engine", metadata.get("engine", "NautilusTrader BacktestEngine")) - payload.setdefault("instrument_id", metadata.get("instrument_id") or _first_symbol(result)) - payload.setdefault("bar_type", metadata.get("bar_type")) - payload.setdefault("timeframe", metadata.get("timeframe") or _bar_timeframe(metadata.get("bar_type"))) - payload.setdefault("initial_capital", _safe_float(metadata.get("initial_capital")) or float(result.initial_capital)) - payload.setdefault("leverage", _safe_float(metadata.get("leverage")) or float(result.leverage)) - payload.setdefault("maintenance_ratio", _safe_float(metadata.get("maintenance_ratio"))) - payload.setdefault("fee_rate", _safe_float(metadata.get("fee_rate"))) - payload.setdefault("fee_round_trip", _safe_float(metadata.get("fee_round_trip"))) - payload.setdefault("slippage", _safe_float(metadata.get("slippage"))) - payload.setdefault("slippage_bps", _safe_float(metadata.get("slippage_bps"))) - payload.setdefault("alloc_per_trade", metadata.get("alloc_per_trade", metadata.get("trade_notional"))) - payload.setdefault("trade_notional", metadata.get("trade_notional")) - payload.setdefault("sizing_mode", metadata.get("sizing_mode")) - payload.setdefault("use_pyramiding", metadata.get("use_pyramiding")) - payload.setdefault("use_funding", metadata.get("use_funding")) - payload.setdefault("funding_rate", metadata.get("funding_rate")) - payload.setdefault("close_positions_on_stop", metadata.get("close_positions_on_stop")) - payload.setdefault("orders_count", int(metadata.get("orders_count", 0) or 0)) - payload.setdefault("fills_count", int(metadata.get("fills_count", 0) or 0)) - payload.setdefault("positions_count", int(metadata.get("positions_count", 0) or 0)) - return payload + run_config = dict(metadata.get("run_config") or {}) + account = dict(run_config.get("account") or {}) + execution = dict(run_config.get("execution") or {}) + fees = dict(run_config.get("fees") or {}) + sizing = dict(run_config.get("sizing") or {}) + funding = dict(run_config.get("funding") or {}) + nautilus = dict(run_config.get("nautilus") or {}) + + requested_fee_rate = _safe_float(metadata.get("fee_rate")) + if requested_fee_rate is None: + requested_fee_rate = _safe_float(fees.get("one_way_fee_rate")) + requested_slippage = _safe_float(metadata.get("slippage")) + if requested_slippage is None: + requested_slippage = _safe_float(execution.get("legacy_slippage_rate")) + requested_slippage_bps = _safe_float(metadata.get("slippage_bps")) + if requested_slippage_bps is None: + requested_slippage_bps = _safe_float(execution.get("slippage_bps")) + + return { + "schema_version": 2, + "backend": metadata.get("backend", "nautilus"), + "engine": metadata.get("engine", "NautilusTrader BacktestEngine"), + "instrument": { + "instrument_id": metadata.get("instrument_id") or _first_symbol(result), + "bar_type": metadata.get("bar_type"), + "timeframe": metadata.get("timeframe") or _bar_timeframe(metadata.get("bar_type")), + }, + "effective_account": { + "initial_capital": _safe_float(metadata.get("initial_capital")) or float(result.initial_capital), + "leverage": _safe_float(metadata.get("leverage")) or float(result.leverage), + "maintenance_ratio": _safe_float(metadata.get("maintenance_ratio")), + "margin_mode": account.get("margin_mode"), + "oms_mode": account.get("oms_mode"), + "base_currency": account.get("base_currency"), + }, + "effective_sizing": { + "sizing_mode": metadata.get("sizing_mode") or sizing.get("hedge_type"), + "hedge_type": sizing.get("hedge_type"), + "trade_notional": metadata.get("trade_notional"), + "alloc_per_trade": metadata.get("alloc_per_trade", sizing.get("alloc_per_trade")), + "use_pyramiding": metadata.get("use_pyramiding", sizing.get("use_pyramiding")), + "contract_size": sizing.get("contract_size"), + }, + "effective_fees": { + "requested_fee_rate": requested_fee_rate, + "requested_fee_convention": "one_way", + "requested_fee_source": "endpoint.fee_rate", + "legacy_fee_round_trip_ignored": fees.get("round_trip_fee"), + "applied_by": "NautilusTrader MakerTakerFeeModel", + "custom_fee_rate_applied_to_nautilus": False, + }, + "effective_execution": { + "requested_slippage_rate": requested_slippage, + "requested_slippage_bps": requested_slippage_bps, + "requested_slippage_source": "endpoint.slippage", + "applied_by": "NautilusTrader bar market execution", + "custom_slippage_applied_to_nautilus": False, + "fill_price_policy": execution.get("fill_price_policy"), + "same_bar_policy": execution.get("same_bar_policy"), + "allow_partial_fill": execution.get("allow_partial_fill"), + "reject_on_insufficient_margin": execution.get("reject_on_insufficient_margin"), + }, + "funding": { + "use_funding": metadata.get("use_funding", funding.get("use_funding")), + "funding_rate": metadata.get("funding_rate", funding.get("funding_rate")), + }, + "nautilus": nautilus, + "diagnostics": { + "orders_count": int(metadata.get("orders_count", 0) or 0), + "fills_count": int(metadata.get("fills_count", 0) or 0), + "positions_count": int(metadata.get("positions_count", 0) or 0), + }, + "annotations": {}, + } def _report_frame(value: Any) -> pd.DataFrame: diff --git a/tests/test_endpoint.py b/tests/test_endpoint.py index bfb13cd..ce8272a 100644 --- a/tests/test_endpoint.py +++ b/tests/test_endpoint.py @@ -292,7 +292,12 @@ def run_signal_series(self, data, signal): closes=closes, symbols=["ETHUSDT-PERP.BINANCE"], initial_capital=10_000.0, - metadata={"use_pyramiding": captured["config"].use_pyramiding}, + metadata={ + "instrument_id": captured["config"].instrument_id, + "sizing_mode": captured["config"].sizing_mode, + "trade_notional": captured["config"].trade_notional, + "use_pyramiding": captured["config"].use_pyramiding, + }, ) monkeypatch.setattr(nautilus_module, "NautilusBacktestEngine", FakeNautilusBacktestEngine) @@ -318,6 +323,9 @@ def run_signal_series(self, data, signal): assert captured["config"].sizing_mode == "%_equity" assert captured["config"].trade_notional == 0.5 assert result.metadata["use_pyramiding"] is False + assert result.metadata["run_config"]["nautilus"]["sizing_mode"] == "%_equity" + assert result.metadata["run_config"]["nautilus"]["trade_notional"] == 0.5 + assert result.metadata["run_config"]["nautilus"]["use_pyramiding"] is False def test_simulate_can_print_bounded_nautilus_order_logs(monkeypatch, capsys): diff --git a/tests/test_phase5_1_nautilus_report_bundle.py b/tests/test_phase5_1_nautilus_report_bundle.py index d2fa723..9677eca 100644 --- a/tests/test_phase5_1_nautilus_report_bundle.py +++ b/tests/test_phase5_1_nautilus_report_bundle.py @@ -1,6 +1,7 @@ from __future__ import annotations import types +import json import pandas as pd @@ -57,6 +58,10 @@ def _synthetic_result(): "bar_type": "BTCUSDT-PERP.BINANCE-12-HOUR-LAST-EXTERNAL", "sizing_mode": "%_equity", "trade_notional": 0.5, + "fee_rate": 0.0005, + "fee_round_trip": 0.0004, + "slippage": 0.0002, + "slippage_bps": 0.0, "orders_count": 3, "fills_count": 3, "positions_count": 1, @@ -67,6 +72,37 @@ def _synthetic_result(): "orders_report": fills.copy(), "fills_report": fills, "positions_report": positions_report, + "run_config": { + "account": { + "initial_capital": 10_000.0, + "leverage": 3.0, + "maintenance_ratio": 0.005, + "margin_mode": "cross", + "oms_mode": "netting", + "base_currency": "USDT", + }, + "sizing": { + "hedge_type": "%_equity", + "alloc_per_trade": 0.5, + "contract_size": 1.0, + "use_pyramiding": False, + }, + "fees": { + "explicit_fee_rate": 0.0005, + "one_way_fee_rate": 0.0005, + "round_trip_fee": 0.0004, + }, + "execution": { + "legacy_slippage_rate": 0.0002, + "slippage_bps": 0.0, + "fill_price_policy": "bar_market", + "same_bar_policy": "close", + "allow_partial_fill": False, + "reject_on_insufficient_margin": True, + }, + "funding": {"use_funding": False, "funding_rate": 0.0}, + "nautilus": {"timeframe": "12h", "close_positions_on_stop": False}, + }, }, ) @@ -117,13 +153,14 @@ def fake_html(returns, benchmark=None, output=None, title=None, periods_per_year assert manifest["positions_count"] == 1 assert manifest["account_reconstructed_diff"] == 0 - config = pd.read_json(report_dir / "config.json", typ="series") - assert config["initial_capital"] == 10_000.0 - assert config["leverage"] == 3.0 - assert config["trade_notional"] == 0.5 - assert config["sizing_mode"] == "%_equity" - assert config["timeframe"] == "12-HOUR" - assert config["note"] == "manual annotation" + config = json.loads((report_dir / "config.json").read_text(encoding="utf-8")) + assert config["schema_version"] == 2 + assert config["effective_account"]["initial_capital"] == 10_000.0 + assert config["effective_account"]["leverage"] == 3.0 + assert config["effective_sizing"]["trade_notional"] == 0.5 + assert config["effective_sizing"]["sizing_mode"] == "%_equity" + assert config["instrument"]["timeframe"] == "12-HOUR" + assert config["annotations"]["note"] == "manual annotation" trade_log = pd.read_csv(report_dir / "trade_log.csv") assert list(trade_log.columns)[:5] == ["strategy_id", "symbol", "exchange", "instrument_id", "position_type"] @@ -135,6 +172,31 @@ def fake_html(returns, benchmark=None, output=None, title=None, periods_per_year assert "BTCUSDT-PERP.BINANCE" in fill_log[0] +def test_config_json_has_single_effective_fee_and_execution_view(tmp_path): + result = _synthetic_result() + + report_dir = export_nautilus_report_bundle( + result=result, + output_dir=tmp_path, + strategy_id="clean-config", + make_quantstats=False, + ) + + config = json.loads((report_dir / "config.json").read_text(encoding="utf-8")) + assert "fee_rate" not in config + assert "fee_round_trip" not in config + assert "slippage" not in config + assert "slippage_bps" not in config + assert "fees" not in config + assert "execution" not in config + assert config["effective_fees"]["requested_fee_rate"] == 0.0005 + assert config["effective_fees"]["requested_fee_convention"] == "one_way" + assert config["effective_fees"]["legacy_fee_round_trip_ignored"] == 0.0004 + assert config["effective_fees"]["custom_fee_rate_applied_to_nautilus"] is False + assert config["effective_execution"]["requested_slippage_rate"] == 0.0002 + assert config["effective_execution"]["custom_slippage_applied_to_nautilus"] is False + + def test_export_nautilus_report_bundle_handles_empty_raw_reports(tmp_path): idx = pd.date_range("2024-01-01", periods=3, freq="1D", tz="UTC") equity = pd.Series([10_000.0, 10_010.0, 10_005.0], index=idx) diff --git a/tests/test_walkforward_phase1.py b/tests/test_walkforward_phase1.py index 8451b9d..73f8da9 100644 --- a/tests/test_walkforward_phase1.py +++ b/tests/test_walkforward_phase1.py @@ -18,6 +18,7 @@ WalkForwardTrialRecord, benchmark_walkforward_kernels, score_strategy_output, + select_is_plateau_robust_record, select_flat_minima_record, stationary_bootstrap_sharpes, synthetic_walkforward_sharpes, @@ -116,6 +117,112 @@ def strategy(data, params, train_index, test_index, fold): assert result.positions["Position_BTC"].loc["2022-01-01"] > 0.0 +def test_train_test_split_metrics_default_to_test_scope(): + idx = _idx() + data = _bars(idx) + data["close"] = 100.0 + pd.Series(range(len(idx)), index=idx) * 0.1 + + def strategy(data, params, train_index, test_index, fold): + signal = pd.Series(1.0, index=test_index) + signal.iloc[0] = 0.0 + return signal + + tts = QuantBTEndpoint.train_test_split( + strategy_class=strategy, + test_start="2022-01-01", + target_mode="pct_equity", + initial_capital=20_000.0, + leverage=5.0, + alloc_per_trade=0.5, + fee=0.0, + use_funding=False, + ) + tts.backtest(data=data, params={}) + + native = QuantBTEndpoint.pct_equity( + initial_capital=20_000.0, + leverage=5.0, + alloc_per_trade=0.5, + fee=0.0, + use_funding=False, + ) + native.backtest( + data=data.loc["2022-01-01":], + signal=strategy(data, {}, None, data.loc["2022-01-01":].index, None), + ) + + auto_report = tts.full_report() + result_report = tts.result.full_report() + full_report = tts.full_report(scope="full") + native_report = native.full_report() + + assert auto_report["final_equity"] == pytest.approx(native_report["final_equity"]) + assert auto_report["total_return_pct"] == pytest.approx(native_report["total_return_pct"]) + assert auto_report["cagr_pct"] == pytest.approx(native_report["cagr_pct"]) + assert result_report["cagr_pct"] == pytest.approx(auto_report["cagr_pct"]) + assert full_report["final_equity"] == pytest.approx(native_report["final_equity"]) + assert full_report["cagr_pct"] < auto_report["cagr_pct"] + + +def test_endpoint_metrics_scope_survives_missing_module_global(monkeypatch): + import quantbt.endpoint as endpoint_module + + idx = _idx() + data = _bars(idx) + data["close"] = 100.0 + pd.Series(range(len(idx)), index=idx) * 0.1 + + def strategy(data, params, train_index, test_index, fold): + signal = pd.Series(1.0, index=test_index) + signal.iloc[0] = 0.0 + return signal + + bt = QuantBTEndpoint.train_test_split( + strategy_class=strategy, + test_start="2022-01-01", + target_mode="pct_equity", + initial_capital=20_000.0, + leverage=5.0, + alloc_per_trade=0.5, + fee=0.0, + use_funding=False, + ) + bt.backtest(data=data, params={}) + + monkeypatch.delattr(endpoint_module, "scoped_result", raising=False) + + report = bt.full_report() + assert report["final_equity"] > 20_000.0 + + +def test_walkforward_result_quick_plot_accepts_default_oos_scope(monkeypatch): + idx = _idx() + data = _bars(idx) + data["close"] = 100.0 + pd.Series(range(len(idx)), index=idx) * 0.1 + + def strategy(data, params, train_index, test_index, fold): + signal = pd.Series(1.0, index=test_index) + signal.iloc[0] = 0.0 + return signal + + bt = QuantBTEndpoint.train_test_split( + strategy_class=strategy, + test_start="2022-01-01", + target_mode="pct_equity", + initial_capital=20_000.0, + leverage=5.0, + alloc_per_trade=0.5, + fee=0.0, + use_funding=False, + ) + result = bt.backtest(data=data, params={}) + + import matplotlib.pyplot as plt + + monkeypatch.setattr(plt, "show", lambda: None) + bt.quick_plot() + result.quick_plot() + + @pytest.mark.parametrize( "optimization_mode,optimization_config", [ @@ -400,7 +507,7 @@ def test_walkforward_score_strategy_output_is_transparent_return_proxy(): assert metrics["turnover"] == 1.0 assert metrics["trade_count"] == 1.0 - assert metrics["mean_return"] > 0.0 + assert metrics["mean_return"] == pytest.approx(0.075) def test_walkforward_trade_frequency_penalty_formula_is_normalized_linear(): @@ -742,6 +849,111 @@ def test_walkforward_phase3_flat_minima_centroid_snaps_to_valid_param_grid(): assert selected.selection_metadata["medoid_params"]["x"] == 24 +def test_is_plateau_robust_selector_prefers_dense_train_plateau_over_isolated_spike(): + records = [ + WalkForwardTrialRecord(0, {"x": 95}, 12.0, 12.0, 0.0, 0.0, 0.0, []), + WalkForwardTrialRecord(1, {"x": 20}, 9.2, 9.2, 0.0, 0.0, 0.0, []), + WalkForwardTrialRecord(2, {"x": 21}, 9.1, 9.1, 0.0, 0.0, 0.0, []), + WalkForwardTrialRecord(3, {"x": 22}, 9.0, 9.0, 0.0, 0.0, 0.0, []), + WalkForwardTrialRecord(4, {"x": 23}, 8.9, 8.9, 0.0, 0.0, 0.0, []), + ] + cfg = WalkForwardConfig( + optimization_mode="mode_1_decay", + candidate_selection_metric="is_plateau_robust", + top_is_fraction=1.0, + flat_eps=0.04, + flat_min_samples=2, + plateau_quantile=0.25, + plateau_std_penalty=0.5, + ) + + selected = select_is_plateau_robust_record(records, {"x": (0, 100, 1)}, config=cfg) + + assert selected.params["x"] in {20, 21, 22, 23} + assert selected.selection_metadata["selected_by"] == "is_plateau_robust" + assert selected.selection_metadata["oos_used_for_selection"] is False + assert selected.selection_metadata["cluster_size"] == 4 + + +def test_train_test_split_can_select_is_plateau_robust_without_oos_selection(): + idx = _idx() + data = _bars(idx) + data["close"] = 100.0 + pd.Series(range(len(idx)), index=idx) * 0.1 + + def strategy(data, params, train_index, test_index, fold): + signal = pd.Series(1.0 if int(params["go_long"]) == 1 else -1.0, index=test_index) + signal.iloc[0] = 0.0 + return signal + + bt = QuantBTEndpoint.train_test_split( + strategy_class=strategy, + test_start="2022-01-01", + target_mode="pct_equity", + optimization_mode="mode_1_decay", + optimization_config={ + "top_is_fraction": 1.0, + "candidate_selection_metric": "is_plateau_robust", + "flat_eps": 1.0, + "flat_min_samples": 1, + "plateau_quantile": 0.25, + }, + optuna_trials=6, + random_seed=7, + initial_capital=20_000.0, + leverage=5.0, + alloc_per_trade=0.5, + fee=0.0, + use_funding=False, + ) + result = bt.backtest(data=data, param_ranges={"go_long": (0, 1, 1)}) + best_meta = result.metadata["walk_forward"]["best_trial"]["selection_metadata"] + + assert result.metadata["walk_forward"]["candidate_selection_metric"] == "is_plateau_robust" + assert best_meta["selected_by"] == "is_plateau_robust" + assert best_meta["oos_seen_by_optuna"] is False + assert best_meta["oos_used_for_selection"] is False + + +def test_train_test_split_endpoint_scoring_matches_pct_equity_report_on_train_fold(): + idx = pd.date_range("2021-01-01", "2022-03-31 23:00:00", freq="1h", tz="UTC") + data = _bars(idx) + data["close"] = 100.0 + pd.Series(range(len(idx)), index=idx) * 0.01 + + def strategy(data, params, train_index, test_index, fold): + signal = pd.Series(1.0, index=test_index) + signal.iloc[0] = 0.0 + return signal + + tts = QuantBTEndpoint.train_test_split( + strategy_class=strategy, + test_start="2022-01-01", + target_mode="pct_equity", + optimization_config={"scoring_backend": "endpoint"}, + initial_capital=20_000.0, + leverage=5.0, + alloc_per_trade=0.5, + fee=0.0, + use_funding=False, + ) + result = tts.backtest(data=data, params={}) + wf = result.metadata["walk_forward"] + fold = result.metadata["walk_forward_result"].folds[0] + + native = QuantBTEndpoint.pct_equity( + initial_capital=20_000.0, + leverage=5.0, + alloc_per_trade=0.5, + fee=0.0, + use_funding=False, + ) + native.backtest(data=data.reindex(fold.train_index), signal=strategy(data, {}, fold.train_index, fold.train_index, fold)) + native_report = native.full_report() + + assert wf["scoring_backend"] == "endpoint" + assert wf["best_trial"]["fold_metrics"][0]["is_sharpe_raw"] == pytest.approx(native_report["sharpe"]) + assert wf["best_trial"]["fold_metrics"][0]["is_trade_count"] == pytest.approx(native_report["num_trades"]) + + def test_walkforward_phase4_rejects_non_timestamped_strategy_output(): idx = _idx() diff --git a/viz/plots.py b/viz/plots.py index 8aa5c14..826070e 100644 --- a/viz/plots.py +++ b/viz/plots.py @@ -53,11 +53,15 @@ def quick_plot( theme: str = "dark", figsize: tuple = (14, 6), title: Optional[str] = None, + scope: str = "auto", ) -> None: """ Two-panel figure: cumulative return (top) and drawdown (bottom). Suitable as a fast sanity-check or inline notebook output. """ + from ..core.scopes import scoped_result + + result = scoped_result(result, scope=scope) c = apply_theme(theme) eq = result.daily_equity @@ -125,6 +129,7 @@ def tearsheet( trading_days: int = 365, benchmark: Optional[pd.Series] = None, title: Optional[str] = None, + scope: str = "auto", ) -> None: """ Full performance tearsheet. @@ -138,6 +143,9 @@ def tearsheet( 5 Per-symbol PnL contribution 6 Daily position exposure """ + from ..core.scopes import scoped_result + + result = scoped_result(result, scope=scope) c = apply_theme(theme) rpt = full_report(result, trading_days) diff --git a/walkforward.py b/walkforward.py index 2d29532..1077438 100644 --- a/walkforward.py +++ b/walkforward.py @@ -16,7 +16,7 @@ import operator import time import warnings -from typing import Any, Dict, List, Optional, Sequence, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union import numpy as np import pandas as pd @@ -163,6 +163,11 @@ class WalkForwardConfig: flat_eps: float = 0.15 flat_min_samples: int = 3 flat_selector: str = "medoid" + plateau_quantile: float = 0.25 + plateau_median_weight: float = 0.25 + plateau_std_penalty: float = 0.50 + plateau_size_bonus: float = 0.01 + scoring_backend: str = "proxy" scoring_trading_days: int = 365 min_trades_per_year: Optional[float] = None trade_penalty_factor: Optional[float] = None @@ -199,8 +204,12 @@ def __post_init__(self) -> None: if self.top_is_k is not None and self.top_is_k <= 0: raise ValueError("top_is_k must be > 0 when provided") metric = self.candidate_selection_metric.lower().strip() - if metric not in {"robust_decay", "mean_oos_sharpe", "mean_is_sharpe"}: - raise ValueError("candidate_selection_metric must be robust_decay, mean_oos_sharpe, or mean_is_sharpe") + valid_metrics = {"robust_decay", "mean_oos_sharpe", "mean_is_sharpe", "is_plateau_robust"} + if metric not in valid_metrics: + raise ValueError( + "candidate_selection_metric must be robust_decay, mean_oos_sharpe, " + "mean_is_sharpe, or is_plateau_robust" + ) object.__setattr__(self, "candidate_selection_metric", metric) candidate_decay_lambda = None if self.candidate_decay_lambda is None else float(self.candidate_decay_lambda) if candidate_decay_lambda is not None and candidate_decay_lambda < 0.0: @@ -246,6 +255,18 @@ def __post_init__(self) -> None: if selector not in {"medoid", "centroid"}: raise ValueError("flat_selector must be medoid or centroid") object.__setattr__(self, "flat_selector", selector) + if not 0.0 <= self.plateau_quantile <= 1.0: + raise ValueError("plateau_quantile must be in [0, 1]") + if self.plateau_median_weight < 0.0: + raise ValueError("plateau_median_weight must be >= 0") + if self.plateau_std_penalty < 0.0: + raise ValueError("plateau_std_penalty must be >= 0") + scoring_backend = self.scoring_backend.lower().strip() + if scoring_backend not in {"proxy", "endpoint"}: + raise ValueError("scoring_backend must be proxy or endpoint") + if opt_mode == "mode_2_sbb" and scoring_backend == "endpoint": + raise ValueError("mode_2_sbb requires scoring_backend='proxy' because it simulates train return paths") + object.__setattr__(self, "scoring_backend", scoring_backend) try: scoring_days = int(self.scoring_trading_days) except (TypeError, ValueError) as exc: @@ -438,11 +459,15 @@ def __init__( self, strategy: Any, config: Optional[WalkForwardConfig] = None, + scorer: Optional[Callable[..., Dict[str, float]]] = None, ): if strategy is None: raise ValueError("WalkForwardEngine requires a strategy callable or strategy class/object") self.strategy = strategy self.config = config or WalkForwardConfig() + self.scorer = scorer + if self.config.scoring_backend == "endpoint" and self.scorer is None: + raise ValueError("scoring_backend='endpoint' requires a scorer callback") def run( self, @@ -525,6 +550,11 @@ def run( "garch_dist": self.config.garch_dist, "garch_vol_multiplier": self.config.garch_vol_multiplier, "numba_enabled": bool(self.config.use_numba and _NUMBA_AVAILABLE), + "plateau_quantile": self.config.plateau_quantile, + "plateau_median_weight": self.config.plateau_median_weight, + "plateau_std_penalty": self.config.plateau_std_penalty, + "plateau_size_bonus": self.config.plateau_size_bonus, + "scoring_backend": self.config.scoring_backend, **self.config.metadata, }, ) @@ -642,12 +672,13 @@ def evaluate_params_is( fold=fold, context="anti-leakage in-sample search", ) - is_metrics = score_strategy_output( + is_metrics = self._score_strategy_output( data, is_output, fold.train_index, - trading_days=int(self.config.scoring_trading_days), - use_numba=bool(self.config.use_numba), + fold=fold, + params=params, + context="anti-leakage in-sample search", ) required_trades = _required_trades_for_index(fold.train_index, self.config.min_trades_per_year) factor = 1.0 if self.config.trade_penalty_factor is None else float(self.config.trade_penalty_factor) @@ -718,19 +749,21 @@ def evaluate_params( fold=fold, context="out-of-sample scoring", ) - is_metrics = score_strategy_output( + is_metrics = self._score_strategy_output( data, is_output, fold.train_index, - trading_days=int(self.config.scoring_trading_days), - use_numba=bool(self.config.use_numba), + fold=fold, + params=params, + context="in-sample scoring", ) - oos_metrics = score_strategy_output( + oos_metrics = self._score_strategy_output( data, oos_output, fold.test_index, - trading_days=int(self.config.scoring_trading_days), - use_numba=bool(self.config.use_numba), + fold=fold, + params=params, + context="out-of-sample scoring", ) is_required_trades = _required_trades_for_index(fold.train_index, self.config.min_trades_per_year) oos_required_trades = _required_trades_for_index(fold.test_index, self.config.min_trades_per_year) @@ -819,14 +852,19 @@ def evaluate_params_sbb( fold=fold, context="sbb train scoring", ) - is_metrics = score_strategy_output( + is_metrics = self._score_strategy_output( data, is_output, fold.train_index, - trading_days=int(self.config.scoring_trading_days), - use_numba=self.config.use_numba, + fold=fold, + params=params, + context="sbb train scoring", ) - returns = strategy_return_series(data, is_output, fold.train_index).to_numpy(dtype=np.float64) + returns = strategy_return_series( + data, + is_output, + fold.train_index, + ).to_numpy(dtype=np.float64) seed = int(self.config.random_seed) + int(trial_id) * 100_003 + int(fold.fold_id) * 9_176 boot = synthetic_walkforward_sharpes( returns=returns, @@ -1003,6 +1041,34 @@ def build_folds(self, idx: pd.DatetimeIndex) -> List[WalkForwardFold]: raise ValueError("walk-forward split produced no folds") return folds + def _score_strategy_output( + self, + data, + output: StrategyOutput, + index: pd.DatetimeIndex, + fold: WalkForwardFold, + params: Dict[str, Any], + context: str, + ) -> Dict[str, float]: + if self.config.scoring_backend == "endpoint": + assert self.scorer is not None + return self.scorer( + data=data, + output=output, + index=index, + fold=fold, + params=params, + context=context, + trading_days=int(self.config.scoring_trading_days), + ) + return score_strategy_output( + data, + output, + index, + trading_days=int(self.config.scoring_trading_days), + use_numba=bool(self.config.use_numba), + ) + def _call_strategy(self, data, params: Dict[str, Any], fold: WalkForwardFold) -> StrategyOutput: return self._call_strategy_for_indices( data=data, @@ -1228,7 +1294,7 @@ def strategy_position_frame(output: StrategyOutput, index: pd.DatetimeIndex) -> def strategy_return_series(data, output: StrategyOutput, index: pd.DatetimeIndex) -> pd.Series: - """Return the transparent shifted-position return proxy used by WFO scoring.""" + """Return the transparent position return proxy used by WFO scoring.""" idx = validate_datetime(index) if len(idx) == 0: return pd.Series(dtype=float, index=idx) @@ -1237,16 +1303,16 @@ def strategy_return_series(data, output: StrategyOutput, index: pd.DatetimeIndex symbols = list(output.columns) pos = _normalize_frame_output(output, symbols).reindex(idx).fillna(0.0) returns = pd.DataFrame({s: close_map[s].reindex(idx).pct_change().fillna(0.0) for s in symbols}) - strat_returns = (pos.shift(1).fillna(0.0) * returns).mean(axis=1) + strat_returns = (pos * returns).mean(axis=1) elif isinstance(output, dict): symbols = list(output.keys()) pos = pd.DataFrame({s: _normalize_series_output(output[s]).reindex(idx).fillna(0.0) for s in symbols}) returns = pd.DataFrame({s: close_map[s].reindex(idx).pct_change().fillna(0.0) for s in symbols}) - strat_returns = (pos.shift(1).fillna(0.0) * returns).mean(axis=1) + strat_returns = (pos * returns).mean(axis=1) else: series = _normalize_series_output(output).reindex(idx).fillna(0.0) close = next(iter(close_map.values())).reindex(idx) - strat_returns = series.shift(1).fillna(0.0) * close.pct_change().fillna(0.0) + strat_returns = series * close.pct_change().fillna(0.0) return strat_returns.fillna(0.0).astype(float) @@ -1876,6 +1942,142 @@ def select_flat_minima_record( ) +def select_is_plateau_robust_record( + records: Sequence[WalkForwardTrialRecord], + param_ranges: Dict[str, Any], + config: WalkForwardConfig, +) -> WalkForwardTrialRecord: + """ + Select robust train-only params from the top IS/search trial plateau. + + The selector first takes the top `top_is_fraction`/`top_is_k` trials by the + train-side objective. Inside that candidate pool it prefers dense parameter + regions whose lower-tail and median scores remain strong while penalizing + noisy, isolated peaks. OOS metrics are intentionally not used. + """ + completed = [record for record in records if not record.pruned and np.isfinite(record.objective)] + if not completed: + raise ValueError("is_plateau_robust selection received no completed trials") + ranked = sorted(completed, key=lambda record: record.objective, reverse=True) + top_n = _candidate_count(len(ranked), config) + top = ranked[:top_n] + matrix, names = _param_matrix(top, param_ranges) + if matrix.shape[0] == 1 or matrix.shape[1] == 0: + return _with_selection_metadata( + top[0], + { + "objective_mode": config.optimization_mode, + "selector": "fallback_best_train_objective", + "selected_by": "is_plateau_robust", + "oos_used_for_selection": False, + "reason": "insufficient_cluster_points", + "top_trials": int(top_n), + }, + ) + + labels, cluster_method = _dbscan_cluster_labels( + matrix, + eps=float(config.flat_eps), + min_samples=int(config.flat_min_samples), + ) + cluster_ids = sorted(label for label in set(labels.tolist()) if label >= 0) + if not cluster_ids: + return _with_selection_metadata( + top[0], + { + "objective_mode": config.optimization_mode, + "selector": "fallback_best_train_objective", + "selected_by": "is_plateau_robust", + "oos_used_for_selection": False, + "reason": "no_dense_train_plateau", + "top_trials": int(top_n), + "eps": float(config.flat_eps), + "min_samples": int(config.flat_min_samples), + "cluster_method": cluster_method, + }, + ) + + best_cluster = None + best_key = None + best_cluster_stats = None + for cluster_id in cluster_ids: + member_idx = np.flatnonzero(labels == cluster_id) + values = np.array([top[i].objective for i in member_idx], dtype=np.float64) + q = float(np.quantile(values, float(config.plateau_quantile))) + median = float(np.median(values)) + std = float(np.std(values, ddof=1)) if len(values) > 1 else 0.0 + cluster_score = ( + q + + float(config.plateau_median_weight) * median + - float(config.plateau_std_penalty) * std + + float(config.plateau_size_bonus) * float(np.log1p(len(member_idx))) + ) + key = (cluster_score, q, median, len(member_idx), float(np.max(values))) + if best_key is None or key > best_key: + best_key = key + best_cluster = member_idx + best_cluster_stats = { + "plateau_score": float(cluster_score), + "plateau_quantile_score": q, + "plateau_median_score": median, + "plateau_std_score": std, + "cluster_best_objective": float(np.max(values)), + } + assert best_cluster is not None and best_cluster_stats is not None + + centroid = np.mean(matrix[best_cluster], axis=0) + distances = np.sqrt(((matrix[best_cluster] - centroid) ** 2).sum(axis=1)) + selected_idx = int(best_cluster[int(np.argmin(distances))]) + medoid = top[selected_idx] + centroid_params = _centroid_params( + centroid=centroid, + names=names, + param_ranges=param_ranges, + base_params=medoid.params, + ) + selected = medoid + requires_evaluation = False + if config.flat_selector == "centroid": + selected = WalkForwardTrialRecord( + trial_id=-1, + params=centroid_params, + objective=float(best_cluster_stats["plateau_score"]), + mean_is_sharpe=float(np.mean([top[i].mean_is_sharpe for i in best_cluster])), + mean_oos_sharpe=0.0, + mean_decay=0.0, + std_decay=0.0, + fold_metrics=[], + ) + requires_evaluation = True + + return _with_selection_metadata( + selected, + { + **best_cluster_stats, + "objective_mode": config.optimization_mode, + "selector": str(config.flat_selector), + "selected_by": "is_plateau_robust", + "oos_used_for_selection": False, + "param_names": names, + "selected_trial_id": int(selected.trial_id), + "medoid_trial_id": int(medoid.trial_id), + "medoid_params": dict(medoid.params), + "centroid_params": centroid_params, + "centroid_normalized": [float(x) for x in centroid.tolist()], + "requires_evaluation": requires_evaluation, + "cluster_size": int(len(best_cluster)), + "top_trials": int(top_n), + "eps": float(config.flat_eps), + "min_samples": int(config.flat_min_samples), + "plateau_quantile": float(config.plateau_quantile), + "plateau_median_weight": float(config.plateau_median_weight), + "plateau_std_penalty": float(config.plateau_std_penalty), + "plateau_size_bonus": float(config.plateau_size_bonus), + "cluster_method": cluster_method, + }, + ) + + def _select_is_candidate_records( records: Sequence[WalkForwardTrialRecord], param_ranges: Dict[str, Any], @@ -1887,6 +2089,9 @@ def _select_is_candidate_records( ranked = sorted(completed, key=lambda record: record.objective, reverse=True) top_n = _candidate_count(len(ranked), config) top = ranked[:top_n] + if config.candidate_selection_metric == "is_plateau_robust": + plateau = select_is_plateau_robust_record(completed, param_ranges, config=config) + return [plateau, *top] if config.optimization_mode == "mode_3_flat_minima": flat = select_flat_minima_record(completed, param_ranges, config=config) return [flat, *top] @@ -1912,6 +2117,28 @@ def _select_oos_candidate_record( key = lambda record: record.mean_oos_sharpe elif metric == "mean_is_sharpe": key = lambda record: record.mean_is_sharpe + elif metric == "is_plateau_robust": + selected = next( + ( + record + for record in records + if record.selection_metadata.get("selected_by") == "is_plateau_robust" + ), + None, + ) + if selected is None: + key = lambda record: record.selection_metadata.get("plateau_score", record.mean_is_sharpe) + selected = max(records, key=key) + return _with_selection_metadata( + selected, + { + **selected.selection_metadata, + "selected_by": metric, + "candidate_selection_complete": True, + "oos_seen_by_optuna": False, + "oos_used_for_selection": False, + }, + ) else: # pragma: no cover - validated in config raise ValueError(f"unsupported candidate_selection_metric: {metric}") selected = max(records, key=key) @@ -2395,6 +2622,11 @@ def _config_hash(config: WalkForwardConfig) -> str: "flat_eps": config.flat_eps, "flat_min_samples": config.flat_min_samples, "flat_selector": config.flat_selector, + "plateau_quantile": config.plateau_quantile, + "plateau_median_weight": config.plateau_median_weight, + "plateau_std_penalty": config.plateau_std_penalty, + "plateau_size_bonus": config.plateau_size_bonus, + "scoring_backend": config.scoring_backend, "scoring_trading_days": config.scoring_trading_days, "min_trades_per_year": config.min_trades_per_year, "trade_penalty_factor": config.trade_penalty_factor,