diff --git a/backends/native_event.py b/backends/native_event.py index 5dfa1b6..feebb12 100644 --- a/backends/native_event.py +++ b/backends/native_event.py @@ -486,6 +486,7 @@ def run_stat_arb_pair_arbitrage( close_dict = align_series(closes, symbols, idx) contract_sizes = self._contract_size_for_spec(spec, contract_size) fee_rates = self._fee_rate_for_spec(spec) + stat_funding = self._funding_for_spec(spec, funding_rate) rebalance_threshold = spec.hedge_policy.rebalance_threshold if not spec.hedge_policy.freeze_on_entry and rebalance_threshold is None: rebalance_threshold = 0.0 @@ -523,18 +524,34 @@ def run_stat_arb_pair_arbitrage( closes=close_dict, highs=highs, lows=lows, - funding_rate=funding_rate, + funding_rate=stat_funding, contract_size=contract_sizes, leverage=leverage, fee_rate=fee_rates, symbols=symbols, ) + funding_dict = prepare_funding(stat_funding if self.config.use_funding else 0.0, symbols, idx) + roles = self._stat_arb_roles(spec) + leg_pnl_report = self._leg_pnl_report( + idx=idx, + symbols=symbols, + roles=roles, + result=result, + closes=close_dict, + funding=funding_dict, + contract_sizes=contract_sizes, + ) + package_report = self._package_pnl_report(idx, result, leg_pnl_report) beta_drift_report = self._stat_arb_beta_drift_report( idx=idx, spec=spec, plan=arb_plan, rebalance_threshold=rebalance_threshold, ) + diagnostics = result.diagnostics.copy() + diagnostics["package_pnl"] = package_report["package_pnl"] + diagnostics["package_pnl_residual"] = package_report["pnl_residual"] + result.diagnostics = diagnostics result.metadata.update( { "backend": "native_event", @@ -547,6 +564,9 @@ def run_stat_arb_pair_arbitrage( "basket_plan": plan, "basket_target_units": arb_plan.target_units, "beta_drift_report": beta_drift_report, + "spread_report": self._stat_arb_spread_report(idx, spec, close_dict, arb_plan), + "leg_pnl_report": leg_pnl_report, + "package_pnl_report": package_report, "rebalance_threshold": rebalance_threshold, "fee_rate_oneway": fee_rates, "contract_size": contract_sizes, @@ -671,7 +691,11 @@ def run_package_arbitrage( """ unsupported = (CrossExchangeArbSpec, TriangularArbSpec, OptionsVolArbSpec) if isinstance(spec, unsupported): - raise NotImplementedError(f"{type(spec).__name__} requires a specialized Phase G+ engine") + raise NotImplementedError( + f"{type(spec).__name__} is schema-validated but requires a specialized arbitrage engine; " + "do not route it through generic package execution. " + "Use QuantBTEndpoint.arbitrage_support_matrix() to inspect supported routes." + ) supported = (CalendarSpreadSpec, FundingArbitrageSpec, SpotPerpCashCarrySpec, IndexBasketArbSpec) if not isinstance(spec, supported): raise TypeError("run_package_arbitrage requires a Phase G package-style arbitrage spec") @@ -1048,6 +1072,15 @@ def _stat_arb_basket_from_spec(spec: StatArbPairSpec) -> BasketSpec: }, ) + @staticmethod + def _stat_arb_roles(spec: StatArbPairSpec) -> Dict[str, str]: + symbols = [leg.symbol for leg in spec.legs] + roles = {leg.symbol: str(leg.role or "leg") for leg in spec.legs} + if len(symbols) >= 2 and len(set(roles.values())) == 1: + roles[symbols[0]] = "leg" + roles[symbols[1]] = "hedge" + return roles + @staticmethod def _stat_arb_beta_drift_report( idx: pd.DatetimeIndex, @@ -1095,6 +1128,129 @@ def _stat_arb_beta_drift_report( ) return pd.DataFrame(rows) + @staticmethod + def _stat_arb_spread_report( + idx: pd.DatetimeIndex, + spec: StatArbPairSpec, + closes: Dict[str, pd.Series], + plan, + ) -> pd.DataFrame: + symbols = [leg.symbol for leg in spec.legs] + leg_symbol = symbols[0] + hedge_symbol = symbols[1] if len(symbols) > 1 else symbols[0] + leg_close = closes[leg_symbol].astype(float) + hedge_close = closes[hedge_symbol].astype(float) + ref_ratio = plan.entry_ratios[leg_symbol].replace(0.0, np.nan).astype(float) + hedge_ratio = (plan.entry_ratios[hedge_symbol].astype(float) / ref_ratio).fillna(0.0) + spread = leg_close + hedge_ratio * hedge_close + return pd.DataFrame( + { + "leg_symbol": leg_symbol, + "hedge_symbol": hedge_symbol, + "leg_close": leg_close, + "hedge_close": hedge_close, + "hedge_ratio_to_leg": hedge_ratio, + "spread": spread, + "abs_spread": spread.abs(), + }, + index=idx, + ) + + def _leg_pnl_report( + self, + idx: pd.DatetimeIndex, + symbols: List[str], + roles: Dict[str, str], + result: BacktestResultV2, + closes: Dict[str, pd.Series], + funding: Dict[str, pd.Series], + contract_sizes: Dict[str, float], + ) -> pd.DataFrame: + fill_rows = {} + for fill in result.fills: + ts = pd.Timestamp(fill.timestamp) + if ts.tz is None: + ts = ts.tz_localize("UTC") + else: + ts = ts.tz_convert("UTC") + key = (ts, fill.symbol) + fee, fill_pnl = fill_rows.get(key, (0.0, 0.0)) + close_price = float(closes[fill.symbol].loc[ts]) + cs = float(contract_sizes[fill.symbol]) + fill_pnl += fill.signed_qty * (close_price - float(fill.price)) * cs + fee += float(fill.fee) + fill_rows[key] = (fee, fill_pnl) + + funding_mask = make_funding_mask(idx) + cumulative = {symbol: 0.0 for symbol in symbols} + rows = [] + for i, ts in enumerate(idx): + for symbol in symbols: + cs = float(contract_sizes[symbol]) + close_price = float(closes[symbol].iloc[i]) + prev_units = 0.0 if i == 0 else float(result.positions[f"Position_{symbol}"].iloc[i - 1]) + units = float(result.positions[f"Position_{symbol}"].iloc[i]) + price_pnl = 0.0 + if i > 0: + price_pnl = prev_units * (close_price - float(closes[symbol].iloc[i - 1])) * cs + funding_cost = 0.0 + if self.config.use_funding and funding_mask[i]: + funding_cost = prev_units * close_price * cs * float(funding[symbol].iloc[i]) + fee, fill_pnl = fill_rows.get((ts, symbol), (0.0, 0.0)) + total_pnl = price_pnl + fill_pnl - fee - funding_cost + cumulative[symbol] += total_pnl + rows.append( + { + "timestamp": ts, + "symbol": symbol, + "role": roles.get(symbol, "leg"), + "units": units, + "close": close_price, + "notional": abs(units) * close_price * cs, + "price_pnl": price_pnl, + "fill_pnl": fill_pnl, + "fee": fee, + "funding_pnl": -funding_cost, + "total_pnl": total_pnl, + "cumulative_pnl": cumulative[symbol], + } + ) + return pd.DataFrame(rows) + + @staticmethod + def _package_pnl_report(idx: pd.DatetimeIndex, result: BacktestResultV2, leg_pnl_report: pd.DataFrame) -> pd.DataFrame: + grouped = leg_pnl_report.groupby("timestamp", sort=False) + package_pnl = grouped["total_pnl"].sum().reindex(idx, fill_value=0.0) + price_pnl = grouped["price_pnl"].sum().reindex(idx, fill_value=0.0) + fill_pnl = grouped["fill_pnl"].sum().reindex(idx, fill_value=0.0) + fees = grouped["fee"].sum().reindex(idx, fill_value=0.0) + funding_pnl = grouped["funding_pnl"].sum().reindex(idx, fill_value=0.0) + role_pnl = leg_pnl_report.pivot_table( + index="timestamp", + columns="role", + values="total_pnl", + aggfunc="sum", + fill_value=0.0, + ).reindex(idx, fill_value=0.0) + leg_pnl = role_pnl["leg"] if "leg" in role_pnl else pd.Series(0.0, index=idx) + hedge_pnl = role_pnl["hedge"] if "hedge" in role_pnl else pd.Series(0.0, index=idx) + report = pd.DataFrame( + { + "price_pnl": price_pnl, + "fill_pnl": fill_pnl, + "fees": fees, + "funding_pnl": funding_pnl, + "leg_pnl": leg_pnl, + "hedge_pnl": hedge_pnl, + "spread_pnl": leg_pnl + hedge_pnl, + "package_pnl": package_pnl, + "equity_delta": result.equity.diff().fillna(0.0), + }, + index=idx, + ) + report["pnl_residual"] = report["equity_delta"] - report["package_pnl"] + return report + def _basis_leg_pnl_report( self, idx: pd.DatetimeIndex, diff --git a/backends/native_portfolio.py b/backends/native_portfolio.py index a4c8ce4..0a61a9e 100644 --- a/backends/native_portfolio.py +++ b/backends/native_portfolio.py @@ -464,7 +464,8 @@ def _build_result( exposure_report = self._build_exposure_report( accepted_notional_arr=accepted_notional_arr, target_notional_arr=target_notional_arr, - equity=equity, + equity_arr=equity_arr, + idx=idx, leverages=leverages, maintenance_ratio=maintenance_ratio, betas=betas, @@ -482,29 +483,41 @@ def _build_result( fee_arr=fee_arr, ) rebalance_report = self._build_rebalance_report( - target_units=target_units_report, - accepted_units=accepted_units_report, - closes=close_report, - contract_sizes=cs, + idx=idx, + symbols=symbol_list, + target_units_arr=target_m, + accepted_units_arr=pos_arr, + closes_arr=closes_m, + contract_sizes=contract_sizes, ) positions = pd.DataFrame(pos_arr, index=idx, columns=[f"Position_{s}" for s in symbol_list], copy=False) closes = pd.DataFrame(closes_m, index=idx, columns=[f"Close_{s}" for s in symbol_list], copy=False) fees = pd.Series(fee_arr, index=idx, name="fees") turnover = pd.Series(turnover_arr, index=idx, name="turnover") - funding_cost = symbol_pnl_report.groupby("timestamp", sort=False)["funding_cost"].sum().reindex(idx, fill_value=0.0) + prev_units = np.vstack([np.zeros((1, len(symbol_list)), dtype=np.float64), pos_arr[:-1]]) + funding_cost_arr = prev_units * closes_m * cs_row * funding_m + funding_cost_arr = np.where(is_funding_bar.reshape(-1, 1).astype(bool), funding_cost_arr, 0.0).sum(axis=1) margin = exposure_report[["initial_margin", "maintenance_margin"]].copy() diagnostics = pd.DataFrame( { - "turnover": turnover, - "rejected_rebalances": (target_units_report - accepted_units_report).abs().sum(axis=1) > 1e-10, + "turnover": turnover_arr, + "rejected_rebalances": np.abs(target_m - pos_arr).sum(axis=1) > 1e-10, }, index=idx, ) + returns_arr = np.zeros_like(equity_arr, dtype=np.float64) + if len(equity_arr) > 1: + returns_arr[1:] = np.divide( + equity_arr[1:] - equity_arr[:-1], + equity_arr[:-1], + out=np.zeros(len(equity_arr) - 1, dtype=np.float64), + where=equity_arr[:-1] != 0.0, + ) return BacktestResultV2( equity=equity, - returns=equity.pct_change().fillna(0.0), + returns=pd.Series(returns_arr, index=idx, name="returns"), positions=positions, closes=closes, symbols=symbol_list, @@ -513,7 +526,7 @@ def _build_result( liquidated=liquidated, liquidation_bar=liquidation_bar, fees=fees, - funding=pd.Series(funding_cost.to_numpy(dtype=float), index=idx, name="funding"), + funding=pd.Series(funding_cost_arr, index=idx, name="funding"), margin=margin, diagnostics=diagnostics, metadata={ @@ -598,13 +611,13 @@ def _build_exposure_report( *, accepted_notional_arr: np.ndarray, target_notional_arr: np.ndarray, - equity: pd.Series, + equity_arr: np.ndarray, + idx: pd.DatetimeIndex, leverages: np.ndarray, maintenance_ratio: float, betas: np.ndarray, ) -> pd.DataFrame: abs_accepted = np.abs(accepted_notional_arr) - equity_arr = equity.to_numpy(dtype=np.float64) gross = abs_accepted.sum(axis=1) net = accepted_notional_arr.sum(axis=1) initial_margin = (abs_accepted / leverages.reshape(1, -1)).sum(axis=1) @@ -613,7 +626,9 @@ def _build_exposure_report( target_gross = np.abs(target_notional_arr).sum(axis=1) target_beta_exposure = (target_notional_arr * betas.reshape(1, -1)).sum(axis=1) mean_leverage = float(np.mean(leverages)) - out = pd.DataFrame( + gross_leverage = np.divide(gross, equity_arr, out=np.zeros_like(gross), where=equity_arr != 0.0) + net_exposure_pct = np.divide(net, equity_arr, out=np.zeros_like(net), where=equity_arr != 0.0) + return pd.DataFrame( { "long_notional": np.where(accepted_notional_arr > 0.0, accepted_notional_arr, 0.0).sum(axis=1), "short_notional": np.where(accepted_notional_arr < 0.0, -accepted_notional_arr, 0.0).sum(axis=1), @@ -627,38 +642,39 @@ def _build_exposure_report( "equity": equity_arr, "available_equity_after_im": equity_arr - initial_margin, "buying_power": equity_arr * mean_leverage, + "gross_leverage": gross_leverage, + "net_exposure_pct": net_exposure_pct, }, - index=equity.index, + index=idx, ) - out["gross_leverage"] = out["gross_notional"] / out["equity"].replace(0.0, np.nan) - out["net_exposure_pct"] = out["net_notional"] / out["equity"].replace(0.0, np.nan) - return out.fillna(0.0) @staticmethod def _build_rebalance_report( *, - target_units: pd.DataFrame, - accepted_units: pd.DataFrame, - closes: pd.DataFrame, - contract_sizes: pd.Series, + idx: pd.DatetimeIndex, + symbols: List[str], + target_units_arr: np.ndarray, + accepted_units_arr: np.ndarray, + closes_arr: np.ndarray, + contract_sizes: np.ndarray, ) -> pd.DataFrame: - diff = target_units - accepted_units - mask = diff.abs() > 1e-10 - if not mask.to_numpy().any(): + diff = target_units_arr - accepted_units_arr + row_idx, col_idx = np.nonzero(np.abs(diff) > 1e-10) + if len(row_idx) == 0: return pd.DataFrame( columns=["timestamp", "symbol", "target_units", "accepted_units", "unit_diff", "notional_diff", "reason"] ) - notional_diff = diff.mul(closes, axis=0).mul(contract_sizes, axis=1) - stacked = diff.where(mask).stack(future_stack=True).dropna() - index = stacked.index + unit_diff = diff[row_idx, col_idx] + notional_diff = unit_diff * closes_arr[row_idx, col_idx] * contract_sizes[col_idx] + symbol_arr = np.asarray(symbols, dtype=object) return pd.DataFrame( { - "timestamp": index.get_level_values(0), - "symbol": index.get_level_values(1), - "target_units": target_units.stack(future_stack=True).reindex(index).to_numpy(dtype=float), - "accepted_units": accepted_units.stack(future_stack=True).reindex(index).to_numpy(dtype=float), - "unit_diff": stacked.to_numpy(dtype=float), - "notional_diff": notional_diff.stack(future_stack=True).reindex(index).to_numpy(dtype=float), + "timestamp": idx.take(row_idx), + "symbol": symbol_arr[col_idx], + "target_units": target_units_arr[row_idx, col_idx], + "accepted_units": accepted_units_arr[row_idx, col_idx], + "unit_diff": unit_diff, + "notional_diff": notional_diff, "reason": "margin_or_portfolio_gate", } ) diff --git a/backends/native_vectorized.py b/backends/native_vectorized.py index 2451ccb..75ba334 100644 --- a/backends/native_vectorized.py +++ b/backends/native_vectorized.py @@ -466,6 +466,7 @@ def run_stat_arb_pair_arbitrage( ) contract_sizes = self._contract_size_for_spec(spec, contract_size) fee_rates = self._fee_rate_for_spec(spec) + stat_funding = self._funding_for_spec(spec, funding_rate) arb_plan = self._apply_atomic_package_margin_policy( idx=idx, plan=ArbitragePlan( @@ -490,17 +491,17 @@ def run_stat_arb_pair_arbitrage( closes=close_dict, highs=highs, lows=lows, - funding_rate=funding_rate, + funding_rate=stat_funding, contract_size=contract_sizes, leverage=leverage, fee_rate=fee_rates, symbols=symbols, ) - funding_dict = prepare_funding(funding_rate if self.config.use_funding else 0.0, symbols, idx) + funding_dict = prepare_funding(stat_funding if self.config.use_funding else 0.0, symbols, idx) leg_pnl_report = self._leg_pnl_report( idx=idx, symbols=symbols, - roles={leg.symbol: leg.role for leg in spec.legs}, + roles=self._stat_arb_roles(spec), result=result, closes=close_dict, funding=funding_dict, @@ -520,6 +521,7 @@ def run_stat_arb_pair_arbitrage( "basket_plan": plan, "basket_target_units": arb_plan.target_units, "beta_drift_report": self._stat_arb_beta_drift_report(idx, spec, arb_plan, rebalance_threshold), + "spread_report": self._stat_arb_spread_report(idx, spec, close_dict, arb_plan), "leg_pnl_report": leg_pnl_report, "package_pnl_report": package_report, "rebalance_threshold": rebalance_threshold, @@ -544,7 +546,11 @@ def run_package_arbitrage( ) -> BacktestResultV2: unsupported = (CrossExchangeArbSpec, TriangularArbSpec, OptionsVolArbSpec) if isinstance(spec, unsupported): - raise NotImplementedError(f"{type(spec).__name__} requires a specialized Phase G+ engine") + raise NotImplementedError( + f"{type(spec).__name__} is schema-validated but requires a specialized arbitrage engine; " + "do not route it through generic package execution. " + "Use QuantBTEndpoint.arbitrage_support_matrix() to inspect supported routes." + ) supported = (CalendarSpreadSpec, FundingArbitrageSpec, SpotPerpCashCarrySpec, IndexBasketArbSpec) if not isinstance(spec, supported): raise TypeError("run_package_arbitrage requires a Phase G package-style arbitrage spec") @@ -859,6 +865,43 @@ def _funding_for_spec(spec: ArbitrageSpec, funding_rate: Union[float, pd.Series, } return {leg.symbol: funding_rate if leg.symbol in funding_symbols else 0.0 for leg in spec.legs} + @staticmethod + def _stat_arb_roles(spec: StatArbPairSpec) -> Dict[str, str]: + symbols = [leg.symbol for leg in spec.legs] + roles = {leg.symbol: str(leg.role or "leg") for leg in spec.legs} + if len(symbols) >= 2 and len(set(roles.values())) == 1: + roles[symbols[0]] = "leg" + roles[symbols[1]] = "hedge" + return roles + + @staticmethod + def _stat_arb_spread_report( + idx: pd.DatetimeIndex, + spec: StatArbPairSpec, + closes: Dict[str, pd.Series], + plan, + ) -> pd.DataFrame: + symbols = [leg.symbol for leg in spec.legs] + leg_symbol = symbols[0] + hedge_symbol = symbols[1] if len(symbols) > 1 else symbols[0] + leg_close = closes[leg_symbol].astype(float) + hedge_close = closes[hedge_symbol].astype(float) + ref_ratio = plan.entry_ratios[leg_symbol].replace(0.0, np.nan).astype(float) + hedge_ratio = (plan.entry_ratios[hedge_symbol].astype(float) / ref_ratio).fillna(0.0) + spread = leg_close + hedge_ratio * hedge_close + return pd.DataFrame( + { + "leg_symbol": leg_symbol, + "hedge_symbol": hedge_symbol, + "leg_close": leg_close, + "hedge_close": hedge_close, + "hedge_ratio_to_leg": hedge_ratio, + "spread": spread, + "abs_spread": spread.abs(), + }, + index=idx, + ) + @staticmethod def _stat_arb_basket_from_spec(spec: StatArbPairSpec) -> BasketSpec: if spec.sizing_policy.kind is not SizingPolicyKind.TARGET_GROSS_NOTIONAL: @@ -929,9 +972,30 @@ def _leg_pnl_report( @staticmethod def _package_pnl_report(idx: pd.DatetimeIndex, result: BacktestResultV2, leg_pnl_report: pd.DataFrame) -> pd.DataFrame: - package_pnl = leg_pnl_report.groupby("timestamp", sort=False)["total_pnl"].sum().reindex(idx, fill_value=0.0) + grouped = leg_pnl_report.groupby("timestamp", sort=False) + package_pnl = grouped["total_pnl"].sum().reindex(idx, fill_value=0.0) + price_pnl = grouped["price_pnl"].sum().reindex(idx, fill_value=0.0) + fill_pnl = grouped["fill_pnl"].sum().reindex(idx, fill_value=0.0) + fees = grouped["fee"].sum().reindex(idx, fill_value=0.0) + funding_pnl = grouped["funding_pnl"].sum().reindex(idx, fill_value=0.0) + role_pnl = leg_pnl_report.pivot_table( + index="timestamp", + columns="role", + values="total_pnl", + aggfunc="sum", + fill_value=0.0, + ).reindex(idx, fill_value=0.0) + leg_pnl = role_pnl["leg"] if "leg" in role_pnl else pd.Series(0.0, index=idx) + hedge_pnl = role_pnl["hedge"] if "hedge" in role_pnl else pd.Series(0.0, index=idx) report = pd.DataFrame( { + "price_pnl": price_pnl, + "fill_pnl": fill_pnl, + "fees": fees, + "funding_pnl": funding_pnl, + "leg_pnl": leg_pnl, + "hedge_pnl": hedge_pnl, + "spread_pnl": leg_pnl + hedge_pnl, "package_pnl": package_pnl, "equity_delta": result.equity.diff().fillna(0.0), }, diff --git a/benchmarks/phase12_arbitrage_cert.json b/benchmarks/phase12_arbitrage_cert.json index 7038537..36297a3 100644 --- a/benchmarks/phase12_arbitrage_cert.json +++ b/benchmarks/phase12_arbitrage_cert.json @@ -2,12 +2,12 @@ "status": "pass", "sandbox_path": "/root/bobby/pool_alpha/quantbt/.local_arbitrage_sandboxes/binance_basis_arb", "basis": { - "event_final_equity": 91362.95065353338, - "vectorized_final_equity": 91362.95065353338, - "order_count": 828, - "fill_count": 828, - "fee_total": 3429.5969336746557, - "funding_total": 17.731103806709445, + "event_final_equity": 99636.15975414228, + "vectorized_final_equity": 99636.15975414228, + "order_count": 88, + "fill_count": 88, + "fee_total": 352.77694586750954, + "funding_total": 9.156364234095296, "audit": { "status": "pass", "passed": true, @@ -26,11 +26,11 @@ "final_target_flat": true, "final_position_flat": true }, - "max_abs_package_pnl_residual": 2.6261659513693303e-11, + "max_abs_package_pnl_residual": 1.7973178501051734e-11, "max_abs_leg_vs_package_pnl_diff": 0.0, - "max_abs_fee_residual": 4.547473508864641e-13, + "max_abs_fee_residual": 0.0, "final_gross_target_units": 0.0, - "final_gross_position_units": 8.881784197001252e-16, + "final_gross_position_units": 0.0, "target_symbols": [ "perpetual", "quarterly" @@ -39,8 +39,8 @@ "perpetual", "quarterly" ], - "order_count": 828, - "fill_count": 828, + "order_count": 88, + "fill_count": 88, "rejection_count": 0 }, "parity": { @@ -55,19 +55,54 @@ "target_units_match": true, "package_residuals_ok": true }, - "max_abs_equity_diff": 1.4551915228366852e-11, - "max_abs_position_diff": 4.440892098500626e-16, + "max_abs_equity_diff": 0.0, + "max_abs_position_diff": 0.0, "max_abs_target_unit_diff": 0.0, - "max_abs_package_residual": 2.6261659513693303e-11 + "max_abs_package_residual": 1.7973178501051734e-11 } }, "stat_pair": { - "event_final_equity": 98806.77550546748, - "vectorized_final_equity": 98806.77550546748, + "event_final_equity": 99727.5575962874, + "vectorized_final_equity": 99727.5575962874, "accounting_parity_passed": true, + "audit": { + "status": "pass", + "passed": true, + "tolerance": 1e-09, + "engine": "event_v1_stat_arb_pair", + "backend": "native_event", + "arb_id": "PHASE12_STAT_PAIR", + "arb_type": "stat_arb_pair", + "missing_reports": [], + "checks": { + "has_required_reports": true, + "package_pnl_residual_ok": true, + "leg_pnl_reconciles_to_package": true, + "fees_reconcile": true, + "target_symbols_match": true, + "final_target_flat": true, + "final_position_flat": true + }, + "max_abs_package_pnl_residual": 1.8891554987021664e-11, + "max_abs_leg_vs_package_pnl_diff": 0.0, + "max_abs_fee_residual": 7.105427357601002e-15, + "final_gross_target_units": 0.0, + "final_gross_position_units": 0.0, + "target_symbols": [ + "asset_a", + "asset_b" + ], + "result_symbols": [ + "asset_a", + "asset_b" + ], + "order_count": 20, + "fill_count": 20, + "rejection_count": 0 + }, "parity": { - "status": "fail", - "passed": false, + "status": "pass", + "passed": true, "tolerance": 1e-09, "event_engine": "event_v1_stat_arb_pair", "vectorized_engine": "units_v2_stat_arb_pair", @@ -75,13 +110,26 @@ "equity_matches": true, "positions_match": true, "target_units_match": true, - "package_residuals_ok": false + "package_residuals_ok": true }, "max_abs_equity_diff": 0.0, "max_abs_position_diff": 0.0, "max_abs_target_unit_diff": 0.0, - "max_abs_package_residual": null - } + "max_abs_package_residual": 1.8891554987021664e-11 + }, + "package_report_columns": [ + "price_pnl", + "fill_pnl", + "fees", + "funding_pnl", + "leg_pnl", + "hedge_pnl", + "spread_pnl", + "package_pnl", + "equity_delta", + "pnl_residual" + ], + "max_package_residual": 1.8891554987021664e-11 }, "index_basket": { "status": "pass", @@ -101,7 +149,7 @@ "max_abs_equity_diff": 0.0, "max_abs_position_diff": 0.0, "max_abs_target_unit_diff": 0.0, - "max_abs_package_residual": 1.1126211063583469e-11 + "max_abs_package_residual": 1.3033130130679638e-11 } }, "schema_only": { @@ -112,43 +160,42 @@ "native_event": { "rejected": true, "error": "NotImplementedError", - "message": "CrossExchangeArbSpec requires a specialized Phase G+ engine" + "message": "CrossExchangeArbSpec is schema-validated but requires a specialized arbitrage engine; do not route it through generic package execution. Use QuantBTEndpoint.arbitrage_support_matrix() to inspect supported routes." }, "native_vectorized": { "rejected": true, "error": "NotImplementedError", - "message": "CrossExchangeArbSpec requires a specialized Phase G+ engine" + "message": "CrossExchangeArbSpec is schema-validated but requires a specialized arbitrage engine; do not route it through generic package execution. Use QuantBTEndpoint.arbitrage_support_matrix() to inspect supported routes." } }, "triangular": { "native_event": { "rejected": true, "error": "NotImplementedError", - "message": "TriangularArbSpec requires a specialized Phase G+ engine" + "message": "TriangularArbSpec is schema-validated but requires a specialized arbitrage engine; do not route it through generic package execution. Use QuantBTEndpoint.arbitrage_support_matrix() to inspect supported routes." }, "native_vectorized": { "rejected": true, "error": "NotImplementedError", - "message": "TriangularArbSpec requires a specialized Phase G+ engine" + "message": "TriangularArbSpec is schema-validated but requires a specialized arbitrage engine; do not route it through generic package execution. Use QuantBTEndpoint.arbitrage_support_matrix() to inspect supported routes." } }, "options_vol": { "native_event": { "rejected": true, "error": "NotImplementedError", - "message": "OptionsVolArbSpec requires a specialized Phase G+ engine" + "message": "OptionsVolArbSpec is schema-validated but requires a specialized arbitrage engine; do not route it through generic package execution. Use QuantBTEndpoint.arbitrage_support_matrix() to inspect supported routes." }, "native_vectorized": { "rejected": true, "error": "NotImplementedError", - "message": "OptionsVolArbSpec requires a specialized Phase G+ engine" + "message": "OptionsVolArbSpec is schema-validated but requires a specialized arbitrage engine; do not route it through generic package execution. Use QuantBTEndpoint.arbitrage_support_matrix() to inspect supported routes." } } } }, "nautilus": { - "status": "pass", - "orders": 828, - "fills": 828 + "status": "skipped", + "reason": "run with --include-nautilus" } } diff --git a/benchmarks/phase12_arbitrage_cert.md b/benchmarks/phase12_arbitrage_cert.md index 668ce25..d90a3dd 100644 --- a/benchmarks/phase12_arbitrage_cert.md +++ b/benchmarks/phase12_arbitrage_cert.md @@ -5,19 +5,21 @@ Sandbox path: `/root/bobby/pool_alpha/quantbt/.local_arbitrage_sandboxes/binance ## Basis Perp-Quarterly -- Event final equity: `91362.950654` -- Vectorized final equity: `91362.950654` -- Max equity diff: `1.4551915228366852e-11` +- Event final equity: `99636.159754` +- Vectorized final equity: `99636.159754` +- Max equity diff: `0.0` - Audit status: `pass` -- Orders: `828` -- Fills: `828` -- Fees: `3429.596934` -- Funding: `17.731104` +- Orders: `88` +- Fills: `88` +- Fees: `352.776946` +- Funding: `9.156364` ## Other Certification Checks - Stat pair accounting parity: `True` -- Stat pair package-residual report: `False` +- Stat pair audit status: `pass` +- Stat pair package-residual report: `True` +- Stat pair max package residual: `1.8891554987021664e-11` - Index basket package smoke: `pass` - Schema-only guardrails: `pass` -- Nautilus package parity: `pass` +- Nautilus package parity: `skipped` diff --git a/benchmarks/phase13_portfolio_report.json b/benchmarks/phase13_portfolio_report.json new file mode 100644 index 0000000..7b7ffed --- /dev/null +++ b/benchmarks/phase13_portfolio_report.json @@ -0,0 +1,16 @@ +{ + "array_preparation_seconds": 0.010810222321500381, + "cython_cpp_recommendation": "Cython/C++ is not justified yet; optimize cached array preparation and report construction first.", + "full_facade_seconds": 0.05868112171689669, + "notes": "Phase 13B keeps accounting unchanged and optimizes report construction with ndarray-first calculations for funding, diagnostics, exposure, and rebalance reports.", + "prepared_reuse_facade_seconds": 0.04790363173621396, + "prepared_reuse_speedup": 1.224982732833912, + "pure_kernel_share_pct": 0.34121019806869696, + "pure_numba_kernel_seconds": 0.00020022597163915634, + "repeats": 3, + "report_construction_estimate_seconds": 0.04767067342375715, + "report_construction_share_pct": 81.2368135253809, + "rows": 2000, + "status": "pass", + "symbols": 6 +} diff --git a/benchmarks/phase13_portfolio_report.md b/benchmarks/phase13_portfolio_report.md new file mode 100644 index 0000000..42aa02e --- /dev/null +++ b/benchmarks/phase13_portfolio_report.md @@ -0,0 +1,23 @@ +# Phase 13B Native Portfolio Report Construction + +Status: **pass** + +- Rows: `2000` +- Symbols: `6` +- Repeats: `3` +- Full facade seconds: `0.058681` +- Prepared reuse seconds: `0.047904` +- Array preparation seconds: `0.010810` +- Pure Numba kernel seconds: `0.000200` +- Report construction residual seconds: `0.047671` +- Report construction share: `81.24%` +- Pure kernel share: `0.34%` +- Prepared reuse speedup: `1.225x` + +## Notes + +Phase 13B keeps accounting unchanged and optimizes report construction with ndarray-first calculations for funding, diagnostics, exposure, and rebalance reports. + +## Cython/C++ Decision + +Cython/C++ is not justified yet; optimize cached array preparation and report construction first. diff --git a/benchmarks/phase13_wfo_cache.json b/benchmarks/phase13_wfo_cache.json new file mode 100644 index 0000000..8f3dd3e --- /dev/null +++ b/benchmarks/phase13_wfo_cache.json @@ -0,0 +1,25 @@ +{ + "cache_metadata": { + "available": true, + "backend": "native_portfolio", + "enabled": true, + "fallback_runs": 0, + "market_cache_entries": 2, + "market_cache_hits": 8, + "market_cache_misses": 2, + "prepared_runs": 10, + "target_mode": "portfolio" + }, + "cached_seconds": 0.9311932160053402, + "final_equity_diff": 0.0, + "objective_diff": 0.0, + "params_match": true, + "rows": 720, + "selected_params": { + "scale": 1.5 + }, + "speedup": 0.8895758405827113, + "status": "pass", + "trials": 12, + "uncached_seconds": 0.8283669878728688 +} diff --git a/benchmarks/phase13_wfo_cache.md b/benchmarks/phase13_wfo_cache.md new file mode 100644 index 0000000..b4ba95b --- /dev/null +++ b/benchmarks/phase13_wfo_cache.md @@ -0,0 +1,25 @@ +# Phase 13A WFO Prepared Market Cache + +Status: **pass** + +- Rows: `720` +- Optuna trials: `12` +- Cached seconds: `0.931193` +- Uncached seconds: `0.828367` +- Speedup: `0.890x` +- Final equity diff: `0.0` +- Objective diff: `0.0` +- Params match: `True` +- Selected params: `{'scale': 1.5}` + +## Cache Metadata + +- Enabled: `True` +- Prepared runs: `10` +- Fallback runs: `0` +- Market cache hits: `8` +- Market cache misses: `2` +- Market cache entries: `2` + +The benchmark is a deterministic parity/reuse guard, not a universal speed claim. +Full WFO runtime can still be dominated by Optuna and report construction. diff --git a/benchmarks/run_phase12_arbitrage_cert.py b/benchmarks/run_phase12_arbitrage_cert.py index 2f5ede3..c247435 100644 --- a/benchmarks/run_phase12_arbitrage_cert.py +++ b/benchmarks/run_phase12_arbitrage_cert.py @@ -141,6 +141,7 @@ def run_certification(rows: int = 900, include_nautilus: bool = False) -> Dict: stat_idx, stat_signal, stat_closes, stat_highs, stat_lows, stat_funding = _stat_market(rows=rows) stat_event = event.run_stat_arb_pair_arbitrage(stat_idx, stat_spec(), stat_signal, stat_closes, highs=stat_highs, lows=stat_lows, funding_rate=stat_funding) stat_vector = vector.run_stat_arb_pair_arbitrage(stat_idx, stat_spec(), stat_signal, stat_closes, highs=stat_highs, lows=stat_lows, funding_rate=stat_funding) + stat_audit = build_arbitrage_domain_audit(stat_event, raise_on_fail=False) stat_parity = compare_native_arbitrage_results(stat_event, stat_vector, raise_on_fail=False) basket_report = _index_basket_smoke(event, vector, rows) @@ -163,7 +164,10 @@ def run_certification(rows: int = 900, include_nautilus: bool = False) -> Dict: "event_final_equity": float(stat_event.equity.iloc[-1]), "vectorized_final_equity": float(stat_vector.equity.iloc[-1]), "accounting_parity_passed": _accounting_parity_passed(stat_parity), + "audit": stat_audit, "parity": stat_parity, + "package_report_columns": list(stat_event.metadata["package_pnl_report"].columns), + "max_package_residual": float(stat_event.metadata["package_pnl_report"]["pnl_residual"].abs().max()), }, "index_basket": basket_report, "schema_only": schema_report, @@ -193,7 +197,9 @@ def make_markdown(report: Dict) -> str: "## Other Certification Checks", "", f"- Stat pair accounting parity: `{report['stat_pair']['accounting_parity_passed']}`", + f"- Stat pair audit status: `{report['stat_pair']['audit']['status']}`", f"- Stat pair package-residual report: `{report['stat_pair']['parity']['checks'].get('package_residuals_ok')}`", + f"- Stat pair max package residual: `{report['stat_pair']['max_package_residual']}`", f"- Index basket package smoke: `{report['index_basket']['status']}`", f"- Schema-only guardrails: `{report['schema_only']['status']}`", f"- Nautilus package parity: `{report['nautilus']['status']}`", @@ -360,6 +366,7 @@ def _accounting_parity_passed(parity: Dict) -> bool: checks.get("equity_matches") and checks.get("positions_match") and checks.get("target_units_match") + and checks.get("package_residuals_ok") ) diff --git a/benchmarks/run_phase13_portfolio_report.py b/benchmarks/run_phase13_portfolio_report.py new file mode 100644 index 0000000..a79f952 --- /dev/null +++ b/benchmarks/run_phase13_portfolio_report.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +""" +Phase 13B native portfolio report-construction benchmark. + +The runner reuses the Phase 12B decomposition and writes a focused artifact for +the report-construction optimization pass. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Dict + +PACKAGE_DIR = Path(__file__).resolve().parents[1] +PROJECT_DIR = PACKAGE_DIR.parent +if str(PROJECT_DIR) not in sys.path: + sys.path.insert(0, str(PROJECT_DIR)) + +from quantbt.benchmarks.run_phase12_benchmark_nautilus_cert import run_certification # noqa: E402 + + +def run_report(rows: int = 2_000, symbols: int = 6, repeats: int = 3) -> Dict: + report = run_certification(rows=rows, symbols=symbols, repeats=repeats, include_nautilus=False) + bench = report["benchmark_followup"] + stages = bench["stages"] + return { + "status": "pass" if report["status"] == "pass" and bench["status"] == "pass" else "fail", + "rows": int(rows), + "symbols": int(symbols), + "repeats": int(repeats), + "full_facade_seconds": float(stages["full_facade_seconds"]), + "prepared_reuse_facade_seconds": float(stages["prepared_reuse_facade_seconds"]), + "array_preparation_seconds": float(stages["array_preparation_seconds"]), + "pure_numba_kernel_seconds": float(stages["pure_numba_kernel_seconds"]), + "report_construction_estimate_seconds": float(stages["report_construction_estimate_seconds"]), + "report_construction_share_pct": float(stages["report_construction_share_pct"]), + "pure_kernel_share_pct": float(stages["pure_kernel_share_pct"]), + "prepared_reuse_speedup": float(stages["prepared_reuse_speedup"]), + "cython_cpp_recommendation": report["cython_cpp_recommendation"], + "notes": ( + "Phase 13B keeps accounting unchanged and optimizes report construction " + "with ndarray-first calculations for funding, diagnostics, exposure, " + "and rebalance reports." + ), + } + + +def make_markdown(report: Dict) -> str: + return "\n".join( + [ + "# Phase 13B Native Portfolio Report Construction", + "", + f"Status: **{report['status']}**", + "", + f"- Rows: `{report['rows']}`", + f"- Symbols: `{report['symbols']}`", + f"- Repeats: `{report['repeats']}`", + f"- Full facade seconds: `{report['full_facade_seconds']:.6f}`", + f"- Prepared reuse seconds: `{report['prepared_reuse_facade_seconds']:.6f}`", + f"- Array preparation seconds: `{report['array_preparation_seconds']:.6f}`", + f"- Pure Numba kernel seconds: `{report['pure_numba_kernel_seconds']:.6f}`", + f"- Report construction residual seconds: `{report['report_construction_estimate_seconds']:.6f}`", + f"- Report construction share: `{report['report_construction_share_pct']:.2f}%`", + f"- Pure kernel share: `{report['pure_kernel_share_pct']:.2f}%`", + f"- Prepared reuse speedup: `{report['prepared_reuse_speedup']:.3f}x`", + "", + "## Notes", + "", + report["notes"], + "", + "## Cython/C++ Decision", + "", + report["cython_cpp_recommendation"], + ] + ) + "\n" + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--rows", type=int, default=2_000) + parser.add_argument("--symbols", type=int, default=6) + parser.add_argument("--repeats", type=int, default=3) + parser.add_argument("--json", type=Path, default=PACKAGE_DIR / "benchmarks" / "phase13_portfolio_report.json") + parser.add_argument("--markdown", type=Path, default=PACKAGE_DIR / "benchmarks" / "phase13_portfolio_report.md") + args = parser.parse_args() + report = run_report(rows=args.rows, symbols=args.symbols, repeats=args.repeats) + args.json.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + args.markdown.write_text(make_markdown(report)) + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 if report["status"] == "pass" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/run_phase13_wfo_cache.py b/benchmarks/run_phase13_wfo_cache.py new file mode 100644 index 0000000..4bf1a08 --- /dev/null +++ b/benchmarks/run_phase13_wfo_cache.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +""" +Phase 13A WFO prepared market-cache benchmark. + +This runner verifies that portfolio WFO endpoint scoring can reuse prepared +market arrays across Optuna trials without changing selected params, objective, +or final backtest equity. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path +from typing import Dict + +import numpy as np +import pandas as pd + +PACKAGE_DIR = Path(__file__).resolve().parents[1] +PROJECT_DIR = PACKAGE_DIR.parent +if str(PROJECT_DIR) not in sys.path: + sys.path.insert(0, str(PROJECT_DIR)) + +from quantbt import QuantBTEndpoint # noqa: E402 + + +def run_benchmark(rows: int = 720, trials: int = 16) -> Dict: + data = _make_data(rows) + _run_wfo(data, trials=max(2, min(int(trials), 4)), use_cache=True) + _run_wfo(data, trials=max(2, min(int(trials), 4)), use_cache=False) + cached_seconds, cached = _time_run(data, trials=trials, use_cache=True) + uncached_seconds, uncached = _time_run(data, trials=trials, use_cache=False) + cached_wf = cached.metadata["walk_forward"] + uncached_wf = uncached.metadata["walk_forward"] + cache_meta = cached_wf.get("prepared_scoring_cache", {}) + final_equity_diff = float(abs(cached.equity.iloc[-1] - uncached.equity.iloc[-1])) + objective_diff = float(abs(cached_wf["best_trial"]["objective"] - uncached_wf["best_trial"]["objective"])) + params_match = cached_wf["params"] == uncached_wf["params"] + speedup = float(uncached_seconds / cached_seconds) if cached_seconds > 0.0 else 0.0 + status = "pass" if final_equity_diff <= 1e-9 and objective_diff <= 1e-12 and params_match else "fail" + return { + "status": status, + "rows": int(rows), + "trials": int(trials), + "cached_seconds": float(cached_seconds), + "uncached_seconds": float(uncached_seconds), + "speedup": speedup, + "final_equity_diff": final_equity_diff, + "objective_diff": objective_diff, + "params_match": bool(params_match), + "selected_params": cached_wf["params"], + "cache_metadata": cache_meta, + } + + +def make_markdown(report: Dict) -> str: + cache = report["cache_metadata"] + lines = [ + "# Phase 13A WFO Prepared Market Cache", + "", + f"Status: **{report['status']}**", + "", + f"- Rows: `{report['rows']}`", + f"- Optuna trials: `{report['trials']}`", + f"- Cached seconds: `{report['cached_seconds']:.6f}`", + f"- Uncached seconds: `{report['uncached_seconds']:.6f}`", + f"- Speedup: `{report['speedup']:.3f}x`", + f"- Final equity diff: `{report['final_equity_diff']}`", + f"- Objective diff: `{report['objective_diff']}`", + f"- Params match: `{report['params_match']}`", + f"- Selected params: `{report['selected_params']}`", + "", + "## Cache Metadata", + "", + f"- Enabled: `{cache.get('enabled')}`", + f"- Prepared runs: `{cache.get('prepared_runs')}`", + f"- Fallback runs: `{cache.get('fallback_runs')}`", + f"- Market cache hits: `{cache.get('market_cache_hits')}`", + f"- Market cache misses: `{cache.get('market_cache_misses')}`", + f"- Market cache entries: `{cache.get('market_cache_entries')}`", + "", + "The benchmark is a deterministic parity/reuse guard, not a universal speed claim.", + "Full WFO runtime can still be dominated by Optuna and report construction.", + ] + return "\n".join(lines) + "\n" + + +def _time_run(data: Dict[str, pd.DataFrame], *, trials: int, use_cache: bool): + start = time.perf_counter() + result = _run_wfo(data, trials=trials, use_cache=use_cache) + return time.perf_counter() - start, result + + +def _run_wfo(data: Dict[str, pd.DataFrame], *, trials: int, use_cache: bool): + def strategy(data, params, train_index, test_index, fold): + scale = float(params["scale"]) + return pd.DataFrame({"BTC": scale, "ETH": -scale}, index=test_index) + + endpoint = QuantBTEndpoint.train_test_split( + strategy_class=strategy, + test_start="2022-01-01", + target_mode="portfolio", + portfolio_mode="longshort", + optimization_mode="mode_1_decay", + optimization_config={ + "scoring_backend": "endpoint", + "use_prepared_scoring_cache": bool(use_cache), + }, + optuna_trials=int(trials), + random_seed=123, + initial_capital=100_000.0, + leverage=5.0, + alloc_per_trade=1_000.0, + fee=0.0, + use_funding=False, + ) + return endpoint.backtest(data=data, param_ranges={"scale": (0.5, 1.5, 0.05)}) + + +def _make_data(rows: int) -> Dict[str, pd.DataFrame]: + idx = pd.date_range("2021-01-01", periods=int(rows), freq="1D", tz="UTC") + x = np.linspace(0.0, 16.0, len(idx)) + btc_close = 100.0 + np.sin(x) * 2.0 + np.arange(len(idx)) * 0.01 + eth_close = 50.0 + np.cos(x) * 1.5 + np.arange(len(idx)) * 0.005 + return { + "BTC": _frame(idx, btc_close), + "ETH": _frame(idx, eth_close), + } + + +def _frame(idx: pd.DatetimeIndex, close: np.ndarray) -> pd.DataFrame: + return pd.DataFrame( + { + "open": close, + "high": close * 1.01, + "low": close * 0.99, + "close": close, + "volume": 1_000.0, + }, + index=idx, + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--rows", type=int, default=720) + parser.add_argument("--trials", type=int, default=16) + parser.add_argument("--json", type=Path, default=PACKAGE_DIR / "benchmarks" / "phase13_wfo_cache.json") + parser.add_argument("--markdown", type=Path, default=PACKAGE_DIR / "benchmarks" / "phase13_wfo_cache.md") + args = parser.parse_args() + report = run_benchmark(rows=args.rows, trials=args.trials) + args.json.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + args.markdown.write_text(make_markdown(report)) + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 if report["status"] == "pass" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/endpoint.md b/docs/endpoint.md index dd609c0..b301c27 100644 --- a/docs/endpoint.md +++ b/docs/endpoint.md @@ -797,6 +797,7 @@ result = bt.backtest( bt.show_metrics() result.metadata["package_target_units"] result.metadata["leg_pnl_report"] +result.metadata["package_pnl_report"] result.metadata["beta_drift_report"] ``` @@ -805,7 +806,7 @@ Supported executable specs: | Spec | Native event | Native vectorized | Nautilus | Notes | |---|---:|---:|---:|---| | `BasisArbitrageSpec` | yes | yes | package validation | linear USDM-style legs only today | -| `StatArbPairSpec` | yes | yes | package validation | frozen hedge-ratio pair; dynamic `hedge_ratios` supported | +| `StatArbPairSpec` | yes | yes | package validation | frozen or rebalance-threshold hedge-ratio pair; dynamic `hedge_ratios` supported | | `CalendarSpreadSpec` | yes | yes | no | package-style futures spread | | `FundingArbitrageSpec` | yes | yes | no | funding-enabled leg required | | `SpotPerpCashCarrySpec` | yes | yes | no | spot plus funding-enabled derivative | @@ -844,6 +845,19 @@ Package execution policy: with enough margin can open, rejected legs are recorded as `insufficient_margin_best_effort`, and only actual open legs are later closed. +Stat-arb reporting: + +- `leg_pnl_report`: row-level per-symbol PnL with `role`, `price_pnl`, + `fill_pnl`, `fee`, `funding_pnl`, `total_pnl`, and `cumulative_pnl`; +- `package_pnl_report`: timestamp-level reconciliation with `leg_pnl`, + `hedge_pnl`, `spread_pnl`, `fees`, `funding_pnl`, `package_pnl`, + `equity_delta`, and `pnl_residual`; +- `spread_report`: pair spread/residual computed from the frozen or current + hedge ratio used by the package planner; +- `beta_drift_report`: hedge-ratio drift diagnostics and threshold breaches; +- funding is charged only to legs marked `funding_enabled=True`, matching the + rest of the arbitrage package routes. + Current hard guards: - inverse and quanto contract sizing raises `NotImplementedError` until proper diff --git a/endpoint.py b/endpoint.py index 90e1cbe..637c885 100644 --- a/endpoint.py +++ b/endpoint.py @@ -16,7 +16,14 @@ import pandas as pd from .backtester import BacktestEngine -from .backends import NativeEventBackend, NativeEventConfig, NativeVectorizedBackend, NativeVectorizedConfig +from .backends import ( + NativeEventBackend, + NativeEventConfig, + NativePortfolioBackend, + NativePortfolioConfig, + NativeVectorizedBackend, + NativeVectorizedConfig, +) from .core.arbitrage import ( BasisArbitrageSpec, CalendarSpreadSpec, @@ -531,6 +538,8 @@ def walk_forward( _default_walkforward_scoring_backend(target_mode=target_mode, optimization_mode=optimization_mode), ) ) + wf_metadata = dict(optimization_config.get("metadata", {}) or {}) + wf_metadata.setdefault("use_prepared_scoring_cache", bool(optimization_config.get("use_prepared_scoring_cache", True))) if wf_config is None: wf_config = WalkForwardConfig( split_mode=split_mode, @@ -595,6 +604,7 @@ def walk_forward( min_trades_per_year=optimization_config.get("min_trades_per_year"), trade_penalty_factor=optimization_config.get("trade_penalty_factor"), use_numba=bool(optimization_config.get("use_numba", True)), + metadata=wf_metadata, ) default_sizing = "signal_notional" if target_mode in {"portfolio", "basket", "arbitrage"} else target_mode sizing = kwargs.pop("sizing", kwargs.pop("hedge_type", default_sizing)) @@ -1216,7 +1226,21 @@ def _run_walk_forward( raise ValueError("walk_forward endpoint requires strategy_class") wf_config = self.config.walkforward_config or WalkForwardConfig(target_mode=self.config.walkforward_target_mode) target_mode = self.config.walkforward_target_mode.lower().strip() - scorer = _make_walkforward_endpoint_scorer(self.config, target_mode=target_mode, symbols=symbols) if wf_config.scoring_backend == "endpoint" else None + scorer = ( + _make_walkforward_endpoint_scorer( + self.config, + target_mode=target_mode, + symbols=symbols, + wf_config=wf_config, + market_data=data, + market_closes=closes, + market_highs=highs, + market_lows=lows, + market_datetime_index=datetime_index, + ) + if wf_config.scoring_backend == "endpoint" + else None + ) engine = WalkForwardEngine(strategy=self.config.strategy_class, config=wf_config, scorer=scorer) wf_result = engine.run( data=data if data is not None else closes, @@ -1330,6 +1354,8 @@ def _run_walk_forward( "scoring_backend": wf_result.metadata.get("scoring_backend"), "numba_enabled": wf_result.metadata.get("numba_enabled"), } + if scorer is not None and hasattr(scorer, "prepared_cache_metadata"): + result.metadata["walk_forward"]["prepared_scoring_cache"] = scorer.prepared_cache_metadata() result.metadata["walk_forward_result"] = wf_result self.engine = engine self.result = result @@ -1897,23 +1923,87 @@ def _default_walkforward_scoring_backend(target_mode: str, optimization_mode: st return "proxy" -def _make_walkforward_endpoint_scorer(config: EndpointConfig, target_mode: str, symbols=None): - score_config = _walkforward_scoring_config(config, target_mode) - symbol_list = list(symbols or config.symbols or ["DEFAULT"]) +def _make_walkforward_endpoint_scorer( + config: EndpointConfig, + target_mode: str, + symbols=None, + wf_config: Optional[WalkForwardConfig] = None, + market_data=None, + market_closes=None, + market_highs=None, + market_lows=None, + market_datetime_index=None, +): + return _WalkForwardEndpointScorer( + config=config, + target_mode=target_mode, + symbols=symbols, + wf_config=wf_config, + market_data=market_data, + market_closes=market_closes, + market_highs=market_highs, + market_lows=market_lows, + market_datetime_index=market_datetime_index, + ) - def scorer(data, output, index, fold, params, context: str, trading_days: int) -> Dict[str, float]: - temp = QuantBTEndpoint(score_config) - sliced_data = _slice_wf_data_to_index(data, index) + +class _WalkForwardEndpointScorer: + """ + Endpoint-backed WFO scorer with run-local prepared market array reuse. + + The cache is intentionally scoped to one scorer instance, which is created + for one `QuantBTEndpoint.backtest(...)` call. It never caches by pandas + object identity and every prepared reuse is validated by backend signatures. + """ + + def __init__( + self, + config: EndpointConfig, + target_mode: str, + symbols=None, + wf_config: Optional[WalkForwardConfig] = None, + market_data=None, + market_closes=None, + market_highs=None, + market_lows=None, + market_datetime_index=None, + ): + self.config = config + self.target_mode = str(target_mode).lower().strip() + self.score_config = _walkforward_scoring_config(config, self.target_mode) + self.symbols = None if symbols is None and config.symbols is None else list(symbols or config.symbols or []) + self.wf_config = wf_config + self.market_data = market_data + self.market_closes = market_closes + self.market_highs = market_highs + self.market_lows = market_lows + self.market_datetime_index = market_datetime_index + self.use_prepared_cache = bool((wf_config.metadata if wf_config is not None else {}).get("use_prepared_scoring_cache", True)) + self._portfolio_backend = None + self._portfolio_market_maps = {} + self._portfolio_market_cache = {} + self._stats = { + "enabled": bool(self.use_prepared_cache), + "target_mode": self.target_mode, + "backend": self.score_config.backend, + "market_cache_hits": 0, + "market_cache_misses": 0, + "market_cache_entries": 0, + "prepared_runs": 0, + "fallback_runs": 0, + } + + def __call__(self, data, output, index, fold, params, context: str, trading_days: int) -> Dict[str, float]: try: - if score_config.mode == "portfolio": - result = temp.backtest(data=sliced_data, positions=output, symbols=symbol_list) + if self._can_score_portfolio_prepared(output): + result = self._score_portfolio_prepared(output=output, index=index) else: - result = temp.backtest(data=sliced_data, signal=output, symbols=symbol_list) + result = self._score_fallback(data=data, output=output, index=index) report = result.full_report(trading_days=trading_days, scope="full") except Exception as exc: raise RuntimeError( "walk-forward endpoint scoring failed during " - f"{context} for fold_id={fold.fold_id}; target_mode={target_mode!r}; params={params}" + f"{context} for fold_id={fold.fold_id}; target_mode={self.target_mode!r}; params={params}" ) from exc return { "sharpe": float(report.get("sharpe", 0.0)), @@ -1925,7 +2015,124 @@ def scorer(data, output, index, fold, params, context: str, trading_days: int) - "profit_factor": float(report.get("profit_factor", 0.0)), } - return scorer + def prepared_cache_metadata(self) -> Dict[str, object]: + meta = dict(self._stats) + meta["market_cache_entries"] = len(self._portfolio_market_cache) + meta["available"] = self.target_mode == "portfolio" and self.score_config.backend == "native_portfolio" + return meta + + def _can_score_portfolio_prepared(self, output) -> bool: + return ( + self.use_prepared_cache + and self.score_config.mode == "portfolio" + and self.score_config.backend == "native_portfolio" + and isinstance(output, (pd.DataFrame, dict)) + ) + + def _score_fallback(self, data, output, index): + self._stats["fallback_runs"] += 1 + temp = QuantBTEndpoint(self.score_config) + sliced_data = _slice_wf_data_to_index(data, index) + symbol_list = self._symbol_list(output) + if self.score_config.mode == "portfolio": + return temp.backtest(data=sliced_data, positions=output, symbols=symbol_list) + return temp.backtest(data=sliced_data, signal=output, symbols=symbol_list) + + def _score_portfolio_prepared(self, output, index): + idx = _ensure_utc_index(index) + symbol_list = self._symbol_list(output) + close_map, high_map, low_map = self._portfolio_maps(symbol_list) + backend = self._portfolio_backend_instance() + cache_key = self._market_cache_key(idx, symbol_list) + market = self._portfolio_market_cache.get(cache_key) + if market is None: + market = backend.prepare_market_arrays( + datetime_index=idx, + closes=close_map, + highs=high_map, + lows=low_map, + funding_rate=self.score_config.funding_rate, + symbols=symbol_list, + ) + self._portfolio_market_cache[cache_key] = market + self._stats["market_cache_misses"] += 1 + else: + self._stats["market_cache_hits"] += 1 + + pos_map = _positions_to_map(output) + raw_signals = NativePortfolioBackend.prepare_signal_matrix(pos_map, idx, symbol_list) + self._stats["prepared_runs"] += 1 + return backend.run_signals( + positions=None, + closes=close_map, + highs=high_map, + lows=low_map, + datetime_index=idx, + mode=self.score_config.portfolio_mode, + alloc_per_trade=self.score_config.alloc_per_trade, + contract_size=self.score_config.contract_size, + hedge_type=self.score_config.sizing if self.score_config.sizing else "notional", + funding_rate=self.score_config.funding_rate, + leverage=self.score_config.account.leverage, + maintenance_ratio=self.score_config.account.maintenance_ratio, + asset_type=self.score_config.asset_type, + use_pyramiding=self.score_config.use_pyramiding, + betas=self.score_config.betas, + risk_lookback=self.score_config.risk_lookback, + market_arrays=market, + raw_signal_matrix=raw_signals, + instruments=self.score_config.instruments, + qty_step=self.score_config.qty_step, + lot_size=self.score_config.lot_size, + slot_size=self.score_config.slot_size, + min_qty=self.score_config.min_qty, + min_notional=self.score_config.min_notional, + ) + + def _portfolio_backend_instance(self) -> NativePortfolioBackend: + if self._portfolio_backend is None: + asset_type = self.score_config.asset_type.lower() + default_fee = 0.0004 if asset_type == "crypto" else 0.0001 + fee_oneway = (self.score_config.fee if self.score_config.fee is not None else default_fee) / 2.0 + self._portfolio_backend = NativePortfolioBackend( + NativePortfolioConfig( + account=self.score_config.account, + execution=self.score_config.execution, + fee_rate=fee_oneway, + use_funding=bool(self.score_config.use_funding), + ) + ) + return self._portfolio_backend + + def _portfolio_maps(self, symbol_list): + key = tuple(symbol_list) + if key not in self._portfolio_market_maps: + close_map, high_map, low_map, _idx, _symbols = _normalize_symbol_data( + data=self.market_data, + closes=self.market_closes, + highs=self.market_highs, + lows=self.market_lows, + datetime_index=self.market_datetime_index, + symbols=symbol_list, + ) + self._portfolio_market_maps[key] = (close_map, high_map, low_map) + return self._portfolio_market_maps[key] + + def _symbol_list(self, output) -> list: + if self.symbols: + return list(self.symbols) + if isinstance(output, pd.DataFrame): + return list(output.columns) + if isinstance(output, dict): + return list(output.keys()) + return ["DEFAULT"] + + @staticmethod + def _market_cache_key(index: pd.DatetimeIndex, symbols: Sequence[str]): + idx = _ensure_utc_index(index) + first = None if len(idx) == 0 else int(idx.asi8[0]) + last = None if len(idx) == 0 else int(idx.asi8[-1]) + return (tuple(symbols), int(len(idx)), first, last) def _walkforward_scoring_config(config: EndpointConfig, target_mode: str) -> EndpointConfig: diff --git a/methodology/walk_forward.md b/methodology/walk_forward.md new file mode 100644 index 0000000..4b5b4e0 --- /dev/null +++ b/methodology/walk_forward.md @@ -0,0 +1,654 @@ +# Walk Forward Methodology + +Tài liệu này mô tả methodology Walk Forward Optimization (WFO) đang được +implementation trong QuantBT. Mục tiêu không phải là chọn params đẹp nhất trên +quá khứ, mà là chọn params có khả năng generalize tốt hơn, ít overfit hơn, và có +audit trail rõ ràng cho từng fold, trial, candidate và final backtest. + +- `IS`: in-sample train set. +- `OOS`: out-of-sample test set. +- `theta`: một bộ tham số strategy. +- `r_theta`: return series sinh ra bởi strategy với tham số `theta`. +- `S_IS`, `S_OOS`: Sharpe trên IS và OOS. +- `J(theta)`: objective value dùng để xếp hạng params. + +--- + +## 1. Vai Trò Của WFO Trong QuantBT + +WFO nằm giữa research layer và execution/accounting backtest engine. Research +layer chịu trách nhiệm feature, alpha logic, signal và tránh look-ahead bias +trong strategy. WFO chịu trách nhiệm chia thời gian, gọi strategy theo fold, +chạy Optuna nếu cần, chọn params, stitch OOS signal, rồi gửi signal cuối vào +endpoint backtest thật. + +Thiết kế này cố ý tách alpha logic khỏi engine: research alpha, model selection, +execution simulation, accounting, report và audit là các lớp riêng. + +--- + +## 2. Time-Safe Fold Construction + +Mỗi fold có `train_index`, `test_index`, `train_start`, `train_end`, +`test_start`, `test_end`. Với WFO chuẩn: + +$$ +\mathrm{train\_end}_{k}<\mathrm{test\_start}_{k} +$$ + +Tham số split: `split_mode`, `split_frequency`, `window_mode`, `train_window`, +`min_train_bars`, `min_test_bars`. `expanding` dùng toàn bộ lịch sử trước OOS; +`rolling` chỉ dùng cửa sổ train gần nhất. + +--- + +## 3. Strategy Contract + +Strategy có thể là callable, class, hoặc object có `build_signal` / +`generate_signal`. QuantBT gọi theo contract: + +```python +strategy(data=data, params=params, train_index=train_index, test_index=test_index, fold=fold) +``` + +Output hợp lệ là `pd.Series`, `pd.DataFrame`, hoặc `dict[str, pd.Series]`, và +bắt buộc có `DatetimeIndex` để tránh reindex sai. + +--- + +## 4. Scoring Backend + +QuantBT WFO có hai scoring backend: `proxy` và `endpoint`. `proxy` tính return +proxy trực tiếp từ signal và data, nhanh hơn cho bootstrap/Optuna. `endpoint` +gọi đúng QuantBT endpoint như `pct_equity`, `signal_notional`, portfolio, basket +hoặc arbitrage, chậm hơn nhưng sát accounting thật hơn. `mode_2_sbb` bắt buộc +dùng `proxy`, vì nó cần mô phỏng nhiều synthetic paths từ IS return. + +--- + +## 5. Metric Nền Và Annualization + +Metric nền là Sharpe: + +$$ +\mathrm{Sharpe}(\theta)=\frac{\operatorname{mean}(r_{\theta})}{\operatorname{std}(r_{\theta})}\sqrt{N} +$$ + +Trong đó `N = scoring_trading_days`; crypto thường dùng `365`, equity daily +thường dùng `252`. Sharpe luôn phải được đọc cùng trade count, drawdown, +exposure và số bar. + +--- + +## 6. Trade Frequency Penalty + +Một alpha đánh quá ít có thể tạo Sharpe cao giả tạo. QuantBT có optional penalty +qua `min_trades_per_year` và `trade_penalty_factor`. Số trade yêu cầu được scale +theo thời lượng: + +$$ +T_{\mathrm{req}}=\mathrm{min\_trades\_per\_year}\times\frac{\mathrm{duration\_days}}{365} +$$ + +Penalty: + +$$ +\mathrm{Penalty}=\alpha\max\left(0,1-\frac{T_{\mathrm{actual}}}{T_{\mathrm{req}}}\right) +$$ + +Sharpe sau penalty: + +$$ +S_{\mathrm{penalized}}=S_{\mathrm{raw}}-\mathrm{Penalty} +$$ + +Nếu không khai báo penalty, hành vi cũ được giữ nguyên. Đây là guardrail chống +bẫy overfit của alpha ít giao dịch nhưng giữ position lâu để làm Sharpe đẹp. + +--- + +## 7. Optuna Search Và Candidate Pool + +Khi truyền `param_ranges`, QuantBT dùng Optuna để sample trial. Các tham số chính +là `optuna_trials`, `optuna_early_stopping`, `random_seed`, `use_numba`, +`top_is_fraction`, `top_is_k`. + +Nếu `top_is_k` được khai báo, QuantBT lấy đúng top K trial. Nếu không, số +candidate được tính: + +$$ +K=\left\lceil n_{\mathrm{trials}}\times\mathrm{top\_is\_fraction}\right\rceil +$$ + +Candidate selection không nhất thiết lấy trial tốt nhất tuyệt đối. Một số mode +ưu tiên vùng tham số robust hơn, dù objective đơn điểm thấp hơn một chút. + +--- + +## 8. Candidate Selection Metrics + +Các selector hiện có: + +- `robust_decay`: chọn theo OOS Sharpe sau khi phạt decay. +- `mean_oos_sharpe`: chọn OOS Sharpe trung bình. +- `mean_is_sharpe`: chọn IS Sharpe trung bình. +- `is_plateau_robust`: chọn plateau trên train/search objective. +- `is_only_robust`: strict IS-only temporal plus plateau selector. +- `full_robust`: full-sample temporal plus plateau calibration. +- `full_plateau_robust`: full-sample plateau-only calibration. +- `full_temporal_robust`: full-sample temporal-only calibration. +- `full_best`: best full-sample objective, rủi ro overfit cao nhất. + +Default mode 1 là `robust_decay`. Default mode 4 là `is_only_robust`. Default +mode 5 là `full_robust`. Config intentionally reject metric sai cho mode 4/5 để +tránh user tưởng đang chạy anti-leakage nhưng thực tế lại dùng selector khác. + +--- + +## 9. Mode 1: `mode_1_decay` + +Mode 1 là WFO decay control. Với mỗi fold, QuantBT tính Sharpe trên IS và OOS, +rồi đo decay: + +$$ +d_k(\theta)=S_{\mathrm{IS},k}(\theta)-S_{\mathrm{OOS},k}(\theta) +$$ + +Mean decay và decay volatility: + +$$ +\overline{d}(\theta)=\operatorname{mean}_{k}\left(d_k(\theta)\right) +$$ + +$$ +\sigma_d(\theta)=\operatorname{std}_{k}\left(d_k(\theta)\right) +$$ + +Objective: + +$$ +J_1(\theta)=\overline{S}_{\mathrm{OOS}}(\theta)-\lambda\sigma_d(\theta)-\gamma\max\left(0,\overline{d}(\theta)\right) +$$ + +Tham số chính: `decay_lambda`, `decay_gamma`, `candidate_decay_lambda`, +`candidate_decay_gamma`. + +Dụng ý của mode 1 là không thưởng params chỉ thắng IS. Một params tốt phải giữ +được hiệu năng khi sang OOS, và decay không được quá bất ổn giữa các fold. Mode +này phù hợp khi user chấp nhận dùng OOS như validation stage. + +### 9.1. Dụng Ý Chuẩn Quỹ Của Mode 1 + +Mode 1 phản ánh bài toán rất thực tế trong quỹ: một alpha có thể đẹp trên train +nhưng mất edge ngay khi sang giai đoạn kế tiếp. Vì vậy objective không chỉ hỏi +OOS Sharpe cao hay không, mà còn hỏi: + +- IS có đang quá đẹp so với OOS không? +- decay có ổn định giữa nhiều fold không? +- một fold thắng lớn có đang che giấu nhiều fold yếu không? + +Trong research nghiêm túc, `S_IS - S_OOS` là tín hiệu quan trọng. Nếu decay luôn +dương và lớn, strategy có thể đang overfit. Nếu decay lúc rất âm, lúc rất dương, +params có thể quá nhạy regime. + +### 9.2. Selection Choices Trong Mode 1 + +Mode 1 thường dùng `candidate_selection_metric="robust_decay"`. Đây là lựa chọn +khớp nhất với objective vì nó chọn candidate theo OOS performance sau khi phạt +decay. + +Các lựa chọn phụ: + +- `mean_oos_sharpe`: aggressive hơn, ưu tiên OOS cao. +- `mean_is_sharpe`: chủ yếu dùng diagnostic, dễ overfit nếu dùng làm selector. +- `is_plateau_robust`: dùng khi muốn chọn vùng IS/search ổn định trước khi nhìn + OOS candidate. + +Nếu mục tiêu là báo cáo validation, `robust_decay` hợp lý hơn `mean_oos_sharpe` +vì nó không bị một vài OOS fold đẹp làm lệch quyết định. + +### 9.3. Tham Số Cần Tune Cẩn Thận + +`decay_lambda` càng cao thì càng phạt params có decay biến động. `decay_gamma` +càng cao thì càng phạt params có IS đẹp hơn OOS quá nhiều. + +Nếu alpha có bản chất regime-following và decay tự nhiên thay đổi mạnh, đặt hai +tham số này quá cao có thể làm selector quá conservative. Nếu alpha có nhiều +params và search space rộng, nên tăng penalty để giảm data snooping. + +Một cấu hình cân bằng thường bắt đầu từ: + +```text +decay_lambda = 0.5 +decay_gamma = 0.5 +``` + +Sau đó điều chỉnh theo số fold, độ dài OOS và mức noisy của strategy. + +--- + +## 10. Mode 2: `mode_2_sbb` + +Mode 2 là Synthetic Block Bootstrap robustness. Nó không dùng OOS thật trong +Optuna objective. Strategy chạy trên IS, rồi return proxy của IS được mô phỏng +thành nhiều synthetic paths. Với mỗi synthetic path, QuantBT tính synthetic +Sharpe. + +Mean synthetic Sharpe: + +$$ +\mu_{\mathrm{boot}}(\theta)=\operatorname{mean}\left(S_{\mathrm{boot}}(\theta)\right) +$$ + +Synthetic dispersion: + +$$ +\sigma_{\mathrm{boot}}(\theta)=\operatorname{std}\left(S_{\mathrm{boot}}(\theta)\right) +$$ + +Synthetic decay: + +$$ +d_{\mathrm{boot}}(\theta)=S_{\mathrm{IS}}(\theta)-\mu_{\mathrm{boot}}(\theta) +$$ + +Objective: + +$$ +J_2(\theta)=\mu_{\mathrm{boot}}(\theta)-\lambda_{\mathrm{sbb}}\max\left(0,d_{\mathrm{boot}}(\theta)\right)-\eta_{\mathrm{sbb}}\sigma_{\mathrm{boot}}(\theta) +$$ + +Tham số chính: `sbb_samples`, `sbb_block_length`, `sbb_decay_lambda`, +`sbb_std_penalty`, `sbb_simulation`. + +`stationary` là block bootstrap mặc định. `regime` sample theo volatility regime. +`stress` scale volatility bằng `stress_vol_multiplier`. `garch` mô phỏng +volatility clustering nếu có dependency `arch`. + +Dụng ý chuẩn quỹ là kiểm tra path dependency, giữ autocorrelation cục bộ, và +stress alpha mà không mở OOS thật cho optimizer. + +### 10.1. Dụng Ý Chuẩn Quỹ Của Mode 2 + +Mode 2 phục vụ câu hỏi khác mode 1: nếu ta giữ nguyên alpha nhưng thị trường đi +theo một đường giá hơi khác, params còn sống không? + +Backtest lịch sử chỉ là một path đã xảy ra. Trong thực tế, cùng distribution có +thể sinh ra nhiều path khác nhau. Stationary Block Bootstrap cố giữ cụm return +liền kề để không phá autocorrelation ngắn hạn, đồng thời tạo nhiều biến thể +khác của train path. + +Điều này đặc biệt quan trọng với: + +- scalping; +- mean reversion; +- grid/DCA; +- basis hoặc spread strategy; +- alpha có stop-loss/take-profit nhạy path. + +### 10.2. Simulation Choices + +`stationary` là lựa chọn mặc định. Nó nhanh, ổn định, và ít giả định. + +`regime` phù hợp khi user muốn stress theo volatility bucket. Ví dụ một alpha +chỉ sống trong low-vol regime thì cần xem high-vol synthetic path có làm Sharpe +sụp không. + +`stress` phù hợp để kiểm tra nhanh khi volatility tăng 1.5x hoặc 2x. + +`garch` phù hợp khi cần mô phỏng volatility clustering có cấu trúc hơn, nhưng +chậm hơn và phụ thuộc data đủ dài. Không nên dùng GARCH chỉ vì nghe “xịn”; nếu +train sample quá ngắn, fitted volatility process có thể không đáng tin. + +### 10.3. Tham Số Ảnh Hưởng Objective + +`sbb_samples` càng cao thì distribution Sharpe synthetic càng ổn định nhưng +runtime tăng. + +`sbb_block_length` quá ngắn sẽ phá cấu trúc path; quá dài sẽ tạo quá ít biến thể. + +`sbb_std_penalty` cao làm selector tránh params có Sharpe synthetic phân tán +mạnh. + +`sbb_decay_lambda` cao làm selector tránh params có IS Sharpe cao nhưng synthetic +Sharpe giảm mạnh. + +Mode 2 không nên được đọc như OOS proof. Nó là train-only robustness stress. + +--- + +## 11. Mode 3: `mode_3_flat_minima` + +Mode 3 chọn vùng tham số phẳng thay vì sharp peak. Sau Optuna, QuantBT lấy top +trials theo `flat_top_fraction`, normalize param space về `[0, 1]`, rồi cluster +bằng logic DBSCAN-style với `flat_eps` và `flat_min_samples`. + +Medoid của cluster: + +$$ +\theta_{\mathrm{medoid}}=\operatorname*{arg\,min}_{\theta_i\in C}\left\lVert x_i-\overline{x}_C\right\rVert_2 +$$ + +Centroid được snap về grid: + +$$ +\theta_{\mathrm{centroid}}=\operatorname{snap}\left(\overline{x}_C\right) +$$ + +Tham số chính: + +- `flat_top_fraction`: phần top trial đưa vào clustering. +- `flat_eps`: bán kính density. +- `flat_min_samples`: số điểm tối thiểu để thành cụm. +- `flat_selector`: `medoid` hoặc `centroid`. + +`medoid` là trial thật đã chạy. `centroid` là trung tâm cụm, được snap về range +đã khai báo và có thể cần evaluate lại. Nếu không có cluster, QuantBT fallback +về best trial; đây là tín hiệu param surface có thể quá sắc hoặc rời rạc. + +### 11.1. Dụng Ý Chuẩn Quỹ Của Mode 3 + +Mode 3 dựa trên một quan sát rất quan trọng: một tham số tốt thật sự thường có +hàng xóm cũng tương đối tốt. Nếu chỉ một điểm duy nhất trong param space thắng +lớn, còn xung quanh thua mạnh, đó thường là dấu hiệu overfit. + +Trong ngôn ngữ optimization, ta muốn vùng phẳng: + +$$ +\left\lvert J(\theta)-J(\theta+\epsilon)\right\rvert +\text{ nhỏ với } \epsilon \text{ nhỏ} +$$ + +QuantBT không cần tính gradient phức tạp. Nó dùng top trials, normalize param +space, rồi tìm cụm dày. Đây là cách thực dụng, minh bạch và dễ audit. + +### 11.2. Medoid Và Centroid + +`flat_selector="medoid"` chọn trial thật nằm gần trung tâm cụm. Đây là lựa chọn +an toàn nhất vì params đó đã được evaluate. + +`flat_selector="centroid"` lấy trung tâm cụm rồi snap về grid. Lựa chọn này có +thể tốt hơn nếu cluster bị lệch bởi noise, nhưng cần evaluate lại vì centroid +có thể là params chưa từng chạy. + +Với production research, `medoid` thường là default tốt hơn. `centroid` phù hợp +khi param grid dày và chi phí evaluate lại thấp. + +### 11.3. Tham Số DBSCAN-Style + +`flat_eps` là bán kính để xem các params có gần nhau không. Nếu quá nhỏ, mọi +điểm thành noise. Nếu quá lớn, nhiều vùng khác nhau bị gộp sai. + +`flat_min_samples` là số điểm tối thiểu để cụm có ý nghĩa. Với ít trial, giá trị +này nên nhỏ; với nhiều trial, có thể tăng để tránh cluster giả. + +`flat_top_fraction` quyết định top region. Nếu quá thấp, cụm thiếu điểm; nếu quá +cao, đưa cả params trung bình vào cluster. + +Mode 3 không thay thế OOS validation. Nó kiểm tra hình học của param surface. + +--- + +## 12. Plateau Robustness + +Plateau score được dùng trong mode 3, mode 4 và mode 5. Với cluster `C`: + +$$ +P_C=Q_C+w_mM_C-w_sV_C+w_n\log\left(1+\lvert C\rvert\right) +$$ + +Trong đó: + +$$ +Q_C=\operatorname{quantile}\left(J(\theta_i),q\right),\quad M_C=\operatorname{median}\left(J(\theta_i)\right),\quad V_C=\operatorname{std}\left(J(\theta_i)\right) +$$ + +Tham số: + +- `plateau_quantile`: lower-tail quantile. +- `plateau_median_weight`: trọng số median. +- `plateau_std_penalty`: phạt dispersion. +- `plateau_size_bonus`: thưởng cluster size. + +Ý nghĩa: lower-tail tốt cho thấy cụm không mong manh; median tốt cho thấy cụm +không chỉ có một điểm thắng; std thấp cho thấy cụm ổn định; cluster lớn cho thấy +vùng tham số có độ dày. + +Plateau score là phần QuantBT dùng để biến trực giác “vùng phẳng tốt hơn điểm +đơn lẻ” thành công thức. `plateau_quantile` nhìn lower-tail của cluster. Nếu +lower-tail tốt, nghĩa là ngay cả những member yếu trong cụm vẫn không quá tệ. + +`plateau_median_weight` thưởng median của cụm, tránh cụm có một vài outlier đẹp. +`plateau_std_penalty` phạt cụm phân tán. `plateau_size_bonus` thưởng nhẹ cho cụm +dày bằng log-size để không làm size áp đảo chất lượng. + +Vì dùng log-size, phần thưởng cụm lớn tăng chậm: + +$$ +\log(1+\lvert C\rvert) +$$ + +Điều này có ý nghĩa domain: cụm lớn là tốt, nhưng cụm lớn mà score thấp không +nên thắng cụm nhỏ hơn nhưng robust hơn. + +--- + +## 13. Mode 4: `mode_4_is_only_robust` + +Mode 4 là strict anti-leakage selection. OOS không được dùng để chọn params. +Optuna objective chỉ dùng IS. Sau đó QuantBT chia IS thành `is_subperiods` +shards và đo sự ổn định temporal. + +Sharpe từng shard: + +$$ +S_i(\theta)=\mathrm{Sharpe}\left(r_{\theta,i}\right) +$$ + +Median, lower-tail và MAD: + +$$ +M_T(\theta)=\operatorname{median}\left(S_i(\theta)\right) +$$ + +$$ +Q_T(\theta)=Q_{25}\left(S_i(\theta)\right) +$$ + +$$ +D_T(\theta)=\operatorname{median}\left(\left\lvert S_i(\theta)-M_T(\theta)\right\rvert\right) +$$ + +Temporal score: + +$$ +T(\theta)=M_T(\theta)+w_qQ_T(\theta)-w_dD_T(\theta) +$$ + +Final IS-only robust score: + +$$ +J_4(\theta)=w_TT(\theta)+w_PP(\theta)-B(\theta)-C(\theta) +$$ + +Tham số chính: + +- `is_subperiods`: số shard trong IS. +- `q25_weight`: trọng số lower-tail Sharpe. +- `dispersion_penalty`: phạt MAD. +- `temporal_weight`: trọng số temporal score. +- `plateau_weight`: trọng số plateau score. +- `use_bootstrap_penalty`, `use_complexity_penalty`: optional extension. + +Hiện tại bootstrap/complexity penalty mặc định không phạt thêm. Mode 4 trả +params để giao dịch OOS fold kế tiếp. OOS chỉ được dùng sau selection để report +và stitch validation, nên đây là mode phù hợp nhất khi cần anti-leakage nghiêm. + +### 13.1. Dụng Ý Chuẩn Quỹ Của Mode 4 + +Mode 4 được thiết kế cho trường hợp user muốn selection tuyệt đối không nhìn +OOS. Đây là khác biệt quan trọng so với mode 1. Trong mode 4, OOS đóng vai trò +report/validation sau khi params đã bị freeze. + +Nó trả lời câu hỏi: + +```text +Chỉ nhìn train, có thể chọn params nào robust nhất để đi vào giai đoạn kế tiếp? +``` + +Đây là tư duy gần hơn với cách vận hành thật: tại thời điểm live, ta không biết +OOS tương lai. Ta chỉ có thể chọn params dựa trên train history và robustness +test bên trong train. + +### 13.2. Temporal Robustness + +Mode 4 chia IS thành nhiều shard để tránh params chỉ thắng một giai đoạn. Nếu +Sharpe chỉ cao ở một shard nhưng thấp ở các shard khác, median và Q25 sẽ yếu, +MAD có thể cao. + +Temporal score dùng: + +- median để đo hiệu năng trung tâm; +- Q25 để nhìn downside của subperiods; +- MAD để phạt dispersion robust hơn standard deviation. + +MAD được dùng vì nó ít nhạy outlier hơn: + +$$ +\operatorname{MAD}(S)=\operatorname{median}\left(\left\lvert S_i-\operatorname{median}(S)\right\rvert\right) +$$ + +### 13.3. Selection Choices Trong Mode 4 + +Mode 4 bắt buộc `candidate_selection_metric="is_only_robust"`. + +Selector này kết hợp: + +- top IS candidates; +- temporal robustness; +- plateau robustness; +- medoid hoặc centroid. + +Nếu không có cluster đủ tốt, selector fallback về best temporal record. Fallback +này không phải lỗi, nhưng là tín hiệu rằng param surface chưa đủ dày để kết luận +plateau mạnh. + +### 13.4. Khi Nào Dùng Mode 4 + +Dùng mode 4 khi: + +- muốn anti-leakage nghiêm; +- strategy có nhiều tham số; +- OOS data ít và không muốn optimizer học OOS; +- mục tiêu là chọn params cho kỳ tiếp theo. + +Không nên kỳ vọng mode 4 luôn chọn params có IS cao nhất. Nó cố tình đánh đổi +một phần IS score để lấy stability. + +--- + +## 14. Mode 5: `mode_5_full_robust` + +Mode 5 là full-sample robust calibration, không phải WFO validation. QuantBT tạo +một fold duy nhất: + +$$ +\mathrm{train\_index}=\mathrm{test\_index}=\mathrm{full\_index} +$$ + +Toàn bộ dữ liệu được xem là calibration sample. Metadata ghi rõ: + +```text +validation_claim = "none_full_sample_calibration" +full_sample_used_for_selection = True +``` + +Selector: + +- `full_robust`: temporal plus plateau, mặc định. +- `full_plateau_robust`: ưu tiên plateau. +- `full_temporal_robust`: ưu tiên subperiod stability. +- `full_best`: objective cao nhất, rủi ro overfit cao nhất. + +Dụng ý của mode 5 là chọn một bộ params production sau khi alpha đã qua +validation độc lập. Nó ép params sống qua toàn bộ lịch sử đã biết và nhiều +regime, nhưng không được dùng như bằng chứng OOS. + +### 14.1. Dụng Ý Chuẩn Quỹ Của Mode 5 + +Mode 5 dành cho production calibration sau khi alpha đã qua validation bằng cách +khác. Nó không trả lời “alpha có generalize không?”. Nó trả lời: + +```text +Sau khi đã tin alpha, nếu phải chọn một params để deploy, +params nào robust nhất trên toàn bộ lịch sử đã biết? +``` + +Nhiều workflow quỹ có bước tương tự: validate model bằng holdout/WFO trước, sau +đó refit hoặc recalibrate trên toàn bộ data trước deployment. Điểm quan trọng là +không được trộn lẫn calibration result với validation proof. + +### 14.2. Selector Con Trong Mode 5 + +`full_robust` là default vì nó kết hợp temporal stability và plateau stability. + +`full_plateau_robust` phù hợp khi user tin rằng độ phẳng param surface quan +trọng hơn subperiod Sharpe. + +`full_temporal_robust` phù hợp khi regime stability là ưu tiên số một. + +`full_best` chỉ nên dùng để benchmark upper bound, không nên dùng làm production +selector nếu search space lớn. + +### 14.3. Cách Đọc Metadata Mode 5 + +Mode 5 luôn phải được đọc cùng: + +```text +validation_claim = "none_full_sample_calibration" +``` + +và: + +```text +full_sample_used_for_selection = True +``` + +Nếu report dùng mode 5, cần nói rõ đây là calibration. Không nên trình bày nó +như OOS performance. + +--- + +## 15. Operational Notes + +`target_mode` quyết định OOS output được gửi vào engine nào. Các target hiện hỗ +trợ gồm `signal_notional`, `notional`, `unit`, `pct_equity`, `dca_ladder`, +`portfolio`, `basket`, và `arbitrage`. Single-symbol thường trả `pd.Series`; +portfolio trả `pd.DataFrame` hoặc `dict[str, pd.Series]`. + +Nếu truyền `params`, WFO dùng params cố định. Nếu truyền `param_ranges`, WFO chạy +Optuna. Range hợp lệ gồm `(low, high)`, `(low, high, step)`, categorical list, +hoặc fixed value khác `None`. Fixed flags nên merge trong strategy, ví dụ: + +```python +run_params = {"issl": True, **params} +``` + +Metadata lưu `optimization_mode`, `candidate_selection_metric`, `n_folds`, +`n_trials`, `n_candidates`, `data_hash`, `config_hash`, `scoring_backend`, +`scoring_trading_days`, `validation_claim`, trial table và fold metrics. Đây là +audit trail để biết params cuối cùng đến từ đâu và có dùng OOS để chọn hay không. + +Nếu mode 1 tốt nhưng mode 4 xấu, params có thể đang phụ thuộc OOS selection. Nếu +mode 2 xấu, alpha có thể yếu khi return path bị perturb. Nếu mode 3 fallback +best, param surface có thể quá sắc. Nếu mode 5 đẹp, vẫn không được xem là OOS +proof; nó chỉ là câu trả lời cho production calibration trên toàn bộ lịch sử đã +biết. + +Workflow khuyến nghị: research signal, chống look-ahead trong strategy, chạy +`mode_1_decay` hoặc `mode_4_is_only_robust`, stress bằng `mode_2_sbb`, kiểm tra +surface bằng `mode_3_flat_minima`, validate bằng endpoint thật, rồi dùng +`mode_5_full_robust` để chọn params production trước forward test/paper/live. + +Không có mode nào đúng cho mọi alpha. Lựa chọn mode phụ thuộc vào số tham số, độ +dài data, trade frequency, regime sensitivity và mức cần anti-leakage. diff --git a/tests/test_phase12_arbitrage_certification.py b/tests/test_phase12_arbitrage_certification.py index daff36d..6e1ae19 100644 --- a/tests/test_phase12_arbitrage_certification.py +++ b/tests/test_phase12_arbitrage_certification.py @@ -10,6 +10,8 @@ def test_phase12_arbitrage_certification_smoke(): assert report["basis"]["audit"]["passed"] is True assert report["basis"]["parity"]["passed"] is True assert report["stat_pair"]["accounting_parity_passed"] is True + assert report["stat_pair"]["audit"]["passed"] is True + assert report["stat_pair"]["max_package_residual"] < 1e-9 assert report["index_basket"]["passed"] is True assert report["schema_only"]["passed"] is True assert report["nautilus"]["status"] == "skipped" diff --git a/tests/test_phase12_benchmark_nautilus_cert.py b/tests/test_phase12_benchmark_nautilus_cert.py index 145c6db..85c3312 100644 --- a/tests/test_phase12_benchmark_nautilus_cert.py +++ b/tests/test_phase12_benchmark_nautilus_cert.py @@ -78,6 +78,23 @@ def test_phase12_native_portfolio_prepared_reuse_matches_normal_run(): rtol=0.0, atol=1e-12, ) + for key in ( + "target_units_report", + "accepted_units_report", + "target_notional_report", + "accepted_notional_report", + "exposure_report", + ): + np.testing.assert_allclose( + reused.metadata[key].to_numpy(dtype=float), + normal.metadata[key].to_numpy(dtype=float), + rtol=0.0, + atol=1e-10, + ) + np.testing.assert_allclose(reused.funding.to_numpy(), normal.funding.to_numpy(), rtol=0.0, atol=1e-12) + assert reused.metadata["rebalance_report"].reset_index(drop=True).equals( + normal.metadata["rebalance_report"].reset_index(drop=True) + ) def test_phase12_native_portfolio_prepared_reuse_rejects_stale_signature(): diff --git a/tests/test_phase13_arbitrage_certification_cleanup.py b/tests/test_phase13_arbitrage_certification_cleanup.py new file mode 100644 index 0000000..a88f03f --- /dev/null +++ b/tests/test_phase13_arbitrage_certification_cleanup.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from quantbt import ( + ArbitrageLeg, + CrossExchangeArbSpec, + HedgePolicy, + HedgePolicyKind, + NativeEventBackend, + NativeEventConfig, + NativeVectorizedBackend, + NativeVectorizedConfig, + SizingPolicy, + SizingPolicyKind, + StatArbPairSpec, + build_arbitrage_domain_audit, + compare_native_arbitrage_results, +) +from quantbt.core.schema import AccountConfig + + +def _idx(): + return pd.date_range("2024-01-01", periods=6, freq="8h", tz="UTC") + + +def _event(): + return NativeEventBackend( + NativeEventConfig( + account=AccountConfig(initial_capital=50_000.0, leverage=10.0), + fee_rate=0.0002, + use_funding=True, + ) + ) + + +def _vectorized(): + return NativeVectorizedBackend( + NativeVectorizedConfig( + account=AccountConfig(initial_capital=50_000.0, leverage=10.0), + fee_rate=0.0002, + use_funding=True, + ) + ) + + +def _stat_spec(): + return StatArbPairSpec( + arb_id="PHASE13C_STAT_ARB", + legs=( + ArbitrageLeg("ETH-PERP", 1.0, asset_class="crypto", funding_enabled=True), + ArbitrageLeg("SOL-PERP", -1.0, asset_class="crypto"), + ), + hedge_policy=HedgePolicy( + HedgePolicyKind.BETA_NEUTRAL, + freeze_on_entry=False, + rebalance_threshold=0.15, + ), + sizing_policy=SizingPolicy(SizingPolicyKind.TARGET_GROSS_NOTIONAL, notional=10_000.0), + ) + + +def test_phase13c_stat_arb_package_pnl_report_reconciles_event_and_vectorized(): + idx = _idx() + closes = { + "ETH-PERP": pd.Series([100.0, 100.0, 102.0, 104.0, 103.0, 101.0], index=idx), + "SOL-PERP": pd.Series([50.0, 50.0, 49.0, 48.0, 50.0, 51.0], index=idx), + } + signal = pd.Series([0.0, 1.0, 1.0, 1.0, 0.0, 0.0], index=idx) + hedge_ratios = { + "ETH-PERP": pd.Series([1.0, 1.0, 1.0, 1.0, 1.0, 1.0], index=idx), + "SOL-PERP": pd.Series([-1.0, -1.0, -1.35, -1.35, -0.8, -0.8], index=idx), + } + funding = { + "ETH-PERP": pd.Series([0.0, 0.0, 0.001, 0.0, -0.0005, 0.0], index=idx), + "SOL-PERP": pd.Series([0.01, 0.01, 0.01, 0.01, 0.01, 0.01], index=idx), + } + spec = _stat_spec() + + event = _event().run_stat_arb_pair_arbitrage( + idx, + spec, + signal, + closes, + hedge_ratios=hedge_ratios, + funding_rate=funding, + ) + vectorized = _vectorized().run_stat_arb_pair_arbitrage( + idx, + spec, + signal, + closes, + hedge_ratios=hedge_ratios, + funding_rate=funding, + ) + + for result in (event, vectorized): + audit = build_arbitrage_domain_audit(result, raise_on_fail=True) + package = result.metadata["package_pnl_report"] + leg_report = result.metadata["leg_pnl_report"] + + assert audit["status"] == "pass" + assert set( + [ + "price_pnl", + "fill_pnl", + "fees", + "funding_pnl", + "leg_pnl", + "hedge_pnl", + "spread_pnl", + "package_pnl", + "equity_delta", + "pnl_residual", + ] + ).issubset(package.columns) + np.testing.assert_allclose( + package["package_pnl"].to_numpy(dtype=float), + ( + package["price_pnl"] + + package["fill_pnl"] + - package["fees"] + + package["funding_pnl"] + ).to_numpy(dtype=float), + rtol=0.0, + atol=1e-10, + ) + np.testing.assert_allclose(package["pnl_residual"].to_numpy(dtype=float), 0.0, rtol=0.0, atol=1e-10) + np.testing.assert_allclose( + package["spread_pnl"].to_numpy(dtype=float), + (package["leg_pnl"] + package["hedge_pnl"]).to_numpy(dtype=float), + rtol=0.0, + atol=1e-12, + ) + assert leg_report.groupby("symbol")["funding_pnl"].sum().loc["SOL-PERP"] == pytest.approx(0.0) + assert abs(float(leg_report.groupby("symbol")["funding_pnl"].sum().loc["ETH-PERP"])) > 0.0 + + parity = compare_native_arbitrage_results(event, vectorized, raise_on_fail=True) + assert parity["status"] == "pass" + pd.testing.assert_frame_equal( + event.metadata["package_pnl_report"], + vectorized.metadata["package_pnl_report"], + check_dtype=False, + check_exact=False, + rtol=0.0, + atol=1e-10, + ) + + +def test_phase13c_schema_only_arbitrage_specs_reject_with_actionable_message(): + idx = _idx() + closes = { + "BTC-BINANCE": pd.Series(100.0, index=idx), + "BTC-OKX": pd.Series(101.0, index=idx), + } + spec = CrossExchangeArbSpec( + arb_id="SCHEMA_ONLY_CROSS_EXCHANGE", + legs=( + ArbitrageLeg("BTC-BINANCE", 1.0, venue="BINANCE"), + ArbitrageLeg("BTC-OKX", -1.0, venue="OKX"), + ), + hedge_policy=HedgePolicy(HedgePolicyKind.BASE_QTY_EQUAL), + sizing_policy=SizingPolicy( + SizingPolicyKind.TARGET_NOTIONAL_TO_BASE_QTY, + notional=10_000.0, + reference_symbol="BTC-BINANCE", + ), + ) + + with pytest.raises(NotImplementedError, match="schema-validated.*specialized arbitrage engine"): + _event().run_package_arbitrage(idx, spec, pd.Series(1.0, index=idx), closes) + with pytest.raises(NotImplementedError, match="arbitrage_support_matrix"): + _vectorized().run_package_arbitrage(idx, spec, pd.Series(1.0, index=idx), closes) diff --git a/tests/test_phase13_portfolio_report_parity.py b/tests/test_phase13_portfolio_report_parity.py new file mode 100644 index 0000000..bf70662 --- /dev/null +++ b/tests/test_phase13_portfolio_report_parity.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd + +from quantbt import AccountConfig +from quantbt.backends import NativePortfolioBackend, NativePortfolioConfig + + +def test_phase13_native_portfolio_report_optimization_matches_legacy_report_formulas(): + idx = pd.date_range("2024-01-01", periods=12, freq="1h", tz="UTC") + symbols = ["BTC", "ETH", "SOL"] + closes = { + "BTC": pd.Series([100, 101, 103, 102, 104, 105, 103, 102, 101, 100, 102, 103], index=idx, dtype=float), + "ETH": pd.Series([50, 49, 48, 50, 51, 52, 53, 52, 51, 50, 49, 50], index=idx, dtype=float), + "SOL": pd.Series([20, 21, 20, 19, 20, 21, 22, 23, 22, 21, 20, 19], index=idx, dtype=float), + } + positions = { + "BTC": pd.Series([0, 1, 1, 0, -1, -1, 0, 1, 1, 0, 0, 0], index=idx, dtype=float), + "ETH": pd.Series([0, -1, -1, 0, 1, 1, 0, -1, -1, 0, 0, 0], index=idx, dtype=float), + "SOL": pd.Series([0, 1, 0, 0, 1, 0, 0, -1, 0, 0, 1, 0], index=idx, dtype=float), + } + funding_rate = { + "BTC": pd.Series([0.0, 0.0, 0.0001, 0.0, 0.0, 0.0001, 0.0, 0.0, 0.0001, 0.0, 0.0, 0.0001], index=idx), + "ETH": pd.Series([0.0, 0.0, -0.00005, 0.0, 0.0, -0.00005, 0.0, 0.0, -0.00005, 0.0, 0.0, -0.00005], index=idx), + "SOL": pd.Series(0.0, index=idx), + } + backend = NativePortfolioBackend( + NativePortfolioConfig( + account=AccountConfig(initial_capital=100_000.0, leverage=4.0, maintenance_ratio=0.005), + fee_rate=0.0002, + use_funding=True, + ) + ) + result = backend.run_signals( + positions=positions, + closes=closes, + highs=closes, + lows=closes, + datetime_index=idx, + symbols=symbols, + mode="market_neutral", + hedge_type="notional", + alloc_per_trade={"BTC": 1_000.0, "ETH": 1_500.0, "SOL": 500.0}, + funding_rate=funding_rate, + contract_size={"BTC": 1.0, "ETH": 1.0, "SOL": 1.0}, + use_pyramiding=True, + ) + + old = _legacy_report_formula_snapshot(result, symbols) + + np.testing.assert_allclose(result.returns.to_numpy(), old["returns"].to_numpy(), rtol=0.0, atol=1e-12) + np.testing.assert_allclose(result.funding.to_numpy(), old["funding"].to_numpy(), rtol=0.0, atol=1e-12) + np.testing.assert_array_equal( + result.diagnostics["rejected_rebalances"].to_numpy(dtype=bool), + old["rejected_rebalances"].to_numpy(dtype=bool), + ) + np.testing.assert_allclose( + result.metadata["exposure_report"].to_numpy(dtype=float), + old["exposure_report"].to_numpy(dtype=float), + rtol=0.0, + atol=1e-10, + ) + pd.testing.assert_frame_equal( + result.metadata["rebalance_report"].reset_index(drop=True), + old["rebalance_report"].reset_index(drop=True), + check_dtype=False, + ) + + +def _legacy_report_formula_snapshot(result, symbols): + idx = result.equity.index + target_units = result.metadata["target_units_report"] + accepted_units = result.metadata["accepted_units_report"] + closes = result.metadata["accepted_notional_report"] / accepted_units.replace(0.0, np.nan) + closes = closes.fillna(result.metadata["target_notional_report"] / target_units.replace(0.0, np.nan)).fillna(0.0) + contract_sizes = pd.Series(result.metadata["contract_size"], dtype=float).reindex(symbols) + target_notional = result.metadata["target_notional_report"] + accepted_notional = result.metadata["accepted_notional_report"] + leverages = pd.Series(result.metadata["initial_buying_power"] / result.initial_capital, index=symbols, dtype=float) + betas = pd.Series(result.metadata["beta"], dtype=float).reindex(symbols) + + returns = result.equity.pct_change().fillna(0.0) + funding = result.metadata["symbol_pnl_report"].groupby("timestamp", sort=False)["funding_cost"].sum().reindex(idx, fill_value=0.0) + rejected_rebalances = (target_units - accepted_units).abs().sum(axis=1) > 1e-10 + + abs_accepted = accepted_notional.abs() + gross = abs_accepted.sum(axis=1) + net = accepted_notional.sum(axis=1) + initial_margin = abs_accepted.div(leverages, axis=1).sum(axis=1) + maintenance_margin = gross * float(result.margin["maintenance_margin"].div(gross.replace(0.0, np.nan)).replace([np.inf, -np.inf], np.nan).dropna().iloc[0]) + beta_exposure = accepted_notional.mul(betas, axis=1).sum(axis=1) + target_gross = target_notional.abs().sum(axis=1) + target_beta_exposure = target_notional.mul(betas, axis=1).sum(axis=1) + exposure_report = pd.DataFrame( + { + "long_notional": accepted_notional.where(accepted_notional > 0.0, 0.0).sum(axis=1), + "short_notional": (-accepted_notional.where(accepted_notional < 0.0, 0.0)).sum(axis=1), + "gross_notional": gross, + "net_notional": net, + "beta_exposure_notional": beta_exposure, + "target_gross_notional": target_gross, + "target_beta_exposure_notional": target_beta_exposure, + "initial_margin": initial_margin, + "maintenance_margin": maintenance_margin, + "equity": result.equity, + "available_equity_after_im": result.equity - initial_margin, + "buying_power": result.equity * float(leverages.mean()), + }, + index=idx, + ) + exposure_report["gross_leverage"] = exposure_report["gross_notional"] / exposure_report["equity"].replace(0.0, np.nan) + exposure_report["net_exposure_pct"] = exposure_report["net_notional"] / exposure_report["equity"].replace(0.0, np.nan) + exposure_report = exposure_report.fillna(0.0) + + diff = target_units - accepted_units + mask = diff.abs() > 1e-10 + if not mask.to_numpy().any(): + rebalance_report = pd.DataFrame( + columns=["timestamp", "symbol", "target_units", "accepted_units", "unit_diff", "notional_diff", "reason"] + ) + else: + notional_diff = diff.mul(closes, axis=0).mul(contract_sizes, axis=1) + stacked = diff.where(mask).stack(future_stack=True).dropna() + index = stacked.index + rebalance_report = pd.DataFrame( + { + "timestamp": index.get_level_values(0), + "symbol": index.get_level_values(1), + "target_units": target_units.stack(future_stack=True).reindex(index).to_numpy(dtype=float), + "accepted_units": accepted_units.stack(future_stack=True).reindex(index).to_numpy(dtype=float), + "unit_diff": stacked.to_numpy(dtype=float), + "notional_diff": notional_diff.stack(future_stack=True).reindex(index).to_numpy(dtype=float), + "reason": "margin_or_portfolio_gate", + } + ) + + return { + "returns": returns, + "funding": funding, + "rejected_rebalances": rejected_rebalances, + "exposure_report": exposure_report, + "rebalance_report": rebalance_report, + } diff --git a/tests/test_walkforward_phase1.py b/tests/test_walkforward_phase1.py index edd7a72..cefcc73 100644 --- a/tests/test_walkforward_phase1.py +++ b/tests/test_walkforward_phase1.py @@ -1197,6 +1197,60 @@ def strategy(data, params, train_index, test_index, fold): assert wf["best_trial"]["fold_metrics"][0]["is_trade_count"] == pytest.approx(native_report["num_trades"]) +def test_walkforward_portfolio_endpoint_scoring_reuses_prepared_market_arrays_without_metric_drift(): + idx = pd.date_range("2021-01-01", "2022-03-31", freq="1D", tz="UTC") + data = { + "BTC": _bars(idx, 100.0), + "ETH": _bars(idx, 50.0), + } + data["BTC"]["close"] = 100.0 + np.sin(np.linspace(0.0, 8.0, len(idx))) * 2.0 + data["ETH"]["close"] = 50.0 + np.cos(np.linspace(0.0, 8.0, len(idx))) * 1.5 + + def strategy(data, params, train_index, test_index, fold): + scale = float(params["scale"]) + return pd.DataFrame({"BTC": scale, "ETH": -scale}, index=test_index) + + def run(use_cache: bool): + endpoint = QuantBTEndpoint.train_test_split( + strategy_class=strategy, + test_start="2022-01-01", + target_mode="portfolio", + portfolio_mode="longshort", + optimization_mode="mode_1_decay", + optimization_config={ + "scoring_backend": "endpoint", + "use_prepared_scoring_cache": use_cache, + }, + optuna_trials=8, + random_seed=123, + initial_capital=100_000.0, + leverage=5.0, + alloc_per_trade=1_000.0, + fee=0.0, + use_funding=False, + ) + return endpoint.backtest(data=data, param_ranges={"scale": (0.5, 1.5, 0.1)}) + + cached = run(True) + uncached = run(False) + cached_wf = cached.metadata["walk_forward"] + uncached_wf = uncached.metadata["walk_forward"] + cache_meta = cached_wf["prepared_scoring_cache"] + uncached_meta = uncached_wf["prepared_scoring_cache"] + + assert cached.equity.iloc[-1] == pytest.approx(uncached.equity.iloc[-1]) + assert cached_wf["params"] == uncached_wf["params"] + assert cached_wf["best_trial"]["objective"] == pytest.approx(uncached_wf["best_trial"]["objective"]) + assert cache_meta["available"] is True + assert cache_meta["enabled"] is True + assert cache_meta["prepared_runs"] > 0 + assert cache_meta["market_cache_hits"] > 0 + assert cache_meta["market_cache_misses"] > 0 + assert uncached_meta["enabled"] is False + assert uncached_meta["prepared_runs"] == 0 + assert uncached_meta["fallback_runs"] > 0 + + def test_walkforward_phase4_rejects_non_timestamped_strategy_output(): idx = _idx() diff --git a/upgrade/implement.md b/upgrade/implement.md index df3836c..cc65751 100644 --- a/upgrade/implement.md +++ b/upgrade/implement.md @@ -2112,6 +2112,245 @@ Acceptance: --- +## Phase 13 - WFO Cache, Portfolio Report, And Arbitrage Certification Cleanup + +Goal: + +Close the remaining non-Nautilus-depth technical debt before starting another +large high-fidelity Nautilus phase. This phase intentionally avoids true L2 +queue simulation, dynamic in-Nautilus DCA/grid state machines, exchange-native +OCO lists, and venue-specific portfolio-margin cloning. + +### Phase 13A - WFO Prepared Market Cache Integration + +Purpose: + +Use the prepared market-array APIs added in Phase 9/10/12 inside higher-level +walk-forward / service loops. When Optuna/WFO evaluates many parameter trials +over the same market tape, QuantBT should not repeatedly normalize pandas data +and pack identical close/high/low arrays. + +Scope: + +- Audit `_run_walk_forward`, endpoint scoring, native event, and native + portfolio scoring paths. +- Add a run-local prepared context for one `.backtest(...)` call: + - prepare market arrays once for invariant `data`, `symbols`, and + `datetime_index`; + - pass prepared arrays into endpoint scoring when the backend supports it; + - validate reuse by datetime/symbol signature, never by mutable pandas object + identity alone. +- Reuse `NativeEventBackend.prepare_market_arrays(...)` when event/order scoring + is used. +- Reuse `NativePortfolioBackend.prepare_market_arrays(...)` when portfolio WFO + scoring is used. +- Reuse `NativePortfolioBackend.prepare_signal_matrix(...)` only when a signal + matrix is truly invariant; normally only market arrays are invariant across + trials. +- Keep all caches local to the WFO run. No process-wide mutable result cache. +- Preserve accounting, fill policy, sizing, fees, funding, margin, and + liquidation semantics exactly. + +Acceptance: + +- WFO endpoint scoring with and without prepared market arrays produces identical + metrics and final backtest outputs. +- Portfolio WFO/native portfolio scoring can reuse market arrays across trials + without stale-data risk. +- Signature mismatch rejects reuse with a clear error. +- Benchmark/mock WFO report shows prepared-cache reuse, parity, and timing so + future optimization work can separate cache benefit from report/Optuna + overhead. +- Existing endpoint behavior remains unchanged when prepared arrays are not + applicable. + +Status: + +- Implemented run-local WFO prepared scoring cache for + `target_mode="portfolio"` with endpoint scoring and `native_portfolio`. +- Added `optimization_config["use_prepared_scoring_cache"]`, default `True`, + stored in `WalkForwardConfig.metadata` for debug/parity runs. +- Cache is scoped to one WFO `.backtest(...)` call and keyed by symbol tuple, + index length, first timestamp, and last timestamp. Backend signature checks + still guard prepared reuse. +- Existing non-portfolio endpoint scoring keeps the previous fallback path. +- Added `prepared_scoring_cache` metadata to `result.metadata["walk_forward"]`. +- Added deterministic parity test proving cached and uncached portfolio WFO + endpoint scoring select the same params, objective, and final equity. +- Added `benchmarks/run_phase13_wfo_cache.py` plus committed JSON/Markdown + artifacts: + - `benchmarks/phase13_wfo_cache.json`; + - `benchmarks/phase13_wfo_cache.md`. +- Current mock report is a parity/reuse guard, not a universal speed claim: + cache hits occur, but full WFO runtime on the small fixture is still dominated + by Optuna/report construction. This reinforces Phase 13B as the next + optimization target. + +### Phase 13B - Native Portfolio Report Construction Optimization + +Purpose: + +Reduce the measured residual report-construction cost after the native portfolio +kernel without changing portfolio domain logic. + +Scope: + +- Profile and optimize construction of: + - equity/returns series; + - position matrix; + - exposure matrix; + - target-units and accepted-notional reports; + - symbol PnL reports; + - diagnostics metadata. +- Prefer ndarray block construction over per-symbol pandas concat. +- Keep default reports backward-compatible for existing notebooks. +- Add optional report-level controls only if they do not break current endpoint + behavior. + +Acceptance: + +- Native portfolio equity, positions, exposure, target units, accepted notional, + and symbol PnL totals match pre-optimization output exactly or within documented + floating tolerance. +- Tests cover multi-symbol mock data, missing data, `%_equity`, `target_weight`, + `gross_exposure`, `risk_parity`, and `beta_neutral`. +- Benchmark report separates kernel runtime from report construction. + +Status: + +- Optimized native portfolio report construction without changing kernel or + accounting semantics: + - funding series now uses ndarray calculations instead of grouping the long + symbol PnL report; + - diagnostics rejected-rebalance flags use ndarray diffs directly; + - returns are built from ndarray equity changes instead of pandas pct-change; + - exposure report computes leverage/exposure columns array-first; + - rebalance report uses `np.nonzero` over target-vs-accepted unit diffs + instead of repeated pandas stack/reindex operations. +- Strengthened prepared-vs-normal native portfolio parity tests for: + - target units; + - accepted units; + - target notional; + - accepted notional; + - exposure report; + - funding series; + - rebalance report. +- Added old-formula report parity regression in + `tests/test_phase13_portfolio_report_parity.py`, comparing the optimized + report surface against the previous pandas formulas for: + - returns; + - funding; + - rejected-rebalance diagnostics; + - exposure report; + - rebalance report. +- Re-ran targeted portfolio/report tests and the full internal regression suite + before Phase 13C; no production report formulas changed after the parity lock. +- Added Phase 13B benchmark artifacts: + - `benchmarks/run_phase13_portfolio_report.py`; + - `benchmarks/phase13_portfolio_report.json`; + - `benchmarks/phase13_portfolio_report.md`. +- Latest benchmark on the standard Phase 13B fixture: + - full facade: `0.058681s`; + - prepared reuse: `0.047904s`; + - pure Numba kernel: `0.000200s`; + - report construction residual: `0.047671s`; + - prepared reuse speedup: `1.225x`. +- Conclusion: report construction improved, but remains the dominant residual + bucket. Cython/C++ is still not justified because pure Numba kernel share is + only about `0.34%`; further gains should come from optional/lazy heavy reports + or more compact report-level controls. + +### Phase 13C - Arbitrage Production Certification Cleanup + +Purpose: + +Move native arbitrage from controlled research usability toward clearer +production-certification artifacts without pretending unsupported arbitrage +families are complete. + +Scope: + +- Add `package_pnl_report` / residual artifact for `StatArbPairSpec`, matching + the basis/index-basket reporting style: + - leg PnL; + - hedge PnL; + - spread/residual PnL; + - fees; + - funding where relevant. +- Preserve native event/vectorized parity for basis, stat-arb, and index-basket + packages. +- Keep real perp/quarterly Nautilus parity explicitly dependent on a delivery + futures instrument provider or adapter extension. +- Keep `CrossExchangeArbSpec`, `TriangularArbSpec`, and `OptionsVolArbSpec` + schema-safe with actionable `NotImplemented` messages until specialized + engines are built. + +Acceptance: + +- Stat-arb package PnL reconciles: + - leg PnL + fees + funding = package PnL; + - residual/spread PnL sign is correct; + - dynamic hedge ratios do not break accounting. +- Event vs vectorized parity remains within tolerance. +- Schema-only specs reject clearly. +- Optional Nautilus smoke skips or passes cleanly depending on installed + dependency and instrument support. + +Status: + +- Added native-event `StatArbPairSpec` package reporting to match the + vectorized/package-style arbitrage surface: + - `leg_pnl_report`; + - `package_pnl_report`; + - `spread_report`; + - diagnostics `package_pnl` and `package_pnl_residual`. +- Tightened native-vectorized stat-arb reporting with the same detailed package + columns: + - `price_pnl`; + - `fill_pnl`; + - `fees`; + - `funding_pnl`; + - `leg_pnl`; + - `hedge_pnl`; + - `spread_pnl`; + - `package_pnl`; + - `equity_delta`; + - `pnl_residual`. +- Stat-arb funding now follows the same arbitrage package policy as + basis/funding/carry routes: only legs with `funding_enabled=True` receive + funding rates. +- Schema-only arbitrage specs now reject with actionable messages pointing to + `QuantBTEndpoint.arbitrage_support_matrix()`. +- Added Phase 13C regression coverage in + `tests/test_phase13_arbitrage_certification_cleanup.py`: + - dynamic hedge-ratio stat-arb package PnL reconciliation; + - event vs vectorized stat-arb package report parity; + - funding-enabled leg filtering; + - schema-only spec guardrails. +- Upgraded the arbitrage certification runner and artifacts: + - `benchmarks/run_phase12_arbitrage_cert.py`; + - `benchmarks/phase12_arbitrage_cert.json`; + - `benchmarks/phase12_arbitrage_cert.md`. +- Latest certification smoke with `--rows 240`: + - status: `pass`; + - basis event/vectorized parity: `pass`; + - stat-pair audit: `pass`; + - stat-pair max package residual: `1.8891554987021664e-11`; + - index-basket smoke: `pass`; + - schema-only guardrails: `pass`; + - Nautilus package parity: `skipped` unless explicitly requested. + +### Explicit Non-Goals For Phase 13 + +- Dynamic in-Nautilus DCA/grid state machine. +- True L2 order-book queue/latency simulation. +- Exchange-native OCO order-list semantics. +- Venue-specific portfolio-margin clone. + +Those belong to a later dedicated Nautilus Depth phase. + +--- + ## Backend Selection Guide Use `native_vectorized` when: