From b4642a2914fa6a411674c5ae9d26954f8e44256d Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 17 Jul 2026 13:50:27 +0000 Subject: [PATCH 1/2] diagnose nautilus pct equity parity --- README.md | 10 + __init__.py | 2 + adapters/nautilus/backend.py | 16 ++ docs/endpoint.md | 30 +++ docs/nautilus_backend.md | 25 ++ endpoint.py | 36 +++ reporting/__init__.py | 2 + reporting/nautilus_bundle.py | 9 + reporting/nautilus_diagnostics.py | 225 ++++++++++++++++++ tests/test_phase5_1_nautilus_report_bundle.py | 31 ++- 10 files changed, 385 insertions(+), 1 deletion(-) create mode 100644 reporting/nautilus_diagnostics.py diff --git a/README.md b/README.md index c5d22f1..f0fa2fc 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,8 @@ account reports. controls. - `use_pyramiding=False` snaps signals to `-1/0/1`; `True` preserves fractional scales such as `1.4`. +- For crypto, `contract_size` is a notional/PnL multiplier. Exchange fractional + lots are governed by `qty_step`/`lot_size`/`min_qty`/`min_notional`. ### DCA And Grid @@ -325,6 +327,14 @@ result = bt.simulate( result.show_metrics() +diag = bt.nautilus_pct_equity_diagnostic( + data=df, + signal_col="pos_weight", + native_fee_round_trip=0.0005, + native_use_funding=False, + native_slippage=0.0002, +) + report_dir = export_nautilus_report_bundle( result=result, output_dir="reports", diff --git a/__init__.py b/__init__.py index 1d1330b..f3847eb 100644 --- a/__init__.py +++ b/__init__.py @@ -194,6 +194,7 @@ build_native_nautilus_parity_report, build_nautilus_depth_execution_report, build_nautilus_depth_parity_summary, + build_nautilus_pct_equity_diagnostic, build_portfolio_domain_audit, build_portfolio_nautilus_position_report, build_portfolio_nautilus_validation_report, @@ -236,6 +237,7 @@ "build_native_nautilus_parity_report", "build_nautilus_depth_execution_report", "build_nautilus_depth_parity_summary", + "build_nautilus_pct_equity_diagnostic", "build_portfolio_domain_audit", "build_portfolio_nautilus_position_report", "build_portfolio_nautilus_validation_report", diff --git a/adapters/nautilus/backend.py b/adapters/nautilus/backend.py index 9719905..fbb2bd4 100644 --- a/adapters/nautilus/backend.py +++ b/adapters/nautilus/backend.py @@ -156,6 +156,7 @@ def run_signal_series( "trade_notional": self.config.trade_notional, "use_pyramiding": self.config.use_pyramiding, "close_positions_on_stop": self.config.close_positions_on_stop, + **self._instrument_constraint_metadata(instrument), **self.config.metadata, **(params or {}), }, @@ -284,6 +285,21 @@ def _make_instrument(self, nt): raise NotImplementedError("custom Nautilus instruments are not wired yet") return make_binance_perpetual(self.config.instrument_id, nt) + @staticmethod + def _instrument_constraint_metadata(instrument) -> Dict: + size_increment = getattr(instrument, "size_increment", None) + min_quantity = getattr(instrument, "min_quantity", None) + min_notional = getattr(instrument, "min_notional", None) + price_increment = getattr(instrument, "price_increment", None) + return { + "qty_step": None if size_increment is None else str(size_increment), + "lot_size": None if size_increment is None else str(size_increment), + "min_qty": None if min_quantity is None else str(min_quantity), + "min_notional": None if min_notional is None else str(min_notional), + "price_increment": None if price_increment is None else str(price_increment), + "quantity_constraint_note": "lot_size/qty_step controls fractional crypto order acceptance; contract_size remains multiplier", + } + @staticmethod def _align_signal(signal: pd.Series, idx: pd.DatetimeIndex) -> pd.Series: sig = signal.copy() diff --git a/docs/endpoint.md b/docs/endpoint.md index 244e9bd..73eed0b 100644 --- a/docs/endpoint.md +++ b/docs/endpoint.md @@ -1067,6 +1067,16 @@ Requirements: - `use_pyramiding` is controlled at the endpoint level and forwarded into the Nautilus strategy adapter. `False` snaps raw signals to `-1/0/1`; `True` preserves fractional scales such as `1.4` in `%_equity` sizing; +- `%_equity` native-vs-Nautilus comparisons must align semantics before + interpreting performance differences. Native legacy `fee=` is round-trip and + is halved internally, while Nautilus `fee_rate=` is currently metadata for + reporting and the signal adapter uses the Nautilus instrument fee model. + Custom endpoint slippage and funding are also not applied by the current + Nautilus signal-series adapter. +- for crypto fractional trading, use venue quantity constraints + `qty_step`/`lot_size`/`min_qty`/`min_notional`. `contract_size` is a PnL and + notional multiplier, not the Binance lot size. Do not set + `contract_size=0.001` just to allow fractional ETH/BTC orders. - DCA/grid, explicit order replay, pair trading, and multi-symbol portfolio validation remain on native QuantBT backends until their Nautilus event adapters are added; @@ -1087,6 +1097,26 @@ Requirements: equity returns by default with `quantstats_periods_per_year=365` for crypto. +Diagnostic helper for `%_equity` comparisons: + +```python +diag = bt.nautilus_pct_equity_diagnostic( + data=df_result, + signal_col="pos_weight", + native_fee_round_trip=0.0005, + native_use_funding=True, + native_slippage=0.0002, +) + +display(diag["checks"]) +display(diag["signal"]["transition_report"].head()) +``` + +Use this before comparing `QuantBTEndpoint.pct_equity(...)` with +`QuantBTEndpoint.nautilus_validation(hedge_type="%_equity", ...)`; it reports +signal transition count, Nautilus order/fill count, fee/slippage/funding +semantic differences, and Binance quantity-step constraints. + Nautilus metadata: ```python diff --git a/docs/nautilus_backend.md b/docs/nautilus_backend.md index aee2d6a..cd21179 100644 --- a/docs/nautilus_backend.md +++ b/docs/nautilus_backend.md @@ -51,6 +51,15 @@ Scope: - sizing modes: `signal_notional`, `notional`, `unit`, and `%_equity`; - endpoint/engine-level `use_pyramiding`, where `False` snaps raw signals to direction only and `True` preserves fractional signal scale; +- `%_equity` signal-series validation currently uses Nautilus instrument + maker/taker fees and bar market execution. Endpoint `fee_rate` and + `slippage` are preserved in metadata/report bundles, but custom fee/slippage + are not injected into Nautilus' signal adapter yet. Funding/carry is also not + applied in the current Nautilus signal-series path. +- Binance-style fractional crypto constraints are represented by + `qty_step`/`lot_size`/`min_qty`/`min_notional`. `contract_size` remains a + notional/PnL multiplier; it should not be changed to `0.001` just to allow + fractional crypto lots. - explicit single-symbol `OrderIntent` replay through `QuantBTEndpoint.orders(backend="nautilus", ...)`; - explicit order types mapped to Nautilus order factory: market, limit, @@ -92,6 +101,22 @@ Report bundle: `cancelled_count`, and `rejected_count` to `run_manifest.json` and `metrics_summary.json`. +`%_equity` diagnostic: + +```python +diag = bt.nautilus_pct_equity_diagnostic( + data=df_result, + signal_col="pos_weight", + native_fee_round_trip=0.0005, + native_use_funding=True, + native_slippage=0.0002, +) +``` + +The diagnostic reports signal transitions vs Nautilus orders/fills, fee +convention differences, unsupported custom slippage/funding, and lot-size +constraints. + Parity audit: - `build_native_nautilus_parity_report(native_result, nautilus_result)` builds diff --git a/endpoint.py b/endpoint.py index 87fc75c..74befbf 100644 --- a/endpoint.py +++ b/endpoint.py @@ -859,6 +859,42 @@ def fills_report(self) -> pd.DataFrame: """Return latest fills report, or an empty DataFrame.""" return self._require_result().metadata.get("fills_report", pd.DataFrame()) + def nautilus_pct_equity_diagnostic( + self, + *, + data, + signal=None, + signal_col: Optional[str] = None, + native_fee_round_trip: Optional[float] = None, + native_fee_one_way: Optional[float] = None, + native_use_funding: Optional[bool] = None, + native_slippage: Optional[float] = None, + ) -> Dict: + """ + Diagnose why a Nautilus `%_equity` validation run differs from native. + + This helper reports signal transition count, Nautilus order/fill count, + fee/slippage/funding semantic differences, and exchange lot-size + constraints. It is diagnostic-only and does not mutate the result. + """ + from .reporting import build_nautilus_pct_equity_diagnostic + + frame, _, sig = _normalize_single_data( + data=data, + signal=signal, + signal_col=signal_col, + datetime_index=None, + ) + return build_nautilus_pct_equity_diagnostic( + self._require_result(), + data=frame, + signal=sig, + native_fee_round_trip=native_fee_round_trip, + native_fee_one_way=native_fee_one_way, + native_use_funding=native_use_funding, + native_slippage=native_slippage, + ) + def _run_single(self, data, signal, signal_col, datetime_index, symbols): frame, idx, sig = _normalize_single_data(data=data, signal=signal, signal_col=signal_col, datetime_index=datetime_index) backend = _resolve_backend(self.config) diff --git a/reporting/__init__.py b/reporting/__init__.py index 12a9ede..e73ef90 100644 --- a/reporting/__init__.py +++ b/reporting/__init__.py @@ -2,6 +2,7 @@ from .arbitrage_audit import build_arbitrage_domain_audit, compare_native_arbitrage_results from .nautilus_bundle import export_nautilus_report_bundle +from .nautilus_diagnostics import build_nautilus_pct_equity_diagnostic from .parity import ( build_native_nautilus_parity_report, build_nautilus_depth_execution_report, @@ -19,6 +20,7 @@ "build_native_nautilus_parity_report", "build_nautilus_depth_execution_report", "build_nautilus_depth_parity_summary", + "build_nautilus_pct_equity_diagnostic", "build_portfolio_domain_audit", "build_portfolio_nautilus_position_report", "build_portfolio_nautilus_validation_report", diff --git a/reporting/nautilus_bundle.py b/reporting/nautilus_bundle.py index c573ed7..8dc5e05 100644 --- a/reporting/nautilus_bundle.py +++ b/reporting/nautilus_bundle.py @@ -450,6 +450,15 @@ def _config_payload_from_result(result: BacktestResultV2, metadata: Dict[str, An "alloc_per_trade": metadata.get("alloc_per_trade", sizing.get("alloc_per_trade")), "use_pyramiding": metadata.get("use_pyramiding", sizing.get("use_pyramiding")), "contract_size": sizing.get("contract_size"), + "contract_size_note": "contract_size is a multiplier for notional/PnL, not the exchange lot size", + "quantity_constraints": { + "qty_step": metadata.get("qty_step"), + "lot_size": metadata.get("lot_size", metadata.get("qty_step")), + "min_qty": metadata.get("min_qty"), + "min_notional": metadata.get("min_notional"), + "price_increment": metadata.get("price_increment"), + "note": "Use qty_step/lot_size/min_qty/min_notional for Binance-style fractional order constraints.", + }, }, "effective_fees": { "requested_fee_rate": requested_fee_rate, diff --git a/reporting/nautilus_diagnostics.py b/reporting/nautilus_diagnostics.py new file mode 100644 index 0000000..ac48f63 --- /dev/null +++ b/reporting/nautilus_diagnostics.py @@ -0,0 +1,225 @@ +"""Nautilus validation diagnostics.""" + +from __future__ import annotations + +from typing import Dict, List, Optional + +import numpy as np +import pandas as pd + +from ..adapters.nautilus.instruments import SUPPORTED_BINANCE_PERP_SPECS, normalize_binance_perp_symbol +from ..core.results import BacktestResultV2 + + +def build_nautilus_pct_equity_diagnostic( + result: BacktestResultV2, + *, + data: pd.DataFrame, + signal: pd.Series, + native_fee_round_trip: Optional[float] = None, + native_fee_one_way: Optional[float] = None, + native_use_funding: Optional[bool] = None, + native_slippage: Optional[float] = None, +) -> Dict: + """ + Compare a Nautilus `%_equity` validation run against native expectations. + + The helper is intentionally diagnostic-only: it does not claim that + Nautilus and native legacy should match. It highlights the most common + sources of divergence: fee convention/application, funding, slippage, + signal transitions, and Binance lot-size constraints. + """ + if not isinstance(result, BacktestResultV2): + raise TypeError("build_nautilus_pct_equity_diagnostic requires BacktestResultV2") + if "close" not in data: + raise ValueError("data must contain a close column") + + metadata = result.metadata or {} + idx = _utc_index(data.index) + sig = _align_signal(signal, idx) + use_pyramiding = bool(metadata.get("use_pyramiding", _nested(metadata, "run_config", "sizing", "use_pyramiding", default=True))) + effective_signal = sig.astype(float) if use_pyramiding else np.sign(sig.astype(float)) + transitions = effective_signal.ne(effective_signal.shift(1).fillna(0.0)) + transition_report = pd.DataFrame( + { + "timestamp": idx[transitions.to_numpy()], + "raw_signal": sig.loc[transitions].to_numpy(dtype=float), + "effective_signal": effective_signal.loc[transitions].to_numpy(dtype=float), + "close": data.reindex(idx)["close"].loc[transitions].to_numpy(dtype=float), + } + ) + + orders_count = int(metadata.get("orders_count", len(_frame(metadata.get("orders_report", metadata.get("order_report")))))) + fills_count = int(metadata.get("fills_count", len(_frame(metadata.get("fills_report"))))) + sizing_mode = str(metadata.get("sizing_mode", _nested(metadata, "run_config", "sizing", "hedge_type", default=""))).lower() + requested_fee = _safe_float(metadata.get("fee_rate")) + requested_slippage = _safe_float(metadata.get("slippage")) + expected_fee = native_fee_one_way + if expected_fee is None and native_fee_round_trip is not None: + expected_fee = float(native_fee_round_trip) / 2.0 + + constraints = _instrument_constraints(metadata) + lot_report = _lot_size_risk_report( + transition_report=transition_report, + initial_capital=float(result.initial_capital), + alloc_per_trade=float(metadata.get("trade_notional", metadata.get("alloc_per_trade", 0.0)) or 0.0), + constraints=constraints, + ) + + checks = { + "sizing_mode_is_pct_equity": sizing_mode in {"%_equity", "pct_equity"}, + "orders_not_more_than_signal_transitions": orders_count <= int(len(transition_report)), + "fills_not_more_than_orders": fills_count <= orders_count, + "fee_convention_matches_native": True if expected_fee is None or requested_fee is None else abs(float(expected_fee) - float(requested_fee)) <= 1e-15, + "custom_fee_rate_applied_to_nautilus": False, + "funding_matches_native": True if native_use_funding is None else bool(native_use_funding) is False, + "slippage_matches_native": True if native_slippage is None else abs(float(native_slippage)) <= 1e-15, + "custom_slippage_applied_to_nautilus": False, + "has_lot_size_constraints": constraints.get("qty_step") is not None, + } + recommendations = _recommendations(checks, native_use_funding=native_use_funding, native_slippage=native_slippage) + status = "ok" if all(checks.values()) else "diff" + return { + "status": status, + "checks": checks, + "signal": { + "rows": int(len(idx)), + "raw_transition_count": int(sig.ne(sig.shift(1).fillna(0.0)).sum()), + "effective_transition_count": int(len(transition_report)), + "use_pyramiding": use_pyramiding, + "signal_index_matches_data_index": bool(_utc_index(signal.index).equals(idx)), + "transition_report": transition_report, + }, + "orders": { + "orders_count": orders_count, + "fills_count": fills_count, + "positions_count": int(metadata.get("positions_count", 0) or 0), + "missing_order_events_vs_transitions": max(0, int(len(transition_report)) - orders_count), + }, + "execution_semantics": { + "requested_fee_rate": requested_fee, + "expected_native_one_way_fee_rate": expected_fee, + "custom_fee_rate_applied_to_nautilus": False, + "requested_slippage": requested_slippage, + "native_slippage": native_slippage, + "custom_slippage_applied_to_nautilus": False, + "native_use_funding": native_use_funding, + "nautilus_signal_funding_supported": False, + "adapter_fill_model": "NautilusTrader bar market execution with instrument maker/taker fee model", + }, + "instrument_constraints": constraints, + "lot_size_risk": lot_report, + "recommendations": recommendations, + } + + +def _instrument_constraints(metadata: Dict) -> Dict: + instrument_id = metadata.get("instrument_id") or _nested(metadata, "run_config", "nautilus", "instrument_id") + out = { + "instrument_id": instrument_id, + "qty_step": metadata.get("qty_step") or metadata.get("lot_size") or metadata.get("size_increment"), + "lot_size": metadata.get("lot_size") or metadata.get("qty_step") or metadata.get("size_increment"), + "min_qty": metadata.get("min_qty") or metadata.get("min_quantity"), + "min_notional": metadata.get("min_notional"), + "contract_size_note": "contract_size is a multiplier; lot_size/qty_step controls fractional crypto order acceptance", + } + if instrument_id: + try: + spec = SUPPORTED_BINANCE_PERP_SPECS[normalize_binance_perp_symbol(str(instrument_id))] + out.update( + { + "qty_step": out["qty_step"] or spec.size_increment, + "lot_size": out["lot_size"] or spec.size_increment, + "min_qty": out["min_qty"] or spec.min_quantity, + "min_notional": out["min_notional"] or "10.0", + "price_increment": spec.price_increment, + "quantity_precision": spec.size_precision, + } + ) + except ValueError: + pass + return out + + +def _lot_size_risk_report( + *, + transition_report: pd.DataFrame, + initial_capital: float, + alloc_per_trade: float, + constraints: Dict, +) -> Dict: + qty_step = _safe_float(constraints.get("qty_step")) + min_qty = _safe_float(constraints.get("min_qty")) + if transition_report.empty or qty_step is None: + return {"status": "unknown", "potential_small_delta_count": 0} + alloc = alloc_per_trade / 100.0 if alloc_per_trade > 1.0 else alloc_per_trade + close = pd.to_numeric(transition_report["close"], errors="coerce").replace(0.0, np.nan) + signal = pd.to_numeric(transition_report["effective_signal"], errors="coerce").abs() + approx_qty = (initial_capital * alloc * signal / close).fillna(0.0) + threshold = max(qty_step, min_qty or 0.0) + small = approx_qty < threshold + return { + "status": "ok" if not bool(small.any()) else "risk", + "potential_small_delta_count": int(small.sum()), + "qty_step": float(qty_step), + "min_qty": None if min_qty is None else float(min_qty), + "min_transition_approx_qty": float(approx_qty.min()) if len(approx_qty) else 0.0, + "note": "Approximation uses initial capital only; live equity and current position can create smaller deltas later.", + } + + +def _recommendations(checks: Dict[str, bool], *, native_use_funding, native_slippage) -> List[str]: + out: List[str] = [] + if not checks["fee_convention_matches_native"]: + out.append("Align fee convention: legacy `fee` is round-trip; Nautilus `fee_rate` is metadata one-way today.") + if not checks["custom_fee_rate_applied_to_nautilus"]: + out.append("Current Nautilus signal adapter uses Nautilus instrument maker/taker fees, not endpoint custom fee_rate.") + if native_use_funding: + out.append("Disable native funding for apples-to-apples, or implement Nautilus funding/carry adapter.") + if native_slippage and abs(float(native_slippage)) > 0.0: + out.append("Disable native slippage for apples-to-apples, or implement Nautilus slippage model.") + if not checks["orders_not_more_than_signal_transitions"]: + out.append("Inspect signal timestamp alignment and Nautilus order reports; order count exceeds transition count.") + return out + + +def _align_signal(signal: pd.Series, idx: pd.DatetimeIndex) -> pd.Series: + sig = signal.copy() + if not isinstance(sig.index, pd.DatetimeIndex): + sig.index = pd.to_datetime(sig.index, utc=True) + if sig.index.tz is None: + sig.index = sig.index.tz_localize("UTC") + else: + sig.index = sig.index.tz_convert("UTC") + return sig.reindex(idx, method="ffill").fillna(0.0) + + +def _utc_index(index) -> pd.DatetimeIndex: + idx = pd.DatetimeIndex(pd.to_datetime(index, utc=True)) + if idx.tz is None: + idx = idx.tz_localize("UTC") + else: + idx = idx.tz_convert("UTC") + return idx + + +def _nested(mapping: Dict, *keys, default=None): + cur = mapping + for key in keys: + if not isinstance(cur, dict) or key not in cur: + return default + cur = cur[key] + return cur + + +def _frame(value) -> pd.DataFrame: + return value if isinstance(value, pd.DataFrame) else pd.DataFrame() + + +def _safe_float(value): + try: + if value is None: + return None + return float(value) + except (TypeError, ValueError): + return None diff --git a/tests/test_phase5_1_nautilus_report_bundle.py b/tests/test_phase5_1_nautilus_report_bundle.py index 9b88955..2f758a6 100644 --- a/tests/test_phase5_1_nautilus_report_bundle.py +++ b/tests/test_phase5_1_nautilus_report_bundle.py @@ -5,7 +5,7 @@ import pandas as pd -from quantbt import BacktestResultV2, export_nautilus_report_bundle +from quantbt import BacktestResultV2, build_nautilus_pct_equity_diagnostic, export_nautilus_report_bundle from quantbt.reporting.nautilus_bundle import ( build_nautilus_trade_log, format_nautilus_event_log, @@ -227,6 +227,35 @@ def test_config_json_has_single_effective_fee_and_execution_view(tmp_path): assert config["effective_fees"]["custom_fee_rate_applied_to_nautilus"] is False assert config["effective_execution"]["requested_slippage_rate"] == 0.0002 assert config["effective_execution"]["custom_slippage_applied_to_nautilus"] is False + assert config["effective_sizing"]["contract_size_note"].startswith("contract_size is a multiplier") + assert config["effective_sizing"]["quantity_constraints"]["note"].startswith("Use qty_step") + + +def test_pct_equity_nautilus_diagnostic_flags_non_apples_to_apples_settings(): + result = _synthetic_result() + idx = result.equity.index + data = pd.DataFrame({"close": result.closes["Close_BTCUSDT-PERP.BINANCE"]}, index=idx) + signal = pd.Series([0.0, 1.0, 1.0, 0.0, 0.0, -1.0, -1.0, 0.0], index=idx) + + report = build_nautilus_pct_equity_diagnostic( + result, + data=data, + signal=signal, + native_fee_round_trip=0.0005, + native_use_funding=True, + native_slippage=0.0002, + ) + + assert report["status"] == "diff" + assert report["checks"]["fee_convention_matches_native"] is False + assert report["checks"]["custom_fee_rate_applied_to_nautilus"] is False + assert report["checks"]["funding_matches_native"] is False + assert report["checks"]["slippage_matches_native"] is False + assert report["signal"]["effective_transition_count"] == 4 + assert report["orders"]["orders_count"] == 3 + assert report["orders"]["missing_order_events_vs_transitions"] == 1 + assert report["instrument_constraints"]["qty_step"] == "0.001" + assert "contract_size is a multiplier" in report["instrument_constraints"]["contract_size_note"] def test_export_nautilus_report_bundle_handles_empty_raw_reports(tmp_path): From b398cf69995cb368791599dc32d5964dd74e5f96 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Fri, 17 Jul 2026 14:49:57 +0000 Subject: [PATCH 2/2] test: add pct equity nautilus smoke audit --- benchmarks/pct_equity_nautilus_smoke.json | 235 ++++++++++++++++++++ benchmarks/pct_equity_nautilus_smoke.md | 33 +++ benchmarks/run_pct_equity_nautilus_smoke.py | 227 +++++++++++++++++++ tests/test_pct_equity_nautilus_smoke.py | 43 ++++ 4 files changed, 538 insertions(+) create mode 100644 benchmarks/pct_equity_nautilus_smoke.json create mode 100644 benchmarks/pct_equity_nautilus_smoke.md create mode 100644 benchmarks/run_pct_equity_nautilus_smoke.py create mode 100644 tests/test_pct_equity_nautilus_smoke.py diff --git a/benchmarks/pct_equity_nautilus_smoke.json b/benchmarks/pct_equity_nautilus_smoke.json new file mode 100644 index 0000000..8a415ba --- /dev/null +++ b/benchmarks/pct_equity_nautilus_smoke.json @@ -0,0 +1,235 @@ +{ + "status": "pass", + "rows": 300, + "symbol": "ETHUSDT-PERP.BINANCE", + "scenarios": [ + { + "name": "aligned_fee_no_funding_no_slippage", + "note": "Native one-way fee approximates ETH taker fee; custom Nautilus fee_rate is not applied.", + "native": { + "final_equity": 20219.18903580209, + "total_return_pct": 1.0959451790104502, + "num_trades": 7 + }, + "nautilus": { + "final_equity": 20219.237180320004, + "total_return_pct": 1.09618590160002, + "num_trades": 7 + }, + "final_equity_diff": 0.04814451791389729, + "diagnostic": { + "status": "diff", + "checks": { + "sizing_mode_is_pct_equity": true, + "orders_not_more_than_signal_transitions": true, + "fills_not_more_than_orders": true, + "fee_convention_matches_native": true, + "custom_fee_rate_applied_to_nautilus": false, + "funding_matches_native": true, + "slippage_matches_native": true, + "custom_slippage_applied_to_nautilus": false, + "has_lot_size_constraints": true + }, + "signal": { + "rows": 300, + "raw_transition_count": 6, + "effective_transition_count": 6, + "use_pyramiding": false, + "signal_index_matches_data_index": true, + "transition_report_head": [ + { + "timestamp": "2024-01-01T10:00:00+00:00", + "raw_signal": 1.0, + "effective_signal": 1.0, + "close": 2068.3791793982627 + }, + { + "timestamp": "2024-01-04T08:00:00+00:00", + "raw_signal": 0.0, + "effective_signal": 0.0, + "close": 1981.0965643123966 + }, + { + "timestamp": "2024-01-05T14:00:00+00:00", + "raw_signal": -1.0, + "effective_signal": -1.0, + "close": 2074.124871437053 + }, + { + "timestamp": "2024-01-08T02:00:00+00:00", + "raw_signal": 0.0, + "effective_signal": 0.0, + "close": 2145.0084364453314 + }, + { + "timestamp": "2024-01-09T18:00:00+00:00", + "raw_signal": 1.0, + "effective_signal": 1.0, + "close": 2087.01814134276 + }, + { + "timestamp": "2024-01-11T20:00:00+00:00", + "raw_signal": 0.0, + "effective_signal": 0.0, + "close": 2303.985396693068 + } + ] + }, + "orders": { + "orders_count": 6, + "fills_count": 6, + "positions_count": 3, + "missing_order_events_vs_transitions": 0 + }, + "execution_semantics": { + "requested_fee_rate": 0.0004, + "expected_native_one_way_fee_rate": 0.0004, + "custom_fee_rate_applied_to_nautilus": false, + "requested_slippage": 0.0, + "native_slippage": 0.0, + "custom_slippage_applied_to_nautilus": false, + "native_use_funding": false, + "nautilus_signal_funding_supported": false, + "adapter_fill_model": "NautilusTrader bar market execution with instrument maker/taker fee model" + }, + "instrument_constraints": { + "instrument_id": "ETHUSDT-PERP.BINANCE", + "qty_step": "0.001", + "lot_size": "0.001", + "min_qty": "0.001", + "min_notional": "10.00000000 USDT", + "contract_size_note": "contract_size is a multiplier; lot_size/qty_step controls fractional crypto order acceptance", + "price_increment": "0.01", + "quantity_precision": 3 + }, + "lot_size_risk": { + "status": "risk", + "potential_small_delta_count": 3, + "qty_step": 0.001, + "min_qty": 0.001, + "min_transition_approx_qty": 0.0, + "note": "Approximation uses initial capital only; live equity and current position can create smaller deltas later." + }, + "recommendations": [ + "Current Nautilus signal adapter uses Nautilus instrument maker/taker fees, not endpoint custom fee_rate." + ] + } + }, + { + "name": "user_like_mismatch", + "note": "Matches the observed notebook-style mismatch: fee convention, funding, and slippage differ.", + "native": { + "final_equity": 20209.303087300526, + "total_return_pct": 1.046515436502632, + "num_trades": 7 + }, + "nautilus": { + "final_equity": 20219.237180320004, + "total_return_pct": 1.09618590160002, + "num_trades": 7 + }, + "final_equity_diff": 9.934093019477586, + "diagnostic": { + "status": "diff", + "checks": { + "sizing_mode_is_pct_equity": true, + "orders_not_more_than_signal_transitions": true, + "fills_not_more_than_orders": true, + "fee_convention_matches_native": false, + "custom_fee_rate_applied_to_nautilus": false, + "funding_matches_native": false, + "slippage_matches_native": false, + "custom_slippage_applied_to_nautilus": false, + "has_lot_size_constraints": true + }, + "signal": { + "rows": 300, + "raw_transition_count": 6, + "effective_transition_count": 6, + "use_pyramiding": false, + "signal_index_matches_data_index": true, + "transition_report_head": [ + { + "timestamp": "2024-01-01T10:00:00+00:00", + "raw_signal": 1.0, + "effective_signal": 1.0, + "close": 2068.3791793982627 + }, + { + "timestamp": "2024-01-04T08:00:00+00:00", + "raw_signal": 0.0, + "effective_signal": 0.0, + "close": 1981.0965643123966 + }, + { + "timestamp": "2024-01-05T14:00:00+00:00", + "raw_signal": -1.0, + "effective_signal": -1.0, + "close": 2074.124871437053 + }, + { + "timestamp": "2024-01-08T02:00:00+00:00", + "raw_signal": 0.0, + "effective_signal": 0.0, + "close": 2145.0084364453314 + }, + { + "timestamp": "2024-01-09T18:00:00+00:00", + "raw_signal": 1.0, + "effective_signal": 1.0, + "close": 2087.01814134276 + }, + { + "timestamp": "2024-01-11T20:00:00+00:00", + "raw_signal": 0.0, + "effective_signal": 0.0, + "close": 2303.985396693068 + } + ] + }, + "orders": { + "orders_count": 6, + "fills_count": 6, + "positions_count": 3, + "missing_order_events_vs_transitions": 0 + }, + "execution_semantics": { + "requested_fee_rate": 0.0005, + "expected_native_one_way_fee_rate": 0.00025, + "custom_fee_rate_applied_to_nautilus": false, + "requested_slippage": 0.0002, + "native_slippage": 0.0002, + "custom_slippage_applied_to_nautilus": false, + "native_use_funding": true, + "nautilus_signal_funding_supported": false, + "adapter_fill_model": "NautilusTrader bar market execution with instrument maker/taker fee model" + }, + "instrument_constraints": { + "instrument_id": "ETHUSDT-PERP.BINANCE", + "qty_step": "0.001", + "lot_size": "0.001", + "min_qty": "0.001", + "min_notional": "10.00000000 USDT", + "contract_size_note": "contract_size is a multiplier; lot_size/qty_step controls fractional crypto order acceptance", + "price_increment": "0.01", + "quantity_precision": 3 + }, + "lot_size_risk": { + "status": "risk", + "potential_small_delta_count": 3, + "qty_step": 0.001, + "min_qty": 0.001, + "min_transition_approx_qty": 0.0, + "note": "Approximation uses initial capital only; live equity and current position can create smaller deltas later." + }, + "recommendations": [ + "Align fee convention: legacy `fee` is round-trip; Nautilus `fee_rate` is metadata one-way today.", + "Current Nautilus signal adapter uses Nautilus instrument maker/taker fees, not endpoint custom fee_rate.", + "Disable native funding for apples-to-apples, or implement Nautilus funding/carry adapter.", + "Disable native slippage for apples-to-apples, or implement Nautilus slippage model." + ] + } + } + ], + "conclusion": "When fee/funding/slippage semantics are aligned as closely as the current adapters allow, the synthetic final-equity gap is only `0.048145` USD and order/fill counts match. The user-like setup intentionally differs: legacy `fee` is round-trip, Nautilus `fee_rate` is metadata today, native funding/slippage are applied while Nautilus signal validation does not apply custom funding/slippage. That scenario shows a larger synthetic gap of `9.934093` USD. Large real-alpha gaps should be audited with the diagnostic helper first; if transition counts match, the next production task is implementing custom fee/slippage/funding in the Nautilus signal adapter." +} diff --git a/benchmarks/pct_equity_nautilus_smoke.md b/benchmarks/pct_equity_nautilus_smoke.md new file mode 100644 index 0000000..4f3ab68 --- /dev/null +++ b/benchmarks/pct_equity_nautilus_smoke.md @@ -0,0 +1,33 @@ +# `%_equity` Native vs Nautilus Smoke + +Status: **pass** +Rows: `300` +Symbol: `ETHUSDT-PERP.BINANCE` + +## aligned_fee_no_funding_no_slippage + +- Native final equity: `20219.189036` +- Nautilus final equity: `20219.237180` +- Final equity diff: `0.048145` +- Native trades: `7` +- Nautilus trades: `7` +- Signal transitions: `6` +- Nautilus orders/fills: `6` / `6` +- Checks: `{'sizing_mode_is_pct_equity': True, 'orders_not_more_than_signal_transitions': True, 'fills_not_more_than_orders': True, 'fee_convention_matches_native': True, 'custom_fee_rate_applied_to_nautilus': False, 'funding_matches_native': True, 'slippage_matches_native': True, 'custom_slippage_applied_to_nautilus': False, 'has_lot_size_constraints': True}` +- Note: Native one-way fee approximates ETH taker fee; custom Nautilus fee_rate is not applied. + +## user_like_mismatch + +- Native final equity: `20209.303087` +- Nautilus final equity: `20219.237180` +- Final equity diff: `9.934093` +- Native trades: `7` +- Nautilus trades: `7` +- Signal transitions: `6` +- Nautilus orders/fills: `6` / `6` +- Checks: `{'sizing_mode_is_pct_equity': True, 'orders_not_more_than_signal_transitions': True, 'fills_not_more_than_orders': True, 'fee_convention_matches_native': False, 'custom_fee_rate_applied_to_nautilus': False, 'funding_matches_native': False, 'slippage_matches_native': False, 'custom_slippage_applied_to_nautilus': False, 'has_lot_size_constraints': True}` +- Note: Matches the observed notebook-style mismatch: fee convention, funding, and slippage differ. + +## Conclusion + +When fee/funding/slippage semantics are aligned as closely as the current adapters allow, the synthetic final-equity gap is only `0.048145` USD and order/fill counts match. The user-like setup intentionally differs: legacy `fee` is round-trip, Nautilus `fee_rate` is metadata today, native funding/slippage are applied while Nautilus signal validation does not apply custom funding/slippage. That scenario shows a larger synthetic gap of `9.934093` USD. Large real-alpha gaps should be audited with the diagnostic helper first; if transition counts match, the next production task is implementing custom fee/slippage/funding in the Nautilus signal adapter. diff --git a/benchmarks/run_pct_equity_nautilus_smoke.py b/benchmarks/run_pct_equity_nautilus_smoke.py new file mode 100644 index 0000000..df9bdc0 --- /dev/null +++ b/benchmarks/run_pct_equity_nautilus_smoke.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Smoke compare native legacy `%_equity` and Nautilus `%_equity` validation.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Dict, Optional + +import numpy as np +import pandas as pd + +PACKAGE_DIR = Path(__file__).resolve().parents[1] +PROJECT_DIR = PACKAGE_DIR.parent +if str(PROJECT_DIR) not in sys.path: + sys.path.insert(0, str(PROJECT_DIR)) + +from quantbt import QuantBTEndpoint # noqa: E402 +from quantbt.adapters.nautilus import NautilusBackendConfig # noqa: E402 + + +def run_smoke(rows: int = 300) -> Dict: + data = _synthetic_eth_data(rows=rows) + scenarios = [ + { + "name": "aligned_fee_no_funding_no_slippage", + "native_fee_round_trip": 0.0008, + "native_use_funding": False, + "native_slippage": 0.0, + "nautilus_fee_rate": 0.0004, + "nautilus_use_funding": False, + "nautilus_slippage": 0.0, + "note": "Native one-way fee approximates ETH taker fee; custom Nautilus fee_rate is not applied.", + }, + { + "name": "user_like_mismatch", + "native_fee_round_trip": 0.0005, + "native_use_funding": True, + "native_slippage": 0.0002, + "nautilus_fee_rate": 0.0005, + "nautilus_use_funding": False, + "nautilus_slippage": 0.0002, + "note": "Matches the observed notebook-style mismatch: fee convention, funding, and slippage differ.", + }, + ] + results = [] + for scenario in scenarios: + results.append(_run_scenario(data, scenario)) + return { + "status": "pass", + "rows": int(rows), + "symbol": "ETHUSDT-PERP.BINANCE", + "scenarios": results, + "conclusion": _conclusion(results), + } + + +def make_markdown(report: Dict) -> str: + lines = [ + "# `%_equity` Native vs Nautilus Smoke", + "", + f"Status: **{report['status']}**", + f"Rows: `{report['rows']}`", + f"Symbol: `{report['symbol']}`", + "", + ] + for item in report["scenarios"]: + lines.extend( + [ + f"## {item['name']}", + "", + f"- Native final equity: `{item['native']['final_equity']:.6f}`", + f"- Nautilus final equity: `{item['nautilus']['final_equity']:.6f}`", + f"- Final equity diff: `{item['final_equity_diff']:.6f}`", + f"- Native trades: `{item['native']['num_trades']}`", + f"- Nautilus trades: `{item['nautilus']['num_trades']}`", + f"- Signal transitions: `{item['diagnostic']['signal']['effective_transition_count']}`", + f"- Nautilus orders/fills: `{item['diagnostic']['orders']['orders_count']}` / `{item['diagnostic']['orders']['fills_count']}`", + f"- Checks: `{item['diagnostic']['checks']}`", + f"- Note: {item['note']}", + "", + ] + ) + lines.extend(["## Conclusion", "", report["conclusion"], ""]) + return "\n".join(lines) + + +def _run_scenario(data: pd.DataFrame, scenario: Dict) -> Dict: + native = QuantBTEndpoint.pct_equity( + initial_capital=20_000, + leverage=5, + maintenance_ratio=0.005, + contract_size=1.0, + use_funding=bool(scenario["native_use_funding"]), + funding_rate=0.0001, + alloc_per_trade=0.5, + fee=float(scenario["native_fee_round_trip"]), + slippage=float(scenario["native_slippage"]), + use_pyramiding=False, + ) + native_result = native.backtest(data=data, signal_col="pos_weight") + + nautilus = QuantBTEndpoint.nautilus_validation( + initial_capital=20_000, + leverage=5, + alloc_per_trade=0.5, + hedge_type="%_equity", + fee_rate=float(scenario["nautilus_fee_rate"]), + use_funding=bool(scenario["nautilus_use_funding"]), + use_pyramiding=False, + slippage=float(scenario["nautilus_slippage"]), + nautilus_config=NautilusBackendConfig( + timeframe="1h", + starting_balance=20_000, + trade_notional=0.5, + close_positions_on_stop=False, + bypass_logging=True, + log_level="ERROR", + ), + ) + nautilus_result = nautilus.simulate( + data=data, + signal_col="pos_weight", + symbols=["ETHUSDT-PERP.BINANCE"], + show_order_logs=False, + ) + diagnostic = nautilus.nautilus_pct_equity_diagnostic( + data=data, + signal_col="pos_weight", + native_fee_round_trip=float(scenario["native_fee_round_trip"]), + native_use_funding=bool(scenario["native_use_funding"]), + native_slippage=float(scenario["native_slippage"]), + ) + native_report = native_result.full_report() + nautilus_report = nautilus_result.full_report() + return { + "name": scenario["name"], + "note": scenario["note"], + "native": { + "final_equity": float(native_result.equity.iloc[-1]), + "total_return_pct": float(native_report["total_return_pct"]), + "num_trades": int(native_report["num_trades"]), + }, + "nautilus": { + "final_equity": float(nautilus_result.equity.iloc[-1]), + "total_return_pct": float(nautilus_report["total_return_pct"]), + "num_trades": int(nautilus_report["num_trades"]), + }, + "final_equity_diff": float(nautilus_result.equity.iloc[-1] - native_result.equity.iloc[-1]), + "diagnostic": _jsonable_diagnostic(diagnostic), + } + + +def _synthetic_eth_data(rows: int) -> pd.DataFrame: + idx = pd.date_range("2024-01-01", periods=rows, freq="1h", tz="UTC") + grid = np.arange(rows) + close = pd.Series(2000 + 80 * np.sin(grid / 18) + 0.8 * grid + 20 * np.sin(grid / 5), index=idx) + signal = pd.Series(0.0, index=idx) + signal.iloc[10 : min(80, rows)] = 1.0 + signal.iloc[min(110, rows) : min(170, rows)] = -1.0 + signal.iloc[min(210, rows) : min(260, rows)] = 1.0 + return pd.DataFrame( + { + "open": close, + "high": close * 1.002, + "low": close * 0.998, + "close": close, + "volume": 10_000.0, + "pos_weight": signal, + }, + index=idx, + ) + + +def _conclusion(results) -> str: + aligned = next(item for item in results if item["name"] == "aligned_fee_no_funding_no_slippage") + mismatch = next(item for item in results if item["name"] == "user_like_mismatch") + return ( + "When fee/funding/slippage semantics are aligned as closely as the current adapters allow, " + f"the synthetic final-equity gap is only `{aligned['final_equity_diff']:.6f}` USD and order/fill counts match. " + "The user-like setup intentionally differs: legacy `fee` is round-trip, Nautilus `fee_rate` is metadata today, " + "native funding/slippage are applied while Nautilus signal validation does not apply custom funding/slippage. " + f"That scenario shows a larger synthetic gap of `{mismatch['final_equity_diff']:.6f}` USD. " + "Large real-alpha gaps should be audited with the diagnostic helper first; if transition counts match, the next " + "production task is implementing custom fee/slippage/funding in the Nautilus signal adapter." + ) + + +def _jsonable_diagnostic(diagnostic: Dict) -> Dict: + out = dict(diagnostic) + out["signal"] = dict(out["signal"]) + transition = out["signal"].pop("transition_report") + out["signal"]["transition_report_head"] = transition.head(20).to_dict(orient="records") + return out + + +def _json_default(value): + if isinstance(value, (np.integer,)): + return int(value) + if isinstance(value, (np.floating,)): + return float(value) + if isinstance(value, (np.bool_,)): + return bool(value) + if isinstance(value, pd.Timestamp): + return value.isoformat() + raise TypeError(f"{type(value).__name__} is not JSON serializable") + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rows", type=int, default=300) + parser.add_argument("--json-out", type=Path, default=PACKAGE_DIR / "benchmarks" / "pct_equity_nautilus_smoke.json") + parser.add_argument("--md-out", type=Path, default=PACKAGE_DIR / "benchmarks" / "pct_equity_nautilus_smoke.md") + args = parser.parse_args(argv) + report = run_smoke(rows=args.rows) + args.json_out.parent.mkdir(parents=True, exist_ok=True) + args.md_out.parent.mkdir(parents=True, exist_ok=True) + args.json_out.write_text(json.dumps(report, indent=2, default=_json_default) + "\n", encoding="utf-8") + args.md_out.write_text(make_markdown(report), encoding="utf-8") + print(make_markdown(report)) + return 0 if report["status"] == "pass" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_pct_equity_nautilus_smoke.py b/tests/test_pct_equity_nautilus_smoke.py new file mode 100644 index 0000000..73aeeb4 --- /dev/null +++ b/tests/test_pct_equity_nautilus_smoke.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + +from quantbt.adapters.nautilus import NautilusBacktestEngine + + +def _load_smoke_module(): + path = Path(__file__).resolve().parents[1] / "benchmarks" / "run_pct_equity_nautilus_smoke.py" + spec = importlib.util.spec_from_file_location("pct_equity_nautilus_smoke", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_pct_equity_nautilus_smoke_explains_aligned_and_user_like_diff(): + try: + NautilusBacktestEngine.check_available() + except ImportError as exc: + pytest.skip(f"NautilusTrader is not installed: {exc}") + + smoke = _load_smoke_module() + report = smoke.run_smoke(rows=300) + scenarios = {item["name"]: item for item in report["scenarios"]} + aligned = scenarios["aligned_fee_no_funding_no_slippage"] + mismatch = scenarios["user_like_mismatch"] + + assert report["status"] == "pass" + assert abs(aligned["final_equity_diff"]) < 1.0 + assert aligned["native"]["num_trades"] == aligned["nautilus"]["num_trades"] + assert aligned["diagnostic"]["checks"]["fee_convention_matches_native"] is True + assert aligned["diagnostic"]["checks"]["funding_matches_native"] is True + assert aligned["diagnostic"]["checks"]["slippage_matches_native"] is True + + assert mismatch["diagnostic"]["checks"]["fee_convention_matches_native"] is False + assert mismatch["diagnostic"]["checks"]["funding_matches_native"] is False + assert mismatch["diagnostic"]["checks"]["slippage_matches_native"] is False + assert mismatch["diagnostic"]["checks"]["custom_fee_rate_applied_to_nautilus"] is False + assert mismatch["diagnostic"]["checks"]["custom_slippage_applied_to_nautilus"] is False