From 9bbcf756633a41bd44ede58ae53547d2c565465e Mon Sep 17 00:00:00 2001 From: yebof Date: Thu, 16 Jul 2026 15:29:30 +0800 Subject: [PATCH 1/8] =?UTF-8?q?feat(cash):=20idle-cash=20sweep=20=E2=80=94?= =?UTF-8?q?=20park=20excess=20cash=20in=20SGOV,=20release=20on=20demand?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2026-07-16 forensics: the account sat at ~84% idle cash for weeks (~$8.9k cash drag over the quarter, ~$300/mo of forgone risk-free carry). This adds a deterministic, zero-LLM sweep with a strict cash-equivalence contract: - CashSweeper (src/execution/cash_sweep.py): park_excess() buys the vehicle with cash above a reserve, minus open-BUY-order holds (Alpaca's cash field doesn't subtract them — sweeping that cash would starve pending fills; unknowable holds → park nothing). fund_buys() releases exactly enough parked cash before the BUY phase, via the same _submit_protected_sell + _finalize_pending_protections discipline as every other SELL path. - The vehicle is hidden from every LLM view (PM decision stage, position reviewer, evening builders) and its market value is credited as CASH in _filter_hard_risk_decisions — parked cash can never block a real BUY via the net-exposure/cash rules. - force_delever sells it FIRST (tier -1); _reconcile_stop_coverage exempts it (deliberately stopless); SWEEP_BUY/SWEEP_SELL action names keep it out of every grading/calibration consumer. - config: cash_sweep section (enabled/symbol/reserve_pct/min_order_usd), default-disabled for backwards compat; production settings.yaml enables SGOV with 1% reserve. - broker.open_buy_notional(): None-vs-0.0 distinction so a failed order query reads as 'unknowable', never as 'no pending buys'. Also pins QUANT_AGENT_MAX_RETRIES in the retry-deadline test (was sensitive to ambient env overrides). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PQkESoSTYx2bCy7WYnTPXR --- config/settings.yaml | 12 ++ src/config.py | 39 +++++ src/execution/broker.py | 53 +++++++ src/execution/cash_sweep.py | 285 ++++++++++++++++++++++++++++++++++ src/pipeline.py | 79 +++++++++- src/pipeline_stages.py | 46 ++++++ tests/test_base_agent.py | 3 + tests/test_cash_sweep.py | 299 ++++++++++++++++++++++++++++++++++++ 8 files changed, 815 insertions(+), 1 deletion(-) create mode 100644 src/execution/cash_sweep.py create mode 100644 tests/test_cash_sweep.py diff --git a/config/settings.yaml b/config/settings.yaml index 6996bd32..1f2a11ad 100644 --- a/config/settings.yaml +++ b/config/settings.yaml @@ -49,6 +49,18 @@ risk: # a mandatory de-lever directive when cash is already negative. allow_margin: false +# Idle-cash sweep — park cash above the reserve in a T-bill ETF, release it +# automatically before BUYs. The vehicle is cash-equivalent everywhere: +# hidden from every LLM view, counted as cash by the risk engine, stopless +# by design, first to liquidate in force_delever. Deterministic, zero-LLM. +# (2026-07-16 forensics: ~84% idle cash for weeks = ~$300/mo risk-free carry +# left on the table; see src/execution/cash_sweep.py.) +cash_sweep: + enabled: true + symbol: "SGOV" # iShares 0-3mo T-bills; BIL is the fallback choice + reserve_pct: 1.0 # % of equity kept as raw cash for fees/slippage + min_order_usd: 500 # don't churn sub-$500 parking orders + trading: universe: # Index ETF diff --git a/src/config.py b/src/config.py index 2260f672..36872cfb 100644 --- a/src/config.py +++ b/src/config.py @@ -126,6 +126,42 @@ class RiskConfig(BaseModel): allow_margin: bool = False +class CashSweepConfig(BaseModel): + """Idle-cash sweep into a T-bill ETF (default SGOV). + + The sweep vehicle is treated as CASH-EQUIVALENT everywhere: excluded + from every LLM-facing position view, excluded from risk-engine exposure + math (its market value counts toward cash in the cash_only filter), + exempt from stop-coverage audits (it deliberately carries no stop), and + force_delever liquidates it FIRST. Deterministic and zero-LLM — the + LLM never decides to park or unpark; the pipeline bookends do. + """ + enabled: bool = False + """Master switch. False = the sweeper is inert everywhere (no view + filtering, no funding sells, no parking buys).""" + + symbol: str = "SGOV" + """The parking vehicle. Must be a cash-like T-bill ETF (SGOV/BIL); + anything with real market beta breaks the cash-equivalence assumption + that justifies every exemption listed above.""" + + reserve_pct: float = Field(default=1.0, ge=0, le=20) + """% of equity kept as raw cash (fees, slippage, partial fills). + Excess above the reserve is parked.""" + + min_order_usd: float = Field(default=500.0, ge=0) + """Don't churn sub-$500 parking orders — spread + noise beat the + few cents of yield.""" + + @field_validator("symbol") + @classmethod + def _symbol_nonempty(cls, v: str) -> str: + v = (v or "").strip().upper() + if not v: + raise ValueError("cash_sweep.symbol must be a non-empty ticker") + return v + + class ScheduleConfig(BaseModel): earnings_preprocess: str = "08:00" morning: str @@ -243,6 +279,9 @@ class AppConfig(BaseModel): trading: TradingConfig storage: StorageConfig evolution: EvolutionConfig = Field(default_factory=EvolutionConfig) + # Optional section — a settings.yaml without it gets a disabled sweeper + # (enabled=False default), so older configs keep working unchanged. + cash_sweep: CashSweepConfig = Field(default_factory=CashSweepConfig) @model_validator(mode="after") def _check_llm_provider_keys(self): diff --git a/src/execution/broker.py b/src/execution/broker.py index 194da77e..fc94d33b 100644 --- a/src/execution/broker.py +++ b/src/execution/broker.py @@ -762,6 +762,59 @@ def cancel_open_entry_orders(self) -> int: logger.warning("Failed to cancel open entry orders: %s", exc) return 0 + def open_buy_notional(self) -> float | None: + """Dollar notional of all OPEN BUY orders, or None when the query fails. + + Used by the cash sweeper: Alpaca's `cash` field does not subtract + open-order holds, so parking must leave room for still-working BUY + limits. The None-vs-0.0 distinction matters — a transient API failure + must read as "unknowable" (caller skips parking), never as "no + pending buys" (caller would sweep cash a pending fill needs). + """ + try: + from alpaca.trading.requests import GetOrdersRequest + + orders = self.client.get_orders( + filter=GetOrdersRequest( + status=QueryOrderStatus.OPEN, + side=OrderSide.BUY, + nested=True, + ) + ) + total = 0.0 + for order in orders or []: + order_side = getattr(getattr(order, "side", None), "value", getattr(order, "side", "")) + if str(order_side).lower() != "buy": + continue + try: + qty = float(getattr(order, "qty", 0) or 0) + except (TypeError, ValueError): + qty = 0.0 + price = None + for attr in ("limit_price", "stop_price"): + raw = getattr(order, attr, None) + if raw is not None: + try: + candidate = float(raw) + except (TypeError, ValueError): + continue + if candidate > 0: + price = candidate + break + if price is None: + # Market order with no price attached — estimate from the + # live quote; on failure treat the whole answer as + # unknowable rather than under-counting the hold. + live = self.get_latest_price(getattr(order, "symbol", "")) + if not live or live <= 0: + return None + price = live + total += qty * price + return total + except Exception as exc: + logger.warning("open_buy_notional query failed: %s", exc) + return None + def list_recent_orders( self, symbol: str, side: str, after, ) -> list[dict] | None: diff --git a/src/execution/cash_sweep.py b/src/execution/cash_sweep.py new file mode 100644 index 00000000..a9b4d78c --- /dev/null +++ b/src/execution/cash_sweep.py @@ -0,0 +1,285 @@ +"""Idle-cash sweep: park excess cash in a T-bill ETF, release it on demand. + +Motivation (2026-07-16 forensics): the account sat at ~84% idle cash for +weeks while short-dated T-bills yielded 4%+. On a ~$100k book that is +~$300/month of risk-free carry left on the table — and unlike everything +else in this system, capturing it requires no forecast at all. + +Design contract (mirrors CLAUDE.md 金额/仓位语义): + +1. The sweep vehicle (default SGOV) is CASH-EQUIVALENT, never a position: + - excluded from every LLM-facing view (PM / position_reviewer / evening + builders) — the LLM never reasons about it, never sells it, never + counts it toward exposure; + - its market value counts as CASH in `_filter_hard_risk_decisions` + (cash_only) and is excluded from net-exposure math, so parked cash can + never block a legitimate BUY; + - exempt from `_reconcile_stop_coverage` (it deliberately carries no + protective stop — a T-bill ladder gapping 5% is not a scenario stops + defend against); + - `_force_delever` liquidates it FIRST (before any real long) when the + account drifts into margin. + +2. Deterministic and zero-LLM. Two bookend operations: + - `fund_buys(ctx, planned_notional)` — before the BUY phase, sell just + enough of the vehicle that raw cash covers the planned notional; + - `park_excess(ctx)` — after a session's trading completes, buy the + vehicle with cash above the configured reserve, minus the notional of + any still-open BUY orders (Alpaca's `cash` does not subtract open-order + holds; sweeping that cash would starve pending fills). + +3. SELL discipline: funding sells go through + `pipeline._submit_protected_sell` + `_finalize_pending_protections`, + exactly like FORCE_DELEVER — the vehicle has no stops so the + cancel/restore halves are no-ops, but the WAL bookkeeping stays uniform + with every other SELL path (a future stop on the vehicle would be + handled instead of orphaned). + +4. Ledger isolation: trades are recorded as SWEEP_BUY / SWEEP_SELL. Those + action names are deliberately ABSENT from every action-tuple consumer + (evening grading, calibration, recent-sells builders), so parking churn + never pollutes the learning loops. + +Failure posture: every operation is best-effort and conservative. Any +uncertainty (broker query failed, non-finite numbers, open-order notional +unknowable) resolves to "do nothing this session" — an unswept dollar +costs basis points; an over-swept dollar can reject a real trade. +""" +import logging +import math + +logger = logging.getLogger(__name__) + +# Raw-cash cushion added on top of planned BUY notional when deciding how +# much of the vehicle to liquidate — covers limit-price drift between +# sizing and fill. Generous is fine: leftover cash is re-parked at the +# session bookend. +_FUND_BUFFER_FRAC = 0.01 +_FUND_BUFFER_MIN_USD = 50.0 + +# Limit-price paddings. The vehicle trades at ~1bp spreads; ±0.1% crosses +# the book immediately while still capping a pathological fill. +_BUY_LIMIT_PAD = 1.001 +_SELL_LIMIT_PAD = 0.999 + + +class CashSweeper: + """Pipeline-owned helper; all broker/DB access goes through `pipeline`.""" + + def __init__(self, *, pipeline): + self._pipeline = pipeline + + # ---------- config / views ---------- + + @property + def _cfg(self): + return getattr(getattr(self._pipeline, "config", None), "cash_sweep", None) + + def enabled(self) -> bool: + # `is True` (not truthiness): tests stub pipeline.config with + # MagicMock, whose auto-created attributes are truthy — a sweeping + # MagicMock must read as DISABLED, never as configured-on. + cfg = self._cfg + return cfg is not None and getattr(cfg, "enabled", False) is True + + @property + def symbol(self) -> str | None: + cfg = self._cfg + return getattr(cfg, "symbol", None) if cfg is not None else None + + def split_positions(self, positions): + """(investable_positions, parked_position_or_None). + + The investable list is what every LLM view and the risk engine + should see; `parked` is the sweep-vehicle position when held. + Disabled sweeper → passthrough (positions, None). + """ + if not self.enabled() or not positions: + return positions, None + sym = self.symbol + investable = [p for p in positions if getattr(p, "symbol", None) != sym] + parked = next((p for p in positions if getattr(p, "symbol", None) == sym), None) + return investable, parked + + def parked_value(self, positions) -> float: + """Market value of the parked vehicle (0.0 when none / non-finite).""" + _, parked = self.split_positions(positions) + if parked is None: + return 0.0 + mv = getattr(parked, "market_value", 0.0) + try: + mv = float(mv) + except (TypeError, ValueError): + return 0.0 + return mv if math.isfinite(mv) and mv > 0 else 0.0 + + def reserve_usd(self, total_value: float) -> float: + cfg = self._cfg + if cfg is None or not math.isfinite(total_value) or total_value <= 0: + return 0.0 + return total_value * cfg.reserve_pct / 100.0 + + # ---------- funding (un-park before BUYs) ---------- + + def fund_buys(self, ctx, planned_notional: float) -> float: + """Sell enough of the vehicle that raw cash covers `planned_notional`. + + Returns the estimated dollars freed (0.0 when nothing was done). + Refreshes ctx.positions / ctx.cash / ctx.total_value from the broker + after a fill so the BUY phase runs on truth. + """ + if not self.enabled(): + return 0.0 + if not math.isfinite(planned_notional) or planned_notional <= 0: + return 0.0 + _, parked = self.split_positions(ctx.positions) + if parked is None or parked.qty <= 0: + return 0.0 + + buffer_usd = max(_FUND_BUFFER_MIN_USD, planned_notional * _FUND_BUFFER_FRAC) + cash = ctx.cash if math.isfinite(ctx.cash) else 0.0 + needed = planned_notional + buffer_usd - cash + if needed <= 0: + return 0.0 + + price = parked.current_price + if not (isinstance(price, (int, float)) and math.isfinite(price) and price > 0): + logger.warning("cash sweep: no usable price for %s — skipping funding sell", + parked.symbol) + return 0.0 + + qty = math.ceil(needed / price) + full_exit = qty >= parked.qty + if full_exit: + qty = self._pipeline._full_sell_qty(parked.qty) + if qty is None: + return 0.0 + + sell_limit = round(price * _SELL_LIMIT_PAD, 2) + sale = self._pipeline._submit_protected_sell( + symbol=parked.symbol, qty=qty, limit_price=sell_limit, + reference_price=price, position_qty_before_sell=parked.qty, + label="SWEEP_SELL", + ) + if sale is None: + return 0.0 + order, prot = sale + try: + self._pipeline.db.insert_trade( + symbol=parked.symbol, action="SWEEP_SELL", qty=qty, price=price, + reasoning=( + f"cash sweep: releasing parked cash to fund " + f"${planned_notional:,.0f} of planned BUYs " + f"(cash=${cash:,.0f}, buffer=${buffer_usd:,.0f})" + ), + run_id=ctx.run_id, broker_order_id=order.get("id"), + fill_status="submitted", + ) + except Exception as e: # noqa: BLE001 — ledger failure must not strand the fill wait + logger.warning("cash sweep: insert_trade failed for SWEEP_SELL: %s", e) + + # Block until terminal + finalize protection bookkeeping (no-op for a + # stopless vehicle, but keeps the SELL discipline uniform). + self._pipeline._finalize_pending_protections([prot], context="CASH SWEEP") + + freed = qty * price + try: + account = self._pipeline.broker.get_account() + ctx.positions = self._pipeline.broker.get_positions() + ctx.cash = account["cash"] + ctx.total_value = account["portfolio_value"] + logger.info( + "cash sweep: released ~$%.0f from %s (%s sh) — post-refresh " + "cash=$%.2f", freed, parked.symbol, + self._pipeline._format_qty(qty), ctx.cash, + ) + except Exception as e: # noqa: BLE001 + logger.warning("cash sweep: broker refresh after funding sell failed: %s", e) + return freed + + # ---------- parking (after a session's trading is done) ---------- + + def park_excess(self, ctx) -> dict | None: + """Buy the vehicle with cash above the reserve. Returns the order + dict (with action=SWEEP_BUY) or None when nothing was parked. + + Always refreshes account state from the broker first — callers run + this after an execution phase whose local cash bookkeeping is stale. + """ + if not self.enabled(): + return None + pipeline = self._pipeline + try: + account = pipeline.broker.get_account() + positions = pipeline.broker.get_positions() + except Exception as e: # noqa: BLE001 + logger.warning("cash sweep: account refresh failed — skipping park: %s", e) + return None + cash = account.get("cash") + total_value = account.get("portfolio_value") + if not (isinstance(cash, (int, float)) and math.isfinite(cash)): + return None + if not (isinstance(total_value, (int, float)) and math.isfinite(total_value)): + return None + ctx.positions = positions + ctx.cash = cash + ctx.total_value = total_value + + # Alpaca's `cash` does not subtract open-order holds. Sweeping cash + # that a pending BUY limit needs would make its fill reject later. + # Unknowable pending notional (query failure) → park nothing. + pending = pipeline.broker.open_buy_notional() + if pending is None: + logger.warning("cash sweep: open-order query failed — skipping park " + "(conservative: unknown pending BUY holds)") + return None + + cfg = self._cfg + excess = cash - self.reserve_usd(total_value) - pending + if excess < cfg.min_order_usd: + logger.info( + "cash sweep: nothing to park (cash=$%.0f, reserve=$%.0f, " + "pending BUYs=$%.0f, min order=$%.0f)", + cash, self.reserve_usd(total_value), pending, cfg.min_order_usd, + ) + return None + + price = pipeline.broker.get_latest_price(cfg.symbol) + if not (isinstance(price, (int, float)) and price > 0 and math.isfinite(price)): + logger.warning("cash sweep: no price for %s — skipping park", cfg.symbol) + return None + qty = int(excess / price) + if qty <= 0: + return None + + limit_price = round(price * _BUY_LIMIT_PAD, 2) + # Write-ahead row before the broker call — same crash-recovery + # pattern as ExecutionStage BUYs (orphan sweep matches on + # fill_status='pending_submit'). + pending_row_id = pipeline.db.insert_trade( + symbol=cfg.symbol, action="SWEEP_BUY", qty=qty, price=limit_price, + reasoning=( + f"cash sweep: parking idle cash (cash=${cash:,.0f}, " + f"reserve=${self.reserve_usd(total_value):,.0f}, " + f"pending BUYs=${pending:,.0f})" + ), + run_id=ctx.run_id, broker_order_id=None, + fill_status="pending_submit", + ) + order = pipeline.broker.submit_order( + symbol=cfg.symbol, qty=qty, side="buy", + limit_price=limit_price, + stop_loss_price=None, # cash-equivalent: deliberately stopless + reference_price=price, + ) + if not pipeline._order_accepted(order, cfg.symbol, "buy"): + pipeline.db.mark_trade_submit_failed(pending_row_id) + return None + pipeline.db.confirm_trade_submitted(pending_row_id, broker_order_id=order.get("id")) + if isinstance(order, dict): + order.setdefault("action", "SWEEP_BUY") + logger.info( + "cash sweep: parked ~$%.0f into %s (%d sh @ limit $%.2f)", + qty * price, cfg.symbol, qty, limit_price, + ) + return order diff --git a/src/pipeline.py b/src/pipeline.py index 78ff171d..5c853135 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -348,6 +348,27 @@ def _key_for(model: str) -> str: self.decision_stage = DecisionStage(pipeline=self) self.risk_stage = RiskStage(pipeline=self) self.execution_stage = ExecutionStage(pipeline=self) + # Idle-cash sweeper (SGOV parking). All consumers access it through + # self._sweeper() so tests that build the pipeline via __new__ (no + # __init__) degrade to a disabled sweeper instead of AttributeError. + from src.execution.cash_sweep import CashSweeper + self.cash_sweeper = CashSweeper(pipeline=self) + + def _sweeper(self): + """The cash sweeper, or None when absent/disabled. + + getattr-guarded because ~58 tests build TradingPipeline via + __new__() without __init__ — for them (and for enabled=False + configs) every sweep hook must be a structural no-op. + """ + from src.execution.cash_sweep import CashSweeper + sweeper = getattr(self, "cash_sweeper", None) + if not isinstance(sweeper, CashSweeper): + return None + try: + return sweeper if sweeper.enabled() else None + except Exception: # noqa: BLE001 — a broken config must not take down a session + return None @staticmethod def _format_qty(qty: float) -> str: @@ -443,6 +464,19 @@ def _filter_hard_risk_decisions( pending_symbol_investment: dict[str, float] = {} pending_cash_outflow = 0.0 + # Cash-sweep view: the parked T-bill vehicle is cash-equivalent — + # exclude it from the position list (net-exposure / cluster math must + # not count parked cash as market exposure) and credit its market + # value to the cash budget (ExecutionStage liquidates it before BUYs + # submit, mirroring how SELL proceeds are pre-credited below). + sweeper = self._sweeper() + if sweeper is not None: + positions, parked = sweeper.split_positions(positions) + if parked is not None and cash is not None: + mv = parked.market_value + if isinstance(mv, (int, float)) and math.isfinite(mv) and mv > 0: + cash = cash + mv + # Pre-pass: sum the cash SELLs in this session will return. The # execution stage always runs SELLs before BUYs and waits for fills, # so by the time a BUY submits, `cash + sell_proceeds` is available. @@ -759,6 +793,8 @@ def _reconcile_stop_coverage(self) -> list[dict]: gaps: list[dict] = [] longs_checked = 0 + sweeper = self._sweeper() + sweep_symbol = sweeper.symbol if sweeper is not None else None for p in positions: symbol = getattr(p, "symbol", None) try: @@ -769,6 +805,11 @@ def _reconcile_stop_coverage(self) -> list[dict]: # hedges have their own handling. Skip symbols the drain already owns. if not symbol or qty <= 0 or symbol in pending_syms: continue + # The cash-sweep vehicle is deliberately stopless (cash-equivalent; + # see src/execution/cash_sweep.py) — flagging it every session + # would train the operator to ignore the 🔴 banner. + if sweep_symbol is not None and symbol == sweep_symbol: + continue longs_checked += 1 try: _ok, specs = self.broker.snapshot_protective_stops(symbol) @@ -4711,6 +4752,10 @@ def _force_delever(self, ctx: RunContext) -> list[dict]: # risk-wise they're opposite. # # Tier key (lower = sells earlier): + # -1 → cash-sweep vehicle (parked T-bills ARE cash — always the + # first thing to liquidate; selling anything else first would + # realize market risk to cover a deficit that parked cash + # can cover for free) # 0 → long (effective_mul > 0) # 1 → inverse-ETF hedge (effective_mul < 0) # Within each tier, classic biggest-loser-first ordering: @@ -4718,7 +4763,11 @@ def _force_delever(self, ctx: RunContext) -> list[dict]: # - then larger market_value (clear deficit in fewer orders) # - then symbol alphabetical (deterministic across runs) from src.risk.rules import _effective_multiplier + sweeper = self._sweeper() + sweep_symbol = sweeper.symbol if sweeper is not None else None def _tier(p): + if sweep_symbol is not None and p.symbol == sweep_symbol: + return -1 return 0 if _effective_multiplier(p.symbol) > 0 else 1 targets = sorted( sellable, @@ -4951,6 +5000,18 @@ def run_morning(self) -> dict: # Phase 4 #1: execution stage — HOLDs logged, SELLs then BUYs submitted. orders = self._execution_stage(ctx) + # Bookend: park idle cash above the reserve into the sweep vehicle. + # After the BUY phase so open BUY limits are subtracted from the + # parkable excess (see CashSweeper.park_excess). + sweeper = self._sweeper() + if sweeper is not None: + try: + sweep_order = sweeper.park_excess(ctx) + if sweep_order: + orders.append(sweep_order) + except Exception as e: + logger.warning("cash sweep: park_excess failed (non-fatal): %s", e) + logger.info("=== Morning run complete: %d orders executed ===", len(orders)) return { "status": "executed", "orders": orders, "run_id": run_id, @@ -5262,7 +5323,16 @@ def run_position_review(self, session_type: str = "midday") -> dict: # Pre-LLM orders (take-profit + ex-div) feed into the same bucket. orders = list(auto_tp_orders) + list(exdiv_orders) - if positions: + # LLM view: the cash-sweep vehicle is cash, not a position — the + # reviewer must never see it, hold-grade it, or sell it. Raw + # `positions` stays in scope for the paths that need broker truth + # (emergency liquidate below sells EVERYTHING, parked cash included). + review_positions = positions + sweeper = self._sweeper() + if sweeper is not None: + review_positions, _parked = sweeper.split_positions(positions) + + if review_positions: # Sweep any straggler fills before building the reviewer prompt. # run_morning's final reconcile is run_id-scoped, so a BUY whose # fill landed AFTER morning's wait window stays at fill_status= @@ -5685,6 +5755,13 @@ def run_evening(self) -> dict: ctx.last_equity = last_equity ctx.daily_pnl = daily_pnl + # LLM view: hide the cash-sweep vehicle from evening's position + # narratives (facts / thesis-health / missed-ops held-set) — parked + # T-bills have no thesis to review. ctx keeps broker truth. + sweeper = self._sweeper() + if sweeper is not None: + positions, _parked = sweeper.split_positions(positions) + # Sweep submitted orders before building the evening prompt so # canceled/expired orders do not get narrated as real trades, and # partial terminal fills are reflected in the trade list. diff --git a/src/pipeline_stages.py b/src/pipeline_stages.py index 4b864783..022b23d1 100644 --- a/src/pipeline_stages.py +++ b/src/pipeline_stages.py @@ -384,6 +384,23 @@ def run(self, ctx: RunContext) -> RunContext: cash = ctx.cash last_equity = ctx.last_equity + # Cash-sweep view for the PM: the parked T-bill vehicle is presented + # as CASH, not as a position — PM sizes deployment against + # cash + parked (ExecutionStage liquidates the vehicle before BUYs + # submit), and never reasons about the vehicle itself. + # isinstance guard: stage tests stub `pipeline` with MagicMock, whose + # auto-attrs would otherwise duck-type as an enabled sweeper. + from src.execution.cash_sweep import CashSweeper + sweeper = getattr(pipeline, "_sweeper", None) + sweeper = sweeper() if callable(sweeper) else None + if isinstance(sweeper, CashSweeper): + positions, parked = sweeper.split_positions(positions) + if parked is not None: + import math as _math + mv = parked.market_value + if isinstance(mv, (int, float)) and _math.isfinite(mv) and mv > 0: + cash = cash + mv + yesterday_insights = pipeline.db.get_latest_insights(before_date=session_date_key()) recent_performance = pipeline._compute_recent_performance(last_equity) if yesterday_insights: @@ -864,6 +881,35 @@ def run(self, ctx: RunContext) -> list[dict]: ) buy_decisions = [] + # Cash-sweep funding: the risk filter counted the parked T-bill + # vehicle's value as cash (cash-equivalent contract), so BUYs that + # passed it may exceed RAW cash. Release just enough parked cash + # to cover the planned notional before the BUY loop sizes against + # `available_cash`. Waits for the fill and refreshes ctx. + # isinstance guard: stage tests stub `pipeline` with MagicMock. + if buy_decisions: + from src.execution.cash_sweep import CashSweeper + sweeper = getattr(pipeline, "_sweeper", None) + sweeper = sweeper() if callable(sweeper) else None + if not isinstance(sweeper, CashSweeper): + sweeper = None + if sweeper is not None: + planned_notional = sum( + total_value * d.allocation_pct / 100.0 + for d in buy_decisions + if d.allocation_pct > 0 + ) + try: + freed = sweeper.fund_buys(ctx, planned_notional) + except Exception as e: + logger.warning("cash sweep: fund_buys failed (BUYs will " + "use raw cash only): %s", e) + freed = 0.0 + if freed > 0: + positions = ctx.positions + cash = ctx.cash + total_value = ctx.total_value + available_cash = cash for decision in buy_decisions: if decision.action != "BUY": diff --git a/tests/test_base_agent.py b/tests/test_base_agent.py index 42d5fea7..6effeb68 100644 --- a/tests/test_base_agent.py +++ b/tests/test_base_agent.py @@ -955,6 +955,9 @@ def test_retry_deadline_abandons_primary_for_failover(monkeypatch): ticks = [0.0, 200.0, 600.0] monkeypatch.setattr("src.agents.base.time.monotonic", lambda: ticks.pop(0) if ticks else 600.0) + # Pin the budget: the assertion below counts attempts, so an ambient + # QUANT_AGENT_MAX_RETRIES override must not change the arithmetic. + monkeypatch.setenv("QUANT_AGENT_MAX_RETRIES", "7") oai = MagicMock() oai.chat.completions.create.side_effect = ConnectionError("relay 524 storm") anth = MagicMock() diff --git a/tests/test_cash_sweep.py b/tests/test_cash_sweep.py new file mode 100644 index 00000000..5976d1ad --- /dev/null +++ b/tests/test_cash_sweep.py @@ -0,0 +1,299 @@ +"""Idle-cash sweep (SGOV parking) invariants. + +The sweep vehicle is CASH-EQUIVALENT everywhere: + 1. Hidden from LLM views (split_positions), counted as cash by the risk + filter, excluded from net-exposure math. + 2. force_delever liquidates it FIRST (tier -1, before real longs). + 3. _reconcile_stop_coverage never flags it (deliberately stopless). + 4. fund_buys releases exactly enough parked cash before the BUY phase; + park_excess parks only cash above reserve + open-BUY holds. + 5. Disabled / unconfigured / MagicMock'd pipelines are structural no-ops. +""" +from types import SimpleNamespace +from unittest.mock import MagicMock + +from src.config import CashSweepConfig +from src.execution.cash_sweep import CashSweeper +from src.models import Position, TradeDecision +from src.pipeline import TradingPipeline +from src.pipeline_context import RunContext +from src.risk.rules import RiskRuleEngine +from src.config import RiskConfig + + +SGOV = Position(symbol="SGOV", qty=800, avg_entry=100.5, current_price=100.6, + market_value=80_480, unrealized_pnl=80, sector="Unknown") +NVDA = Position(symbol="NVDA", qty=10, avg_entry=900, current_price=950, + market_value=9_500, unrealized_pnl=500, sector="Technology") + + +def _sweep_pipeline(enabled=True, reserve_pct=1.0, min_order_usd=500.0): + pipeline = TradingPipeline.__new__(TradingPipeline) + pipeline.config = SimpleNamespace( + cash_sweep=CashSweepConfig( + enabled=enabled, symbol="SGOV", + reserve_pct=reserve_pct, min_order_usd=min_order_usd, + ), + risk=RiskConfig( + max_position_pct=20, max_total_position_pct=90, + max_daily_loss_pct=3, max_sector_pct=40, + require_stop_loss=True, allow_margin=False, + ), + ) + pipeline.broker = MagicMock() + pipeline.db = MagicMock() + pipeline.cash_sweeper = CashSweeper(pipeline=pipeline) + pipeline.risk_engine = RiskRuleEngine(pipeline.config.risk) + return pipeline + + +# ---------- views ---------- + +def test_split_positions_hides_vehicle(): + p = _sweep_pipeline() + investable, parked = p.cash_sweeper.split_positions([SGOV, NVDA]) + assert [x.symbol for x in investable] == ["NVDA"] + assert parked is not None and parked.symbol == "SGOV" + assert p.cash_sweeper.parked_value([SGOV, NVDA]) == SGOV.market_value + + +def test_split_positions_passthrough_when_disabled(): + p = _sweep_pipeline(enabled=False) + investable, parked = p.cash_sweeper.split_positions([SGOV, NVDA]) + assert investable == [SGOV, NVDA] + assert parked is None + assert p._sweeper() is None + + +def test_sweeper_none_for_bare_new_pipeline(): + """__new__-built pipelines (no cash_sweeper attr) degrade to disabled.""" + pipeline = TradingPipeline.__new__(TradingPipeline) + assert pipeline._sweeper() is None + + +def test_magicmock_config_reads_as_disabled(): + """MagicMock auto-attrs are truthy — enabled() must use `is True`.""" + pipeline = TradingPipeline.__new__(TradingPipeline) + pipeline.config = MagicMock() + pipeline.cash_sweeper = CashSweeper(pipeline=pipeline) + assert pipeline.cash_sweeper.enabled() is False + assert pipeline._sweeper() is None + + +# ---------- risk filter: parked value is cash, not exposure ---------- + +def _buy(symbol="AAPL", alloc=10.0): + return TradeDecision(action="BUY", symbol=symbol, allocation_pct=alloc, + entry_price=100.0, stop_loss=95.0, take_profit=120.0, + reasoning="test") + + +def test_filter_credits_parked_value_as_cash(): + """A BUY that raw cash can't cover passes when parked SGOV covers it + (ExecutionStage releases the cash before the BUY submits).""" + p = _sweep_pipeline() + # $100k book: $9.5k NVDA, $80.5k SGOV, $1k raw cash. 10% BUY = $10k. + allowed, _, blocked = p._filter_hard_risk_decisions( + [_buy(alloc=10.0)], [SGOV, NVDA], total_value=100_000.0, + daily_pnl=0.0, baseline=100_000.0, cash=1_000.0, + ) + assert [d.symbol for d in allowed] == ["AAPL"] + assert not blocked + + +def test_filter_blocks_same_buy_when_sweep_disabled(): + p = _sweep_pipeline(enabled=False) + allowed, _, blocked = p._filter_hard_risk_decisions( + [_buy(alloc=10.0)], [SGOV, NVDA], total_value=100_000.0, + daily_pnl=0.0, baseline=100_000.0, cash=1_000.0, + ) + assert allowed == [] + assert any("cash" in r for r in blocked) + + +def test_filter_excludes_vehicle_from_net_exposure(): + """80% parked + 9.5% stock must not trip the 90% net-exposure cap for a + new BUY — parked cash is not market exposure.""" + p = _sweep_pipeline() + allowed, _, blocked = p._filter_hard_risk_decisions( + [_buy(alloc=15.0)], [SGOV, NVDA], total_value=100_000.0, + daily_pnl=0.0, baseline=100_000.0, cash=20_000.0, + ) + assert [d.symbol for d in allowed] == ["AAPL"], blocked + + +# ---------- force_delever: vehicle first ---------- + +def test_force_delever_sells_vehicle_before_real_longs(): + p = _sweep_pipeline() + p.broker.submit_order.return_value = {"id": "o1", "status": "accepted"} + p.broker.wait_for_order_terminal.return_value = "filled" + p.broker.snapshot_protective_stops.return_value = (True, []) + p.broker.cancel_snapshotted_stops.return_value = True + p.broker.get_account.return_value = { + "cash": 100.0, "portfolio_value": 90_000.0, "last_equity": 90_000.0, + } + p.broker.get_positions.return_value = [] + p.broker.get_order_fill_info.return_value = {"fill_qty": 800, "status": "filled"} + + ctx = RunContext.start("morning") + ctx.cash = -500.0 + loser = Position(symbol="LOSER", qty=5, avg_entry=300, current_price=250, + market_value=1_250, unrealized_pnl=-250, sector="Tech") + ctx.positions = [loser, SGOV] + + p._force_delever(ctx) + first = p.broker.submit_order.call_args_list[0].kwargs + assert first["symbol"] == "SGOV" # parked cash first, not the loser + + +# ---------- stop-coverage audit exemption ---------- + +def test_reconcile_stop_coverage_skips_vehicle(): + p = _sweep_pipeline() + p.broker.get_positions.return_value = [SGOV] + p.db.get_pending_protection_restores.return_value = [] + # No stops exist for SGOV; without the exemption this would be a gap. + p.broker.snapshot_protective_stops.return_value = (True, []) + gaps = p._reconcile_stop_coverage() + assert gaps == [] + p.broker.snapshot_protective_stops.assert_not_called() + + +# ---------- fund_buys ---------- + +def _funding_pipeline(): + p = _sweep_pipeline() + p._submit_protected_sell = MagicMock(return_value=( + {"id": "sell-1", "status": "accepted"}, + {"symbol": "SGOV", "order_id": "sell-1"}, + )) + p._finalize_pending_protections = MagicMock() + p.broker.get_account.return_value = { + "cash": 50_000.0, "portfolio_value": 100_000.0, + } + p.broker.get_positions.return_value = [NVDA] + return p + + +def test_fund_buys_releases_enough_for_planned_notional(): + p = _funding_pipeline() + ctx = RunContext.start("morning") + ctx.cash = 1_000.0 + ctx.positions = [SGOV, NVDA] + + freed = p.cash_sweeper.fund_buys(ctx, planned_notional=30_000.0) + + assert freed > 0 + kwargs = p._submit_protected_sell.call_args.kwargs + assert kwargs["symbol"] == "SGOV" + assert kwargs["label"] == "SWEEP_SELL" + # needed = 30k + max(50, 1%·30k=300) - 1k = 29.3k → ceil(29300/100.6)=292 + assert kwargs["qty"] == 292 + p._finalize_pending_protections.assert_called_once() + assert ctx.cash == 50_000.0 # refreshed from broker + + +def test_fund_buys_noop_when_cash_already_covers(): + p = _funding_pipeline() + ctx = RunContext.start("morning") + ctx.cash = 50_000.0 + ctx.positions = [SGOV, NVDA] + assert p.cash_sweeper.fund_buys(ctx, planned_notional=10_000.0) == 0.0 + p._submit_protected_sell.assert_not_called() + + +def test_fund_buys_caps_at_full_position(): + """Needing more than parked → full exit via _full_sell_qty, no oversell.""" + p = _funding_pipeline() + ctx = RunContext.start("morning") + ctx.cash = 0.0 + ctx.positions = [SGOV, NVDA] + p.cash_sweeper.fund_buys(ctx, planned_notional=200_000.0) + kwargs = p._submit_protected_sell.call_args.kwargs + assert kwargs["qty"] == SGOV.qty + + +def test_fund_buys_noop_without_vehicle_position(): + p = _funding_pipeline() + ctx = RunContext.start("morning") + ctx.cash = 0.0 + ctx.positions = [NVDA] + assert p.cash_sweeper.fund_buys(ctx, planned_notional=10_000.0) == 0.0 + + +# ---------- park_excess ---------- + +def _parking_pipeline(cash=90_000.0, total=100_000.0, pending=0.0): + p = _sweep_pipeline() + p.broker.get_account.return_value = {"cash": cash, "portfolio_value": total} + p.broker.get_positions.return_value = [NVDA] + p.broker.open_buy_notional.return_value = pending + p.broker.get_latest_price.return_value = 100.60 + p.broker.submit_order.return_value = {"id": "buy-1", "status": "accepted"} + p.db.insert_trade.return_value = 42 + p._order_accepted = MagicMock(return_value=True) + return p + + +def test_park_excess_buys_vehicle_with_idle_cash(): + p = _parking_pipeline(cash=90_000.0, total=100_000.0) + ctx = RunContext.start("morning") + order = p.cash_sweeper.park_excess(ctx) + assert order is not None and order["action"] == "SWEEP_BUY" + kwargs = p.broker.submit_order.call_args.kwargs + assert kwargs["symbol"] == "SGOV" and kwargs["side"] == "buy" + # excess = 90k - 1%·100k - 0 = 89k → int(89000/100.60) = 884 shares + assert kwargs["qty"] == 884 + assert kwargs["stop_loss_price"] is None # deliberately stopless + p.db.confirm_trade_submitted.assert_called_once() + + +def test_park_excess_subtracts_open_buy_holds(): + """Cash reserved by still-open BUY limits must not be swept.""" + p = _parking_pipeline(cash=90_000.0, total=100_000.0, pending=88_600.0) + ctx = RunContext.start("morning") + assert p.cash_sweeper.park_excess(ctx) is None # 90k-1k-88.6k = 400 < 500 + + +def test_park_excess_skips_when_open_orders_unknowable(): + p = _parking_pipeline() + p.broker.open_buy_notional.return_value = None # query failed + ctx = RunContext.start("morning") + assert p.cash_sweeper.park_excess(ctx) is None + p.broker.submit_order.assert_not_called() + + +def test_park_excess_respects_min_order(): + p = _parking_pipeline(cash=1_400.0, total=100_000.0) # excess 400 < 500 + ctx = RunContext.start("morning") + assert p.cash_sweeper.park_excess(ctx) is None + + +def test_park_excess_marks_row_failed_on_reject(): + p = _parking_pipeline() + p._order_accepted = MagicMock(return_value=False) + ctx = RunContext.start("morning") + assert p.cash_sweeper.park_excess(ctx) is None + p.db.mark_trade_submit_failed.assert_called_once_with(42) + p.db.confirm_trade_submitted.assert_not_called() + + +def test_park_excess_disabled_is_inert(): + p = _parking_pipeline() + p.config.cash_sweep = CashSweepConfig(enabled=False) + ctx = RunContext.start("morning") + assert p.cash_sweeper.park_excess(ctx) is None + p.broker.get_account.assert_not_called() + + +# ---------- config ---------- + +def test_cash_sweep_config_defaults_disabled(): + cfg = CashSweepConfig() + assert cfg.enabled is False + assert cfg.symbol == "SGOV" + + +def test_cash_sweep_config_uppercases_symbol(): + assert CashSweepConfig(symbol=" bil ").symbol == "BIL" From f4f5be42c1cb23ab6c66656adb3d28a39b413f7d Mon Sep 17 00:00:00 2001 From: yebof Date: Thu, 16 Jul 2026 15:41:43 +0800 Subject: [PATCH 2/8] =?UTF-8?q?feat(exits):=20deterministic=20exit-quality?= =?UTF-8?q?=20guards=20=E2=80=94=20stop=20the=20winner-whipsaw?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2026-07-16 sell autopsy: 28/53 realized exits since 5/1 were EARLY (stock >=5% higher within 20 days), $8.5k-16k left on the table; 5 trail-stop fills missed avg +30.7% post-exit; LLY was trail-whipsawed twice identically 4 weeks apart. The mechanical share of the leak gets code guards (prompts alone demonstrably didn't hold): - TRAIL_STOP noise-band clamp: a new stop inside 1.25×ATR14 of current price sits inside one day's normal range — rejected, old stop kept. A cited hard trigger bypasses (same philosophy as the same-day-trim gate). Unknowable ATR degrades open (no clamp). - TRAIL_STOP ratchet cooldown: at most one accepted tighten per ~2 trading days per symbol (the reviewer's >=1.02×old_stop min-bump rule made every accepted trail tighten >=2%; GE was ratcheted 325->350 in 8 sessions on one flag). Hard triggers bypass. - Live stop truth in position facts: prefer broker.get_current_stop_price over the stale-wide BUY-row stop (after any trail the reviewer saw a fat distance_to_stop and kept ratcheting), plus atr_pct / stop_distance_atrs so the reviewer reasons in vol units. - Entry ATR floor (ExecutionStage): a BUY stop closer than 1×ATR14 (computed from bars already fetched by research) is widened to 1×ATR; qty_by_risk sizes against the wider distance so per-trade $ risk is unchanged. No bars -> no floor. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PQkESoSTYx2bCy7WYnTPXR --- src/pipeline.py | 106 +++++++++++++++ src/pipeline_stages.py | 41 +++++- tests/test_exit_quality.py | 266 +++++++++++++++++++++++++++++++++++++ 3 files changed, 408 insertions(+), 5 deletions(-) create mode 100644 tests/test_exit_quality.py diff --git a/src/pipeline.py b/src/pipeline.py index 5c853135..459a178b 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -4522,6 +4522,58 @@ def _symbols_already_trimmed_today(self) -> set[str]: out.add(sym) return out + def _atr_for_symbol(self, symbol: str) -> float | None: + """ATR(14) from ~30 days of daily bars; None when unknowable. + + Used by the TRAIL_STOP noise-band clamp and the position-facts + vol-unit metrics. Failure is always None (callers degrade to the + pre-clamp behavior) — never raises. + """ + try: + bars = self.market.get_ohlcv(symbol, 30) or [] + if len(bars) < 15: + return None + from src.data.technical import compute_indicators + atr = compute_indicators(symbol, bars).atr_14 + return float(atr) if atr and atr > 0 else None + except Exception as e: # noqa: BLE001 + logger.warning("ATR fetch failed for %s: %s", symbol, e) + return None + + def _trail_tightened_recently(self, symbol: str, calendar_days: int = 4) -> bool: + """True when a non-canceled TRAIL_STOP for `symbol` landed within the + last `calendar_days` days (~2 trading days across a weekend). + + RC1 forensics (2026-07-16): the reviewer's ≥1.02×old_stop min-bump + rule means every ACCEPTED trail tightens ≥2%; per-session trailing + marched stops into the daily-noise band in 3-4 sessions (GE was + ratcheted 325→350 in 8 sessions on one flag). A cooldown makes + tightening a considered, at-most-every-other-day act. + """ + try: + rows = self.db.get_trades(symbol=symbol, limit=10) + except Exception as e: # noqa: BLE001 + logger.warning("trail cooldown query failed for %s: %s", symbol, e) + return False + from datetime import datetime as _dt, timedelta, timezone + cutoff = _dt.now(timezone.utc) - timedelta(days=calendar_days) + for row in rows: + if (row.get("action") or "").upper() != "TRAIL_STOP": + continue + if (row.get("fill_status") or "") == "canceled": + continue + ts = row.get("timestamp") or "" + try: + dt = _dt.fromisoformat(ts.replace("Z", "+00:00")) if "T" in ts \ + else _dt.strptime(ts, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc) + except (TypeError, ValueError): + continue + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + if dt >= cutoff: + return True + return False + def _midday_execute_llm_actions( self, positions, review, run_id: str, blocked_symbols: set[str] | None = None, already_trimmed_today: set[str] | None = None, @@ -4631,6 +4683,38 @@ def _midday_execute_llm_actions( symbol, new_stop, existing[0].current_price, ) continue + # RC1 exit-quality clamps (2026-07-16 forensics: 5 trail + # fills missed avg +30.7% post-exit; LLY was whipsawed + # twice identically). A hard-trigger citation in the + # reason bypasses both — mirroring the SELL/REDUCE gate. + if not _reason_cites_hard_trigger(action_item.get("reason", "")): + # (a) Ratchet cooldown: at most one accepted tighten + # per ~2 trading days per symbol. + if self._trail_tightened_recently(symbol): + logger.warning( + "Midday: TRAIL_STOP %s skipped — a trail was " + "already tightened within the last 2 trading " + "days (ratchet cooldown; cite a hard trigger " + "to bypass)", symbol, + ) + continue + # (b) Noise-band clamp: a stop inside 1.25×ATR14 of + # the current price sits inside one day's normal + # range — it converts routine volatility into a + # realized exit. Keep the old stop instead. + atr = self._atr_for_symbol(symbol) + if atr is not None: + noise_floor = existing[0].current_price - 1.25 * atr + if new_stop > noise_floor: + logger.warning( + "Midday: TRAIL_STOP %s skipped — new_stop " + "$%.2f is inside the 1.25×ATR noise band " + "(floor $%.2f, ATR14 $%.2f); routine " + "volatility would fill it. Old stop kept; " + "cite a hard trigger to bypass.", + symbol, new_stop, noise_floor, atr, + ) + continue order = self.broker.replace_stop_loss(symbol, new_stop) if order: if isinstance(order, dict): @@ -5067,6 +5151,16 @@ def _build_position_facts(self, positions, morning_trades, total_value, avg_hold stop_loss = float((buy or {}).get("stop_loss") or 0) take_profit = float((buy or {}).get("take_profit") or 0) + # RC1: after any TRAIL_STOP the BUY row's stop is stale-WIDE — + # the reviewer would see a fat distance_to_stop and keep + # ratcheting. Prefer live broker truth; fall back to the BUY row. + try: + live_stop = self.broker.get_current_stop_price(sym) + except Exception: # noqa: BLE001 + live_stop = None + if isinstance(live_stop, (int, float)) and live_stop > 0: + stop_loss = float(live_stop) + # days_held — from BUY timestamp; fall back to None. days_held = None buy_ts = (buy or {}).get("timestamp") @@ -5112,6 +5206,16 @@ def _build_position_facts(self, positions, morning_trades, total_value, avg_hold drift_flag = weight_pct > 12 and pnl_pct > 10 target_breach_flag = progress_pct is not None and progress_pct > 150 + # Vol-unit context so the reviewer reasons about stop distance + # in ATRs, not raw % (a 3% gap is roomy for KO, suicidal for + # RKLB). None when bars are unavailable — the prompt treats + # missing as "unknown", never as zero. + atr = self._atr_for_symbol(sym) + atr_pct = round(atr / cur * 100, 2) if (atr and cur > 0) else None + stop_distance_atrs = None + if atr and stop_loss and cur > stop_loss: + stop_distance_atrs = round((cur - stop_loss) / atr, 2) + facts[sym] = { "days_held": days_held, "thesis_progress_pct": progress_pct, @@ -5122,6 +5226,8 @@ def _build_position_facts(self, positions, morning_trades, total_value, avg_hold "parabolic_flag": parabolic_flag, "drift_flag": drift_flag, "target_breach_flag": target_breach_flag, + "atr_pct": atr_pct, + "stop_distance_atrs": stop_distance_atrs, } return facts diff --git a/src/pipeline_stages.py b/src/pipeline_stages.py index 022b23d1..5bd013b5 100644 --- a/src/pipeline_stages.py +++ b/src/pipeline_stages.py @@ -976,11 +976,42 @@ def run(self, ctx: RunContext) -> list[dict]: ) continue + # RC1: code-enforced ATR stop-distance floor at entry. The + # P1 prompt rule ("fresh-entry stops never tighter than + # 1×ATR") is advisory — LLM output still occasionally lands + # stops inside one day's range, which converts routine + # volatility into a same-week exit. Widen to 1×ATR(14) from + # bars already fetched by research; qty_by_risk below sizes + # against the wider distance, so per-trade $ risk is + # unchanged. No bars → no floor (behavior identical). + stop_price = decision.stop_loss + if stop_price > 0 and sizing_price > stop_price: + try: + bars = ctx.symbols_bars.get(decision.symbol) or [] + atr14 = None + if len(bars) >= 15: + from src.data.technical import compute_indicators + atr14 = compute_indicators(decision.symbol, bars).atr_14 + if atr14 and atr14 > 0 and (sizing_price - stop_price) < atr14: + widened = round(sizing_price - atr14, 2) + logger.warning( + "BUY %s: stop $%.2f is %.2f×ATR from entry " + "$%.2f — widening to $%.2f (1×ATR14=$%.2f " + "floor; qty sizing compensates)", + decision.symbol, stop_price, + (sizing_price - stop_price) / atr14, + sizing_price, widened, atr14, + ) + stop_price = widened + except Exception as e: + logger.warning("ATR stop floor skipped for %s: %s", + decision.symbol, e) + qty_by_alloc = int((total_value * decision.allocation_pct / 100) / sizing_price) qty_by_risk = None RISK_BUDGET_PCT = 0.5 - if decision.stop_loss > 0 and sizing_price > decision.stop_loss: - risk_per_share = sizing_price - decision.stop_loss + if stop_price > 0 and sizing_price > stop_price: + risk_per_share = sizing_price - stop_price if risk_per_share > 0: risk_dollars = total_value * RISK_BUDGET_PCT / 100 qty_by_risk = int(risk_dollars / risk_per_share) @@ -989,7 +1020,7 @@ def run(self, ctx: RunContext) -> list[dict]: "Vol-adjusted sizing for %s: qty_by_alloc=%d → qty_by_risk=%d " "(risk %.2f/share, budget $%.0f = %.1f%% of equity)", decision.symbol, qty_by_alloc, qty_by_risk, - sizing_price - decision.stop_loss, + sizing_price - stop_price, total_value * RISK_BUDGET_PCT / 100, RISK_BUDGET_PCT, ) qty = qty_by_risk @@ -1022,7 +1053,7 @@ def run(self, ctx: RunContext) -> list[dict]: pending_row_id = pipeline.db.insert_trade( symbol=decision.symbol, action="BUY", qty=qty, price=executed_price, reasoning=decision.reasoning, run_id=run_id, - stop_loss=decision.stop_loss, take_profit=decision.take_profit, + stop_loss=stop_price, take_profit=decision.take_profit, broker_order_id=None, fill_status="pending_submit", ) @@ -1031,7 +1062,7 @@ def run(self, ctx: RunContext) -> list[dict]: order = pipeline.broker.submit_order( symbol=decision.symbol, qty=qty, side="buy", limit_price=limit_price, - stop_loss_price=decision.stop_loss if decision.stop_loss > 0 else None, + stop_loss_price=stop_price if stop_price > 0 else None, reference_price=market_price, ) except Exception: diff --git a/tests/test_exit_quality.py b/tests/test_exit_quality.py new file mode 100644 index 00000000..5ec3415b --- /dev/null +++ b/tests/test_exit_quality.py @@ -0,0 +1,266 @@ +"""RC1 exit-quality guards (2026-07-16 forensics). + +The sell autopsy found 28/53 realized exits were EARLY (stock ≥5% higher +within 20 days); 5 trail-stop fills missed an average +30.7% post-exit, and +LLY was trail-whipsawed twice identically in 4 weeks. Three deterministic +guards close the mechanical part: + + 1. Noise-band clamp — a TRAIL_STOP inside 1.25×ATR14 of current price is + rejected (routine volatility would fill it); hard-trigger reasons bypass. + 2. Ratchet cooldown — at most one accepted tighten per ~2 trading days per + symbol; hard-trigger reasons bypass. + 3. Entry ATR floor — a BUY whose stop is closer than 1×ATR14 gets the stop + widened (qty sizing compensates, per-trade $ risk unchanged). + +Plus: position facts must show the LIVE broker stop (post-trail), not the +stale-wide BUY-row stop that kept the ratchet feedback going. +""" +from datetime import date as _date +from unittest.mock import MagicMock + +from src.models import Position, PositionAction, PositionReview, PositionReasoningChain +from src.pipeline import TradingPipeline + + +def _review_rc() -> PositionReasoningChain: + return PositionReasoningChain( + macro_continuity_check="stable", + thesis_progress_check="on pace", + thesis_integrity_check="intact", + winners_discipline_check="no flags", + session_disposition_check="patient", + execution_rationale="n/a", + ) + + +def _mk_pipeline(position: Position) -> TradingPipeline: + pipeline = TradingPipeline.__new__(TradingPipeline) + pipeline.broker = MagicMock() + pipeline.broker.replace_stop_loss.return_value = {"id": "stop-1", "status": "accepted"} + pipeline.db = MagicMock() + pipeline.db.get_trades.return_value = [] + pipeline._format_qty = lambda q: str(q) + return pipeline + + +def _trail_review(symbol: str, new_stop: float, reason: str) -> PositionReview: + return PositionReview( + reasoning_chain=_review_rc(), + actions=[PositionAction(action="TRAIL_STOP", symbol=symbol, + reason=reason, new_stop_price=new_stop)], + overall_assessment="trail", risk_level="low", + ) + + +GE = Position(symbol="GE", qty=26.0, avg_entry=316.0, current_price=360.0, + market_value=9360.0, unrealized_pnl=1144.0, + unrealized_intraday_pnl=0.0, sector="Industrials") + + +def test_trail_inside_noise_band_is_rejected(): + """ATR14=$8 → noise floor = 360 - 1.25*8 = $350. A $355 stop sits inside + one day's range — keep the old stop.""" + pipeline = _mk_pipeline(GE) + pipeline._atr_for_symbol = lambda s: 8.0 + orders = pipeline._midday_execute_llm_actions( + positions=[GE], run_id="r-1", + review=_trail_review("GE", 355.0, "TARGET_BREACH — locking in gains"), + ) + assert orders == [] + pipeline.broker.replace_stop_loss.assert_not_called() + + +def test_trail_outside_noise_band_passes(): + pipeline = _mk_pipeline(GE) + pipeline._atr_for_symbol = lambda s: 8.0 + orders = pipeline._midday_execute_llm_actions( + positions=[GE], run_id="r-1", + review=_trail_review("GE", 344.0, "TARGET_BREACH — locking in gains"), + ) + assert len(orders) == 1 + pipeline.broker.replace_stop_loss.assert_called_once_with("GE", 344.0) + + +def test_trail_hard_trigger_bypasses_noise_clamp(): + """A cited hard trigger (thesis_invalid_if) may tighten into the band — + same bypass philosophy as the SELL/REDUCE same-day-trim gate.""" + pipeline = _mk_pipeline(GE) + pipeline._atr_for_symbol = lambda s: 8.0 + orders = pipeline._midday_execute_llm_actions( + positions=[GE], run_id="r-1", + review=_trail_review("GE", 355.0, + "thesis_invalid_if triggered — guidance withdrawn"), + ) + assert len(orders) == 1 + + +def test_trail_unknowable_atr_degrades_open(): + """No bars → no clamp — blocking ALL trails on a data gap would leave + runners unprotected from legitimate tightening.""" + pipeline = _mk_pipeline(GE) + pipeline._atr_for_symbol = lambda s: None + orders = pipeline._midday_execute_llm_actions( + positions=[GE], run_id="r-1", + review=_trail_review("GE", 355.0, "TARGET_BREACH"), + ) + assert len(orders) == 1 + + +def test_trail_ratchet_cooldown_blocks_repeat_tighten(): + """A non-canceled TRAIL_STOP within the last ~2 trading days blocks a + second soft-reason tighten (GE was ratcheted 7× in 8 sessions).""" + from datetime import datetime, timezone + pipeline = _mk_pipeline(GE) + pipeline._atr_for_symbol = lambda s: 8.0 + pipeline.db.get_trades.return_value = [{ + "action": "TRAIL_STOP", "fill_status": "submitted", + "timestamp": datetime.now(timezone.utc).isoformat(), + }] + orders = pipeline._midday_execute_llm_actions( + positions=[GE], run_id="r-1", + review=_trail_review("GE", 344.0, "TARGET_BREACH — trail up again"), + ) + assert orders == [] + pipeline.broker.replace_stop_loss.assert_not_called() + + +def test_trail_cooldown_ignores_old_and_canceled_rows(): + pipeline = _mk_pipeline(GE) + pipeline._atr_for_symbol = lambda s: 8.0 + pipeline.db.get_trades.return_value = [ + {"action": "TRAIL_STOP", "fill_status": "canceled", + "timestamp": "2026-07-16T14:00:00+00:00"}, + {"action": "TRAIL_STOP", "fill_status": "submitted", + "timestamp": "2026-06-01T14:00:00+00:00"}, # weeks old + ] + assert pipeline._trail_tightened_recently("GE") is False + + +# ---------- live stop reference in position facts ---------- + +def test_position_facts_prefer_live_broker_stop(): + """After a trail to $350, the BUY row still says $300 — the reviewer + must see the live $350 (distance 2.8%), not a stale-wide 16.7%.""" + pipeline = TradingPipeline.__new__(TradingPipeline) + pipeline.broker = MagicMock() + pipeline.broker.get_current_stop_price.return_value = 350.0 + pipeline.db = MagicMock() + pipeline.db.get_symbol_last_buy.return_value = { + "stop_loss": 300.0, "take_profit": 400.0, + "timestamp": "2026-07-01 14:00:00", + } + pipeline._atr_for_symbol = lambda s: 8.0 + facts = pipeline._build_position_facts( + [GE], morning_trades=[], total_value=100_000.0, avg_hold_days=10.0, + ) + dist = facts["GE"]["distance_to_stop_pct"] + assert dist is not None and abs(dist - (360 - 350) / 360 * 100) < 0.01 + # vol-unit context present: (360-350)/8 = 1.25 ATRs + assert facts["GE"]["stop_distance_atrs"] == 1.25 + assert facts["GE"]["atr_pct"] == round(8.0 / 360.0 * 100, 2) + + +def test_position_facts_fall_back_to_buy_row_stop(): + pipeline = TradingPipeline.__new__(TradingPipeline) + pipeline.broker = MagicMock() + pipeline.broker.get_current_stop_price.return_value = None + pipeline.db = MagicMock() + pipeline.db.get_symbol_last_buy.return_value = { + "stop_loss": 300.0, "take_profit": 400.0, + "timestamp": "2026-07-01 14:00:00", + } + pipeline._atr_for_symbol = lambda s: None + facts = pipeline._build_position_facts( + [GE], morning_trades=[], total_value=100_000.0, avg_hold_days=10.0, + ) + dist = facts["GE"]["distance_to_stop_pct"] + assert dist is not None and abs(dist - (360 - 300) / 360 * 100) < 0.01 + assert facts["GE"]["stop_distance_atrs"] is None # unknown ≠ zero + + +def _pm_rc(): + from src.models import ReasoningChain + return ReasoningChain( + macro_filter="x", news_check="x", earnings_check="x", + signal_conflicts="x", sizing_logic="x", + portfolio_balance="x", cash_target="x", + ) + +# ---------- entry ATR stop floor (ExecutionStage) ---------- + +def test_entry_stop_floor_widens_tight_stop_and_resizes(): + """Stop 0.5×ATR from entry gets widened to 1×ATR; qty_by_risk sizes + against the wider distance so per-trade $ risk is unchanged.""" + from src.pipeline_stages import ExecutionStage + from src.pipeline_context import RunContext + from src.models import TradeDecision, PortfolioDecision, OHLCV, ReasoningChain + + pipeline = MagicMock() + pipeline.db.insert_trade.return_value = 7 + pipeline.broker.submit_order.return_value = {"id": "b1", "status": "accepted"} + pipeline._order_accepted.return_value = True + pipeline.risk_engine.check_daily_loss.return_value = None + pipeline._refresh_account_state.return_value = ( + {"cash": 50_000.0, "portfolio_value": 100_000.0}, [], {"NVDA": 100.0}, + ) + pipeline.broker.get_latest_price.return_value = 100.0 + + # 20 synthetic daily bars with true range ≈ $4 → ATR14 ≈ 4. + bars = [OHLCV(date=_date(2026, 6, d + 1), open=100, high=102, + low=98, close=100, volume=1_000_000) for d in range(20)] + + ctx = RunContext.start("morning") + ctx.positions, ctx.cash, ctx.total_value, ctx.last_equity = [], 50_000.0, 100_000.0, 100_000.0 + ctx.symbols_bars = {"NVDA": bars} + ctx.portfolio_decision = PortfolioDecision( + reasoning_chain=_pm_rc(), + decisions=[TradeDecision(action="BUY", symbol="NVDA", allocation_pct=10.0, + entry_price=100.0, stop_loss=98.0, # 0.5×ATR — too tight + take_profit=120.0, reasoning="test")], + portfolio_view="test", + ) + + ExecutionStage(pipeline=pipeline).run(ctx) + + submit = pipeline.broker.submit_order.call_args.kwargs + assert submit["symbol"] == "NVDA" + assert submit["stop_loss_price"] == 96.0 # 100 - 1×ATR(=4) + # alloc sizing binds here (10% / $100 = 100 sh; risk budget $500/$4 = 125 + # doesn't). The floor's effect on qty shows via qty_by_risk when alloc is + # large; what matters structurally: sizing used the WIDENED distance. + assert submit["qty"] == 100 + # write-ahead row carries the widened stop too + insert = pipeline.db.insert_trade.call_args_list[0].kwargs + assert insert["stop_loss"] == 96.0 + + +def test_entry_stop_floor_leaves_wide_stop_alone(): + from src.pipeline_stages import ExecutionStage + from src.pipeline_context import RunContext + from src.models import TradeDecision, PortfolioDecision, OHLCV, ReasoningChain + + pipeline = MagicMock() + pipeline.db.insert_trade.return_value = 7 + pipeline.broker.submit_order.return_value = {"id": "b1", "status": "accepted"} + pipeline._order_accepted.return_value = True + pipeline.risk_engine.check_daily_loss.return_value = None + pipeline._refresh_account_state.return_value = ( + {"cash": 50_000.0, "portfolio_value": 100_000.0}, [], {"NVDA": 100.0}, + ) + pipeline.broker.get_latest_price.return_value = 100.0 + bars = [OHLCV(date=_date(2026, 6, d + 1), open=100, high=102, + low=98, close=100, volume=1_000_000) for d in range(20)] + + ctx = RunContext.start("morning") + ctx.positions, ctx.cash, ctx.total_value, ctx.last_equity = [], 50_000.0, 100_000.0, 100_000.0 + ctx.symbols_bars = {"NVDA": bars} + ctx.portfolio_decision = PortfolioDecision( + reasoning_chain=_pm_rc(), + decisions=[TradeDecision(action="BUY", symbol="NVDA", allocation_pct=10.0, + entry_price=100.0, stop_loss=90.0, # 2.5×ATR — fine + take_profit=120.0, reasoning="test")], + portfolio_view="test", + ) + ExecutionStage(pipeline=pipeline).run(ctx) + submit = pipeline.broker.submit_order.call_args.kwargs + assert submit["stop_loss_price"] == 90.0 From 988ea9c7372f0e9b33e3c33d2b555ae37ae9506a Mon Sep 17 00:00:00 2001 From: yebof Date: Thu, 16 Jul 2026 15:50:37 +0800 Subject: [PATCH 3/8] feat(feedback): repair the self-exculpatory grading loop + tape-truth exit audit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RC4 (2026-07-16 forensics): the learning loop couldn't emit 'we were wrong to sell' or 'we were wrong to skip': - value_entry_missed was code-filtered OUT of the recurring-miss digest — the exact category evening uses for actionable entries (SNDK flagged 16×, ORCL 7×; PM was shown '(no recurring missed themes)' every run since 7/01 while evening wrote 'zero execution — this is process failure'). - Misses were grouped by theme_if_any, LLM free text that never repeats verbatim (45 distinct themes, 0 recurring) — now grouped by SYMBOL, theme kept as annotation. - Sell grades are scored by the LLM at t+1..t+3 with thesis-rationalization framing (32/33 'correct' while the tape had 53% of exits ≥5% higher within 20d; grader error ~50% on checkable sells). A deterministic post-exit reality block (_build_post_exit_reality: trades × live prices, no LLM) now rides with the grade summary into the reviewer prompt, with a hard escalation line when over half of recent exits kept running. SWEEP_SELL and <2-day-old exits excluded. - Reviewer prompt: to_stop is advisory distance, never a trigger (GS 5/18 pre-empted its own stop); single-source bearish news on a >10% winner caps first-day action at REDUCE≤50% (AAPL 6/25); think in ATR units; documents the trail cooldown + noise-band enforcement from the exits commit. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PQkESoSTYx2bCy7WYnTPXR --- config/prompts/position_reviewer.md | 28 ++++- src/agents/position_reviewer.py | 28 +++++ src/pipeline.py | 156 +++++++++++++++++++++++++--- tests/test_feedback_loops.py | 153 +++++++++++++++++++++++++++ 4 files changed, 348 insertions(+), 17 deletions(-) create mode 100644 tests/test_feedback_loops.py diff --git a/config/prompts/position_reviewer.md b/config/prompts/position_reviewer.md index 416c18e7..e89635f3 100644 --- a/config/prompts/position_reviewer.md +++ b/config/prompts/position_reviewer.md @@ -109,7 +109,13 @@ A SELL or REDUCE must point to ONE of: entry thesis has actually occurred (not "I worry it might") - **HIGH-conviction state_change that reverses the thesis** — not any news, specifically a state_change labeled HIGH that contradicts the entry - rationale + rationale. **Single-source cap on winners:** when the ONLY trigger is one + news state_change (nothing else corroborates — tech rating unchanged, + thesis_invalid_if not met, no earnings signal) and the position is a >10% + winner, first-day action is capped at REDUCE (≤50%); a full SELL requires + either a second corroborating signal or the story surviving into the next + session. (2026-06-25 autopsy: a +18% AAPL position was fully exited + same-day on one component-cost story; the story faded, the stock didn't.) - **Earnings filing bearish for this position** — the just-filed 10-Q/10-K analysis comes back with `sentiment=bearish` AND `conviction ∈ {medium, high}` on a name you're long. A `bearish` + `low` conviction filing is mixed-signal @@ -131,7 +137,17 @@ Every position has deterministic numbers: don't trim a fast winner). <0.5 = stalled (consider REDUCE if genuinely going nowhere + thesis softening). - `to_stop` / `to_target` = % distance to the respective levels. <2% to stop - = critical zone. + = critical zone. **`to_stop` is ADVISORY DISTANCE, never a trigger: only + the broker fills stops.** "Close to stop" or "will gap through the stop + overnight" is NOT a reason to SELL ahead of it — pre-empting the stop + converts protection into a realized whipsaw (GS 2026-05-18: sold at + +0.4%-to-stop "before the gap"; no gap came, the stock ran). +- `atr_pct` = ATR(14) as % of price — one day's normal range. `stop_distance_atrs` + = stop distance in ATR units. **Think in ATRs, not raw %**: a 3% gap is roomy + for a staples name and suicidal for a high-beta one. A stop <1.25 ATRs away + is inside daily noise — the pipeline will REJECT a TRAIL_STOP into that band + (without a hard trigger), so don't propose one; if you genuinely want out, + say SELL/REDUCE with the trigger named. - `weight_pct` = current $ weight of book. Flags the pipeline may attach: @@ -194,6 +210,14 @@ Respond ONLY with valid JSON matching `PositionReview`: negligible protection gain — if the right new stop is within 2% of the old one, just HOLD. The stop can only go UP; you cannot widen it later, so do not ratchet a young position's stop up into its own noise band. + **Pipeline enforcement (don't fight it, plan around it):** without a hard + trigger cited in `reason`, a TRAIL_STOP is REJECTED when (a) a trail on the + same symbol was already accepted within the last ~2 trading days (ratchet + cooldown — the ×1.02 minimum means back-to-back trails walk the stop ≥2% + per session straight into the noise band; GE was ratcheted 7× in 8 sessions + this way), or (b) the new stop lands within 1.25×ATR14 of the current price + (inside one day's range — routine volatility would fill it). One considered + trail beats daily nudges. - **REDUCE** — sells 50% of the position. Use for: drift_flag firing, parabolic exhaustion confirmed, target_breach with momentum fading, correlation cluster rebalance. **If a 50% reduce would still leave `weight_pct > 12%` diff --git a/src/agents/position_reviewer.py b/src/agents/position_reviewer.py index c18877ba..1fe71020 100644 --- a/src/agents/position_reviewer.py +++ b/src/agents/position_reviewer.py @@ -371,6 +371,34 @@ def _opt_section(title: str, body: str) -> str: f"{tilt_note}\n" ).rstrip() + "\n" + # Deterministic post-exit reality (computed from trades × live + # prices, NO self-assessment): what the tape did after our recent + # exits. Rendered even when nightly grades are absent — the + # 2026-07-16 audit found grades said 97% "correct" while 53% of + # exits ran ≥5% higher within 20 days. + reality = trade_grade_summary.get("post_exit_reality") if trade_grade_summary else None + if reality and reality.get("n"): + worst_lines = "; ".join( + f"{w['symbol']} exited {w['date']} → {w['move_pct']:+.1f}% since" + for w in (reality.get("worst") or []) + ) + frac = reality["n_higher_5pct"] / reality["n"] + reality_note = "" + if frac >= 0.5: + reality_note = ( + "⚠️ Over HALF of recent exits ran ≥5% after you sold — the " + "tape says your exits fire too early. Any SELL/tighten today " + "must name a hard trigger, not price action." + ) + grade_section += ( + f"### Post-exit reality check (deterministic, from price data)\n" + f"Exits in window: {reality['n']}; ran ≥5% higher after exit: " + f"{reality['n_higher_5pct']}; avg move since exit: " + f"{reality['avg_move_pct']:+.1f}%\n" + f"Biggest post-exit runs: {worst_lines}\n" + f"{reality_note}\n" + ).rstrip() + "\n" + if recent_performance: r5 = recent_performance.get("rolling_5d_pct") r20 = recent_performance.get("rolling_20d_pct") diff --git a/src/pipeline.py b/src/pipeline.py index 459a178b..c297cb4a 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -2893,13 +2893,26 @@ def _build_trade_grade_summary(self, lookback_days: int = 14) -> dict: "repeat_premature_symbols": [], "repeat_wrong_symbols": [], } + def _with_reality(base: dict) -> dict: + # The deterministic post-exit block must ride along even when + # nightly grades are absent/corrupt — it's tape-derived, not + # grade-derived, and it's the part the grader can't sugar-coat. + try: + base["post_exit_reality"] = self._build_post_exit_reality( + lookback_days=max(lookback_days, 14), + ) + except Exception as e: # noqa: BLE001 + logger.warning("post_exit_reality failed (summary degrades): %s", e) + base["post_exit_reality"] = None + return base + try: rows = self.db.get_recent_insights(limit=lookback_days + 5) except Exception as e: logger.warning("trade_grade_summary: insights fetch failed: %s", e) - return empty + return _with_reality(empty) if not rows: - return empty + return _with_reality(empty) sell_counts = {"correct": 0, "premature": 0, "wrong": 0} buy_counts = {"correct": 0, "premature": 0, "wrong": 0} @@ -2954,7 +2967,7 @@ def _load(col: str, row: dict) -> list[dict]: if grade in buy_counts: buy_counts[grade] += 1 - return { + summary = { "n_sells": sum(sell_counts.values()), "n_buys": sum(buy_counts.values()), "sell_counts": sell_counts, @@ -2966,6 +2979,101 @@ def _load(col: str, row: dict) -> list[dict]: s for s, c in sell_wrong_by_symbol.items() if c >= 2 ), } + # RC4 (2026-07-16): deterministic post-exit reality. The LLM grader + # scored 32/33 recent sells "correct" at t+1..t+3 while the tape + # showed 28/53 exits ≥5% higher within 20 days — self-assessment + # cannot be the only input to the patience tilt. These numbers come + # from trades × live prices, no LLM in the loop. + return _with_reality(summary) + + # Realized-exit actions whose post-exit trajectory is worth auditing. + # SWEEP_SELL is deliberately absent — parking churn is not a decision. + _EXIT_AUDIT_ACTIONS = ( + "SELL", "REDUCE", "EMERGENCY_SELL", "FORCE_DELEVER", "TAKE_PROFIT", + ) + + def _build_post_exit_reality( + self, lookback_days: int = 14, min_age_days: int = 2, max_symbols: int = 12, + ) -> dict | None: + """What actually happened after our recent exits — from the tape. + + For every realized exit in the window (SELL family + filled + TRAIL_STOPs) at least `min_age_days` old, compare the exit price to + the live price. Returns None when there's nothing to audit. + + {"n": int, "n_higher_5pct": int, "avg_move_pct": float, + "worst": [{"symbol", "date", "move_pct"} × ≤3]} # worst = ran most + """ + from datetime import datetime as _dt, timedelta, timezone + try: + rows = self.db.get_trades(limit=120) + except Exception as e: # noqa: BLE001 + logger.warning("post_exit_reality: trades fetch failed: %s", e) + return None + now = _dt.now(timezone.utc) + window_start = now - timedelta(days=lookback_days) + age_cutoff = now - timedelta(days=min_age_days) + exits: list[dict] = [] + for row in rows: + action = (row.get("action") or "").upper() + is_exit = ( + action in self._EXIT_AUDIT_ACTIONS + or action.startswith("PARTIAL_SELL") + or (action == "TRAIL_STOP" + and (row.get("fill_status") or "") == "filled") + ) + if not is_exit: + continue + if action != "TRAIL_STOP" and (row.get("fill_status") or "") not in ( + "filled", "submitted", + ): + continue + ts = row.get("timestamp") or "" + try: + dt = _dt.fromisoformat(ts.replace("Z", "+00:00")) if "T" in ts \ + else _dt.strptime(ts, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc) + except (TypeError, ValueError): + continue + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + if not (window_start <= dt <= age_cutoff): + continue + exit_px = row.get("fill_price") or row.get("price") + if not (isinstance(exit_px, (int, float)) and exit_px > 0): + continue + exits.append({ + "symbol": row.get("symbol"), "date": ts[:10], + "exit_px": float(exit_px), + }) + if not exits: + return None + # Live prices — one broker call per distinct symbol, capped. + prices: dict[str, float] = {} + for sym in list(dict.fromkeys(e["symbol"] for e in exits))[:max_symbols]: + try: + px = self.broker.get_latest_price(sym) + except Exception: # noqa: BLE001 + px = None + if isinstance(px, (int, float)) and px > 0: + prices[sym] = float(px) + moves: list[dict] = [] + for e in exits: + cur = prices.get(e["symbol"]) + if cur is None: + continue + moves.append({ + "symbol": e["symbol"], "date": e["date"], + "move_pct": round((cur - e["exit_px"]) / e["exit_px"] * 100, 1), + }) + if not moves: + return None + moves.sort(key=lambda m: -m["move_pct"]) + return { + "n": len(moves), + "n_higher_5pct": sum(1 for m in moves if m["move_pct"] >= 5.0), + "avg_move_pct": round(sum(m["move_pct"] for m in moves) / len(moves), 1), + "worst": moves[:3], + } def _build_recent_missed_lessons(self, lookback_days: int = 14) -> str: """PM L3d memory: themes that evening flagged ≥ 2 times as missed. @@ -2989,8 +3097,14 @@ def _build_recent_missed_lessons(self, lookback_days: int = 14) -> str: return "" if not rows: return "" + # RC4 (2026-07-16): value_entry_missed IS a real, actionable miss — + # it's the category evening uses for "we identified the entry and + # didn't take it" (SNDK was flagged 16×, ORCL 7×, and PM never saw + # any of it because this set filtered them out). Only the two + # "not-really-a-miss" categories stay excluded. real_miss_cats = { - "trend_timing_miss", "theme_blindspot", "fundamentals_mispricing" + "trend_timing_miss", "theme_blindspot", "fundamentals_mispricing", + "value_entry_missed", } theme_dates: dict[str, set[str]] = {} theme_symbols: dict[str, list[str]] = {} @@ -3024,18 +3138,30 @@ def _build_recent_missed_lessons(self, lookback_days: int = 14) -> str: continue theme = (m.get("theme_if_any") or "").strip() sym = (m.get("symbol") or "").strip().upper() - # Group key: theme name when present, else symbol (fall back so - # "no theme tagged but same symbol missed twice" still surfaces). - key = theme or f"sym:{sym}" - if not key: + # DUAL grouping keys. RC4 (2026-07-16): theme_if_any is LLM + # free text that almost never repeats verbatim (45 distinct + # themes, 0 recurring in the audit window) — keyed ONLY by + # theme, a symbol missed 16 times (SNDK) diluted into 16 + # one-off "themes" and PM was shown "(no recurring missed + # themes)" every run. Symbol-keyed counting fixes that; + # theme-keyed counting is KEPT because cross-symbol theme + # recurrence (VST + OKLO both "nuclear/power") is a real, + # distinct signal a symbol key can't see. + keys = set() + if sym: + keys.add(f"sym:{sym}") + if theme: + keys.add(theme) + if not keys: continue - theme_dates.setdefault(key, set()).add(row_date) - theme_symbols.setdefault(key, []).append(sym) - # Rows are newest-first; first lesson we see is the freshest. - if key not in theme_lessons: - lesson = (m.get("lesson") or "").strip() - if lesson: - theme_lessons[key] = lesson[:200] + for key in keys: + theme_dates.setdefault(key, set()).add(row_date) + theme_symbols.setdefault(key, []).append(sym) + # Rows are newest-first; first lesson seen is freshest. + if key not in theme_lessons: + lesson = (m.get("lesson") or "").strip() + if lesson: + theme_lessons[key] = lesson[:200] # Keep themes seen on ≥ 2 distinct dates. recurring = [ (k, len(theme_dates[k])) for k in theme_dates diff --git a/tests/test_feedback_loops.py b/tests/test_feedback_loops.py new file mode 100644 index 00000000..994882d0 --- /dev/null +++ b/tests/test_feedback_loops.py @@ -0,0 +1,153 @@ +"""RC4 feedback-loop repairs (2026-07-16 forensics). + +The self-grading loop was structurally self-exculpatory: + - `value_entry_missed` (evening's actionable "we identified the entry and + skipped it" category — SNDK flagged 16×) was code-filtered OUT of the + recurring-miss digest, so PM was shown "(no recurring missed themes)" + every run. + - Misses were grouped by `theme_if_any` — LLM free text that never repeats + verbatim — so symbol-level recurrence could not surface. + - Sell grades were scored at t+1..t+3 by the LLM (97% "correct" while the + tape showed 53% of exits ≥5% higher within 20 days). A deterministic + post-exit reality block now rides along with the grade summary. +""" +import json +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock + +from src.pipeline import TradingPipeline + + +def _mk_pipeline(): + pipeline = TradingPipeline.__new__(TradingPipeline) + pipeline.db = MagicMock() + pipeline.broker = MagicMock() + return pipeline + + +def _insights_row(date: str, misses: list[dict]) -> dict: + return {"date": date, "missed_opportunities_json": json.dumps(misses)} + + +def test_value_entry_missed_counts_as_real_miss(): + pipeline = _mk_pipeline() + pipeline.db.get_recent_insights.return_value = [ + _insights_row("2026-07-15", [ + {"miss_category": "value_entry_missed", "symbol": "SNDK", + "theme_if_any": "memory upcycle pricing power", "lesson": "buy the dip"}, + ]), + _insights_row("2026-07-14", [ + {"miss_category": "value_entry_missed", "symbol": "SNDK", + "theme_if_any": "NAND tightness into H2", "lesson": "still cheap"}, + ]), + ] + out = pipeline._build_recent_missed_lessons() + assert "SNDK" in out, "value_entry_missed must surface as a real miss" + + +def test_misses_group_by_symbol_not_freetext_theme(): + """Same symbol, different free-text themes on 2 dates → must still + recur. Under the old theme-keyed grouping each date was a distinct + 'theme' seen once, and the ≥2-dates filter emitted nothing.""" + pipeline = _mk_pipeline() + pipeline.db.get_recent_insights.return_value = [ + _insights_row("2026-07-15", [ + {"miss_category": "trend_timing_miss", "symbol": "ORCL", + "theme_if_any": "AI capex second wave", "lesson": "x"}, + ]), + _insights_row("2026-07-13", [ + {"miss_category": "trend_timing_miss", "symbol": "ORCL", + "theme_if_any": "hyperscaler backlog acceleration", "lesson": "y"}, + ]), + ] + out = pipeline._build_recent_missed_lessons() + assert "ORCL" in out + + +def test_noise_categories_still_excluded(): + pipeline = _mk_pipeline() + pipeline.db.get_recent_insights.return_value = [ + _insights_row("2026-07-15", [ + {"miss_category": "noise_rally", "symbol": "GME", "theme_if_any": ""}, + ]), + _insights_row("2026-07-14", [ + {"miss_category": "noise_rally", "symbol": "GME", "theme_if_any": ""}, + ]), + ] + assert pipeline._build_recent_missed_lessons() == "" + + +# ---------- deterministic post-exit reality ---------- + +def _trade(symbol: str, action: str, price: float, days_ago: int, + fill_status: str = "filled") -> dict: + ts = (datetime.now(timezone.utc) - timedelta(days=days_ago)).isoformat() + return {"symbol": symbol, "action": action, "price": price, + "fill_price": price, "fill_status": fill_status, "timestamp": ts} + + +def test_post_exit_reality_measures_the_tape(): + pipeline = _mk_pipeline() + pipeline.db.get_trades.return_value = [ + _trade("LLY", "SELL", 1000.0, days_ago=5), + _trade("VST", "TRAIL_STOP", 150.0, days_ago=7), + _trade("KO", "REDUCE", 100.0, days_ago=4), + ] + pipeline.broker.get_latest_price.side_effect = ( + lambda s: {"LLY": 1120.0, "VST": 150.0, "KO": 95.0}[s] + ) + r = pipeline._build_post_exit_reality() + assert r["n"] == 3 + assert r["n_higher_5pct"] == 1 # only LLY (+12%) + assert r["worst"][0]["symbol"] == "LLY" + assert r["worst"][0]["move_pct"] == 12.0 + + +def test_post_exit_reality_excludes_sweep_and_fresh_exits(): + pipeline = _mk_pipeline() + pipeline.db.get_trades.return_value = [ + _trade("SGOV", "SWEEP_SELL", 100.6, days_ago=5), # parking churn + _trade("NVDA", "SELL", 900.0, days_ago=0), # too fresh (<2d) + _trade("GE", "TRAIL_STOP", 350.0, days_ago=5, + fill_status="submitted"), # trail not FILLED + ] + pipeline.broker.get_latest_price.return_value = 999.0 + assert pipeline._build_post_exit_reality() is None + + +def test_trade_grade_summary_carries_reality_block(): + pipeline = _mk_pipeline() + pipeline.db.get_recent_insights.return_value = [] + pipeline.db.get_trades.return_value = [_trade("LLY", "SELL", 1000.0, days_ago=5)] + pipeline.broker.get_latest_price.return_value = 1100.0 + summary = pipeline._build_trade_grade_summary() + assert summary["post_exit_reality"]["n"] == 1 + assert summary["post_exit_reality"]["avg_move_pct"] == 10.0 + + +# ---------- reviewer prompt renders the reality section ---------- + +def test_reviewer_prompt_renders_post_exit_reality(): + from src.agents.position_reviewer import PositionReviewerAgent + from unittest.mock import patch + with patch("anthropic.Anthropic"): + agent = PositionReviewerAgent( + api_key="k", model="claude-opus-4-7", max_tokens=1024, + ) + msg = agent.build_user_message( + positions=[], macro_summary={}, cash_balance=1000.0, + total_value=100_000.0, session_type="midday", + trade_grade_summary={ + "n_sells": 0, "n_buys": 0, + "sell_counts": {}, "buy_counts": {}, + "repeat_premature_symbols": [], "repeat_wrong_symbols": [], + "post_exit_reality": { + "n": 4, "n_higher_5pct": 3, "avg_move_pct": 8.5, + "worst": [{"symbol": "LLY", "date": "2026-06-18", "move_pct": 12.4}], + }, + }, + ) + assert "Post-exit reality check" in msg + assert "LLY" in msg and "+12.4%" in msg + # ≥50% of exits ran → the hard-trigger escalation line must render + assert "tape says your exits fire too early" in msg From 838e42d0b1a65984e38c27d218bc199362c82b37 Mon Sep 17 00:00:00 2001 From: yebof Date: Thu, 16 Jul 2026 16:00:07 +0800 Subject: [PATCH 4/8] =?UTF-8?q?feat(pm):=20deployment-gap=20convergence=20?= =?UTF-8?q?=E2=80=94=20the=20under-deployment=20must=20be=20answered?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RC3 (2026-07-16 forensics): macro demanded 72-75% invested for three months (risk-on the whole window); realized invested% averaged 39.2% and declined monotonically 72.7%→15.4% over 13 weeks. Every layer shaved sizes independently (PM on calibration, RM on R/R, evening tilt) and nothing reconciled the compounded result against the target — ~$8.9k of cash drag, 81% of the SPY shortfall. - PMFacts gains macro_target_invested_pct + deployment_gap_pp; a >15pp under-target book renders a ⚠️ DEPLOYMENT GAP section the PM prompt now requires be answered in the cash_target step: close the gap this session OR name a checkable blocker per unfilled slot. 'Staying cautious' without a named blocker is explicitly ruled out. Does not override RM, caps, or drawdown-halving — it forces the gap to be addressed, not auto-filled. - macro_exposure_deviation advisory is now direction-aware: for an UNDER- deployed book it tells RM NOT to scale the remaining BUYs down for this reason (the old symmetric 'consider scale_all_buys' amplified the drag it should have been correcting). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PQkESoSTYx2bCy7WYnTPXR --- config/prompts/portfolio_manager.md | 12 +++ src/pipeline.py | 31 +++++++- src/pipeline_context.py | 31 +++++++- src/pipeline_stages.py | 1 + tests/test_deployment_gap.py | 111 ++++++++++++++++++++++++++++ 5 files changed, 184 insertions(+), 2 deletions(-) create mode 100644 tests/test_deployment_gap.py diff --git a/config/prompts/portfolio_manager.md b/config/prompts/portfolio_manager.md index 265aa26c..28fa0970 100644 --- a/config/prompts/portfolio_manager.md +++ b/config/prompts/portfolio_manager.md @@ -391,6 +391,18 @@ Rules: - Cash **above** ceiling and macro is risk-on / transitional → you are under-deploying; either size up high-conviction names or lower your hurdle one notch +- **DEPLOYMENT GAP fact (when present in your facts block): answer it + here, explicitly.** The facts may show invested% more than 15pp under + macro's `target_invested_pct`. That gap was measured as the single + largest P&L drag over Apr–Jul 2026 (idle cash while macro said 72–75% + invested, book sat at ~39%). Your `cash_target` step must then contain + ONE of: (a) targets this session that close most of the gap, or (b) a + named, checkable blocker per unfilled slot — "no candidate passed the + R/R filter today", "regime gate: macro flipped transitional", + "top candidates all earnings-queued". A generic "staying selective / + cautious" is NOT a valid answer; it's how three months of drag + happened. (This does not override RM, caps, or the drawdown-halving — + it forces the gap to be ADDRESSED, not auto-filled.) - Align with Macro's `position_guidance.cash_recommendation_pct` when present, but **regime-based floors ALWAYS override** (advisory is soft; floor is hard) diff --git a/src/pipeline.py b/src/pipeline.py index c297cb4a..df03e71d 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -589,12 +589,26 @@ def _filter_hard_risk_decisions( projected_invested_pct = abs(existing_net + pending_investment) / total_value * 100 deviation = projected_invested_pct - macro_target_invested_pct if abs(deviation) > 15: + # RC3: direction matters. The old symmetric message told RM + # to "consider scale_all_buys" for BOTH directions — for an + # UNDER-deployed book that advice compounds the exact drag + # it should be correcting (three months of 39% invested vs + # a 72-75% target). + if deviation < 0: + guidance = ( + "advisory — book is UNDER macro's target; do NOT " + "scale down the remaining BUYs for this reason. If " + "cutting anything, name a risk specific to the trade, " + "not the gap." + ) + else: + guidance = "advisory — RM should consider scale_all_buys" remaining_violations.append(RiskViolation( rule="macro_exposure_deviation", message=( f"Projected net exposure {projected_invested_pct:.0f}% deviates " f"from Macro target {macro_target_invested_pct:.0f}% by {deviation:+.0f}pp " - f"(advisory — RM should consider scale_all_buys)" + f"({guidance})" ), value=projected_invested_pct, limit=macro_target_invested_pct, @@ -4166,6 +4180,7 @@ def _build_pm_facts( total_value: float, cash: float, recent_performance: dict, + macro_analysis=None, ) -> PMFacts: """Quantitative snapshot surfaced to PM as structured fields. @@ -4270,6 +4285,20 @@ def _build_pm_facts( f.rolling_20d_pct = recent_performance.get("rolling_20d_pct") f.in_drawdown = bool(recent_performance.get("in_drawdown")) + # RC3: deployment gap vs the macro target as a hard fact in PM's + # face. `invested_pct` above is sweep-aware (the DecisionStage view + # already counts parked T-bills as cash, not exposure). + try: + target = None + if macro_analysis is not None: + guidance = getattr(macro_analysis, "position_guidance", None) + target = getattr(guidance, "target_invested_pct", None) + if isinstance(target, (int, float)) and math.isfinite(target): + f.macro_target_invested_pct = float(target) + f.deployment_gap_pp = round(f.invested_pct - float(target), 1) + except Exception as e: # noqa: BLE001 + logger.warning("pm_facts: deployment gap failed: %s", e) + return f def _build_calibration_note(self, lookback_days: int = 45) -> str: diff --git a/src/pipeline_context.py b/src/pipeline_context.py index 5ee1ccaa..2b18ec6d 100644 --- a/src/pipeline_context.py +++ b/src/pipeline_context.py @@ -132,6 +132,14 @@ class PMFacts: rolling_20d_pct: float | None = None in_drawdown: bool = False + # RC3 (2026-07-16): deployment vs the macro target. Macro demanded + # 72-75% invested for three months while realized invested% averaged + # 39% and NOTHING forced the gap into PM's face — every layer shaved + # sizes independently and no one reconciled the compound. None when + # macro didn't provide a target this session. + macro_target_invested_pct: float | None = None + deployment_gap_pp: float | None = None # invested - target (negative = under) + def render(self) -> str: """Format as a compact markdown block for PM's prompt.""" def _pct(v: float | None) -> str: @@ -162,4 +170,25 @@ def _num(v: float | int | None) -> str: - signals={self.tech_signals_count} · median_age={_num(self.tech_signals_median_age_days)}d · stale(≥8d)={self.tech_signals_stale_count} ### System Performance -- rolling 5d={_pct(self.rolling_5d_pct)} · 20d={_pct(self.rolling_20d_pct)} · in_drawdown={self.in_drawdown}""" +- rolling 5d={_pct(self.rolling_5d_pct)} · 20d={_pct(self.rolling_20d_pct)} · in_drawdown={self.in_drawdown}{self._render_deployment_gap()}""" + + def _render_deployment_gap(self) -> str: + if self.macro_target_invested_pct is None or self.deployment_gap_pp is None: + return "" + if self.deployment_gap_pp >= -15: + return ( + f"\n\n### Deployment vs Macro Target" + f"\n- invested={self.invested_pct:.1f}% vs macro target=" + f"{self.macro_target_invested_pct:.0f}% (gap {self.deployment_gap_pp:+.0f}pp — within band)" + ) + return ( + f"\n\n### ⚠️ DEPLOYMENT GAP (address in cash_target step)" + f"\n- invested={self.invested_pct:.1f}% vs macro target=" + f"{self.macro_target_invested_pct:.0f}% — you are {-self.deployment_gap_pp:.0f}pp UNDER the target" + f"\n- This gap has been the single largest P&L drag (idle cash in a" + f" rising market). In `cash_target`, either (a) close it with" + f" qualified candidates THIS session, or (b) name the concrete" + f" blocker per unfilled slot (no-qualified-setups after filters /" + f" regime gate / earnings-queue). \"Staying cautious\" without a" + f" named blocker is not an answer." + ) diff --git a/src/pipeline_stages.py b/src/pipeline_stages.py index 5bd013b5..ee09332a 100644 --- a/src/pipeline_stages.py +++ b/src/pipeline_stages.py @@ -430,6 +430,7 @@ def run(self, ctx: RunContext) -> RunContext: positions=positions, analyses=analyses, total_value=total_value, cash=cash, recent_performance=recent_performance, + macro_analysis=macro_analysis, ) ctx.facts = pm_facts diff --git a/tests/test_deployment_gap.py b/tests/test_deployment_gap.py new file mode 100644 index 00000000..023f2bca --- /dev/null +++ b/tests/test_deployment_gap.py @@ -0,0 +1,111 @@ +"""RC3 deployment-convergence facts (2026-07-16 forensics). + +Macro demanded 72-75% invested for three months; realized invested% +averaged 39% and declined monotonically while every layer shaved sizes +independently. Nothing reconciled the compound. Two fixes: + + 1. PMFacts carries macro_target_invested_pct + deployment_gap_pp and + renders a ⚠️ DEPLOYMENT GAP section (>15pp under) that the PM prompt + requires be answered in the cash_target step. + 2. The macro_exposure_deviation advisory is direction-aware — for an + UNDER-deployed book it must NOT tell RM to scale_all_buys down. +""" +from types import SimpleNamespace +from unittest.mock import MagicMock + +from src.models import Position, TradeDecision +from src.pipeline import TradingPipeline +from src.pipeline_context import PMFacts +from src.risk.rules import RiskRuleEngine +from src.config import RiskConfig + + +def test_pm_facts_render_deployment_gap_when_under(): + f = PMFacts() + f.invested_pct = 39.0 + f.macro_target_invested_pct = 74.0 + f.deployment_gap_pp = -35.0 + out = f.render() + assert "DEPLOYMENT GAP" in out + assert "35pp UNDER" in out + assert "cash_target" in out + + +def test_pm_facts_render_within_band_is_calm(): + f = PMFacts() + f.invested_pct = 70.0 + f.macro_target_invested_pct = 74.0 + f.deployment_gap_pp = -4.0 + out = f.render() + assert "within band" in out + assert "DEPLOYMENT GAP" not in out + + +def test_pm_facts_render_no_target_no_section(): + out = PMFacts().render() + assert "Deployment vs Macro Target" not in out + assert "DEPLOYMENT GAP" not in out + + +def test_build_pm_facts_computes_gap_from_macro(): + pipeline = TradingPipeline.__new__(TradingPipeline) + pipeline.db = MagicMock() + pipeline.db.compute_trade_calibration.return_value = {} + pipeline.db.get_recent_agent_outputs.return_value = [] + pipeline._build_position_history = MagicMock(return_value={}) + macro = SimpleNamespace( + position_guidance=SimpleNamespace(target_invested_pct=75.0), + ) + pos = Position(symbol="GE", qty=26, avg_entry=316, current_price=360, + market_value=9_360, unrealized_pnl=1_144, + unrealized_intraday_pnl=0.0, sector="Industrials") + f = pipeline._build_pm_facts( + positions=[pos], analyses=[], total_value=100_000.0, + cash=90_640.0, recent_performance={}, macro_analysis=macro, + ) + assert f.macro_target_invested_pct == 75.0 + # invested ≈ 9.4% → gap ≈ -65.6pp + assert f.deployment_gap_pp is not None and f.deployment_gap_pp < -60 + + +def _engine_pipeline(): + pipeline = TradingPipeline.__new__(TradingPipeline) + pipeline.risk_engine = RiskRuleEngine(RiskConfig( + max_position_pct=50, max_total_position_pct=200, + max_daily_loss_pct=10, max_sector_pct=100, + require_stop_loss=True, allow_margin=False, + )) + return pipeline + + +def test_under_deployment_advisory_does_not_ask_for_scale_down(): + pipeline = _engine_pipeline() + decision = TradeDecision(action="BUY", symbol="NVDA", allocation_pct=5.0, + entry_price=100.0, stop_loss=90.0, + take_profit=130.0, reasoning="x") + _, violations, _ = pipeline._filter_hard_risk_decisions( + [decision], [], total_value=100_000.0, daily_pnl=0.0, + baseline=100_000.0, macro_target_invested_pct=75.0, cash=95_000.0, + ) + dev = [v for v in violations if v.rule == "macro_exposure_deviation"] + assert dev, "5% projected vs 75% target must fire the advisory" + assert "UNDER" in dev[0].message + assert "do NOT scale down" in dev[0].message or "do NOT" in dev[0].message + + +def test_over_deployment_advisory_still_asks_for_scale_down(): + pipeline = _engine_pipeline() + positions = [Position(symbol="NVDA", qty=100, avg_entry=800, + current_price=900, market_value=90_000, + unrealized_pnl=10_000, unrealized_intraday_pnl=0.0, + sector="Technology")] + decision = TradeDecision(action="BUY", symbol="AAPL", allocation_pct=20.0, + entry_price=100.0, stop_loss=90.0, + take_profit=130.0, reasoning="x") + _, violations, _ = pipeline._filter_hard_risk_decisions( + [decision], positions, total_value=100_000.0, daily_pnl=0.0, + baseline=100_000.0, macro_target_invested_pct=50.0, cash=100_000.0, + ) + dev = [v for v in violations if v.rule == "macro_exposure_deviation"] + assert dev + assert "scale_all_buys" in dev[0].message From 95b71f4846851ca9c70b58b2b766c0749d1dd0d8 Mon Sep 17 00:00:00 2001 From: yebof Date: Thu, 16 Jul 2026 16:09:55 +0800 Subject: [PATCH 5/8] feat(pipeline): post-PM decision checkpoint + zero-LLM morning resume lane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RC2 (2026-07-16 forensics): when LLM latency inflates, morning dies at the wrapper's outer timeout exactly at the PM→RM boundary — research (3 tech chunks + news + macro) plus PM eat the budget and the kill lands as RM starts. 61/61 PM BUY-proposal days between 6/30 and 7/15 were destroyed this way (zero RM vetoes, 100% mechanical attrition), and every 30-min retry tick re-burned the FULL research pipeline to die identically (run-57b752bd: 'Constructor: 5 targets → 4 BUY' logged at 1196s of a 1200s budget). The relay outage that triggered it is fixed, but the structural fragility — the only BUY-capable session being the first casualty of any latency inflation — is not. - src/decision_checkpoint.py: the plan is persisted (atomic tmp+rename) the moment DecisionStage produces decisions; ET-date-keyed, versioned, 90-min max age. - run_morning resume lane: after the FULL normal preamble (WAL drains, orphan sweep, coverage audit, stale-order cancel, force_delever, circuit breaker, fresh account snapshot), an unconsumed same-day checkpoint skips research+PM and re-enters at RiskStage — ~2 LLM calls instead of ~8. RM ALWAYS re-runs (no resume-past-RM variant, by design); the hard-risk filter re-runs on fresh cash/positions; ExecutionStage's 5% entry-staleness skip and pre-BUY daily-loss recheck guard stale prices. - At-most-once: the checkpoint is marked consumed BEFORE ExecutionStage submits and on ANY RiskStage early-exit — an RM-rejected plan is never re-offered (that would be a veto bypass), and a kill mid-execution is owned by the existing BUY write-ahead orphan sweep, never by re-running the plan. - Every checkpoint operation is best-effort: failures degrade to a normal full run, never a crashed session. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PQkESoSTYx2bCy7WYnTPXR --- src/decision_checkpoint.py | 159 ++++++++++++++++++++++ src/pipeline.py | 123 ++++++++++------- tests/test_decision_checkpoint.py | 210 ++++++++++++++++++++++++++++++ 3 files changed, 448 insertions(+), 44 deletions(-) create mode 100644 src/decision_checkpoint.py create mode 100644 tests/test_decision_checkpoint.py diff --git a/src/decision_checkpoint.py b/src/decision_checkpoint.py new file mode 100644 index 00000000..da378a67 --- /dev/null +++ b/src/decision_checkpoint.py @@ -0,0 +1,159 @@ +"""Post-PM decision checkpoint + zero-LLM resume lane for the morning session. + +RC2 (2026-07-16 forensics): when LLM latency inflates (relay outage, +provider degradation), the morning session dies at the wrapper's outer +timeout EXACTLY at the PM→RM boundary — research (3 tech chunks + news + +macro) plus PM consume the whole budget, then the kill lands while RM +starts. 61/61 PM BUY-proposal-days between 6/30 and 7/15 were destroyed +this way (zero RM vetoes — 100% mechanical attrition), and every 30-min +retry tick re-burned the full research pipeline just to die identically. + +The fix: persist the PM's plan the moment it exists. The next tick's +morning run — after executing its FULL normal preamble (WAL drains, orphan +sweep, coverage audit, stale-order cancel, force_delever, circuit breaker, +fresh account snapshot) — finds the unconsumed checkpoint and re-enters at +the RiskStage with ~2 LLM calls left instead of ~8. + +Safety contract (from the adversarial design review): + - Resume NEVER skips the preamble; divergence starts exactly where + research would have started. + - RM ALWAYS re-runs on resume — the two-layer risk philosophy is not + diluted; there is deliberately no "resume after RM" variant. + - The checkpoint is marked consumed BEFORE ExecutionStage submits + (at-most-once: a kill during execution must not re-execute; the BUY + write-ahead orphan sweep owns partial submits) and also on any + RiskStage early-exit (an RM-rejected plan must never be retried). + - Stale protection: same-ET-date only, max age 90 minutes, plus the + existing ExecutionStage guards (5% entry-price staleness skip, + pre-BUY daily-loss recheck) run against FRESH market state. + - Every function is best-effort: any error degrades to "no checkpoint" + (normal full run), never to a crashed session. +""" +import json +import logging +from datetime import datetime, timezone +from pathlib import Path + +logger = logging.getLogger(__name__) + +CHECKPOINT_VERSION = 1 +CHECKPOINT_DIR = Path("data/checkpoints") +MAX_AGE_MINUTES = 90.0 + + +def checkpoint_path(session: str, et_date: str | None = None) -> Path: + from src.trading_calendar import session_date_key + return CHECKPOINT_DIR / f"{et_date or session_date_key()}-{session}.json" + + +def write(ctx) -> Path | None: + """Persist the decided-but-not-yet-risk-reviewed plan. Never raises. + + Only writes when there are actual decisions to preserve — an empty + plan has nothing to resume. + """ + try: + pd = ctx.portfolio_decision + if pd is None or not pd.decisions: + return None + payload = { + "version": CHECKPOINT_VERSION, + "session": ctx.session, + "run_id": ctx.run_id, + "created_at_utc": datetime.now(timezone.utc).isoformat(), + "consumed": False, + "portfolio_decision": pd.model_dump(mode="json"), + "analyses": [a.model_dump(mode="json") for a in (ctx.analyses or [])], + "news_intel": (ctx.news_intel.model_dump(mode="json") + if ctx.news_intel is not None else None), + "macro_analysis": (ctx.macro_analysis.model_dump(mode="json") + if ctx.macro_analysis is not None else None), + "macro_summary": ctx.macro_summary or {}, + "earnings_results": ctx.earnings_results or [], + "data_status": dict(ctx.data_status or {}), + } + path = checkpoint_path(ctx.session) + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".json.tmp") + tmp.write_text(json.dumps(payload)) + tmp.replace(path) # atomic on POSIX — no torn half-written checkpoint + logger.info( + "decision checkpoint written: %s (%d decisions) — a killed run " + "can resume at RiskStage next tick", path, len(pd.decisions), + ) + return path + except Exception as e: # noqa: BLE001 — checkpointing must never hurt the live run + logger.warning("decision checkpoint write failed (non-fatal): %s", e) + return None + + +def load(session: str, max_age_minutes: float = MAX_AGE_MINUTES) -> dict | None: + """Load today's unconsumed checkpoint with models reconstructed. + + Returns {run_id, age_minutes, portfolio_decision, analyses, news_intel, + macro_analysis, macro_summary, earnings_results, data_status} or None + (missing / consumed / stale / wrong version / unparseable). Never raises. + """ + try: + path = checkpoint_path(session) + if not path.exists(): + return None + payload = json.loads(path.read_text()) + if payload.get("version") != CHECKPOINT_VERSION: + return None + if payload.get("consumed") is not False: + return None + created = datetime.fromisoformat(payload["created_at_utc"]) + age_min = (datetime.now(timezone.utc) - created).total_seconds() / 60.0 + if not (0 <= age_min <= max_age_minutes): + logger.info( + "decision checkpoint %s ignored: age %.0f min exceeds %.0f", + path, age_min, max_age_minutes, + ) + return None + from src.models import ( + MacroAnalysis, NewsIntelligenceReport, PortfolioDecision, + TechAnalysisResult, + ) + return { + "run_id": payload.get("run_id"), + "age_minutes": age_min, + "portfolio_decision": PortfolioDecision.model_validate( + payload["portfolio_decision"]), + "analyses": [TechAnalysisResult.model_validate(a) + for a in payload.get("analyses") or []], + "news_intel": (NewsIntelligenceReport.model_validate(payload["news_intel"]) + if payload.get("news_intel") else None), + "macro_analysis": (MacroAnalysis.model_validate(payload["macro_analysis"]) + if payload.get("macro_analysis") else None), + "macro_summary": payload.get("macro_summary") or {}, + "earnings_results": payload.get("earnings_results") or [], + "data_status": payload.get("data_status") or {}, + } + except Exception as e: # noqa: BLE001 + logger.warning("decision checkpoint load failed (treating as absent): %s", e) + return None + + +def mark_consumed(session: str) -> None: + """Flip consumed=true in place. Never raises. + + Called (a) right before ExecutionStage submits (at-most-once for BUYs) + and (b) on any RiskStage early-exit — an RM-rejected or hard-blocked + plan must never be re-offered by the resume lane. + """ + try: + path = checkpoint_path(session) + if not path.exists(): + return + payload = json.loads(path.read_text()) + if payload.get("consumed") is True: + return + payload["consumed"] = True + payload["consumed_at_utc"] = datetime.now(timezone.utc).isoformat() + tmp = path.with_suffix(".json.tmp") + tmp.write_text(json.dumps(payload)) + tmp.replace(path) + logger.info("decision checkpoint %s marked consumed", path) + except Exception as e: # noqa: BLE001 + logger.warning("decision checkpoint consume-mark failed: %s", e) diff --git a/src/pipeline.py b/src/pipeline.py index df03e71d..a9ae27ed 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -5169,50 +5169,78 @@ def run_morning(self) -> dict: "run_id": run_id, } - # Phase 4 #1: research stage runs the parallel fan-out (macro / news / - # tech / earnings). Populates ctx fields; we unpack to local names so - # the downstream code keeps reading legibly. - self.morning_research_stage.run(ctx) - macro_summary = ctx.macro_summary - macro_analysis = ctx.macro_analysis - news_intel = ctx.news_intel - analyses = ctx.analyses - earnings_results = ctx.earnings_results - data_status = ctx.data_status - - # Late-breach check: research can take 5-10 min on slow OpenAI - # days. The pre-research circuit breaker (#45) caught open-gap - # losses; this catches the case where the tape crosses the - # daily-loss limit DURING research and the morning would - # otherwise bail to no_data/no_trades, leaving the breach for - # the next intra tick (30 min away). Mirror the pre-research - # bypass: deterministic emergency liquidate, no LLM dependency. - late_breach = self._check_late_breach_and_emergency_liquidate( - run_id, "post-research", - ) - if late_breach is not None: - return late_breach - - if not analyses: - logger.warning("No analyses produced, skipping trading") - return {"status": "no_data", "orders": [], "run_id": run_id} - - # Phase 4 #1: decision stage — memory layers + PM + Constructor. - self._decision_stage(ctx) - portfolio_decision = ctx.portfolio_decision - - # Second late-breach check: PM is itself a multi-second LLM - # call (memory layers + Constructor sizing). The post-research - # check (#60) caught breaches during research but a parse-fail - # or empty-plan exit at this point would still skip - # deterministic liquidation until the next intra tick. Codex - # r8 #1 caught this gap — same fix as #60, just one stage - # later in the pipeline. - late_breach = self._check_late_breach_and_emergency_liquidate( - run_id, "post-decision", - ) - if late_breach is not None: - return late_breach + # RC2 resume lane: a prior morning tick may have been killed by + # the wrapper timeout AFTER the PM produced a plan but BEFORE the + # RiskStage reviewed it (the observed death mode: 61/61 BUY- + # proposal days destroyed at the PM→RM boundary during the + # 6/30-7/15 relay outage). If today's unconsumed checkpoint + # exists and is fresh, skip research+PM entirely — the full + # preamble above (drains, coverage audit, force_delever, circuit + # breaker, FRESH account snapshot) has already run, and the + # RiskStage + execution guards below all operate on live state. + # RM always re-runs; there is no resume-past-RM. + from src import decision_checkpoint as _dc + resumed = _dc.load("morning") + if resumed is not None: + logger.warning( + "RESUME LANE: unconsumed decision checkpoint from %s " + "(age %.0f min, %d decisions) — skipping research+PM, " + "re-entering at RiskStage on fresh account state", + resumed["run_id"], resumed["age_minutes"], + len(resumed["portfolio_decision"].decisions), + ) + ctx.macro_summary = resumed["macro_summary"] + ctx.macro_analysis = resumed["macro_analysis"] + ctx.news_intel = resumed["news_intel"] + ctx.analyses = resumed["analyses"] + ctx.earnings_results = resumed["earnings_results"] + ctx.data_status = resumed["data_status"] + ctx.portfolio_decision = resumed["portfolio_decision"] + portfolio_decision = ctx.portfolio_decision + else: + # Phase 4 #1: research stage runs the parallel fan-out (macro / + # news / tech / earnings). Populates ctx fields. + self.morning_research_stage.run(ctx) + analyses = ctx.analyses + + # Late-breach check: research can take 5-10 min on slow OpenAI + # days. The pre-research circuit breaker (#45) caught open-gap + # losses; this catches the case where the tape crosses the + # daily-loss limit DURING research and the morning would + # otherwise bail to no_data/no_trades, leaving the breach for + # the next intra tick (30 min away). Mirror the pre-research + # bypass: deterministic emergency liquidate, no LLM dependency. + late_breach = self._check_late_breach_and_emergency_liquidate( + run_id, "post-research", + ) + if late_breach is not None: + return late_breach + + if not analyses: + logger.warning("No analyses produced, skipping trading") + return {"status": "no_data", "orders": [], "run_id": run_id} + + # Phase 4 #1: decision stage — memory layers + PM + Constructor. + self._decision_stage(ctx) + portfolio_decision = ctx.portfolio_decision + + # Persist the plan the moment it exists — a kill anywhere + # between here and execution leaves a resumable checkpoint + # instead of a wasted research+PM spend. + _dc.write(ctx) + + # Second late-breach check: PM is itself a multi-second LLM + # call (memory layers + Constructor sizing). The post-research + # check (#60) caught breaches during research but a parse-fail + # or empty-plan exit at this point would still skip + # deterministic liquidation until the next intra tick. Codex + # r8 #1 caught this gap — same fix as #60, just one stage + # later in the pipeline. + late_breach = self._check_late_breach_and_emergency_liquidate( + run_id, "post-decision", + ) + if late_breach is not None: + return late_breach if not portfolio_decision: logger.info("Portfolio manager: parse failed, no decision object") @@ -5231,6 +5259,13 @@ def run_morning(self) -> dict: # Phase 4 #1: risk stage — hard filter + earnings cap + RM review + mods. early_exit = self._risk_stage(ctx) + # The plan has now been risk-reviewed — whatever the outcome, it + # must never be re-offered by the resume lane (an RM-rejected + # plan retried next tick would be a veto bypass), and marking + # BEFORE execution makes the execution at-most-once (a kill + # mid-execution is owned by the BUY write-ahead orphan sweep, + # not by re-running the plan). + _dc.mark_consumed("morning") if early_exit is not None: early_exit["run_id"] = run_id early_exit["data_status"] = dict(ctx.data_status) diff --git a/tests/test_decision_checkpoint.py b/tests/test_decision_checkpoint.py new file mode 100644 index 00000000..fc42fb16 --- /dev/null +++ b/tests/test_decision_checkpoint.py @@ -0,0 +1,210 @@ +"""RC2 decision checkpoint + resume lane. + +The 6/30-7/15 death mode: morning killed by the wrapper timeout at the +PM→RM boundary, 61/61 BUY-proposal days destroyed, every retry tick +re-burning the full research pipeline. The checkpoint persists the plan +when it exists; the next tick resumes at RiskStage after the full +preamble. Safety contract pinned here: + + - resume path NEVER calls research/PM again, but RM ALWAYS re-runs; + - the checkpoint is consumed on ANY RiskStage outcome (incl. reject) and + BEFORE ExecutionStage submits (at-most-once); + - stale (>90min), consumed, wrong-version, or corrupt checkpoints are + ignored; + - checkpoint failures never crash the session. +""" +import json +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from unittest.mock import MagicMock + +from src import decision_checkpoint as dc +from src.models import ( + PortfolioDecision, ReasoningChain, TradeDecision, +) +from src.pipeline import TradingPipeline +from src.pipeline_context import RunContext + + +def _pm_rc(): + return ReasoningChain( + macro_filter="x", news_check="x", earnings_check="x", + signal_conflicts="x", sizing_logic="x", + portfolio_balance="x", cash_target="x", + ) + + +def _decision(symbol="NVDA"): + return TradeDecision(action="BUY", symbol=symbol, allocation_pct=10.0, + entry_price=100.0, stop_loss=90.0, + take_profit=130.0, reasoning="test") + + +def _ctx_with_plan() -> RunContext: + ctx = RunContext.start("morning") + ctx.portfolio_decision = PortfolioDecision( + reasoning_chain=_pm_rc(), decisions=[_decision()], portfolio_view="v", + ) + ctx.analyses = [] + ctx.news_intel = None + ctx.macro_analysis = None + ctx.macro_summary = {"vix": {"current": 18.0}} + ctx.earnings_results = [{"symbol": "NKE", "queued": True, "analysis": None}] + ctx.data_status = {"tech": "ok"} + return ctx + + +def _point_dir_at(monkeypatch, tmp_path): + monkeypatch.setattr(dc, "CHECKPOINT_DIR", tmp_path) + + +# ---------- module round-trip ---------- + +def test_checkpoint_roundtrip(monkeypatch, tmp_path): + _point_dir_at(monkeypatch, tmp_path) + ctx = _ctx_with_plan() + path = dc.write(ctx) + assert path is not None and path.exists() + + loaded = dc.load("morning") + assert loaded is not None + assert loaded["run_id"] == ctx.run_id + pd = loaded["portfolio_decision"] + assert isinstance(pd, PortfolioDecision) + assert pd.decisions[0].symbol == "NVDA" + assert loaded["earnings_results"][0]["symbol"] == "NKE" + assert loaded["macro_summary"]["vix"]["current"] == 18.0 + + +def test_checkpoint_empty_plan_not_written(monkeypatch, tmp_path): + _point_dir_at(monkeypatch, tmp_path) + ctx = _ctx_with_plan() + ctx.portfolio_decision = PortfolioDecision( + reasoning_chain=_pm_rc(), decisions=[], portfolio_view="v", + ) + assert dc.write(ctx) is None + assert dc.load("morning") is None + + +def test_checkpoint_consumed_not_loaded(monkeypatch, tmp_path): + _point_dir_at(monkeypatch, tmp_path) + dc.write(_ctx_with_plan()) + dc.mark_consumed("morning") + assert dc.load("morning") is None + + +def test_checkpoint_stale_not_loaded(monkeypatch, tmp_path): + _point_dir_at(monkeypatch, tmp_path) + path = dc.write(_ctx_with_plan()) + payload = json.loads(path.read_text()) + payload["created_at_utc"] = ( + datetime.now(timezone.utc) - timedelta(minutes=120) + ).isoformat() + path.write_text(json.dumps(payload)) + assert dc.load("morning") is None + + +def test_checkpoint_corrupt_is_ignored(monkeypatch, tmp_path): + _point_dir_at(monkeypatch, tmp_path) + dc.checkpoint_path("morning").parent.mkdir(parents=True, exist_ok=True) + dc.checkpoint_path("morning").write_text("{not json") + assert dc.load("morning") is None # no raise + + +def test_checkpoint_wrong_version_ignored(monkeypatch, tmp_path): + _point_dir_at(monkeypatch, tmp_path) + path = dc.write(_ctx_with_plan()) + payload = json.loads(path.read_text()) + payload["version"] = 999 + path.write_text(json.dumps(payload)) + assert dc.load("morning") is None + + +# ---------- run_morning resume behavior ---------- + +def _resume_pipeline(): + """__new__-built pipeline with every preamble dependency stubbed.""" + p = TradingPipeline.__new__(TradingPipeline) + p._is_trading_day = lambda: True + p._drain_pending_protection_restores = MagicMock() + p._reconcile_orphan_pending_submits = MagicMock() + p._reconcile_stop_coverage = MagicMock(return_value=[]) + p._reconcile_fills = MagicMock() + p._force_delever = MagicMock(return_value=[]) + p.broker = MagicMock() + p.broker.get_account.return_value = { + "cash": 50_000.0, "portfolio_value": 100_000.0, "last_equity": 100_000.0, + } + p.broker.get_positions.return_value = [] + p.risk_engine = MagicMock() + p.risk_engine.check_daily_loss.return_value = None + p.morning_research_stage = MagicMock() + p.risk_stage = MagicMock() + p.risk_stage.run.return_value = None # RM approved, proceed + p.execution_stage = MagicMock() + p.execution_stage.run.return_value = [{"id": "o1", "action": "BUY"}] + p.decision_stage = MagicMock() + return p + + +def test_run_morning_resumes_and_rm_still_runs(monkeypatch, tmp_path): + _point_dir_at(monkeypatch, tmp_path) + dc.write(_ctx_with_plan()) + + p = _resume_pipeline() + result = p.run_morning() + + # research + PM skipped; RM and execution ran on the checkpointed plan + p.morning_research_stage.run.assert_not_called() + p.decision_stage.run.assert_not_called() + p.risk_stage.run.assert_called_once() + p.execution_stage.run.assert_called_once() + assert result["status"] == "executed" + # consumed BEFORE execution → a second morning tick does a normal run + assert dc.load("morning") is None + + +def test_run_morning_rm_reject_consumes_checkpoint(monkeypatch, tmp_path): + """An RM-rejected plan must never be re-offered by the resume lane — + retrying it next tick would be a veto bypass.""" + _point_dir_at(monkeypatch, tmp_path) + dc.write(_ctx_with_plan()) + + p = _resume_pipeline() + p.risk_stage.run.return_value = {"status": "rejected", "orders": [], + "reason": "cluster risk"} + result = p.run_morning() + + assert result["status"] == "rejected" + p.execution_stage.run.assert_not_called() + assert dc.load("morning") is None # consumed despite the reject + + +def test_run_morning_without_checkpoint_runs_full_pipeline(monkeypatch, tmp_path): + _point_dir_at(monkeypatch, tmp_path) + + p = _resume_pipeline() + + def _research(ctx): + ctx.analyses = [SimpleNamespace( + symbol="NVDA", rating="buy", + model_dump=lambda mode=None: {"symbol": "NVDA", "rating": "buy"}, + )] + ctx.data_status = {"tech": "ok"} + p.morning_research_stage.run.side_effect = _research + + def _decide(ctx): + ctx.portfolio_decision = PortfolioDecision( + reasoning_chain=_pm_rc(), decisions=[_decision()], portfolio_view="v", + ) + p.decision_stage.run.side_effect = _decide + p._check_late_breach_and_emergency_liquidate = MagicMock(return_value=None) + + result = p.run_morning() + + p.morning_research_stage.run.assert_called_once() + p.decision_stage.run.assert_called_once() + assert result["status"] == "executed" + # the plan was checkpointed mid-run, then consumed before execution + assert dc.checkpoint_path("morning").exists() + assert json.loads(dc.checkpoint_path("morning").read_text())["consumed"] is True From caec0cd5b6c623516000c41928d486e6e25006c2 Mon Sep 17 00:00:00 2001 From: yebof Date: Thu, 16 Jul 2026 16:16:14 +0800 Subject: [PATCH 6/8] feat(obs): report violent session deaths + completion-aware dead-man check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RC5 (2026-07-16 forensics): a 1-day incident became 13+ days because both observability layers were blind to kills: (a) timeout's SIGTERM/SIGKILL skips python's finally-block notifier — 13 straight mornings died with ZERO Telegram pushes; (b) evening's _expected_sessions_missing_today only checked that ANY run- row existed — partial runs (research logged, then killed) satisfied it, so the 🔴 banner never fired either. - Wrapper: on violent deaths only (124/137/143 — the codes where python's own notifier cannot run) push a Telegram message directly from bash; ordinary non-zero exits stay python-owned to avoid duplicate pushes. TELEGRAM_DISABLED kill switch honored. - Wrapper: optional HEALTHCHECKS_URL ping on success, /fail on failure — the external dead-man's switch CLAUDE.md has wishlisted since May; covers total host death and evening-not-firing, which no in-process check can. - Dead-man check: two sharper probes when 'run-' rows exist — (a) research ran but portfolio_manager never logged (killed mid-research), (b) the decision checkpoint exists unconsumed (killed at the PM→RM boundary, the observed death mode). New db.agent_names_logged_on() supports the probe. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PQkESoSTYx2bCy7WYnTPXR --- scripts/run_if_et_window.sh | 33 +++++++++++++ src/pipeline.py | 28 +++++++++++- src/storage/db.py | 20 ++++++++ tests/test_et_window_script.py | 84 ++++++++++++++++++++++++++++++++++ 4 files changed, 164 insertions(+), 1 deletion(-) diff --git a/scripts/run_if_et_window.sh b/scripts/run_if_et_window.sh index 673b038a..a114bcdf 100755 --- a/scripts/run_if_et_window.sh +++ b/scripts/run_if_et_window.sh @@ -190,16 +190,49 @@ if [[ -f "${PROJECT_ROOT}/.env" ]]; then set +a fi +# Best-effort Telegram push from BASH. RC5 (2026-07-16): when `timeout` +# SIGTERM/SIGKILLs python, the finally-block notifier never runs — 13 +# straight days of morning kills produced ZERO failure notifications. +# Python cannot be trusted to report its own violent death; the wrapper can. +notify_telegram() { + local text="$1" + [[ "${TELEGRAM_DISABLED:-0}" == "1" ]] && return 0 + [[ -z "${TELEGRAM_BOT_TOKEN:-}" || -z "${TELEGRAM_CHAT_ID:-}" ]] && return 0 + curl -sS --max-time 10 \ + "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ + --data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \ + --data-urlencode "text=${text}" >/dev/null 2>&1 || true +} + +# External dead-man's switch (CLAUDE.md wishlist): if HEALTHCHECKS_URL is +# set in .env, ping it on success and /fail on failure. An absent ping +# then alerts from OUTSIDE the host — the only coverage for total host +# death / evening itself not firing. +ping_healthcheck() { + local suffix="${1:-}" + [[ -z "${HEALTHCHECKS_URL:-}" ]] && return 0 + curl -fsS --max-time 10 --retry 2 "${HEALTHCHECKS_URL}${suffix}" >/dev/null 2>&1 || true +} + if "$TIMEOUT" --kill-after=30 1200 "$PYTHON" main.py --mode "$MODE"; then # intra_check is intentionally guard-less (see last-run guard block above) — # we don't write the marker for it, so the next 30-min tick can fire freely. if [[ "$MODE" != "intra_check" ]]; then echo "${ET_DATE} ${NOW_UNIX}" > "$LAST_FILE" fi + ping_healthcheck exit 0 else STATUS=$? fi echo "[$(date '+%Y-%m-%d %H:%M:%S %Z')] ${MODE} failed with status ${STATUS}; not updating last-run guard" >&2 +# Bash-side push ONLY for violent deaths (124=timeout, 137=SIGKILL, +# 143=SIGTERM) — those skip python's finally-block notifier entirely. For +# ordinary non-zero exits python already pushed its own FAILED message; +# pushing again here would just teach the operator to ignore duplicates. +if [[ "$STATUS" -eq 124 || "$STATUS" -eq 137 || "$STATUS" -eq 143 ]]; then + notify_telegram "🔴 quant-agent ${MODE} KILLED (status ${STATUS}) on ${ET_DATE} — python got no chance to notify. Next tick retries (morning resumes from the decision checkpoint if one was written)." +fi +ping_healthcheck "/fail" exit "$STATUS" diff --git a/src/pipeline.py b/src/pipeline.py index a9ae27ed..2cdbcdbb 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -6408,7 +6408,33 @@ def _expected_sessions_missing_today(self) -> list[str]: return [] # run_id prefix -> display name; morning's prefix is 'run'. expected = {"run": "morning", "midday": "midday", "close": "close"} - return [name for prefix, name in expected.items() if prefix not in present] + missing = [name for prefix, name in expected.items() if prefix not in present] + + # RC5 (2026-07-16): "any run- row exists" cannot tell a completed + # morning from one killed mid-flight — research rows land BEFORE the + # kill, so 13 straight days of morning deaths passed this check and + # the 🔴 banner never fired. Two sharper probes: + if "morning" not in missing and "run" in present: + # (a) research logged but the PM never ran → died during research. + try: + agents = self.db.agent_names_logged_on("run-") + if agents and "portfolio_manager" not in agents: + missing.append("morning (research ran, PM never did — killed mid-run?)") + except Exception as exc: # noqa: BLE001 + logger.warning("missing-session check: agent probe failed: %s", exc) + # (b) PM plan checkpointed but never consumed → killed before the + # RiskStage reviewed it (the observed 6/30-7/15 death mode). + try: + import json as _json + from src import decision_checkpoint as _dc + p = _dc.checkpoint_path("morning") + if p.exists() and _json.loads(p.read_text()).get("consumed") is False: + missing.append( + "morning (PM plan never risk-reviewed — checkpoint unconsumed)" + ) + except Exception as exc: # noqa: BLE001 + logger.warning("missing-session check: checkpoint probe failed: %s", exc) + return missing def _maybe_run_quarterly_meta(self) -> dict | None: """Evening-time piggyback for the quarterly meta-reflection loop. diff --git a/src/storage/db.py b/src/storage/db.py index 17e8ebca..325dae05 100644 --- a/src/storage/db.py +++ b/src/storage/db.py @@ -846,6 +846,26 @@ def session_prefixes_logged_on(self, trading_day: date | None = None) -> set[str prefixes.add(rid.rsplit("-", 1)[0] if "-" in rid else rid) return prefixes + def agent_names_logged_on(self, run_id_prefix: str, + trading_day: date | None = None) -> set[str]: + """Distinct agent_name values logged on the given ET trading day for + run_ids starting with `run_id_prefix` (e.g. 'run-' for morning). + + RC5 (2026-07-16): the prefix check above can't tell a COMPLETED + morning from one killed mid-flight — research rows land before the + kill, so 'run' shows present while PM/RM never ran. This lets the + dead-man's check ask "did the pipeline actually reach the decision + stage?" + """ + start_utc, end_utc = self._et_day_utc_bounds(trading_day) + with self._lock: + rows = self.conn.execute( + "SELECT DISTINCT agent_name FROM agent_logs " + "WHERE timestamp >= ? AND timestamp < ? AND run_id LIKE ?", + (start_utc, end_utc, f"{run_id_prefix}%"), + ).fetchall() + return {r[0] for r in rows if r[0]} + def sum_session_cost(self, run_id: str) -> tuple[float | None, int]: """Total cost + per-call count for a session's run_id. diff --git a/tests/test_et_window_script.py b/tests/test_et_window_script.py index c4025528..bf74c595 100644 --- a/tests/test_et_window_script.py +++ b/tests/test_et_window_script.py @@ -339,3 +339,87 @@ def test_run_if_et_window_intra_check_skips_outside_window(tmp_path): # Neither tick should have exec'd the fake-python assert not counter_file.exists() + + +# ============================================================================ +# RC5 (2026-07-16): bash-side kill notification + external dead-man ping. +# When `timeout` SIGTERM/SIGKILLs python, the in-process finally-block +# notifier never runs — 13 straight days of morning kills produced zero +# failure pushes. The wrapper must report violent deaths itself, and ping +# HEALTHCHECKS_URL so an EXTERNAL monitor sees liveness. +# ============================================================================ + +def _base_env(tmp_path, python_body: str) -> dict: + project_root = tmp_path / "project" + project_root.mkdir(exist_ok=True) + (project_root / ".env").write_text("") + timeout_bin = tmp_path / "timeout" + python_bin = tmp_path / "fake-python" + curl_log = tmp_path / "curl.log" + curl_bin = tmp_path / "curl" + _write_executable(timeout_bin, "#!/bin/bash\nshift 2\nexec \"$@\"\n") + _write_executable(python_bin, f"#!/bin/bash\n{python_body}\n") + _write_executable(curl_bin, f"#!/bin/bash\necho \"$@\" >> {curl_log}\nexit 0\n") + env = os.environ | { + "PROJECT_ROOT_OVERRIDE": str(project_root), + "PYTHON_OVERRIDE": str(python_bin), + "TIMEOUT_OVERRIDE": str(timeout_bin), + "LAST_RUN_DIR_OVERRIDE": str(tmp_path / "cache"), + "ET_DOW_OVERRIDE": "1", + "ET_HOUR_OVERRIDE": "08", + "ET_MIN_OVERRIDE": "30", + "ET_DATE_OVERRIDE": "2026-07-16", + "NOW_UNIX_OVERRIDE": "1234567890", + "PATH": f"{tmp_path}:{os.environ['PATH']}", # fake curl first + } + return env + + +def _run(script_env, mode="earnings_preprocess"): + script = Path(__file__).resolve().parents[1] / "scripts" / "run_if_et_window.sh" + return subprocess.run(["bash", str(script), mode], env=script_env, + capture_output=True, text=True, check=False) + + +def test_wrapper_notifies_telegram_on_timeout_kill(tmp_path): + env = _base_env(tmp_path, "exit 124") + env |= {"TELEGRAM_BOT_TOKEN": "tok", "TELEGRAM_CHAT_ID": "42"} + result = _run(env) + assert result.returncode == 124 + log = (tmp_path / "curl.log").read_text() + assert "sendMessage" in log + assert "KILLED" in log + + +def test_wrapper_does_not_duplicate_python_notifier_on_plain_failure(tmp_path): + """Ordinary non-zero exits: python's own finally-block already pushed — + a second bash push would train the operator to ignore duplicates.""" + env = _base_env(tmp_path, "exit 1") + env |= {"TELEGRAM_BOT_TOKEN": "tok", "TELEGRAM_CHAT_ID": "42"} + result = _run(env) + assert result.returncode == 1 + log_file = tmp_path / "curl.log" + assert not log_file.exists() or "sendMessage" not in log_file.read_text() + + +def test_wrapper_respects_telegram_kill_switch(tmp_path): + env = _base_env(tmp_path, "exit 137") + env |= {"TELEGRAM_BOT_TOKEN": "tok", "TELEGRAM_CHAT_ID": "42", + "TELEGRAM_DISABLED": "1"} + _run(env) + log_file = tmp_path / "curl.log" + assert not log_file.exists() or "sendMessage" not in log_file.read_text() + + +def test_wrapper_pings_healthchecks_on_success_and_fail(tmp_path): + env = _base_env(tmp_path, "exit 0") + env |= {"HEALTHCHECKS_URL": "https://hc-ping.example/uuid-1"} + assert _run(env).returncode == 0 + assert "https://hc-ping.example/uuid-1" in (tmp_path / "curl.log").read_text() + + second = tmp_path / "second" + second.mkdir() + env2 = _base_env(second, "exit 124") + env2 |= {"HEALTHCHECKS_URL": "https://hc-ping.example/uuid-1"} + _run(env2) + assert "https://hc-ping.example/uuid-1/fail" in (second / "curl.log").read_text() From 18a2889d65dd3c939b877791f651292998f9fda0 Mon Sep 17 00:00:00 2001 From: yebof Date: Thu, 16 Jul 2026 16:22:30 +0800 Subject: [PATCH 7/8] fix(cash): reviewer view actually excludes the vehicle; midday/close park bookend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps caught in self-review before the adversarial pass: - run_position_review computed the SGOV-free view but still passed raw positions into position facts / the reviewer / the action dispatcher — the reviewer would have hold-graded parked cash. - park_excess was hooked into morning only; midday/close SELL proceeds would have sat unswept overnight. Both sessions now park at the bookend (emergency paths return earlier and deliberately skip parking). Adds an end-to-end run_midday test pinning both behaviors. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PQkESoSTYx2bCy7WYnTPXR --- src/pipeline.py | 23 ++++++++++++--- tests/test_cash_sweep.py | 62 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 4 deletions(-) diff --git a/src/pipeline.py b/src/pipeline.py index 2cdbcdbb..770a81d7 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -5660,7 +5660,7 @@ def run_position_review(self, session_type: str = "midday") -> dict: raw_avg = calib.get("avg_hold_days") avg_hold_days = raw_avg if isinstance(raw_avg, (int, float)) else None position_facts = self._build_position_facts( - positions, morning_trades, total_value, avg_hold_days, + review_positions, morning_trades, total_value, avg_hold_days, ) # Memory layers — share the same helpers PM uses. @@ -5681,7 +5681,7 @@ def run_position_review(self, session_type: str = "midday") -> dict: recent_performance = self._compute_recent_performance(last_equity) review, md_result = self.position_reviewer.review( - positions=positions, + positions=review_positions, macro_summary=macro_summary, cash_balance=cash, total_value=total_value, @@ -5705,7 +5705,7 @@ def run_position_review(self, session_type: str = "midday") -> dict: self.db.insert_agent_log( agent_name="position_reviewer", run_id=run_id, input_summary=( - f"{session_type} | {len(positions)} positions, ${total_value:.0f} total" + f"{session_type} | {len(review_positions)} positions, ${total_value:.0f} total" ), input_message=md_result.user_message, output_summary=review.overall_assessment if review else "parse_error", @@ -5727,7 +5727,7 @@ def run_position_review(self, session_type: str = "midday") -> dict: )) else: orders.extend(self._midday_execute_llm_actions( - positions, review, run_id, + review_positions, review, run_id, blocked_symbols=blocked_position_symbols, already_trimmed_today=already_trimmed_today, )) @@ -5739,6 +5739,21 @@ def run_position_review(self, session_type: str = "midday") -> dict: # Reconcile everything still marked submitted (today's new orders + # any lingering from morning that didn't reach terminal in time). self._reconcile_fills() + + # Bookend: park cash freed by this session's sells (and any still-idle + # excess) — without this, midday/close SELL proceeds sit unswept until + # tomorrow's morning bookend. park_excess refreshes account state and + # subtracts open-BUY holds itself; emergency paths returned earlier and + # deliberately skip parking. + sweeper = self._sweeper() + if sweeper is not None: + try: + sweep_order = sweeper.park_excess(ctx) + if sweep_order: + orders.append(sweep_order) + except Exception as e: # noqa: BLE001 + logger.warning("cash sweep: park_excess failed (non-fatal): %s", e) + return { "status": "reviewed", "session": session_type, diff --git a/tests/test_cash_sweep.py b/tests/test_cash_sweep.py index 5976d1ad..2455d99f 100644 --- a/tests/test_cash_sweep.py +++ b/tests/test_cash_sweep.py @@ -297,3 +297,65 @@ def test_cash_sweep_config_defaults_disabled(): def test_cash_sweep_config_uppercases_symbol(): assert CashSweepConfig(symbol=" bil ").symbol == "BIL" + + +# ---------- session integration: reviewer never sees the vehicle; midday parks ---------- + +def test_position_review_hides_vehicle_and_parks_at_end(tmp_path): + """End-to-end through run_midday: (a) the reviewer's position list must + exclude the sweep vehicle (it would otherwise hold-grade / sell parked + cash), (b) the session bookend parks idle cash left by sells.""" + from unittest.mock import patch + from src.models import PositionReview, PositionReasoningChain + from src.storage.db import Database + + db = Database(str(tmp_path / "t.db")) + db.initialize() + + p = _sweep_pipeline() + p.db = db + p.broker.is_trading_day.return_value = True + p.broker.get_session_close = MagicMock(return_value=None) + p.broker.get_account.return_value = { + "cash": 85_000.0, "portfolio_value": 100_000.0, "last_equity": 100_000.0, + } + p.broker.get_positions.return_value = [SGOV, NVDA] + p.broker.open_buy_notional.return_value = 0.0 + p.broker.get_latest_price.return_value = 100.60 + p.broker.submit_order.return_value = {"id": "sweep-1", "status": "accepted"} + p.broker.snapshot_protective_stops.return_value = (True, []) + p.macro = MagicMock() + p.macro.get_macro_summary.return_value = {} + p.macro_store = MagicMock() + p.macro_store.load_last_state.return_value = None + p.config.llm = MagicMock() + p.config.llm.position_reviewer_model = "test-model" + p._auto_take_profit = MagicMock(return_value=[]) + p._handle_ex_dividends = MagicMock(return_value=[]) + p._run_news_update = MagicMock(return_value=None) + p._load_earnings_analyses = MagicMock(return_value=(None, [])) + p._midday_execute_llm_actions = MagicMock(return_value=[]) + p._reconcile_stop_coverage = MagicMock(return_value=[]) + p.risk_engine = MagicMock() + p.risk_engine.check_daily_loss.return_value = None + p.position_reviewer = MagicMock() + p.position_reviewer.review.return_value = ( + PositionReview( + reasoning_chain=PositionReasoningChain( + macro_continuity_check="x", thesis_progress_check="x", + thesis_integrity_check="x", winners_discipline_check="x", + session_disposition_check="x", execution_rationale="x", + ), + actions=[], overall_assessment="stable", risk_level="low", + ), + MagicMock(user_message="m", raw_text="{}", tokens_used=1, + input_tokens=1, output_tokens=1, cost_usd=0.0), + ) + + result = p.run_midday() + + assert result["status"] == "reviewed" + seen = p.position_reviewer.review.call_args.kwargs["positions"] + assert [x.symbol for x in seen] == ["NVDA"], "reviewer must not see SGOV" + # bookend parked the idle cash: a SWEEP_BUY order rides in the result + assert any(o.get("action") == "SWEEP_BUY" for o in result["orders"]) From c05d003f67df39476d451ea947483b8c5885217f Mon Sep 17 00:00:00 2001 From: yebof Date: Thu, 16 Jul 2026 17:04:42 +0800 Subject: [PATCH 8/8] fix(review): apply 22 confirmed findings from the adversarial review pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 34-agent review workflow (5 lenses × independent skeptic per finding) confirmed 22 defects (7 claims refuted). Grouped fixes: - RM sweep view (major): RiskStage passed raw positions to the RiskManager — the VETO layer saw parked SGOV as an 84%-of-book position while PM saw the same dollars as cash, desynchronizing the two risk layers with veto power on the corrupted view. RM/correlation/has_book now use the scrubbed list; the hard filter keeps RAW positions (it derives the parked-cash credit from finding the vehicle itself). Regression test added. - Emergency×park oscillation (major): the post-review breach branch in run_position_review fell through to the park bookend — the system would buy SGOV with ~all equity minutes after force-selling everything, then the next intra tick would emergency-sell the fresh SGOV lot. Now returns emergency_sold immediately, mirroring the pre-review breaker. - Ex-div false cooldown (major): ex-div adjustments write TRAIL_STOP rows (stop LOWERING) — they no longer count as a 'tighten' for the ratchet cooldown, so dividend names don't get spurious 2-day trail freezes. - Resume-lane bars (major): the checkpoint deliberately omits symbols_bars; resume now rehydrates OHLCV for the plan's BUY symbols deterministically, restoring the entry ATR floor and the correlation advisory on resume. - Checkpoint lifecycle: mark_consumed is fail-CLOSED (falls back to deleting the file — unlink survives ENOSPC — and returns bool); both emergency-liquidation exits consume the checkpoint and record a status marker; legit no_data mornings record status too. - Dead-man accuracy: probes skip legitimately-PM-less mornings via the status marker; the notifier matches decorated 'morning (...)' entries so the sharpened diagnoses actually reach the 🔴 banner. - Sweep hygiene: evening today_trades excludes SWEEP_* churn (limit 30→20 after filter); post-exit reality skips the sweep symbol (emergency exits of SGOV are not decision-quality data); park_excess sizes qty against the LIMIT price so a padded fill can't overdraw a thin reserve. - Entry ATR floor: after widening, recompute R/R vs the unchanged target and skip the BUY below 1.2 — RM approved the tight-stop geometry, not a collapsed one. - PMFacts renders the OVER-deployment case instead of calling +40pp 'within band'. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PQkESoSTYx2bCy7WYnTPXR --- src/decision_checkpoint.py | 63 ++++++++++++++++++++---- src/execution/cash_sweep.py | 9 ++-- src/notifier.py | 15 ++++-- src/pipeline.py | 80 +++++++++++++++++++++++++++++-- src/pipeline_context.py | 8 ++++ src/pipeline_stages.py | 39 +++++++++++++-- tests/test_bugfixes.py | 4 +- tests/test_cash_sweep.py | 65 ++++++++++++++++++++++++- tests/test_decision_checkpoint.py | 4 ++ tests/test_notifier.py | 2 +- 10 files changed, 261 insertions(+), 28 deletions(-) diff --git a/src/decision_checkpoint.py b/src/decision_checkpoint.py index da378a67..fabbbfde 100644 --- a/src/decision_checkpoint.py +++ b/src/decision_checkpoint.py @@ -135,25 +135,68 @@ def load(session: str, max_age_minutes: float = MAX_AGE_MINUTES) -> dict | None: return None -def mark_consumed(session: str) -> None: - """Flip consumed=true in place. Never raises. - - Called (a) right before ExecutionStage submits (at-most-once for BUYs) - and (b) on any RiskStage early-exit — an RM-rejected or hard-blocked - plan must never be re-offered by the resume lane. +def mark_consumed(session: str) -> bool: + """Flip consumed=true in place; returns True when the checkpoint is + guaranteed dead (consumed or gone). Never raises. + + Called (a) right before ExecutionStage submits (at-most-once for BUYs), + (b) on any RiskStage early-exit — an RM-rejected or hard-blocked plan + must never be re-offered — and (c) on emergency-liquidation exits. + + Fail-CLOSED: if rewriting the file fails (disk full, permissions), fall + back to deleting it — for load() a missing checkpoint equals a consumed + one, and unlink succeeds under ENOSPC where write_text cannot. A + swallowed failure here would leave the plan live, which is the unsafe + direction for the at-most-once contract. """ + path = checkpoint_path(session) try: - path = checkpoint_path(session) if not path.exists(): - return + return True payload = json.loads(path.read_text()) if payload.get("consumed") is True: - return + return True payload["consumed"] = True payload["consumed_at_utc"] = datetime.now(timezone.utc).isoformat() tmp = path.with_suffix(".json.tmp") tmp.write_text(json.dumps(payload)) tmp.replace(path) logger.info("decision checkpoint %s marked consumed", path) + return True + except Exception as e: # noqa: BLE001 + logger.error("decision checkpoint consume-mark failed (%s) — " + "deleting the checkpoint instead (fail-closed)", e) + try: + path.unlink(missing_ok=True) + return True + except Exception as e2: # noqa: BLE001 + logger.error("decision checkpoint delete also failed: %s — " + "resume lane may re-offer this plan!", e2) + return False + + +def write_status(session: str, status: str) -> None: + """Record a legitimate PM-less terminal status for the ET day + (no_data / emergency_sold). The evening dead-man probe reads this to + avoid false 'morning killed mid-run' alarms. Never raises. + """ + try: + path = checkpoint_path(session).with_suffix(".status") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({ + "status": status, + "at_utc": datetime.now(timezone.utc).isoformat(), + })) except Exception as e: # noqa: BLE001 - logger.warning("decision checkpoint consume-mark failed: %s", e) + logger.warning("session status write failed: %s", e) + + +def read_status(session: str) -> str | None: + """Today's recorded terminal status, or None. Never raises.""" + try: + path = checkpoint_path(session).with_suffix(".status") + if not path.exists(): + return None + return json.loads(path.read_text()).get("status") + except Exception: # noqa: BLE001 + return None diff --git a/src/execution/cash_sweep.py b/src/execution/cash_sweep.py index a9b4d78c..7e9b6fba 100644 --- a/src/execution/cash_sweep.py +++ b/src/execution/cash_sweep.py @@ -248,11 +248,14 @@ def park_excess(self, ctx) -> dict | None: if not (isinstance(price, (int, float)) and price > 0 and math.isfinite(price)): logger.warning("cash sweep: no price for %s — skipping park", cfg.symbol) return None - qty = int(excess / price) + # Size against the LIMIT price (what a fill can actually cost), not + # the quote — sizing on the quote could overdraw raw cash by the pad + # amount when the reserve is configured thin (review finding; SWEEP + # orders don't pass through the cash_only engine). + limit_price = round(price * _BUY_LIMIT_PAD, 2) + qty = int(excess / limit_price) if qty <= 0: return None - - limit_price = round(price * _BUY_LIMIT_PAD, 2) # Write-ahead row before the broker call — same crash-recovery # pattern as ExecutionStage BUYs (orphan sweep matches on # fill_status='pending_submit'). diff --git a/src/notifier.py b/src/notifier.py index 14ea076f..71ffea6f 100644 --- a/src/notifier.py +++ b/src/notifier.py @@ -326,12 +326,17 @@ def _append_evening_body(lines: list[str], result: dict) -> None: # legitimately skipped on some early-close days → softer ⚠️. missing = result.get("missing_sessions") if isinstance(missing, list) and missing: - if "morning" in missing: - lines.append( - "🔴 SESSION DID NOT RUN TODAY: morning — no agent activity " - "logged; check the timer/scheduler" + # Prefix match: the sharpened probes emit decorated entries like + # "morning (PM plan never risk-reviewed — checkpoint unconsumed)" — + # they carry the diagnosis and must hit the hard banner too. + hard = [m for m in missing + if m == "morning" or str(m).startswith("morning (")] + for m in hard: + detail = m if m != "morning" else ( + "morning — no agent activity logged; check the timer/scheduler" ) - soft = [m for m in missing if m != "morning"] + lines.append(f"🔴 MORNING SESSION INCOMPLETE TODAY: {detail}") + soft = [m for m in missing if m not in hard] if soft: lines.append(f"⚠️ no activity logged today for: {', '.join(soft)}") diff --git a/src/pipeline.py b/src/pipeline.py index 770a81d7..028e58a0 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -3027,9 +3027,16 @@ def _build_post_exit_reality( now = _dt.now(timezone.utc) window_start = now - timedelta(days=lookback_days) age_cutoff = now - timedelta(days=min_age_days) + sweeper = self._sweeper() + sweep_symbol = sweeper.symbol if sweeper is not None else None exits: list[dict] = [] for row in rows: action = (row.get("action") or "").upper() + # Belt on top of the SWEEP_* action exclusion: an emergency + # liquidation can exit the sweep vehicle under EMERGENCY_SELL — + # a ~0% T-bill "move" is noise in a decision-quality audit. + if sweep_symbol is not None and (row.get("symbol") or "") == sweep_symbol: + continue is_exit = ( action in self._EXIT_AUDIT_ACTIONS or action.startswith("PARTIAL_SELL") @@ -4717,6 +4724,12 @@ def _trail_tightened_recently(self, symbol: str, calendar_days: int = 4) -> bool continue if (row.get("fill_status") or "") == "canceled": continue + # Ex-div adjustments also write TRAIL_STOP rows, but they LOWER + # the stop (dividend-drop compensation) — counting them as a + # "tighten" would hand every dividend payer a spurious cooldown. + # Same idiom as the ex-div idempotence check. + if "ex-div" in (row.get("reasoning") or "").lower(): + continue ts = row.get("timestamp") or "" try: dt = _dt.fromisoformat(ts.replace("Z", "+00:00")) if "T" in ts \ @@ -5163,6 +5176,12 @@ def run_morning(self) -> dict: loss_violation.message, ) orders = self._midday_emergency_liquidate(positions, loss_violation, run_id) + # Any same-day plan is superseded by the liquidation — a + # stale unconsumed checkpoint must not resume, and the + # dead-man probe must not read this as a killed morning. + from src import decision_checkpoint as _dc + _dc.mark_consumed("morning") + _dc.write_status("morning", "emergency_sold") return { "status": "emergency_sold", "orders": orders, @@ -5197,6 +5216,22 @@ def run_morning(self) -> dict: ctx.data_status = resumed["data_status"] ctx.portfolio_decision = resumed["portfolio_decision"] portfolio_decision = ctx.portfolio_decision + # Rehydrate bars for the plan's BUY symbols (zero-LLM, fresh + # data). The checkpoint deliberately omits symbols_bars + # (huge); without this the entry ATR stop floor silently + # no-ops and the correlation advisory false-fires on resume. + bars: dict = {} + for d in portfolio_decision.decisions: + if d.action != "BUY": + continue + try: + bars[d.symbol] = self.market.get_ohlcv( + d.symbol, self.config.trading.lookback_days, + ) or [] + except Exception as e: # noqa: BLE001 + logger.warning("resume: bar rehydrate failed for %s: %s", + d.symbol, e) + ctx.symbols_bars = bars else: # Phase 4 #1: research stage runs the parallel fan-out (macro / # news / tech / earnings). Populates ctx fields. @@ -5214,10 +5249,16 @@ def run_morning(self) -> dict: run_id, "post-research", ) if late_breach is not None: + _dc.mark_consumed("morning") + _dc.write_status("morning", "emergency_sold") return late_breach if not analyses: logger.warning("No analyses produced, skipping trading") + # Legit PM-less completion — record it so the evening + # dead-man probe doesn't read "research rows, no PM row" + # as a killed morning. + _dc.write_status("morning", "no_data") return {"status": "no_data", "orders": [], "run_id": run_id} # Phase 4 #1: decision stage — memory layers + PM + Constructor. @@ -5240,6 +5281,10 @@ def run_morning(self) -> dict: run_id, "post-decision", ) if late_breach is not None: + # The just-written checkpoint is superseded by the + # emergency liquidation — never resume it. + _dc.mark_consumed("morning") + _dc.write_status("morning", "emergency_sold") return late_breach if not portfolio_decision: @@ -5722,9 +5767,25 @@ def run_position_review(self, session_type: str = "midday") -> dict: daily_pnl = total_value - last_equity loss_violation = self.risk_engine.check_daily_loss(last_equity, daily_pnl) if loss_violation: + # Review fix: this branch previously FELL THROUGH to the park + # bookend — the system would buy SGOV with ~all equity minutes + # after force-selling everything, and the next intra tick + # would emergency-sell the fresh SGOV lot (spurious 🚨 alert + + # a full round-trip on the worst possible day). Mirror the + # pre-review breaker: reconcile and return, never park. orders.extend(self._midday_emergency_liquidate( positions, loss_violation, run_id, )) + self._reconcile_fills() + return { + "status": "emergency_sold", + "session": session_type, + "positions": len(positions), + "review": review.model_dump() if review else None, + "orders": orders, + "run_id": run_id, + "stop_coverage_gaps": coverage_gaps, + } else: orders.extend(self._midday_execute_llm_actions( review_positions, review, run_id, @@ -6093,10 +6154,15 @@ def run_evening(self) -> dict: # 3. LLM evening analysis — daily review and tomorrow outlook macro_summary = self.macro.get_macro_summary() + # Sweep churn (SWEEP_BUY/SWEEP_SELL) is cash parking, not a trading + # decision — narrating it to the evening analyst would feed the + # learning loops noise (review finding). Fetch extra rows so the + # filter doesn't shrink the real-trade view. today_trades = [ self._actualize_trade_row(t) - for t in self.db.get_trades(limit=20, today_only=True, executed_only=True) - ] + for t in self.db.get_trades(limit=30, today_only=True, executed_only=True) + if (t.get("action") or "") not in ("SWEEP_BUY", "SWEEP_SELL") + ][:20] # Feed yesterday's insights back so evening can grade its own prior outlook # against today's reality — enables calibration over time. prior_outlook = self.db.get_latest_insights(before_date=today_str) @@ -6430,10 +6496,18 @@ def _expected_sessions_missing_today(self) -> list[str]: # kill, so 13 straight days of morning deaths passed this check and # the 🔴 banner never fired. Two sharper probes: if "morning" not in missing and "run" in present: + # A legit PM-less completion (no_data / emergency_sold) records a + # status marker — skip both probes for it. + try: + from src import decision_checkpoint as _dc0 + legit_early_exit = _dc0.read_status("morning") is not None + except Exception: # noqa: BLE001 + legit_early_exit = False # (a) research logged but the PM never ran → died during research. try: agents = self.db.agent_names_logged_on("run-") - if agents and "portfolio_manager" not in agents: + if (not legit_early_exit and agents + and "portfolio_manager" not in agents): missing.append("morning (research ran, PM never did — killed mid-run?)") except Exception as exc: # noqa: BLE001 logger.warning("missing-session check: agent probe failed: %s", exc) diff --git a/src/pipeline_context.py b/src/pipeline_context.py index 2b18ec6d..fc278ce3 100644 --- a/src/pipeline_context.py +++ b/src/pipeline_context.py @@ -175,6 +175,14 @@ def _num(v: float | int | None) -> str: def _render_deployment_gap(self) -> str: if self.macro_target_invested_pct is None or self.deployment_gap_pp is None: return "" + if self.deployment_gap_pp > 15: + return ( + f"\n\n### Deployment vs Macro Target" + f"\n- invested={self.invested_pct:.1f}% vs macro target=" + f"{self.macro_target_invested_pct:.0f}% — {self.deployment_gap_pp:.0f}pp OVER" + f" the target. The RM advisory will flag this; trims/rotation" + f" are a valid response, especially if macro is not risk-on." + ) if self.deployment_gap_pp >= -15: return ( f"\n\n### Deployment vs Macro Target" diff --git a/src/pipeline_stages.py b/src/pipeline_stages.py index ee09332a..9f6e5f6f 100644 --- a/src/pipeline_stages.py +++ b/src/pipeline_stages.py @@ -548,6 +548,21 @@ def run(self, ctx: RunContext) -> dict | None: news_intel = ctx.news_intel data_status = ctx.data_status + # Cash-sweep view — same contract as DecisionStage: the RiskManager + # must see parked T-bills as CASH, never as an 84%-of-book "position" + # (review finding: PM and RM otherwise get contradictory views of the + # same dollars in the same run, and RM's veto acts on the corrupted + # one). IMPORTANT: only the LLM-facing uses (RM prompt, correlation + # pool, has_book_to_check) take the scrubbed list — the hard filter + # keeps RAW positions because it derives the parked-cash credit from + # finding the vehicle in the list itself. + from src.execution.cash_sweep import CashSweeper + sweeper = getattr(pipeline, "_sweeper", None) + sweeper = sweeper() if callable(sweeper) else None + rm_positions = positions + if isinstance(sweeper, CashSweeper): + rm_positions, _parked = sweeper.split_positions(positions) + # Symbol guard portfolio_decision.decisions, symbol_blocked_reasons = pipeline._filter_supported_symbols( portfolio_decision.decisions, analyses, positions, @@ -577,7 +592,7 @@ def run(self, ctx: RunContext) -> dict | None: try: from src.data.correlation import build_correlation_matrix pool_bars = dict(ctx.symbols_bars) - for p in positions: + for p in rm_positions: if p.symbol not in pool_bars: pool_bars[p.symbol] = pipeline.market.get_ohlcv( p.symbol, pipeline.config.trading.lookback_days, @@ -622,7 +637,7 @@ def run(self, ctx: RunContext) -> dict | None: )) logger.warning("Morning data degradation: %s", data_status) - has_book_to_check = len(positions) >= 2 or any( + has_book_to_check = len(rm_positions) >= 2 or any( d.action == "BUY" for d in portfolio_decision.decisions ) if (not correlation_matrix) and has_book_to_check: @@ -647,7 +662,7 @@ def run(self, ctx: RunContext) -> dict | None: verdict, rm_result = pipeline.risk_manager.review( portfolio_decision=portfolio_decision, - positions=positions, + positions=rm_positions, macro_summary=ctx.macro_summary, rule_violations=rule_violations, tech_analyses=analyses, @@ -1004,6 +1019,24 @@ def run(self, ctx: RunContext) -> list[dict]: sizing_price, widened, atr14, ) stop_price = widened + # Review fix: widening happens AFTER the RM + # audited this trade's R/R. If the honest + # geometry (real stop distance vs the same + # target) collapses the R/R below a sane floor, + # the setup RM approved never existed — skip + # rather than execute a trade nobody reviewed. + if decision.take_profit > 0: + reward = decision.take_profit - sizing_price + risk = sizing_price - stop_price + if risk > 0 and reward / risk < 1.2: + logger.warning( + "BUY %s skipped: ATR-widened stop " + "makes R/R %.2f (<1.2) — RM approved " + "a tighter-stop geometry that daily " + "noise would have destroyed.", + decision.symbol, reward / risk, + ) + continue except Exception as e: logger.warning("ATR stop floor skipped for %s: %s", decision.symbol, e) diff --git a/tests/test_bugfixes.py b/tests/test_bugfixes.py index 93a56efa..1b288be3 100644 --- a/tests/test_bugfixes.py +++ b/tests/test_bugfixes.py @@ -577,7 +577,9 @@ def _get_trades(*args, **kwargs): pipeline.run_evening() assert events[0] == "reconcile" - assert ("get_trades", {"limit": 20, "today_only": True, "executed_only": True}) in events + # limit 30: evening fetches headroom so the SWEEP_* filter can't shrink + # the real-trade view below the original 20 (feat/returns-optimization) + assert ("get_trades", {"limit": 30, "today_only": True, "executed_only": True}) in events assert pipeline._reconcile_fills.call_count == 2 diff --git a/tests/test_cash_sweep.py b/tests/test_cash_sweep.py index 2455d99f..c43a5710 100644 --- a/tests/test_cash_sweep.py +++ b/tests/test_cash_sweep.py @@ -243,8 +243,10 @@ def test_park_excess_buys_vehicle_with_idle_cash(): assert order is not None and order["action"] == "SWEEP_BUY" kwargs = p.broker.submit_order.call_args.kwargs assert kwargs["symbol"] == "SGOV" and kwargs["side"] == "buy" - # excess = 90k - 1%·100k - 0 = 89k → int(89000/100.60) = 884 shares - assert kwargs["qty"] == 884 + # excess = 90k - 1%·100k - 0 = 89k; sized on the LIMIT price + # (100.60×1.001 → 100.70) so a padded fill can't overdraw raw cash: + # int(89000/100.70) = 883 shares + assert kwargs["qty"] == 883 assert kwargs["stop_loss_price"] is None # deliberately stopless p.db.confirm_trade_submitted.assert_called_once() @@ -359,3 +361,62 @@ def test_position_review_hides_vehicle_and_parks_at_end(tmp_path): assert [x.symbol for x in seen] == ["NVDA"], "reviewer must not see SGOV" # bookend parked the idle cash: a SWEEP_BUY order rides in the result assert any(o.get("action") == "SWEEP_BUY" for o in result["orders"]) + + +def test_risk_stage_rm_view_excludes_vehicle(): + """Review finding: RM (the veto layer) must see parked T-bills as cash, + not as an 84%-of-book position — otherwise PM and RM get contradictory + views of the same dollars in the same run and RM's veto acts on the + corrupted one.""" + from src.pipeline_stages import RiskStage + from src.models import PortfolioDecision, ReasoningChain, RiskVerdict, RiskReasoningChain + + p = _sweep_pipeline() + p.market = MagicMock() + p.market.get_ohlcv.return_value = [] + p._filter_supported_symbols = MagicMock(side_effect=lambda d, a, pos: (d, [])) + p._clamp_queued_earnings_buys = MagicMock(side_effect=lambda d, e: d) + p._filter_hard_risk_decisions = MagicMock(side_effect=lambda d, *a, **k: (d, [], [])) + p.risk_manager = MagicMock() + p.risk_manager.review.return_value = ( + RiskVerdict( + approved=True, modifications=[], reasoning="ok", + reasoning_chain=RiskReasoningChain( + rr_audit="x", signal_fidelity="x", correlation_check="x", + event_risk="x", sizing_sanity="x", overall="x", + ), + ), + MagicMock(user_message="m", raw_text="{}", tokens_used=1, + input_tokens=1, output_tokens=1, cost_usd=0.0), + ) + p.db = MagicMock() + p.config.llm = MagicMock() + p.config.llm.risk_manager_model = "test-model" + p.config.trading = MagicMock() + p.config.trading.lookback_days = 120 + + from src.pipeline_context import RunContext + ctx = RunContext.start("morning") + ctx.positions = [SGOV, NVDA] + ctx.total_value = 100_000.0 + ctx.last_equity = 100_000.0 + ctx.cash = 10_000.0 + ctx.portfolio_decision = PortfolioDecision( + reasoning_chain=ReasoningChain( + macro_filter="x", news_check="x", earnings_check="x", + signal_conflicts="x", sizing_logic="x", + portfolio_balance="x", cash_target="x", + ), + decisions=[_buy(alloc=5.0)], portfolio_view="v", + ) + ctx.symbols_bars = {} + ctx.data_status = {} + + RiskStage(pipeline=p).run(ctx) + + rm_seen = p.risk_manager.review.call_args.kwargs["positions"] + assert [x.symbol for x in rm_seen] == ["NVDA"], "RM must not see SGOV" + # but the HARD filter received the RAW list (it derives the parked-cash + # credit from finding the vehicle itself) + filter_positions = p._filter_hard_risk_decisions.call_args_list[0].args[1] + assert any(x.symbol == "SGOV" for x in filter_positions) diff --git a/tests/test_decision_checkpoint.py b/tests/test_decision_checkpoint.py index fc42fb16..e925d1f3 100644 --- a/tests/test_decision_checkpoint.py +++ b/tests/test_decision_checkpoint.py @@ -144,6 +144,10 @@ def _resume_pipeline(): p.execution_stage = MagicMock() p.execution_stage.run.return_value = [{"id": "o1", "action": "BUY"}] p.decision_stage = MagicMock() + p.market = MagicMock() + p.market.get_ohlcv.return_value = [] + p.config = MagicMock() + p.config.trading.lookback_days = 120 return p diff --git a/tests/test_notifier.py b/tests/test_notifier.py index 21ff40a2..1a7a99b8 100644 --- a/tests/test_notifier.py +++ b/tests/test_notifier.py @@ -933,7 +933,7 @@ def test_format_evening_missing_morning_session_is_red(): "analysis": {"risk_rating": "low"}, } msg = format_session_result("evening", result, 10.0) - assert "🔴 SESSION DID NOT RUN TODAY: morning" in msg + assert "🔴 MORNING SESSION INCOMPLETE TODAY: morning" in msg assert "midday" in msg # soft warning for the non-morning miss