From 1cb60c1f0103cc428babb263277771c05e2bee0f Mon Sep 17 00:00:00 2001 From: Brent Palmer Date: Tue, 4 Aug 2026 15:30:55 -0400 Subject: [PATCH] fix: make the quantity-zero tolerance scale-relative The engine decides a position is closed when the post-fill quantity is under an absolute 1e-12. float64 spacing grows with magnitude, so that threshold is only correct over one binade: one ULP is 9.09e-13 just below 8192 units and 1.82e-12 from 8192 upward. At and above 2**13 units an exact close that lands one ULP off zero is no longer recognised, the position key survives in broker.positions, and the engine reports an open position holding ~1e-12 shares. Since n_positions is len(broker.positions), the reported count is one too high until the name is traded again. Observed on a short cover of 10,927.322882 shares: the residual was exactly 2**-39, one ULP of that binade, and the count stayed high for 20 sessions. Replace the absolute epsilon with quantity_zero_tolerance() in core/shared.py, which scales with the operands that produced the residual: tol = max(1e-12, 16 * ulp(max(|old_qty|, |signed_qty|))) The scale is the operands, not the result -- the result is ~0 by construction, so a tolerance taken against it could never fire. The floor keeps the previous absolute behaviour everywhere float64 spacing is finer than it, which is every magnitude below 8192; a call passing a single operand is therefore exactly equivalent to the old rule. A larger fixed epsilon is not the repair: it moves the failure point instead of removing it, and erases genuine small positions. For the same reason the tolerance carries no upper cap, which would be scale-blind in turn. Every quantity-zero comparison in the package now routes through the one primitive: position closure in the fill executor, the submission-precheck shadow book, the shadow-queue validation and commit paths, and the gatekeeper's position normalisation. The sites previously disagreed at the threshold itself, mixing `<` and `<=`; they are unified on the `<=` form the majority already used, so one predicate now holds everywhere. OrderBook._MIN_ORDER_SIZE is deliberately unchanged. It is an economic policy -- the smallest order the engine accepts -- not a statement about floating-point residue. Adds tests/test_quantity_zero_tolerance.py (66 tests) covering the binade boundary, ULP-graded residues on both sides of the bound, genuine small long and short positions in the same magnitude range, partial fills, repeated fills, position-key/count consistency, cross-site agreement, sign symmetry, subnormal and non-finite inputs, and the sites left unchanged. Expected boundaries are reconstructed from frexp rather than from the production helper. Co-Authored-By: Claude Opus 5 --- docs/concepts/quantity-zero-tolerance.md | 164 ++++++++ mkdocs.yml | 1 + src/ml4t/backtest/accounting/gatekeeper.py | 3 +- src/ml4t/backtest/core/execution_engine.py | 16 +- src/ml4t/backtest/core/order_book.py | 23 +- src/ml4t/backtest/core/shared.py | 51 +++ src/ml4t/backtest/execution/fill_executor.py | 4 +- tests/test_quantity_zero_tolerance.py | 405 +++++++++++++++++++ 8 files changed, 644 insertions(+), 23 deletions(-) create mode 100644 docs/concepts/quantity-zero-tolerance.md create mode 100644 tests/test_quantity_zero_tolerance.py diff --git a/docs/concepts/quantity-zero-tolerance.md b/docs/concepts/quantity-zero-tolerance.md new file mode 100644 index 00000000..609b9db0 --- /dev/null +++ b/docs/concepts/quantity-zero-tolerance.md @@ -0,0 +1,164 @@ +# Quantity-zero tolerance + +When a fill closes a position the engine must decide whether the resulting +quantity is *zero*. Floating-point arithmetic rarely lands on exactly `0.0`, so +that decision needs a tolerance. This page states the contract that tolerance +obeys. + +The implementation is `quantity_zero_tolerance()` in +`ml4t/backtest/core/shared.py`. + +## The arithmetic that creates a residual + +A fill updates an existing position with one addition: + +```python +old_qty = pos.quantity +new_qty = old_qty + ctx.signed_qty # execution/fill_executor.py +``` + +When the close is economically exact, `old_qty` and `ctx.signed_qty` are equal +and opposite. Their magnitudes are then within a factor of two of each other, so +by Sterbenz's lemma **the addition itself is exact**. It contributes no error. + +A non-zero `new_qty` therefore means the two operands were not exact negatives as +float64 values. They were produced by different routes — the book quantity by +accumulating prior fills, the closing quantity by sizing an order — and those +routes round differently in the last bits. The residual is the gap between them. + +That gap is bounded by the spacing of float64 **at the magnitude of the +quantities being cancelled**. It is not bounded by any fixed absolute quantity. + +## Controlling scale + +The scale is `max(|old_qty|, |signed_qty|)`: the magnitudes actually +participating in the cancellation. + +It is emphatically **not** `|new_qty|`. `new_qty` is approximately zero by +construction, so a tolerance derived from it would be approximately zero as well: +`abs(new_qty) < k * ulp(abs(new_qty))` is false for every `k < 1` and every +normal float. A relative tolerance taken against zero is mathematically +ineffective, which is why the tolerance takes the *operands* rather than the +result. + +`max()` rather than either operand alone, because the two are interchangeable in +the closure case and `max()` is the only choice that is symmetric in them. It is +also the correct upper bound on the last-place spacing of both. + +## Why the previous absolute epsilon failed + +The engine previously compared against an absolute `1e-12`. Float64 spacing grows +with magnitude, so a single absolute threshold can only be correct over one +binade: + +| position size | one ULP | caught by `1e-12`? | +|---|---|---| +| 4096 | 9.094947e-13 | yes | +| 8191 | 9.094947e-13 | yes | +| **8192** | **1.818989e-12** | **no** | +| 10927 | 1.818989e-12 | no | +| 874442 | 1.164153e-10 | no | + +At exactly 213 = 8192 units the spacing crosses `1e-12` and the rule +stops working. From there upward a one-ULP residual survives, the position key is +retained, and the engine reports a position that holds a quantity of the order of +1e-12 shares. + +A **larger** fixed epsilon is not the repair. It moves the failure point to a +larger magnitude without removing it — every absolute threshold is scale-blind +somewhere — and it erases genuine small positions below the new threshold. The +defect is the absoluteness, not the value. + +For the same reason the tolerance carries **no upper cap**. A ceiling is itself a +fixed absolute value, so capping would reintroduce exactly the scale-blindness +being removed. + +## The rule + +``` +scale = max(|operand|) over the finite operands +tol = QTY_ZERO_FLOOR if scale == 0 + max(QTY_ZERO_FLOOR, QTY_ZERO_ULPS * ulp(scale)) otherwise + +QTY_ZERO_FLOOR = 1e-12 +QTY_ZERO_ULPS = 16 +``` + +A quantity counts as zero when `|q| <= tol`, and is open when `|q| > tol`. That +single predicate holds at every site. The engine previously mixed `<` and `<=` +against its absolute epsilon, so its sites disagreed with each other exactly at +the threshold; they are unified here on the `<=` form the majority already used. + +### Why 16 ULP + +* **Mechanism.** The residual is the last-bits gap between the book quantity and + the order quantity. Order sizing (a target weight through equity and price, or + a re-read of the book) costs a small number of ULP; accumulating a position + over *n* fills adds at most `n/2` ULP of drift between the book value and the + exact sum. +* **Margin over observation.** Across every retained run available for this + repair, every unsnapped residual is **exactly one ULP** of the operation scale. + 16 ULP is a sixteen-fold margin over the worst case actually seen, and covers + roughly 32 accumulation steps at the worst-case half-ULP each. +* **Relative size.** 16 ULP is `16 * 2**-52 ≈ 3.6e-15` of the operation scale — + the fifteenth significant digit. float64 carries about 15.95 significant + decimal digits, so a distinction that small cannot be *reliably* produced by + any float64 computation working at that scale. +* **Headroom below a submittable order.** `OrderBook._MIN_ORDER_SIZE` is `1e-8`: + the engine refuses to submit anything smaller. At the largest operation scale + observed in practice (874,443 units) the tolerance is 1.86e-9 — still well + under that floor. + +### Why the absolute floor + +`ulp(scale)` collapses toward zero as the scale does, so a pure ULP rule would +have no useful width for small positions and none at all at `scale == 0`. The +floor keeps the historical absolute behaviour everywhere float64 spacing is finer +than `1e-12`, which is every magnitude below 8192. + +This gives the migration a provable property: **a call with a single operand is +exactly equivalent to the old `abs(q) < 1e-12` rule**. For any normal `x`, +`16 * ulp(x) < x`, so the ULP term can never make such a predicate true on its +own; only the floor can. Call sites that merely ask "is this book quantity zero?" +therefore keep their previous behaviour bit for bit, while call sites that pass +both closure operands gain the scale-aware rule. + +## Residue versus a genuine small position + +A **residue** is what remains after an economically exact close: its magnitude is +a handful of ULP of the quantities that cancelled. + +A **genuine** small position is one a strategy meant to hold. It is preserved +because the tolerance is relative to *the operation that produced it*, not to a +global constant: + +* Opening 1e-9 units on an empty book is an operation at scale 1e-9, where the + tolerance is the floor, 1e-12. The position survives. +* Holding 10,000 units and selling 9,999.999999999 leaves a genuine 1e-9. The + operation scale is 10,000, where the tolerance is 1.46e-11. 1e-9 is seventy + times larger, so the position survives. + +The only quantity the rule erases is one arrived at by cancelling two much larger +quantities — and at that point the result is below what float64 arithmetic at +that scale can warrant. + +## Edge cases + +| input | behaviour | +|---|---| +| `scale == 0` (no position, zero fill) | tolerance is the floor, `1e-12` | +| subnormal operands | `ulp` is the minimum subnormal; the floor dominates | +| `inf` / `NaN` operand | ignored when computing the scale; the tolerance stays finite and positive, and falls back to the floor if no operand is finite | +| `NaN` quantity under test | `abs(nan) < tol` is false, so it is not treated as zero — unchanged from the previous rule | +| long versus short | the tolerance is built from magnitudes, so it is identical for `+q` and `-q` | + +## Sites + +One shared primitive serves every site that asks whether a *quantity* is zero: +position closure in the fill executor, the submission-precheck shadow book, the +shadow-queue validation and commit paths, and the gatekeeper's position +normalisation. + +`OrderBook._MIN_ORDER_SIZE` is deliberately **not** migrated. It is an economic +policy — the smallest order the engine will accept — not a statement about +floating-point residue. diff --git a/mkdocs.yml b/mkdocs.yml index 226446a4..f42b60f3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -119,6 +119,7 @@ nav: - Quickstart: getting-started/quickstart.md - Concepts: - How It Works: concepts/how-it-works.md + - Quantity-Zero Tolerance: concepts/quantity-zero-tolerance.md - User Guide: - Strategies: user-guide/strategies.md - Order Types: user-guide/orders.md diff --git a/src/ml4t/backtest/accounting/gatekeeper.py b/src/ml4t/backtest/accounting/gatekeeper.py index d5375df1..1567238a 100644 --- a/src/ml4t/backtest/accounting/gatekeeper.py +++ b/src/ml4t/backtest/accounting/gatekeeper.py @@ -4,6 +4,7 @@ ensuring they meet account policy constraints and preventing invalid trades. """ +from ..core.shared import quantity_zero_tolerance from ..models import CommissionModel from ..types import Order, OrderSide from .account import AccountState @@ -123,7 +124,7 @@ def validate_order(self, order: Order, price: float) -> tuple[bool, str]: """ # Get current position quantity (0 if no position) current_qty = self.account.get_position_quantity(order.asset) - if abs(current_qty) < 1e-12: + if abs(current_qty) <= quantity_zero_tolerance(current_qty): current_qty = 0.0 # Determine order direction (positive=buy, negative=sell) diff --git a/src/ml4t/backtest/core/execution_engine.py b/src/ml4t/backtest/core/execution_engine.py index adc3f86b..047bcd1c 100644 --- a/src/ml4t/backtest/core/execution_engine.py +++ b/src/ml4t/backtest/core/execution_engine.py @@ -5,7 +5,7 @@ import copy from ..types import ExecutionMode, OrderSide, OrderStatus, OrderType, Position -from .shared import is_exit_order +from .shared import is_exit_order, quantity_zero_tolerance class ExecutionEngine: @@ -238,15 +238,15 @@ def _validate_shadow_queue_order( ) new_qty = current_qty + qty_delta is_reversal = ( - abs(current_qty) > 1e-12 - and abs(new_qty) > 1e-12 + abs(current_qty) > quantity_zero_tolerance(current_qty) + and abs(new_qty) > quantity_zero_tolerance(current_qty, qty_delta) and ((current_qty > 0 and new_qty < 0) or (current_qty < 0 and new_qty > 0)) ) commission = broker.commission_model.calculate( order.asset, order.quantity, validation_price ) - if abs(current_qty) <= 1e-12: + if abs(current_qty) <= quantity_zero_tolerance(current_qty): return policy.validate_new_position( asset=order.asset, quantity=qty_delta, @@ -290,7 +290,7 @@ def _commit_shadow_queue_fill( commission = broker.commission_model.calculate(order.asset, order.quantity, fill_price) shadow_cash += -qty_delta * fill_price * broker.get_multiplier(order.asset) - commission - if abs(new_qty) <= 1e-12: + if abs(new_qty) <= quantity_zero_tolerance(current_qty, qty_delta): shadow_positions.pop(order.asset, None) return shadow_cash @@ -307,13 +307,13 @@ def _commit_shadow_queue_fill( return shadow_cash is_reversal = ( - abs(current_qty) > 1e-12 - and abs(new_qty) > 1e-12 + abs(current_qty) > quantity_zero_tolerance(current_qty) + and abs(new_qty) > quantity_zero_tolerance(current_qty, qty_delta) and ((current_qty > 0 and new_qty < 0) or (current_qty < 0 and new_qty > 0)) ) position.quantity = new_qty position.current_price = broker._current_prices.get(order.asset, fill_price) - if abs(current_qty) <= 1e-12 or is_reversal: + if abs(current_qty) <= quantity_zero_tolerance(current_qty) or is_reversal: position.entry_price = fill_price return shadow_cash diff --git a/src/ml4t/backtest/core/order_book.py b/src/ml4t/backtest/core/order_book.py index 4af120d5..eb8bace4 100644 --- a/src/ml4t/backtest/core/order_book.py +++ b/src/ml4t/backtest/core/order_book.py @@ -5,7 +5,7 @@ from datetime import datetime from ..types import ExecutionMode, Order, OrderSide, OrderStatus, OrderType, Position -from .shared import SubmitOrderOptions, is_exit_order +from .shared import SubmitOrderOptions, is_exit_order, quantity_zero_tolerance class OrderBook: @@ -15,7 +15,6 @@ class OrderBook: {"quantity", "limit_price", "stop_price", "trail_amount"} ) _MIN_ORDER_SIZE: float = 1e-8 - _QTY_EPS: float = 1e-12 def __init__(self, broker): self.broker = broker @@ -254,7 +253,7 @@ def _reset_submission_shadow_if_needed(self) -> None: or pos.entry_price, ) for asset, pos in broker.positions.items() - if abs(pos.quantity) > self._QTY_EPS + if abs(pos.quantity) > quantity_zero_tolerance(pos.quantity) } def _build_shadow_policy_positions(self) -> dict[str, Position]: @@ -262,7 +261,7 @@ def _build_shadow_policy_positions(self) -> dict[str, Position]: ts = broker._current_time or datetime(1970, 1, 1) positions: dict[str, Position] = {} for asset, (qty, basis_price) in self._submission_shadow_positions.items(): - if abs(qty) <= self._QTY_EPS: + if abs(qty) <= quantity_zero_tolerance(qty): continue mark_price = broker.get_mark_price(asset, quantity=qty) or basis_price positions[asset] = Position( @@ -281,10 +280,10 @@ def _simulate_position_update( ) -> tuple[float, float, float, float]: """Mirror Backtrader Position.update for pseudo-exec prechecks.""" new_qty = old_qty + size - if abs(new_qty) < OrderBook._QTY_EPS: + if abs(new_qty) <= quantity_zero_tolerance(old_qty, size): return 0.0, 0.0, 0.0, size - if abs(old_qty) < OrderBook._QTY_EPS: + if abs(old_qty) <= quantity_zero_tolerance(old_qty): return new_qty, price, size, 0.0 if old_qty > 0: @@ -352,7 +351,7 @@ def _passes_submission_precheck(self, order: Order) -> bool: # Keep shadow effects even for rejected orders to mirror Backtrader's # sequential submitted-queue pseudo-execution behavior. self._submission_shadow_cash = shadow_cash - if abs(new_qty) <= self._QTY_EPS: + if abs(new_qty) <= quantity_zero_tolerance(old_qty, size): self._submission_shadow_positions.pop(order.asset, None) else: self._submission_shadow_positions[order.asset] = (new_qty, new_price) @@ -438,7 +437,7 @@ def _passes_buying_power_check(self, order: Order) -> bool: # Accepted — commit shadow changes self._submission_shadow_cash = shadow_cash - if abs(new_qty) <= self._QTY_EPS: + if abs(new_qty) <= quantity_zero_tolerance(old_qty, size): self._submission_shadow_positions.pop(order.asset, None) else: self._submission_shadow_positions[order.asset] = (new_qty, new_price) @@ -465,12 +464,12 @@ def _passes_margin_submission_precheck(self, order: Order, signal_price: float) shadow_positions = self._build_shadow_policy_positions() is_reversal = ( - abs(old_qty) > self._QTY_EPS - and abs(new_qty) > self._QTY_EPS + abs(old_qty) > quantity_zero_tolerance(old_qty) + and abs(new_qty) > quantity_zero_tolerance(old_qty, size) and ((old_qty > 0 and new_qty < 0) or (old_qty < 0 and new_qty > 0)) ) - if abs(old_qty) <= self._QTY_EPS: + if abs(old_qty) <= quantity_zero_tolerance(old_qty): valid, reason = broker.account.policy.validate_new_position( asset=order.asset, quantity=size, @@ -510,7 +509,7 @@ def _passes_margin_submission_precheck(self, order: Order, signal_price: float) multiplier = broker.get_multiplier(order.asset) self._submission_shadow_cash += -size * signal_price * multiplier - commission - if abs(new_qty) <= self._QTY_EPS: + if abs(new_qty) <= quantity_zero_tolerance(old_qty, size): self._submission_shadow_positions.pop(order.asset, None) else: self._submission_shadow_positions[order.asset] = (new_qty, new_price) diff --git a/src/ml4t/backtest/core/shared.py b/src/ml4t/backtest/core/shared.py index 69026768..abf7ed74 100644 --- a/src/ml4t/backtest/core/shared.py +++ b/src/ml4t/backtest/core/shared.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass +from math import fabs, isfinite, ulp from typing import TYPE_CHECKING from ..types import ExitReason, OrderSide @@ -14,6 +15,56 @@ # Prevents order rejections due to rounding in equity/price arithmetic. CASH_TOLERANCE: float = 0.01 +# Quantity-zero tolerance. +# +# A fill updates a position with ``new_qty = old_qty + signed_qty``. When the +# close is economically exact the two operands cancel as real numbers, but they +# are floats produced by different routes, so the sum can land a few units in +# the last place away from zero instead of on it. The size of that residue is +# set by the spacing of float64 at the scale of the quantities being cancelled, +# and float64 spacing grows with magnitude. +# +# A single absolute epsilon cannot express that. 1e-12 is wider than one ULP +# while |q| < 8192 and narrower than one ULP from |q| = 8192 upward, so an +# absolute rule silently stops closing positions at exactly 2**13 units. +# +# The floor preserves the historical absolute behaviour wherever float64 spacing +# is finer than it, which is every magnitude below 8192. +QTY_ZERO_FLOOR: float = 1e-12 +QTY_ZERO_ULPS: int = 16 + + +def quantity_zero_tolerance(*operands: float) -> float: + """Return the tolerance under which a residual quantity counts as zero. + + The tolerance is the larger of :data:`QTY_ZERO_FLOOR` and + :data:`QTY_ZERO_ULPS` units in the last place of the largest finite operand. + + Pass the quantities that *produced* the residual — for a fill, the pre-fill + position and the signed fill size. Do not pass the residual itself: it is + approximately zero by construction, so it carries no scale and a tolerance + derived from it could never fire. + + Non-finite operands are ignored, so the tolerance is always a finite + positive number; if no operand is finite the floor applies. The tolerance is + built from magnitudes, so it is symmetric for long and short quantities. + + Args: + *operands: Quantities defining the scale of the operation. + + Returns: + The tolerance, always >= ``QTY_ZERO_FLOOR``. + """ + scale = 0.0 + for operand in operands: + if isfinite(operand): + magnitude = fabs(operand) + if magnitude > scale: + scale = magnitude + if scale == 0.0: + return QTY_ZERO_FLOOR + return max(QTY_ZERO_FLOOR, QTY_ZERO_ULPS * ulp(scale)) + @dataclass class SubmitOrderOptions: diff --git a/src/ml4t/backtest/execution/fill_executor.py b/src/ml4t/backtest/execution/fill_executor.py index d4afb347..213e2f5d 100644 --- a/src/ml4t/backtest/execution/fill_executor.py +++ b/src/ml4t/backtest/execution/fill_executor.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING from ..config import InitialHwmSource, ShareType +from ..core.shared import quantity_zero_tolerance from ..types import ( ExitReason, Fill, @@ -82,7 +83,6 @@ def __init__(self, broker: Broker): broker: The Broker instance whose state we'll modify """ self.broker = broker - self._qty_zero_epsilon = 1e-12 def execute(self, order: Order, base_price: float) -> bool: """Execute a fill and update positions. @@ -260,7 +260,7 @@ def _update_position(self, ctx: FillContext) -> float: else: old_qty = pos.quantity new_qty = old_qty + ctx.signed_qty - if abs(new_qty) < self._qty_zero_epsilon: + if abs(new_qty) <= quantity_zero_tolerance(old_qty, ctx.signed_qty): new_qty = 0.0 if new_qty == 0: diff --git a/tests/test_quantity_zero_tolerance.py b/tests/test_quantity_zero_tolerance.py new file mode 100644 index 00000000..236456ad --- /dev/null +++ b/tests/test_quantity_zero_tolerance.py @@ -0,0 +1,405 @@ +"""Scale-relative quantity-zero tolerance. + +Covers the primitive in ``ml4t.backtest.core.shared`` and every call site that was +migrated onto it, plus the sites deliberately left alone. + +Expected boundaries here are built independently of the production helper. For a +float64 ``x`` in the binade ``[2**e, 2**(e+1))`` the spacing is exactly +``2**(e-52)``; ``_ulp_from_binade`` reconstructs that from ``frexp`` alone, using +neither ``math.ulp`` nor ``quantity_zero_tolerance``. A test that asked the +production code for its own expected answer would pass no matter what the rule +said. +""" + +from __future__ import annotations + +import math +from datetime import datetime + +import pytest + +from ml4t.backtest.broker import Broker +from ml4t.backtest.config import ShareType +from ml4t.backtest.core.order_book import OrderBook +from ml4t.backtest.core.shared import ( + QTY_ZERO_FLOOR, + QTY_ZERO_ULPS, + quantity_zero_tolerance, +) +from ml4t.backtest.models import NoCommission, NoSlippage +from ml4t.backtest.types import OrderSide, Position + +TS = datetime(2024, 1, 2, 16, 0) +PRICE = 10.0 + +# The residual observed in the field, on a short cover of ~10,927 shares. +OBSERVED_RESIDUAL = 2.0**-39 +OBSERVED_QUANTITY = 10927.322882 + + +def _ulp_from_binade(x: float) -> float: + """Spacing of float64 at ``x``, derived from the exponent alone. + + ``frexp`` returns ``(m, e)`` with ``0.5 <= |m| < 1`` and ``x = m * 2**e``, so + ``x`` lies in the binade ``[2**(e-1), 2**e)`` and the spacing there is + ``2**(e-1-52)``. Independent of ``math.ulp`` and of the production helper. + """ + _, exponent = math.frexp(abs(x)) + return 2.0 ** (exponent - 1 - 52) + + +def _expected_tolerance(*operands: float) -> float: + """Reference implementation of the contract, written from the spec.""" + finite = [abs(o) for o in operands if math.isfinite(o)] + scale = max(finite) if finite else 0.0 + if scale == 0.0: + return QTY_ZERO_FLOOR + return max(QTY_ZERO_FLOOR, QTY_ZERO_ULPS * _ulp_from_binade(scale)) + + +def _broker() -> Broker: + return Broker( + initial_cash=100_000_000.0, + commission_model=NoCommission(), + slippage_model=NoSlippage(), + allow_short_selling=True, + allow_leverage=True, + share_type=ShareType.FRACTIONAL, + ) + + +def _seed(broker: Broker, asset: str, quantity: float) -> None: + broker.positions[asset] = Position( + asset=asset, + quantity=quantity, + entry_price=PRICE, + current_price=PRICE, + entry_time=TS, + ) + + +def _fill(broker: Broker, asset: str, signed_qty: float) -> None: + """Submit and process one market order of ``signed_qty`` shares.""" + broker._update_time( + timestamp=TS, + prices={asset: PRICE}, + opens={asset: PRICE}, + highs={asset: PRICE}, + lows={asset: PRICE}, + volumes={asset: 1e12}, + signals={}, + ) + side = OrderSide.BUY if signed_qty > 0 else OrderSide.SELL + broker.submit_order(asset, abs(signed_qty), side) + broker._process_orders() + + +def _close_with_residual(position_qty: float, residual_ulps: int) -> Broker: + """Seed ``position_qty`` and fill the offsetting order, off by ``residual_ulps``. + + The offsetting quantity is nudged so the sum lands exactly ``residual_ulps`` + units in the last place away from zero. Both operands sit within a factor of + two of each other, so by Sterbenz's lemma the addition is exact and the + residual is precisely what was constructed. + """ + broker = _broker() + _seed(broker, "AAA", position_qty) + magnitude = abs(position_qty) + offset = residual_ulps * _ulp_from_binade(magnitude) + closing = math.copysign(magnitude + offset, -position_qty) + _fill(broker, "AAA", closing) + return broker + + +# -------------------------------------------------------------------------- +# The primitive +# -------------------------------------------------------------------------- + + +class TestTolerancePrimitive: + def test_floor_applies_where_spacing_is_finer(self): + # 16 ULP of 1.0 is 3.55e-15, far below the floor. + assert quantity_zero_tolerance(1.0) == QTY_ZERO_FLOOR + + @pytest.mark.parametrize("magnitude", [4096.0, 8191.0, 8192.0, 8192.5, 10927.322882, 874442.6]) + def test_matches_independent_reference(self, magnitude): + assert quantity_zero_tolerance(magnitude) == _expected_tolerance(magnitude) + assert quantity_zero_tolerance(-magnitude, magnitude) == _expected_tolerance(magnitude) + + def test_binade_boundary_is_8192(self): + """The absolute 1e-12 rule failed at exactly 2**13; document the crossing.""" + assert _ulp_from_binade(8191.0) < 1e-12 + assert _ulp_from_binade(8192.0) > 1e-12 + assert _ulp_from_binade(8192.0) == OBSERVED_RESIDUAL + + def test_scale_is_the_largest_operand(self): + assert quantity_zero_tolerance(3.0, -20000.0) == quantity_zero_tolerance(20000.0) + assert quantity_zero_tolerance(-20000.0, 3.0) == quantity_zero_tolerance(20000.0) + + def test_symmetric_in_sign(self): + for magnitude in (1.0, 8192.0, OBSERVED_QUANTITY, 874442.6): + assert quantity_zero_tolerance(magnitude) == quantity_zero_tolerance(-magnitude) + assert quantity_zero_tolerance(magnitude, -magnitude) == quantity_zero_tolerance( + -magnitude, magnitude + ) + + def test_zero_scale_falls_back_to_floor(self): + assert quantity_zero_tolerance() == QTY_ZERO_FLOOR + assert quantity_zero_tolerance(0.0) == QTY_ZERO_FLOOR + assert quantity_zero_tolerance(0.0, -0.0) == QTY_ZERO_FLOOR + + def test_subnormal_operands_fall_back_to_floor(self): + assert quantity_zero_tolerance(5e-324) == QTY_ZERO_FLOOR + assert quantity_zero_tolerance(1e-320, -1e-322) == QTY_ZERO_FLOOR + + @pytest.mark.parametrize( + "operands", + [ + (float("inf"),), + (float("-inf"),), + (float("nan"),), + (float("nan"), float("inf")), + ], + ) + def test_non_finite_only_operands_fall_back_to_floor(self, operands): + result = quantity_zero_tolerance(*operands) + assert result == QTY_ZERO_FLOOR + assert math.isfinite(result) + + def test_non_finite_operands_are_ignored_not_propagated(self): + result = quantity_zero_tolerance(float("nan"), -8192.0, float("inf")) + assert math.isfinite(result) + assert result == _expected_tolerance(8192.0) + + def test_tolerance_never_reaches_the_operand_scale(self): + """A single-operand call can only fire through the floor, never the ULP term.""" + for magnitude in (1e-300, 1e-10, 1.0, 8192.0, 1e10, 1e300): + assert quantity_zero_tolerance(magnitude) < max(magnitude, QTY_ZERO_FLOOR * 2) + + def test_single_operand_is_equivalent_to_the_old_absolute_rule(self): + """Sites that only ask 'is this book quantity zero?' keep prior behaviour. + + The engine's own boundary convention is ``<=``; against the old ``<= 1e-12`` + sites the migration is exactly behaviour-preserving. + """ + for magnitude in (0.0, 5e-324, 1e-13, 1e-12, 2e-12, 1.0, 8192.0, 1e9): + old = abs(magnitude) <= 1e-12 + new = abs(magnitude) <= quantity_zero_tolerance(magnitude) + assert old == new, magnitude + + +# -------------------------------------------------------------------------- +# Position closure through the real fill path +# -------------------------------------------------------------------------- + + +class TestExactClose: + @pytest.mark.parametrize( + "quantity", + [ + 4096.5, # below the old failure point + 8191.9, # immediately below 2**13 + 8192.0, # exactly at 2**13 + math.nextafter(8192.0, math.inf), # immediately above 2**13 + OBSERVED_QUANTITY, # the case observed in the field + 16384.0, # 2**14 + 1048576.0, # 2**20 + ], + ) + def test_bitwise_exact_close_removes_the_key(self, quantity): + for signed in (quantity, -quantity): + broker = _close_with_residual(signed, residual_ulps=0) + assert "AAA" not in broker.positions + assert len(broker.positions) == 0 + + +class TestResidualClose: + @pytest.mark.parametrize("ulps", [1, 2, 8, QTY_ZERO_ULPS - 1, QTY_ZERO_ULPS]) + @pytest.mark.parametrize("sign", [1.0, -1.0]) + def test_residue_inside_the_bound_closes(self, ulps, sign): + broker = _close_with_residual(sign * OBSERVED_QUANTITY, residual_ulps=ulps) + assert "AAA" not in broker.positions + + @pytest.mark.parametrize("ulps", [QTY_ZERO_ULPS + 1, QTY_ZERO_ULPS + 2, 64]) + @pytest.mark.parametrize("sign", [1.0, -1.0]) + def test_residue_outside_the_bound_is_retained(self, ulps, sign): + broker = _close_with_residual(sign * OBSERVED_QUANTITY, residual_ulps=ulps) + assert "AAA" in broker.positions + assert abs(broker.positions["AAA"].quantity) > 0.0 + + def test_long_and_short_boundaries_are_identical(self): + """The last closing and first retained ULP count must match across signs.""" + edges = {} + for sign in (1.0, -1.0): + closed = [ + n + for n in range(0, 2 * QTY_ZERO_ULPS + 2) + if "AAA" not in _close_with_residual(sign * OBSERVED_QUANTITY, n).positions + ] + edges[sign] = (max(closed), len(closed)) + assert edges[1.0] == edges[-1.0] + # The predicate is |q| <= tol, so the tolerance itself still closes. + assert edges[1.0][0] == QTY_ZERO_ULPS + + def test_the_observed_field_case(self): + """Short 10,927.322882 covered exactly, residual 2**-39, must close.""" + broker = _broker() + _seed(broker, "AAA", -OBSERVED_QUANTITY) + closing = math.nextafter(OBSERVED_QUANTITY, math.inf) + # Sterbenz: the operands are within a factor of two, so this is exact. + assert -OBSERVED_QUANTITY + closing == OBSERVED_RESIDUAL + assert OBSERVED_RESIDUAL > 1e-12 # the old absolute rule could not fire + _fill(broker, "AAA", closing) + assert "AAA" not in broker.positions + assert len(broker.positions) == 0 + + +class TestGenuineSmallPositions: + @pytest.mark.parametrize("sign", [1.0, -1.0]) + def test_genuine_small_position_survives_reduction_of_a_large_one(self, sign): + """Trading 10,000 down to 1e-9 leaves a real position, not residue.""" + broker = _broker() + _seed(broker, "AAA", sign * 10_000.0) + genuine = 1e-9 + _fill(broker, "AAA", -sign * (10_000.0 - genuine)) + assert "AAA" in broker.positions + assert broker.positions["AAA"].quantity == pytest.approx(sign * genuine, rel=1e-6) + + @pytest.mark.parametrize("sign", [1.0, -1.0]) + def test_genuine_small_position_opened_at_its_own_scale_survives(self, sign): + broker = _broker() + _fill(broker, "AAA", sign * 1e-7) + assert "AAA" in broker.positions + assert broker.positions["AAA"].quantity == pytest.approx(sign * 1e-7, rel=1e-9) + + def test_a_genuine_position_at_the_residual_magnitude_is_not_erased(self): + """1e-9 shares is 550x the tolerance at scale 10,000 - it must persist.""" + broker = _broker() + _seed(broker, "AAA", 10_000.0) + _fill(broker, "AAA", -9_999.999999999) + assert "AAA" in broker.positions + assert 0.0 < broker.positions["AAA"].quantity < 1e-8 + + +class TestPartialFills: + def test_partial_long_reduction(self): + broker = _broker() + _seed(broker, "AAA", 20_000.0) + _fill(broker, "AAA", -7_500.0) + assert broker.positions["AAA"].quantity == pytest.approx(12_500.0) + + def test_partial_short_cover(self): + broker = _broker() + _seed(broker, "AAA", -20_000.0) + _fill(broker, "AAA", 7_500.0) + assert broker.positions["AAA"].quantity == pytest.approx(-12_500.0) + + def test_repeated_fills_ending_flat_close_the_key(self): + broker = _broker() + _seed(broker, "AAA", 12_000.0) + for _ in range(4): + _fill(broker, "AAA", -3_000.0) + assert "AAA" not in broker.positions + + def test_repeated_fills_leaving_a_position_keep_the_key(self): + broker = _broker() + _seed(broker, "AAA", 12_000.0) + for _ in range(3): + _fill(broker, "AAA", -3_000.0) + assert broker.positions["AAA"].quantity == pytest.approx(3_000.0) + + +class TestPositionCountConsistency: + def test_book_holds_no_residual_keys_after_a_close(self): + """n_positions is len(broker.positions); a residual key inflates it.""" + broker = _broker() + for asset, qty in (("AAA", -OBSERVED_QUANTITY), ("BBB", 5_000.0), ("CCC", 30_000.0)): + _seed(broker, asset, qty) + _fill(broker, "AAA", math.nextafter(OBSERVED_QUANTITY, math.inf)) + assert set(broker.positions) == {"BBB", "CCC"} + assert len(broker.positions) == 2 + assert all(p.quantity != 0.0 for p in broker.positions.values()) + + +# -------------------------------------------------------------------------- +# Cross-site consistency +# -------------------------------------------------------------------------- + + +class TestMigratedSiteConsistency: + @pytest.mark.parametrize("ulps", [0, 1, QTY_ZERO_ULPS, QTY_ZERO_ULPS + 1, 64]) + def test_precheck_simulation_agrees_with_the_executor(self, ulps): + """OrderBook._simulate_position_update models the same closure decision.""" + magnitude = OBSERVED_QUANTITY + offset = ulps * _ulp_from_binade(magnitude) + old_qty = -magnitude + size = magnitude + offset + + new_qty, _price, _opened, _closed = OrderBook._simulate_position_update( + old_qty, PRICE, size, PRICE + ) + simulated_closed = new_qty == 0.0 + + broker = _close_with_residual(old_qty, residual_ulps=ulps) + executed_closed = "AAA" not in broker.positions + + assert simulated_closed == executed_closed + + @pytest.mark.parametrize("ulps", [0, 1, QTY_ZERO_ULPS, QTY_ZERO_ULPS + 1]) + def test_shadow_queue_commit_agrees_with_the_executor(self, ulps): + """ExecutionEngine._commit_shadow_queue_fill drops the same keys.""" + magnitude = OBSERVED_QUANTITY + offset = ulps * _ulp_from_binade(magnitude) + broker = _broker() + broker._update_time( + timestamp=TS, + prices={"AAA": PRICE}, + opens={"AAA": PRICE}, + highs={"AAA": PRICE}, + lows={"AAA": PRICE}, + volumes={"AAA": 1e12}, + signals={}, + ) + shadow = { + "AAA": Position( + asset="AAA", + quantity=-magnitude, + entry_price=PRICE, + current_price=PRICE, + entry_time=TS, + ) + } + order = broker.submit_order("AAA", magnitude + offset, OrderSide.BUY) + assert order is not None + broker._execution_engine._commit_shadow_queue_fill( + order=order, fill_price=PRICE, shadow_cash=0.0, shadow_positions=shadow + ) + shadow_closed = "AAA" not in shadow + + executed_closed = "AAA" not in _close_with_residual(-magnitude, ulps).positions + assert shadow_closed == executed_closed + + +class TestRetainedSites: + def test_minimum_order_size_is_unchanged(self): + """_MIN_ORDER_SIZE is an economic policy, not a residue rule.""" + assert OrderBook._MIN_ORDER_SIZE == 1e-8 + + def test_orders_at_or_below_the_minimum_are_still_refused(self): + broker = _broker() + broker._update_time( + timestamp=TS, + prices={"AAA": PRICE}, + opens={"AAA": PRICE}, + highs={"AAA": PRICE}, + lows={"AAA": PRICE}, + volumes={"AAA": 1e12}, + signals={}, + ) + assert broker.submit_order("AAA", 1e-8, OrderSide.BUY) is None + assert broker.submit_order("AAA", 1e-9, OrderSide.BUY) is None + assert broker.submit_order("AAA", 1e-7, OrderSide.BUY) is not None + + def test_tolerance_stays_below_the_minimum_order_size_in_practice(self): + """Largest operation scale seen in the field is ~874k shares.""" + assert quantity_zero_tolerance(874_442.6149965813) < OrderBook._MIN_ORDER_SIZE