Phase 2: realized volatility, volume and VWAP metrics

README.md

@@ -35,7 +35,7 @@ Early scaffolding. Roadmap:
 
 - [x] Phase 0 — Scaffolding: monorepo layout, linters, CI.
 - [x] Phase 1 — Vertical slice: sample tick CSV → order-flow imbalance → SQLite → API endpoint.
-- [ ] Phase 2 — More metrics (realized volatility, volume), tests against known data.
+- [x] Phase 2 — More metrics (realized volatility, volume, VWAP), tests against known data.
 - [ ] Phase 3 — Trade journal model + Claude API behavioral agent.
 - [ ] Phase 4 — Next.js dashboard.
 - [ ] Phase 5 — v0.1.0 release with sample-data demo.
@@ -49,16 +49,18 @@ trades or amounts.
 # 1. (Optional) regenerate the sample tape — deterministic, seeded
 python3 data/generate_ticks.py
 
-# 2. Compute order-flow imbalance per 1s window into metrics.db
+# 2. Compute microstructure metrics per 1s window into metrics.db
 cargo run --manifest-path engine/Cargo.toml --release -- \
   --input data/sample_mnq_ticks.csv --db metrics.db --window 1s
 
 # 3. Serve the metrics
 python3 -m venv .venv && .venv/bin/pip install -e "backtest[dev]"
 METRICS_DB=metrics.db .venv/bin/uvicorn backtest.api:app
 
-# 4. Query them
-curl 'localhost:8000/metrics/ofi?limit=5'
+# 4. Query them — one table, three views
+curl 'localhost:8000/metrics/ofi?limit=5'         # order-flow imbalance
+curl 'localhost:8000/metrics/volatility?limit=5'  # realized volatility + trade count
+curl 'localhost:8000/metrics/volume?limit=5'      # buy/sell/total volume + VWAP
 ```
 
 Run the checks the same way CI does:
@@ -95,3 +97,18 @@ calls were made by me — as the project progresses.
   bugs — sign errors produce equally plausible noise. The generator drifts its
   buy-pressure between 0.35 and 0.65 so imbalance windows show persistent, verifiable
   signal.
+- **One unified `metrics` table instead of a table per metric** (Phase 2, Claude's
+  proposal, accepted): every metric shares the same `(bucket_start_ns, window_ns)` key
+  and is produced in a single windowing pass, so a wide row is the natural shape. The
+  Python API projects the columns each endpoint needs (`/metrics/ofi`,
+  `/metrics/volatility`, `/metrics/volume`) out of that one table; the Phase 1
+  `/metrics/ofi` contract is unchanged.
+- **Realized volatility is continuous across window boundaries** (Phase 2, mine): a
+  window's realized variance includes the log return from the previous window's last
+  trade, so summing per-window variances recovers the whole tape's realized variance and
+  a window with a single tick still carries the volatility of its move. The alternative —
+  resetting returns at each window edge — silently discards the boundary moves and
+  understates volatility.
+- **VWAP folded into the same pass** (Phase 2, Claude's proposal, accepted): it needs
+  only the running `Σ price·size` the engine already touches per tick, so it comes almost
+  for free and gives the dashboard a price anchor alongside the volume figures.

backtest/src/backtest/api.py

@@ -21,9 +21,12 @@ def _db_path() -> Path:
     return Path(os.environ.get(DB_PATH_ENV, "metrics.db"))
 
 
-@app.get("/metrics/ofi")
-def get_ofi(limit: int = Query(default=100, ge=1, le=10_000)) -> list[dict[str, Any]]:
-    """Return the most recent OFI buckets, newest first."""
+def _query_metrics(columns: str, limit: int) -> list[dict[str, Any]]:
+    """Return `columns` from the newest metric buckets, newest first.
+
+    Shared by every endpoint: they differ only in which columns they project
+    out of the single `metrics` table the engine writes.
+    """
     path = _db_path()
     if not path.exists():
         raise HTTPException(
@@ -34,13 +37,33 @@ def get_ofi(limit: int = Query(default=100, ge=1, le=10_000)) -> list[dict[str,
         conn.row_factory = sqlite3.Row
         try:
             rows = conn.execute(
-                "SELECT bucket_start_ns, window_ns, buy_volume, sell_volume, ofi"
-                " FROM ofi ORDER BY bucket_start_ns DESC LIMIT ?",
+                f"SELECT {columns} FROM metrics ORDER BY bucket_start_ns DESC LIMIT ?",
                 (limit,),
             ).fetchall()
         except sqlite3.OperationalError as exc:
             raise HTTPException(
                 status_code=503,
-                detail=f"metrics database at '{path}' has no OFI data yet: {exc}",
+                detail=f"metrics database at '{path}' has no metrics yet: {exc}",
             ) from exc
     return [dict(row) for row in rows]
+
+
+@app.get("/metrics/ofi")
+def get_ofi(limit: int = Query(default=100, ge=1, le=10_000)) -> list[dict[str, Any]]:
+    """Return the most recent order-flow-imbalance buckets, newest first."""
+    return _query_metrics("bucket_start_ns, window_ns, buy_volume, sell_volume, ofi", limit)
+
+
+@app.get("/metrics/volatility")
+def get_volatility(limit: int = Query(default=100, ge=1, le=10_000)) -> list[dict[str, Any]]:
+    """Return the most recent realized-volatility buckets, newest first."""
+    return _query_metrics("bucket_start_ns, window_ns, realized_volatility, trade_count", limit)
+
+
+@app.get("/metrics/volume")
+def get_volume(limit: int = Query(default=100, ge=1, le=10_000)) -> list[dict[str, Any]]:
+    """Return the most recent volume buckets (with VWAP), newest first."""
+    return _query_metrics(
+        "bucket_start_ns, window_ns, buy_volume, sell_volume, total_volume, vwap, trade_count",
+        limit,
+    )

backtest/tests/test_api.py

@@ -5,30 +5,35 @@
 
 from backtest import api
 
+COLUMNS = (
+    "bucket_start_ns, window_ns, buy_volume, sell_volume, total_volume,"
+    " trade_count, ofi, vwap, realized_volatility"
+)
 ROWS = [
-    # (bucket_start_ns, window_ns, buy_volume, sell_volume, ofi)
-    (1_000_000_000, 1_000_000_000, 30, 10, 0.5),
-    (2_000_000_000, 1_000_000_000, 5, 15, -0.5),
-    (3_000_000_000, 1_000_000_000, 7, 7, 0.0),
+    # bucket_start_ns, window_ns, buy, sell, total, count, ofi, vwap, rvol
+    (1_000_000_000, 1_000_000_000, 30, 10, 40, 5, 0.5, 21_400.0, 0.001),
+    (2_000_000_000, 1_000_000_000, 5, 15, 20, 4, -0.5, 21_401.0, 0.002),
+    (3_000_000_000, 1_000_000_000, 7, 7, 14, 3, 0.0, 21_402.0, 0.0),
 ]
 
 
 @pytest.fixture
 def client(tmp_path, monkeypatch):
     db = tmp_path / "metrics.db"
     with sqlite3.connect(db) as conn:
-        conn.execute(
-            "CREATE TABLE ofi (bucket_start_ns INTEGER, window_ns INTEGER,"
-            " buy_volume INTEGER, sell_volume INTEGER, ofi REAL)"
-        )
-        conn.executemany("INSERT INTO ofi VALUES (?, ?, ?, ?, ?)", ROWS)
+        conn.execute(f"CREATE TABLE metrics ({COLUMNS})")
+        conn.executemany(f"INSERT INTO metrics VALUES ({', '.join(['?'] * len(ROWS[0]))})", ROWS)
     monkeypatch.setenv(api.DB_PATH_ENV, str(db))
     return TestClient(app=api.app)
 
 
-def test_returns_buckets_newest_first(client):
+def test_ofi_returns_buckets_newest_first(client):
     body = client.get("/metrics/ofi").json()
-    assert [row["bucket_start_ns"] for row in body] == [3_000_000_000, 2_000_000_000, 1_000_000_000]
+    assert [row["bucket_start_ns"] for row in body] == [
+        3_000_000_000,
+        2_000_000_000,
+        1_000_000_000,
+    ]
     assert body[1] == {
         "bucket_start_ns": 2_000_000_000,
         "window_ns": 1_000_000_000,
@@ -38,23 +43,50 @@ def test_returns_buckets_newest_first(client):
     }
 
 
-def test_limit_caps_rows(client):
-    assert len(client.get("/metrics/ofi", params={"limit": 2}).json()) == 2
+def test_volatility_endpoint_projects_expected_columns(client):
+    body = client.get("/metrics/volatility").json()
+    assert body[0] == {
+        "bucket_start_ns": 3_000_000_000,
+        "window_ns": 1_000_000_000,
+        "realized_volatility": 0.0,
+        "trade_count": 3,
+    }
+
+
+def test_volume_endpoint_projects_expected_columns(client):
+    body = client.get("/metrics/volume").json()
+    assert body[2] == {
+        "bucket_start_ns": 1_000_000_000,
+        "window_ns": 1_000_000_000,
+        "buy_volume": 30,
+        "sell_volume": 10,
+        "total_volume": 40,
+        "vwap": 21_400.0,
+        "trade_count": 5,
+    }
+
+
+@pytest.mark.parametrize("endpoint", ["ofi", "volatility", "volume"])
+def test_limit_caps_rows(client, endpoint):
+    assert len(client.get(f"/metrics/{endpoint}", params={"limit": 2}).json()) == 2
 
 
-def test_limit_is_validated(client):
-    assert client.get("/metrics/ofi", params={"limit": 0}).status_code == 422
+@pytest.mark.parametrize("endpoint", ["ofi", "volatility", "volume"])
+def test_limit_is_validated(client, endpoint):
+    assert client.get(f"/metrics/{endpoint}", params={"limit": 0}).status_code == 422
 
 
-def test_missing_database_returns_503(monkeypatch, tmp_path):
+@pytest.mark.parametrize("endpoint", ["ofi", "volatility", "volume"])
+def test_missing_database_returns_503(monkeypatch, tmp_path, endpoint):
     monkeypatch.setenv(api.DB_PATH_ENV, str(tmp_path / "nope.db"))
-    response = TestClient(app=api.app).get("/metrics/ofi")
+    response = TestClient(app=api.app).get(f"/metrics/{endpoint}")
     assert response.status_code == 503
     assert "run the engine first" in response.json()["detail"]
 
 
-def test_empty_database_returns_503(monkeypatch, tmp_path):
+@pytest.mark.parametrize("endpoint", ["ofi", "volatility", "volume"])
+def test_empty_database_returns_503(monkeypatch, tmp_path, endpoint):
     db = tmp_path / "empty.db"
     sqlite3.connect(db).close()
     monkeypatch.setenv(api.DB_PATH_ENV, str(db))
-    assert TestClient(app=api.app).get("/metrics/ofi").status_code == 503
+    assert TestClient(app=api.app).get(f"/metrics/{endpoint}").status_code == 503

engine/src/main.rs

@@ -1,4 +1,4 @@
-mod ofi;
+mod metrics;
 mod storage;
 mod tick;
 
@@ -28,14 +28,14 @@ fn main() -> anyhow::Result<()> {
 
     let ticks = tick::read_csv(&args.input)
         .with_context(|| format!("reading ticks from {}", args.input.display()))?;
-    let buckets = ofi::compute(&ticks, window_ns);
+    let buckets = metrics::compute(&ticks, window_ns);
 
     let mut conn = rusqlite::Connection::open(&args.db)
         .with_context(|| format!("opening database {}", args.db.display()))?;
-    storage::write_ofi(&mut conn, window_ns, &buckets)?;
+    storage::write_metrics(&mut conn, window_ns, &buckets)?;
 
     println!(
-        "{} ticks -> {} OFI buckets (window {}) -> {}",
+        "{} ticks -> {} metric buckets (window {}) -> {}",
         ticks.len(),
         buckets.len(),
         args.window,