Phase 3 polish: journal validation, CRUD, time filters, agent robustness

README.md

@@ -81,17 +81,33 @@ python3 -m backtest.coach --load data/sample_journal.csv --db metrics.db
 curl -X POST localhost:8000/journal -H 'content-type: application/json' \
   -d '{"symbol":"MNQ","side":"long","entered_at_ns":1760000030000000000,
        "entry_price":21400,"size":1,"notes":"followed my plan","emotion":"calm"}'
-curl 'localhost:8000/journal?limit=5'             # entries joined to the market regime
-curl -X POST 'localhost:8000/journal/analyze'     # Claude behavioral analysis (503 without a key)
+curl 'localhost:8000/journal?limit=5'                 # entries joined to the market regime
+curl 'localhost:8000/journal?since_ns=...&until_ns=...' # bound by entry time (since incl., until excl.)
+curl 'localhost:8000/journal/1'                       # one entry by id (404 if absent)
+curl -X PATCH localhost:8000/journal/1 -H 'content-type: application/json' \
+  -d '{"emotion":"disciplined"}'                       # partial update
+curl -X DELETE 'localhost:8000/journal/1'             # 204 on success, 404 if absent
+curl -X POST 'localhost:8000/journal/analyze'         # Claude behavioral analysis (503 without a key)
 ```
 
+`side` must be `long`/`short`, `size`/`entry_price`/`entered_at_ns` must be positive, and
+`exited_at_ns` (if given) must be ≥ `entered_at_ns` — invalid bodies return 422. If Claude
+declines or returns no structured analysis, `/journal/analyze` returns 502.
+
 Run the checks the same way CI does:
 
 ```bash
 cd engine && cargo fmt --check && cargo clippy --all-targets -- -D warnings && cargo test
 cd backtest && ruff check . && ruff format --check . && pytest
 ```
 
+Verify the whole Phase 1–3 pipeline end to end (tape → engine → metrics → journal → regime
+join; no API key needed):
+
+```bash
+bash scripts/verify_pipeline.sh
+```
+
 ## Design decisions
 
 This project is built in deliberate collaboration with Claude. This section documents
@@ -147,3 +163,17 @@ calls were made by me — as the project progresses.
   design **not** trading advice and **not** price predictions. The Anthropic client is
   dependency-injected, so the analysis is unit-tested against a mock and CI needs neither
   a key nor network access.
+- **Phase 3 polish: validation, CRUD, refusal handling, time filters** (Phase 3, mine):
+  journal entries are validated at the API boundary (`side` ∈ `long`/`short`, positive
+  size/price/timestamps, `exited_at_ns` ≥ `entered_at_ns`) → 422 on bad input. The journal
+  is a full resource (`GET`/`PATCH`/`DELETE /journal/{id}`) with `since_ns`/`until_ns`
+  filters for scoping analysis to a session. Because `messages.parse().parsed_output` is
+  `Optional` (it is `None` on a safety refusal or an empty structured parse), the agent
+  raises `AnalysisUnavailable` and the endpoint maps it to a 502 rather than surfacing a
+  bare `null`. `scripts/verify_pipeline.sh` exercises the whole tape → engine → journal →
+  regime-join chain without an API key.
+- **Nanosecond timestamps deferred as a Phase 4 JSON concern** (Phase 3, mine): the API
+  returns every `*_ns` field as a JSON integer, and these epochs (~1.76e18) exceed
+  JavaScript's safe-integer range (2^53 ≈ 9e15). Rather than change the Phase 1–2 metrics
+  contract now, the Phase 4 dashboard will read them as strings/BigInt; documented in the
+  `backtest.api` module docstring so it is not forgotten.

backtest/src/backtest/api.py

@@ -6,15 +6,21 @@
 same database, so entries can be joined to the market regime at entry time. Point
 the app at a database with the METRICS_DB environment variable (default:
 metrics.db in the working directory).
+
+Note on timestamps: every `*_ns` field is serialized as a JSON integer, and these
+nanosecond epochs (~1.76e18) exceed JavaScript's safe-integer range (2^53 ≈ 9e15).
+A JS client (the Phase 4 dashboard) must read them as strings/BigInt to avoid
+precision loss. This is left as a Phase 4 concern to keep the Phase 1–3 contract
+unchanged.
 """
 
 import os
 import sqlite3
 from pathlib import Path
-from typing import Annotated, Any
+from typing import Annotated, Any, Literal
 
 from fastapi import Depends, FastAPI, HTTPException, Query
-from pydantic import BaseModel
+from pydantic import BaseModel, Field, model_validator
 
 from backtest import coach, journal
 
@@ -26,17 +32,51 @@
 class JournalEntryIn(BaseModel):
     """A trade the user is logging to their journal."""
 
-    symbol: str
-    side: str
-    entered_at_ns: int
-    entry_price: float
-    size: int
-    exited_at_ns: int | None = None
-    exit_price: float | None = None
+    symbol: str = Field(min_length=1)
+    side: Literal["long", "short"]
+    entered_at_ns: int = Field(gt=0)
+    entry_price: float = Field(gt=0)
+    size: int = Field(gt=0)
+    exited_at_ns: int | None = Field(default=None, gt=0)
+    exit_price: float | None = Field(default=None, gt=0)
     pnl: float | None = None
     notes: str | None = None
     emotion: str | None = None
 
+    @model_validator(mode="after")
+    def _check_exit_after_entry(self) -> "JournalEntryIn":
+        if self.exited_at_ns is not None and self.exited_at_ns < self.entered_at_ns:
+            raise ValueError("exited_at_ns must be >= entered_at_ns")
+        return self
+
+
+class JournalEntryPatch(BaseModel):
+    """A partial update to a journal entry — only the provided fields change."""
+
+    symbol: str | None = Field(default=None, min_length=1)
+    side: Literal["long", "short"] | None = None
+    entered_at_ns: int | None = Field(default=None, gt=0)
+    entry_price: float | None = Field(default=None, gt=0)
+    size: int | None = Field(default=None, gt=0)
+    exited_at_ns: int | None = Field(default=None, gt=0)
+    exit_price: float | None = Field(default=None, gt=0)
+    pnl: float | None = None
+    notes: str | None = None
+    emotion: str | None = None
+
+    @model_validator(mode="after")
+    def _validate(self) -> "JournalEntryPatch":
+        if not self.model_fields_set:
+            raise ValueError("at least one field must be provided")
+        if (
+            {"entered_at_ns", "exited_at_ns"} <= self.model_fields_set
+            and self.entered_at_ns is not None
+            and self.exited_at_ns is not None
+            and self.exited_at_ns < self.entered_at_ns
+        ):
+            raise ValueError("exited_at_ns must be >= entered_at_ns")
+        return self
+
 
 def get_anthropic_client() -> Any:
     """Provide an Anthropic client, or 503 when no API key is configured.
@@ -113,21 +153,59 @@ def create_journal_entry(entry: JournalEntryIn) -> dict[str, Any]:
 
 
 @app.get("/journal")
-def get_journal(limit: int = Query(default=100, ge=1, le=10_000)) -> list[dict[str, Any]]:
+def get_journal(
+    limit: int = Query(default=100, ge=1, le=10_000),
+    since_ns: int | None = Query(default=None),
+    until_ns: int | None = Query(default=None),
+) -> list[dict[str, Any]]:
     """Return journal entries newest first, each joined to its market regime.
 
+    ``since_ns`` (inclusive) and ``until_ns`` (exclusive) bound the entry time.
     An empty journal (or a database that does not exist yet) returns ``[]`` —
     unlike the metrics endpoints, the journal is created by this layer, so
     "no entries yet" is a normal state rather than a 503.
     """
-    return journal.list_enriched(_db_path(), limit)
+    return journal.list_enriched(_db_path(), limit, since_ns=since_ns, until_ns=until_ns)
+
+
+@app.get("/journal/{entry_id}")
+def get_journal_entry(entry_id: int) -> dict[str, Any]:
+    """Return one journal entry (joined to its market regime), or 404."""
+    entry = journal.get_entry(_db_path(), entry_id)
+    if entry is None:
+        raise HTTPException(status_code=404, detail=f"journal entry {entry_id} not found")
+    return entry
+
+
+@app.patch("/journal/{entry_id}")
+def patch_journal_entry(entry_id: int, patch: JournalEntryPatch) -> dict[str, Any]:
+    """Apply a partial update to a journal entry, returning the updated row."""
+    updated = journal.update_entry(_db_path(), entry_id, patch.model_dump(exclude_unset=True))
+    if updated is None:
+        raise HTTPException(status_code=404, detail=f"journal entry {entry_id} not found")
+    return updated
+
+
+@app.delete("/journal/{entry_id}", status_code=204)
+def delete_journal_entry(entry_id: int) -> None:
+    """Delete a journal entry by id (204 on success, 404 if absent)."""
+    if not journal.delete_entry(_db_path(), entry_id):
+        raise HTTPException(status_code=404, detail=f"journal entry {entry_id} not found")
 
 
 @app.post("/journal/analyze")
 def analyze_journal(
     client: Annotated[Any, Depends(get_anthropic_client)],
     limit: int = Query(default=100, ge=1, le=10_000),
+    since_ns: int | None = Query(default=None),
+    until_ns: int | None = Query(default=None),
 ) -> coach.BehavioralAnalysis:
-    """Run the Claude behavioral-analysis agent over the journal."""
-    entries = journal.list_enriched(_db_path(), limit)
-    return coach.analyze(entries, client=client)
+    """Run the Claude behavioral-analysis agent over the journal.
+
+    Returns 502 when the model declines or yields no structured analysis.
+    """
+    entries = journal.list_enriched(_db_path(), limit, since_ns=since_ns, until_ns=until_ns)
+    try:
+        return coach.analyze(entries, client=client)
+    except coach.AnalysisUnavailable as exc:
+        raise HTTPException(status_code=502, detail=str(exc)) from exc

backtest/src/backtest/coach.py

@@ -53,6 +53,15 @@
 """
 
 
+class AnalysisUnavailable(Exception):
+    """Raised when Claude returns no usable structured analysis.
+
+    Happens on a safety ``refusal`` or when structured parsing yields nothing.
+    The API maps this to a 502 and the CLI reports it, rather than surfacing a
+    bare ``None``.
+    """
+
+
 class BehavioralObservation(BaseModel):
     """One recurring behavioral pattern found in the journal."""
 
@@ -126,7 +135,13 @@ def analyze(
         messages=messages,
         output_format=BehavioralAnalysis,
     )
-    return response.parsed_output
+    analysis = response.parsed_output
+    if analysis is None:
+        stop_reason = getattr(response, "stop_reason", None)
+        raise AnalysisUnavailable(
+            f"the model returned no structured analysis (stop_reason={stop_reason!r})"
+        )
+    return analysis
 
 
 def _format_entry(entry: dict[str, Any]) -> str:
@@ -181,7 +196,11 @@ def main(argv: list[str] | None = None) -> None:
 
     import anthropic  # lazy: keeps the SDK out of the library import path
 
-    analysis = analyze(entries, client=anthropic.Anthropic())
+    try:
+        analysis = analyze(entries, client=anthropic.Anthropic())
+    except AnalysisUnavailable as exc:
+        print(f"analysis unavailable: {exc}")
+        return
     print(_render(analysis))