feat: Phase 3 — Dawarich location-history validation (advisory)

.env.example

@@ -39,3 +39,11 @@ RUN_ON_START=true          # also run once when the container starts
 # LLM_URL=http://airtrail-host:3000
 # LLM_MODEL=llama3.1:8b
 # LLM_API_KEY=REPLACE_ME
+
+# --- Dawarich location validation (Phase 3, optional, opt-in) ---
+# Cross-checks each candidate against your Dawarich point history: confirms a flight
+# when you were near the airport, flags a likely cancellation/rebooking when tracking
+# was active but you were clearly elsewhere. Advisory only — never writes or deletes.
+# DAWARICH_VALIDATE=false
+# DAWARICH_URL=http://airtrail-host:3005
+# DAWARICH_API_KEY=REPLACE_ME

Dockerfile

@@ -13,7 +13,7 @@ ENV PYTHONUNBUFFERED=1 \
 WORKDIR /app
 
 # Copy only the code (no secrets, no output — see .dockerignore).
-COPY layover.py flightparse.py airdata.py airtrail.py llm.py notify.py populate.py scheduler.py ./
+COPY layover.py flightparse.py airdata.py airtrail.py llm.py dawarich.py notify.py populate.py scheduler.py ./
 
 # Run unprivileged; /data (state.json watermark + candidates) is a persisted volume.
 RUN useradd --system --uid 10001 --home-dir /app layover \

README.md

@@ -297,9 +297,13 @@ Phase 1 (above) is deterministic templates + digest, no LLM in the loop. Where t
   and asks for strict JSON. Every LLM candidate is marked `uncertain` on purpose, so it always lands
   in the human-review bucket and can never reach the unattended `--auto-write` path. Enable with
   `LLM_FALLBACK=1` + `LLM_URL`/`LLM_MODEL` (see `.env.example`) or `populate.py --llm`.
-- **Phase 3 — location-history validation:** cross-check candidates against
-  [Dawarich](https://github.com/Freika/dawarich) point history to auto-confirm a flight or flag a
-  cancelled/rebooked one (the signal a phone app gets, self-hosted).
+- **Phase 3 — location-history validation** (shipped, opt-in): cross-checks candidates against
+  [Dawarich](https://github.com/Freika/dawarich) point history — confirms a flight when you were
+  near the airport, flags a likely cancellation/rebooking when tracking was active but you were
+  clearly elsewhere (the signal a phone app gets, self-hosted). Advisory only: it annotates and, on
+  a contradiction, adds a review flag that also blocks unattended auto-write — it never writes or
+  deletes. Absence of data is always "unknown", never a false alarm. Enable with
+  `DAWARICH_VALIDATE=1` + `DAWARICH_URL`/`DAWARICH_API_KEY` or `populate.py --validate`.
 - **Miles & More** stays a periodic manual cross-check — highest-precision source, used as a
   completeness checksum, not the engine.
 

airdata.py

@@ -157,6 +157,85 @@
 }
 
 
+# --- Airport coordinates: IATA -> (lat, lon) ------------------------------
+# Approximate (≈1 decimal is plenty) — used only for a coarse "was I near this
+# airport?" proximity test against Dawarich location history (radius ~100 km), so
+# metro-area precision is more than enough. Missing airports degrade to "unknown"
+# in the validator; add rows as new routes appear. Not exhaustive by design.
+AIRPORT_LATLON = {
+    # DACH
+    "MUC": (48.35, 11.79), "FRA": (50.04, 8.56), "TXL": (52.56, 13.29),
+    "BER": (52.37, 13.50), "DUS": (51.29, 6.77), "HAM": (53.63, 9.99),
+    "STR": (48.69, 9.22), "CGN": (50.87, 7.14), "NUE": (49.50, 11.08),
+    "ZRH": (47.46, 8.55), "GVA": (46.24, 6.11), "BSL": (47.59, 7.53),
+    "VIE": (48.11, 16.57), "SZG": (47.79, 13.00), "INN": (47.26, 11.34),
+    # UK / Ireland
+    "LHR": (51.47, -0.46), "LGW": (51.15, -0.19), "LCY": (51.51, 0.05),
+    "STN": (51.89, 0.24), "LTN": (51.87, -0.37), "MAN": (53.35, -2.27),
+    "EDI": (55.95, -3.37), "DUB": (53.42, -6.27),
+    # France / Benelux
+    "CDG": (49.01, 2.55), "ORY": (48.73, 2.36), "NCE": (43.66, 7.22),
+    "LYS": (45.73, 5.08), "AMS": (52.31, 4.76), "BRU": (50.90, 4.48),
+    "CRL": (50.46, 4.45), "LUX": (49.63, 6.21),
+    # Iberia
+    "MAD": (40.47, -3.56), "BCN": (41.30, 2.08), "PMI": (39.55, 2.74),
+    "AGP": (36.67, -4.50), "VLC": (39.49, -0.48), "IBZ": (38.87, 1.37),
+    "LPA": (27.93, -15.39), "TFS": (28.04, -16.57), "LIS": (38.77, -9.13),
+    "OPO": (41.24, -8.68), "FAO": (37.01, -7.97),
+    # Italy
+    "FCO": (41.80, 12.24), "CIA": (41.80, 12.59), "MXP": (45.63, 8.72),
+    "LIN": (45.45, 9.28), "BGY": (45.67, 9.70), "VCE": (45.51, 12.35),
+    "NAP": (40.89, 14.29), "BLQ": (44.53, 11.30), "CTA": (37.47, 15.07),
+    # Nordics
+    "ARN": (59.65, 17.92), "BMA": (59.35, 17.94), "GOT": (57.67, 12.29),
+    "CPH": (55.62, 12.65), "OSL": (60.19, 11.10), "BGO": (60.29, 5.22),
+    "HEL": (60.32, 24.96), "KEF": (63.99, -22.61),
+    # Central / Eastern Europe & Balkans
+    "PRG": (50.10, 14.26), "WAW": (52.17, 20.97), "WMI": (52.45, 20.65),
+    "KRK": (50.08, 19.79), "BUD": (47.44, 19.26), "OTP": (44.57, 26.09),
+    "SOF": (42.70, 23.41), "BEG": (44.82, 20.29), "ZAG": (45.74, 16.07),
+    "SPU": (43.54, 16.30), "DBV": (42.56, 18.27), "LJU": (46.22, 14.46),
+    # Greece / Cyprus / Malta / Turkey
+    "ATH": (37.94, 23.95), "SKG": (40.52, 22.97), "HER": (35.34, 25.18),
+    "JMK": (37.44, 25.35), "JTR": (36.40, 25.48), "LCA": (34.88, 33.63),
+    "PFO": (34.72, 32.49), "MLA": (35.86, 14.48), "IST": (41.26, 28.74),
+    "SAW": (40.90, 29.31), "AYT": (36.90, 30.79),
+    # Middle East / Africa
+    "DOH": (25.27, 51.61), "DXB": (25.25, 55.36), "AUH": (24.43, 54.65),
+    "TLV": (32.01, 34.89), "CAI": (30.11, 31.41), "JNB": (-26.13, 28.24),
+    "CPT": (-33.97, 18.60), "NBO": (-1.32, 36.93), "ADD": (8.98, 38.80),
+    "RAK": (31.61, -8.04), "CMN": (33.37, -7.59),
+    # Americas
+    "JFK": (40.64, -73.78), "EWR": (40.69, -74.17), "IAD": (38.94, -77.46),
+    "BOS": (42.36, -71.01), "ORD": (41.98, -87.90), "ATL": (33.64, -84.43),
+    "MIA": (25.79, -80.29), "FLL": (26.07, -80.15), "LAX": (33.94, -118.41),
+    "SFO": (37.62, -122.38), "SEA": (47.45, -122.31), "DEN": (39.86, -104.67),
+    "DFW": (32.90, -97.04), "BIL": (45.81, -108.54), "YYZ": (43.68, -79.61),
+    "GRU": (-23.43, -46.47), "GIG": (-22.81, -43.25), "SCL": (-33.39, -70.79),
+    "EZE": (-34.82, -58.54), "BOG": (4.70, -74.15), "LIM": (-12.02, -77.11),
+    "MEX": (19.44, -99.07), "CUN": (21.04, -86.87),
+    # Asia / Pacific
+    "SIN": (1.36, 103.99), "HKG": (22.31, 113.91), "BKK": (13.69, 100.75),
+    "DEL": (28.57, 77.10), "BOM": (19.09, 72.87), "NRT": (35.77, 140.39),
+    "HND": (35.55, 139.78), "ICN": (37.46, 126.44), "PVG": (31.14, 121.81),
+    "SYD": (-33.95, 151.18), "MEL": (-37.67, 144.84), "AKL": (-37.01, 174.79),
+}
+
+# reverse IATA<-ICAO for coordinate lookups by ICAO
+_ICAO_IATA = {v: k for k, v in AIRPORT_IATA_ICAO.items()}
+
+
+def airport_latlon(code):
+    """Return (lat, lon) for an airport given IATA or ICAO; None if unknown."""
+    if not code:
+        return None
+    c = code.strip().upper()
+    if c in AIRPORT_LATLON:
+        return AIRPORT_LATLON[c]
+    iata = _ICAO_IATA.get(c)          # accept an ICAO in
+    return AIRPORT_LATLON.get(iata) if iata else None
+
+
 def airport_icao(code):
     """Return the ICAO for an airport given IATA or ICAO; None if unknown."""
     if not code:

dawarich.py

@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+"""
+dawarich — location-history validation of candidate flights (Phase 3).
+
+Dawarich already knows where you actually were (your phone's point history). This
+module uses that as a second, independent signal on a parsed candidate — the same
+thing a flight-tracker app gets from GPS, but self-hosted:
+
+  * points near the destination shortly after arrival  -> "confirmed"
+  * tracking clearly active that day but nowhere near either endpoint
+        -> "contradicted"  (a likely cancellation/rebooking — the ghost-flight trap)
+  * no coordinates for the airport, or no points in the window -> "unknown"
+
+It only ever ANNOTATES: it sets candidate["location"] and, on a contradiction, adds
+an issue (which also blocks unattended auto-write, since _blocked_reason trips on
+"cancellation"). It never changes confidence or writes anything. Absence of data is
+"unknown", never "contradicted" — a Dawarich tracking gap must not invent a problem.
+
+Config (env, or the matching populate.py flags):
+    DAWARICH_URL       base URL              (e.g. http://airtrail-host:3005)
+    DAWARICH_API_KEY   API key
+    DAWARICH_VALIDATE  truthy to enable
+
+Zero dependencies (urllib + math).
+"""
+
+import json
+import math
+import os
+import urllib.parse
+import urllib.request
+from datetime import datetime, timedelta
+
+import airdata
+
+RADIUS_KM = 100          # metro-area proximity for "near an airport"
+WINDOW_DAYS = 1          # look ±1 day around the flight date
+MIN_POINTS_FOR_CONTRADICTION = 5   # need real tracking before calling a miss a problem
+
+
+# ---------------------------------------------------------------------------
+# config
+# ---------------------------------------------------------------------------
+
+def load_config(url=None, api_key=None):
+    return (url or os.environ.get("DAWARICH_URL"),
+            api_key or os.environ.get("DAWARICH_API_KEY"))
+
+
+def enabled(url=None, api_key=None):
+    on = os.environ.get("DAWARICH_VALIDATE", "").strip().lower() in (
+        "1", "true", "yes", "on")
+    u, k = load_config(url, api_key)
+    return bool(on and u and k)
+
+
+# ---------------------------------------------------------------------------
+# geo
+# ---------------------------------------------------------------------------
+
+def haversine_km(a, b):
+    """Great-circle distance in km between two (lat, lon) pairs."""
+    lat1, lon1, lat2, lon2 = map(math.radians, (a[0], a[1], b[0], b[1]))
+    dlat, dlon = lat2 - lat1, lon2 - lon1
+    h = math.sin(dlat / 2) ** 2 + math.cos(lat1) * math.cos(lat2) * math.sin(dlon / 2) ** 2
+    return 2 * 6371.0 * math.asin(math.sqrt(h))
+
+
+# ---------------------------------------------------------------------------
+# fetching points
+# ---------------------------------------------------------------------------
+
+def fetch_points(url, api_key, start, end, timeout=30):
+    """GET points in [start, end] (date strings/ISO). Returns [(lat, lon), ...].
+
+    Tolerant of Dawarich response shape: accepts a bare list or {data|points|
+    features:[...]}, and either flat lat/lon keys or GeoJSON geometry."""
+    q = urllib.parse.urlencode({
+        "api_key": api_key,
+        "start_at": start,
+        "end_at": end,
+        "per_page": 1000,
+    })
+    endpoint = url.rstrip("/") + "/api/v1/points?" + q
+    req = urllib.request.Request(endpoint, headers={
+        "Authorization": f"Bearer {api_key}", "Accept": "application/json"})
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        payload = json.loads(resp.read().decode())
+    return _coords(payload)
+
+
+def _coords(payload):
+    rows = payload
+    if isinstance(payload, dict):
+        for k in ("data", "points", "features", "results"):
+            if isinstance(payload.get(k), list):
+                rows = payload[k]
+                break
+        else:
+            rows = []
+    out = []
+    for r in rows or []:
+        if not isinstance(r, dict):
+            continue
+        lat = r.get("latitude", r.get("lat"))
+        lon = r.get("longitude", r.get("lon", r.get("lng")))
+        if lat is None and isinstance(r.get("geometry"), dict):
+            coords = r["geometry"].get("coordinates")     # GeoJSON [lon, lat]
+            if isinstance(coords, (list, tuple)) and len(coords) >= 2:
+                lon, lat = coords[0], coords[1]
+        try:
+            out.append((float(lat), float(lon)))
+        except (TypeError, ValueError):
+            continue
+    return out
+
+
+# ---------------------------------------------------------------------------
+# validation
+# ---------------------------------------------------------------------------
+
+def _near(points, airport_code, radius_km=RADIUS_KM):
+    """True/False if any point is within radius of the airport; None if the
+    airport has no known coordinates (can't tell)."""
+    ap = airdata.airport_latlon(airport_code)
+    if not ap:
+        return None
+    return any(haversine_km(ap, p) <= radius_km for p in points)
+
+
+def classify_location(cand, points):
+    """Return 'confirmed' | 'contradicted' | 'unknown' for one candidate given the
+    points already filtered to its date window."""
+    near_to = _near(points, cand.get("to") or cand.get("to_iata"))
+    near_from = _near(points, cand.get("from") or cand.get("from_iata"))
+    if near_to or near_from:
+        return "confirmed"
+    # only call it a contradiction if tracking was genuinely active and we had a
+    # coordinate to test against — otherwise we simply don't know.
+    testable = near_to is not None or near_from is not None
+    if testable and len(points) >= MIN_POINTS_FOR_CONTRADICTION:
+        return "contradicted"
+    return "unknown"
+
+
+def validate(cands, url=None, api_key=None, fetch=fetch_points):
+    """Annotate each candidate with candidate['location']; add a review issue on a
+    contradiction. `fetch` is injectable for testing. Returns the same list."""
+    url, api_key = load_config(url, api_key)
+    for c in cands:
+        if not c.get("date"):
+            c["location"] = "unknown"
+            continue
+        try:
+            d = datetime.strptime(c["date"][:10], "%Y-%m-%d")
+        except (ValueError, TypeError):
+            c["location"] = "unknown"
+            continue
+        start = (d - timedelta(days=WINDOW_DAYS)).strftime("%Y-%m-%d")
+        end = (d + timedelta(days=WINDOW_DAYS)).strftime("%Y-%m-%d")
+        try:
+            points = fetch(url, api_key, start, end) if url and api_key else []
+        except Exception as exc:  # noqa: BLE001 - a Dawarich hiccup must not break the run
+            c["location"] = "unknown"
+            c.setdefault("issues", []).append(f"Dawarich check failed: {exc}")
+            continue
+        loc = classify_location(c, points)
+        c["location"] = loc
+        if loc == "contradicted":
+            c.setdefault("issues", []).append(
+                f"Dawarich shows you elsewhere around {c['date']} "
+                f"(no points near {c.get('from_iata')}/{c.get('to_iata')}) — "
+                "possible cancellation/rebooking")
+    return cands

populate.py

@@ -40,6 +40,7 @@
 from collections import defaultdict
 
 import airtrail
+import dawarich
 import flightparse
 import llm
 import notify as notify_mod
@@ -151,11 +152,13 @@ def show(title, items):
         if not items:
             return
         print(f"--- {title} ({len(items)}) ---")
+        _loc = {"confirmed": " 📍ok", "contradicted": " 📍?!"}
         for c in items:
             seat = f" seat {c['seatNumber']}" if c.get("seatNumber") else ""
+            loc = _loc.get(c.get("location"), "")
             line = (f"  {c['date']}  {(c['flightNumber'] or '?'):8} "
                     f"{c['from'] or c['from_iata']}->{c['to'] or c['to_iata']}"
-                    f"  {(c['departure'] or '')[:16]}{seat}  [{c['extractor']}]")
+                    f"  {(c['departure'] or '')[:16]}{seat}{loc}  [{c['extractor']}]")
             print(line)
             for issue in c.get("issues", []):
                 print(f"        ! {issue}")
@@ -270,6 +273,11 @@ def main():
                          "miss (also via LLM_FALLBACK=1 + LLM_URL/LLM_MODEL)")
     ap.add_argument("--no-llm", dest="llm", action="store_false",
                     help="force the LLM fallback off even if configured")
+    ap.add_argument("--validate", dest="validate", action="store_true", default=None,
+                    help="cross-check candidates against Dawarich location history "
+                         "(also via DAWARICH_VALIDATE=1 + DAWARICH_URL/API_KEY)")
+    ap.add_argument("--no-validate", dest="validate", action="store_false",
+                    help="skip Dawarich validation even if configured")
     args = ap.parse_args()
 
     if args.pull:
@@ -278,6 +286,8 @@ def main():
     cands = parse_candidates(args.out_dir, use_llm=args.llm)
     existing, source_label = get_existing(args)
     classify(cands, existing)
+    if args.validate if args.validate is not None else dawarich.enabled():
+        dawarich.validate(cands)
     rebookings = find_rebookings(cands)
 
     if args.out: