diff --git a/.env.example b/.env.example index 6c663cf..6368cda 100644 --- a/.env.example +++ b/.env.example @@ -39,3 +39,11 @@ RUN_ON_START=true # also run once when the container starts # LLM_URL=http://airtrail-host:3000 # LLM_MODEL=llama3.1:8b # LLM_API_KEY=REPLACE_ME + +# --- Dawarich location validation (Phase 3, optional, opt-in) --- +# Cross-checks each candidate against your Dawarich point history: confirms a flight +# when you were near the airport, flags a likely cancellation/rebooking when tracking +# was active but you were clearly elsewhere. Advisory only — never writes or deletes. +# DAWARICH_VALIDATE=false +# DAWARICH_URL=http://airtrail-host:3005 +# DAWARICH_API_KEY=REPLACE_ME diff --git a/Dockerfile b/Dockerfile index 6ceccb9..1452f69 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ ENV PYTHONUNBUFFERED=1 \ WORKDIR /app # Copy only the code (no secrets, no output — see .dockerignore). -COPY layover.py flightparse.py airdata.py airtrail.py llm.py notify.py populate.py scheduler.py ./ +COPY layover.py flightparse.py airdata.py airtrail.py llm.py dawarich.py notify.py populate.py scheduler.py ./ # Run unprivileged; /data (state.json watermark + candidates) is a persisted volume. RUN useradd --system --uid 10001 --home-dir /app layover \ diff --git a/README.md b/README.md index 292c817..5d760ff 100644 --- a/README.md +++ b/README.md @@ -297,9 +297,13 @@ Phase 1 (above) is deterministic templates + digest, no LLM in the loop. Where t and asks for strict JSON. Every LLM candidate is marked `uncertain` on purpose, so it always lands in the human-review bucket and can never reach the unattended `--auto-write` path. Enable with `LLM_FALLBACK=1` + `LLM_URL`/`LLM_MODEL` (see `.env.example`) or `populate.py --llm`. -- **Phase 3 — location-history validation:** cross-check candidates against - [Dawarich](https://github.com/Freika/dawarich) point history to auto-confirm a flight or flag a - cancelled/rebooked one (the signal a phone app gets, self-hosted). +- **Phase 3 — location-history validation** (shipped, opt-in): cross-checks candidates against + [Dawarich](https://github.com/Freika/dawarich) point history — confirms a flight when you were + near the airport, flags a likely cancellation/rebooking when tracking was active but you were + clearly elsewhere (the signal a phone app gets, self-hosted). Advisory only: it annotates and, on + a contradiction, adds a review flag that also blocks unattended auto-write — it never writes or + deletes. Absence of data is always "unknown", never a false alarm. Enable with + `DAWARICH_VALIDATE=1` + `DAWARICH_URL`/`DAWARICH_API_KEY` or `populate.py --validate`. - **Miles & More** stays a periodic manual cross-check — highest-precision source, used as a completeness checksum, not the engine. diff --git a/airdata.py b/airdata.py index 6b14cfb..af575bf 100644 --- a/airdata.py +++ b/airdata.py @@ -157,6 +157,85 @@ } +# --- Airport coordinates: IATA -> (lat, lon) ------------------------------ +# Approximate (≈1 decimal is plenty) — used only for a coarse "was I near this +# airport?" proximity test against Dawarich location history (radius ~100 km), so +# metro-area precision is more than enough. Missing airports degrade to "unknown" +# in the validator; add rows as new routes appear. Not exhaustive by design. +AIRPORT_LATLON = { + # DACH + "MUC": (48.35, 11.79), "FRA": (50.04, 8.56), "TXL": (52.56, 13.29), + "BER": (52.37, 13.50), "DUS": (51.29, 6.77), "HAM": (53.63, 9.99), + "STR": (48.69, 9.22), "CGN": (50.87, 7.14), "NUE": (49.50, 11.08), + "ZRH": (47.46, 8.55), "GVA": (46.24, 6.11), "BSL": (47.59, 7.53), + "VIE": (48.11, 16.57), "SZG": (47.79, 13.00), "INN": (47.26, 11.34), + # UK / Ireland + "LHR": (51.47, -0.46), "LGW": (51.15, -0.19), "LCY": (51.51, 0.05), + "STN": (51.89, 0.24), "LTN": (51.87, -0.37), "MAN": (53.35, -2.27), + "EDI": (55.95, -3.37), "DUB": (53.42, -6.27), + # France / Benelux + "CDG": (49.01, 2.55), "ORY": (48.73, 2.36), "NCE": (43.66, 7.22), + "LYS": (45.73, 5.08), "AMS": (52.31, 4.76), "BRU": (50.90, 4.48), + "CRL": (50.46, 4.45), "LUX": (49.63, 6.21), + # Iberia + "MAD": (40.47, -3.56), "BCN": (41.30, 2.08), "PMI": (39.55, 2.74), + "AGP": (36.67, -4.50), "VLC": (39.49, -0.48), "IBZ": (38.87, 1.37), + "LPA": (27.93, -15.39), "TFS": (28.04, -16.57), "LIS": (38.77, -9.13), + "OPO": (41.24, -8.68), "FAO": (37.01, -7.97), + # Italy + "FCO": (41.80, 12.24), "CIA": (41.80, 12.59), "MXP": (45.63, 8.72), + "LIN": (45.45, 9.28), "BGY": (45.67, 9.70), "VCE": (45.51, 12.35), + "NAP": (40.89, 14.29), "BLQ": (44.53, 11.30), "CTA": (37.47, 15.07), + # Nordics + "ARN": (59.65, 17.92), "BMA": (59.35, 17.94), "GOT": (57.67, 12.29), + "CPH": (55.62, 12.65), "OSL": (60.19, 11.10), "BGO": (60.29, 5.22), + "HEL": (60.32, 24.96), "KEF": (63.99, -22.61), + # Central / Eastern Europe & Balkans + "PRG": (50.10, 14.26), "WAW": (52.17, 20.97), "WMI": (52.45, 20.65), + "KRK": (50.08, 19.79), "BUD": (47.44, 19.26), "OTP": (44.57, 26.09), + "SOF": (42.70, 23.41), "BEG": (44.82, 20.29), "ZAG": (45.74, 16.07), + "SPU": (43.54, 16.30), "DBV": (42.56, 18.27), "LJU": (46.22, 14.46), + # Greece / Cyprus / Malta / Turkey + "ATH": (37.94, 23.95), "SKG": (40.52, 22.97), "HER": (35.34, 25.18), + "JMK": (37.44, 25.35), "JTR": (36.40, 25.48), "LCA": (34.88, 33.63), + "PFO": (34.72, 32.49), "MLA": (35.86, 14.48), "IST": (41.26, 28.74), + "SAW": (40.90, 29.31), "AYT": (36.90, 30.79), + # Middle East / Africa + "DOH": (25.27, 51.61), "DXB": (25.25, 55.36), "AUH": (24.43, 54.65), + "TLV": (32.01, 34.89), "CAI": (30.11, 31.41), "JNB": (-26.13, 28.24), + "CPT": (-33.97, 18.60), "NBO": (-1.32, 36.93), "ADD": (8.98, 38.80), + "RAK": (31.61, -8.04), "CMN": (33.37, -7.59), + # Americas + "JFK": (40.64, -73.78), "EWR": (40.69, -74.17), "IAD": (38.94, -77.46), + "BOS": (42.36, -71.01), "ORD": (41.98, -87.90), "ATL": (33.64, -84.43), + "MIA": (25.79, -80.29), "FLL": (26.07, -80.15), "LAX": (33.94, -118.41), + "SFO": (37.62, -122.38), "SEA": (47.45, -122.31), "DEN": (39.86, -104.67), + "DFW": (32.90, -97.04), "BIL": (45.81, -108.54), "YYZ": (43.68, -79.61), + "GRU": (-23.43, -46.47), "GIG": (-22.81, -43.25), "SCL": (-33.39, -70.79), + "EZE": (-34.82, -58.54), "BOG": (4.70, -74.15), "LIM": (-12.02, -77.11), + "MEX": (19.44, -99.07), "CUN": (21.04, -86.87), + # Asia / Pacific + "SIN": (1.36, 103.99), "HKG": (22.31, 113.91), "BKK": (13.69, 100.75), + "DEL": (28.57, 77.10), "BOM": (19.09, 72.87), "NRT": (35.77, 140.39), + "HND": (35.55, 139.78), "ICN": (37.46, 126.44), "PVG": (31.14, 121.81), + "SYD": (-33.95, 151.18), "MEL": (-37.67, 144.84), "AKL": (-37.01, 174.79), +} + +# reverse IATA<-ICAO for coordinate lookups by ICAO +_ICAO_IATA = {v: k for k, v in AIRPORT_IATA_ICAO.items()} + + +def airport_latlon(code): + """Return (lat, lon) for an airport given IATA or ICAO; None if unknown.""" + if not code: + return None + c = code.strip().upper() + if c in AIRPORT_LATLON: + return AIRPORT_LATLON[c] + iata = _ICAO_IATA.get(c) # accept an ICAO in + return AIRPORT_LATLON.get(iata) if iata else None + + def airport_icao(code): """Return the ICAO for an airport given IATA or ICAO; None if unknown.""" if not code: diff --git a/dawarich.py b/dawarich.py new file mode 100644 index 0000000..8059537 --- /dev/null +++ b/dawarich.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +""" +dawarich — location-history validation of candidate flights (Phase 3). + +Dawarich already knows where you actually were (your phone's point history). This +module uses that as a second, independent signal on a parsed candidate — the same +thing a flight-tracker app gets from GPS, but self-hosted: + + * points near the destination shortly after arrival -> "confirmed" + * tracking clearly active that day but nowhere near either endpoint + -> "contradicted" (a likely cancellation/rebooking — the ghost-flight trap) + * no coordinates for the airport, or no points in the window -> "unknown" + +It only ever ANNOTATES: it sets candidate["location"] and, on a contradiction, adds +an issue (which also blocks unattended auto-write, since _blocked_reason trips on +"cancellation"). It never changes confidence or writes anything. Absence of data is +"unknown", never "contradicted" — a Dawarich tracking gap must not invent a problem. + +Config (env, or the matching populate.py flags): + DAWARICH_URL base URL (e.g. http://airtrail-host:3005) + DAWARICH_API_KEY API key + DAWARICH_VALIDATE truthy to enable + +Zero dependencies (urllib + math). +""" + +import json +import math +import os +import urllib.parse +import urllib.request +from datetime import datetime, timedelta + +import airdata + +RADIUS_KM = 100 # metro-area proximity for "near an airport" +WINDOW_DAYS = 1 # look ±1 day around the flight date +MIN_POINTS_FOR_CONTRADICTION = 5 # need real tracking before calling a miss a problem + + +# --------------------------------------------------------------------------- +# config +# --------------------------------------------------------------------------- + +def load_config(url=None, api_key=None): + return (url or os.environ.get("DAWARICH_URL"), + api_key or os.environ.get("DAWARICH_API_KEY")) + + +def enabled(url=None, api_key=None): + on = os.environ.get("DAWARICH_VALIDATE", "").strip().lower() in ( + "1", "true", "yes", "on") + u, k = load_config(url, api_key) + return bool(on and u and k) + + +# --------------------------------------------------------------------------- +# geo +# --------------------------------------------------------------------------- + +def haversine_km(a, b): + """Great-circle distance in km between two (lat, lon) pairs.""" + lat1, lon1, lat2, lon2 = map(math.radians, (a[0], a[1], b[0], b[1])) + dlat, dlon = lat2 - lat1, lon2 - lon1 + h = math.sin(dlat / 2) ** 2 + math.cos(lat1) * math.cos(lat2) * math.sin(dlon / 2) ** 2 + return 2 * 6371.0 * math.asin(math.sqrt(h)) + + +# --------------------------------------------------------------------------- +# fetching points +# --------------------------------------------------------------------------- + +def fetch_points(url, api_key, start, end, timeout=30): + """GET points in [start, end] (date strings/ISO). Returns [(lat, lon), ...]. + + Tolerant of Dawarich response shape: accepts a bare list or {data|points| + features:[...]}, and either flat lat/lon keys or GeoJSON geometry.""" + q = urllib.parse.urlencode({ + "api_key": api_key, + "start_at": start, + "end_at": end, + "per_page": 1000, + }) + endpoint = url.rstrip("/") + "/api/v1/points?" + q + req = urllib.request.Request(endpoint, headers={ + "Authorization": f"Bearer {api_key}", "Accept": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout) as resp: + payload = json.loads(resp.read().decode()) + return _coords(payload) + + +def _coords(payload): + rows = payload + if isinstance(payload, dict): + for k in ("data", "points", "features", "results"): + if isinstance(payload.get(k), list): + rows = payload[k] + break + else: + rows = [] + out = [] + for r in rows or []: + if not isinstance(r, dict): + continue + lat = r.get("latitude", r.get("lat")) + lon = r.get("longitude", r.get("lon", r.get("lng"))) + if lat is None and isinstance(r.get("geometry"), dict): + coords = r["geometry"].get("coordinates") # GeoJSON [lon, lat] + if isinstance(coords, (list, tuple)) and len(coords) >= 2: + lon, lat = coords[0], coords[1] + try: + out.append((float(lat), float(lon))) + except (TypeError, ValueError): + continue + return out + + +# --------------------------------------------------------------------------- +# validation +# --------------------------------------------------------------------------- + +def _near(points, airport_code, radius_km=RADIUS_KM): + """True/False if any point is within radius of the airport; None if the + airport has no known coordinates (can't tell).""" + ap = airdata.airport_latlon(airport_code) + if not ap: + return None + return any(haversine_km(ap, p) <= radius_km for p in points) + + +def classify_location(cand, points): + """Return 'confirmed' | 'contradicted' | 'unknown' for one candidate given the + points already filtered to its date window.""" + near_to = _near(points, cand.get("to") or cand.get("to_iata")) + near_from = _near(points, cand.get("from") or cand.get("from_iata")) + if near_to or near_from: + return "confirmed" + # only call it a contradiction if tracking was genuinely active and we had a + # coordinate to test against — otherwise we simply don't know. + testable = near_to is not None or near_from is not None + if testable and len(points) >= MIN_POINTS_FOR_CONTRADICTION: + return "contradicted" + return "unknown" + + +def validate(cands, url=None, api_key=None, fetch=fetch_points): + """Annotate each candidate with candidate['location']; add a review issue on a + contradiction. `fetch` is injectable for testing. Returns the same list.""" + url, api_key = load_config(url, api_key) + for c in cands: + if not c.get("date"): + c["location"] = "unknown" + continue + try: + d = datetime.strptime(c["date"][:10], "%Y-%m-%d") + except (ValueError, TypeError): + c["location"] = "unknown" + continue + start = (d - timedelta(days=WINDOW_DAYS)).strftime("%Y-%m-%d") + end = (d + timedelta(days=WINDOW_DAYS)).strftime("%Y-%m-%d") + try: + points = fetch(url, api_key, start, end) if url and api_key else [] + except Exception as exc: # noqa: BLE001 - a Dawarich hiccup must not break the run + c["location"] = "unknown" + c.setdefault("issues", []).append(f"Dawarich check failed: {exc}") + continue + loc = classify_location(c, points) + c["location"] = loc + if loc == "contradicted": + c.setdefault("issues", []).append( + f"Dawarich shows you elsewhere around {c['date']} " + f"(no points near {c.get('from_iata')}/{c.get('to_iata')}) — " + "possible cancellation/rebooking") + return cands diff --git a/populate.py b/populate.py index ff1d7c2..6d1317a 100644 --- a/populate.py +++ b/populate.py @@ -40,6 +40,7 @@ from collections import defaultdict import airtrail +import dawarich import flightparse import llm import notify as notify_mod @@ -151,11 +152,13 @@ def show(title, items): if not items: return print(f"--- {title} ({len(items)}) ---") + _loc = {"confirmed": " 📍ok", "contradicted": " 📍?!"} for c in items: seat = f" seat {c['seatNumber']}" if c.get("seatNumber") else "" + loc = _loc.get(c.get("location"), "") line = (f" {c['date']} {(c['flightNumber'] or '?'):8} " f"{c['from'] or c['from_iata']}->{c['to'] or c['to_iata']}" - f" {(c['departure'] or '')[:16]}{seat} [{c['extractor']}]") + f" {(c['departure'] or '')[:16]}{seat}{loc} [{c['extractor']}]") print(line) for issue in c.get("issues", []): print(f" ! {issue}") @@ -270,6 +273,11 @@ def main(): "miss (also via LLM_FALLBACK=1 + LLM_URL/LLM_MODEL)") ap.add_argument("--no-llm", dest="llm", action="store_false", help="force the LLM fallback off even if configured") + ap.add_argument("--validate", dest="validate", action="store_true", default=None, + help="cross-check candidates against Dawarich location history " + "(also via DAWARICH_VALIDATE=1 + DAWARICH_URL/API_KEY)") + ap.add_argument("--no-validate", dest="validate", action="store_false", + help="skip Dawarich validation even if configured") args = ap.parse_args() if args.pull: @@ -278,6 +286,8 @@ def main(): cands = parse_candidates(args.out_dir, use_llm=args.llm) existing, source_label = get_existing(args) classify(cands, existing) + if args.validate if args.validate is not None else dawarich.enabled(): + dawarich.validate(cands) rebookings = find_rebookings(cands) if args.out: diff --git a/tests/test_dawarich.py b/tests/test_dawarich.py new file mode 100644 index 0000000..90e700f --- /dev/null +++ b/tests/test_dawarich.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +""" +Unit tests for dawarich.py — Phase 3 location-history validation. + +No network: point-fetching is injected. The contract under test is the safety +logic — confirm on proximity, contradict ONLY when tracking was active and clearly +elsewhere, and treat any absence of data as "unknown" (never a false alarm). +""" + +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import airdata # noqa: E402 +import dawarich # noqa: E402 + + +def cand(**kw): + base = {"flightNumber": "LX1246", "date": "2022-03-17", + "from": "LSZH", "to": "ESSA", "from_iata": "ZRH", "to_iata": "ARN", + "issues": []} + base.update(kw) + return base + + +ZRH = airdata.airport_latlon("ZRH") # (47.46, 8.55) +ARN = airdata.airport_latlon("ARN") # (59.65, 17.92) + + +class TestGeo(unittest.TestCase): + def test_haversine_known_distance(self): + # ZRH -> ARN great circle is ~1490 km; allow a generous tolerance + self.assertAlmostEqual(dawarich.haversine_km(ZRH, ARN), 1490, delta=60) + + def test_coords_lookup_by_icao(self): + self.assertEqual(airdata.airport_latlon("LSZH"), ZRH) + + +class TestClassifyLocation(unittest.TestCase): + def test_confirmed_near_destination(self): + pts = [(59.60, 17.90)] # right by ARN + self.assertEqual(dawarich.classify_location(cand(), pts), "confirmed") + + def test_confirmed_near_origin(self): + pts = [(47.40, 8.60)] # by ZRH + self.assertEqual(dawarich.classify_location(cand(), pts), "confirmed") + + def test_contradicted_when_active_but_far(self): + pts = [(40.0, -3.5)] * 6 # Madrid, plenty of points + self.assertEqual(dawarich.classify_location(cand(), pts), "contradicted") + + def test_unknown_when_few_points(self): + pts = [(40.0, -3.5)] # far but only one point + self.assertEqual(dawarich.classify_location(cand(), pts), "unknown") + + def test_unknown_when_no_coords(self): + # airport not in the coordinate table -> can't tell -> unknown + c = cand(**{"from": None, "to": None, "from_iata": "XXX", "to_iata": "YYY"}) + self.assertEqual(dawarich.classify_location(c, [(0, 0)] * 10), "unknown") + + +class TestValidate(unittest.TestCase): + def test_contradiction_adds_issue(self): + cands = [cand()] + dawarich.validate(cands, url="http://d", api_key="k", + fetch=lambda u, k, s, e: [(40.0, -3.5)] * 6) + self.assertEqual(cands[0]["location"], "contradicted") + self.assertTrue(any("rebooking" in i.lower() for i in cands[0]["issues"])) + + def test_confirmed_no_issue(self): + cands = [cand()] + dawarich.validate(cands, url="http://d", api_key="k", + fetch=lambda u, k, s, e: [(59.6, 17.9)]) + self.assertEqual(cands[0]["location"], "confirmed") + self.assertEqual(cands[0]["issues"], []) + + def test_no_config_is_unknown(self): + cands = [cand()] + dawarich.validate(cands, url=None, api_key=None) + self.assertEqual(cands[0]["location"], "unknown") + + def test_fetch_failure_is_swallowed(self): + def boom(u, k, s, e): + raise RuntimeError("dawarich down") + cands = [cand()] + dawarich.validate(cands, url="http://d", api_key="k", fetch=boom) + self.assertEqual(cands[0]["location"], "unknown") # must not raise + + def test_bad_date_is_unknown(self): + cands = [cand(date=None)] + dawarich.validate(cands, url="http://d", api_key="k", + fetch=lambda *a: [(59.6, 17.9)]) + self.assertEqual(cands[0]["location"], "unknown") + + +class TestCoordParsing(unittest.TestCase): + def test_flat_latlon(self): + p = dawarich._coords([{"latitude": 47.4, "longitude": 8.5}]) + self.assertEqual(p, [(47.4, 8.5)]) + + def test_wrapped_and_geojson(self): + payload = {"data": [ + {"lat": 59.6, "lng": 17.9}, + {"geometry": {"coordinates": [8.5, 47.4]}}, # GeoJSON lon,lat + {"latitude": None, "longitude": None}, # dropped + ]} + self.assertEqual(dawarich._coords(payload), [(59.6, 17.9), (47.4, 8.5)]) + + +class TestEnabledGating(unittest.TestCase): + def setUp(self): + self._env = {k: os.environ.get(k) for k in + ("DAWARICH_VALIDATE", "DAWARICH_URL", "DAWARICH_API_KEY")} + + def tearDown(self): + for k, v in self._env.items(): + os.environ.pop(k, None) if v is None else os.environ.__setitem__(k, v) + + def test_off_by_default(self): + for k in ("DAWARICH_VALIDATE", "DAWARICH_URL", "DAWARICH_API_KEY"): + os.environ.pop(k, None) + self.assertFalse(dawarich.enabled()) + + def test_on_when_configured(self): + os.environ.update({"DAWARICH_VALIDATE": "1", "DAWARICH_URL": "http://d", + "DAWARICH_API_KEY": "k"}) + self.assertTrue(dawarich.enabled()) + + +if __name__ == "__main__": + unittest.main()