From b293d42d805dcc509aa780c325ed98726c8af6d2 Mon Sep 17 00:00:00 2001 From: Constantin Gemmingen <49587165+constgemm@users.noreply.github.com> Date: Tue, 7 Jul 2026 23:33:44 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20Phase=202=20=E2=80=94=20local-LLM=20fal?= =?UTF-8?q?lback=20for=20the=20long-tail=20parsers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds llm.py: an opt-in OpenAI-compatible fallback (Open WebUI) that parses the emails the deterministic templates miss (Ryanair/Wizz/LATAM/OTAs). It runs only on files the templates didn't match, and every candidate it returns is extractor="llm" + confidence="uncertain" by construction — so classify() routes it to the human-review bucket and _blocked_reason() keeps it out of --auto-write. An LLM guess never writes itself. - populate.parse_candidates gains a per-file fallback (env LLM_FALLBACK or --llm). - Dockerfile copies llm.py (populate imports it). - 15 tests: JSON/fence parsing, provenance+confidence contract, enable gating, and the populate integration (fallback used only when templates miss). Co-Authored-By: Claude Opus 4.8 --- .env.example | 10 +++ Dockerfile | 2 +- README.md | 8 +- llm.py | 193 ++++++++++++++++++++++++++++++++++++++++++++++ populate.py | 28 ++++++- tests/test_llm.py | 163 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 398 insertions(+), 6 deletions(-) create mode 100644 llm.py create mode 100644 tests/test_llm.py diff --git a/.env.example b/.env.example index 89f041d..5a69dea 100644 --- a/.env.example +++ b/.env.example @@ -24,3 +24,13 @@ RUN_ON_START=true # also run once when the container starts # Writes only high-confidence, non-rebooking NEW flights straight to AirTrail; # anything uncertain or flagged still waits for `populate.py --commit`. # AUTO_WRITE=false + +# --- local-LLM fallback (Phase 2, optional, opt-in) --- +# Parses the long tail (Ryanair/Wizz/LATAM/OTAs) the deterministic templates miss, +# via an OpenAI-compatible endpoint (e.g. Open WebUI). LLM candidates are ALWAYS +# uncertain — they surface for review and never auto-write. Off unless LLM_FALLBACK +# is truthy AND LLM_URL + LLM_MODEL are set. +# LLM_FALLBACK=false +# LLM_URL=http://airtrail-host:3000 +# LLM_MODEL=llama3.1:8b +# LLM_API_KEY=REPLACE_ME diff --git a/Dockerfile b/Dockerfile index bbdebad..6ceccb9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ ENV PYTHONUNBUFFERED=1 \ WORKDIR /app # Copy only the code (no secrets, no output — see .dockerignore). -COPY layover.py flightparse.py airdata.py airtrail.py notify.py populate.py scheduler.py ./ +COPY layover.py flightparse.py airdata.py airtrail.py llm.py notify.py populate.py scheduler.py ./ # Run unprivileged; /data (state.json watermark + candidates) is a persisted volume. RUN useradd --system --uid 10001 --home-dir /app layover \ diff --git a/README.md b/README.md index 82d8b80..6e6ce57 100644 --- a/README.md +++ b/README.md @@ -263,8 +263,12 @@ python3 -m unittest discover -s tests # parser contract, dedup, key normalisa Phase 1 (above) is deterministic templates + digest, no LLM in the loop. Where this is headed: -- **Phase 2 — local-LLM fallback** for the long tail (Ryanair, Wizz, LATAM, OTAs) whose emails - have no stable structured markup. Still behind the same human-confirm gate. +- **Phase 2 — local-LLM fallback** (shipped, opt-in) for the long tail (Ryanair, Wizz, LATAM, + OTAs) whose emails have no stable structured markup. It runs **only on emails the deterministic + templates miss**, calls an **OpenAI-compatible** endpoint (e.g. Open WebUI on the homelab box), + and asks for strict JSON. Every LLM candidate is marked `uncertain` on purpose, so it always lands + in the human-review bucket and can never reach the unattended `--auto-write` path. Enable with + `LLM_FALLBACK=1` + `LLM_URL`/`LLM_MODEL` (see `.env.example`) or `populate.py --llm`. - **Phase 3 — location-history validation:** cross-check candidates against [Dawarich](https://github.com/Freika/dawarich) point history to auto-confirm a flight or flag a cancelled/rebooked one (the signal a phone app gets, self-hosted). diff --git a/llm.py b/llm.py new file mode 100644 index 0000000..aad1783 --- /dev/null +++ b/llm.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +""" +llm — optional local-LLM fallback parser for the long tail (Phase 2). + +The deterministic extractors in flightparse.py cover the formats that dominate and +are stable (SWISS/Edelweiss JSON-LD, Lufthansa check-in, BA e-tickets). This module +picks up the *rest* — Ryanair, Wizz, LATAM, OTAs — whose emails have no reliable +structured markup, by asking a local **OpenAI-compatible** chat endpoint (Open WebUI +on the homelab box) to extract flights as strict JSON. + +Safety contract, on purpose and non-negotiable: + * every candidate this returns is `extractor="llm"` and `confidence="uncertain"`, + so classify() drops it in the human-reviewed bucket and _blocked_reason() keeps + it out of the unattended --auto-write path. An LLM guess never writes itself. + * it only runs on emails the deterministic pass missed (see populate.py), so a + normal weekly run makes zero LLM calls unless there's genuinely new long-tail mail. + * disabled unless configured — no endpoint, no calls. + +Config (env, or the matching populate.py flags): + LLM_URL base URL of the OpenAI-compatible API (e.g. http://vmgpu:3000) + LLM_MODEL model name to request + LLM_API_KEY Bearer key (Open WebUI issues one; optional for keyless servers) + LLM_FALLBACK set truthy (1/true/yes/on) to enable the fallback + +Zero dependencies (urllib + json). +""" + +import json +import os +import re +import urllib.request + +import airdata +import flightparse + +# The schema we force the model to emit. Kept tiny and flat so a small local model +# can hit it reliably; anything richer (seat class, terminals) is best-effort. +SYSTEM_PROMPT = ( + "You extract flight bookings from the text of a single airline or travel-agency " + "email. Return ONLY a JSON array (no prose, no code fence). Each element is one " + "flown/booked flight leg with these keys:\n" + ' airline_iata two-char IATA airline code (e.g. "FR", "W6", "LA") or null\n' + ' flight_number digits only, no airline prefix (e.g. "1834") or null\n' + ' from_iata 3-letter IATA departure airport or null\n' + ' to_iata 3-letter IATA arrival airport or null\n' + ' date departure date as YYYY-MM-DD or null\n' + ' departure local departure timestamp YYYY-MM-DDTHH:MM or null\n' + ' arrival local arrival timestamp YYYY-MM-DDTHH:MM or null\n' + ' seat_number e.g. "14C" or null\n' + ' status "Confirmed" / "Cancelled" / null\n' + "Rules: one element per leg (a round trip is two). Use IATA codes, never names. " + "If the email is not a flight booking, or you are unsure, return []. Never invent " + "a flight number, airport, or date — use null for anything not clearly stated." +) + + +# --------------------------------------------------------------------------- +# config +# --------------------------------------------------------------------------- + +def load_config(url=None, model=None, api_key=None): + url = url or os.environ.get("LLM_URL") + model = model or os.environ.get("LLM_MODEL") + api_key = api_key or os.environ.get("LLM_API_KEY") + return url, model, api_key + + +def enabled(url=None, model=None): + """Fallback runs only when explicitly switched on AND an endpoint+model exist.""" + on = os.environ.get("LLM_FALLBACK", "").strip().lower() in ("1", "true", "yes", "on") + u, m, _ = load_config(url, model) + return bool(on and u and m) + + +# --------------------------------------------------------------------------- +# transport (OpenAI-compatible /v1/chat/completions) +# --------------------------------------------------------------------------- + +def chat(url, model, api_key, system, user, timeout=60): + """POST one chat completion, return the assistant message content (str).""" + endpoint = url.rstrip("/") + "/v1/chat/completions" + payload = { + "model": model, + "temperature": 0, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + } + headers = {"Content-Type": "application/json", "Accept": "application/json"} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + req = urllib.request.Request(endpoint, data=json.dumps(payload).encode(), + method="POST", headers=headers) + with urllib.request.urlopen(req, timeout=timeout) as resp: + data = json.loads(resp.read().decode()) + return data["choices"][0]["message"]["content"] + + +# --------------------------------------------------------------------------- +# response parsing +# --------------------------------------------------------------------------- + +def parse_json_flights(text): + """Pull the JSON array out of a model reply, tolerating ```json fences and + stray prose around it. Returns a list (possibly empty); never raises.""" + if not text: + return [] + # strip a ```json ... ``` (or ``` ... ```) fence if present + fenced = re.search(r"```(?:json)?\s*(.*?)```", text, re.S) + body = fenced.group(1) if fenced else text + try: + obj = json.loads(body) + except json.JSONDecodeError: + # last resort: grab the first [...] span + m = re.search(r"\[.*\]", body, re.S) + if not m: + return [] + try: + obj = json.loads(m.group(0)) + except json.JSONDecodeError: + return [] + if isinstance(obj, dict): # a lone object -> wrap it + obj = [obj] + return [x for x in obj if isinstance(x, dict)] if isinstance(obj, list) else [] + + +def to_candidate(obj, source_file): + """Map one LLM-extracted flight dict to the canonical candidate dict shape + (same keys as flightparse.Candidate.to_dict), forced to LLM provenance and + uncertain confidence so it can never auto-write.""" + airline_iata = (obj.get("airline_iata") or "").strip().upper() or None + number = obj.get("flight_number") + flight_number = None + if airline_iata and number not in (None, ""): + flight_number = flightparse.norm_flight_number(airline_iata, number) + from_iata = (obj.get("from_iata") or "").strip().upper() or None + to_iata = (obj.get("to_iata") or "").strip().upper() or None + seat = obj.get("seat_number") + seat = seat.strip().upper() if isinstance(seat, str) and seat.strip() else None + date = (obj.get("date") or "") or None + + issues = ["LLM-parsed — verify before writing"] + status = obj.get("status") + if (status or "").lower() in ("cancelled", "canceled", "storniert"): + issues.append(f"reservation status {status!r} — verify not a ghost") + + return { + "flightNumber": flight_number, + "airline": airdata.airline_icao(airline_iata), + "airline_iata": airline_iata, + "from": airdata.airport_icao(from_iata), + "to": airdata.airport_icao(to_iata), + "from_iata": from_iata, + "to_iata": to_iata, + "date": date[:10] if date else None, + "departure": obj.get("departure") or None, + "arrival": obj.get("arrival") or None, + "seatNumber": seat, + "seat": flightparse.seat_type(seat), + "seatClass": flightparse.seat_class(obj.get("seat_class")), + "flightReason": "leisure", + "source_file": source_file, + "extractor": "llm", + "confidence": "uncertain", # never "high": keeps it out of auto-write + "issues": issues, + } + + +# --------------------------------------------------------------------------- +# public entry points +# --------------------------------------------------------------------------- + +def extract_flights(body, source_file, url=None, model=None, api_key=None): + """Ask the model for flights in `body`; return candidate dicts (may be empty).""" + url, model, api_key = load_config(url, model, api_key) + if not (url and model): + return [] + reply = chat(url, model, api_key, SYSTEM_PROMPT, body[:12000]) + cands = [] + for obj in parse_json_flights(reply): + cand = to_candidate(obj, source_file) + # require at least a route or a flight number to be worth a human's time + if cand["flightNumber"] or (cand["from_iata"] and cand["to_iata"]): + cands.append(cand) + return cands + + +def extract_file(path, url=None, model=None, api_key=None): + """Load a Layover-saved .txt and run the fallback on its body.""" + _, body = flightparse.load_email(path) + rel = os.path.basename(os.path.dirname(path)) + "/" + os.path.basename(path) + return extract_flights(body, rel, url, model, api_key) diff --git a/populate.py b/populate.py index b1cb839..ff1d7c2 100644 --- a/populate.py +++ b/populate.py @@ -41,6 +41,7 @@ import airtrail import flightparse +import llm import notify as notify_mod @@ -49,13 +50,29 @@ def pull(accounts_ini, out_dir): subprocess.run([sys.executable, "layover.py", accounts_ini, out_dir], check=True) -def parse_candidates(out_dir): +def parse_candidates(out_dir, use_llm=None): + """Deterministic parse of every saved email; for the ones the templates miss, + fall back to the local LLM (Phase 2) when it's enabled. LLM candidates are + always uncertain — they surface for review, never auto-write. + + use_llm: None -> honour env (llm.enabled()); True/False -> force on/off.""" + llm_on = llm.enabled() if use_llm is None else use_llm raw = [] for path in flightparse.iter_paths([out_dir]): try: - raw += [c.to_dict() for c in flightparse.parse_file(path)] + file_cands = [c.to_dict() for c in flightparse.parse_file(path)] except Exception as exc: # noqa: BLE001 print(f" !! {path}: {exc}", file=sys.stderr) + file_cands = [] + if not file_cands and llm_on: + try: + file_cands = llm.extract_file(path) + if file_cands: + print(f" [llm] {path}: {len(file_cands)} candidate(s)", + file=sys.stderr) + except Exception as exc: # noqa: BLE001 - LLM problems must not abort the run + print(f" !! llm {path}: {exc}", file=sys.stderr) + raw += file_cands cands = flightparse.dedup_candidates(raw) cands.sort(key=lambda d: (d.get("date") or "", d.get("flightNumber") or "")) return cands @@ -248,12 +265,17 @@ def main(): ap.add_argument("--webhook-url", help="POST a JSON digest here when new flights " "are found (or set WEBHOOK_URL)") ap.add_argument("--user-id", help="AirTrail user id to assign seats to on writes") + ap.add_argument("--llm", dest="llm", action="store_true", default=None, + help="force the local-LLM fallback on for emails the templates " + "miss (also via LLM_FALLBACK=1 + LLM_URL/LLM_MODEL)") + ap.add_argument("--no-llm", dest="llm", action="store_false", + help="force the LLM fallback off even if configured") args = ap.parse_args() if args.pull: pull(args.pull, args.out_dir) - cands = parse_candidates(args.out_dir) + cands = parse_candidates(args.out_dir, use_llm=args.llm) existing, source_label = get_existing(args) classify(cands, existing) rebookings = find_rebookings(cands) diff --git a/tests/test_llm.py b/tests/test_llm.py new file mode 100644 index 0000000..66cadab --- /dev/null +++ b/tests/test_llm.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +""" +Unit tests for llm.py — the Phase 2 local-LLM fallback parser. + +No real network and no real model: llm.chat is monkeypatched to return canned +replies. The point of these tests is the *contract*, not model quality: + * replies parse out of prose / code fences, + * candidates carry LLM provenance and uncertain confidence (so they can never + reach the unattended auto-write path), + * the fallback stays off unless explicitly enabled. +""" + +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import llm # noqa: E402 +import populate # noqa: E402 + + +class TestParseJsonFlights(unittest.TestCase): + def test_plain_array(self): + self.assertEqual(llm.parse_json_flights('[{"a": 1}]'), [{"a": 1}]) + + def test_code_fence(self): + text = 'Sure!\n```json\n[{"flight_number": "1834"}]\n```\nHope that helps.' + self.assertEqual(llm.parse_json_flights(text), [{"flight_number": "1834"}]) + + def test_prose_wrapped_array(self): + text = 'Here is the data: [{"x": 2}] and nothing else.' + self.assertEqual(llm.parse_json_flights(text), [{"x": 2}]) + + def test_lone_object_wrapped(self): + self.assertEqual(llm.parse_json_flights('{"x": 1}'), [{"x": 1}]) + + def test_garbage_is_empty(self): + self.assertEqual(llm.parse_json_flights("no json here"), []) + self.assertEqual(llm.parse_json_flights(""), []) + + +class TestToCandidate(unittest.TestCase): + def test_maps_and_resolves_icao(self): + obj = {"airline_iata": "fr", "flight_number": "1834", + "from_iata": "STN", "to_iata": "DUB", "date": "2019-06-01", + "departure": "2019-06-01T07:10", "seat_number": "12a"} + c = llm.to_candidate(obj, "acct/x.txt") + self.assertEqual(c["flightNumber"], "FR1834") + self.assertEqual(c["airline"], "RYR") # IATA FR -> ICAO + self.assertEqual(c["from"], "EGSS") # STN -> ICAO + self.assertEqual(c["to"], "EIDW") # DUB -> ICAO + self.assertEqual(c["seatNumber"], "12A") + self.assertEqual(c["seat"], "window") # A -> window + self.assertEqual(c["extractor"], "llm") + + def test_confidence_is_never_high(self): + obj = {"airline_iata": "FR", "flight_number": "1", "from_iata": "STN", + "to_iata": "DUB", "date": "2019-06-01"} + c = llm.to_candidate(obj, "x") + self.assertNotEqual(c["confidence"], "high") + self.assertTrue(any("verify" in i.lower() for i in c["issues"])) + + def test_cancelled_flagged(self): + c = llm.to_candidate({"airline_iata": "FR", "flight_number": "1", + "from_iata": "STN", "to_iata": "DUB", + "date": "2019-06-01", "status": "Cancelled"}, "x") + self.assertTrue(any("ghost" in i.lower() for i in c["issues"])) + + +class TestExtractFlights(unittest.TestCase): + def setUp(self): + self._orig = llm.chat + llm.chat = lambda url, model, key, sys_, usr, timeout=60: ( + '[{"airline_iata":"W6","flight_number":"2310","from_iata":"LTN",' + '"to_iata":"OTP","date":"2021-08-14","departure":"2021-08-14T06:00"}]') + + def tearDown(self): + llm.chat = self._orig + + def test_returns_uncertain_candidate(self): + cands = llm.extract_flights("some ryanair-ish email body", "acct/e.txt", + url="http://x:3000", model="m") + self.assertEqual(len(cands), 1) + self.assertEqual(cands[0]["flightNumber"], "W62310") + self.assertEqual(cands[0]["confidence"], "uncertain") + self.assertEqual(cands[0]["extractor"], "llm") + + def test_no_config_is_noop(self): + # no url/model -> no call, empty result (env not set in test) + self.assertEqual(llm.extract_flights("body", "s", url=None, model=None), []) + + def test_drops_contentless_rows(self): + llm.chat = lambda *a, **k: '[{"seat_number": "1A"}]' # no route, no number + self.assertEqual( + llm.extract_flights("b", "s", url="http://x", model="m"), []) + + +class TestEnabledGating(unittest.TestCase): + def setUp(self): + self._env = {k: os.environ.get(k) for k in + ("LLM_FALLBACK", "LLM_URL", "LLM_MODEL")} + + def tearDown(self): + for k, v in self._env.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + def test_off_by_default(self): + for k in ("LLM_FALLBACK", "LLM_URL", "LLM_MODEL"): + os.environ.pop(k, None) + self.assertFalse(llm.enabled()) + + def test_needs_flag_and_endpoint(self): + os.environ["LLM_FALLBACK"] = "1" + os.environ.pop("LLM_URL", None) + self.assertFalse(llm.enabled()) # flag but no endpoint + os.environ["LLM_URL"] = "http://x:3000" + os.environ["LLM_MODEL"] = "m" + self.assertTrue(llm.enabled()) + + +class TestPopulateIntegration(unittest.TestCase): + """parse_candidates should call the fallback only for emails the deterministic + extractors miss, and surface the result as an uncertain candidate.""" + + def setUp(self): + import tempfile + self.tmp = tempfile.mkdtemp() + acct = os.path.join(self.tmp, "acct") + os.makedirs(acct) + # an email the three templates don't recognise + with open(os.path.join(acct, "1_ryanair.txt"), "w") as fh: + fh.write("From: noreply@ryanair.com\nSubject: Your trip\n\n" + "Thanks for booking. STN to DUB on 1 June.\n") + self._orig = llm.extract_file + llm.extract_file = lambda path, *a, **k: [llm.to_candidate( + {"airline_iata": "FR", "flight_number": "1834", "from_iata": "STN", + "to_iata": "DUB", "date": "2019-06-01"}, "acct/1_ryanair.txt")] + + def tearDown(self): + llm.extract_file = self._orig + import shutil + shutil.rmtree(self.tmp, ignore_errors=True) + + def test_fallback_used_when_templates_miss(self): + cands = populate.parse_candidates(self.tmp, use_llm=True) + self.assertEqual(len(cands), 1) + self.assertEqual(cands[0]["extractor"], "llm") + # and it classifies as uncertain (needs a human), never new/auto-writable + populate.classify(cands, existing=set()) + self.assertEqual(cands[0]["status"], "uncertain") + self.assertEqual(populate.auto_writable(cands), []) + + def test_fallback_skipped_when_off(self): + cands = populate.parse_candidates(self.tmp, use_llm=False) + self.assertEqual(cands, []) + + +if __name__ == "__main__": + unittest.main()