From 3cae48a61eaf03737cc0563ed794662d2bb06cbe Mon Sep 17 00:00:00 2001 From: raulkolaric Date: Fri, 7 Aug 2026 11:23:38 -0300 Subject: [PATCH 1/4] feat: generalize student timetable parsing - grid.py: parse and validate student-specific timetable slots and semester bounds\n- llm.py: reconcile extracted subjects against the parsed timetable\n- dates.py: require explicit resolution for unmatched subjects and times\n- output.py: remove the hard-coded campus location from ICS output\n- main.py: add interactive subject/time selection and user-facing failures\n- docs and tests: document the dynamic workflow and cover generalized behavior --- .env.example | 11 +- README.md | 40 +++- files/README.md | 4 +- src/dates.py | 104 +++++---- src/grid.py | 220 ++++++++++-------- src/llm.py | 396 +++++++++++++++----------------- src/main.py | 130 ++++++++++- src/output.py | 10 +- src/test.py | 37 ++- tests/test_academic_calendar.py | 4 +- tests/test_generalization.py | 121 ++++++++++ 11 files changed, 686 insertions(+), 391 deletions(-) create mode 100644 tests/test_generalization.py diff --git a/.env.example b/.env.example index c6edfd0..bdc9f6d 100644 --- a/.env.example +++ b/.env.example @@ -1,12 +1,11 @@ -# OpenAI-compatible API configuration. -# Recommended: Alibaba DashScope (Qwen) - one key covers vision + text. +# OpenAI-compatible API configuration. Both models are required, and the +# vision model must accept image inputs. +# +# Example: Alibaba DashScope (Qwen) - one key covers vision + text. # BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 # MODEL_TEXT=qwen-turbo # MODEL_VISION=qwen-vl-plus -# Alternative for text: DeepSeek (no vision support) -# BASE_URL=https://api.deepseek.com -# MODEL_TEXT=deepseek-chat API_KEY= BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 MODEL_TEXT=qwen-turbo -MODEL_VISION=qwen-vl-plus \ No newline at end of file +MODEL_VISION=qwen-vl-plus diff --git a/README.md b/README.md index 42a8537..4294bec 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,10 @@ Turn Brazilian university teaching plans into calendar events. -Give it your teaching-plan PDFs and weekly timetable. It finds tests and -activities, assigns dates and class times, then writes files you can import -into Google Calendar, Apple Calendar, or Outlook. +Give it your teaching-plan PDFs and weekly timetable. It reads that student's +subjects, semester dates, weekdays, and class times; finds tests and activities; +then writes files that can be imported into Google Calendar, Apple Calendar, +or Outlook. ## Quick start @@ -28,7 +29,8 @@ The terminal guides you through the rest. Put these in `files/`: -- `image.png` — your weekly timetable screenshot. +- `image.png` — your weekly timetable screenshot. It must show the semester + start/end dates, subjects, weekdays, and class times. - One PDF teaching plan for each subject. - Optional: an academic calendar as PDF, PNG, JPEG, or WebP. @@ -42,9 +44,8 @@ The optional calendar keeps estimated dates from drifting after holidays, recesses, or cancelled class days. When a calendar has different rules for different campuses, the terminal asks -for your campus and, if useful, your unit, course, and shift. A Sorocaba-only -recess is excluded for a São Paulo student. If a row is unclear, the tool asks -you instead of guessing. +for your campus and, if useful, your unit, course, and shift. If a row is +unclear or belongs to a different campus, the tool asks instead of guessing. The calendar may be normal text, scanned pages, or a mixture of both. Its page results are saved under `output/academic-calendar-cache/`, so rerunning the @@ -66,10 +67,25 @@ MODEL_TEXT=qwen-turbo MODEL_VISION=qwen-vl-plus ``` -The defaults work with Alibaba DashScope's Qwen models. Any compatible provider -with a text model and an image-capable model should work. Your key is read from +The example values use Alibaba DashScope's Qwen models. They are examples, not +university requirements. Another OpenAI-compatible provider works when it has +both a text model and a model that accepts image inputs. Your key is read from `.env` and is not printed or saved in output files. +## How student-specific data is handled + +No semester, subject, campus, or class time is built into the program. Every +run reads those values from the student's timetable and teaching plans. + +If the timetable image is incomplete or cannot be parsed, the run stops and +explains what is missing. If a teaching-plan subject or event time cannot be +matched safely, the terminal asks the student to select the correct timetable +entry. Pressing Enter aborts the run, and no new CSV or ICS file is written. + +Teaching-plan extraction is designed first for Brazilian Portuguese. It +recognizes common assessment terms such as prova, avaliação, atividade, +trabalho, projeto, seminário, and substitutiva. + ## Output After a successful run, look in `output/`: @@ -80,8 +96,10 @@ After a successful run, look in `output/`: - `academic-calendar-review.json` — present only when you use an academic calendar. Regular classes are not exported. The output contains tests and activities -found in the teaching plans. Dates written directly in a plan are used when -they look valid; other dates are estimated from the class number and timetable. +found in the teaching plans. Literal dates in a plan are used directly; other +dates are estimated from the class number, timetable, and accepted academic +calendar closures. Events without a trustworthy date or time are not silently +given defaults. ## Project layout diff --git a/files/README.md b/files/README.md index 5c79119..f12b9a4 100644 --- a/files/README.md +++ b/files/README.md @@ -1,6 +1,8 @@ # Input Files -Place teaching-plan PDFs and `image.png` (the weekly timetable) here. +Place teaching-plan PDFs and `image.png` (the weekly timetable) here. The +timetable image must show semester start/end dates, subjects, weekdays, and +class times. The program does not contain a fallback schedule. You may also add one optional academic calendar as PDF, PNG, JPEG, or WebP. Text-readable, scanned, and mixed PDFs are supported. A filename containing diff --git a/src/dates.py b/src/dates.py index b292f12..e2bfb6a 100644 --- a/src/dates.py +++ b/src/dates.py @@ -4,11 +4,18 @@ import unicodedata from dataclasses import dataclass from datetime import date, time, timedelta +from typing import Callable from .grid import Slot from .llm import Fact -DEFAULT_SLOT = (time(7, 15), time(8, 55)) + +class EventResolutionError(ValueError): + """An event is missing information required for a trustworthy calendar entry.""" + + +SubjectResolver = Callable[[Fact, list[Slot]], list[Slot] | None] +SlotResolver = Callable[[Fact, list[Slot], date], Slot | None] @dataclass @@ -34,14 +41,19 @@ def normalize(s: str) -> str: def _subject_slots(code: str, name: str, slots: list[Slot]) -> list[Slot]: code = code.strip().upper() - by_code = [s for s in slots if s.code == code] - if by_code: - return sorted(by_code, key=lambda s: s.weekday) + if code: + by_code = [s for s in slots if s.code == code] + if by_code: + return sorted(by_code, key=lambda s: s.weekday) nname = normalize(name) if not nname: return [] - by_name = [s for s in slots if nname in normalize(s.name) or normalize(s.name) in nname] - return sorted(by_name, key=lambda s: s.weekday) + exact_name = [s for s in slots if normalize(s.name) == nname] + if exact_name: + return sorted(exact_name, key=lambda s: s.weekday) + partial = [s for s in slots if nname in normalize(s.name) or normalize(s.name) in nname] + identities = {(slot.code, normalize(slot.name)) for slot in partial} + return sorted(partial, key=lambda s: s.weekday) if len(identities) == 1 else [] def nth_meeting( @@ -71,67 +83,71 @@ def compute_events( facts: list[Fact], slots: list[Slot], semester_start: date, + semester_end: date | None = None, non_teaching_dates: set[date] | None = None, + subject_resolver: SubjectResolver | None = None, + slot_resolver: SlotResolver | None = None, ) -> list[Event]: print(f"[dates] computing events from {len(facts)} fact(s), {len(slots)} grid slot(s), semester start {semester_start}") events = [] non_teaching_dates = non_teaching_dates or set() for fact in facts: subj_slots = _subject_slots(fact.subject_code, fact.subject_name, slots) + if not subj_slots and subject_resolver is not None: + subj_slots = subject_resolver(fact, slots) or [] + if not subj_slots: + raise EventResolutionError( + f"Could not match subject {fact.subject_code or fact.subject_name!r} " + f"from {fact.source_pdf!r} to the timetable." + ) print(f"[dates] fact: {fact.title} | code={fact.subject_code} | matched {len(subj_slots)} slot(s)") # --- date --- if fact.explicit_date: - # Validate: if the explicit date's weekday doesn't match any grid slot - # for this subject, the date was likely hallucinated. Fall back to class_number. - subj_weekdays = {s.weekday for s in subj_slots} - if subj_weekdays and fact.explicit_date.weekday() not in subj_weekdays: - print(f"[dates] explicit date {fact.explicit_date} (weekday={fact.explicit_date.weekday()}) not in subject weekdays {subj_weekdays} — likely hallucinated, falling back to class_number") - if fact.class_number: - weekdays = sorted(subj_weekdays) - day, estimated = nth_meeting(weekdays, semester_start, fact.class_number, excluded_dates=non_teaching_dates), True - if day: - print(f"[dates] class #{fact.class_number} on weekdays {weekdays} → {day} (estimated)") - else: - print(f"[dates] class #{fact.class_number} on weekdays {weekdays} → could not compute date") - continue - else: - print(f"[dates] no class_number to fall back to — skipping") - continue - else: - day, estimated = fact.explicit_date, False - print(f"[dates] using explicit date: {day}") + day, estimated = fact.explicit_date, False + print(f"[dates] using explicit date: {day}") elif fact.class_number: weekdays = sorted({s.weekday for s in subj_slots}) - day, estimated = nth_meeting(weekdays, semester_start, fact.class_number, excluded_dates=non_teaching_dates), True + day, estimated = nth_meeting( + weekdays, + semester_start, + fact.class_number, + limit=semester_end, + excluded_dates=non_teaching_dates, + ), True if day: print(f"[dates] class #{fact.class_number} on weekdays {weekdays} → {day} (estimated)") else: - print(f"[dates] class #{fact.class_number} on weekdays {weekdays} → could not compute date") + raise EventResolutionError( + f"Could not place class #{fact.class_number} for {fact.title!r} " + "inside the active semester." + ) else: - print(f"[dates] skip (no date info)") - continue + raise EventResolutionError( + f"Event {fact.title!r} from {fact.source_pdf!r} has neither a " + "literal date nor a class number." + ) if day is None: - print(f"[dates] skip (uncomputable date)") - continue + raise EventResolutionError(f"Could not compute a date for {fact.title!r}.") # --- hours --- note = "" if day in non_teaching_dates: note = "explicit date falls on an accepted non-teaching day; review calendar conflict" - slot = next((s for s in subj_slots if s.weekday == day.weekday()), None) - if slot is None: - slot = subj_slots[0] if subj_slots else None + same_day_slots = [slot for slot in subj_slots if slot.weekday == day.weekday()] + slot = same_day_slots[0] if len(same_day_slots) == 1 else None + if slot is None and slot_resolver is not None: + slot = slot_resolver(fact, same_day_slots or subj_slots, day) if slot is not None: - note = _append_note(note, "weekday not in grid; used subject's regular slot") - if slot is not None: - start_t, end_t = slot.start, slot.end - print(f"[dates] slot: {slot.code} {slot.start}-{slot.end} (weekday={slot.weekday})") - else: - start_t, end_t = DEFAULT_SLOT - note = _append_note(note, "subject not in grid; used default slot 07:15-08:55") - print(f"[dates] no slot found; using default {DEFAULT_SLOT[0]}-{DEFAULT_SLOT[1]}") + note = _append_note(note, "timetable slot selected by user") + if slot is None: + reason = "multiple timetable slots match" if same_day_slots else "no timetable slot matches the event weekday" + raise EventResolutionError( + f"Could not assign a time to {fact.title!r}: {reason}." + ) + start_t, end_t = slot.start, slot.end + print(f"[dates] slot: {slot.code} {slot.start}-{slot.end} (weekday={slot.weekday})") events.append(Event( - subject_code=fact.subject_code or (slot.code if slot else ""), - subject_name=fact.subject_name or (slot.name if slot else ""), + subject_code=slot.code or fact.subject_code, + subject_name=slot.name or fact.subject_name, title=fact.title, kind=fact.kind, date=day, diff --git a/src/grid.py b/src/grid.py index 032aba1..9ff19d1 100644 --- a/src/grid.py +++ b/src/grid.py @@ -1,28 +1,44 @@ -"""Weekly class grid. - -Parses the timetable image with a vision LLM when a client is available, -otherwise falls back to the hardcoded grid extracted from files/image.png. -""" +"""Parse a student's weekly timetable into validated class slots.""" import base64 import json +import mimetypes import re from dataclasses import dataclass from datetime import date, time +from pathlib import Path -# Semester range shown in the timetable image. -SEMESTER_START = date(2026, 8, 3) -SEMESTER_END = date(2026, 12, 12) WEEKDAYS = { - "monday": 0, "segunda": 0, - "tuesday": 1, "terça": 1, "terca": 1, - "wednesday": 2, "quarta": 2, - "thursday": 3, "quinta": 3, - "friday": 4, "sexta": 4, + "monday": 0, + "segunda": 0, + "segunda-feira": 0, + "tuesday": 1, + "terça": 1, + "terca": 1, + "terça-feira": 1, + "terca-feira": 1, + "wednesday": 2, + "quarta": 2, + "quarta-feira": 2, + "thursday": 3, + "quinta": 3, + "quinta-feira": 3, + "friday": 4, + "sexta": 4, + "sexta-feira": 4, + "saturday": 5, + "sábado": 5, + "sabado": 5, + "sunday": 6, + "domingo": 6, } +class GridParseError(ValueError): + """The supplied timetable could not be converted into a usable grid.""" + + @dataclass class Slot: weekday: int # 0 = Monday ... 6 = Sunday @@ -32,95 +48,115 @@ class Slot: end: time -def _t(hhmm: str) -> time: - hh, mm = hhmm.split(":") - return time(int(hh), int(mm)) - - -# Fallback grid, transcribed from files/image.png. -HARDCODED_SLOTS = [ - Slot(0, "CGPI", "COMPUTAÇÃO GRÁFICA E PROCESSAMENTO DE IMAGENS", _t("07:15"), _t("08:55")), - Slot(0, "TNC", "TEORIA DOS NÚMEROS E CRIPTOGRAFIA", _t("09:05"), _t("10:45")), - Slot(1, "CGPI", "COMPUTAÇÃO GRÁFICA E PROCESSAMENTO DE IMAGENS (PRÁTICA)", _t("07:15"), _t("08:55")), - Slot(1, "LMA", "LABORATÓRIO DE MODELAGEM ALGORÍTMICA", _t("09:05"), _t("10:45")), - Slot(1, "EDNL", "ESTRUTURAS DE DADOS - NÃO LINEARES", _t("10:55"), _t("12:35")), - Slot(2, "OP", "ORIENTAÇÃO PROFISSIONAL", _t("07:15"), _t("08:55")), - Slot(2, "TA", "TEORIA DE AUTÔMATOS", _t("09:05"), _t("10:45")), - Slot(2, "PE", "PROBABILIDADE E ESTATÍSTICA", _t("10:55"), _t("12:35")), - Slot(3, "EDNL", "ESTRUTURAS DE DADOS - NÃO LINEARES", _t("07:15"), _t("08:55")), - Slot(3, "PE", "PROBABILIDADE E ESTATÍSTICA", _t("09:05"), _t("10:45")), - Slot(4, "TA", "TEORIA DE AUTÔMATOS", _t("07:15"), _t("08:55")), - Slot(4, "TNC", "TEORIA DOS NÚMEROS E CRIPTOGRAFIA", _t("09:05"), _t("10:45")), -] - -_VISION_PROMPT = """You are reading a weekly class timetable grid (Portuguese). +_VISION_PROMPT = """Read this university timetable image. It may be in Portuguese. Return ONLY valid JSON with this exact shape: {"semester_start": "YYYY-MM-DD", "semester_end": "YYYY-MM-DD", - "slots": [{"weekday": "monday"|"tuesday"|"wednesday"|"thursday"|"friday", - "code": "short code e.g. EDNL", - "name": "FULL SUBJECT NAME", - "start": "HH:MM", "end": "HH:MM"}]} -Merge consecutive time rows belonging to the same subject into one slot. -No markdown, no commentary.""" + "slots": [{"weekday": "monday", "code": "subject code or empty string", + "name": "full subject name", "start": "HH:MM", "end": "HH:MM"}]} + +Rules: +- Read the semester start and end dates printed in the image. Do not guess them. +- Include every class slot, including evening and weekend classes. +- Use lowercase English weekday names from monday through sunday. +- Preserve the subject code when one is printed. Use an empty string when none is shown. +- Merge adjacent time rows only when they belong to the same subject and class session. +- Use 24-hour HH:MM times. +- Do not include breaks, headers, or empty cells as slots. +No markdown and no commentary.""" + + +def _strip_fences(raw: str) -> str: + raw = raw.strip() + if raw.startswith("```"): + raw = re.sub(r"^```[a-zA-Z0-9_-]*\n?", "", raw) + raw = re.sub(r"\n?```$", "", raw) + return raw.strip() def _parse_hhmm(value: str) -> time: - m = re.match(r"(\d{1,2}):(\d{2})", value or "") - if not m: - raise ValueError(f"bad time: {value!r}") - return time(int(m.group(1)), int(m.group(2))) - - -def _slots_from_json(payload: dict) -> list[Slot]: - slots = [] - for item in payload["slots"]: - weekday = WEEKDAYS[str(item["weekday"]).strip().lower()] - slots.append(Slot( - weekday=weekday, - code=str(item["code"]).strip().upper(), - name=str(item["name"]).strip().upper(), - start=_parse_hhmm(item["start"]), - end=_parse_hhmm(item["end"]), - )) - if not slots: - raise ValueError("vision returned no slots") - return slots + match = re.fullmatch(r"\s*(\d{1,2}):(\d{2})\s*", value or "") + if not match: + raise GridParseError(f"invalid timetable time: {value!r}") + try: + return time(int(match.group(1)), int(match.group(2))) + except ValueError as exc: + raise GridParseError(f"invalid timetable time: {value!r}") from exc + + +def _parse_date(value, field: str) -> date: + try: + return date.fromisoformat(str(value)) + except (TypeError, ValueError) as exc: + raise GridParseError(f"missing or invalid {field}; expected YYYY-MM-DD") from exc + + +def _grid_from_json(payload: dict) -> tuple[list[Slot], date, date]: + if not isinstance(payload, dict): + raise GridParseError("timetable response was not a JSON object") + + semester_start = _parse_date(payload.get("semester_start"), "semester_start") + semester_end = _parse_date(payload.get("semester_end"), "semester_end") + if semester_end < semester_start: + raise GridParseError("semester_end is earlier than semester_start") + + raw_slots = payload.get("slots") + if not isinstance(raw_slots, list) or not raw_slots: + raise GridParseError("timetable response contained no class slots") + + slots: list[Slot] = [] + for index, item in enumerate(raw_slots, start=1): + if not isinstance(item, dict): + raise GridParseError(f"timetable slot {index} was not an object") + weekday_key = str(item.get("weekday", "")).strip().casefold() + if weekday_key not in WEEKDAYS: + raise GridParseError(f"invalid weekday in timetable slot {index}: {weekday_key!r}") + code = str(item.get("code", "")).strip().upper() + name = str(item.get("name", "")).strip().upper() + if not code and not name: + raise GridParseError(f"timetable slot {index} has no subject code or name") + start = _parse_hhmm(str(item.get("start", ""))) + end = _parse_hhmm(str(item.get("end", ""))) + if end <= start: + raise GridParseError(f"timetable slot {index} ends before it starts") + slots.append(Slot(WEEKDAYS[weekday_key], code, name, start, end)) + + return slots, semester_start, semester_end def parse_with_vision(image_path: str, client, model: str) -> tuple[list[Slot], date, date]: - print(f"[grid] reading image: {image_path}") - with open(image_path, "rb") as fh: - raw_bytes = fh.read() - print(f"[grid] image size: {len(raw_bytes):,} bytes, encoding to base64...") - b64 = base64.b64encode(raw_bytes).decode() - print(f"[grid] base64 length: {len(b64):,} chars, sending to vision model '{model}'...") - resp = client.chat.completions.create( + source = Path(image_path) + print(f"[grid] reading timetable image: {source}") + raw_bytes = source.read_bytes() + if not raw_bytes: + raise GridParseError("the timetable image is empty") + + mime = mimetypes.guess_type(source.name)[0] or "image/png" + encoded = base64.b64encode(raw_bytes).decode("ascii") + response = client.chat.completions.create( model=model, messages=[{"role": "user", "content": [ {"type": "text", "text": _VISION_PROMPT}, - {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}}, + {"type": "image_url", "image_url": {"url": f"data:{mime};base64,{encoded}"}}, ]}], ) - raw = resp.choices[0].message.content or "" - print(f"[grid] vision response received ({len(raw)} chars)") - raw = raw.strip() - if raw.startswith("```"): - raw = re.sub(r"^```[a-z]*\n?|\n?```$", "", raw) - payload = json.loads(raw) - slots = _slots_from_json(payload) - # Vision models frequently misread dates (DD/MM vs MM/DD confusion). - # Always use the hardcoded semester range as the source of truth. - print(f"[grid] parsed {len(slots)} slots, semester {SEMESTER_START} → {SEMESTER_END} (hardcoded)") - return slots, SEMESTER_START, SEMESTER_END - - -def load_grid(image_path: str, client=None, model: str = None): - """Return (slots, semester_start, semester_end). - - Uses the hardcoded grid transcribed from files/image.png. Vision parsing - is available but disabled by default because models frequently misread - subject codes and merge unrelated time blocks. - """ - _ = image_path, client, model # kept for API compatibility - print(f"[grid] using hardcoded grid: {len(HARDCODED_SLOTS)} slots, semester start {SEMESTER_START}") - return list(HARDCODED_SLOTS), SEMESTER_START, SEMESTER_END + raw = response.choices[0].message.content or "" + try: + payload = json.loads(_strip_fences(raw)) + except json.JSONDecodeError as exc: + raise GridParseError("the vision model did not return valid timetable JSON") from exc + + slots, semester_start, semester_end = _grid_from_json(payload) + print(f"[grid] parsed {len(slots)} slots, semester {semester_start} → {semester_end}") + return slots, semester_start, semester_end + + +def load_grid(image_path: str, client=None, model: str | None = None): + """Return slots and semester bounds parsed from the supplied timetable.""" + if not Path(image_path).is_file(): + raise GridParseError( + f"No timetable image found at {image_path}. " + "Add a complete timetable image with semester dates, subjects, weekdays, and times." + ) + if client is None or not model: + raise GridParseError("a configured vision model is required to read the timetable image") + return parse_with_vision(image_path, client, model) diff --git a/src/llm.py b/src/llm.py index 22669bb..bef8350 100644 --- a/src/llm.py +++ b/src/llm.py @@ -1,13 +1,16 @@ -"""LLM helpers: OpenAI-compatible client creation and fact extraction.""" +"""LLM helpers for provider-neutral teaching-plan event extraction.""" import json import os import re +import unicodedata from dataclasses import dataclass from datetime import date from openai import OpenAI +from .grid import Slot + @dataclass class Fact: @@ -34,231 +37,202 @@ def make_client() -> OpenAI | None: def _strip_fences(raw: str) -> str: raw = raw.strip() if raw.startswith("```"): - raw = re.sub(r"^```[a-z]*\n?|\n?```$", "", raw) - return raw - - -_PROMPT = """You extract evaluation events from a collection of Brazilian university -teaching plans ("planos de ensino"). The input is a JSON array of documents, each -with a filename and its complete extracted text. + raw = re.sub(r"^```[a-zA-Z0-9_-]*\n?", "", raw) + raw = re.sub(r"\n?```$", "", raw) + return raw.strip() + + +def _normalize(value: str) -> str: + decomposed = unicodedata.normalize("NFD", value or "") + unaccented = "".join(char for char in decomposed if not unicodedata.combining(char)) + return re.sub(r"[^A-Z0-9]+", " ", unaccented.upper()).strip() + + +def _subject_catalog(slots: list[Slot]) -> list[dict[str, str]]: + seen: set[tuple[str, str]] = set() + subjects: list[dict[str, str]] = [] + for slot in slots: + key = (slot.code, slot.name) + if key not in seen: + seen.add(key) + subjects.append({"code": slot.code, "name": slot.name}) + return subjects + + +def _build_prompt( + semester_start: date, + semester_end: date, + subjects: list[dict[str, str]], +) -> str: + catalog_json = json.dumps(subjects, ensure_ascii=False) + return f"""Extract evaluation events from Brazilian university teaching plans +(\"planos de ensino\"). The input is a JSON array of documents with filenames and +complete extracted text. + +The active semester is {semester_start.isoformat()} through {semester_end.isoformat()}. +The student's timetable contains these subjects: +{catalog_json} Rules: -- Extract ONLY tests and activities: entries like "PROVA P1", "Atividade At1", - "Avaliação bimestral", "PROVA substitutiva". -- The title must contain ONLY the human-readable evaluation label, such as - "Atividade At1" or "PROVA P1". Do NOT put a course/program identifier, - degree/shift/semester signal, subject code, or subject name in the title. - For example, if a document contains "COM-MA4" and "PROVA P1", the title is - "PROVA P1", not "COM-MA4" and not "COM-MA4 - PROVA P1". -- Compare all documents before extracting. Repeated signals such as a code - appearing in filenames, headers, and schedules are context for identifying - the subject, not activity names. -- EXCLUDE regular class topics and "VISTA DE PROVAS"/"Vista de provas". -- EXCLUDE sections from other years (e.g. a stale "CRONOGRAMA ... 2023"). -- kind: "test" for provas/avaliações bimestrais/semestrais/substitutiva, - "activity" for atividades. -- class_number: the numbered class (Aula) the event appears at, if visible. -- source_pdf: the exact filename of the document containing the event. -- subject_code and subject_name: return them on every event, using the actual - discipline identifier and name, not a program/shift/semester identifier. - NEVER use "COM-MA4", "COM-MA4A", "CC-MA4A" or similar as subject_code or - subject_name — these are degree/shift/semester identifiers, NOT subject codes. -- explicit_date: ONLY when a literal date (DD/MM or YYYY-MM-DD) is printed in - the document next to the event. NEVER fabricate a date from week numbers like - "4ª semana", "7ª semana", or class numbers like "4ª Aula". If the document - only says "4ª semana" or "4ª Aula" without a literal date, set explicit_date - to null and use class_number instead. - IMPORTANT: Dates in Brazilian documents use DD/MM format (day first, then month). - For example "22/04" means April 22 (NOT February 22), "15/09" means September 15. - Convert to "YYYY-MM-DD" using the year in the relevant filename. Otherwise null. -- CRITICAL: Many documents have a dated "Avaliação" / assessment table near the - end with explicit YYYY-MM-DD or DD/MM dates. ALWAYS extract every entry from - these tables. These tables are the authoritative source for dates. - For LMA documents, the dated entries are named C11, C12, C13, P1, C21, C22, - C23, P2, Prova Substitutiva — extract ALL of them with their explicit dates. -- When a document has BOTH a cronograma (class schedule) AND a dated assessment - table, prefer the assessment table for explicit dates. Use the cronograma only - to find class_number when no explicit date is available. - -Document → expected subject_code + subject_name (use these EXACTLY): - Plans containing "EDNL" in filename → EDNL, ESTRUTURAS DE DADOS - NÃO LINEARES - Plans containing "PE" or "Probabilidade" → PE, PROBABILIDADE E ESTATÍSTICA - Plans containing "TNC" in filename → TNC, TEORIA DOS NÚMEROS E CRIPTOGRAFIA - Plans containing "CGPI" in filename → CGPI, COMPUTAÇÃO GRÁFICA E PROCESSAMENTO DE IMAGENS - Plans containing "LMA" in filename → LMA, LABORATÓRIO DE MODELAGEM ALGORÍTMICA - Plans containing "TA" in filename → TA, TEORIA DE AUTÔMATOS - Plans containing "OP" or "Orientacao" in filename → OP, ORIENTAÇÃO PROFISSIONAL - -Return ONLY valid JSON: -{{"events": [{{"source_pdf": "plan-2026.pdf", "subject_code": "EDNL", - "subject_name": "ESTRUTURAS DE DADOS - NÃO LINEARES", "title": "Atividade At1", - "kind": "activity", "class_number": 15, "explicit_date": "2026-04-22"}}]}} -No markdown, no commentary.""" - - -def _clean_title(title: str, subject_code: str) -> str: - """Remove an accidentally repeated subject code from an LLM title.""" +- Extract only assessments and student deliverables, including provas, + avaliações, atividades, trabalhos, projetos, seminários, and substitutivas. +- Exclude ordinary class topics and \"vista de provas\". +- Ignore schedules or sections from a different semester or year. +- Use kind \"test\" for provas and formal assessments; use \"activity\" for + other graded deliverables. +- Return the exact source PDF filename. +- Return a concise human-readable title without degree, shift, semester, + subject code, or subject name prefixes. +- Infer the actual discipline code and name from the document, then use the + matching code and name from the timetable catalog whenever possible. Never + invent a timetable subject. If no catalog subject can be identified, return + the best code/name printed in the document so the student can resolve it. +- class_number is the numbered Aula containing the event, when printed. +- explicit_date is allowed only when a literal date is printed next to the + event or in an authoritative assessment table. Brazilian dates are DD/MM. + Convert dates to YYYY-MM-DD using the active semester. Never turn week or + class numbers into dates. +- When a schedule and a dated assessment table disagree, prefer the literal + date in the assessment table. +- Extract every assessment-table row; do not add document-specific exceptions. + +Return only valid JSON in this shape: +{{"events": [{{"source_pdf": "plano.pdf", "subject_code": "ABC", + "subject_name": "NOME DA DISCIPLINA", "title": "PROVA P1", + "kind": "test", "class_number": 12, "explicit_date": "YYYY-MM-DD"}}]}} +Use null for unknown class_number or explicit_date. No markdown or commentary.""" + + +def _clean_title(title: str, subject_code: str, subject_name: str) -> str: title = re.sub(r"\s+", " ", title or "").strip() - if subject_code: - title = re.sub(rf"^{re.escape(subject_code)}\s*[-:–—]\s*", "", title, flags=re.I) - return title - - -# Degree/shift/semester identifiers that should NEVER be used as subject_code. -_DEGREE_NOISE = re.compile( - r"^(COM[\s-]*MA\s*4\s*A?|CC[\s-]*MA\s*4\s*A?)$", re.IGNORECASE -) - -# Filename → correct (subject_code, subject_name) lookup. Order matters: -# earlier keys are tried first. -_FILENAME_MAP = [ - (re.compile(r"ednl", re.I), ("EDNL", "ESTRUTURAS DE DADOS - NÃO LINEARES")), - (re.compile(r"lma", re.I), ("LMA", "LABORATÓRIO DE MODELAGEM ALGORÍTMICA")), - (re.compile(r"cgpi", re.I), ("CGPI", "COMPUTAÇÃO GRÁFICA E PROCESSAMENTO DE IMAGENS")), - (re.compile(r"\bta\b|automato", re.I), ("TA", "TEORIA DE AUTÔMATOS")), - (re.compile(r"tnc|teoria\s+(dos\s+)?numeros", re.I), ("TNC", "TEORIA DOS NÚMEROS E CRIPTOGRAFIA")), - (re.compile(r"\bpe\b|probabilidade", re.I), ("PE", "PROBABILIDADE E ESTATÍSTICA")), - (re.compile(r"\bop\b|orientacao", re.I), ("OP", "ORIENTAÇÃO PROFISSIONAL")), -] - -_SUBJECT_NAME_OVERRIDES = { - "EDNL": "ESTRUTURAS DE DADOS - NÃO LINEARES", - "LMA": "LABORATÓRIO DE MODELAGEM ALGORÍTMICA", - "CGPI": "COMPUTAÇÃO GRÁFICA E PROCESSAMENTO DE IMAGENS", - "TA": "TEORIA DE AUTÔMATOS", - "TNC": "TEORIA DOS NÚMEROS E CRIPTOGRAFIA", - "PE": "PROBABILIDADE E ESTATÍSTICA", - "OP": "ORIENTAÇÃO PROFISSIONAL", -} - -_TITLE_NOISE = re.compile( - r"^(COM[\s-]*MA\s*4\s*A?|CC[\s-]*MA\s*4\s*A?)[\s-]*[-:–—]*[\s-]*", re.IGNORECASE -) - - -def _clean_facts(facts: list[Fact]) -> list[Fact]: - """Post-process facts to fix known LLM mistakes (bad codes, names, titles).""" - for fact in facts: - # 1. Fix subject_code if it's a degree identifier - if _DEGREE_NOISE.match(fact.subject_code): - for pattern, (code, name) in _FILENAME_MAP: - if pattern.search(fact.source_pdf) or pattern.search(fact.subject_name): - print(f"[llm] FIXED subject_code: {fact.subject_code} → {code} (from filename '{fact.source_pdf}')") - fact.subject_code = code - if fact.subject_name.upper() in ("", "PROBABILIDADE E ESTATÍSTICA", "TEORIA DOS NÚMEROS E CRIPTOGRAFIA (TNC)"): - fact.subject_name = name - break - - # 2. Normalize subject_name to canonical form - if fact.subject_code in _SUBJECT_NAME_OVERRIDES: - canonical = _SUBJECT_NAME_OVERRIDES[fact.subject_code] - if fact.subject_name.upper() != canonical: - print(f"[llm] FIXED subject_name: {fact.subject_name} → {canonical}") - fact.subject_name = canonical - - # 3. Strip COM-MA4 noise from titles - cleaned = _TITLE_NOISE.sub("", fact.title).strip() - if cleaned != fact.title: - print(f"[llm] FIXED title: '{fact.title}' → '{cleaned}'") - fact.title = cleaned - - return facts + for prefix in (subject_code, subject_name): + if prefix: + title = re.sub(rf"^{re.escape(prefix)}\s*[-:–—]\s*", "", title, flags=re.I) + return title.strip() + + +def _catalog_match(code: str, name: str, slots: list[Slot]) -> Slot | None: + normalized_code = _normalize(code) + normalized_name = _normalize(name) + if normalized_code: + code_matches = [slot for slot in slots if _normalize(slot.code) == normalized_code] + if code_matches: + return code_matches[0] + if normalized_name: + exact = [slot for slot in slots if _normalize(slot.name) == normalized_name] + if exact: + return exact[0] + partial = [ + slot for slot in slots + if normalized_name in _normalize(slot.name) or _normalize(slot.name) in normalized_name + ] + unique_subjects = {(slot.code, slot.name): slot for slot in partial} + if len(unique_subjects) == 1: + return next(iter(unique_subjects.values())) + return None + + +def _parse_class_number(value) -> int | None: + if value in (None, ""): + return None + try: + number = int(value) + except (TypeError, ValueError): + return None + return number if number > 0 else None -# Regex to extract LMA assessment table entries from raw PDF text. -# Matches lines like: "2026-08-25 Atividade Contínua (C11) Individual ..." -_LMA_ASSESSMENT_RE = re.compile( - r"^(?P\d{4}-\d{2}-\d{2})\s+" - r"(?PAtividade Contínua\s*\(C\d+\)|Prova Presencial\s*\(P\d+\)|Prova Substitutiva)", - re.MULTILINE | re.IGNORECASE, -) - -_LMA_KIND_MAP = { - "prova presencial": "test", - "prova substitutiva": "test", - "prova presencial (p1)": "test", - "prova presencial (p2)": "test", -} - - -def _extract_lma_from_text(documents: list[dict[str, str]], existing: list[Fact]) -> list[Fact]: - """Deterministic extraction of LMA assessment table entries. - - The LLM frequently misses the dated assessment table at the end of LMA - documents. This regex-based fallback catches every entry. - Skips entries that already exist (same date from an LMA PDF) to avoid - duplicates when the LLM also extracts from the same table. - """ - seen = {f.explicit_date for f in existing if f.explicit_date and re.search(r"lma", f.source_pdf, re.I)} - extra: list[Fact] = [] - for doc in documents: - if not re.search(r"lma", doc["filename"], re.I): +def _parse_explicit_date(value, semester_start: date, semester_end: date) -> date | None: + if not value: + return None + text = str(value).strip() + try: + candidate = date.fromisoformat(text[:10]) + return candidate if semester_start <= candidate <= semester_end else None + except ValueError: + pass + + match = re.fullmatch(r"(\d{1,2})/(\d{1,2})(?:/(\d{2,4}))?", text) + if not match: + return None + day_number, month_number = int(match.group(1)), int(match.group(2)) + explicit_year = match.group(3) + if explicit_year: + year = int(explicit_year) + year = 2000 + year if year < 100 else year + candidate_years = [year] + else: + candidate_years = list(range(semester_start.year, semester_end.year + 1)) + for year in candidate_years: + try: + candidate = date(year, month_number, day_number) + except ValueError: continue - text = doc["text"] - for m in _LMA_ASSESSMENT_RE.finditer(text): - title = m.group("title").strip() - kind = _LMA_KIND_MAP.get(title.lower(), "activity") - try: - explicit_date = date.fromisoformat(m.group("date")) - except ValueError: - continue - if explicit_date in seen: - continue - extra.append(Fact( - subject_code="LMA", - subject_name="LABORATÓRIO DE MODELAGEM ALGORÍTMICA", - title=title, - kind=kind, - class_number=None, - explicit_date=explicit_date, - source_pdf=doc["filename"], - )) - if extra: - print(f"[llm] LMA regex extracted {len(extra)} additional fact(s) from assessment table") - return extra - - -def extract_facts(documents: list[dict[str, str]], client: OpenAI, model: str) -> list[Fact]: - """Ask the LLM for evaluation facts from all labeled teaching plans.""" + if semester_start <= candidate <= semester_end: + return candidate + return None + + +def _parse_response(raw: str) -> list[dict]: + payload = json.loads(_strip_fences(raw)) + if isinstance(payload, list): + return [item for item in payload if isinstance(item, dict)] + if isinstance(payload, dict): + events = payload.get("events", []) + return [item for item in events if isinstance(item, dict)] if isinstance(events, list) else [] + return [] + + +def extract_facts( + documents: list[dict[str, str]], + client: OpenAI, + model: str, + slots: list[Slot], + semester_start: date, + semester_end: date, +) -> list[Fact]: + """Extract assessment facts and reconcile subjects against the timetable.""" + subjects = _subject_catalog(slots) + prompt = _build_prompt(semester_start, semester_end, subjects) payload_json = json.dumps(documents, ensure_ascii=False) print(f"[llm] extracting facts from {len(documents)} document(s) (model={model})...") - print(f"[llm] prompt size: {len(_PROMPT):,} chars, JSON payload size: {len(payload_json):,} chars") - resp = client.chat.completions.create( + print(f"[llm] subject catalog: {len(subjects)} subject(s)") + response = client.chat.completions.create( model=model, - messages=[{"role": "user", "content": _PROMPT + "\n\nDOCUMENTS_JSON:\n" + payload_json}], + messages=[{"role": "user", "content": prompt + "\n\nDOCUMENTS_JSON:\n" + payload_json}], ) - raw_content = resp.choices[0].message.content or "" - print(f"[llm] response received ({len(raw_content)} chars)") - payload = json.loads(_strip_fences(raw_content)) - # Handle both {"events": [...]} and bare [...] formats - if isinstance(payload, list): - events_list = payload - elif isinstance(payload, dict): - events_list = payload.get("events", []) - else: - events_list = [] - facts = [] - for ev in events_list: - source_pdf = str(ev.get("source_pdf", "")).strip() - year_match = re.search(r"(19|20)\d{2}", source_pdf) - year = year_match.group(0) if year_match else "2026" - subject_code = str(ev.get("subject_code", "")).strip().upper() - subject_name = str(ev.get("subject_name", "")).strip().upper() - explicit = ev.get("explicit_date") + events_list = _parse_response(response.choices[0].message.content or "") + + facts: list[Fact] = [] + for event in events_list: + source_pdf = str(event.get("source_pdf", "")).strip() + subject_code = str(event.get("subject_code", "")).strip().upper() + subject_name = str(event.get("subject_name", "")).strip().upper() + catalog_slot = _catalog_match(subject_code, subject_name, slots) + if catalog_slot is not None: + subject_code = catalog_slot.code or subject_code + subject_name = catalog_slot.name or subject_name + kind = str(event.get("kind", "activity")).strip().lower() + if kind not in {"test", "activity"}: + kind = "activity" fact = Fact( subject_code=subject_code, subject_name=subject_name, - title=_clean_title(str(ev.get("title", "")), subject_code), - kind=ev.get("kind", "activity"), - class_number=ev.get("class_number"), - explicit_date=date.fromisoformat(explicit) if explicit else None, + title=_clean_title(str(event.get("title", "")), subject_code, subject_name), + kind=kind, + class_number=_parse_class_number(event.get("class_number")), + explicit_date=_parse_explicit_date( + event.get("explicit_date"), semester_start, semester_end + ), source_pdf=source_pdf, ) + if not fact.title: + print(f"[llm] skipped event with an empty title from {source_pdf or 'unknown source'}") + continue facts.append(fact) - print(f"[llm] event: {fact.title} | kind={fact.kind} | class_number={fact.class_number} | explicit_date={fact.explicit_date}") + print( + f"[llm] event: {fact.title} | subject={fact.subject_code or fact.subject_name} " + f"| class_number={fact.class_number} | explicit_date={fact.explicit_date}" + ) print(f"[llm] extracted {len(facts)} fact(s) across {len(documents)} document(s)") - facts = _clean_facts(facts) - # Deterministic fallback: extract LMA assessment table entries - lma_facts = _extract_lma_from_text(documents, facts) - facts.extend(lma_facts) return facts diff --git a/src/main.py b/src/main.py index 5352f87..72ff63f 100644 --- a/src/main.py +++ b/src/main.py @@ -17,8 +17,8 @@ review as review_calendar, summarize as summarize_calendar, ) -from .dates import compute_events -from .grid import load_grid +from .dates import EventResolutionError, compute_events +from .grid import Slot, load_grid from .output import write_csv, write_ics PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -32,8 +32,75 @@ CALENDAR_REPORT_PATH = os.path.join(OUTPUT_DIR, "academic-calendar-review.json") CALENDAR_CACHE_DIR = os.path.join(OUTPUT_DIR, "academic-calendar-cache") +WEEKDAY_NAMES = [ + "segunda-feira", "terça-feira", "quarta-feira", "quinta-feira", + "sexta-feira", "sábado", "domingo", +] +_SUBJECT_RESOLUTIONS: dict[tuple[str, str, str], tuple[str, str]] = {} + + +def _ask_number(prompt: str, maximum: int) -> int | None: + """Ask for a 1-based menu choice; blank input cancels resolution.""" + while True: + try: + raw = input(prompt).strip() + except EOFError: + return None + if not raw: + return None + try: + choice = int(raw) + except ValueError: + print(f"Enter a number from 1 to {maximum}, or press Enter to abort.") + continue + if 1 <= choice <= maximum: + return choice - 1 + print(f"Enter a number from 1 to {maximum}, or press Enter to abort.") + + +def _resolve_subject(fact, slots: list[Slot]) -> list[Slot] | None: + """Let the student map an unmatched teaching plan to a timetable subject.""" + cache_key = (fact.source_pdf, fact.subject_code, fact.subject_name) + if cache_key in _SUBJECT_RESOLUTIONS: + code, name = _SUBJECT_RESOLUTIONS[cache_key] + fact.subject_code, fact.subject_name = code or fact.subject_code, name + return [slot for slot in slots if (slot.code, slot.name) == (code, name)] + + subjects: list[tuple[str, str]] = [] + for slot in slots: + identity = (slot.code, slot.name) + if identity not in subjects: + subjects.append(identity) + + print("\n[resolve] Teaching-plan subject was not found in the timetable:") + print(f" File: {fact.source_pdf}") + print(f" Extracted: {fact.subject_code or '(no code)'} — {fact.subject_name or '(no name)'}") + for index, (code, name) in enumerate(subjects, start=1): + label = f"{code} — {name}" if code else name + print(f" {index}. {label}") + choice = _ask_number("Choose the correct subject, or press Enter to abort: ", len(subjects)) + if choice is None: + return None + code, name = subjects[choice] + _SUBJECT_RESOLUTIONS[cache_key] = (code, name) + fact.subject_code, fact.subject_name = code or fact.subject_code, name + return [slot for slot in slots if (slot.code, slot.name) == (code, name)] + + +def _resolve_slot(fact, candidates: list[Slot], day) -> Slot | None: + """Let the student choose a time when the timetable match is ambiguous.""" + print(f"\n[resolve] Choose a time for {fact.title!r} on {day.isoformat()} ({WEEKDAY_NAMES[day.weekday()]}):") + for index, slot in enumerate(candidates, start=1): + print( + f" {index}. {WEEKDAY_NAMES[slot.weekday]} " + f"{slot.start:%H:%M}–{slot.end:%H:%M} — {slot.code or slot.name}" + ) + choice = _ask_number("Choose the timetable slot, or press Enter to abort: ", len(candidates)) + return candidates[choice] if choice is not None else None + def main() -> int: + _SUBJECT_RESOLUTIONS.clear() banner() print(" teaching plans + timetable + academic calendar → events\n") load_dotenv() @@ -47,8 +114,14 @@ def main() -> int: print("=" * 60) return 1 - model_text = os.getenv("MODEL_TEXT", "qwen-turbo") - model_vision = os.getenv("MODEL_VISION", "qwen-vl-plus") + model_text = os.getenv("MODEL_TEXT", "").strip() + model_vision = os.getenv("MODEL_VISION", "").strip() + if not model_text or not model_vision: + print("=" * 60) + print(" ERROR: MODEL_TEXT and MODEL_VISION must be set in .env.") + print(" MODEL_VISION must support image inputs.") + print("=" * 60) + return 1 print(f"[main] models: text={model_text}, vision={model_vision}") # Select this before collecting teaching plans so the academic calendar is @@ -59,9 +132,16 @@ def main() -> int: print("\n" + "-" * 40) print(" STEP 1: Parse timetable grid") print("-" * 40) - if not os.path.isfile(IMAGE_PATH): - print(f"[main] WARNING: No timetable image at {IMAGE_PATH} — using hardcoded grid") - slots, semester_start, semester_end = load_grid(IMAGE_PATH, client, model_vision) + try: + slots, semester_start, semester_end = load_grid(IMAGE_PATH, client, model_vision) + except Exception as exc: + print("=" * 60) + print(" ERROR: Could not read the timetable image.") + print(f" {exc}") + print(" Provide files/image.png with visible semester dates, subjects,") + print(" weekdays, and start/end times, then run the tool again.") + print("=" * 60) + return 1 print(f"[main] grid result: {len(slots)} slots, semester start {semester_start.isoformat()}") print("\n" + "-" * 40) @@ -88,7 +168,22 @@ def main() -> int: json.dump(documents, fh, ensure_ascii=False, indent=2) print(f"[main] debug payload: {PAYLOAD_PATH}") - facts = llm.extract_facts(documents, client, model_text) + try: + facts = llm.extract_facts( + documents, + client, + model_text, + slots, + semester_start, + semester_end, + ) + except Exception as exc: + print("=" * 60) + print(" ERROR: Could not extract events from the teaching plans.") + print(f" {exc}") + print(" Check the PDFs and text-model configuration, then run again.") + print("=" * 60) + return 1 print(f"[main] total facts across all PDFs: {len(facts)}") non_teaching_dates = set() @@ -128,7 +223,24 @@ def main() -> int: print("\n" + "-" * 40) print(" STEP 4: Compute event dates & times") print("-" * 40) - events = compute_events(facts, slots, semester_start, non_teaching_dates=non_teaching_dates) + try: + events = compute_events( + facts, + slots, + semester_start, + semester_end=semester_end, + non_teaching_dates=non_teaching_dates, + subject_resolver=_resolve_subject, + slot_resolver=_resolve_slot, + ) + except EventResolutionError as exc: + print("=" * 60) + print(" ERROR: An event could not be resolved safely.") + print(f" {exc}") + print(" No new CSV or ICS file was written. Fix the source document or") + print(" timetable image, then run the tool again.") + print("=" * 60) + return 1 print("\n" + "-" * 40) print(" STEP 5: Write output files") diff --git a/src/output.py b/src/output.py index d5e4c9f..aa7a11e 100644 --- a/src/output.py +++ b/src/output.py @@ -52,13 +52,14 @@ def write_ics(events: list[Event], path: str) -> None: lines: list[str] = [ "BEGIN:VCALENDAR", "VERSION:2.0", - "PRODID:-//iso-parse//iso-parse//EN", + "PRODID:-//ics-parse//ics-parse//EN", "METHOD:PUBLISH", - "X-WR-CALNAME:iso-parse events", + "X-WR-CALNAME:ics-parse events", ] for e in events: - uid = _sanitize_uid(f"{e.date.isoformat()}-{e.subject_code}-{e.title}@iso-parse") - summary = f"{e.subject_code} - {e.title}" + subject_label = e.subject_code or e.subject_name + uid = _sanitize_uid(f"{e.date.isoformat()}-{subject_label}-{e.title}@ics-parse") + summary = f"{subject_label} - {e.title}" if subject_label else e.title desc = f"{e.subject_name} ({e.kind})" if e.estimated: desc += " - estimated" @@ -72,7 +73,6 @@ def write_ics(events: list[Event], path: str) -> None: f"DTEND:{dtend}", f"SUMMARY:{summary}", f"DESCRIPTION:{desc}", - "LOCATION:Campus", "END:VEVENT", ] lines.append("END:VCALENDAR") diff --git a/src/test.py b/src/test.py index c8cebeb..1708535 100644 --- a/src/test.py +++ b/src/test.py @@ -8,8 +8,9 @@ from . import llm, pdfs from .academic_calendar import discover_calendar_files -from .dates import compute_events -from .grid import load_grid +from .dates import EventResolutionError, compute_events +from .grid import GridParseError, load_grid +from .main import _resolve_slot, _resolve_subject @@ -26,14 +27,30 @@ def main() -> int: print("ERROR: No API_KEY found. Copy .env.example to .env and fill in your key.") return 1 - model_text = os.getenv("MODEL_TEXT", "qwen-turbo") - model_vision = os.getenv("MODEL_VISION", "qwen-vl-plus") - with redirect_stdout(StringIO()): - slots, semester_start, _ = load_grid(IMAGE_PATH, client, model_vision) - calendar_paths = {str(path) for path in discover_calendar_files(FILES_DIR)} - documents = pdfs.collect_documents(FILES_DIR, excluded_paths=calendar_paths) - facts = llm.extract_facts(documents, client, model_text) - events = compute_events(facts, slots, semester_start) + model_text = os.getenv("MODEL_TEXT", "").strip() + model_vision = os.getenv("MODEL_VISION", "").strip() + if not model_text or not model_vision: + print("ERROR: MODEL_TEXT and MODEL_VISION must be configured in .env.") + return 1 + try: + with redirect_stdout(StringIO()): + slots, semester_start, semester_end = load_grid(IMAGE_PATH, client, model_vision) + calendar_paths = {str(path) for path in discover_calendar_files(FILES_DIR)} + documents = pdfs.collect_documents(FILES_DIR, excluded_paths=calendar_paths) + facts = llm.extract_facts( + documents, client, model_text, slots, semester_start, semester_end + ) + events = compute_events( + facts, + slots, + semester_start, + semester_end=semester_end, + subject_resolver=_resolve_subject, + slot_resolver=_resolve_slot, + ) + except Exception as exc: + print(f"ERROR: {exc}") + return 1 print("\nEVENT PREVIEW") print("=============") diff --git a/tests/test_academic_calendar.py b/tests/test_academic_calendar.py index fd78d83..85c09cb 100644 --- a/tests/test_academic_calendar.py +++ b/tests/test_academic_calendar.py @@ -58,7 +58,7 @@ def test_scope_accepts_global_and_matching_campus_but_rejects_other_campus(self) entries = [ CalendarEntry(date(2026, 9, 7), "Feriado", "holiday", True, scope_kind="global"), CalendarEntry(date(2026, 10, 12), "Recesso", "recess", True, scope_kind="restricted", campuses=["Campus Centro"]), - CalendarEntry(date(2026, 10, 15), "Recesso", "recess", True, scope_kind="restricted", campuses=["Sorocaba"]), + CalendarEntry(date(2026, 10, 15), "Recesso", "recess", True, scope_kind="restricted", campuses=["Campus Norte"]), ] decisions = resolve_entries(entries, UserProfile(campus="Campus Centro")) self.assertEqual([decision.decision for decision in decisions], ["include", "include", "exclude"]) @@ -99,7 +99,7 @@ def test_string_false_is_parsed_as_false(self): def test_same_day_rows_for_different_campuses_are_not_deduplicated(self): entries = [ CalendarEntry(date(2026, 10, 15), "Recesso", "recess", True, campuses=["Centro"]), - CalendarEntry(date(2026, 10, 15), "Recesso", "recess", True, campuses=["Sorocaba"]), + CalendarEntry(date(2026, 10, 15), "Recesso", "recess", True, campuses=["Norte"]), ] self.assertEqual(len(_deduplicate(entries)), 2) diff --git a/tests/test_generalization.py b/tests/test_generalization.py new file mode 100644 index 0000000..2649bac --- /dev/null +++ b/tests/test_generalization.py @@ -0,0 +1,121 @@ +"""Tests that student-specific timetable data is never built into the tool.""" + +import tempfile +import unittest +from datetime import date, time +from pathlib import Path + +from src.dates import Event, EventResolutionError, compute_events +from src.grid import GridParseError, Slot, _grid_from_json, load_grid +from src.llm import Fact, _catalog_match, _parse_explicit_date +from src.output import write_ics + + +class GeneralizationTests(unittest.TestCase): + def test_grid_uses_arbitrary_subjects_and_semester_dates(self): + slots, semester_start, semester_end = _grid_from_json({ + "semester_start": "2027-02-01", + "semester_end": "2027-06-30", + "slots": [{ + "weekday": "saturday", + "code": "BIO42", + "name": "Biologia Computacional", + "start": "13:30", + "end": "15:10", + }], + }) + self.assertEqual(semester_start, date(2027, 2, 1)) + self.assertEqual(semester_end, date(2027, 6, 30)) + self.assertEqual(slots[0], Slot(5, "BIO42", "BIOLOGIA COMPUTACIONAL", time(13, 30), time(15, 10))) + + def test_missing_timetable_has_no_built_in_fallback(self): + with self.assertRaises(GridParseError): + load_grid("/path/that/does/not/exist.png", client=object(), model="vision-model") + + def test_invalid_semester_bounds_are_rejected(self): + with self.assertRaises(GridParseError): + _grid_from_json({ + "semester_start": "2027-07-01", + "semester_end": "2027-02-01", + "slots": [{ + "weekday": "monday", "code": "X", "name": "Example", + "start": "08:00", "end": "09:00", + }], + }) + + def test_subjects_are_reconciled_from_the_supplied_timetable(self): + slots = [Slot(2, "BIO42", "BIOLOGIA COMPUTACIONAL", time(13, 30), time(15, 10))] + self.assertEqual(_catalog_match("bio42", "", slots), slots[0]) + self.assertEqual(_catalog_match("", "Biologia Computacional", slots), slots[0]) + self.assertIsNone(_catalog_match("OLD", "Outra disciplina", slots)) + + def test_document_code_is_preserved_when_timetable_has_names_only(self): + slots = [Slot(2, "", "BIOLOGIA COMPUTACIONAL", time(13, 30), time(15, 10))] + match = _catalog_match("BIO42", "Biologia Computacional", slots) + self.assertEqual(match, slots[0]) + + def test_short_date_uses_the_active_semester_year(self): + parsed = _parse_explicit_date( + "15/08", date(2027, 8, 1), date(2027, 12, 20) + ) + self.assertEqual(parsed, date(2027, 8, 15)) + + def test_stale_explicit_year_is_rejected(self): + parsed = _parse_explicit_date( + "2024-08-15", date(2027, 8, 1), date(2027, 12, 20) + ) + self.assertIsNone(parsed) + + def test_arbitrary_subject_event_uses_its_timetable_slot(self): + slots = [Slot(0, "BIO42", "BIOLOGIA COMPUTACIONAL", time(13, 30), time(15, 10))] + fact = Fact("BIO42", "BIOLOGIA COMPUTACIONAL", "Projeto 1", "activity", 2, None, "plano.pdf") + events = compute_events( + [fact], slots, date(2027, 2, 1), semester_end=date(2027, 6, 30) + ) + self.assertEqual(events[0].date, date(2027, 2, 8)) + self.assertEqual((events[0].start, events[0].end), (time(13, 30), time(15, 10))) + + def test_unmatched_subject_does_not_receive_a_default_time(self): + slots = [Slot(0, "BIO42", "BIOLOGIA COMPUTACIONAL", time(13, 30), time(15, 10))] + fact = Fact("OTHER", "OUTRA DISCIPLINA", "Prova", "test", None, date(2027, 3, 1), "plano.pdf") + with self.assertRaises(EventResolutionError): + compute_events( + [fact], slots, date(2027, 2, 1), semester_end=date(2027, 6, 30) + ) + + def test_ambiguous_partial_subject_name_requires_resolution(self): + slots = [ + Slot(0, "", "CÁLCULO I", time(8), time(9)), + Slot(1, "", "CÁLCULO II", time(9), time(10)), + ] + fact = Fact("", "CÁLCULO", "Prova", "test", None, date(2027, 3, 1), "plano.pdf") + with self.assertRaises(EventResolutionError): + compute_events( + [fact], slots, date(2027, 2, 1), semester_end=date(2027, 6, 30) + ) + + def test_ics_omits_location(self): + event = Event( + "BIO42", "BIOLOGIA COMPUTACIONAL", "Projeto 1", "activity", + date(2027, 2, 8), time(13, 30), time(15, 10), False, "plano.pdf", + ) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory, "events.ics") + write_ics([event], str(path)) + content = path.read_text(encoding="utf-8") + self.assertNotIn("LOCATION:", content) + + def test_ics_uses_subject_name_when_code_is_unavailable(self): + event = Event( + "", "BIOLOGIA COMPUTACIONAL", "Projeto 1", "activity", + date(2027, 2, 8), time(13, 30), time(15, 10), False, "plano.pdf", + ) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory, "events.ics") + write_ics([event], str(path)) + content = path.read_text(encoding="utf-8") + self.assertIn("SUMMARY:BIOLOGIA COMPUTACIONAL - Projeto 1", content) + + +if __name__ == "__main__": + unittest.main() From 14054278a7580df3d7b4f53449a897e2e98894a4 Mon Sep 17 00:00:00 2001 From: raulkolaric <raul.kolaric@gmail.com> Date: Fri, 7 Aug 2026 11:29:23 -0300 Subject: [PATCH 2/4] fix: merge consecutive timetable rows - grid.py: combine adjacent slots for the same subject into one class block\n- test_generalization.py: cover consecutive timetable rows from one class session --- src/grid.py | 32 +++++++++++++++++++++++++++++++- tests/test_generalization.py | 26 ++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/grid.py b/src/grid.py index 9ff19d1..c8e1409 100644 --- a/src/grid.py +++ b/src/grid.py @@ -90,6 +90,36 @@ def _parse_date(value, field: str) -> date: raise GridParseError(f"missing or invalid {field}; expected YYYY-MM-DD") from exc +def _same_subject(left: Slot, right: Slot) -> bool: + """Match repeated timetable rows without confusing adjacent subjects.""" + if left.code and right.code and left.code != right.code: + return False + return bool(left.name and right.name and left.name == right.name) + + +def _merge_adjacent_slots(slots: list[Slot]) -> list[Slot]: + """Merge consecutive timetable rows belonging to one class session.""" + merged: list[Slot] = [] + for slot in sorted(slots, key=lambda item: (item.weekday, item.start, item.end)): + previous = merged[-1] if merged else None + if ( + previous is not None + and previous.weekday == slot.weekday + and previous.end == slot.start + and _same_subject(previous, slot) + ): + merged[-1] = Slot( + weekday=previous.weekday, + code=previous.code or slot.code, + name=previous.name or slot.name, + start=previous.start, + end=slot.end, + ) + else: + merged.append(slot) + return merged + + def _grid_from_json(payload: dict) -> tuple[list[Slot], date, date]: if not isinstance(payload, dict): raise GridParseError("timetable response was not a JSON object") @@ -120,7 +150,7 @@ def _grid_from_json(payload: dict) -> tuple[list[Slot], date, date]: raise GridParseError(f"timetable slot {index} ends before it starts") slots.append(Slot(WEEKDAYS[weekday_key], code, name, start, end)) - return slots, semester_start, semester_end + return _merge_adjacent_slots(slots), semester_start, semester_end def parse_with_vision(image_path: str, client, model: str) -> tuple[list[Slot], date, date]: diff --git a/tests/test_generalization.py b/tests/test_generalization.py index 2649bac..785aa47 100644 --- a/tests/test_generalization.py +++ b/tests/test_generalization.py @@ -28,6 +28,32 @@ def test_grid_uses_arbitrary_subjects_and_semester_dates(self): self.assertEqual(semester_end, date(2027, 6, 30)) self.assertEqual(slots[0], Slot(5, "BIO42", "BIOLOGIA COMPUTACIONAL", time(13, 30), time(15, 10))) + def test_consecutive_rows_for_one_subject_merge_into_one_class_block(self): + slots, _, _ = _grid_from_json({ + "semester_start": "2026-08-03", + "semester_end": "2026-12-12", + "slots": [ + { + "weekday": "monday", + "code": "TNC", + "name": "TEORIA DOS NÚMEROS E CRIPTOGRAFIA", + "start": "09:05", + "end": "09:55", + }, + { + "weekday": "monday", + "code": "TNC", + "name": "TEORIA DOS NÚMEROS E CRIPTOGRAFIA", + "start": "09:55", + "end": "10:45", + }, + ], + }) + self.assertEqual( + slots, + [Slot(0, "TNC", "TEORIA DOS NÚMEROS E CRIPTOGRAFIA", time(9, 5), time(10, 45))], + ) + def test_missing_timetable_has_no_built_in_fallback(self): with self.assertRaises(GridParseError): load_grid("/path/that/does/not/exist.png", client=object(), model="vision-model") From d3d07fa6b620b8937d3c3b49243b078b63be8a15 Mon Sep 17 00:00:00 2001 From: raulkolaric <raul.kolaric@gmail.com> Date: Fri, 7 Aug 2026 11:35:07 -0300 Subject: [PATCH 3/4] fix: auto-select sole subject slot - dates.py: use the only subject timetable slot when an explicit date falls on another weekday\n- test_generalization.py: cover automatic resolution for a single available slot --- src/dates.py | 3 +++ tests/test_generalization.py | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/dates.py b/src/dates.py index e2bfb6a..77b6ad1 100644 --- a/src/dates.py +++ b/src/dates.py @@ -134,6 +134,9 @@ def compute_events( note = "explicit date falls on an accepted non-teaching day; review calendar conflict" same_day_slots = [slot for slot in subj_slots if slot.weekday == day.weekday()] slot = same_day_slots[0] if len(same_day_slots) == 1 else None + if slot is None and not same_day_slots and len(subj_slots) == 1: + slot = subj_slots[0] + note = _append_note(note, "weekday not in timetable; used subject's only slot") if slot is None and slot_resolver is not None: slot = slot_resolver(fact, same_day_slots or subj_slots, day) if slot is not None: diff --git a/tests/test_generalization.py b/tests/test_generalization.py index 785aa47..2dd2e6f 100644 --- a/tests/test_generalization.py +++ b/tests/test_generalization.py @@ -109,6 +109,18 @@ def test_unmatched_subject_does_not_receive_a_default_time(self): [fact], slots, date(2027, 2, 1), semester_end=date(2027, 6, 30) ) + def test_single_subject_slot_is_used_without_prompt_for_explicit_other_weekday(self): + slots = [Slot(3, "PE", "PROBABILIDADE E ESTATÍSTICA", time(9, 5), time(10, 45))] + fact = Fact( + "PE", "PROBABILIDADE E ESTATÍSTICA", "A1 – Atividade Individual", + "activity", None, date(2026, 9, 4), "plano.pdf", + ) + events = compute_events( + [fact], slots, date(2026, 8, 3), semester_end=date(2026, 12, 12) + ) + self.assertEqual((events[0].start, events[0].end), (time(9, 5), time(10, 45))) + self.assertIn("used subject's only slot", events[0].note) + def test_ambiguous_partial_subject_name_requires_resolution(self): slots = [ Slot(0, "", "CÁLCULO I", time(8), time(9)), From 15a344fe054ef0f0a34f37682018bdf971a8daf8 Mon Sep 17 00:00:00 2001 From: raulkolaric <raul.kolaric@gmail.com> Date: Fri, 7 Aug 2026 11:40:28 -0300 Subject: [PATCH 4/4] feat: derive academic context from plans Extract calendar-scoping context from the teaching-plan Markdown content and preserve missing values as blank instead of prompting the student. --- README.md | 9 ++++-- src/llm.py | 59 ++++++++++++++++++++++++++++++++++++ src/main.py | 11 +++++-- tests/test_generalization.py | 34 ++++++++++++++++++++- 4 files changed, 107 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4294bec..e5c7b65 100644 --- a/README.md +++ b/README.md @@ -43,9 +43,12 @@ enter its path in the terminal. The optional calendar keeps estimated dates from drifting after holidays, recesses, or cancelled class days. -When a calendar has different rules for different campuses, the terminal asks -for your campus and, if useful, your unit, course, and shift. If a row is -unclear or belongs to a different campus, the tool asks instead of guessing. +When a calendar has different rules for different campuses, the tool derives +the academic context from the teaching-plan text saved under `output/extracted/`. +It does not ask for institution, campus, faculty, program, or shift; a field +that is missing or inconsistent across plans remains blank. If a calendar row +is unclear or has a different scope, the tool asks for a decision instead of +guessing. The calendar may be normal text, scanned pages, or a mixture of both. Its page results are saved under `output/academic-calendar-cache/`, so rerunning the diff --git a/src/llm.py b/src/llm.py index bef8350..9767ebc 100644 --- a/src/llm.py +++ b/src/llm.py @@ -9,6 +9,7 @@ from openai import OpenAI +from .academic_calendar.models import UserProfile from .grid import Slot @@ -103,6 +104,30 @@ class numbers into dates. Use null for unknown class_number or explicit_date. No markdown or commentary.""" +def _build_academic_context_prompt() -> str: + return """Extract the shared academic context from the supplied university +teaching plans ("planos de ensino"). The input is a JSON array containing the +same extracted document text that is saved as Markdown in output/extracted/. + +Return only values literally stated in the documents. Do not use institutional +knowledge, infer a value, or ask the student for confirmation. When documents +conflict or a value is not stated, return an empty string. + +Return only valid JSON in this shape: +{"institution": "", "campus": "", "unit": "", "program": "", "shift": ""} + +Field rules: +- institution: university or institution name. +- campus: an explicitly named physical campus or location only. Leave blank + when the plan does not state one. +- unit: faculty, school, institute, or academic unit. +- program: degree program or course. +- shift: an explicitly stated class shift (morning, afternoon, evening, night, + or its local equivalent). A numbered curricular period such as "4º" is not a + shift and must leave this field blank. +No markdown or commentary.""" + + def _clean_title(title: str, subject_code: str, subject_name: str) -> str: title = re.sub(r"\s+", " ", title or "").strip() for prefix in (subject_code, subject_name): @@ -183,6 +208,40 @@ def _parse_response(raw: str) -> list[dict]: return [] +def _context_value(payload: dict, key: str) -> str: + value = payload.get(key, "") + return value.strip() if isinstance(value, str) else "" + + +def extract_academic_context( + documents: list[dict[str, str]], client: OpenAI, model: str +) -> UserProfile: + """Derive calendar-scoping context from extracted teaching-plan Markdown text.""" + payload_json = json.dumps(documents, ensure_ascii=False) + print(f"[llm] extracting academic context from {len(documents)} document(s) (model={model})...") + response = client.chat.completions.create( + model=model, + messages=[{ + "role": "user", + "content": _build_academic_context_prompt() + "\n\nDOCUMENTS_JSON:\n" + payload_json, + }], + ) + payload = json.loads(_strip_fences(response.choices[0].message.content or "")) + if not isinstance(payload, dict): + raise ValueError("academic-context response was not a JSON object") + profile = UserProfile( + institution=_context_value(payload, "institution"), + campus=_context_value(payload, "campus"), + unit=_context_value(payload, "unit"), + program=_context_value(payload, "program"), + shift=_context_value(payload, "shift"), + ) + print("[llm] academic context: " + ", ".join( + f"{key}={value or '(unknown)'}" for key, value in profile.as_dict().items() + )) + return profile + + def extract_facts( documents: list[dict[str, str]], client: OpenAI, diff --git a/src/main.py b/src/main.py index 72ff63f..a562380 100644 --- a/src/main.py +++ b/src/main.py @@ -9,8 +9,8 @@ from . import llm from . import pdfs from .academic_calendar import analyze_calendar, resolve_entries +from .academic_calendar.models import UserProfile from .academic_calendar.terminal import ( - ask_profile, banner, choose_calendar, progress as calendar_progress, @@ -127,7 +127,6 @@ def main() -> int: # Select this before collecting teaching plans so the academic calendar is # never sent to the evaluation-event extractor as if it were a course PDF. calendar_path = choose_calendar(FILES_DIR) - profile = ask_profile() if calendar_path else None print("\n" + "-" * 40) print(" STEP 1: Parse timetable grid") @@ -168,6 +167,14 @@ def main() -> int: json.dump(documents, fh, ensure_ascii=False, indent=2) print(f"[main] debug payload: {PAYLOAD_PATH}") + profile = UserProfile() + if calendar_path: + try: + profile = llm.extract_academic_context(documents, client, model_text) + except Exception as exc: + print(f"[main] WARNING: could not derive academic context from teaching plans: {exc}") + print("[main] calendar scope will use blank fields; no questionnaire will be shown") + try: facts = llm.extract_facts( documents, diff --git a/tests/test_generalization.py b/tests/test_generalization.py index 2dd2e6f..c8c4ef9 100644 --- a/tests/test_generalization.py +++ b/tests/test_generalization.py @@ -7,10 +7,24 @@ from src.dates import Event, EventResolutionError, compute_events from src.grid import GridParseError, Slot, _grid_from_json, load_grid -from src.llm import Fact, _catalog_match, _parse_explicit_date +from src.llm import Fact, _catalog_match, _parse_explicit_date, extract_academic_context from src.output import write_ics +class _FakeCompletions: + def __init__(self, content): + self.content = content + + def create(self, **_kwargs): + message = type("Message", (), {"content": self.content})() + return type("Response", (), {"choices": [type("Choice", (), {"message": message})()]})() + + +class _FakeClient: + def __init__(self, content): + self.chat = type("Chat", (), {"completions": _FakeCompletions(content)})() + + class GeneralizationTests(unittest.TestCase): def test_grid_uses_arbitrary_subjects_and_semester_dates(self): slots, semester_start, semester_end = _grid_from_json({ @@ -92,6 +106,24 @@ def test_stale_explicit_year_is_rejected(self): ) self.assertIsNone(parsed) + def test_academic_context_is_derived_without_guessing_campus_or_shift(self): + client = _FakeClient('''{ + "institution": "Pontifícia Universidade Católica de São Paulo", + "campus": "", + "unit": "Faculdade de Ciências Exatas e Tecnologia", + "program": "Ciência da Computação", + "shift": "" + }''') + profile = extract_academic_context([{ + "filename": "tnc.pdf", + "text": "CURSO Ciência da Computação\nPERÍODO 4º\nFACULDADE FCET", + }], client, "test-model") + self.assertEqual(profile.institution, "Pontifícia Universidade Católica de São Paulo") + self.assertEqual(profile.unit, "Faculdade de Ciências Exatas e Tecnologia") + self.assertEqual(profile.program, "Ciência da Computação") + self.assertEqual(profile.campus, "") + self.assertEqual(profile.shift, "") + def test_arbitrary_subject_event_uses_its_timetable_slot(self): slots = [Slot(0, "BIO42", "BIOLOGIA COMPUTACIONAL", time(13, 30), time(15, 10))] fact = Fact("BIO42", "BIOLOGIA COMPUTACIONAL", "Projeto 1", "activity", 2, None, "plano.pdf")