diff --git a/backend/src/data/main.py b/backend/src/data/main.py index 6b0e64c3..7d0585d0 100644 --- a/backend/src/data/main.py +++ b/backend/src/data/main.py @@ -74,13 +74,13 @@ def process_year( objs = process_year_epa(objs, all_team_years) timer.print(str(year_num) + " EPA") - write_objs_db(year_num, objs, orig_objs if partial else None, not partial) - timer.print(str(year_num) + " Write DB") - if year_num == CURR_YEAR and not DISABLE_GCS: write_objs_storage(objs, orig_objs if partial else None) timer.print(str(year_num) + " Write Storage") + write_objs_db(year_num, objs, orig_objs if partial else None, not partial) + timer.print(str(year_num) + " Write DB") + return teams diff --git a/backend/src/data/utils.py b/backend/src/data/utils.py index 669be1af..ad2f0246 100644 --- a/backend/src/data/utils.py +++ b/backend/src/data/utils.py @@ -1,5 +1,8 @@ from datetime import datetime -from typing import Dict, Optional, Tuple +import math +from typing import Any, Dict, Optional, Tuple + +import attr from src.db.functions import clear_year from src.db.models import ETag, Event, Match, TeamEvent, TeamYear, Year @@ -27,6 +30,24 @@ def create_objs(year: int) -> objs_type: return (Year(year=year), {}, {}, {}, {}, {}) +def _canonical(value: Any) -> Any: + if isinstance(value, float): + return "__nan__" if math.isnan(value) else value + if attr.has(type(value)): + return {f.name: _canonical(getattr(value, f.name)) for f in attr.fields(type(value))} + if isinstance(value, dict): + return {k: _canonical(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_canonical(v) for v in value] + return value + + +def nan_safe_eq(a: Any, b: Any) -> bool: + if a == b: + return True + return _canonical(a) == _canonical(b) + + def read_objs(year: int) -> objs_type: year_obj = get_year_db(year) if year_obj is None: @@ -55,7 +76,7 @@ def write_objs( orig_objs = create_objs(-1) def changed(curr: dict, prev: dict) -> list: - return [obj for obj in curr.values() if str(obj) != str(prev.get(obj.pk(), ""))] + return [obj for obj in curr.values() if not nan_safe_eq(obj, prev.get(obj.pk()))] write_year_db( years=[objs[0]], diff --git a/backend/src/db/write/template.py b/backend/src/db/write/template.py index 21547ee8..9e50898c 100644 --- a/backend/src/db/write/template.py +++ b/backend/src/db/write/template.py @@ -15,7 +15,7 @@ from src.db.models.team_year import TeamYearORM from src.db.models.year import YearORM -CUTOFF = 1000 +CUTOFF = 200 def _primary_key(orm_type: Type[TModelORM]) -> List[str]: diff --git a/backend/src/google/storage.py b/backend/src/google/storage.py index 2913150c..f52b660a 100644 --- a/backend/src/google/storage.py +++ b/backend/src/google/storage.py @@ -9,7 +9,7 @@ from google.cloud import storage from src.constants import CURR_YEAR, PROD -from src.data.utils import objs_type +from src.data.utils import nan_safe_eq, objs_type from src.db.functions import get_noteworthy_matches, get_upcoming_matches from src.db.read.event import get_events as get_events_db from src.db.read.team import get_teams as get_teams_db @@ -41,7 +41,7 @@ def upload_file_to_gcs(data: Any, object_name: str) -> None: def upload_files_to_gcs(data: List[Any], object_names: List[str]) -> None: with ThreadPoolExecutor() as executor: - executor.map(upload_file_to_gcs, data, object_names) + list(executor.map(upload_file_to_gcs, data, object_names)) def write_objs( @@ -76,15 +76,27 @@ def write_objs( upload_file_to_gcs(_read_events(year_obj, events), f"events/{year}") # event/{event.key} - orig_events = orig_objs[2] if orig_objs else {} - new_events = [e for e in events if str(e) != str(orig_events.get(e.pk(), ""))] - if len(new_events) > 0: + def group_by_event(source: Optional[objs_type]): event_to_matches = defaultdict(list) event_to_team_events = defaultdict(list) - for m in objs[4].values(): - event_to_matches[m.event].append(m) - for te in objs[3].values(): - event_to_team_events[te.event].append(te) + if source is not None: + for m in source[4].values(): + event_to_matches[m.event].append(m) + for te in source[3].values(): + event_to_team_events[te.event].append(te) + return event_to_matches, event_to_team_events + + event_to_matches, event_to_team_events = group_by_event(objs) + orig_matches, orig_team_events = group_by_event(orig_objs) + orig_events = orig_objs[2] if orig_objs else {} + new_events = [ + e + for e in events + if not nan_safe_eq(e, orig_events.get(e.pk())) + or not nan_safe_eq(event_to_matches[e.key], orig_matches[e.key]) + or not nan_safe_eq(event_to_team_events[e.key], orig_team_events[e.key]) + ] + if len(new_events) > 0: data = [] object_names = [] for event in new_events: diff --git a/backend/src/tba/read_tba.py b/backend/src/tba/read_tba.py index ae6b83ed..c9ca5664 100644 --- a/backend/src/tba/read_tba.py +++ b/backend/src/tba/read_tba.py @@ -20,6 +20,20 @@ def get_timestamp_from_str(date: str): return int(time.mktime(datetime.strptime(date, "%Y-%m-%d").timetuple())) +BREAKDOWN_GRACE_SECONDS = 24 * 3600 + + +def defer_missing_breakdown( + year: int, completed: bool, has_breakdown: bool, match_time: int, now_ts: int +) -> bool: + return ( + year >= 2016 + and completed + and not has_breakdown + and now_ts - match_time < BREAKDOWN_GRACE_SECONDS + ) + + def get_teams(cache: bool = True) -> List[TeamDict]: out: List[TeamDict] = [] for i in range(50): @@ -180,6 +194,8 @@ def get_event_matches( if type(matches) is bool: return out, new_etag + now_ts = int(datetime.now().timestamp()) + for match in matches: red_teams: List[str] = match["alliances"]["red"]["team_keys"] red_dq_teams: List[str] = match["alliances"]["red"]["dq_team_keys"] @@ -253,6 +269,18 @@ def get_event_matches( event_time, ) + if defer_missing_breakdown( + year, + status == MatchStatus.COMPLETED, + breakdown.get("red") is not None and breakdown.get("blue") is not None, + time, + now_ts, + ): + status = MatchStatus.UPCOMING + winner = None + red_score = None + blue_score = None + comp_level = CompLevel.INVALID if match["comp_level"] == "qm": comp_level = CompLevel.QUAL