Fix EPA consistency across pages

backend/src/data/main.py

@@ -74,13 +74,13 @@ def process_year(
     objs = process_year_epa(objs, all_team_years)
     timer.print(str(year_num) + " EPA")
 
-    write_objs_db(year_num, objs, orig_objs if partial else None, not partial)
-    timer.print(str(year_num) + " Write DB")
-
     if year_num == CURR_YEAR and not DISABLE_GCS:
         write_objs_storage(objs, orig_objs if partial else None)
         timer.print(str(year_num) + " Write Storage")
 
+    write_objs_db(year_num, objs, orig_objs if partial else None, not partial)
+    timer.print(str(year_num) + " Write DB")
+
     return teams
 
 

backend/src/data/utils.py

@@ -1,5 +1,8 @@
 from datetime import datetime
-from typing import Dict, Optional, Tuple
+import math
+from typing import Any, Dict, Optional, Tuple
+
+import attr
 
 from src.db.functions import clear_year
 from src.db.models import ETag, Event, Match, TeamEvent, TeamYear, Year
@@ -27,6 +30,24 @@ def create_objs(year: int) -> objs_type:
     return (Year(year=year), {}, {}, {}, {}, {})
 
 
+def _canonical(value: Any) -> Any:
+    if isinstance(value, float):
+        return "__nan__" if math.isnan(value) else value
+    if attr.has(type(value)):
+        return {f.name: _canonical(getattr(value, f.name)) for f in attr.fields(type(value))}
+    if isinstance(value, dict):
+        return {k: _canonical(v) for k, v in value.items()}
+    if isinstance(value, (list, tuple)):
+        return [_canonical(v) for v in value]
+    return value
+
+
+def nan_safe_eq(a: Any, b: Any) -> bool:
+    if a == b:
+        return True
+    return _canonical(a) == _canonical(b)
+
+
 def read_objs(year: int) -> objs_type:
     year_obj = get_year_db(year)
     if year_obj is None:
@@ -55,7 +76,7 @@ def write_objs(
         orig_objs = create_objs(-1)
 
     def changed(curr: dict, prev: dict) -> list:
-        return [obj for obj in curr.values() if str(obj) != str(prev.get(obj.pk(), ""))]
+        return [obj for obj in curr.values() if not nan_safe_eq(obj, prev.get(obj.pk()))]
 
     write_year_db(
         years=[objs[0]],

backend/src/db/write/template.py

@@ -15,7 +15,7 @@
 from src.db.models.team_year import TeamYearORM
 from src.db.models.year import YearORM
 
-CUTOFF = 1000
+CUTOFF = 200
 
 
 def _primary_key(orm_type: Type[TModelORM]) -> List[str]:

backend/src/google/storage.py

@@ -9,7 +9,7 @@
 from google.cloud import storage
 
 from src.constants import CURR_YEAR, PROD
-from src.data.utils import objs_type
+from src.data.utils import nan_safe_eq, objs_type
 from src.db.functions import get_noteworthy_matches, get_upcoming_matches
 from src.db.read.event import get_events as get_events_db
 from src.db.read.team import get_teams as get_teams_db
@@ -41,7 +41,7 @@ def upload_file_to_gcs(data: Any, object_name: str) -> None:
 
 def upload_files_to_gcs(data: List[Any], object_names: List[str]) -> None:
     with ThreadPoolExecutor() as executor:
-        executor.map(upload_file_to_gcs, data, object_names)
+        list(executor.map(upload_file_to_gcs, data, object_names))
 
 
 def write_objs(
@@ -76,15 +76,27 @@ def write_objs(
     upload_file_to_gcs(_read_events(year_obj, events), f"events/{year}")
 
     # event/{event.key}
-    orig_events = orig_objs[2] if orig_objs else {}
-    new_events = [e for e in events if str(e) != str(orig_events.get(e.pk(), ""))]
-    if len(new_events) > 0:
+    def group_by_event(source: Optional[objs_type]):
         event_to_matches = defaultdict(list)
         event_to_team_events = defaultdict(list)
-        for m in objs[4].values():
-            event_to_matches[m.event].append(m)
-        for te in objs[3].values():
-            event_to_team_events[te.event].append(te)
+        if source is not None:
+            for m in source[4].values():
+                event_to_matches[m.event].append(m)
+            for te in source[3].values():
+                event_to_team_events[te.event].append(te)
+        return event_to_matches, event_to_team_events
+
+    event_to_matches, event_to_team_events = group_by_event(objs)
+    orig_matches, orig_team_events = group_by_event(orig_objs)
+    orig_events = orig_objs[2] if orig_objs else {}
+    new_events = [
+        e
+        for e in events
+        if not nan_safe_eq(e, orig_events.get(e.pk()))
+        or not nan_safe_eq(event_to_matches[e.key], orig_matches[e.key])
+        or not nan_safe_eq(event_to_team_events[e.key], orig_team_events[e.key])
+    ]
+    if len(new_events) > 0:
         data = []
         object_names = []
         for event in new_events:

backend/src/tba/read_tba.py

@@ -20,6 +20,20 @@ def get_timestamp_from_str(date: str):
     return int(time.mktime(datetime.strptime(date, "%Y-%m-%d").timetuple()))
 
 
+BREAKDOWN_GRACE_SECONDS = 24 * 3600
+
+
+def defer_missing_breakdown(
+    year: int, completed: bool, has_breakdown: bool, match_time: int, now_ts: int
+) -> bool:
+    return (
+        year >= 2016
+        and completed
+        and not has_breakdown
+        and now_ts - match_time < BREAKDOWN_GRACE_SECONDS
+    )
+
+
 def get_teams(cache: bool = True) -> List[TeamDict]:
     out: List[TeamDict] = []
     for i in range(50):
@@ -180,6 +194,8 @@ def get_event_matches(
     if type(matches) is bool:
         return out, new_etag
 
+    now_ts = int(datetime.now().timestamp())
+
     for match in matches:
         red_teams: List[str] = match["alliances"]["red"]["team_keys"]
         red_dq_teams: List[str] = match["alliances"]["red"]["dq_team_keys"]
@@ -253,6 +269,18 @@ def get_event_matches(
             event_time,
         )
 
+        if defer_missing_breakdown(
+            year,
+            status == MatchStatus.COMPLETED,
+            breakdown.get("red") is not None and breakdown.get("blue") is not None,
+            time,
+            now_ts,
+        ):
+            status = MatchStatus.UPCOMING
+            winner = None
+            red_score = None
+            blue_score = None
+
         comp_level = CompLevel.INVALID
         if match["comp_level"] == "qm":
             comp_level = CompLevel.QUAL