From bbcc2b182de31e3e45121cd43be0244beb8b4ba1 Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Thu, 9 Jul 2026 22:07:15 -0700 Subject: [PATCH 1/8] Compare all fields when deciding whether to write DB rows The partial-update write filter compared objects via __str__ methods that cover only a subset of columns (TeamYear omits rank, percentile, and norm_epa; TeamEvent omits component EPAs). Drift in the omitted fields never reached the DB, so the public API disagreed with the website, which rebuilds its blobs from in-memory state every cycle (avgupta456#413). attrs-generated equality compares every column and benchmarks at ~0.1s per cycle across all 30K objects. --- backend/src/data/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/data/utils.py b/backend/src/data/utils.py index 669be1af..77bf29cb 100644 --- a/backend/src/data/utils.py +++ b/backend/src/data/utils.py @@ -55,7 +55,7 @@ def write_objs( orig_objs = create_objs(-1) def changed(curr: dict, prev: dict) -> list: - return [obj for obj in curr.values() if str(obj) != str(prev.get(obj.pk(), ""))] + return [obj for obj in curr.values() if obj != prev.get(obj.pk())] write_year_db( years=[objs[0]], From 06f8614614922f3b20bd4c56857d3210ced962c0 Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Thu, 9 Jul 2026 22:07:15 -0700 Subject: [PATCH 2/8] Reduce upsert batch size to fit CockroachDB message limit A 1000-row team_years upsert batch renders to ~15 MiB on average (measured; 18 MiB observed in practice), which exceeds CockroachDB's default 16 MiB sql.conn.max_read_buffer_message_size and fails with a ProtocolViolation. Full-content write gating makes large team_years batches routine, so cap batches at 250 rows (~4 MiB average). --- backend/src/db/write/template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/db/write/template.py b/backend/src/db/write/template.py index 21547ee8..82f7a88d 100644 --- a/backend/src/db/write/template.py +++ b/backend/src/db/write/template.py @@ -15,7 +15,7 @@ from src.db.models.team_year import TeamYearORM from src.db.models.year import YearORM -CUTOFF = 1000 +CUTOFF = 250 def _primary_key(orm_type: Type[TModelORM]) -> List[str]: From 3050e1f3b3587b01feb4b6aedd3779a7f3c53e41 Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Thu, 9 Jul 2026 22:07:15 -0700 Subject: [PATCH 3/8] Defer EPA processing of matches missing score breakdowns TBA sometimes posts a match score before the score breakdown. The match was ingested as completed with all component values imputed to zero, cratering component EPAs for its six teams until the breakdown arrived on a later cycle. For 2016+, treat such a match as upcoming (predictions still publish) until the breakdown arrives or the match is 24 hours old, so matches at events that never post breakdowns are still processed. --- backend/src/tba/read_tba.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/backend/src/tba/read_tba.py b/backend/src/tba/read_tba.py index ae6b83ed..a6dc7c3f 100644 --- a/backend/src/tba/read_tba.py +++ b/backend/src/tba/read_tba.py @@ -20,6 +20,20 @@ def get_timestamp_from_str(date: str): return int(time.mktime(datetime.strptime(date, "%Y-%m-%d").timetuple())) +BREAKDOWN_GRACE_SECONDS = 24 * 3600 + + +def defer_missing_breakdown( + year: int, completed: bool, has_breakdown: bool, match_time: int, now_ts: int +) -> bool: + return ( + year >= 2016 + and completed + and not has_breakdown + and now_ts - match_time < BREAKDOWN_GRACE_SECONDS + ) + + def get_teams(cache: bool = True) -> List[TeamDict]: out: List[TeamDict] = [] for i in range(50): @@ -180,6 +194,8 @@ def get_event_matches( if type(matches) is bool: return out, new_etag + now_ts = int(datetime.now().timestamp()) + for match in matches: red_teams: List[str] = match["alliances"]["red"]["team_keys"] red_dq_teams: List[str] = match["alliances"]["red"]["dq_team_keys"] @@ -253,6 +269,16 @@ def get_event_matches( event_time, ) + if defer_missing_breakdown( + year, + status == MatchStatus.COMPLETED, + breakdown.get("red") is not None and breakdown.get("blue") is not None, + time, + now_ts, + ): + status = MatchStatus.UPCOMING + winner = None + comp_level = CompLevel.INVALID if match["comp_level"] == "qm": comp_level = CompLevel.QUAL From 146b7636aadb2ece7653d94114dae2e2736698b3 Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Thu, 9 Jul 2026 22:07:15 -0700 Subject: [PATCH 4/8] Publish event blobs when their content changes The event/{key} blob upload was gated on Event.__str__, which contains no EPA fields, so during an event's registration window team EPAs kept updating in the DB while the event page blob stayed frozen for weeks (the stale-EPA reports on ChiefDelphi). Gate on the content of what the blob renders instead: the year, event, match, and team_event rows that _read_event serializes. Stale blobs from the old gate need a one-time partial=False run to resync. --- backend/src/google/storage.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/backend/src/google/storage.py b/backend/src/google/storage.py index 2913150c..f247b29b 100644 --- a/backend/src/google/storage.py +++ b/backend/src/google/storage.py @@ -76,15 +76,29 @@ def write_objs( upload_file_to_gcs(_read_events(year_obj, events), f"events/{year}") # event/{event.key} - orig_events = orig_objs[2] if orig_objs else {} - new_events = [e for e in events if str(e) != str(orig_events.get(e.pk(), ""))] - if len(new_events) > 0: + def group_by_event(source: Optional[objs_type]): event_to_matches = defaultdict(list) event_to_team_events = defaultdict(list) - for m in objs[4].values(): - event_to_matches[m.event].append(m) - for te in objs[3].values(): - event_to_team_events[te.event].append(te) + if source is not None: + for m in source[4].values(): + event_to_matches[m.event].append(m) + for te in source[3].values(): + event_to_team_events[te.event].append(te) + return event_to_matches, event_to_team_events + + event_to_matches, event_to_team_events = group_by_event(objs) + orig_matches, orig_team_events = group_by_event(orig_objs) + orig_events = orig_objs[2] if orig_objs else {} + year_changed = orig_objs is None or year_obj != orig_objs[0] + new_events = [ + e + for e in events + if year_changed + or e != orig_events.get(e.pk()) + or event_to_matches[e.key] != orig_matches[e.key] + or event_to_team_events[e.key] != orig_team_events[e.key] + ] + if len(new_events) > 0: data = [] object_names = [] for event in new_events: From 2c83989b924b50914a347b33066fd79366c5a4cf Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Fri, 10 Jul 2026 11:10:16 -0700 Subject: [PATCH 5/8] Gate event blobs on event content; make write/publish gates NaN-stable F1: dropping the year_changed short-circuit stops all ~215 event blobs from republishing every partial cycle (year stats churn constantly in-season). Event blobs now republish only when their own event/match/team_event content changes; the embedded year refreshes with them. F2: nan_safe_eq treats NaN==NaN so a NaN-bearing float field no longer forces a perpetual rewrite/republish. Also consume the upload executor results so per-blob upload failures surface. --- backend/src/data/utils.py | 25 +++++++++++++++++++++++-- backend/src/google/storage.py | 12 +++++------- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/backend/src/data/utils.py b/backend/src/data/utils.py index 77bf29cb..ad2f0246 100644 --- a/backend/src/data/utils.py +++ b/backend/src/data/utils.py @@ -1,5 +1,8 @@ from datetime import datetime -from typing import Dict, Optional, Tuple +import math +from typing import Any, Dict, Optional, Tuple + +import attr from src.db.functions import clear_year from src.db.models import ETag, Event, Match, TeamEvent, TeamYear, Year @@ -27,6 +30,24 @@ def create_objs(year: int) -> objs_type: return (Year(year=year), {}, {}, {}, {}, {}) +def _canonical(value: Any) -> Any: + if isinstance(value, float): + return "__nan__" if math.isnan(value) else value + if attr.has(type(value)): + return {f.name: _canonical(getattr(value, f.name)) for f in attr.fields(type(value))} + if isinstance(value, dict): + return {k: _canonical(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_canonical(v) for v in value] + return value + + +def nan_safe_eq(a: Any, b: Any) -> bool: + if a == b: + return True + return _canonical(a) == _canonical(b) + + def read_objs(year: int) -> objs_type: year_obj = get_year_db(year) if year_obj is None: @@ -55,7 +76,7 @@ def write_objs( orig_objs = create_objs(-1) def changed(curr: dict, prev: dict) -> list: - return [obj for obj in curr.values() if obj != prev.get(obj.pk())] + return [obj for obj in curr.values() if not nan_safe_eq(obj, prev.get(obj.pk()))] write_year_db( years=[objs[0]], diff --git a/backend/src/google/storage.py b/backend/src/google/storage.py index f247b29b..f52b660a 100644 --- a/backend/src/google/storage.py +++ b/backend/src/google/storage.py @@ -9,7 +9,7 @@ from google.cloud import storage from src.constants import CURR_YEAR, PROD -from src.data.utils import objs_type +from src.data.utils import nan_safe_eq, objs_type from src.db.functions import get_noteworthy_matches, get_upcoming_matches from src.db.read.event import get_events as get_events_db from src.db.read.team import get_teams as get_teams_db @@ -41,7 +41,7 @@ def upload_file_to_gcs(data: Any, object_name: str) -> None: def upload_files_to_gcs(data: List[Any], object_names: List[str]) -> None: with ThreadPoolExecutor() as executor: - executor.map(upload_file_to_gcs, data, object_names) + list(executor.map(upload_file_to_gcs, data, object_names)) def write_objs( @@ -89,14 +89,12 @@ def group_by_event(source: Optional[objs_type]): event_to_matches, event_to_team_events = group_by_event(objs) orig_matches, orig_team_events = group_by_event(orig_objs) orig_events = orig_objs[2] if orig_objs else {} - year_changed = orig_objs is None or year_obj != orig_objs[0] new_events = [ e for e in events - if year_changed - or e != orig_events.get(e.pk()) - or event_to_matches[e.key] != orig_matches[e.key] - or event_to_team_events[e.key] != orig_team_events[e.key] + if not nan_safe_eq(e, orig_events.get(e.pk())) + or not nan_safe_eq(event_to_matches[e.key], orig_matches[e.key]) + or not nan_safe_eq(event_to_team_events[e.key], orig_team_events[e.key]) ] if len(new_events) > 0: data = [] From d9522c487a754ab1270b36b2a428a0e330b9b55d Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Fri, 10 Jul 2026 11:10:16 -0700 Subject: [PATCH 6/8] Publish blobs before advancing the DB write baseline F3: writing storage before the DB means a failed/partial GCS upload raises and aborts the cycle before the DB baseline moves, so the next cycle re-diffs against the same baseline and retries the stale blobs instead of silently skipping them. --- backend/src/data/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/data/main.py b/backend/src/data/main.py index 6b0e64c3..7d0585d0 100644 --- a/backend/src/data/main.py +++ b/backend/src/data/main.py @@ -74,13 +74,13 @@ def process_year( objs = process_year_epa(objs, all_team_years) timer.print(str(year_num) + " EPA") - write_objs_db(year_num, objs, orig_objs if partial else None, not partial) - timer.print(str(year_num) + " Write DB") - if year_num == CURR_YEAR and not DISABLE_GCS: write_objs_storage(objs, orig_objs if partial else None) timer.print(str(year_num) + " Write Storage") + write_objs_db(year_num, objs, orig_objs if partial else None, not partial) + timer.print(str(year_num) + " Write DB") + return teams From 6bc09cafe3cfc09b0e6029e32105708a92a26796 Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Fri, 10 Jul 2026 11:10:16 -0700 Subject: [PATCH 7/8] Lower upsert CUTOFF to a worst-case-safe batch size F4: 200 x 69KB max row ~= 13.5 MiB < 16 MiB (CRDB default message buffer), vs 250 x 69KB ~= 17.25 MiB which could trip ProtocolViolation on a dense batch. --- backend/src/db/write/template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/db/write/template.py b/backend/src/db/write/template.py index 82f7a88d..9e50898c 100644 --- a/backend/src/db/write/template.py +++ b/backend/src/db/write/template.py @@ -15,7 +15,7 @@ from src.db.models.team_year import TeamYearORM from src.db.models.year import YearORM -CUTOFF = 250 +CUTOFF = 200 def _primary_key(orm_type: Type[TModelORM]) -> List[str]: From 5bfb71ce3d498137a2b8a81afd0aaf13a6ddcc82 Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Fri, 10 Jul 2026 11:10:16 -0700 Subject: [PATCH 8/8] Blank scores on matches demoted to Upcoming pending breakdown F5: a deferred match no longer carries a real final score while flagged Upcoming; aggregates already gate on Completed so they are unaffected. --- backend/src/tba/read_tba.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/src/tba/read_tba.py b/backend/src/tba/read_tba.py index a6dc7c3f..c9ca5664 100644 --- a/backend/src/tba/read_tba.py +++ b/backend/src/tba/read_tba.py @@ -278,6 +278,8 @@ def get_event_matches( ): status = MatchStatus.UPCOMING winner = None + red_score = None + blue_score = None comp_level = CompLevel.INVALID if match["comp_level"] == "qm":