From eabcdeca87f83492a5497286215ee6ca7356c6ef Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Thu, 9 Jul 2026 21:59:05 -0700 Subject: [PATCH 1/8] Extract _read_team and _read_team_year site helpers Pull the /team/{num} and /team/{num}/{year} payload shaping out of the route bodies into reusable helpers (verified byte-identical responses) so the blob export and historical backfill can produce payloads identical to the API. Adds the HIST_EPOCH constant that versions the immutable historical blob path. --- backend/src/constants.py | 1 + backend/src/site/team.py | 48 +++++++++++++++++++++++++--------------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/backend/src/constants.py b/backend/src/constants.py index 3e07f453..34ac62f6 100644 --- a/backend/src/constants.py +++ b/backend/src/constants.py @@ -36,6 +36,7 @@ CURR_YEAR = 2026 DISABLE_GCS = False +HIST_EPOCH = 1 # MISC diff --git a/backend/src/site/team.py b/backend/src/site/team.py index 301a02ef..14e69f0e 100644 --- a/backend/src/site/team.py +++ b/backend/src/site/team.py @@ -34,18 +34,8 @@ async def read_all_teams(response: Response, no_cache: bool = False) -> Any: return _read_all_teams(teams) -@router.get("/team/{team_num}") -@async_fail_gracefully_singular -async def read_team_years( - response: Response, team_num: int, no_cache: bool = False -) -> Any: - team: Optional[Team] = await get_team_cached(team=team_num, no_cache=no_cache) - if team is None: - raise Exception("Team not found") - - team_years: List[TeamYear] = await get_team_years_cached( - team=team_num, no_cache=True - ) +def _read_team(team: Team, team_years: List[TeamYear]) -> Dict[str, Any]: + team_years = sorted(team_years, key=lambda x: x.year) team_year_stats = [ { "year": x.year, @@ -64,12 +54,26 @@ async def read_team_years( for x in team_years ] - out = { + return { "team": team.to_dict(), "team_years": team_year_stats, } - return out + +@router.get("/team/{team_num}") +@async_fail_gracefully_singular +async def read_team_years( + response: Response, team_num: int, no_cache: bool = False +) -> Any: + team: Optional[Team] = await get_team_cached(team=team_num, no_cache=no_cache) + if team is None: + raise Exception("Team not found") + + team_years: List[TeamYear] = await get_team_years_cached( + team=team_num, no_cache=True + ) + + return _read_team(team, team_years) @router.get("/team/{team_num}/{year}") @@ -109,6 +113,16 @@ async def read_team_year( team=team_num, year=year, no_cache=no_cache ) + return _read_team_year(year_obj, team, team_year, team_events, matches) + + +def _read_team_year( + year_obj: Year, + team: Team, + team_year: TeamYear, + team_events: List[TeamEvent], + matches: List[Match], +) -> Dict[str, Any]: matches = sorted(matches, key=lambda x: x.time) event_times: Dict[str, Optional[int]] = {e.event: None for e in team_events} @@ -123,13 +137,11 @@ async def read_team_year( team_matches = sorted(team_year.team_matches or [], key=lambda x: x["time"]) - out = { + return { "year": year_obj.to_dict(), "team": team.to_dict(), "team_year": team_year.to_dict(), "team_events": [x.to_dict() for x in team_events], "matches": [x.to_dict() for x in matches], - "team_matches": [{"team": team_num, **tm} for tm in team_matches], + "team_matches": [{"team": team.team, **tm} for tm in team_matches], } - - return out From 202d21ee1b5099594c5aac4202b30d46f776855c Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Thu, 9 Jul 2026 21:59:05 -0700 Subject: [PATCH 2/8] Publish blobs as a versioned set behind a manifest write_objs now renders the current-year blob set, uploads only blobs whose content hash changed (copy-on-write, to immutable content-addressed v2/{path}.{hash} keys with long-lived Cache-Control), and writes manifest.json last so readers always resolve a complete old or new set, never a torn mix. Adds a team/{num} blob per active team. Legacy unversioned paths are still written each cycle for compatibility with the deployed frontend. The lossy str(Event) upload gate is replaced by the content hash, which also stops event blobs going stale between match updates. --- backend/src/google/publish.py | 96 +++++++++++++++++++ backend/src/google/storage.py | 173 ++++++++++++++++++++++++---------- 2 files changed, 221 insertions(+), 48 deletions(-) create mode 100644 backend/src/google/publish.py diff --git a/backend/src/google/publish.py b/backend/src/google/publish.py new file mode 100644 index 00000000..f971fa88 --- /dev/null +++ b/backend/src/google/publish.py @@ -0,0 +1,96 @@ +import hashlib +import json +from dataclasses import dataclass, field +from typing import Dict, Optional + +MANIFEST_OBJECT = "manifest.json" +VERSION_PREFIX = "v2" +HIST_PREFIX = "hist" +HASH_LEN = 12 +SCHEMA = 1 + + +def content_hash(data: bytes) -> str: + return hashlib.sha256(data).hexdigest()[:HASH_LEN] + + +def versioned_key(logical_path: str, digest: str) -> str: + return f"{VERSION_PREFIX}/{logical_path}.{digest}" + + +def historical_key(epoch: int, logical_path: str) -> str: + return f"{HIST_PREFIX}/{epoch}/{logical_path}" + + +@dataclass +class Manifest: + schema: int = SCHEMA + cycle: str = "" + hist_epoch: int = 1 + blobs: Dict[str, str] = field(default_factory=dict) + + def hash_for(self, logical_path: str) -> Optional[str]: + key = self.blobs.get(logical_path) + if key is None: + return None + return key.rsplit(".", 1)[-1] + + def to_json(self) -> str: + return json.dumps( + { + "schema": self.schema, + "cycle": self.cycle, + "hist_epoch": self.hist_epoch, + "blobs": self.blobs, + }, + sort_keys=True, + ) + + @classmethod + def from_json(cls, raw: object) -> "Manifest": + if isinstance(raw, (bytes, bytearray)): + raw = bytes(raw).decode("utf-8") + if isinstance(raw, str): + data = json.loads(raw) + elif isinstance(raw, dict): + data = raw + else: + raise TypeError(f"Cannot parse manifest from {type(raw)!r}") + return cls( + schema=int(data.get("schema", SCHEMA)), + cycle=str(data.get("cycle", "")), + hist_epoch=int(data.get("hist_epoch", 1)), + blobs=dict(data.get("blobs", {})), + ) + + +@dataclass +class UploadPlan: + uploads: Dict[str, bytes] + legacy_uploads: Dict[str, bytes] + manifest: Manifest + + +def plan_uploads( + rendered: Dict[str, bytes], + prev: Optional[Manifest], + cycle: str, + hist_epoch: Optional[int] = None, +) -> UploadPlan: + prev = prev or Manifest() + if hist_epoch is None: + hist_epoch = prev.hist_epoch + + uploads: Dict[str, bytes] = {} + legacy_uploads: Dict[str, bytes] = {} + blobs: Dict[str, str] = dict(prev.blobs) + + for logical_path, data in rendered.items(): + digest = content_hash(data) + blobs[logical_path] = versioned_key(logical_path, digest) + if prev.hash_for(logical_path) != digest: + uploads[versioned_key(logical_path, digest)] = data + legacy_uploads[logical_path] = data + + manifest = Manifest(schema=SCHEMA, cycle=cycle, hist_epoch=hist_epoch, blobs=blobs) + return UploadPlan(uploads=uploads, legacy_uploads=legacy_uploads, manifest=manifest) diff --git a/backend/src/google/storage.py b/backend/src/google/storage.py index 2913150c..eb077f74 100644 --- a/backend/src/google/storage.py +++ b/backend/src/google/storage.py @@ -1,25 +1,35 @@ from collections import defaultdict from concurrent.futures import ThreadPoolExecutor - -# from datetime import datetime +from datetime import datetime, timezone import json -from typing import Any, List, Optional +from typing import Any, Dict, List, Optional import zlib from google.cloud import storage -from src.constants import CURR_YEAR, PROD +from src.constants import CURR_YEAR, HIST_EPOCH, PROD from src.data.utils import objs_type from src.db.functions import get_noteworthy_matches, get_upcoming_matches from src.db.read.event import get_events as get_events_db from src.db.read.team import get_teams as get_teams_db +from src.db.read.team_year import get_team_years as get_team_years_db +from src.google.publish import ( + MANIFEST_OBJECT, + Manifest, + UploadPlan, + historical_key, + plan_uploads, +) from src.site.event import _read_all_events, _read_events, _read_event from src.site.match import _read_noteworthy_matches, _read_upcoming_matches +from src.site.team import _read_all_teams, _read_team from src.site.team_year import _read_team_years -from src.site.team import _read_all_teams BUCKET_NAME = "site_v1" if PROD else "site_dev_v1" +IMMUTABLE_CACHE = "public, max-age=31536000, immutable" +MANIFEST_CACHE = "public, max-age=60" + def compress(data: Any) -> bytes: # start = datetime.now() @@ -31,81 +41,133 @@ def compress(data: Any) -> bytes: return compressed +def _bucket() -> Any: + return storage.Client().bucket(BUCKET_NAME) + + +def _upload_bytes( + bucket: Any, object_name: str, data: bytes, cache_control: Optional[str] +) -> None: + blob = bucket.blob(object_name) + if cache_control is not None: + blob.cache_control = cache_control + blob.upload_from_string(data, "application/octet-stream") + + def upload_file_to_gcs(data: Any, object_name: str) -> None: - # start = datetime.now() - storage.Client().bucket(BUCKET_NAME).blob(object_name).upload_from_string( - compress(data), "application/octet-stream" - ) - # print(f"Uploaded {object_name} to GCS in {datetime.now() - start}") + _upload_bytes(_bucket(), object_name, compress(data), None) + + +def read_manifest() -> Optional[Manifest]: + try: + raw = _bucket().blob(MANIFEST_OBJECT).download_as_bytes() + except Exception: + return None + try: + return Manifest.from_json(raw) + except Exception: + return None + + +def write_manifest(manifest: Manifest, bucket: Any = None) -> None: + bucket = bucket or _bucket() + blob = bucket.blob(MANIFEST_OBJECT) + blob.cache_control = MANIFEST_CACHE + blob.upload_from_string(manifest.to_json().encode("utf-8"), "application/json") + + +def _publish(plan: UploadPlan) -> None: + bucket = _bucket() + + jobs: List[Any] = [] + for versioned, data in plan.uploads.items(): + jobs.append((versioned, data, IMMUTABLE_CACHE)) + for logical, data in plan.legacy_uploads.items(): + jobs.append((logical, data, None)) + if jobs: + with ThreadPoolExecutor() as executor: + list(executor.map(lambda job: _upload_bytes(bucket, *job), jobs)) -def upload_files_to_gcs(data: List[Any], object_names: List[str]) -> None: - with ThreadPoolExecutor() as executor: - executor.map(upload_file_to_gcs, data, object_names) + # manifest is written last so readers always resolve a complete blob set + write_manifest(plan.manifest, bucket) def write_objs( objs: objs_type, orig_objs: Optional[objs_type] = None, ) -> None: + year = CURR_YEAR + year_obj = objs[0] + + rendered: Dict[str, bytes] = {} + + def add(object_name: str, data: Any) -> None: + rendered[object_name] = compress(data) + # teams/all teams = get_teams_db() - upload_file_to_gcs(_read_all_teams(teams), "teams/all") + add("teams/all", _read_all_teams(teams)) # team_years/{CURR_YEAR} - year = CURR_YEAR - year_obj = objs[0] team_years = list(objs[1].values()) - upload_file_to_gcs( - _read_team_years(year, year_obj, team_years), f"team_years/{year}" - ) + add(f"team_years/{year}", _read_team_years(year, year_obj, team_years)) # team_years/{CURR_YEAR}?limit=100&metric=epa - team_years = sorted(team_years, key=lambda x: -x.epa)[:100] - upload_file_to_gcs( - _read_team_years(year, year_obj, team_years), + top_team_years = sorted(team_years, key=lambda x: -x.epa)[:100] + add( f"team_years/{year}.limit=100.metric=epa", + _read_team_years(year, year_obj, top_team_years), ) # events/all - events = get_events_db() - upload_file_to_gcs(_read_all_events(events), "events/all") + add("events/all", _read_all_events(get_events_db())) # events/{CURR_YEAR} events = list(objs[2].values()) - upload_file_to_gcs(_read_events(year_obj, events), f"events/{year}") + add(f"events/{year}", _read_events(year_obj, events)) # event/{event.key} - orig_events = orig_objs[2] if orig_objs else {} - new_events = [e for e in events if str(e) != str(orig_events.get(e.pk(), ""))] - if len(new_events) > 0: - event_to_matches = defaultdict(list) - event_to_team_events = defaultdict(list) - for m in objs[4].values(): - event_to_matches[m.event].append(m) - for te in objs[3].values(): - event_to_team_events[te.event].append(te) - data = [] - object_names = [] - for event in new_events: - matches = event_to_matches.get(event.key, []) - team_events = event_to_team_events.get(event.key, []) - data.append(_read_event(year_obj, event, matches, team_events)) - object_names.append(f"event/{event.key}") - upload_files_to_gcs(data, object_names) + event_to_matches = defaultdict(list) + event_to_team_events = defaultdict(list) + for m in objs[4].values(): + event_to_matches[m.event].append(m) + for te in objs[3].values(): + event_to_team_events[te.event].append(te) + for event in events: + add( + f"event/{event.key}", + _read_event( + year_obj, + event, + event_to_matches.get(event.key, []), + event_to_team_events.get(event.key, []), + ), + ) + # team_to_events team_to_events = defaultdict(list) for team_event in objs[3].values(): team_to_events[team_event.team].append(team_event.event) - upload_file_to_gcs(team_to_events, "team_to_events") + add("team_to_events", team_to_events) + + # team/{team.team} + all_team_years = get_team_years_db() + team_years_by_team: Dict[int, List[Any]] = defaultdict(list) + for ty in all_team_years: + team_years_by_team[ty.team].append(ty) + teams_by_num = {t.team: t for t in teams} + for num in {ty.team for ty in team_years}: + team_obj = teams_by_num.get(num) + if team_obj is None: + continue + add(f"team/{num}", _read_team(team_obj, team_years_by_team.get(num, []))) # noteworthy_matches/{year} noteworthy_matches = get_noteworthy_matches( year=year, country=None, state=None, district=None, elim=None, week=None ) - upload_file_to_gcs( - _read_noteworthy_matches(noteworthy_matches), f"noteworthy_matches/{year}" - ) + add(f"noteworthy_matches/{year}", _read_noteworthy_matches(noteworthy_matches)) # upcoming_matches?limit=20&metric={predicted_time | max_epa | sum_epa | diff_epa} for metric in ["predicted_time", "max_epa", "sum_epa", "diff_epa"]: @@ -118,9 +180,24 @@ def write_objs( limit=20, metric=metric, ) - upload_file_to_gcs( - _read_upcoming_matches(upcoming_matches), + add( f"upcoming_matches.limit=20.metric={metric}", + _read_upcoming_matches(upcoming_matches), ) + prev = read_manifest() + cycle = datetime.now(timezone.utc).isoformat() + plan = plan_uploads(rendered, prev, cycle, hist_epoch=HIST_EPOCH) + _publish(plan) + return + + +def upload_historical(logical_path: str, data: Any, bucket: Any = None) -> bool: + bucket = bucket or _bucket() + blob = bucket.blob(historical_key(HIST_EPOCH, logical_path)) + if blob.exists(): + return False + blob.cache_control = IMMUTABLE_CACHE + blob.upload_from_string(compress(data), "application/octet-stream") + return True From f2f09cd958babdd165d22476a08ae70e19acffd8 Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Thu, 9 Jul 2026 21:59:05 -0700 Subject: [PATCH 3/8] Add historical blob backfill script One-time script exporting team_years/{year}, events/{year}, event/{key} and team/{num}/{year} for every past season (2021 skipped) to the immutable hist/{HIST_EPOCH} path, using the shared site _read_* helpers. Idempotent per object and resumable via a bucket-side progress checkpoint. --- backend/backfill_blobs.py | 156 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 backend/backfill_blobs.py diff --git a/backend/backfill_blobs.py b/backend/backfill_blobs.py new file mode 100644 index 00000000..f4427013 --- /dev/null +++ b/backend/backfill_blobs.py @@ -0,0 +1,156 @@ +"""One-time historical blob backfill. + + python backfill_blobs.py # all past years + python backfill_blobs.py 2018 2019 # specific years + python backfill_blobs.py --force # ignore the progress checkpoint +""" + +import json +import sys +from collections import defaultdict +from typing import Dict, List, Set + +from src.constants import CURR_YEAR, HIST_EPOCH +from src.db.read.event import get_events as get_events_db +from src.db.read.match import get_matches as get_matches_db +from src.db.read.team import get_teams as get_teams_db +from src.db.read.team_event import get_team_events as get_team_events_db +from src.db.read.team_year import get_team_years as get_team_years_db +from src.db.read.year import get_year as get_year_db +from src.google.publish import Manifest +from src.google.storage import ( + _bucket, + read_manifest, + upload_historical, + write_manifest, +) +from src.site.event import _read_event, _read_events +from src.site.team import _read_team_year +from src.site.team_year import _read_team_years + +PROGRESS_OBJECT = "backfill/progress.json" +SKIP_YEARS = {2021} + + +def _read_progress(bucket) -> Dict: + try: + raw = bucket.blob(PROGRESS_OBJECT).download_as_bytes() + return json.loads(raw) + except Exception: + return {"epoch": HIST_EPOCH, "completed_years": []} + + +def _write_progress(bucket, progress: Dict) -> None: + blob = bucket.blob(PROGRESS_OBJECT) + blob.cache_control = "no-cache" + blob.upload_from_string(json.dumps(progress).encode("utf-8"), "application/json") + + +def backfill_year(year: int, bucket) -> int: + year_obj = get_year_db(year) + if year_obj is None: + print(f" {year}: no Year row, skipping") + return 0 + + team_years = get_team_years_db(year=year) + events = get_events_db(year=year) + matches = get_matches_db(year=year) + team_events = get_team_events_db(year=year) + teams_by_num = {t.team: t for t in get_teams_db()} + + matches_by_event: Dict[str, List] = defaultdict(list) + team_events_by_event: Dict[str, List] = defaultdict(list) + for m in matches: + matches_by_event[m.event].append(m) + for te in team_events: + team_events_by_event[te.event].append(te) + + matches_by_team: Dict[int, List] = defaultdict(list) + for m in matches: + for num in set(m.get_red()) | set(m.get_blue()): + matches_by_team[num].append(m) + team_events_by_team: Dict[int, List] = defaultdict(list) + for te in team_events: + team_events_by_team[te.team].append(te) + + written = 0 + + if upload_historical( + f"team_years/{year}", _read_team_years(year, year_obj, team_years), bucket + ): + written += 1 + if upload_historical(f"events/{year}", _read_events(year_obj, events), bucket): + written += 1 + + for event in events: + payload = _read_event( + year_obj, + event, + matches_by_event.get(event.key, []), + team_events_by_event.get(event.key, []), + ) + if upload_historical(f"event/{event.key}", payload, bucket): + written += 1 + + for ty in team_years: + team_obj = teams_by_num.get(ty.team) + if team_obj is None: + continue + payload = _read_team_year( + year_obj, + team_obj, + ty, + team_events_by_team.get(ty.team, []), + matches_by_team.get(ty.team, []), + ) + if upload_historical(f"team/{ty.team}/{year}", payload, bucket): + written += 1 + + return written + + +def _ensure_manifest_epoch(bucket) -> None: + manifest = read_manifest() + if manifest is None: + manifest = Manifest(cycle="backfill", hist_epoch=HIST_EPOCH, blobs={}) + else: + manifest.hist_epoch = HIST_EPOCH + write_manifest(manifest, bucket) + + +def main(argv: List[str]) -> None: + force = "--force" in argv + year_args = [int(a) for a in argv if a.isdigit()] + + if year_args: + years = year_args + else: + years = [y for y in range(2002, CURR_YEAR) if y not in SKIP_YEARS] + + bucket = _bucket() + progress = _read_progress(bucket) + if progress.get("epoch") != HIST_EPOCH: + progress = {"epoch": HIST_EPOCH, "completed_years": []} + completed: Set[int] = set(progress.get("completed_years", [])) + + total = 0 + for year in years: + if year in SKIP_YEARS: + continue + if year in completed and not force: + print(f"{year}: already complete, skipping") + continue + print(f"{year}: backfilling...") + written = backfill_year(year, bucket) + total += written + completed.add(year) + progress["completed_years"] = sorted(completed) + _write_progress(bucket, progress) + print(f"{year}: {written} objects written") + + _ensure_manifest_epoch(bucket) + print(f"Done. {total} objects written this run (epoch {HIST_EPOCH}).") + + +if __name__ == "__main__": + main(sys.argv[1:]) From 032db396045eb36a199fa23dca662433bcee720d Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Thu, 9 Jul 2026 21:59:05 -0700 Subject: [PATCH 4/8] Serve team and event pages bucket-first via the manifest Blob URLs now resolve through manifest.json: immutable versioned URLs for the current-year set, epoch-prefixed paths for historical years, with no per-request cache-buster. When no manifest exists (backend not yet deployed) fetches fall back to the legacy path plus ?t= buster, and the backend keeps writing legacy paths, so either deploy order works. getTeam, getYearTeamYears, and historical getTeamYear become bucket-first; the two team-match fetches read the event blob and team-year payload instead of API-only endpoints. Expired IndexedDB entries are retained and served as a last resort when both bucket and API fail, instead of rendering an empty page. --- frontend/src/api/event.tsx | 8 +-- frontend/src/api/storage.tsx | 114 ++++++++++++++++++++++++++++++----- frontend/src/api/team.tsx | 23 ++----- frontend/src/api/teams.tsx | 9 ++- 4 files changed, 111 insertions(+), 43 deletions(-) diff --git a/frontend/src/api/event.tsx b/frontend/src/api/event.tsx index 44e783db..9718fffb 100644 --- a/frontend/src/api/event.tsx +++ b/frontend/src/api/event.tsx @@ -13,8 +13,8 @@ export async function getTeamEventTeamMatches( team: number, event: string ): Promise { - const urlSuffix = `/event/${event}/team_matches/${team}`; - const storageKey = `event_${event}_team_matches_${team}_${version}`; - - return query(storageKey, urlSuffix, false, 0, 60); // 1 minute + const eventData = await getEvent(event); + return ((eventData?.team_matches ?? []) as APITeamMatch[]).filter( + (tm: any) => tm.team === team + ); } diff --git a/frontend/src/api/storage.tsx b/frontend/src/api/storage.tsx index d5b17b19..78f8afae 100644 --- a/frontend/src/api/storage.tsx +++ b/frontend/src/api/storage.tsx @@ -1,4 +1,4 @@ -import { del, get, set } from "idb-keyval"; +import { get, set } from "idb-keyval"; import pako from "pako"; import { BACKEND_URL, BUCKET_URL, DISABLE_GCS } from "../constants"; @@ -6,6 +6,55 @@ import { log, round } from "../utils"; export const version = "v4"; +const MANIFEST_TTL_MS = 60 * 1000; + +interface Manifest { + schema?: number; + cycle?: string; + hist_epoch?: number; + blobs?: { [logicalPath: string]: string }; +} + +let manifestPromise: { promise: Promise; fetchedAt: number } | null = null; + +async function fetchManifest(): Promise { + try { + const res = await fetch(`${BUCKET_URL}/manifest.json`, { next: { revalidate: 0 } }); + if (res.ok) { + return (await res.json()) as Manifest; + } + } catch (e) { + log("manifest fetch failed", e); + } + return null; +} + +async function getManifest(): Promise { + if (DISABLE_GCS) return null; + const now = Date.now(); + if (!manifestPromise || now - manifestPromise.fetchedAt > MANIFEST_TTL_MS) { + manifestPromise = { promise: fetchManifest(), fetchedAt: now }; + } + return manifestPromise.promise; +} + +function toLogicalPath(apiPath: string): string { + return apiPath.replace("?", ".").replace("&", ".").replace(/^\//, ""); +} + +function resolveBucketUrl(logicalPath: string, manifest: Manifest | null): string { + if (manifest) { + const versioned = manifest.blobs?.[logicalPath]; + if (versioned) { + return `${BUCKET_URL}/${versioned}`; + } + if (manifest.hist_epoch != null) { + return `${BUCKET_URL}/hist/${manifest.hist_epoch}/${logicalPath}`; + } + } + return `${BUCKET_URL}/${logicalPath}?t=${Date.now() / 1000 / 60}`; +} + async function setWithExpiry(key: string, value: any, ttl: number) { const now = new Date(); @@ -24,8 +73,14 @@ async function getWithExpiry(key: string) { } const now = new Date(); if (now.getTime() > expiry) { - await del(`${key}_expiry`); - await del(key); + return null; + } + return get(key); +} + +async function getStale(key: string) { + const expiry = await get(`${key}_expiry`); + if (!expiry) { return null; } return get(key); @@ -37,6 +92,27 @@ export function decompress(buffer: any) { return data; } +const bucketInFlight: { [logicalPath: string]: Promise } = {}; + +async function fetchBucketDataImpl(logicalPath: string): Promise { + const manifest = await getManifest(); + const url = resolveBucketUrl(logicalPath, manifest); + const res = await fetch(url, { next: { revalidate: 0 } }); + if (!res.ok) { + throw new Error(`Failed to fetch from bucket: ${res.status}`); + } + return decompress(await res.arrayBuffer()); +} + +export async function fetchBucketData(logicalPath: string): Promise { + if (!bucketInFlight[logicalPath]) { + bucketInFlight[logicalPath] = fetchBucketDataImpl(logicalPath).finally(() => { + delete bucketInFlight[logicalPath]; + }); + } + return bucketInFlight[logicalPath]; +} + async function query( storageKey: string, apiPath: string, @@ -57,25 +133,25 @@ async function query( if (!checkBucket || DISABLE_GCS) { throw new Error("Skip bucket check"); } - const fileName = apiPath.replace("?", ".").replace("&", "."); - const res = await fetch(`${BUCKET_URL}${fileName}?t=${Date.now() / 1000 / 60}`, { - next: { revalidate: 0 }, - headers: { - "Cache-Control": "no-cache", - "Content-Type": "application/octet-stream", - }, - }); - log(`${fileName} (bucket) took ${round(performance.now() - start, 0)}ms`); + const logicalPath = toLogicalPath(apiPath); + const manifest = await getManifest(); + const url = resolveBucketUrl(logicalPath, manifest); + const res = await fetch(url, { next: { revalidate: 0 } }); + log(`${logicalPath} (bucket) took ${round(performance.now() - start, 0)}ms`); if (res.ok) { buffer = decompress(await res.arrayBuffer()); } else { throw new Error(`Failed to fetch from bucket: ${res.status}`); } } catch (e) { - const res = await fetch(`${BACKEND_URL}${apiPath}`, { next: { revalidate: 0 } }); - log(`${apiPath} (backend) took ${round(performance.now() - start, 0)}ms`); - if (res.ok) { - buffer = await res.json(); + try { + const res = await fetch(`${BACKEND_URL}${apiPath}`, { next: { revalidate: 0 } }); + log(`${apiPath} (backend) took ${round(performance.now() - start, 0)}ms`); + if (res.ok) { + buffer = await res.json(); + } + } catch (apiErr) { + log(`${apiPath} (backend) failed`, apiErr); } } @@ -83,6 +159,12 @@ async function query( await setWithExpiry(storageKey, buffer, expiry); return buffer; } + + const stale = await getStale(storageKey); + if (stale && (minLength === 0 || stale?.length > minLength)) { + log(`Served stale cache: ${storageKey}`); + return stale; + } } export default query; diff --git a/frontend/src/api/team.tsx b/frontend/src/api/team.tsx index a0a5d54b..8c590501 100644 --- a/frontend/src/api/team.tsx +++ b/frontend/src/api/team.tsx @@ -1,15 +1,15 @@ -import { BUCKET_URL, CURR_YEAR } from "../constants"; +import { CURR_YEAR } from "../constants"; import { APITeam, APITeamEvent } from "../types/api"; import { TeamYearData, TeamYearRedirect } from "../types/data"; import { getEvent } from "./event"; -import query, { decompress, version } from "./storage"; +import query, { fetchBucketData, version } from "./storage"; import { getYearTeamYears } from "./teams"; export async function getTeam(team: number): Promise<{ team: APITeam; team_years: any[] }> { const urlSuffix = `/team/${team}`; const storageKey = `team_${team}_${version}`; - return query(storageKey, urlSuffix, false, 0, 60); // 1 minute + return query(storageKey, urlSuffix, true, 0, 60); // 1 minute } export async function getTeamYear( @@ -19,26 +19,13 @@ export async function getTeamYear( const urlSuffix = `/team/${team}/${year}`; const storageKey = `team_${team}_${year}_${version}`; - const readBucket = async (url: string) => { - const res = await fetch(url, { - next: { revalidate: 0 }, - }); - if (res.ok) { - const buffer = await res.arrayBuffer(); - const data = decompress(buffer); - return data; - } else { - throw new Error(`Failed to fetch from bucket: ${res.status}`); - } - }; - try { // try to reconstruct output from team, team_to_events, and events if (year !== CURR_YEAR) { throw new Error("Not current year"); } const [teamToEvents, teamData, teamYearData] = await Promise.all([ - readBucket(`${BUCKET_URL}/team_to_events`) as Promise<{ [key: number]: string[] }>, + fetchBucketData("team_to_events") as Promise<{ [key: number]: string[] }>, getTeam(team), getYearTeamYears(year), ]); @@ -85,7 +72,7 @@ export async function getTeamYear( team_matches: teamMatches, }; } catch (e) { - return query(storageKey, urlSuffix, false, 0, year === CURR_YEAR ? 60 : 60 * 60); // 1 minute / 1 hour + return query(storageKey, urlSuffix, true, 0, year === CURR_YEAR ? 60 : 60 * 60); // 1 minute / 1 hour } } diff --git a/frontend/src/api/teams.tsx b/frontend/src/api/teams.tsx index a4b6b633..2df7b491 100644 --- a/frontend/src/api/teams.tsx +++ b/frontend/src/api/teams.tsx @@ -1,6 +1,7 @@ import { CURR_YEAR } from "../constants"; import { APITeamMatch, APITeamYear, APIYear } from "../types/api"; import query, { version } from "./storage"; +import { getTeamYear } from "./team"; export async function getYearTeamYears( year: number, @@ -17,15 +18,13 @@ export async function getYearTeamYears( } storageKey += "_v3"; - return query(storageKey, urlSuffix, year === CURR_YEAR, 0, year === CURR_YEAR ? 60 : 60 * 60); // 1 minute / 1 hour + return query(storageKey, urlSuffix, true, 0, year === CURR_YEAR ? 60 : 60 * 60); // 1 minute / 1 hour } export async function getTeamYearTeamMatches( year: number, teamNum: number ): Promise { - const urlSuffix = `/team_year/${year}/${teamNum}/matches`; - const storageKey = `team_year_matches_${year}_${teamNum}_${version}`; - - return query(storageKey, urlSuffix, false, 0, year === CURR_YEAR ? 60 : 60 * 60); // 1 minute / 1 hour + const teamYear = await getTeamYear(teamNum, year); + return ((teamYear as any)?.team_matches ?? []) as APITeamMatch[]; } From ca35b7cb04feefed15cf0276a99a9781094df4bf Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Fri, 10 Jul 2026 11:16:36 -0700 Subject: [PATCH 5/8] Gate event blobs on event content so year drift alone stops republishing F1/F2: the event blob embeds the full year object, whose stats churn every in-season cycle, so content-addressing re-uploaded all ~215 event blobs each cycle (defeating the immutable edge cache). Render an event blob only when its own event/match/team_event content changes (NaN-stable comparison); unchanged events carry forward their prior versioned key and stale year snapshot. --- backend/src/data/utils.py | 23 +++++++++++++++++++- backend/src/google/storage.py | 40 ++++++++++++++++++++++++++--------- 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/backend/src/data/utils.py b/backend/src/data/utils.py index 669be1af..560670b3 100644 --- a/backend/src/data/utils.py +++ b/backend/src/data/utils.py @@ -1,5 +1,8 @@ from datetime import datetime -from typing import Dict, Optional, Tuple +import math +from typing import Any, Dict, Optional, Tuple + +import attr from src.db.functions import clear_year from src.db.models import ETag, Event, Match, TeamEvent, TeamYear, Year @@ -27,6 +30,24 @@ def create_objs(year: int) -> objs_type: return (Year(year=year), {}, {}, {}, {}, {}) +def _canonical(value: Any) -> Any: + if isinstance(value, float): + return "__nan__" if math.isnan(value) else value + if attr.has(type(value)): + return {f.name: _canonical(getattr(value, f.name)) for f in attr.fields(type(value))} + if isinstance(value, dict): + return {k: _canonical(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_canonical(v) for v in value] + return value + + +def nan_safe_eq(a: Any, b: Any) -> bool: + if a == b: + return True + return _canonical(a) == _canonical(b) + + def read_objs(year: int) -> objs_type: year_obj = get_year_db(year) if year_obj is None: diff --git a/backend/src/google/storage.py b/backend/src/google/storage.py index eb077f74..86ecb83d 100644 --- a/backend/src/google/storage.py +++ b/backend/src/google/storage.py @@ -8,7 +8,7 @@ from google.cloud import storage from src.constants import CURR_YEAR, HIST_EPOCH, PROD -from src.data.utils import objs_type +from src.data.utils import nan_safe_eq, objs_type from src.db.functions import get_noteworthy_matches, get_upcoming_matches from src.db.read.event import get_events as get_events_db from src.db.read.team import get_teams as get_teams_db @@ -128,22 +128,43 @@ def add(object_name: str, data: Any) -> None: add(f"events/{year}", _read_events(year_obj, events)) # event/{event.key} + prev = read_manifest() + prev_blobs = prev.blobs if prev else {} event_to_matches = defaultdict(list) event_to_team_events = defaultdict(list) for m in objs[4].values(): event_to_matches[m.event].append(m) for te in objs[3].values(): event_to_team_events[te.event].append(te) + + orig_events = orig_objs[2] if orig_objs else {} + orig_matches: Dict[str, List[Any]] = defaultdict(list) + orig_team_events: Dict[str, List[Any]] = defaultdict(list) + if orig_objs is not None: + for m in orig_objs[4].values(): + orig_matches[m.event].append(m) + for te in orig_objs[3].values(): + orig_team_events[te.event].append(te) + for event in events: - add( - f"event/{event.key}", - _read_event( - year_obj, - event, - event_to_matches.get(event.key, []), - event_to_team_events.get(event.key, []), - ), + logical = f"event/{event.key}" + event_changed = ( + not nan_safe_eq(event, orig_events.get(event.pk())) + or not nan_safe_eq(event_to_matches[event.key], orig_matches[event.key]) + or not nan_safe_eq( + event_to_team_events[event.key], orig_team_events[event.key] + ) ) + if event_changed or logical not in prev_blobs: + add( + logical, + _read_event( + year_obj, + event, + event_to_matches.get(event.key, []), + event_to_team_events.get(event.key, []), + ), + ) # team_to_events team_to_events = defaultdict(list) @@ -185,7 +206,6 @@ def add(object_name: str, data: Any) -> None: _read_upcoming_matches(upcoming_matches), ) - prev = read_manifest() cycle = datetime.now(timezone.utc).isoformat() plan = plan_uploads(rendered, prev, cycle, hist_epoch=HIST_EPOCH) _publish(plan) From 4de126d8e9bf92175a322d93862a42eaeab9a4c5 Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Fri, 10 Jul 2026 11:16:36 -0700 Subject: [PATCH 6/8] Harden manifest resolution: retry a blipped fetch; rewrite all query params A7: a single failed manifest.json fetch no longer pins every client to the uncached legacy ?t= path for a full 60s TTL (a null result is dropped from the cache so the next call retries). toLogicalPath now replaces all ? and & so blob keys with 3+ query params resolve instead of missing to the backend. --- frontend/src/api/storage.tsx | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/frontend/src/api/storage.tsx b/frontend/src/api/storage.tsx index 78f8afae..00403e34 100644 --- a/frontend/src/api/storage.tsx +++ b/frontend/src/api/storage.tsx @@ -33,13 +33,21 @@ async function getManifest(): Promise { if (DISABLE_GCS) return null; const now = Date.now(); if (!manifestPromise || now - manifestPromise.fetchedAt > MANIFEST_TTL_MS) { - manifestPromise = { promise: fetchManifest(), fetchedAt: now }; + const promise = fetchManifest(); + manifestPromise = { promise, fetchedAt: now }; + // Don't pin every client to the uncached legacy path for a full TTL after one + // blipped fetch: drop a null result from the cache so the next call retries. + promise.then((manifest) => { + if (manifest === null && manifestPromise?.promise === promise) { + manifestPromise = null; + } + }); } return manifestPromise.promise; } function toLogicalPath(apiPath: string): string { - return apiPath.replace("?", ".").replace("&", ".").replace(/^\//, ""); + return apiPath.replace(/[?&]/g, ".").replace(/^\//, ""); } function resolveBucketUrl(logicalPath: string, manifest: Manifest | null): string { From 3d747b9bc386bc83cf5aa825a5bb01a6e53fc945 Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Fri, 10 Jul 2026 13:26:43 -0700 Subject: [PATCH 7/8] Serve match pages from the edge-cached event blob; unblock team event wave MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - getMatch: derive the match view from the (bucket-first, edge-cached) event blob — the match key encodes its event, and the event blob already carries the match + its team_matches + team_events. Falls back to the /match API when the blob lacks the match. Removes the Cloud-Run round trip (and its cold-start tail) from the match page's critical path. - getTeamYear: start the event-blob fetches as soon as team_to_events resolves instead of gating them on the whole metadata Promise.all, which included the large team_years/{year} blob the event wave does not need. --- frontend/src/api/match.tsx | 18 +++++++++++++++++- frontend/src/api/team.tsx | 12 +++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/frontend/src/api/match.tsx b/frontend/src/api/match.tsx index 43efa499..68da6f3f 100644 --- a/frontend/src/api/match.tsx +++ b/frontend/src/api/match.tsx @@ -1,9 +1,25 @@ import { MatchData } from "../types/data"; +import { getEvent } from "./event"; import query, { version } from "./storage"; export async function getMatch(match: string): Promise { const urlSuffix = `/match/${match}`; const storageKey = `match_${match}_${version}`; - return query(storageKey, urlSuffix, false, 0, 60); // 1 minute + try { + const eventData = await getEvent(match.split("_")[0]); + const matchData = eventData.matches.find((m) => m.key === match); + if (!matchData) { + throw new Error("Match not found in event blob"); + } + return { + year: eventData.year, + event: eventData.event, + team_events: eventData.team_events, + match: matchData, + team_matches: eventData.team_matches.filter((tm) => tm.match === match), + }; + } catch (e) { + return query(storageKey, urlSuffix, false, 0, 60); // 1 minute + } } diff --git a/frontend/src/api/team.tsx b/frontend/src/api/team.tsx index 8c590501..5cb6b860 100644 --- a/frontend/src/api/team.tsx +++ b/frontend/src/api/team.tsx @@ -24,15 +24,17 @@ export async function getTeamYear( if (year !== CURR_YEAR) { throw new Error("Not current year"); } - const [teamToEvents, teamData, teamYearData] = await Promise.all([ - fetchBucketData("team_to_events") as Promise<{ [key: number]: string[] }>, + const eventsPromise = ( + fetchBucketData("team_to_events") as Promise<{ [key: number]: string[] }> + ).then((teamToEvents) => + Promise.all(teamToEvents[team].map(async (eventKey) => await getEvent(eventKey))) + ); + const [teamData, teamYearData, events] = await Promise.all([ getTeam(team), getYearTeamYears(year), + eventsPromise, ]); const teamYear = teamYearData?.team_years?.find((teamYear) => teamYear.team === team); - const events = await Promise.all( - teamToEvents[team].map(async (eventKey) => await getEvent(eventKey)) - ); const matches = events.flatMap((event) => event.matches .filter( From 740397e597db65cce12d4d708edb9d364fdbcc69 Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Thu, 9 Jul 2026 23:46:25 -0700 Subject: [PATCH 8/8] Reduce not-found render delay from 8s to 1.5s The not-found placeholder waited 8000ms before rendering, leaving a blank content area for 8 full seconds on any nonexistent or slow-to-load entity (/team/99999, /event/2026zzzzz, etc.). The debounce exists only to avoid a flash of the not-found message before data arrives; 1.5s is ample for that while no longer looking like a hung page. (The comment already described the intent as 'one second'; the 8000 value grew from 1000 over prior commits.) --- frontend/src/pagesContent/shared/notFound.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/pagesContent/shared/notFound.tsx b/frontend/src/pagesContent/shared/notFound.tsx index 0f03fa67..269bd50f 100644 --- a/frontend/src/pagesContent/shared/notFound.tsx +++ b/frontend/src/pagesContent/shared/notFound.tsx @@ -11,7 +11,7 @@ const NotFound = ({ type }: { type: string }) => { useEffect(() => { setTimeout(() => { setRender(true); - }, 8000); + }, 1500); }, []); if (!render) {