diff --git a/backend/backfill_blobs.py b/backend/backfill_blobs.py new file mode 100644 index 00000000..f4427013 --- /dev/null +++ b/backend/backfill_blobs.py @@ -0,0 +1,156 @@ +"""One-time historical blob backfill. + + python backfill_blobs.py # all past years + python backfill_blobs.py 2018 2019 # specific years + python backfill_blobs.py --force # ignore the progress checkpoint +""" + +import json +import sys +from collections import defaultdict +from typing import Dict, List, Set + +from src.constants import CURR_YEAR, HIST_EPOCH +from src.db.read.event import get_events as get_events_db +from src.db.read.match import get_matches as get_matches_db +from src.db.read.team import get_teams as get_teams_db +from src.db.read.team_event import get_team_events as get_team_events_db +from src.db.read.team_year import get_team_years as get_team_years_db +from src.db.read.year import get_year as get_year_db +from src.google.publish import Manifest +from src.google.storage import ( + _bucket, + read_manifest, + upload_historical, + write_manifest, +) +from src.site.event import _read_event, _read_events +from src.site.team import _read_team_year +from src.site.team_year import _read_team_years + +PROGRESS_OBJECT = "backfill/progress.json" +SKIP_YEARS = {2021} + + +def _read_progress(bucket) -> Dict: + try: + raw = bucket.blob(PROGRESS_OBJECT).download_as_bytes() + return json.loads(raw) + except Exception: + return {"epoch": HIST_EPOCH, "completed_years": []} + + +def _write_progress(bucket, progress: Dict) -> None: + blob = bucket.blob(PROGRESS_OBJECT) + blob.cache_control = "no-cache" + blob.upload_from_string(json.dumps(progress).encode("utf-8"), "application/json") + + +def backfill_year(year: int, bucket) -> int: + year_obj = get_year_db(year) + if year_obj is None: + print(f" {year}: no Year row, skipping") + return 0 + + team_years = get_team_years_db(year=year) + events = get_events_db(year=year) + matches = get_matches_db(year=year) + team_events = get_team_events_db(year=year) + teams_by_num = {t.team: t for t in get_teams_db()} + + matches_by_event: Dict[str, List] = defaultdict(list) + team_events_by_event: Dict[str, List] = defaultdict(list) + for m in matches: + matches_by_event[m.event].append(m) + for te in team_events: + team_events_by_event[te.event].append(te) + + matches_by_team: Dict[int, List] = defaultdict(list) + for m in matches: + for num in set(m.get_red()) | set(m.get_blue()): + matches_by_team[num].append(m) + team_events_by_team: Dict[int, List] = defaultdict(list) + for te in team_events: + team_events_by_team[te.team].append(te) + + written = 0 + + if upload_historical( + f"team_years/{year}", _read_team_years(year, year_obj, team_years), bucket + ): + written += 1 + if upload_historical(f"events/{year}", _read_events(year_obj, events), bucket): + written += 1 + + for event in events: + payload = _read_event( + year_obj, + event, + matches_by_event.get(event.key, []), + team_events_by_event.get(event.key, []), + ) + if upload_historical(f"event/{event.key}", payload, bucket): + written += 1 + + for ty in team_years: + team_obj = teams_by_num.get(ty.team) + if team_obj is None: + continue + payload = _read_team_year( + year_obj, + team_obj, + ty, + team_events_by_team.get(ty.team, []), + matches_by_team.get(ty.team, []), + ) + if upload_historical(f"team/{ty.team}/{year}", payload, bucket): + written += 1 + + return written + + +def _ensure_manifest_epoch(bucket) -> None: + manifest = read_manifest() + if manifest is None: + manifest = Manifest(cycle="backfill", hist_epoch=HIST_EPOCH, blobs={}) + else: + manifest.hist_epoch = HIST_EPOCH + write_manifest(manifest, bucket) + + +def main(argv: List[str]) -> None: + force = "--force" in argv + year_args = [int(a) for a in argv if a.isdigit()] + + if year_args: + years = year_args + else: + years = [y for y in range(2002, CURR_YEAR) if y not in SKIP_YEARS] + + bucket = _bucket() + progress = _read_progress(bucket) + if progress.get("epoch") != HIST_EPOCH: + progress = {"epoch": HIST_EPOCH, "completed_years": []} + completed: Set[int] = set(progress.get("completed_years", [])) + + total = 0 + for year in years: + if year in SKIP_YEARS: + continue + if year in completed and not force: + print(f"{year}: already complete, skipping") + continue + print(f"{year}: backfilling...") + written = backfill_year(year, bucket) + total += written + completed.add(year) + progress["completed_years"] = sorted(completed) + _write_progress(bucket, progress) + print(f"{year}: {written} objects written") + + _ensure_manifest_epoch(bucket) + print(f"Done. {total} objects written this run (epoch {HIST_EPOCH}).") + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/backend/src/constants.py b/backend/src/constants.py index 3e07f453..34ac62f6 100644 --- a/backend/src/constants.py +++ b/backend/src/constants.py @@ -36,6 +36,7 @@ CURR_YEAR = 2026 DISABLE_GCS = False +HIST_EPOCH = 1 # MISC diff --git a/backend/src/data/utils.py b/backend/src/data/utils.py index 669be1af..560670b3 100644 --- a/backend/src/data/utils.py +++ b/backend/src/data/utils.py @@ -1,5 +1,8 @@ from datetime import datetime -from typing import Dict, Optional, Tuple +import math +from typing import Any, Dict, Optional, Tuple + +import attr from src.db.functions import clear_year from src.db.models import ETag, Event, Match, TeamEvent, TeamYear, Year @@ -27,6 +30,24 @@ def create_objs(year: int) -> objs_type: return (Year(year=year), {}, {}, {}, {}, {}) +def _canonical(value: Any) -> Any: + if isinstance(value, float): + return "__nan__" if math.isnan(value) else value + if attr.has(type(value)): + return {f.name: _canonical(getattr(value, f.name)) for f in attr.fields(type(value))} + if isinstance(value, dict): + return {k: _canonical(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_canonical(v) for v in value] + return value + + +def nan_safe_eq(a: Any, b: Any) -> bool: + if a == b: + return True + return _canonical(a) == _canonical(b) + + def read_objs(year: int) -> objs_type: year_obj = get_year_db(year) if year_obj is None: diff --git a/backend/src/google/publish.py b/backend/src/google/publish.py new file mode 100644 index 00000000..f971fa88 --- /dev/null +++ b/backend/src/google/publish.py @@ -0,0 +1,96 @@ +import hashlib +import json +from dataclasses import dataclass, field +from typing import Dict, Optional + +MANIFEST_OBJECT = "manifest.json" +VERSION_PREFIX = "v2" +HIST_PREFIX = "hist" +HASH_LEN = 12 +SCHEMA = 1 + + +def content_hash(data: bytes) -> str: + return hashlib.sha256(data).hexdigest()[:HASH_LEN] + + +def versioned_key(logical_path: str, digest: str) -> str: + return f"{VERSION_PREFIX}/{logical_path}.{digest}" + + +def historical_key(epoch: int, logical_path: str) -> str: + return f"{HIST_PREFIX}/{epoch}/{logical_path}" + + +@dataclass +class Manifest: + schema: int = SCHEMA + cycle: str = "" + hist_epoch: int = 1 + blobs: Dict[str, str] = field(default_factory=dict) + + def hash_for(self, logical_path: str) -> Optional[str]: + key = self.blobs.get(logical_path) + if key is None: + return None + return key.rsplit(".", 1)[-1] + + def to_json(self) -> str: + return json.dumps( + { + "schema": self.schema, + "cycle": self.cycle, + "hist_epoch": self.hist_epoch, + "blobs": self.blobs, + }, + sort_keys=True, + ) + + @classmethod + def from_json(cls, raw: object) -> "Manifest": + if isinstance(raw, (bytes, bytearray)): + raw = bytes(raw).decode("utf-8") + if isinstance(raw, str): + data = json.loads(raw) + elif isinstance(raw, dict): + data = raw + else: + raise TypeError(f"Cannot parse manifest from {type(raw)!r}") + return cls( + schema=int(data.get("schema", SCHEMA)), + cycle=str(data.get("cycle", "")), + hist_epoch=int(data.get("hist_epoch", 1)), + blobs=dict(data.get("blobs", {})), + ) + + +@dataclass +class UploadPlan: + uploads: Dict[str, bytes] + legacy_uploads: Dict[str, bytes] + manifest: Manifest + + +def plan_uploads( + rendered: Dict[str, bytes], + prev: Optional[Manifest], + cycle: str, + hist_epoch: Optional[int] = None, +) -> UploadPlan: + prev = prev or Manifest() + if hist_epoch is None: + hist_epoch = prev.hist_epoch + + uploads: Dict[str, bytes] = {} + legacy_uploads: Dict[str, bytes] = {} + blobs: Dict[str, str] = dict(prev.blobs) + + for logical_path, data in rendered.items(): + digest = content_hash(data) + blobs[logical_path] = versioned_key(logical_path, digest) + if prev.hash_for(logical_path) != digest: + uploads[versioned_key(logical_path, digest)] = data + legacy_uploads[logical_path] = data + + manifest = Manifest(schema=SCHEMA, cycle=cycle, hist_epoch=hist_epoch, blobs=blobs) + return UploadPlan(uploads=uploads, legacy_uploads=legacy_uploads, manifest=manifest) diff --git a/backend/src/google/storage.py b/backend/src/google/storage.py index 2913150c..86ecb83d 100644 --- a/backend/src/google/storage.py +++ b/backend/src/google/storage.py @@ -1,25 +1,35 @@ from collections import defaultdict from concurrent.futures import ThreadPoolExecutor - -# from datetime import datetime +from datetime import datetime, timezone import json -from typing import Any, List, Optional +from typing import Any, Dict, List, Optional import zlib from google.cloud import storage -from src.constants import CURR_YEAR, PROD -from src.data.utils import objs_type +from src.constants import CURR_YEAR, HIST_EPOCH, PROD +from src.data.utils import nan_safe_eq, objs_type from src.db.functions import get_noteworthy_matches, get_upcoming_matches from src.db.read.event import get_events as get_events_db from src.db.read.team import get_teams as get_teams_db +from src.db.read.team_year import get_team_years as get_team_years_db +from src.google.publish import ( + MANIFEST_OBJECT, + Manifest, + UploadPlan, + historical_key, + plan_uploads, +) from src.site.event import _read_all_events, _read_events, _read_event from src.site.match import _read_noteworthy_matches, _read_upcoming_matches +from src.site.team import _read_all_teams, _read_team from src.site.team_year import _read_team_years -from src.site.team import _read_all_teams BUCKET_NAME = "site_v1" if PROD else "site_dev_v1" +IMMUTABLE_CACHE = "public, max-age=31536000, immutable" +MANIFEST_CACHE = "public, max-age=60" + def compress(data: Any) -> bytes: # start = datetime.now() @@ -31,81 +41,154 @@ def compress(data: Any) -> bytes: return compressed +def _bucket() -> Any: + return storage.Client().bucket(BUCKET_NAME) + + +def _upload_bytes( + bucket: Any, object_name: str, data: bytes, cache_control: Optional[str] +) -> None: + blob = bucket.blob(object_name) + if cache_control is not None: + blob.cache_control = cache_control + blob.upload_from_string(data, "application/octet-stream") + + def upload_file_to_gcs(data: Any, object_name: str) -> None: - # start = datetime.now() - storage.Client().bucket(BUCKET_NAME).blob(object_name).upload_from_string( - compress(data), "application/octet-stream" - ) - # print(f"Uploaded {object_name} to GCS in {datetime.now() - start}") + _upload_bytes(_bucket(), object_name, compress(data), None) -def upload_files_to_gcs(data: List[Any], object_names: List[str]) -> None: - with ThreadPoolExecutor() as executor: - executor.map(upload_file_to_gcs, data, object_names) +def read_manifest() -> Optional[Manifest]: + try: + raw = _bucket().blob(MANIFEST_OBJECT).download_as_bytes() + except Exception: + return None + try: + return Manifest.from_json(raw) + except Exception: + return None + + +def write_manifest(manifest: Manifest, bucket: Any = None) -> None: + bucket = bucket or _bucket() + blob = bucket.blob(MANIFEST_OBJECT) + blob.cache_control = MANIFEST_CACHE + blob.upload_from_string(manifest.to_json().encode("utf-8"), "application/json") + + +def _publish(plan: UploadPlan) -> None: + bucket = _bucket() + + jobs: List[Any] = [] + for versioned, data in plan.uploads.items(): + jobs.append((versioned, data, IMMUTABLE_CACHE)) + for logical, data in plan.legacy_uploads.items(): + jobs.append((logical, data, None)) + + if jobs: + with ThreadPoolExecutor() as executor: + list(executor.map(lambda job: _upload_bytes(bucket, *job), jobs)) + + # manifest is written last so readers always resolve a complete blob set + write_manifest(plan.manifest, bucket) def write_objs( objs: objs_type, orig_objs: Optional[objs_type] = None, ) -> None: + year = CURR_YEAR + year_obj = objs[0] + + rendered: Dict[str, bytes] = {} + + def add(object_name: str, data: Any) -> None: + rendered[object_name] = compress(data) + # teams/all teams = get_teams_db() - upload_file_to_gcs(_read_all_teams(teams), "teams/all") + add("teams/all", _read_all_teams(teams)) # team_years/{CURR_YEAR} - year = CURR_YEAR - year_obj = objs[0] team_years = list(objs[1].values()) - upload_file_to_gcs( - _read_team_years(year, year_obj, team_years), f"team_years/{year}" - ) + add(f"team_years/{year}", _read_team_years(year, year_obj, team_years)) # team_years/{CURR_YEAR}?limit=100&metric=epa - team_years = sorted(team_years, key=lambda x: -x.epa)[:100] - upload_file_to_gcs( - _read_team_years(year, year_obj, team_years), + top_team_years = sorted(team_years, key=lambda x: -x.epa)[:100] + add( f"team_years/{year}.limit=100.metric=epa", + _read_team_years(year, year_obj, top_team_years), ) # events/all - events = get_events_db() - upload_file_to_gcs(_read_all_events(events), "events/all") + add("events/all", _read_all_events(get_events_db())) # events/{CURR_YEAR} events = list(objs[2].values()) - upload_file_to_gcs(_read_events(year_obj, events), f"events/{year}") + add(f"events/{year}", _read_events(year_obj, events)) # event/{event.key} - orig_events = orig_objs[2] if orig_objs else {} - new_events = [e for e in events if str(e) != str(orig_events.get(e.pk(), ""))] - if len(new_events) > 0: - event_to_matches = defaultdict(list) - event_to_team_events = defaultdict(list) - for m in objs[4].values(): - event_to_matches[m.event].append(m) - for te in objs[3].values(): - event_to_team_events[te.event].append(te) - data = [] - object_names = [] - for event in new_events: - matches = event_to_matches.get(event.key, []) - team_events = event_to_team_events.get(event.key, []) - data.append(_read_event(year_obj, event, matches, team_events)) - object_names.append(f"event/{event.key}") - upload_files_to_gcs(data, object_names) + prev = read_manifest() + prev_blobs = prev.blobs if prev else {} + event_to_matches = defaultdict(list) + event_to_team_events = defaultdict(list) + for m in objs[4].values(): + event_to_matches[m.event].append(m) + for te in objs[3].values(): + event_to_team_events[te.event].append(te) + orig_events = orig_objs[2] if orig_objs else {} + orig_matches: Dict[str, List[Any]] = defaultdict(list) + orig_team_events: Dict[str, List[Any]] = defaultdict(list) + if orig_objs is not None: + for m in orig_objs[4].values(): + orig_matches[m.event].append(m) + for te in orig_objs[3].values(): + orig_team_events[te.event].append(te) + + for event in events: + logical = f"event/{event.key}" + event_changed = ( + not nan_safe_eq(event, orig_events.get(event.pk())) + or not nan_safe_eq(event_to_matches[event.key], orig_matches[event.key]) + or not nan_safe_eq( + event_to_team_events[event.key], orig_team_events[event.key] + ) + ) + if event_changed or logical not in prev_blobs: + add( + logical, + _read_event( + year_obj, + event, + event_to_matches.get(event.key, []), + event_to_team_events.get(event.key, []), + ), + ) + + # team_to_events team_to_events = defaultdict(list) for team_event in objs[3].values(): team_to_events[team_event.team].append(team_event.event) - upload_file_to_gcs(team_to_events, "team_to_events") + add("team_to_events", team_to_events) + + # team/{team.team} + all_team_years = get_team_years_db() + team_years_by_team: Dict[int, List[Any]] = defaultdict(list) + for ty in all_team_years: + team_years_by_team[ty.team].append(ty) + teams_by_num = {t.team: t for t in teams} + for num in {ty.team for ty in team_years}: + team_obj = teams_by_num.get(num) + if team_obj is None: + continue + add(f"team/{num}", _read_team(team_obj, team_years_by_team.get(num, []))) # noteworthy_matches/{year} noteworthy_matches = get_noteworthy_matches( year=year, country=None, state=None, district=None, elim=None, week=None ) - upload_file_to_gcs( - _read_noteworthy_matches(noteworthy_matches), f"noteworthy_matches/{year}" - ) + add(f"noteworthy_matches/{year}", _read_noteworthy_matches(noteworthy_matches)) # upcoming_matches?limit=20&metric={predicted_time | max_epa | sum_epa | diff_epa} for metric in ["predicted_time", "max_epa", "sum_epa", "diff_epa"]: @@ -118,9 +201,23 @@ def write_objs( limit=20, metric=metric, ) - upload_file_to_gcs( - _read_upcoming_matches(upcoming_matches), + add( f"upcoming_matches.limit=20.metric={metric}", + _read_upcoming_matches(upcoming_matches), ) + cycle = datetime.now(timezone.utc).isoformat() + plan = plan_uploads(rendered, prev, cycle, hist_epoch=HIST_EPOCH) + _publish(plan) + return + + +def upload_historical(logical_path: str, data: Any, bucket: Any = None) -> bool: + bucket = bucket or _bucket() + blob = bucket.blob(historical_key(HIST_EPOCH, logical_path)) + if blob.exists(): + return False + blob.cache_control = IMMUTABLE_CACHE + blob.upload_from_string(compress(data), "application/octet-stream") + return True diff --git a/backend/src/site/team.py b/backend/src/site/team.py index 301a02ef..14e69f0e 100644 --- a/backend/src/site/team.py +++ b/backend/src/site/team.py @@ -34,18 +34,8 @@ async def read_all_teams(response: Response, no_cache: bool = False) -> Any: return _read_all_teams(teams) -@router.get("/team/{team_num}") -@async_fail_gracefully_singular -async def read_team_years( - response: Response, team_num: int, no_cache: bool = False -) -> Any: - team: Optional[Team] = await get_team_cached(team=team_num, no_cache=no_cache) - if team is None: - raise Exception("Team not found") - - team_years: List[TeamYear] = await get_team_years_cached( - team=team_num, no_cache=True - ) +def _read_team(team: Team, team_years: List[TeamYear]) -> Dict[str, Any]: + team_years = sorted(team_years, key=lambda x: x.year) team_year_stats = [ { "year": x.year, @@ -64,12 +54,26 @@ async def read_team_years( for x in team_years ] - out = { + return { "team": team.to_dict(), "team_years": team_year_stats, } - return out + +@router.get("/team/{team_num}") +@async_fail_gracefully_singular +async def read_team_years( + response: Response, team_num: int, no_cache: bool = False +) -> Any: + team: Optional[Team] = await get_team_cached(team=team_num, no_cache=no_cache) + if team is None: + raise Exception("Team not found") + + team_years: List[TeamYear] = await get_team_years_cached( + team=team_num, no_cache=True + ) + + return _read_team(team, team_years) @router.get("/team/{team_num}/{year}") @@ -109,6 +113,16 @@ async def read_team_year( team=team_num, year=year, no_cache=no_cache ) + return _read_team_year(year_obj, team, team_year, team_events, matches) + + +def _read_team_year( + year_obj: Year, + team: Team, + team_year: TeamYear, + team_events: List[TeamEvent], + matches: List[Match], +) -> Dict[str, Any]: matches = sorted(matches, key=lambda x: x.time) event_times: Dict[str, Optional[int]] = {e.event: None for e in team_events} @@ -123,13 +137,11 @@ async def read_team_year( team_matches = sorted(team_year.team_matches or [], key=lambda x: x["time"]) - out = { + return { "year": year_obj.to_dict(), "team": team.to_dict(), "team_year": team_year.to_dict(), "team_events": [x.to_dict() for x in team_events], "matches": [x.to_dict() for x in matches], - "team_matches": [{"team": team_num, **tm} for tm in team_matches], + "team_matches": [{"team": team.team, **tm} for tm in team_matches], } - - return out diff --git a/frontend/src/api/event.tsx b/frontend/src/api/event.tsx index 44e783db..9718fffb 100644 --- a/frontend/src/api/event.tsx +++ b/frontend/src/api/event.tsx @@ -13,8 +13,8 @@ export async function getTeamEventTeamMatches( team: number, event: string ): Promise { - const urlSuffix = `/event/${event}/team_matches/${team}`; - const storageKey = `event_${event}_team_matches_${team}_${version}`; - - return query(storageKey, urlSuffix, false, 0, 60); // 1 minute + const eventData = await getEvent(event); + return ((eventData?.team_matches ?? []) as APITeamMatch[]).filter( + (tm: any) => tm.team === team + ); } diff --git a/frontend/src/api/match.tsx b/frontend/src/api/match.tsx index 43efa499..68da6f3f 100644 --- a/frontend/src/api/match.tsx +++ b/frontend/src/api/match.tsx @@ -1,9 +1,25 @@ import { MatchData } from "../types/data"; +import { getEvent } from "./event"; import query, { version } from "./storage"; export async function getMatch(match: string): Promise { const urlSuffix = `/match/${match}`; const storageKey = `match_${match}_${version}`; - return query(storageKey, urlSuffix, false, 0, 60); // 1 minute + try { + const eventData = await getEvent(match.split("_")[0]); + const matchData = eventData.matches.find((m) => m.key === match); + if (!matchData) { + throw new Error("Match not found in event blob"); + } + return { + year: eventData.year, + event: eventData.event, + team_events: eventData.team_events, + match: matchData, + team_matches: eventData.team_matches.filter((tm) => tm.match === match), + }; + } catch (e) { + return query(storageKey, urlSuffix, false, 0, 60); // 1 minute + } } diff --git a/frontend/src/api/storage.tsx b/frontend/src/api/storage.tsx index d5b17b19..00403e34 100644 --- a/frontend/src/api/storage.tsx +++ b/frontend/src/api/storage.tsx @@ -1,4 +1,4 @@ -import { del, get, set } from "idb-keyval"; +import { get, set } from "idb-keyval"; import pako from "pako"; import { BACKEND_URL, BUCKET_URL, DISABLE_GCS } from "../constants"; @@ -6,6 +6,63 @@ import { log, round } from "../utils"; export const version = "v4"; +const MANIFEST_TTL_MS = 60 * 1000; + +interface Manifest { + schema?: number; + cycle?: string; + hist_epoch?: number; + blobs?: { [logicalPath: string]: string }; +} + +let manifestPromise: { promise: Promise; fetchedAt: number } | null = null; + +async function fetchManifest(): Promise { + try { + const res = await fetch(`${BUCKET_URL}/manifest.json`, { next: { revalidate: 0 } }); + if (res.ok) { + return (await res.json()) as Manifest; + } + } catch (e) { + log("manifest fetch failed", e); + } + return null; +} + +async function getManifest(): Promise { + if (DISABLE_GCS) return null; + const now = Date.now(); + if (!manifestPromise || now - manifestPromise.fetchedAt > MANIFEST_TTL_MS) { + const promise = fetchManifest(); + manifestPromise = { promise, fetchedAt: now }; + // Don't pin every client to the uncached legacy path for a full TTL after one + // blipped fetch: drop a null result from the cache so the next call retries. + promise.then((manifest) => { + if (manifest === null && manifestPromise?.promise === promise) { + manifestPromise = null; + } + }); + } + return manifestPromise.promise; +} + +function toLogicalPath(apiPath: string): string { + return apiPath.replace(/[?&]/g, ".").replace(/^\//, ""); +} + +function resolveBucketUrl(logicalPath: string, manifest: Manifest | null): string { + if (manifest) { + const versioned = manifest.blobs?.[logicalPath]; + if (versioned) { + return `${BUCKET_URL}/${versioned}`; + } + if (manifest.hist_epoch != null) { + return `${BUCKET_URL}/hist/${manifest.hist_epoch}/${logicalPath}`; + } + } + return `${BUCKET_URL}/${logicalPath}?t=${Date.now() / 1000 / 60}`; +} + async function setWithExpiry(key: string, value: any, ttl: number) { const now = new Date(); @@ -24,8 +81,14 @@ async function getWithExpiry(key: string) { } const now = new Date(); if (now.getTime() > expiry) { - await del(`${key}_expiry`); - await del(key); + return null; + } + return get(key); +} + +async function getStale(key: string) { + const expiry = await get(`${key}_expiry`); + if (!expiry) { return null; } return get(key); @@ -37,6 +100,27 @@ export function decompress(buffer: any) { return data; } +const bucketInFlight: { [logicalPath: string]: Promise } = {}; + +async function fetchBucketDataImpl(logicalPath: string): Promise { + const manifest = await getManifest(); + const url = resolveBucketUrl(logicalPath, manifest); + const res = await fetch(url, { next: { revalidate: 0 } }); + if (!res.ok) { + throw new Error(`Failed to fetch from bucket: ${res.status}`); + } + return decompress(await res.arrayBuffer()); +} + +export async function fetchBucketData(logicalPath: string): Promise { + if (!bucketInFlight[logicalPath]) { + bucketInFlight[logicalPath] = fetchBucketDataImpl(logicalPath).finally(() => { + delete bucketInFlight[logicalPath]; + }); + } + return bucketInFlight[logicalPath]; +} + async function query( storageKey: string, apiPath: string, @@ -57,25 +141,25 @@ async function query( if (!checkBucket || DISABLE_GCS) { throw new Error("Skip bucket check"); } - const fileName = apiPath.replace("?", ".").replace("&", "."); - const res = await fetch(`${BUCKET_URL}${fileName}?t=${Date.now() / 1000 / 60}`, { - next: { revalidate: 0 }, - headers: { - "Cache-Control": "no-cache", - "Content-Type": "application/octet-stream", - }, - }); - log(`${fileName} (bucket) took ${round(performance.now() - start, 0)}ms`); + const logicalPath = toLogicalPath(apiPath); + const manifest = await getManifest(); + const url = resolveBucketUrl(logicalPath, manifest); + const res = await fetch(url, { next: { revalidate: 0 } }); + log(`${logicalPath} (bucket) took ${round(performance.now() - start, 0)}ms`); if (res.ok) { buffer = decompress(await res.arrayBuffer()); } else { throw new Error(`Failed to fetch from bucket: ${res.status}`); } } catch (e) { - const res = await fetch(`${BACKEND_URL}${apiPath}`, { next: { revalidate: 0 } }); - log(`${apiPath} (backend) took ${round(performance.now() - start, 0)}ms`); - if (res.ok) { - buffer = await res.json(); + try { + const res = await fetch(`${BACKEND_URL}${apiPath}`, { next: { revalidate: 0 } }); + log(`${apiPath} (backend) took ${round(performance.now() - start, 0)}ms`); + if (res.ok) { + buffer = await res.json(); + } + } catch (apiErr) { + log(`${apiPath} (backend) failed`, apiErr); } } @@ -83,6 +167,12 @@ async function query( await setWithExpiry(storageKey, buffer, expiry); return buffer; } + + const stale = await getStale(storageKey); + if (stale && (minLength === 0 || stale?.length > minLength)) { + log(`Served stale cache: ${storageKey}`); + return stale; + } } export default query; diff --git a/frontend/src/api/team.tsx b/frontend/src/api/team.tsx index a0a5d54b..5cb6b860 100644 --- a/frontend/src/api/team.tsx +++ b/frontend/src/api/team.tsx @@ -1,15 +1,15 @@ -import { BUCKET_URL, CURR_YEAR } from "../constants"; +import { CURR_YEAR } from "../constants"; import { APITeam, APITeamEvent } from "../types/api"; import { TeamYearData, TeamYearRedirect } from "../types/data"; import { getEvent } from "./event"; -import query, { decompress, version } from "./storage"; +import query, { fetchBucketData, version } from "./storage"; import { getYearTeamYears } from "./teams"; export async function getTeam(team: number): Promise<{ team: APITeam; team_years: any[] }> { const urlSuffix = `/team/${team}`; const storageKey = `team_${team}_${version}`; - return query(storageKey, urlSuffix, false, 0, 60); // 1 minute + return query(storageKey, urlSuffix, true, 0, 60); // 1 minute } export async function getTeamYear( @@ -19,33 +19,22 @@ export async function getTeamYear( const urlSuffix = `/team/${team}/${year}`; const storageKey = `team_${team}_${year}_${version}`; - const readBucket = async (url: string) => { - const res = await fetch(url, { - next: { revalidate: 0 }, - }); - if (res.ok) { - const buffer = await res.arrayBuffer(); - const data = decompress(buffer); - return data; - } else { - throw new Error(`Failed to fetch from bucket: ${res.status}`); - } - }; - try { // try to reconstruct output from team, team_to_events, and events if (year !== CURR_YEAR) { throw new Error("Not current year"); } - const [teamToEvents, teamData, teamYearData] = await Promise.all([ - readBucket(`${BUCKET_URL}/team_to_events`) as Promise<{ [key: number]: string[] }>, + const eventsPromise = ( + fetchBucketData("team_to_events") as Promise<{ [key: number]: string[] }> + ).then((teamToEvents) => + Promise.all(teamToEvents[team].map(async (eventKey) => await getEvent(eventKey))) + ); + const [teamData, teamYearData, events] = await Promise.all([ getTeam(team), getYearTeamYears(year), + eventsPromise, ]); const teamYear = teamYearData?.team_years?.find((teamYear) => teamYear.team === team); - const events = await Promise.all( - teamToEvents[team].map(async (eventKey) => await getEvent(eventKey)) - ); const matches = events.flatMap((event) => event.matches .filter( @@ -85,7 +74,7 @@ export async function getTeamYear( team_matches: teamMatches, }; } catch (e) { - return query(storageKey, urlSuffix, false, 0, year === CURR_YEAR ? 60 : 60 * 60); // 1 minute / 1 hour + return query(storageKey, urlSuffix, true, 0, year === CURR_YEAR ? 60 : 60 * 60); // 1 minute / 1 hour } } diff --git a/frontend/src/api/teams.tsx b/frontend/src/api/teams.tsx index a4b6b633..2df7b491 100644 --- a/frontend/src/api/teams.tsx +++ b/frontend/src/api/teams.tsx @@ -1,6 +1,7 @@ import { CURR_YEAR } from "../constants"; import { APITeamMatch, APITeamYear, APIYear } from "../types/api"; import query, { version } from "./storage"; +import { getTeamYear } from "./team"; export async function getYearTeamYears( year: number, @@ -17,15 +18,13 @@ export async function getYearTeamYears( } storageKey += "_v3"; - return query(storageKey, urlSuffix, year === CURR_YEAR, 0, year === CURR_YEAR ? 60 : 60 * 60); // 1 minute / 1 hour + return query(storageKey, urlSuffix, true, 0, year === CURR_YEAR ? 60 : 60 * 60); // 1 minute / 1 hour } export async function getTeamYearTeamMatches( year: number, teamNum: number ): Promise { - const urlSuffix = `/team_year/${year}/${teamNum}/matches`; - const storageKey = `team_year_matches_${year}_${teamNum}_${version}`; - - return query(storageKey, urlSuffix, false, 0, year === CURR_YEAR ? 60 : 60 * 60); // 1 minute / 1 hour + const teamYear = await getTeamYear(teamNum, year); + return ((teamYear as any)?.team_matches ?? []) as APITeamMatch[]; } diff --git a/frontend/src/pagesContent/shared/notFound.tsx b/frontend/src/pagesContent/shared/notFound.tsx index 0f03fa67..269bd50f 100644 --- a/frontend/src/pagesContent/shared/notFound.tsx +++ b/frontend/src/pagesContent/shared/notFound.tsx @@ -11,7 +11,7 @@ const NotFound = ({ type }: { type: string }) => { useEffect(() => { setTimeout(() => { setRender(true); - }, 8000); + }, 1500); }, []); if (!render) {