[blob-store 4/5] DuckDB read layer + Parquet publish for /v3 (demonstrative)

backend/pyproject.toml

@@ -24,6 +24,8 @@ isort = "^5.12.0"
 pyright = "^1.1.324"
 numpy = "1.26.4"
 google-cloud-storage = "^3.1.0"
+duckdb = "^1.5.4"
+pyarrow = "^20.0.0"
 
 [tool.poetry.group.dev.dependencies]
 pyinstrument = "^4.6.1"

backend/requirements.txt

@@ -8,6 +8,7 @@ certifi==2025.1.31
 charset-normalizer==3.4.1
 click==8.1.8
 colorama==0.4.6
+duckdb==1.5.4
 exceptiongroup==1.2.2
 fastapi==0.101.1
 flake8==6.1.0
@@ -36,6 +37,7 @@ platformdirs==4.3.7
 proto-plus==1.26.1
 protobuf==6.30.2
 psycopg2==2.9.10
+pyarrow==20.0.0
 pyasn1-modules==0.4.2
 pyasn1==0.6.1
 pycodestyle==2.11.1

backend/src/api/backend.py

@@ -0,0 +1,47 @@
+import os
+
+if os.environ.get("API_BACKEND", "").lower() == "duckdb":
+    from src.db_duckdb import (
+        get_event,
+        get_events,
+        get_match,
+        get_matches,
+        get_team,
+        get_team_event,
+        get_team_events,
+        get_team_year,
+        get_team_years,
+        get_teams,
+        get_year,
+        get_years,
+    )
+else:
+    from src.db.read import (
+        get_event,
+        get_events,
+        get_match,
+        get_matches,
+        get_team,
+        get_team_event,
+        get_team_events,
+        get_team_year,
+        get_team_years,
+        get_teams,
+        get_year,
+        get_years,
+    )
+
+__all__ = [
+    "get_event",
+    "get_events",
+    "get_match",
+    "get_matches",
+    "get_team_event",
+    "get_team_events",
+    "get_team_year",
+    "get_team_years",
+    "get_team",
+    "get_teams",
+    "get_year",
+    "get_years",
+]

backend/src/api/event.py

@@ -16,7 +16,7 @@
     year_query,
 )
 from src.db.models import Event
-from src.db.read import get_event, get_events
+from src.api.backend import get_event, get_events
 from src.utils.alru_cache import alru_cache
 from src.utils.decorators import (
     async_fail_gracefully_plural,

backend/src/api/match.py

@@ -15,7 +15,7 @@
     year_query,
 )
 from src.db.models import Match
-from src.db.read import get_match, get_matches
+from src.api.backend import get_match, get_matches
 from src.utils.alru_cache import alru_cache
 from src.utils.decorators import (
     async_fail_gracefully_plural,

backend/src/api/team.py

@@ -14,7 +14,7 @@
     state_query,
 )
 from src.db.models import Team
-from src.db.read import get_team, get_teams
+from src.api.backend import get_team, get_teams
 from src.utils.alru_cache import alru_cache
 from src.utils.decorators import (
     async_fail_gracefully_plural,

backend/src/api/team_event.py

@@ -18,7 +18,7 @@
     year_query,
 )
 from src.db.models import TeamEvent
-from src.db.read import get_team_event, get_team_events
+from src.api.backend import get_team_event, get_team_events
 from src.utils.alru_cache import alru_cache
 from src.utils.decorators import (
     async_fail_gracefully_plural,

backend/src/api/team_year.py

@@ -15,7 +15,7 @@
     year_query,
 )
 from src.db.models import TeamYear
-from src.db.read import get_team_year, get_team_years
+from src.api.backend import get_team_year, get_team_years
 from src.utils.alru_cache import alru_cache
 from src.utils.decorators import (
     async_fail_gracefully_plural,

backend/src/api/year.py

@@ -5,7 +5,7 @@
 
 from src.api.query import ascending_query, limit_query, metric_query, offset_query
 from src.db.models import Year
-from src.db.read import get_year, get_years
+from src.api.backend import get_year, get_years
 from src.utils.alru_cache import alru_cache
 from src.utils.decorators import (
     async_fail_gracefully_plural,

backend/src/data/main.py

@@ -39,6 +39,7 @@
     update_team_years as update_team_years_db,
     update_teams as update_teams_db,
 )
+from src.google.parquet import build_parquet_uploads, write_parquet
 from src.google.snapshot import read_snapshot, write_snapshot
 from src.google.storage import write_objs as write_objs_storage
 
@@ -83,7 +84,8 @@ def process_year(
         write_snapshot(year_num, objs, teams)
         timer.print(str(year_num) + " Write Snapshot")
 
-        write_objs_storage(objs, orig_objs if partial else None, teams)
+        parquet_uploads = build_parquet_uploads(year_num, objs, teams)
+        write_objs_storage(objs, orig_objs if partial else None, teams, parquet_uploads)
         timer.print(str(year_num) + " Write Storage")
 
         try:
@@ -96,6 +98,10 @@ def process_year(
         write_objs_db(year_num, objs, orig_objs if partial else None, not partial)
         timer.print(str(year_num) + " Write DB")
 
+        if not DISABLE_GCS:
+            write_parquet(year_num, objs, teams)
+            timer.print(str(year_num) + " Write Parquet")
+
     return teams
 
 

backend/src/db_duckdb/__init__.py

@@ -0,0 +1,29 @@
+from src.db_duckdb.main import (
+    get_event,
+    get_events,
+    get_match,
+    get_matches,
+    get_team,
+    get_team_event,
+    get_team_events,
+    get_team_year,
+    get_team_years,
+    get_teams,
+    get_year,
+    get_years,
+)
+
+__all__ = [
+    "get_event",
+    "get_events",
+    "get_match",
+    "get_matches",
+    "get_team_event",
+    "get_team_events",
+    "get_team_year",
+    "get_team_years",
+    "get_team",
+    "get_teams",
+    "get_year",
+    "get_years",
+]