From a48c23ef11cc3e88f5bb7b144aa590e89cbac9bf Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Thu, 9 Jul 2026 21:33:29 -0700 Subject: [PATCH 1/4] Allow overriding DB connection string via DATABASE_URL Lets the backend run against plain PostgreSQL (e.g. Cloud SQL) instead of CockroachDB by exporting DATABASE_URL, without disturbing the existing PROD CockroachDB path or local dev default. Retry behavior is selected from the engine dialect (see src/db/transaction.py). --- backend/src/constants.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/src/constants.py b/backend/src/constants.py index 3e07f453..ea70e2ec 100644 --- a/backend/src/constants.py +++ b/backend/src/constants.py @@ -14,7 +14,11 @@ CRDB_PWD = os.getenv("CRDB_PWD", "") CRDB_HOST = os.getenv("CRDB_HOST", "") -CONN_STR = ( +# DATABASE_URL, if set, overrides the CockroachDB connection string entirely. +# Used to run the backend against plain PostgreSQL (e.g. Cloud SQL staging), +# e.g. "postgresql+psycopg2://user:pwd@host:5432/statbotics3". The transaction +# helper (src/db/transaction.py) selects retry behavior from the engine dialect. +CONN_STR = os.getenv("DATABASE_URL") or ( ( "cockroachdb://" + CRDB_USER From f8d80263af8dceb654134909994f2e229723426d Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Thu, 9 Jul 2026 21:33:29 -0700 Subject: [PATCH 2/4] Dispatch run_transaction on engine dialect for PostgreSQL support Introduce src/db/transaction.py with a run_transaction wrapper that keeps the CockroachDB behavior unchanged (delegates to sqlalchemy_cockroachdb, imported lazily) while adding a plain-SQLAlchemy transaction path for other dialects (PostgreSQL). The Postgres path preserves retry-on-serialization-failure (SQLSTATE 40001) semantics. Repoint the 13 db read/write/function modules to import run_transaction from src.db.transaction instead of sqlalchemy_cockroachdb. --- backend/src/db/functions/clear_year.py | 2 +- .../src/db/functions/noteworthy_matches.py | 2 +- .../db/functions/remove_teams_no_events.py | 2 +- backend/src/db/functions/upcoming_matches.py | 2 +- backend/src/db/functions/update_teams.py | 2 +- backend/src/db/read/etag.py | 2 +- backend/src/db/read/event.py | 2 +- backend/src/db/read/match.py | 2 +- backend/src/db/read/team.py | 2 +- backend/src/db/read/team_event.py | 2 +- backend/src/db/read/team_year.py | 2 +- backend/src/db/read/year.py | 2 +- backend/src/db/transaction.py | 75 +++++++++++++++++++ backend/src/db/write/template.py | 2 +- 14 files changed, 88 insertions(+), 13 deletions(-) create mode 100644 backend/src/db/transaction.py diff --git a/backend/src/db/functions/clear_year.py b/backend/src/db/functions/clear_year.py index 4442a457..6242fe65 100644 --- a/backend/src/db/functions/clear_year.py +++ b/backend/src/db/functions/clear_year.py @@ -1,5 +1,4 @@ from sqlalchemy.orm import Session as SessionType -from sqlalchemy_cockroachdb import run_transaction # type: ignore from src.db.main import Session from src.db.models.etag import ETagORM @@ -8,6 +7,7 @@ from src.db.models.team_event import TeamEventORM from src.db.models.team_year import TeamYearORM from src.db.models.year import YearORM +from src.db.transaction import run_transaction def clear_year(year: int) -> None: diff --git a/backend/src/db/functions/noteworthy_matches.py b/backend/src/db/functions/noteworthy_matches.py index 889b238a..e265de49 100644 --- a/backend/src/db/functions/noteworthy_matches.py +++ b/backend/src/db/functions/noteworthy_matches.py @@ -2,11 +2,11 @@ from sqlalchemy import asc, desc, func from sqlalchemy.orm import Session as SessionType -from sqlalchemy_cockroachdb import run_transaction # type: ignore from src.db.main import Session from src.db.models.event import EventORM from src.db.models.match import Match, MatchORM +from src.db.transaction import run_transaction from src.types.enums import MatchStatus diff --git a/backend/src/db/functions/remove_teams_no_events.py b/backend/src/db/functions/remove_teams_no_events.py index ba886d60..0a4969d8 100644 --- a/backend/src/db/functions/remove_teams_no_events.py +++ b/backend/src/db/functions/remove_teams_no_events.py @@ -1,13 +1,13 @@ from typing import List from sqlalchemy.orm import Session as SessionType -from sqlalchemy_cockroachdb import run_transaction # type: ignore from src.constants import CURR_YEAR from src.db.main import Session from src.db.models.team import TeamORM from src.db.models.team_event import TeamEventORM from src.db.models.team_year import TeamYearORM +from src.db.transaction import run_transaction def remove_teams_with_no_events() -> None: diff --git a/backend/src/db/functions/upcoming_matches.py b/backend/src/db/functions/upcoming_matches.py index 098d6f18..9cd8c468 100644 --- a/backend/src/db/functions/upcoming_matches.py +++ b/backend/src/db/functions/upcoming_matches.py @@ -3,12 +3,12 @@ from sqlalchemy import func, text from sqlalchemy.orm import Session as SessionType -from sqlalchemy_cockroachdb import run_transaction # type: ignore from src.constants import CURR_YEAR from src.db.main import Session from src.db.models.event import EventORM from src.db.models.match import Match, MatchORM +from src.db.transaction import run_transaction from src.types.enums import EventStatus diff --git a/backend/src/db/functions/update_teams.py b/backend/src/db/functions/update_teams.py index e7e1cc53..c1aa304e 100644 --- a/backend/src/db/functions/update_teams.py +++ b/backend/src/db/functions/update_teams.py @@ -1,9 +1,9 @@ from sqlalchemy.orm import Session as SessionType -from sqlalchemy_cockroachdb import run_transaction # type: ignore from src.db.main import Session from src.db.models.team import TeamORM from src.db.models.team_year import TeamYearORM +from src.db.transaction import run_transaction def update_team_districts() -> None: diff --git a/backend/src/db/read/etag.py b/backend/src/db/read/etag.py index 6188e3a5..59c1ba91 100644 --- a/backend/src/db/read/etag.py +++ b/backend/src/db/read/etag.py @@ -1,10 +1,10 @@ from typing import List, Optional from sqlalchemy.orm.session import Session as SessionType -from sqlalchemy_cockroachdb import run_transaction # type: ignore from src.db.main import Session from src.db.models.etag import ETag, ETagORM +from src.db.transaction import run_transaction def get_etags(year: Optional[int] = None, path: Optional[str] = None) -> List[ETag]: diff --git a/backend/src/db/read/event.py b/backend/src/db/read/event.py index 4b6f858d..18c8c992 100644 --- a/backend/src/db/read/event.py +++ b/backend/src/db/read/event.py @@ -1,11 +1,11 @@ from typing import List, Optional from sqlalchemy.orm.session import Session as SessionType -from sqlalchemy_cockroachdb import run_transaction # type: ignore from src.db.main import Session from src.db.models.event import Event, EventORM from src.db.read.main import common_filters +from src.db.transaction import run_transaction def get_event(event_id: str) -> Optional[Event]: diff --git a/backend/src/db/read/match.py b/backend/src/db/read/match.py index 23b13738..c715f507 100644 --- a/backend/src/db/read/match.py +++ b/backend/src/db/read/match.py @@ -1,11 +1,11 @@ from typing import List, Optional from sqlalchemy.orm.session import Session as SessionType -from sqlalchemy_cockroachdb import run_transaction # type: ignore from src.db.main import Session from src.db.models.match import Match, MatchORM from src.db.read.main import common_filters +from src.db.transaction import run_transaction def get_match(match: str) -> Optional[Match]: diff --git a/backend/src/db/read/team.py b/backend/src/db/read/team.py index 32da95a1..419e1b0e 100644 --- a/backend/src/db/read/team.py +++ b/backend/src/db/read/team.py @@ -1,11 +1,11 @@ from typing import List, Optional from sqlalchemy.orm.session import Session as SessionType -from sqlalchemy_cockroachdb import run_transaction # type: ignore from src.db.main import Session from src.db.models.team import Team, TeamORM from src.db.read.main import common_filters +from src.db.transaction import run_transaction def get_team(team: int) -> Optional[Team]: diff --git a/backend/src/db/read/team_event.py b/backend/src/db/read/team_event.py index 22eb3be7..17efd726 100644 --- a/backend/src/db/read/team_event.py +++ b/backend/src/db/read/team_event.py @@ -1,11 +1,11 @@ from typing import List, Optional from sqlalchemy.orm.session import Session as SessionType -from sqlalchemy_cockroachdb import run_transaction # type: ignore from src.db.main import Session from src.db.models.team_event import TeamEvent, TeamEventORM from src.db.read.main import common_filters +from src.db.transaction import run_transaction def get_team_event(team: int, event: str) -> Optional[TeamEvent]: diff --git a/backend/src/db/read/team_year.py b/backend/src/db/read/team_year.py index 2f47c79d..5fc50bf3 100644 --- a/backend/src/db/read/team_year.py +++ b/backend/src/db/read/team_year.py @@ -1,11 +1,11 @@ from typing import List, Optional from sqlalchemy.orm.session import Session as SessionType -from sqlalchemy_cockroachdb import run_transaction # type: ignore from src.db.main import Session from src.db.models.team_year import TeamYear, TeamYearORM from src.db.read.main import common_filters +from src.db.transaction import run_transaction def get_team_year(team: int, year: int) -> Optional[TeamYear]: diff --git a/backend/src/db/read/year.py b/backend/src/db/read/year.py index 5b11270a..d1071a31 100644 --- a/backend/src/db/read/year.py +++ b/backend/src/db/read/year.py @@ -1,11 +1,11 @@ from typing import List, Optional from sqlalchemy.orm.session import Session as SessionType -from sqlalchemy_cockroachdb import run_transaction # type: ignore from src.db.main import Session from src.db.models.year import Year, YearORM from src.db.read.main import common_filters +from src.db.transaction import run_transaction def get_year(year: int) -> Optional[Year]: diff --git a/backend/src/db/transaction.py b/backend/src/db/transaction.py new file mode 100644 index 00000000..bb26f111 --- /dev/null +++ b/backend/src/db/transaction.py @@ -0,0 +1,75 @@ +"""Database transaction helper. + +Provides a single ``run_transaction`` entry point used across ``src/db``. + +Historically the codebase imported ``run_transaction`` directly from +``sqlalchemy_cockroachdb``. That helper is CockroachDB-specific (it rewrites +savepoint names for CRDB's transaction-retry protocol). To let the backend run +against plain PostgreSQL (e.g. Cloud SQL) as well as CockroachDB, this wrapper +dispatches on the engine dialect: + +* CockroachDB -> delegate to ``sqlalchemy_cockroachdb.run_transaction`` so + production behavior is byte-for-byte unchanged (imported lazily so a + Postgres-only deployment need not install the CRDB dialect). +* everything else (PostgreSQL) -> a plain SQLAlchemy transaction with the same + retry-on-serialization-failure semantics (SQLSTATE 40001). + +Call signature matches the original: ``run_transaction(Session, callback)`` +where ``Session`` is a ``sessionmaker`` and ``callback(session)`` performs the +work and returns a value. ``callback`` must not commit or roll back; it may be +invoked more than once and so must be free of non-DB side effects. +""" +from typing import Any, Callable, Optional + +from sqlalchemy.exc import DBAPIError +from sqlalchemy.orm.session import Session as SessionType + +from src.db.main import engine + +# PostgreSQL / CockroachDB serialization failure (retryable). +SERIALIZATION_FAILURE = "40001" + +# Default retry budget for the plain-Postgres path (CRDB helper defaults to +# unbounded; a small bounded budget is friendlier for a single-writer pipeline). +DEFAULT_MAX_RETRIES = 3 + + +def _run_plain( + sessionmaker: Any, + callback: Callable[[SessionType], Any], + max_retries: int, +) -> Any: + retry_count = 0 + while True: + session = sessionmaker() + try: + with session.begin(): + return callback(session) + except DBAPIError as exc: + retryable = getattr(exc.orig, "pgcode", None) == SERIALIZATION_FAILURE + if retryable and retry_count < max_retries: + retry_count += 1 + continue + raise + finally: + session.close() + + +def run_transaction( + transactor: Any, + callback: Callable[[SessionType], Any], + max_retries: Optional[int] = None, + max_backoff: int = 0, +) -> Any: + if engine.dialect.name == "cockroachdb": + import sqlalchemy_cockroachdb + + return sqlalchemy_cockroachdb.run_transaction( + transactor, callback, max_retries=max_retries, max_backoff=max_backoff + ) + + return _run_plain( + transactor, + callback, + DEFAULT_MAX_RETRIES if max_retries is None else max_retries, + ) diff --git a/backend/src/db/write/template.py b/backend/src/db/write/template.py index 21547ee8..f40baa9b 100644 --- a/backend/src/db/write/template.py +++ b/backend/src/db/write/template.py @@ -3,7 +3,6 @@ import attr from sqlalchemy.dialects import postgresql from sqlalchemy.orm.session import Session as SessionType -from sqlalchemy_cockroachdb import run_transaction # type: ignore from src.db.main import Session from src.db.models.etag import ETagORM @@ -14,6 +13,7 @@ from src.db.models.team_event import TeamEventORM from src.db.models.team_year import TeamYearORM from src.db.models.year import YearORM +from src.db.transaction import run_transaction CUTOFF = 1000 From c99681dc46146d555ab59b4d7f1c460a6c78d4b9 Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Fri, 10 Jul 2026 10:44:47 -0700 Subject: [PATCH 3/4] Use BigInteger for match/event timestamp columns On Postgres, SQLAlchemy Integer maps to int4 (32-bit). TBA returns a ~1900 placeholder Unix timestamp (-2208988800) for matches/events with an unknown time, which underflows int32 and aborts historical builds at 2006 with psycopg2.errors.NumericValueOutOfRange. CockroachDB INT is 64-bit, so this never surfaced on the production database. Widen the four timestamp columns (match.time, match.predicted_time, event.time, team_event.time) to BigInteger; this matches CockroachDB's width and is future-proof past the 2038 int32 limit. --- backend/src/db/models/event.py | 4 ++-- backend/src/db/models/match.py | 10 +++++++--- backend/src/db/models/team_event.py | 4 ++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/backend/src/db/models/event.py b/backend/src/db/models/event.py index 745bcf98..3ac1a12c 100644 --- a/backend/src/db/models/event.py +++ b/backend/src/db/models/event.py @@ -1,6 +1,6 @@ from typing import Any, Dict -from sqlalchemy import Enum, Float, Integer, String +from sqlalchemy import BigInteger, Enum, Float, Integer, String from sqlalchemy.orm import Mapped, mapped_column from sqlalchemy.sql.schema import ForeignKeyConstraint, PrimaryKeyConstraint @@ -23,7 +23,7 @@ class EventORM(Base, ModelORM): """GENERAL""" name: MS = mapped_column(String(100)) - time: MI = mapped_column(Integer) + time: MI = mapped_column(BigInteger) # Unix timestamp; see match.py note country: MOS = mapped_column(String(30), nullable=True) state: MOS = mapped_column(String(10), nullable=True) district: MOS = mapped_column(String(10), nullable=True) diff --git a/backend/src/db/models/match.py b/backend/src/db/models/match.py index 884a073a..67494863 100644 --- a/backend/src/db/models/match.py +++ b/backend/src/db/models/match.py @@ -1,7 +1,7 @@ from typing import Any, Dict, List, Optional, Tuple import numpy as np -from sqlalchemy import Boolean, Enum, Float, Integer, JSON, String +from sqlalchemy import BigInteger, Boolean, Enum, Float, Integer, JSON, String from sqlalchemy.orm import Mapped, mapped_column from sqlalchemy.sql.schema import ForeignKeyConstraint, PrimaryKeyConstraint @@ -35,8 +35,12 @@ class MatchORM(Base, ModelORM): set_number: MI = mapped_column(Integer) match_number: MI = mapped_column(Integer) - time: MI = mapped_column(Integer) # Enforces ordering - predicted_time: MOI = mapped_column(Integer, nullable=True) # For display + # BigInteger: Unix timestamps. TBA returns a ~1900 placeholder + # (-2,208,988,800) for matches with unknown time, which underflows Postgres + # int32; CockroachDB INT is 64-bit so this never surfaced in prod. BigInteger + # matches that width and is also future-proof past the 2038 int32 limit. + time: MI = mapped_column(BigInteger) # Enforces ordering + predicted_time: MOI = mapped_column(BigInteger, nullable=True) # For display status: Mapped[MatchStatus] = mapped_column( Enum(MatchStatus, values_callable=values_callable) diff --git a/backend/src/db/models/team_event.py b/backend/src/db/models/team_event.py index 7d6da7bd..be262539 100644 --- a/backend/src/db/models/team_event.py +++ b/backend/src/db/models/team_event.py @@ -1,6 +1,6 @@ from typing import Any, Dict, Tuple -from sqlalchemy import Boolean, Enum, Float, Integer, String +from sqlalchemy import BigInteger, Boolean, Enum, Float, Integer, String from sqlalchemy.orm import Mapped, mapped_column from sqlalchemy.sql.schema import ForeignKeyConstraint, PrimaryKeyConstraint @@ -27,7 +27,7 @@ class TeamEventORM(Base, ModelORM): ForeignKeyConstraint(["event"], ["events.key"]) """GENERAL""" - time: MI = mapped_column(Integer) + time: MI = mapped_column(BigInteger) # Unix timestamp; see match.py note """API COMPLETENESS""" team_name: MS = mapped_column(String(100)) From 536bb5a3f568755ac55c60674929b42141cd4adf Mon Sep 17 00:00:00 2001 From: Chris Hondl Date: Fri, 17 Jul 2026 09:29:52 -0700 Subject: [PATCH 4/4] fix: pool_pre_ping + recycle so idle DB connections don't crash the ETL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pooled connection left idle across the hourly-cron gap goes stale (Cloud SQL / db-f1-micro / the Cloud SQL proxy reap idle connections). The next query then raises 'server closed the connection unexpectedly', which 500s the ETL trigger (/v3/site/update_curr_year) and silently stalls ingestion — on the mirror this stopped offseason match schedules from appearing. pool_pre_ping reconnects transparently; pool_recycle=1800 drops old connections proactively. Deployed and verified on the staging mirror (backend rev 00014): the update endpoint returns 200 repeatedly where it had been 500ing, and 2026iri ingested its full match schedule. --- backend/src/db/main.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/src/db/main.py b/backend/src/db/main.py index 4f0719af..dcbf0c32 100644 --- a/backend/src/db/main.py +++ b/backend/src/db/main.py @@ -3,7 +3,12 @@ from src.constants import CONN_STR -engine = create_engine(CONN_STR) +# pool_pre_ping: a pooled connection idle across the hourly-cron gap goes stale +# (Cloud SQL / db-f1-micro / the Cloud SQL proxy reap idle connections), and the +# next query raises "server closed the connection unexpectedly", 500ing the ETL +# trigger and stalling ingestion. pre_ping reconnects transparently; recycle +# drops connections older than 30 min proactively. +engine = create_engine(CONN_STR, pool_pre_ping=True, pool_recycle=1800) Session = sessionmaker(bind=engine)