diff --git a/commissioners/common/adapters.py b/commissioners/common/adapters.py index 21c8ba7..439b1ad 100644 --- a/commissioners/common/adapters.py +++ b/commissioners/common/adapters.py @@ -32,7 +32,7 @@ DescribeDivisionResponse, DivisionDescription as CommissionerDivisionDescription, DivisionConfig as CommissionerDivisionConfig, - DivisionLeaderboardEntry as CommissionerDivisionLeaderboardEntry, + DivisionLeaderboardView as CommissionerDivisionLeaderboardView, ) from commissioners.common.protocol import ( MembershipChange as CommissionerMembershipChange, @@ -312,8 +312,8 @@ def _protocol_round_spec(spec: RoundSpec) -> CommissionerRoundSpec: return CommissionerRoundSpec.model_validate(spec.model_dump(mode="json")) -def _protocol_leaderboard_entry(entry: CommissionerDivisionLeaderboardEntry) -> CommissionerDivisionLeaderboardEntry: - return CommissionerDivisionLeaderboardEntry.model_validate(entry.model_dump(mode="json")) +def _protocol_leaderboard_view(view: Any) -> CommissionerDivisionLeaderboardView: + return CommissionerDivisionLeaderboardView.model_validate(view.model_dump(mode="json")) def _protocol_division_description( @@ -352,6 +352,7 @@ def complete_round_for_round_start( policy_version_id=score.policy_version_id, player_id=score.player_id, score=score.score, + scores=score.scores, ) for score in result.scores ], @@ -516,7 +517,7 @@ def rank_division_for_request( commissioner: Commissioner, request: RankDivisionRequest, ) -> RankDivisionResponse: - rankings = commissioner.rank_division( + leaderboards = commissioner.rank_division_leaderboards( DivisionLeaderboardContext( league=LeagueSnapshot( id=request.league.id, @@ -546,7 +547,10 @@ def rank_division_for_request( ], ) ) - return RankDivisionResponse(rankings=[_protocol_leaderboard_entry(ranking) for ranking in rankings]) + return RankDivisionResponse( + default_view_key=leaderboards.default_view_key, + views=[_protocol_leaderboard_view(view) for view in leaderboards.views], + ) def describe_division_for_request( diff --git a/commissioners/common/commissioners.py b/commissioners/common/commissioners.py index 56d45f0..40190b5 100644 --- a/commissioners/common/commissioners.py +++ b/commissioners/common/commissioners.py @@ -64,7 +64,12 @@ MembershipSnapshot, RoundSnapshot, RoundResultSnapshot, + DivisionLeaderboardAxisSnapshot, + DivisionLeaderboardRowSnapshot, DivisionLeaderboardSnapshot, + DivisionLeaderboardTablesSnapshot, + DivisionLeaderboardViewSnapshot, + DivisionLeaderboardsSnapshot, _LeaderboardAgg, LeaderboardRoundResultSnapshot, RoundSpec, @@ -160,6 +165,45 @@ def migrate_league(self, ctx: LeagueMigrationContext) -> LeagueMigrationResult: @abstractmethod def rank_division(self, ctx: DivisionLeaderboardContext) -> list[DivisionLeaderboardSnapshot]: ... + def rank_division_leaderboards(self, ctx: DivisionLeaderboardContext) -> DivisionLeaderboardsSnapshot: + # TODO: delete compatibility shim after all commissioners implement generic leaderboard views. + entries = self.rank_division(ctx) + return DivisionLeaderboardsSnapshot( + views=[ + DivisionLeaderboardViewSnapshot( + key="score", + title="Score", + axes=[ + DivisionLeaderboardAxisSnapshot(key="rank", label="Rank", value_type="integer", sort="asc"), + DivisionLeaderboardAxisSnapshot(key="score", label="Score", value_type="number", sort="desc"), + DivisionLeaderboardAxisSnapshot( + key="rounds_played", + label="Rounds Played", + value_type="integer", + ), + ], + rows=[ + DivisionLeaderboardRowSnapshot( + subject_id=entry.player_id, + subject_name=entry.player_name, + values={ + "rank": entry.rank, + "score": entry.score, + "rounds_played": entry.rounds_played, + }, + policy_version_ids=entry.policy_version_ids, + recent_rounds=entry.recent_rounds, + ) + for entry in entries + ], + ) + ] + ) + + def rank_division_tables(self, ctx: DivisionLeaderboardContext) -> DivisionLeaderboardTablesSnapshot: + # TODO: delete compatibility shim after callers stop using table terminology. + return self.rank_division_leaderboards(ctx) + @abstractmethod def describe_division(self, ctx: DivisionDescriptionContext) -> DivisionCommissionerDescriptionPublic: ... @@ -211,6 +255,227 @@ class BaselineCommissioner(Commissioner): def _scheduling_config(self, commissioner_config: dict[str, Any] | None) -> RoundSchedulingConfig: return RoundSchedulingConfig.model_validate(commissioner_config or {}) + def _round_result_score(self, result: LeaderboardRoundResultSnapshot, score_key: str) -> float | None: + if score_key == "score": + return result.score + scores = result.result_metadata.get("scores") + if not isinstance(scores, dict): + return None + value = scores.get(score_key) + return float(value) if isinstance(value, (int, float)) else None + + def _rank_division_view_by_metric( + self, + ctx: DivisionLeaderboardContext, + *, + view_key: str = "score", + title: str = "Score", + description: str | None = None, + score_axis_label: str = "Score", + score_key: str = "score", + half_life_hours: float | None = None, + window_hours: float | None = None, + ) -> DivisionLeaderboardViewSnapshot: + axes = [ + DivisionLeaderboardAxisSnapshot(key="rank", label="Rank", value_type="integer", sort="asc"), + DivisionLeaderboardAxisSnapshot(key=score_key, label=score_axis_label, value_type="number", sort="desc"), + DivisionLeaderboardAxisSnapshot(key="rounds_played", label="Rounds Played", value_type="integer"), + ] + if not ctx.completed_rounds or not ctx.round_results: + return DivisionLeaderboardViewSnapshot( + key=view_key, + title=title, + description=description, + axes=axes, + ) + + completed_rounds = ctx.completed_rounds + latest_completed_at = completed_rounds[0].completed_at + assert latest_completed_at is not None, f"Completed round {ctx.completed_rounds[0].id} is missing completed_at" + if window_hours is not None: + cutoff = latest_completed_at - timedelta(hours=window_hours) + completed_rounds = [ + round_row + for round_row in completed_rounds + if round_row.completed_at is not None and round_row.completed_at >= cutoff + ] + if not completed_rounds: + return DivisionLeaderboardViewSnapshot( + key=view_key, + title=title, + description=description, + axes=axes, + ) + + completed_rounds_by_id = {round_row.id: round_row for round_row in completed_rounds} + halflife_seconds = ( + timedelta(hours=half_life_hours).total_seconds() + if half_life_hours is not None + else self._leaderboard_ewma_halflife(ctx).total_seconds() + ) + + player_rounds: dict[tuple[PlayerId, UUID], LeaderboardRoundResultSnapshot] = {} + for result in ctx.round_results: + if int(result.result_metadata.get(RANKED_SCORE_COUNT_METADATA_KEY, 1)) <= 0: + continue + score = self._round_result_score(result, score_key) + if score is None: + continue + key = (result.player_id, result.round_id) + current = player_rounds.get(key) + current_score = self._round_result_score(current, score_key) if current is not None else None + if current is None or current_score is None or (score, -result.rank) > (current_score, -current.rank): + player_rounds[key] = result + + rounds_played_by_player: dict[PlayerId, int] = {} + aggs: dict[PlayerId, _LeaderboardAgg] = {} + for player_round in player_rounds.values(): + round_row = completed_rounds_by_id.get(player_round.round_id) + if round_row is None: + continue + score = self._round_result_score(player_round, score_key) + if score is None: + continue + rounds_played_by_player[player_round.player_id] = rounds_played_by_player.get(player_round.player_id, 0) + 1 + if player_round.player_id not in aggs: + aggs[player_round.player_id] = _LeaderboardAgg( + player_id=player_round.player_id, + player_name=player_round.player_name, + ) + assert round_row.completed_at is not None, f"Completed round {round_row.id} is missing completed_at" + weight = 0.5 ** ((latest_completed_at - round_row.completed_at).total_seconds() / halflife_seconds) + aggs[player_round.player_id].policy_version_ids.add(player_round.policy_version_id) + aggs[player_round.player_id].weighted_score_sum += score * weight + aggs[player_round.player_id].weight_sum += weight + + ranks_by_round_and_player = { + (player_round.round_id, player_round.player_id): player_round.rank + for player_round in player_rounds.values() + } + scores_by_round_and_player = { + (player_round.round_id, player_round.player_id): self._round_result_score(player_round, score_key) + for player_round in player_rounds.values() + } + + def build_recent_rounds(player_id: PlayerId) -> list[LeaderboardRecentRoundPublic] | None: + if not ctx.recent_rounds: + return None + return [ + LeaderboardRecentRoundPublic( + id=round_row.public_id, + round_number=round_row.round_number, + status=round_row.status, + rank=ranks_by_round_and_player.get((round_row.id, player_id)), + score=scores_by_round_and_player.get((round_row.id, player_id)), + started_at=round_row.started_at, + completed_at=round_row.completed_at, + ) + for round_row in ctx.recent_rounds + ] + + ranked_aggs = sorted( + [agg for agg in aggs.values() if agg.weight_sum > 0], + key=lambda agg: ( + -agg.score(), + agg.player_name or "", + str(agg.player_id), + ), + ) + return DivisionLeaderboardViewSnapshot( + key=view_key, + title=title, + description=description, + axes=axes, + rows=[ + DivisionLeaderboardRowSnapshot( + subject_id=agg.player_id, + subject_name=agg.player_name, + values={ + "rank": rank, + score_key: agg.score(), + "rounds_played": rounds_played_by_player[agg.player_id], + }, + policy_version_ids=agg.policy_version_ids, + recent_rounds=build_recent_rounds(agg.player_id), + ) + for rank, agg in enumerate(ranked_aggs, start=1) + ], + ) + + def _rank_division_leaderboards_from_config( + self, + ctx: DivisionLeaderboardContext, + leaderboard_configs: list[dict[str, Any]], + *, + default_half_life_hours: float | None = None, + ) -> DivisionLeaderboardsSnapshot: + if not leaderboard_configs: + leaderboard_configs = [{"key": "score", "title": "Score", "score_key": "score", "default": True}] + + default_view_key = "score" + views: list[DivisionLeaderboardViewSnapshot] = [] + for raw_view in leaderboard_configs: + raw_axes = raw_view.get("axes") + axes_config: list[Any] = raw_axes if isinstance(raw_axes, list) else [] + configured_score_axis = next( + (axis for axis in axes_config if isinstance(axis, dict) and axis.get("sort") == "desc"), + None, + ) + score_key = str( + raw_view.get("score_key") + or raw_view.get("metric") + or (configured_score_axis or {}).get("key") + or raw_view.get("key") + or raw_view.get("id") + or "score" + ) + view_key = str(raw_view.get("key") or raw_view.get("id") or score_key) + if raw_view.get("default", raw_view.get("primary", False)): + default_view_key = view_key + title = str(raw_view.get("title") or raw_view.get("label") or score_key.replace("_", " ").title()) + score_axis_label = str(raw_view.get("score_axis_label") or raw_view.get("score_label") or title) + window_hours = raw_view.get("window_hours", raw_view.get("lookback_hours")) + view = self._rank_division_view_by_metric( + ctx, + view_key=view_key, + title=title, + description=raw_view.get("description"), + score_axis_label=score_axis_label, + score_key=score_key, + half_life_hours=float(raw_view.get("half_life_hours", default_half_life_hours)) + if raw_view.get("half_life_hours", default_half_life_hours) is not None + else None, + window_hours=float(window_hours) if window_hours is not None else None, + ) + if axes_config: + view.axes = [DivisionLeaderboardAxisSnapshot.model_validate(axis) for axis in axes_config] + views.append(view) + + if not any(view.key == default_view_key for view in views) and views: + default_view_key = views[0].key + return DivisionLeaderboardsSnapshot(default_view_key=default_view_key, views=views) + + def _rank_division_tables_from_config( + self, + ctx: DivisionLeaderboardContext, + leaderboard_configs: list[dict[str, Any]], + *, + default_half_life_hours: float | None = None, + ) -> DivisionLeaderboardTablesSnapshot: + # TODO: delete compatibility shim after callers stop using table terminology. + return self._rank_division_leaderboards_from_config( + ctx, + leaderboard_configs, + default_half_life_hours=default_half_life_hours, + ) + + def rank_division_leaderboards(self, ctx: DivisionLeaderboardContext) -> DivisionLeaderboardsSnapshot: + return self._rank_division_leaderboards_from_config(ctx, []) + + def rank_division_tables(self, ctx: DivisionLeaderboardContext) -> DivisionLeaderboardTablesSnapshot: + # TODO: delete compatibility shim after callers stop using table terminology. + return self.rank_division_leaderboards(ctx) + def rank_division(self, ctx: DivisionLeaderboardContext) -> list[DivisionLeaderboardSnapshot]: if not ctx.completed_rounds or not ctx.round_results: return [] @@ -443,6 +708,19 @@ def complete_round( episode_results: list[EpisodeResult], ) -> CommissionerRoundComplete: round_score_by_policy, ranked_score_counts = self._round_scores_by_policy(entries, episode_results) + score_field_lists_by_policy: dict[UUID, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list)) + for result in episode_results: + for score in result.scores: + for key, value in score.scores.items(): + score_field_lists_by_policy[score.policy_version_id][key].append(value) + avg_score_fields_by_policy = { + entry.policy_version_id: { + key: sum(values) / len(values) + for key, values in sorted(score_field_lists_by_policy.get(entry.policy_version_id, {}).items()) + if values + } + for entry in entries + } completed_episode_counts: dict[UUID, int] = defaultdict(int) for result in episode_results: for policy_version_id in {score.policy_version_id for score in result.scores}: @@ -465,6 +743,10 @@ def complete_round( "seed_order": entry.seed_order, COMPLETED_EPISODE_COUNT_METADATA_KEY: completed_episode_counts[entry.policy_version_id], RANKED_SCORE_COUNT_METADATA_KEY: ranked_score_counts[entry.policy_version_id], + "scores": { + "score": round_score_by_policy[entry.policy_version_id], + **avg_score_fields_by_policy[entry.policy_version_id], + }, }, ) for rank, entry in enumerate(ranked_entries, start=1) diff --git a/commissioners/common/models.py b/commissioners/common/models.py index d199f5a..9a4eec4 100644 --- a/commissioners/common/models.py +++ b/commissioners/common/models.py @@ -3,7 +3,7 @@ from datetime import UTC, datetime, timedelta from enum import Enum, StrEnum from os import getenv -from typing import Any +from typing import Any, Literal from uuid import UUID from pydantic import BaseModel, Field, model_validator @@ -155,6 +155,13 @@ class RoundPolicyScore(BaseModel): policy_version_id: UUID player_id: PlayerId | None = None score: float + scores: dict[str, float] = Field(default_factory=dict) + + @model_validator(mode="after") + def include_primary_score(self) -> "RoundPolicyScore": + if "score" not in self.scores: + self.scores = {"score": self.score, **self.scores} + return self class EpisodeResult(BaseModel): @@ -394,6 +401,123 @@ class DivisionLeaderboardSnapshot(BaseModel): recent_rounds: list[LeaderboardRecentRoundPublic] | None = None +LeaderboardValue = str | int | float | bool | None + + +class DivisionLeaderboardAxisSnapshot(BaseModel): + key: str + label: str | None = None + value_type: Literal["number", "integer", "string", "boolean"] = "number" + sort: Literal["asc", "desc"] | None = None + + +class DivisionLeaderboardRowSnapshot(BaseModel): + subject_type: str = "player" + subject_id: str + subject_name: str | None = None + values: dict[str, LeaderboardValue] = Field(default_factory=dict) + policy_version_ids: set[UUID] = Field(default_factory=set) + recent_rounds: list[LeaderboardRecentRoundPublic] | None = None + + +class DivisionLeaderboardViewSnapshot(BaseModel): + key: str = "score" + title: str | None = None + description: str | None = None + axes: list[DivisionLeaderboardAxisSnapshot] = Field(default_factory=list) + rows: list[DivisionLeaderboardRowSnapshot] = Field(default_factory=list) + + +class DivisionLeaderboardTableSnapshot(BaseModel): + # TODO: delete compatibility model after callers stop reading table-shaped leaderboards. + id: str = "score" + label: str = "Score" + description: str | None = None + score_label: str = "Score" + entries: list[DivisionLeaderboardSnapshot] = Field(default_factory=list) + + +class DivisionLeaderboardsSnapshot(BaseModel): + default_view_key: str = "score" + views: list[DivisionLeaderboardViewSnapshot] = Field(default_factory=list) + + def default_view(self) -> DivisionLeaderboardViewSnapshot: + if not self.views: + return DivisionLeaderboardViewSnapshot(key=self.default_view_key) + return next((view for view in self.views if view.key == self.default_view_key), self.views[0]) + + def legacy_entries(self) -> list[DivisionLeaderboardSnapshot]: + # TODO: delete compatibility shim after platform clients read generic leaderboard views. + view = self.default_view() + score_axis_key = _legacy_score_axis_key(view) + return [ + _legacy_entry_from_row(row, rank, score_axis_key) + for rank, row in enumerate(view.rows, start=1) + ] + + @property + def primary_table_id(self) -> str: + # TODO: delete compatibility shim after callers stop using table terminology. + return self.default_view_key + + @property + def tables(self) -> list[DivisionLeaderboardTableSnapshot]: + # TODO: delete compatibility shim after callers stop using table terminology. + return [_table_from_view(view) for view in self.views] + + def primary_table(self) -> DivisionLeaderboardTableSnapshot: + # TODO: delete compatibility shim after callers stop using table terminology. + return _table_from_view(self.default_view()) + + +def _legacy_score_axis_key(view: DivisionLeaderboardViewSnapshot) -> str: + for axis in view.axes: + if axis.key != "rank" and axis.sort == "desc" and axis.value_type in {"number", "integer"}: + return axis.key + for axis in view.axes: + if axis.key != "rank" and axis.value_type in {"number", "integer"}: + return axis.key + return "score" + + +def _legacy_entry_from_row( + row: DivisionLeaderboardRowSnapshot, + rank: int, + score_axis_key: str, +) -> DivisionLeaderboardSnapshot: + score = row.values.get(score_axis_key) + row_rank = row.values.get("rank", rank) + rounds_played = row.values.get("rounds_played", 0) + return DivisionLeaderboardSnapshot( + player_id=row.subject_id, + player_name=row.subject_name, + rank=int(row_rank) if isinstance(row_rank, (int, float)) else rank, + score=float(score) if isinstance(score, (int, float)) else 0.0, + rounds_played=int(rounds_played) if isinstance(rounds_played, (int, float)) else 0, + policy_version_ids=row.policy_version_ids, + recent_rounds=row.recent_rounds, + ) + + +def _table_from_view(view: DivisionLeaderboardViewSnapshot) -> DivisionLeaderboardTableSnapshot: + score_axis_key = _legacy_score_axis_key(view) + score_axis = next((axis for axis in view.axes if axis.key == score_axis_key), None) + return DivisionLeaderboardTableSnapshot( + id=view.key, + label=view.title or view.key, + description=view.description, + score_label=(score_axis.label if score_axis is not None and score_axis.label is not None else score_axis_key), + entries=[ + _legacy_entry_from_row(row, rank, score_axis_key) + for rank, row in enumerate(view.rows, start=1) + ], + ) + + +# TODO: delete compatibility alias after callers stop referring to leaderboard tables. +DivisionLeaderboardTablesSnapshot = DivisionLeaderboardsSnapshot + + class _LeaderboardAgg(BaseModel): player_id: PlayerId player_name: str | None = None diff --git a/commissioners/common/protocol.py b/commissioners/common/protocol.py index 8c16f05..3eb5dd8 100644 --- a/commissioners/common/protocol.py +++ b/commissioners/common/protocol.py @@ -2,7 +2,7 @@ import json from secrets import randbelow -from typing import Any +from typing import Any, Literal from uuid import UUID from pydantic import BaseModel, Field, field_validator, model_validator @@ -81,6 +81,13 @@ class EpisodeScore(BaseModel): policy_version_id: UUID player_id: str | None = None score: float + scores: dict[str, float] = Field(default_factory=dict) + + @model_validator(mode="after") + def include_primary_score(self) -> EpisodeScore: + if "score" not in self.scores: + self.scores = {"score": self.score, **self.scores} + return self class RankingEntry(BaseModel): @@ -178,6 +185,109 @@ class DivisionLeaderboardEntry(BaseModel): recent_rounds: list[dict[str, Any]] | None = None +LeaderboardValue = str | int | float | bool | None + + +class DivisionLeaderboardAxis(BaseModel): + key: str + label: str | None = None + value_type: Literal["number", "integer", "string", "boolean"] = "number" + sort: Literal["asc", "desc"] | None = None + + +class DivisionLeaderboardRow(BaseModel): + subject_type: str = "player" + subject_id: str + subject_name: str | None = None + values: dict[str, LeaderboardValue] = Field(default_factory=dict) + policy_version_ids: set[UUID] = Field(default_factory=set) + recent_rounds: list[dict[str, Any]] | None = None + + +class DivisionLeaderboardView(BaseModel): + key: str = "score" + title: str | None = None + description: str | None = None + axes: list[DivisionLeaderboardAxis] = Field(default_factory=list) + rows: list[DivisionLeaderboardRow] = Field(default_factory=list) + + +class DivisionLeaderboardTable(BaseModel): + # TODO: delete compatibility model after all commissioners publish DivisionLeaderboardView. + # Stable table identifier used by primary_table_id and clients; usually the metric key. + id: str = "score" + # Human-facing table/tab title, e.g. "Winrate 24h". + label: str = "Score" + description: str | None = None + # Human-facing label for entry.score in this table, e.g. "Winrate". + score_label: str = "Score" + rankings: list[DivisionLeaderboardEntry] = Field(default_factory=list) + + +def _legacy_score_axis_key(view: DivisionLeaderboardView) -> str: + for axis in view.axes: + if axis.key != "rank" and axis.sort == "desc" and axis.value_type in {"number", "integer"}: + return axis.key + for axis in view.axes: + if axis.key != "rank" and axis.value_type in {"number", "integer"}: + return axis.key + return "score" + + +def _entry_from_row(row: DivisionLeaderboardRow, rank: int, score_axis_key: str) -> DivisionLeaderboardEntry: + score = row.values.get(score_axis_key) + row_rank = row.values.get("rank", rank) + rounds_played = row.values.get("rounds_played", 0) + return DivisionLeaderboardEntry( + player_id=row.subject_id, + player_name=row.subject_name, + rank=int(row_rank) if isinstance(row_rank, (int, float)) else rank, + score=float(score) if isinstance(score, (int, float)) else 0.0, + rounds_played=int(rounds_played) if isinstance(rounds_played, (int, float)) else 0, + policy_version_ids=row.policy_version_ids, + recent_rounds=row.recent_rounds, + ) + + +def _row_from_entry(entry: DivisionLeaderboardEntry) -> DivisionLeaderboardRow: + return DivisionLeaderboardRow( + subject_type="player", + subject_id=entry.player_id, + subject_name=entry.player_name, + values={"rank": entry.rank, "score": entry.score, "rounds_played": entry.rounds_played}, + policy_version_ids=entry.policy_version_ids, + recent_rounds=entry.recent_rounds, + ) + + +def _view_from_table(table: DivisionLeaderboardTable) -> DivisionLeaderboardView: + # TODO: delete compatibility shim after table-shaped commissioner responses are gone. + return DivisionLeaderboardView( + key=table.id, + title=table.label, + description=table.description, + axes=[ + DivisionLeaderboardAxis(key="rank", label="Rank", value_type="integer", sort="asc"), + DivisionLeaderboardAxis(key="score", label=table.score_label, value_type="number", sort="desc"), + DivisionLeaderboardAxis(key="rounds_played", label="Rounds Played", value_type="integer"), + ], + rows=[_row_from_entry(entry) for entry in table.rankings], + ) + + +def _table_from_view(view: DivisionLeaderboardView) -> DivisionLeaderboardTable: + # TODO: delete compatibility shim after callers stop reading table-shaped rank responses. + score_axis_key = _legacy_score_axis_key(view) + score_axis = next((axis for axis in view.axes if axis.key == score_axis_key), None) + return DivisionLeaderboardTable( + id=view.key, + label=view.title or view.key, + description=view.description, + score_label=(score_axis.label if score_axis is not None and score_axis.label is not None else score_axis_key), + rankings=[_entry_from_row(row, rank, score_axis_key) for rank, row in enumerate(view.rows, start=1)], + ) + + class DivisionDescription(BaseModel): round_schedule: str | None = None next_round: str | None = None @@ -368,8 +478,47 @@ def to_json(self) -> dict[str, Any]: class RankDivisionResponse(BaseModel): + default_view_key: str = "score" + views: list[DivisionLeaderboardView] = Field(default_factory=list) + # TODO: delete compatibility fields after metta and clients read generic `views`. + primary_table_id: str | None = None + tables: list[DivisionLeaderboardTable] = Field(default_factory=list) rankings: list[DivisionLeaderboardEntry] = Field(default_factory=list) + @model_validator(mode="after") + def fill_compatibility_fields(self) -> "RankDivisionResponse": + if not self.views and self.tables: + self.default_view_key = self.primary_table_id or self.tables[0].id + self.views = [_view_from_table(table) for table in self.tables] + if not self.views and self.rankings: + self.views = [ + DivisionLeaderboardView( + key=self.default_view_key, + title="Score", + axes=[ + DivisionLeaderboardAxis(key="rank", label="Rank", value_type="integer", sort="asc"), + DivisionLeaderboardAxis(key="score", label="Score", value_type="number", sort="desc"), + DivisionLeaderboardAxis(key="rounds_played", label="Rounds Played", value_type="integer"), + ], + rows=[_row_from_entry(entry) for entry in self.rankings], + ) + ] + if not self.views: + self.views = [DivisionLeaderboardView(key=self.default_view_key)] + if not any(view.key == self.default_view_key for view in self.views): + self.default_view_key = self.views[0].key + if self.primary_table_id is None: + self.primary_table_id = self.default_view_key + if not self.tables: + self.tables = [_table_from_view(view) for view in self.views] + if not self.rankings: + default_view = next((view for view in self.views if view.key == self.default_view_key), self.views[0]) + score_axis_key = _legacy_score_axis_key(default_view) + self.rankings = [ + _entry_from_row(row, rank, score_axis_key) for rank, row in enumerate(default_view.rows, start=1) + ] + return self + def to_json(self) -> dict[str, Any]: data = self.model_dump(mode="json") data["type"] = "rank_division_response" diff --git a/commissioners/common/ruleset_strategy/commissioner.py b/commissioners/common/ruleset_strategy/commissioner.py index 58a4423..41a48bf 100644 --- a/commissioners/common/ruleset_strategy/commissioner.py +++ b/commissioners/common/ruleset_strategy/commissioner.py @@ -12,6 +12,7 @@ DivisionDescriptionContext, DivisionLeaderboardContext, DivisionLeaderboardSnapshot, + DivisionLeaderboardsSnapshot, EpisodeResult, LeagueMigrationConfigContext, LeagueMigrationContext, @@ -124,6 +125,28 @@ def rank_division(self, ctx: DivisionLeaderboardContext) -> list[DivisionLeaderb ctx = ctx.model_copy(update={"round_results": filtered}) return super().rank_division(ctx) + def rank_division_leaderboards(self, ctx: DivisionLeaderboardContext) -> DivisionLeaderboardsSnapshot: + config = self._config() + if config.ranking.filter_metadata: + filtered = [ + result + for result in ctx.round_results + if all(result.result_metadata.get(key) == value for key, value in config.ranking.filter_metadata.items()) + ] + ctx = ctx.model_copy(update={"round_results": filtered}) + table_configs = [] + default_half_life_hours = config.ranking.ewma_halflife_hours + if config.scoring is not None: + table_configs = [ + table.model_dump(exclude_none=True) + for table in [*config.scoring.leaderboards, *config.scoring.leaderboard_tables] + ] + return self._rank_division_leaderboards_from_config( + ctx, + table_configs, + default_half_life_hours=default_half_life_hours, + ) + def _leaderboard_ewma_halflife(self, ctx: DivisionLeaderboardContext) -> timedelta: config = self._config() return timedelta(hours=config.ranking.ewma_halflife_hours) diff --git a/commissioners/common/ruleset_strategy/config.py b/commissioners/common/ruleset_strategy/config.py index 1d28eaf..034a8a1 100644 --- a/commissioners/common/ruleset_strategy/config.py +++ b/commissioners/common/ruleset_strategy/config.py @@ -159,12 +159,55 @@ class LeaderboardScoringConfig(_ConfigModel): half_life_hours: float = Field(default=2.0, gt=0) +class LeaderboardAxisConfig(_ConfigModel): + key: str + label: str | None = None + value_type: Literal["number", "integer", "string", "boolean"] = "number" + sort: Literal["asc", "desc"] | None = None + + +class LeaderboardViewConfig(_ConfigModel): + key: str = "" + title: str | None = None + description: str | None = None + score_key: str | None = None + score_axis_label: str | None = None + axes: list[LeaderboardAxisConfig] = Field(default_factory=list) + half_life_hours: float | None = Field(default=None, gt=0) + window_hours: float | None = Field(default=None, gt=0) + lookback_hours: float | None = Field(default=None, gt=0) + default: bool = False + # TODO: delete compatibility fields after old config payloads stop using table terminology. + id: str | None = None + label: str | None = None + score_label: str | None = None + primary: bool = False + + @model_validator(mode="after") + def fill_compatibility_names(self) -> "LeaderboardViewConfig": + if self.id is not None and self.key == "": + self.key = self.id + if self.key == "": + score_axis = next((axis for axis in self.axes if axis.sort == "desc"), None) + self.key = self.score_key or (score_axis.key if score_axis is not None else "score") + if self.title is None and self.label is not None: + self.title = self.label + if self.score_axis_label is None and self.score_label is not None: + self.score_axis_label = self.score_label + if self.primary: + self.default = True + return self + + class ScoringConfig(_ConfigModel): # "mean": round score is the mean of a policy's per-episode scores. # "rank": round score is the mean of a policy's per-episode rank points (placement within # each episode, N..1), so margins of victory are discarded and only placement counts. round_score: Literal["mean", "rank"] = "mean" leaderboard: LeaderboardScoringConfig = Field(default_factory=LeaderboardScoringConfig) + leaderboards: list[LeaderboardViewConfig] = Field(default_factory=list) + # TODO: delete compatibility field after old config payloads stop using table terminology. + leaderboard_tables: list[LeaderboardViewConfig] = Field(default_factory=list) mechanics: str | None = None diff --git a/tests/test_commissioner_strategies.py b/tests/test_commissioner_strategies.py index a1f1c58..fe54c06 100644 --- a/tests/test_commissioner_strategies.py +++ b/tests/test_commissioner_strategies.py @@ -312,19 +312,19 @@ def test_default_commissioner_round_robin_generation_and_ranking() -> None: EpisodeResult( episode_request_id=uuid4(), scores=[ - RoundPolicyScore(policy_version_id=policy_version_ids[0], score=4.0), - RoundPolicyScore(policy_version_id=policy_version_ids[1], score=2.0), - RoundPolicyScore(policy_version_id=policy_version_ids[2], score=6.0), - RoundPolicyScore(policy_version_id=policy_version_ids[0], score=8.0), + RoundPolicyScore(policy_version_id=policy_version_ids[0], score=4.0, scores={"territory": 1.0}), + RoundPolicyScore(policy_version_id=policy_version_ids[1], score=2.0, scores={"territory": 2.0}), + RoundPolicyScore(policy_version_id=policy_version_ids[2], score=6.0, scores={"territory": 3.0}), + RoundPolicyScore(policy_version_id=policy_version_ids[0], score=8.0, scores={"territory": 5.0}), ], ), EpisodeResult( episode_request_id=uuid4(), scores=[ - RoundPolicyScore(policy_version_id=policy_version_ids[1], score=10.0), - RoundPolicyScore(policy_version_id=policy_version_ids[2], score=0.0), - RoundPolicyScore(policy_version_id=policy_version_ids[0], score=6.0), - RoundPolicyScore(policy_version_id=policy_version_ids[1], score=4.0), + RoundPolicyScore(policy_version_id=policy_version_ids[1], score=10.0, scores={"territory": 4.0}), + RoundPolicyScore(policy_version_id=policy_version_ids[2], score=0.0, scores={"territory": 1.0}), + RoundPolicyScore(policy_version_id=policy_version_ids[0], score=6.0, scores={"territory": 2.0}), + RoundPolicyScore(policy_version_id=policy_version_ids[1], score=4.0, scores={"territory": 6.0}), ], ), ], @@ -337,6 +337,7 @@ def test_default_commissioner_round_robin_generation_and_ranking() -> None: policy_version_ids[2], ] assert [ranking.score for ranking in rankings] == pytest.approx([6.0, 16.0 / 3.0, 3.0]) + assert rankings[0].result_metadata["scores"] == pytest.approx({"score": 6.0, "territory": 8.0 / 3.0}) def test_ruleset_strategy_rank_round_score_uses_per_episode_placement() -> None: @@ -1662,6 +1663,108 @@ def test_ruleset_strategy_scoring_configures_leaderboard_ewma_halflife() -> None assert response.rankings[0].score == pytest.approx(8.0) +def test_ruleset_strategy_scoring_configures_multiple_leaderboard_views() -> None: + league_id = uuid4() + division_id = uuid4() + recent_round_id = uuid4() + old_round_id = uuid4() + player_a_policy_id = uuid4() + player_b_policy_id = uuid4() + commissioner = RulesetStrategyCommissioner( + { + "scoring": { + "round_score": "mean", + "leaderboards": [ + { + "key": "score_all", + "title": "Score", + "axes": [ + {"key": "rank", "label": "Rank", "value_type": "integer", "sort": "asc"}, + {"key": "score", "label": "Score", "value_type": "number", "sort": "desc"}, + {"key": "rounds_played", "label": "Rounds Played", "value_type": "integer"}, + ], + }, + { + "key": "winrate_24h", + "title": "Winrate 24h", + "window_hours": 24, + "default": True, + "axes": [ + {"key": "rank", "label": "Rank", "value_type": "integer", "sort": "asc"}, + {"key": "winrate", "label": "Winrate", "value_type": "number", "sort": "desc"}, + {"key": "rounds_played", "label": "Rounds Played", "value_type": "integer"}, + ], + }, + ], + }, + "divisions": {"competition": {"match": {"type": "competition"}, "entrants": "champions"}}, + } + ) + + response = rank_division_for_request( + commissioner, + RankDivisionRequest( + league=LeagueInfo(id=league_id), + division=DivisionInfo(id=division_id, name="Competition", level=1), + completed_rounds=[ + RoundInfo( + id=recent_round_id, + public_id="round_recent", + division_id=division_id, + round_number=2, + status="completed", + completed_at="2026-06-23T00:00:00+00:00", + ), + RoundInfo( + id=old_round_id, + public_id="round_old", + division_id=division_id, + round_number=1, + status="completed", + completed_at="2026-06-21T00:00:00+00:00", + ), + ], + recent_rounds=[], + round_results=[ + LeaderboardRoundResultInfo( + round_id=recent_round_id, + policy_version_id=player_a_policy_id, + player_id="player-a", + player_name="Alice", + rank=2, + score=10.0, + result_metadata={"score_kind": MEAN_ROUND_SCORE_KIND, "scores": {"score": 10.0, "winrate": 0.2}}, + ), + LeaderboardRoundResultInfo( + round_id=recent_round_id, + policy_version_id=player_b_policy_id, + player_id="player-b", + player_name="Bob", + rank=1, + score=8.0, + result_metadata={"score_kind": MEAN_ROUND_SCORE_KIND, "scores": {"score": 8.0, "winrate": 0.9}}, + ), + LeaderboardRoundResultInfo( + round_id=old_round_id, + policy_version_id=player_a_policy_id, + player_id="player-a", + player_name="Alice", + rank=1, + score=50.0, + result_metadata={"score_kind": MEAN_ROUND_SCORE_KIND, "scores": {"score": 50.0, "winrate": 1.0}}, + ), + ], + ), + ) + + assert response.default_view_key == "winrate_24h" + assert [view.key for view in response.views] == ["score_all", "winrate_24h"] + assert response.views[0].rows[0].subject_id == "player-a" + assert response.views[1].rows[0].subject_id == "player-b" + assert response.views[1].rows[0].values["winrate"] == pytest.approx(0.9) + assert response.rankings[0].player_id == "player-b" + + def test_baseline_round_start_uses_is_champion_for_competition_entries() -> None: division_id = uuid4() boolean_champion_id = uuid4()