diff --git a/commissioners/common/ruleset_strategy/commissioner.py b/commissioners/common/ruleset_strategy/commissioner.py index 58a4423..6947016 100644 --- a/commissioners/common/ruleset_strategy/commissioner.py +++ b/commissioners/common/ruleset_strategy/commissioner.py @@ -39,9 +39,11 @@ _count_text, _current_schedule_slot, _duration_text, + _episode_points_lists_by_policy, + _episode_rank_points, + _episode_win_points, _leaderboard_rules_description, _plural_word, - _rank_points_lists_by_policy, _round_structure_description, _schedule_slot_description, ) @@ -254,9 +256,10 @@ def _round_scores_by_policy( episode_results: list[EpisodeResult], ) -> tuple[dict[UUID, float], dict[UUID, int]]: scoring = self._config().scoring - if scoring is None or scoring.round_score != "rank": + if scoring is None or scoring.round_score == "mean": return super()._round_scores_by_policy(entries, episode_results) - points_lists = _rank_points_lists_by_policy(episode_results) + episode_points = _episode_win_points if scoring.round_score == "win" else _episode_rank_points + points_lists = _episode_points_lists_by_policy(episode_results, episode_points) scores = { entry.policy_version_id: ( sum(points_lists.get(entry.policy_version_id, [])) diff --git a/commissioners/common/ruleset_strategy/config.py b/commissioners/common/ruleset_strategy/config.py index 1d28eaf..ba6ef2c 100644 --- a/commissioners/common/ruleset_strategy/config.py +++ b/commissioners/common/ruleset_strategy/config.py @@ -21,6 +21,8 @@ MEAN_SCORE_EWMA_SCORING_MECHANICS, RANK_EPISODE_EWMA_SCORING_MECHANICS, RANK_EPISODE_ROUND_SCORE_KIND, + WIN_EPISODE_EWMA_SCORING_MECHANICS, + WIN_EPISODE_ROUND_SCORE_KIND, ) CONFIG_KEY = "ruleset_strategy" @@ -163,7 +165,9 @@ class ScoringConfig(_ConfigModel): # "mean": round score is the mean of a policy's per-episode scores. # "rank": round score is the mean of a policy's per-episode rank points (placement within # each episode, N..1), so margins of victory are discarded and only placement counts. - round_score: Literal["mean", "rank"] = "mean" + # "win": round score is the policy's win rate — 1 for each episode it (co-)won, 0 otherwise — + # so only winning the game matters, not placement or margin. + round_score: Literal["mean", "rank", "win"] = "mean" leaderboard: LeaderboardScoringConfig = Field(default_factory=LeaderboardScoringConfig) mechanics: str | None = None @@ -365,8 +369,12 @@ def insufficient_players(self) -> InsufficientPlayersConfig: @property def round_score_kind(self) -> str: - if self.scoring is not None and self.scoring.round_score == "rank": + if self.scoring is None: + return MEAN_ROUND_SCORE_KIND + if self.scoring.round_score == "rank": return RANK_EPISODE_ROUND_SCORE_KIND + if self.scoring.round_score == "win": + return WIN_EPISODE_ROUND_SCORE_KIND return MEAN_ROUND_SCORE_KIND @property @@ -389,17 +397,27 @@ def scoring_mechanics(self) -> str | None: if self.scoring.mechanics is not None: return self.scoring.mechanics half_life_hours = self.scoring.leaderboard.half_life_hours - is_rank = self.scoring.round_score == "rank" + round_score = self.scoring.round_score if half_life_hours == 2: - return RANK_EPISODE_EWMA_SCORING_MECHANICS if is_rank else MEAN_SCORE_EWMA_SCORING_MECHANICS + return { + "rank": RANK_EPISODE_EWMA_SCORING_MECHANICS, + "win": WIN_EPISODE_EWMA_SCORING_MECHANICS, + }.get(round_score, MEAN_SCORE_EWMA_SCORING_MECHANICS) half_life_text = int(half_life_hours) if half_life_hours.is_integer() else half_life_hours - if is_rank: + if round_score == "rank": return ( "Rounds rank policies by placement within each episode (N points for the episode winner of an " "N-policy game down to 1 for last, ties sharing the better place), averaged across the episodes " "each policy played. The division leaderboard combines completed rounds with a " f"{half_life_text}-hour half-life EWMA, so newer rounds count more than older rounds." ) + if round_score == "win": + return ( + "Rounds score policies by win rate within each episode (1 for the episode winner, 0 for everyone " + "else, a tie for first sharing the win), averaged across the episodes each policy played. The " + "division leaderboard combines completed rounds with a " + f"{half_life_text}-hour half-life EWMA, so newer rounds count more than older rounds." + ) return ( "Rounds rank policies by the average score reported by the game across each policy's episode slots. " "The division leaderboard only uses current average-score round results and combines completed rounds " diff --git a/commissioners/common/utils.py b/commissioners/common/utils.py index dd397e7..d38b9b1 100644 --- a/commissioners/common/utils.py +++ b/commissioners/common/utils.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections import defaultdict +from collections.abc import Callable from datetime import UTC, datetime, timedelta from math import ceil from typing import Any @@ -26,6 +27,7 @@ AMONG_THEM_RESULT_METADATA_VERSION = 2 MEAN_ROUND_SCORE_KIND = "mean_round_score" RANK_EPISODE_ROUND_SCORE_KIND = "rank_episode_round_score" +WIN_EPISODE_ROUND_SCORE_KIND = "win_episode_round_score" COMPLETED_EPISODE_COUNT_METADATA_KEY = "completed_episode_count" RANKED_SCORE_COUNT_METADATA_KEY = "ranked_score_count" MEAN_SCORE_EWMA_SCORING_MECHANICS = ( @@ -42,6 +44,13 @@ "are discarded — only who beat whom each game matters. The division leaderboard combines completed rounds with " "a 2-hour half-life EWMA, so newer rounds count more than older rounds." ) +WIN_EPISODE_EWMA_SCORING_MECHANICS = ( + "Rounds score policies by win rate rather than by raw score or placement: in each episode the " + "top-scoring policy earns 1 and everyone else earns 0 (a tie for first shares the win, so every " + "top scorer gets 1), and a policy's round score is the fraction of its episodes it won. Margins and " + "lower placements are discarded — only winning the game matters. The division leaderboard combines " + "completed rounds with a 2-hour half-life EWMA, so newer rounds count more than older rounds." +) AMONG_THEM_SCORE_KIND = MEAN_ROUND_SCORE_KIND AMONG_THEM_SCORING_MECHANICS = MEAN_SCORE_EWMA_SCORING_MECHANICS @@ -233,17 +242,33 @@ def _episode_rank_points(scores: list[float]) -> list[float]: return [float(n - sum(1 for other in scores if other > score)) for score in scores] -def _rank_points_lists_by_policy(episode_results: list[EpisodeResult]) -> dict[UUID, list[float]]: - """Per-policy lists of per-episode rank points across every episode the policy played. +def _episode_win_points(scores: list[float]) -> list[float]: + """Convert one episode's per-policy scores into binary win points. + + The episode's top scorer earns 1 and everyone else earns 0; a tie for first shares the win, so + every policy at the top score gets 1. Margins and lower placements are discarded — only winning + the episode counts. An empty episode yields no points. + """ + if not scores: + return [] + top = max(scores) + return [1.0 if score == top else 0.0 for score in scores] + + +def _episode_points_lists_by_policy( + episode_results: list[EpisodeResult], + episode_points: Callable[[list[float]], list[float]], +) -> dict[UUID, list[float]]: + """Per-policy lists of per-episode points (rank or win) across every episode the policy played. - Unlike ``_score_lists_by_policy`` no scores are dropped: placement is meaningful for every - seat in an episode, including a zero score, so each episode contributes one rank point per + Unlike ``_score_lists_by_policy`` no scores are dropped: ``episode_points`` is meaningful for + every seat in an episode, including a zero score, so each episode contributes one point per participating policy. """ points_lists: dict[UUID, list[float]] = defaultdict(list) for result in episode_results: episode_scores = [(score.policy_version_id, score.score) for score in result.scores] - points = _episode_rank_points([score for _, score in episode_scores]) + points = episode_points([score for _, score in episode_scores]) for (policy_version_id, _), point in zip(episode_scores, points, strict=True): points_lists[policy_version_id].append(point) return points_lists diff --git a/commissioners/ruleset_strategy_commissioner/configs/agricogla.yaml b/commissioners/ruleset_strategy_commissioner/configs/agricogla.yaml index ad5ba6d..2d24d35 100644 --- a/commissioners/ruleset_strategy_commissioner/configs/agricogla.yaml +++ b/commissioners/ruleset_strategy_commissioner/configs/agricogla.yaml @@ -1,10 +1,11 @@ # agricogla: 4-seat worker-placement game. Same as `default` except: -# - competition rounds run 100 episodes (not 1) so baseline_window seating seats +# - competition rounds run 50 episodes (not 1) so baseline_window seating seats # every champion across the 4 seats each round instead of only 4 of them. -# - rounds are scored by per-episode placement (round_score: rank) instead of mean -# score, so the league rewards consistently beating the table, not blowouts. +# - rounds are scored by per-episode win rate (round_score: win): 1 for the episode +# winner, 0 for everyone else (a tie for first shares the win), so the league +# rewards winning games outright, not placement or margin. scoring: - round_score: rank + round_score: win defaults: seating: baseline_window @@ -12,7 +13,7 @@ defaults: min_entries_to_start: 2 stage: label: Round - episodes: 100 + episodes: 50 min_episodes_per_entrant: 1 divisions: diff --git a/tests/test_commissioner_strategies.py b/tests/test_commissioner_strategies.py index a1f1c58..1d95f2a 100644 --- a/tests/test_commissioner_strategies.py +++ b/tests/test_commissioner_strategies.py @@ -395,6 +395,65 @@ def test_ruleset_strategy_rank_round_score_uses_per_episode_placement() -> None: assert by_policy[policy_version_ids[0]].result_metadata["score_kind"] == "rank_episode_round_score" +def test_ruleset_strategy_win_round_score_uses_binary_win_points() -> None: + # scoring.round_score = "win": each episode's top scorer earns 1 and everyone else 0 (a tie + # for first shares the win), and a policy's round score is its win rate across its seats. + policy_version_ids = [uuid4() for _ in range(3)] + pool = PolicyPool(id=uuid4(), label="Round", pool_type="round", config={"num_episodes": 2}) + entries = [ + PolicyPoolEntry(pool_id=pool.id, policy_version_id=policy_version_id, seed_order=index) + for index, policy_version_id in enumerate(policy_version_ids) + ] + commissioner = RulesetStrategyCommissioner( + { + "scoring": {"round_score": "win"}, + "divisions": {"competition": {"match": {"type": "competition"}, "entrants": "champions"}}, + } + ) + + complete = commissioner.complete_round( + round_row=Round(id=uuid4(), division_id=uuid4(), round_number=1, commissioner_key="ruleset_strategy"), + pool=pool, + entries=entries, + episode_results=[ + EpisodeResult( + episode_request_id=uuid4(), + scores=[ + RoundPolicyScore(policy_version_id=policy_version_ids[0], score=10.0), + RoundPolicyScore(policy_version_id=policy_version_ids[1], score=5.0), + RoundPolicyScore(policy_version_id=policy_version_ids[2], score=3.0), + RoundPolicyScore(policy_version_id=policy_version_ids[0], score=2.0), + ], + ), + EpisodeResult( + episode_request_id=uuid4(), + scores=[ + RoundPolicyScore(policy_version_id=policy_version_ids[1], score=8.0), + RoundPolicyScore(policy_version_id=policy_version_ids[1], score=8.0), + RoundPolicyScore(policy_version_id=policy_version_ids[2], score=4.0), + RoundPolicyScore(policy_version_id=policy_version_ids[0], score=1.0), + ], + ), + ], + ) + + rankings = complete.results[0].rankings + by_policy = {ranking.policy_version_id: ranking for ranking in rankings} + # Per-episode win points (1 for the episode's top score, ties shared), averaged across a policy's seats: + # p0: ep1 10->1, 2->0; ep2 1->0 => (1+0+0)/3 = 1/3 + # p1: ep1 5->0; ep2 8->1, 8->1 => (0+1+1)/3 = 2/3 (the tied-for-first seats both win) + # p2: ep1 3->0; ep2 4->0 => (0+0)/2 = 0.0 + assert by_policy[policy_version_ids[0]].score == pytest.approx(1.0 / 3.0) + assert by_policy[policy_version_ids[1]].score == pytest.approx(2.0 / 3.0) + assert by_policy[policy_version_ids[2]].score == pytest.approx(0.0) + assert [ranking.policy_version_id for ranking in rankings] == [ + policy_version_ids[1], + policy_version_ids[0], + policy_version_ids[2], + ] + assert by_policy[policy_version_ids[1]].result_metadata["score_kind"] == "win_episode_round_score" + + def test_default_commissioner_ignores_neutral_zero_scores_only_when_episode_has_negative_score() -> None: policy_version_ids = [uuid4() for _ in range(3)] pool = PolicyPool(