feat(agricogla): win-rate round scoring (1 for the winner, 0 otherwise), 50 episodes/round

commissioners/common/ruleset_strategy/commissioner.py

@@ -39,9 +39,11 @@
     _count_text,
     _current_schedule_slot,
     _duration_text,
+    _episode_points_lists_by_policy,
+    _episode_rank_points,
+    _episode_win_points,
     _leaderboard_rules_description,
     _plural_word,
-    _rank_points_lists_by_policy,
     _round_structure_description,
     _schedule_slot_description,
 )
@@ -254,9 +256,10 @@ def _round_scores_by_policy(
         episode_results: list[EpisodeResult],
     ) -> tuple[dict[UUID, float], dict[UUID, int]]:
         scoring = self._config().scoring
-        if scoring is None or scoring.round_score != "rank":
+        if scoring is None or scoring.round_score == "mean":
             return super()._round_scores_by_policy(entries, episode_results)
-        points_lists = _rank_points_lists_by_policy(episode_results)
+        episode_points = _episode_win_points if scoring.round_score == "win" else _episode_rank_points
+        points_lists = _episode_points_lists_by_policy(episode_results, episode_points)
         scores = {
             entry.policy_version_id: (
                 sum(points_lists.get(entry.policy_version_id, []))

commissioners/common/ruleset_strategy/config.py

@@ -21,6 +21,8 @@
     MEAN_SCORE_EWMA_SCORING_MECHANICS,
     RANK_EPISODE_EWMA_SCORING_MECHANICS,
     RANK_EPISODE_ROUND_SCORE_KIND,
+    WIN_EPISODE_EWMA_SCORING_MECHANICS,
+    WIN_EPISODE_ROUND_SCORE_KIND,
 )
 
 CONFIG_KEY = "ruleset_strategy"
@@ -163,7 +165,9 @@ class ScoringConfig(_ConfigModel):
     # "mean": round score is the mean of a policy's per-episode scores.
     # "rank": round score is the mean of a policy's per-episode rank points (placement within
     #         each episode, N..1), so margins of victory are discarded and only placement counts.
-    round_score: Literal["mean", "rank"] = "mean"
+    # "win":  round score is the policy's win rate — 1 for each episode it (co-)won, 0 otherwise —
+    #         so only winning the game matters, not placement or margin.
+    round_score: Literal["mean", "rank", "win"] = "mean"
     leaderboard: LeaderboardScoringConfig = Field(default_factory=LeaderboardScoringConfig)
     mechanics: str | None = None
 
@@ -365,8 +369,12 @@ def insufficient_players(self) -> InsufficientPlayersConfig:
 
     @property
     def round_score_kind(self) -> str:
-        if self.scoring is not None and self.scoring.round_score == "rank":
+        if self.scoring is None:
+            return MEAN_ROUND_SCORE_KIND
+        if self.scoring.round_score == "rank":
             return RANK_EPISODE_ROUND_SCORE_KIND
+        if self.scoring.round_score == "win":
+            return WIN_EPISODE_ROUND_SCORE_KIND
         return MEAN_ROUND_SCORE_KIND
 
     @property
@@ -389,17 +397,27 @@ def scoring_mechanics(self) -> str | None:
         if self.scoring.mechanics is not None:
             return self.scoring.mechanics
         half_life_hours = self.scoring.leaderboard.half_life_hours
-        is_rank = self.scoring.round_score == "rank"
+        round_score = self.scoring.round_score
         if half_life_hours == 2:
-            return RANK_EPISODE_EWMA_SCORING_MECHANICS if is_rank else MEAN_SCORE_EWMA_SCORING_MECHANICS
+            return {
+                "rank": RANK_EPISODE_EWMA_SCORING_MECHANICS,
+                "win": WIN_EPISODE_EWMA_SCORING_MECHANICS,
+            }.get(round_score, MEAN_SCORE_EWMA_SCORING_MECHANICS)
         half_life_text = int(half_life_hours) if half_life_hours.is_integer() else half_life_hours
-        if is_rank:
+        if round_score == "rank":
             return (
                 "Rounds rank policies by placement within each episode (N points for the episode winner of an "
                 "N-policy game down to 1 for last, ties sharing the better place), averaged across the episodes "
                 "each policy played. The division leaderboard combines completed rounds with a "
                 f"{half_life_text}-hour half-life EWMA, so newer rounds count more than older rounds."
             )
+        if round_score == "win":
+            return (
+                "Rounds score policies by win rate within each episode (1 for the episode winner, 0 for everyone "
+                "else, a tie for first sharing the win), averaged across the episodes each policy played. The "
+                "division leaderboard combines completed rounds with a "
+                f"{half_life_text}-hour half-life EWMA, so newer rounds count more than older rounds."
+            )
         return (
             "Rounds rank policies by the average score reported by the game across each policy's episode slots. "
             "The division leaderboard only uses current average-score round results and combines completed rounds "

commissioners/common/utils.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from collections import defaultdict
+from collections.abc import Callable
 from datetime import UTC, datetime, timedelta
 from math import ceil
 from typing import Any
@@ -26,6 +27,7 @@
 AMONG_THEM_RESULT_METADATA_VERSION = 2
 MEAN_ROUND_SCORE_KIND = "mean_round_score"
 RANK_EPISODE_ROUND_SCORE_KIND = "rank_episode_round_score"
+WIN_EPISODE_ROUND_SCORE_KIND = "win_episode_round_score"
 COMPLETED_EPISODE_COUNT_METADATA_KEY = "completed_episode_count"
 RANKED_SCORE_COUNT_METADATA_KEY = "ranked_score_count"
 MEAN_SCORE_EWMA_SCORING_MECHANICS = (
@@ -42,6 +44,13 @@
     "are discarded — only who beat whom each game matters. The division leaderboard combines completed rounds with "
     "a 2-hour half-life EWMA, so newer rounds count more than older rounds."
 )
+WIN_EPISODE_EWMA_SCORING_MECHANICS = (
+    "Rounds score policies by win rate rather than by raw score or placement: in each episode the "
+    "top-scoring policy earns 1 and everyone else earns 0 (a tie for first shares the win, so every "
+    "top scorer gets 1), and a policy's round score is the fraction of its episodes it won. Margins and "
+    "lower placements are discarded — only winning the game matters. The division leaderboard combines "
+    "completed rounds with a 2-hour half-life EWMA, so newer rounds count more than older rounds."
+)
 AMONG_THEM_SCORE_KIND = MEAN_ROUND_SCORE_KIND
 AMONG_THEM_SCORING_MECHANICS = MEAN_SCORE_EWMA_SCORING_MECHANICS
 
@@ -233,17 +242,33 @@ def _episode_rank_points(scores: list[float]) -> list[float]:
     return [float(n - sum(1 for other in scores if other > score)) for score in scores]
 
 
-def _rank_points_lists_by_policy(episode_results: list[EpisodeResult]) -> dict[UUID, list[float]]:
-    """Per-policy lists of per-episode rank points across every episode the policy played.
+def _episode_win_points(scores: list[float]) -> list[float]:
+    """Convert one episode's per-policy scores into binary win points.
+
+    The episode's top scorer earns 1 and everyone else earns 0; a tie for first shares the win, so
+    every policy at the top score gets 1. Margins and lower placements are discarded — only winning
+    the episode counts. An empty episode yields no points.
+    """
+    if not scores:
+        return []
+    top = max(scores)
+    return [1.0 if score == top else 0.0 for score in scores]
+
+
+def _episode_points_lists_by_policy(
+    episode_results: list[EpisodeResult],
+    episode_points: Callable[[list[float]], list[float]],
+) -> dict[UUID, list[float]]:
+    """Per-policy lists of per-episode points (rank or win) across every episode the policy played.
 
-    Unlike ``_score_lists_by_policy`` no scores are dropped: placement is meaningful for every
-    seat in an episode, including a zero score, so each episode contributes one rank point per
+    Unlike ``_score_lists_by_policy`` no scores are dropped: ``episode_points`` is meaningful for
+    every seat in an episode, including a zero score, so each episode contributes one point per
     participating policy.
     """
     points_lists: dict[UUID, list[float]] = defaultdict(list)
     for result in episode_results:
         episode_scores = [(score.policy_version_id, score.score) for score in result.scores]
-        points = _episode_rank_points([score for _, score in episode_scores])
+        points = episode_points([score for _, score in episode_scores])
         for (policy_version_id, _), point in zip(episode_scores, points, strict=True):
             points_lists[policy_version_id].append(point)
     return points_lists

commissioners/ruleset_strategy_commissioner/configs/agricogla.yaml

@@ -1,18 +1,19 @@
 # agricogla: 4-seat worker-placement game. Same as `default` except:
-#   - competition rounds run 100 episodes (not 1) so baseline_window seating seats
+#   - competition rounds run 50 episodes (not 1) so baseline_window seating seats
 #     every champion across the 4 seats each round instead of only 4 of them.
-#   - rounds are scored by per-episode placement (round_score: rank) instead of mean
-#     score, so the league rewards consistently beating the table, not blowouts.
+#   - rounds are scored by per-episode win rate (round_score: win): 1 for the episode
+#     winner, 0 for everyone else (a tie for first shares the win), so the league
+#     rewards winning games outright, not placement or margin.
 scoring:
-  round_score: rank
+  round_score: win
 
 defaults:
   seating: baseline_window
   fill_seats: duplicate
   min_entries_to_start: 2
   stage:
     label: Round
-    episodes: 100
+    episodes: 50
     min_episodes_per_entrant: 1
 
 divisions:

tests/test_commissioner_strategies.py

@@ -395,6 +395,65 @@ def test_ruleset_strategy_rank_round_score_uses_per_episode_placement() -> None:
     assert by_policy[policy_version_ids[0]].result_metadata["score_kind"] == "rank_episode_round_score"
 
 
+def test_ruleset_strategy_win_round_score_uses_binary_win_points() -> None:
+    # scoring.round_score = "win": each episode's top scorer earns 1 and everyone else 0 (a tie
+    # for first shares the win), and a policy's round score is its win rate across its seats.
+    policy_version_ids = [uuid4() for _ in range(3)]
+    pool = PolicyPool(id=uuid4(), label="Round", pool_type="round", config={"num_episodes": 2})
+    entries = [
+        PolicyPoolEntry(pool_id=pool.id, policy_version_id=policy_version_id, seed_order=index)
+        for index, policy_version_id in enumerate(policy_version_ids)
+    ]
+    commissioner = RulesetStrategyCommissioner(
+        {
+            "scoring": {"round_score": "win"},
+            "divisions": {"competition": {"match": {"type": "competition"}, "entrants": "champions"}},
+        }
+    )
+
+    complete = commissioner.complete_round(
+        round_row=Round(id=uuid4(), division_id=uuid4(), round_number=1, commissioner_key="ruleset_strategy"),
+        pool=pool,
+        entries=entries,
+        episode_results=[
+            EpisodeResult(
+                episode_request_id=uuid4(),
+                scores=[
+                    RoundPolicyScore(policy_version_id=policy_version_ids[0], score=10.0),
+                    RoundPolicyScore(policy_version_id=policy_version_ids[1], score=5.0),
+                    RoundPolicyScore(policy_version_id=policy_version_ids[2], score=3.0),
+                    RoundPolicyScore(policy_version_id=policy_version_ids[0], score=2.0),
+                ],
+            ),
+            EpisodeResult(
+                episode_request_id=uuid4(),
+                scores=[
+                    RoundPolicyScore(policy_version_id=policy_version_ids[1], score=8.0),
+                    RoundPolicyScore(policy_version_id=policy_version_ids[1], score=8.0),
+                    RoundPolicyScore(policy_version_id=policy_version_ids[2], score=4.0),
+                    RoundPolicyScore(policy_version_id=policy_version_ids[0], score=1.0),
+                ],
+            ),
+        ],
+    )
+
+    rankings = complete.results[0].rankings
+    by_policy = {ranking.policy_version_id: ranking for ranking in rankings}
+    # Per-episode win points (1 for the episode's top score, ties shared), averaged across a policy's seats:
+    #   p0: ep1 10->1, 2->0; ep2 1->0           => (1+0+0)/3 = 1/3
+    #   p1: ep1 5->0;        ep2 8->1, 8->1      => (0+1+1)/3 = 2/3  (the tied-for-first seats both win)
+    #   p2: ep1 3->0;        ep2 4->0           => (0+0)/2   = 0.0
+    assert by_policy[policy_version_ids[0]].score == pytest.approx(1.0 / 3.0)
+    assert by_policy[policy_version_ids[1]].score == pytest.approx(2.0 / 3.0)
+    assert by_policy[policy_version_ids[2]].score == pytest.approx(0.0)
+    assert [ranking.policy_version_id for ranking in rankings] == [
+        policy_version_ids[1],
+        policy_version_ids[0],
+        policy_version_ids[2],
+    ]
+    assert by_policy[policy_version_ids[1]].result_metadata["score_kind"] == "win_episode_round_score"
+
+
 def test_default_commissioner_ignores_neutral_zero_scores_only_when_episode_has_negative_score() -> None:
     policy_version_ids = [uuid4() for _ in range(3)]
     pool = PolicyPool(