From 6f5608eaed42d14d71308a9302a760f5c0279574 Mon Sep 17 00:00:00 2001
From: David Hall <david.hall@openathena.ai>
Date: Wed, 22 Jul 2026 18:29:31 -0700
Subject: [PATCH] [eval] Vary GPQA answer positions by question

Seed each GPQA option shuffle from the question text so correct answers do not occupy the same label across the dataset. Keep the mapping deterministic across runs.

Ports mlfoundations/evalchemy#156.
Fixes marin-community/marin#7527
---
 .../GPQADiamond/eval_instruct.py              |  3 +-
 tests/gpqa/test_answer_shuffle.py             | 31 +++++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 tests/gpqa/test_answer_shuffle.py

diff --git a/eval/chat_benchmarks/GPQADiamond/eval_instruct.py b/eval/chat_benchmarks/GPQADiamond/eval_instruct.py
index dab52fd0..c413a1d1 100644
--- a/eval/chat_benchmarks/GPQADiamond/eval_instruct.py
+++ b/eval/chat_benchmarks/GPQADiamond/eval_instruct.py
@@ -1,3 +1,4 @@
+import hashlib
 import logging
 import os
 import random
@@ -177,7 +178,7 @@ def generate_multiple_choice_answers(self, data: Dict[str, Any]) -> tuple[str, s
             data["Incorrect Answer 2"],
             data["Incorrect Answer 3"],
         ]
-        rnd = random.Random(42)
+        rnd = random.Random(42 + int(hashlib.md5(data["Question"].encode()).hexdigest(), 16))
         rnd.shuffle(answers)
 
         options = ["A", "B", "C", "D"]
diff --git a/tests/gpqa/test_answer_shuffle.py b/tests/gpqa/test_answer_shuffle.py
new file mode 100644
index 00000000..1c65b27b
--- /dev/null
+++ b/tests/gpqa/test_answer_shuffle.py
@@ -0,0 +1,31 @@
+"""Regression tests for GPQA-Diamond answer placement."""
+
+import sys
+from pathlib import Path
+
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(_REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(_REPO_ROOT))
+
+from eval.chat_benchmarks.GPQADiamond.eval_instruct import GPQADiamondBenchmark
+
+
+def _question(index: int) -> dict[str, str]:
+    return {
+        "Question": f"Question {index}",
+        "Correct Answer": "correct",
+        "Incorrect Answer 1": "wrong one",
+        "Incorrect Answer 2": "wrong two",
+        "Incorrect Answer 3": "wrong three",
+    }
+
+
+def test_answer_positions_vary_reproducibly_across_questions():
+    benchmark = GPQADiamondBenchmark.__new__(GPQADiamondBenchmark)
+    questions = [_question(index) for index in range(32)]
+
+    first = [benchmark.generate_multiple_choice_answers(question)[1] for question in questions]
+    second = [benchmark.generate_multiple_choice_answers(question)[1] for question in questions]
+
+    assert set(first) == {"A", "B", "C", "D"}
+    assert first == second