From 3571b7b3c38c4e8bc65fedf49319f4e5918fc3e9 Mon Sep 17 00:00:00 2001 From: Chengqian Date: Fri, 3 Jul 2026 10:40:18 +0000 Subject: [PATCH] fix(data_preprocess): GPQA answer preprocess() strips load-bearing brackets preprocess() in gpqa.py/gpqa_diamond.py (adopted from EleutherAI's lm-evaluation-harness GPQA utils) runs `re.sub("\[.*?\]", "", text)` on every answer-choice field, intended to strip citation-style "[1]"/"[title]" markers. GPQA's answer fields are heavily organic-chemistry/physics, where "[...]" is load-bearing: IUPAC ring-locants (spiro[4.5], bicyclo[5.3.1]), SMILES stereocenters ([C@H]), LaTeX (\left[...\right]), or plain math grouping. The regex silently deletes all of it. Audited gpqa_diamond (198 rows): 12 answer choices corrupted, including one row where all 4 choices collapsed to an identical string ("ln(2) = "), making it unanswerable from the text alone -- and where the pipeline's own ground-truth letter (via the same all_choices.index(correct) pattern hitting the same collision) was also wrong as a result. Audited gpqa_main (448 rows): 19/448 answer choices have similar bracket content. In both, found zero genuine citation markers -- every match was real content. Dropped the line. Co-Authored-By: Claude Sonnet 5 --- data_preprocess/stem/gpqa.py | 9 ++++++++- data_preprocess/stem/gpqa_diamond.py | 8 +++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/data_preprocess/stem/gpqa.py b/data_preprocess/stem/gpqa.py index c06607cc1..a7d5e7016 100644 --- a/data_preprocess/stem/gpqa.py +++ b/data_preprocess/stem/gpqa.py @@ -31,7 +31,14 @@ def preprocess(text): return " " text = text.strip() text = text.replace(" [title]", ". ") - text = re.sub("\\[.*?\\]", "", text) + # NOTE: the upstream harness also does `re.sub("\\[.*?\\]", "", text)` here + # to strip citation-style "[1]" markers, but GPQA's Answer fields are + # dominated by organic-chemistry/physics questions where "[...]" is + # load-bearing content (IUPAC ring-locants like spiro[4.5], SMILES + # stereocenters like [C@H], LaTeX like \left[...\right], or plain math + # grouping). Auditing gpqa_main found 19/448 rows with such content and + # zero genuine citation markers -- dropped it (see gpqa_diamond.py, where + # the same line corrupted 12/198 answer choices). text = text.replace(" ", " ") return text diff --git a/data_preprocess/stem/gpqa_diamond.py b/data_preprocess/stem/gpqa_diamond.py index c1dbad484..72d62b287 100644 --- a/data_preprocess/stem/gpqa_diamond.py +++ b/data_preprocess/stem/gpqa_diamond.py @@ -31,7 +31,13 @@ def preprocess(text): return " " text = text.strip() text = text.replace(" [title]", ". ") - text = re.sub("\\[.*?\\]", "", text) + # NOTE: the upstream harness also does `re.sub("\\[.*?\\]", "", text)` here + # to strip citation-style "[1]" markers, but GPQA-Diamond's Answer fields + # are dominated by organic-chemistry/physics questions where "[...]" is + # load-bearing content (IUPAC ring-locants like spiro[4.5], SMILES + # stereocenters like [C@H], LaTeX like \left[...\right], or plain math + # grouping). Auditing all 198 questions found zero genuine citation + # markers and 12 corrupted answer choices from this line -- dropped it. text = text.replace(" ", " ") return text