Add ArxivRollBench eval

README.md

@@ -82,6 +82,7 @@ This repository currently contains the following evals:
 - HumanEval: Evaluating Large Language Models Trained on Code, reference https://arxiv.org/abs/2107.03374, https://github.com/openai/human-eval, [MIT License](https://github.com/openai/human-eval/blob/master/LICENSE)
 - SimpleQA: Measuring short-form factuality in large language models, reference: https://openai.com/index/introducing-simpleqa, [MIT License](https://github.com/openai/simple-evals/blob/main/LICENSE)
 - BrowseComp: A Simple Yet Challenging Benchmark for Browsing Agents, reference: https://openai.com/index/browsecomp, [MIT License](https://github.com/openai/simple-evals/blob/main/LICENSE)
+- ArxivRollBench: A rolling benchmark for evaluating recent scientific text reasoning from arXiv papers, references: https://ojs.aaai.org/index.php/AAAI/article/view/41098, https://arxivrollbench.github.io/
 - HealthBench: Evaluating Large Language Models Towards Improved Human Health, reference: https://openai.com/index/healthbench, [MIT License](https://github.com/openai/simple-evals/blob/main/LICENSE)
 
 ## Samplers
@@ -113,6 +114,11 @@ For the [Anthropic API](https://docs.anthropic.com/claude/docs/quickstart-guide)
 pip install anthropic
 ```
 
+For ArxivRollBench:
+```bash
+pip install datasets
+```
+
 ## Running the evals
 ```bash
 python -m simple-evals.simple_evals --list-models

arxivrollbench_eval.py

@@ -0,0 +1,169 @@
+"""
+ArxivRollBench: a rolling arXiv benchmark for evaluating recent scientific text reasoning.
+AAAI 2026 paper: https://ojs.aaai.org/index.php/AAAI/article/view/41098
+"""
+
+import re
+from typing import Literal
+
+from . import common
+from .common import ANSWER_PATTERN_MULTICHOICE, HTML_JINJA, format_multichoice_question
+from .types import Eval, EvalResult, SamplerBase, SingleEvalResult
+
+DOMAINS = [
+    ("cs", "cs"),
+    ("q_fin", "q-fin"),
+    ("math", "math"),
+    ("physics", "physics"),
+    ("stat", "stat"),
+    ("q_bio", "q-bio"),
+    ("econ", "econ"),
+    ("eess", "eess"),
+]
+RELEASES = ["2024b", "2025a", "2026a"]
+TASK_TYPES = ["s", "c", "p"]
+TASK_TYPE_NAMES = {
+    "s": "sequencing",
+    "c": "cloze",
+    "p": "prediction",
+}
+
+
+def _dataset_path(
+    release: str,
+    hf_domain: str,
+    task_type: str,
+    split: Literal["compact", "full"],
+) -> str:
+    suffix = "-50" if split == "compact" else ""
+    if release == "2024b":
+        return f"liangzid/robench2024b_all_set{hf_domain}SCP-{task_type}{suffix}"
+    return (
+        f"liangzid/robench{release}_test_all_category_set"
+        f"{hf_domain}SCP-{task_type}{suffix}"
+    )
+
+
+def _selection_to_letter(label: str) -> str:
+    match = re.search(r"\bselection\s*([1-4])\b", str(label), re.IGNORECASE)
+    if match:
+        return chr(ord("A") + int(match.group(1)) - 1)
+    return str(label).strip().upper()
+
+
+def _record_to_example(record: dict, release: str, domain: str, task_type: str) -> dict:
+    if task_type == "p":
+        question = (
+            "Given the context, select the text that is the next sequence.\n\n"
+            f"Context:\n{record['context']}"
+        )
+        answer = str(record["label"]).strip().upper()
+    else:
+        question = (
+            "Select the option that correctly completes the sequencing or cloze task.\n\n"
+            f"{record['shuffled_text']}"
+        )
+        answer = _selection_to_letter(record["label"])
+
+    return {
+        "Question": question,
+        "A": record["A"],
+        "B": record["B"],
+        "C": record["C"],
+        "D": record["D"],
+        "Answer": answer,
+        "release": release,
+        "domain": domain,
+        "task_type": task_type,
+        "task_type_name": TASK_TYPE_NAMES[task_type],
+        "source_label": record["label"],
+    }
+
+
+class ArxivRollBenchEval(Eval):
+    def __init__(
+        self,
+        split: Literal["compact", "full"] = "compact",
+        num_examples: int | None = None,
+        subsets: list[str] | None = None,
+    ):
+        try:
+            from datasets import load_dataset
+        except ImportError as exc:
+            raise ImportError(
+                "ArxivRollBenchEval requires the optional `datasets` package. "
+                "Install it with `pip install datasets`."
+            ) from exc
+
+        wanted_subsets = set(subsets) if subsets is not None else None
+        examples = []
+        for release in RELEASES:
+            for domain, hf_domain in DOMAINS:
+                for task_type in TASK_TYPES:
+                    subset_name = f"{release}_{domain}_{task_type}"
+                    if wanted_subsets is not None and subset_name not in wanted_subsets:
+                        continue
+
+                    remaining = (
+                        None if num_examples is None else num_examples - len(examples)
+                    )
+                    if remaining is not None and remaining <= 0:
+                        break
+
+                    dataset_path = _dataset_path(release, hf_domain, task_type, split)
+                    split_name = "train" if remaining is None else f"train[:{remaining}]"
+                    dataset = load_dataset(dataset_path, split=split_name)
+                    for record in dataset:
+                        examples.append(
+                            _record_to_example(record, release, domain, task_type)
+                        )
+            if num_examples is not None and len(examples) >= num_examples:
+                break
+
+        self.examples = examples
+        self.split = split
+
+    def __call__(self, sampler: SamplerBase) -> EvalResult:
+        def fn(row: dict):
+            prompt_messages = [
+                sampler._pack_message(
+                    content=format_multichoice_question(row),
+                    role="user",
+                )
+            ]
+            sampler_response = sampler(prompt_messages)
+            response_text = sampler_response.response_text
+            actual_queried_prompt_messages = sampler_response.actual_queried_message_list
+            match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text)
+            extracted_answer = match.group(1).upper() if match else None
+            score = 1.0 if extracted_answer == row["Answer"] else 0.0
+            html = common.jinja_env.from_string(HTML_JINJA).render(
+                prompt_messages=actual_queried_prompt_messages,
+                next_message=dict(content=response_text, role="assistant"),
+                score=score,
+                correct_answer=row["Answer"],
+                extracted_answer=extracted_answer,
+            )
+            convo = actual_queried_prompt_messages + [
+                dict(content=response_text, role="assistant")
+            ]
+            return SingleEvalResult(
+                html=html,
+                score=score,
+                convo=convo,
+                metrics={
+                    row["release"]: score,
+                    row["domain"]: score,
+                    row["task_type_name"]: score,
+                    "chars": len(response_text),
+                },
+                example_level_metadata={
+                    "release": row["release"],
+                    "domain": row["domain"],
+                    "task_type": row["task_type"],
+                    "source_label": row["source_label"],
+                },
+            )
+
+        results = common.map_with_progress(fn, self.examples)
+        return common.aggregate_results(results)

arxivrollbench_eval_test.py

@@ -0,0 +1,56 @@
+from .arxivrollbench_eval import (
+    _dataset_path,
+    _record_to_example,
+    _selection_to_letter,
+)
+
+
+def test_dataset_path_compact_and_full():
+    assert (
+        _dataset_path("2026a", "cs", "s", "compact")
+        == "liangzid/robench2026a_test_all_category_setcsSCP-s-50"
+    )
+    assert (
+        _dataset_path("2024b", "q-fin", "p", "full")
+        == "liangzid/robench2024b_all_setq-finSCP-p"
+    )
+
+
+def test_selection_to_letter():
+    assert _selection_to_letter("Selection 1") == "A"
+    assert _selection_to_letter("selection 4") == "D"
+    assert _selection_to_letter("A") == "A"
+    assert _selection_to_letter("1") == "1"
+
+
+def test_record_to_example_prediction():
+    record = {
+        "context": "The introduction describes a new method.",
+        "A": "A candidate",
+        "B": "B candidate",
+        "C": "C candidate",
+        "D": "D candidate",
+        "label": "C",
+    }
+
+    example = _record_to_example(record, "2026a", "cs", "p")
+
+    assert example["Answer"] == "C"
+    assert "Context:\nThe introduction describes a new method." in example["Question"]
+    assert example["task_type_name"] == "prediction"
+
+
+def test_record_to_example_selection():
+    record = {
+        "shuffled_text": "Paragraph with a blank.",
+        "A": "A candidate",
+        "B": "B candidate",
+        "C": "C candidate",
+        "D": "D candidate",
+        "label": "Selection 2",
+    }
+
+    example = _record_to_example(record, "2026a", "math", "s")
+
+    assert example["Answer"] == "B"
+    assert example["task_type_name"] == "sequencing"

simple_evals.py

@@ -6,6 +6,7 @@
 import pandas as pd
 
 from . import common
+from .arxivrollbench_eval import ArxivRollBenchEval
 from .browsecomp_eval import BrowseCompEval
 from .drop_eval import DropEval
 from .gpqa_eval import GPQAEval
@@ -387,6 +388,16 @@ def get_evals(eval_name, debug_mode):
                     grader_model=grading_sampler,
                     num_examples=10 if debug_mode else num_examples,
                 )
+            case "arxivrollbench":
+                return ArxivRollBenchEval(
+                    split="compact",
+                    num_examples=5 if debug_mode else num_examples,
+                )
+            case "arxivrollbench_full":
+                return ArxivRollBenchEval(
+                    split="full",
+                    num_examples=5 if debug_mode else num_examples,
+                )
             case "healthbench":
                 return HealthBenchEval(
                     grader_model=healthbench_grading_sampler,