diff --git a/eval/chat_benchmarks/NUPA/README.md b/eval/chat_benchmarks/NUPA/README.md new file mode 100644 index 00000000..b46cbcb1 --- /dev/null +++ b/eval/chat_benchmarks/NUPA/README.md @@ -0,0 +1,129 @@ +# NUPA + +NUPA is the direct numeric question-answering benchmark from +["Number Cookbook: Number Understanding of Language Models and How to Improve It"](https://arxiv.org/abs/2411.03766). +Evalchemy registers it as one native task named `NUPA`. + +The integration has two stages: + +1. `data_prep/flatten_hf_dataset.py` converts the original nested dataset once + and publishes row-oriented records to Hugging Face. +2. `NUPABenchmark` loads those records, requests model completions, scores each + response, and aggregates the metrics. + +## Dataset repositories + +The conversion source is the MIT-licensed +[`HaotongYang/NUPA_text`](https://huggingface.co/datasets/HaotongYang/NUPA_text) +dataset. The original source has nested task and digit mappings, so it is not +loaded directly during evaluation. + +The flattened dataset repository is currently `TODO_ORG/nupa-text-eval`. +This identifier is a placeholder shared by the conversion command and runtime +loader. Finalize the owning Hugging Face organization and repository name before +publishing the production conversion or merging the integration. Update +`PUBLISHED_DATASET_NAME` in `eval_instruct.py` when the repository is chosen. + +The flattened schema is: + +```json +{ + "id": "test:max_Float_Float_Float:3:000000", + "task_name": "max_Float_Float_Float", + "operation": "max", + "answer_format": "Float", + "digit": 3, + "length_bucket": "S", + "prompt": "Directly return ... Get the maximal number: 9.11 and 9.9 =", + "answer": "9.9" +} +``` + +`answer_format` is one of `Integer`, `Float`, `Fraction`, or +`ScientificNotation`. `length_bucket` is one of `S`, `M`, `L`, or `XL`. + +## Convert and publish + +Install the benchmark dependency and authenticate the Hugging Face CLI before +publishing: + +```bash +uv sync --extra nupa +hf auth login +``` + +Download the original `test.json`, stream-flatten it, and publish the result: + +```bash +uv run python -m eval.chat_benchmarks.NUPA.data_prep.flatten_hf_dataset \ + --dataset-name HaotongYang/NUPA_text \ + --split test \ + --output /tmp/nupa_test.jsonl \ + --repo-id TODO_ORG/nupa-text-eval +``` + +The converter records the source dataset revision in the published dataset card. +It reads one top-level task at a time and writes JSONL incrementally; it does not +hold the complete nested source or flattened result in memory. + +For a publishing smoke test, retain one example from every task-and-digit group: + +```bash +uv run python -m eval.chat_benchmarks.NUPA.data_prep.flatten_hf_dataset \ + --dataset-name HaotongYang/NUPA_text \ + --split test \ + --limit-per-task-digit 1 \ + --output /tmp/nupa_test_smoke.jsonl \ + --repo-id USER/nupa-text-eval-smoke +``` + +The smoke dataset checks conversion coverage and upload behavior. Do not report +benchmark performance from it. + +## Run the benchmark + +Evaluate the published dataset against an OpenAI-compatible endpoint: + +```bash +eval --model local-completions \ + --tasks NUPA \ + --model_args model=served,base_url=http://localhost:8000/v1/completions +``` + +Use `--debug` to load the four checked-in smoke records instead of Hugging Face: + +```bash +eval --model local-completions \ + --tasks NUPA \ + --debug \ + --model_args model=served,base_url=http://localhost:8000/v1/completions +``` + +## Scoring and metrics + +Response extraction and normalization follow the observable behavior of the +official NUPA text evaluator. Evalchemy's scorer is a clean-room implementation; +the Number Cookbook code repository is GPL-3.0 and its code is not copied here. + +The benchmark reports: + +- `exact_match`: representation-sensitive equality after format-specific + extraction and normalization. +- `digit_match`: aligned digit accuracy between the extracted answer and target. +- `dlength`: absolute difference in total digit count; lower is better. +- `format_valid_rate`: fraction of responses accepted by the expected answer + format parser. +- `no_answer_rate`: fraction of responses from which no answer was extracted; + lower is better. +- `dataset_num_samples`: number of evaluated rows. + +Metrics are emitted overall and under these prefixes: + +```text +task:/ +bucket:/ +task:/bucket:/ +``` + +The task key is grouping metadata, not an Evalchemy task. One `NUPA` evaluation +runs dataset rows from multiple task-family and representation combinations. diff --git a/eval/chat_benchmarks/NUPA/__init__.py b/eval/chat_benchmarks/NUPA/__init__.py new file mode 100644 index 00000000..6576ed29 --- /dev/null +++ b/eval/chat_benchmarks/NUPA/__init__.py @@ -0,0 +1 @@ +"""NUPA benchmark package.""" diff --git a/eval/chat_benchmarks/NUPA/data/nupa_smoke.jsonl b/eval/chat_benchmarks/NUPA/data/nupa_smoke.jsonl new file mode 100644 index 00000000..d5ef60b5 --- /dev/null +++ b/eval/chat_benchmarks/NUPA/data/nupa_smoke.jsonl @@ -0,0 +1,4 @@ +{"id":"smoke:max_Float_Float_Float:3:000000","task_name":"max_Float_Float_Float","operation":"max","answer_format":"Float","digit":3,"length_bucket":"S","prompt":"Directly return the answer as a float without any comma separator, like 10.4 . Get the maximal number: 9.11 and 9.9 =","answer":"9.9"} +{"id":"smoke:add_Integer_Integer_Integer:3:000000","task_name":"add_Integer_Integer_Integer","operation":"add","answer_format":"Integer","digit":3,"length_bucket":"S","prompt":"Directly return the answer as an integer without any comma separator, like 123 . Add two numbers: 830 + 70 =","answer":"900"} +{"id":"smoke:truediv_Fraction_Fraction_Fraction:2:000000","task_name":"truediv_Fraction_Fraction_Fraction","operation":"truediv","answer_format":"Fraction","digit":2,"length_bucket":"S","prompt":"Directly return the answer as an irreducible fraction without any comma separator, like 1/2 . Divide two numbers: 3/4 / 1/2 =","answer":"3/2"} +{"id":"smoke:to_scient_Integer_ScientificNotation:5:000000","task_name":"to_scient_Integer_ScientificNotation","operation":"to_scient","answer_format":"ScientificNotation","digit":5,"length_bucket":"M","prompt":"Directly return the answer in scientific notation without any comma separator, like 1.23e4 . Convert the number to scientific notation: 50400 =","answer":"5.04e4"} diff --git a/eval/chat_benchmarks/NUPA/data_prep/__init__.py b/eval/chat_benchmarks/NUPA/data_prep/__init__.py new file mode 100644 index 00000000..c1b48342 --- /dev/null +++ b/eval/chat_benchmarks/NUPA/data_prep/__init__.py @@ -0,0 +1 @@ +"""NUPA data preparation helpers.""" diff --git a/eval/chat_benchmarks/NUPA/data_prep/flatten_hf_dataset.py b/eval/chat_benchmarks/NUPA/data_prep/flatten_hf_dataset.py new file mode 100644 index 00000000..9bf42a66 --- /dev/null +++ b/eval/chat_benchmarks/NUPA/data_prep/flatten_hf_dataset.py @@ -0,0 +1,189 @@ +"""Convert the original nested NUPA JSON to row-oriented JSONL and optionally publish it. + +Example: + uv run --extra nupa python -m eval.chat_benchmarks.NUPA.data_prep.flatten_hf_dataset \ + --split test --output /tmp/nupa_test.jsonl \ + --repo-id TODO_ORG/nupa-text-eval +""" + +from __future__ import annotations + +import argparse +import io +import json +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import ijson +from datasets import load_dataset +from huggingface_hub import HfApi, hf_hub_download + +from eval.chat_benchmarks.NUPA.eval_instruct import PUBLISHED_DATASET_NAME, SOURCE_DATASET_NAME, flatten_nupa_row + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dataset-name", default=SOURCE_DATASET_NAME) + parser.add_argument("--revision") + parser.add_argument("--split", default="test") + parser.add_argument("--source-file", type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument( + "--repo-id", + metavar=PUBLISHED_DATASET_NAME, + help=( + "Optional Hugging Face dataset repository to publish. " + f"The integration placeholder is {PUBLISHED_DATASET_NAME}." + ), + ) + parser.add_argument("--config-name", default="default") + parser.add_argument("--private", action="store_true") + parser.add_argument( + "--limit-per-task-digit", + type=int, + help="Optional deterministic cap applied before flattening each task/digit group.", + ) + args = parser.parse_args() + + source = args.source_file or Path( + hf_hub_download( + repo_id=args.dataset_name, + filename=f"{args.split}.json", + repo_type="dataset", + revision=args.revision, + ) + ) + count = convert_file( + source, + args.output, + split=args.split, + limit_per_task_digit=args.limit_per_task_digit, + ) + print(f"Wrote {count} flattened NUPA records to {args.output}") + + if args.repo_id: + source_revision = args.revision or HfApi().dataset_info(args.dataset_name).sha + publish_dataset( + args.output, + repo_id=args.repo_id, + config_name=args.config_name, + split=args.split, + private=args.private, + source_dataset=args.dataset_name, + source_revision=source_revision, + ) + print(f"Published https://huggingface.co/datasets/{args.repo_id}") + + +def convert_file( + source: Path, + output: Path, + *, + split: str, + limit_per_task_digit: int | None = None, +) -> int: + """Stream a nested NUPA JSON file into row-oriented JSONL records.""" + if limit_per_task_digit is not None and limit_per_task_digit <= 0: + raise ValueError("limit_per_task_digit must be positive") + + output.parent.mkdir(parents=True, exist_ok=True) + count = 0 + with source.open("rb") as source_file, output.open("w", encoding="utf-8") as output_file: + for task_name, by_digit in ijson.kvitems(source_file, ""): + row = {task_name: _limit_task(by_digit, limit_per_task_digit)} + for record in flatten_nupa_row(row, split=split): + output_file.write(json.dumps(record, sort_keys=True) + "\n") + count += 1 + return count + + +def publish_dataset( + path: Path, + *, + repo_id: str, + config_name: str, + split: str, + private: bool, + source_dataset: str, + source_revision: str, +) -> None: + """Upload flattened JSONL plus a provenance-bearing dataset card.""" + dataset = load_dataset("json", data_files=str(path), split="train") + dataset.push_to_hub( + repo_id, + config_name=config_name, + split=split, + private=private, + commit_message=f"Publish flattened NUPA {split} split", + ) + generated_card = Path( + hf_hub_download(repo_id=repo_id, filename="README.md", repo_type="dataset", force_download=True) + ).read_text() + provenance = _provenance(source_dataset, source_revision, config_name, split) + card = generated_card.split("", 1)[0].rstrip() + provenance + HfApi().upload_file( + path_or_fileobj=io.BytesIO(card.encode()), + path_in_repo="README.md", + repo_id=repo_id, + repo_type="dataset", + commit_message="Document NUPA source provenance", + ) + + +def _limit_task(by_digit: Any, limit: int | None) -> dict[str, list[str]]: + if not isinstance(by_digit, Mapping): + raise ValueError(f"Expected digit mapping, got {type(by_digit).__name__}") + limited = {} + for digit, examples in by_digit.items(): + if not isinstance(examples, list): + raise ValueError(f"Expected example list for digit {digit}, got {type(examples).__name__}") + limited[str(digit)] = examples if limit is None else examples[:limit] + return limited + + +def _provenance(source_dataset: str, source_revision: str, config_name: str, split: str) -> str: + return f""" + + + +## NUPA text data for Evalchemy + +This dataset is a row-oriented conversion of +[`{source_dataset}`](https://huggingface.co/datasets/{source_dataset}) for the +native Evalchemy `NUPA` benchmark. It separates the one-time conversion of the +original nested JSON from model evaluation. + +Source revision: `{source_revision}`. Configuration: `{config_name}`. Split: +`{split}`. The source dataset is MIT-licensed; consult its dataset card for the +license terms and original provenance. + +Each row contains: + +- `id`: stable split, task, digit, and example identifier +- `task_name`: original NUPA task-family and representation key +- `operation`: numeric operation derived from the task key +- `answer_format`: `Integer`, `Float`, `Fraction`, or `ScientificNotation` +- `digit`: original digit group +- `length_bucket`: `S`, `M`, `L`, or `XL` +- `prompt`: model input ending at the source answer delimiter +- `answer`: reference representation used for scoring + +Reproduce the conversion from Evalchemy: + +```bash +uv run --extra nupa python -m eval.chat_benchmarks.NUPA.data_prep.flatten_hf_dataset \\ + --dataset-name {source_dataset} \\ + --revision {source_revision} \\ + --split {split} \\ + --output /tmp/nupa_{split}.jsonl \\ + --repo-id OWNER/nupa-text-eval +``` + +Datasets published with `--limit-per-task-digit` are integration fixtures. Do +not use a limited conversion to report benchmark performance. +""" + + +if __name__ == "__main__": + main() diff --git a/eval/chat_benchmarks/NUPA/eval_instruct.py b/eval/chat_benchmarks/NUPA/eval_instruct.py new file mode 100644 index 00000000..08ab442b --- /dev/null +++ b/eval/chat_benchmarks/NUPA/eval_instruct.py @@ -0,0 +1,217 @@ +"""NUPA: direct number understanding and processing evaluation. + +NUPA is the text Q&A benchmark from "Number Cookbook: Number Understanding of +Language Models and How to Improve It". This native Evalchemy benchmark treats +NUPA as one Evalchemy task and reports metrics over flattened prompt/answer +examples, with NUPA task-family and length-bucket breakdowns. + +Design: marin-community/marin#7297. +""" + +from __future__ import annotations + +import json +import logging +import os +from collections import defaultdict +from typing import Any, Dict, Iterable, List, Optional + +from datasets import load_dataset +from lm_eval.api.instance import Instance +from lm_eval.api.model import LM + +from eval.task import BaseBenchmark + +from .scorer import ExampleScore, extract_answer, length_bucket, mean, normalize_answer, score_prediction + +SOURCE_DATASET_NAME = "HaotongYang/NUPA_text" +# TODO(marin-community/marin#7297): Finalize the owning Hugging Face organization +# and repository name before merging the NUPA integration. +PUBLISHED_DATASET_NAME = "TODO_ORG/nupa-text-eval" +DATASET_NAME = PUBLISHED_DATASET_NAME +DEFAULT_SPLIT = "test" +DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") +SMOKE_DATA = os.path.join(DATA_DIR, "nupa_smoke.jsonl") +FLAT_RECORD_FIELDS = { + "answer", + "answer_format", + "digit", + "id", + "length_bucket", + "operation", + "prompt", + "task_name", +} + + +class NUPABenchmark(BaseBenchmark): + """Native Evalchemy wrapper for NUPA direct numeric QA.""" + + def __init__( + self, + dataset_name: str = DATASET_NAME, + dataset_split: str = DEFAULT_SPLIT, + data_file: Optional[str] = None, + max_tokens: int = 256, + debug: bool = False, + logger: Optional[logging.Logger] = None, + system_instruction: Optional[str] = None, + ): + super().__init__(logger=logger, system_instruction=system_instruction) + self.dataset_name = dataset_name + self.dataset_split = dataset_split + self.data_file = data_file + self.max_new_tokens = max_tokens + self.debug = debug + + def _load_records(self) -> List[Dict[str, Any]]: + if self.debug: + records = _read_jsonl(SMOKE_DATA) + elif self.data_file: + records = _read_jsonl(self.data_file) + else: + records = _load_flattened_hf_records(self.dataset_name, self.dataset_split) + return records + + def _build_instances(self, model: LM, records: List[Dict[str, Any]]) -> List[Instance]: + instances = [] + for idx, record in enumerate(records): + prompt = record["prompt"] + messages = [{"role": "user", "content": prompt}] + templated = self._prepare_messages(messages, model) + instances.append( + Instance( + "generate_until", + record, + ( + templated, + {"do_sample": False, "temperature": 0.0, "max_new_tokens": self.max_new_tokens}, + ), + idx, + ) + ) + return instances + + def generate_responses(self, model: LM) -> Optional[Dict[str, Any]]: + records = self._load_records() + self.logger.info("Generating responses for NUPA (%d examples)...", len(records)) + outputs = self.compute(model, self._build_instances(model, records)) + if model.rank != 0: + return None + examples = [] + for record, output in zip(records, outputs): + example = dict(record) + example["output"] = output + examples.append(example) + return {"examples": examples} + + def evaluate_responses(self, results: Optional[Dict[str, Any]]) -> Optional[Dict[str, float]]: + if results is None: + return None + + scored_examples = [] + for example in results["examples"]: + score = score_prediction(example.get("output"), example["answer"], example["answer_format"]) + example["model_answer"] = extract_answer(example.get("output"), example["answer_format"]) + example["normalized_model_answer"] = normalize_answer(example["model_answer"], example["answer_format"]) + example["correct"] = bool(score.exact_match) + example["score"] = score + scored_examples.append(example) + + metrics: Dict[str, float] = {} + metrics.update(_aggregate_scores(scored_examples, prefix="")) + for group_name, items in _group_by(scored_examples, "task_name").items(): + metrics.update(_aggregate_scores(items, prefix=f"task:{group_name}/")) + for group_name, items in _group_by(scored_examples, "length_bucket").items(): + metrics.update(_aggregate_scores(items, prefix=f"bucket:{group_name}/")) + for task_name, task_items in _group_by(scored_examples, "task_name").items(): + for bucket, bucket_items in _group_by(task_items, "length_bucket").items(): + metrics.update(_aggregate_scores(bucket_items, prefix=f"task:{task_name}/bucket:{bucket}/")) + + metrics["dataset_num_samples"] = float(len(scored_examples)) + return metrics + + +def _aggregate_scores(examples: List[Dict[str, Any]], prefix: str) -> Dict[str, float]: + scores: List[ExampleScore] = [example["score"] for example in examples] + return { + f"{prefix}exact_match": mean(score.exact_match for score in scores), + f"{prefix}digit_match": mean(score.digit_match for score in scores), + f"{prefix}dlength": mean(score.dlength for score in scores), + f"{prefix}format_valid_rate": mean(score.format_valid for score in scores), + f"{prefix}no_answer_rate": mean(score.no_answer for score in scores), + } + + +def _group_by(examples: Iterable[Dict[str, Any]], key: str) -> Dict[str, List[Dict[str, Any]]]: + groups: Dict[str, List[Dict[str, Any]]] = defaultdict(list) + for example in examples: + groups[str(example[key])].append(example) + return dict(groups) + + +def _read_jsonl(path: str) -> List[Dict[str, Any]]: + with open(path, encoding="utf-8") as f: + return [json.loads(line) for line in f if line.strip()] + + +def _load_flattened_hf_records(dataset_name: str, split: str) -> List[Dict[str, Any]]: + dataset = load_dataset(dataset_name, split=split) + records: List[Dict[str, Any]] = [] + for row in dataset: + record = dict(row) + if FLAT_RECORD_FIELDS.issubset(record): + records.append(record) + else: + records.extend(flatten_nupa_row(record, split=split)) + return records + + +def flatten_nupa_row(row: Dict[str, Any], split: str) -> List[Dict[str, Any]]: + """Flatten one row from the auto-converted HF NUPA_text dataset.""" + flattened: List[Dict[str, Any]] = [] + for task_name, by_digit in row.items(): + if not isinstance(by_digit, dict): + continue + answer_format = _answer_format_from_task_name(task_name) + operation = _operation_from_task_name(task_name) + max_digit = max(int(digit_key) for digit_key, examples in by_digit.items() if isinstance(examples, list)) + for digit_key, examples in by_digit.items(): + if not isinstance(examples, list): + continue + digit = int(digit_key) + bucket = length_bucket(digit, max_digit=max_digit) + for idx, text in enumerate(examples): + prompt, answer = split_prompt_answer(text) + flattened.append( + { + "id": f"{split}:{task_name}:{digit}:{idx:06d}", + "task_name": task_name, + "operation": operation, + "answer_format": answer_format, + "digit": digit, + "length_bucket": bucket, + "prompt": prompt, + "answer": answer, + } + ) + return flattened + + +def split_prompt_answer(text: str) -> tuple[str, str]: + prompt, answer = text.split("=", 1) + return f"{prompt.rstrip()} =", answer.strip() + + +def _operation_from_task_name(task_name: str) -> str: + parts = task_name.split("_") + if len(parts) >= 2 and parts[0] == "multiply" and parts[1] in {"easy", "hard"}: + return "_".join(parts[:2]) + if len(parts) >= 2 and parts[0] in {"digit", "to"}: + return "_".join(parts[:2]) + return parts[0] + + +def _answer_format_from_task_name(task_name: str) -> str: + answer_format = task_name.split("_")[-1] + return "Integer" if answer_format == "int" else answer_format diff --git a/eval/chat_benchmarks/NUPA/scorer.py b/eval/chat_benchmarks/NUPA/scorer.py new file mode 100644 index 00000000..f85c6786 --- /dev/null +++ b/eval/chat_benchmarks/NUPA/scorer.py @@ -0,0 +1,140 @@ +"""Clean-room implementation of the official NUPA text metrics. + +The implementation reproduces the observable behavior of Number Cookbook's +text evaluator without copying its GPL-licensed source. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Iterable + +INTEGER = "Integer" +FLOAT = "Float" +FRACTION = "Fraction" +SCIENTIFIC = "ScientificNotation" + +_ANSWER_MARKER_RE = re.compile(r"(?i)^(?:the\s+answer\s+is|so\s+the\s+answer\s+is)\s+") +_ANSWER_PATTERNS = { + INTEGER: re.compile(r"^\d+"), + FLOAT: re.compile(r"^\d+\.\d+"), + FRACTION: re.compile(r"^\d+/\d+"), + SCIENTIFIC: re.compile(r"^\d+\.\d+[eE][+-]?\d+"), +} +_READ_FROM_RIGHT = { + INTEGER: (True,), + FLOAT: (True, False), + FRACTION: (True, True), + SCIENTIFIC: (True, False, True), +} + + +@dataclass(frozen=True) +class ExampleScore: + exact_match: float + digit_match: float + dlength: float + format_valid: float + no_answer: float + + +def extract_answer(text: object, answer_format: str) -> str | None: + """Extract an answer at the start of a direct-answer completion.""" + if answer_format not in _ANSWER_PATTERNS: + raise ValueError(f"Unsupported NUPA answer format: {answer_format}") + if not isinstance(text, str): + return None + stripped = text.strip() + stripped = _ANSWER_MARKER_RE.sub("", stripped, count=1) + match = _ANSWER_PATTERNS[answer_format].match(stripped) + if match is None: + return None + return match.group().replace("+", "").replace("-", "").replace("E", "e") + + +def normalize_answer(answer: str | None, answer_format: str) -> str | None: + """Validate an extracted answer without changing its digit representation.""" + if answer is None: + return None + extracted = extract_answer(answer, answer_format) + if extracted is None or extracted != answer.replace("+", "").replace("-", "").replace("E", "e"): + return None + return extracted + + +def score_prediction(prediction: str | None, gold: str, answer_format: str) -> ExampleScore: + """Score a completion with the official text-evaluator semantics.""" + extracted = extract_answer(prediction, answer_format) + gold_parts = _digit_parts(gold, answer_format) + prediction_parts = _digit_parts(extracted or "", answer_format) + format_valid = extracted is not None + return ExampleScore( + exact_match=1.0 if format_valid and prediction_parts == gold_parts else 0.0, + digit_match=_digit_match(prediction_parts, gold_parts, answer_format), + dlength=float(abs(_total_part_length(prediction_parts) - _total_part_length(gold_parts))), + format_valid=1.0 if format_valid else 0.0, + no_answer=0.0 if format_valid else 1.0, + ) + + +def length_bucket(digit: int, *, max_digit: int | None = None) -> str: + """Return the NUPA S/M/L/XL interval for a digit length.""" + if max_digit is None: + max_digit = 20 if digit <= 20 else 100 + if max_digit <= 20: + if digit <= 4: + return "S" + if digit <= 8: + return "M" + if digit <= 14: + return "L" + return "XL" + if digit <= 10: + return "S" + if digit <= 20: + return "M" + if digit <= 60: + return "L" + return "XL" + + +def mean(values: Iterable[float]) -> float: + vals = list(values) + return sum(vals) / len(vals) if vals else 0.0 + + +def _digit_parts(answer: str, answer_format: str) -> tuple[str, ...]: + separators = { + INTEGER: (), + FLOAT: (".",), + FRACTION: ("/",), + SCIENTIFIC: (".", "e"), + } + if answer_format not in separators: + raise ValueError(f"Unsupported NUPA answer format: {answer_format}") + parts = [answer] + for separator in separators[answer_format]: + next_parts = [] + for part in parts: + next_parts.extend(part.replace("E", "e").split(separator, 1)) + parts = next_parts + expected_parts = len(separators[answer_format]) + 1 + if len(parts) != expected_parts: + return tuple("" for _ in range(expected_parts)) + return tuple("".join(character for character in part if character.isdigit()) for part in parts) + + +def _digit_match(prediction: tuple[str, ...], gold: tuple[str, ...], answer_format: str) -> float: + correct = 0 + total = _total_part_length(gold) + for prediction_part, gold_part, align_right in zip(prediction, gold, _READ_FROM_RIGHT[answer_format], strict=True): + if align_right: + prediction_part = prediction_part[::-1] + gold_part = gold_part[::-1] + correct += sum(predicted == expected for predicted, expected in zip(prediction_part, gold_part)) + return correct / total if total else 0.0 + + +def _total_part_length(parts: Iterable[str]) -> int: + return sum(len(part) for part in parts) diff --git a/packages/evalchemy-config/docs/debug-log-nupa-openai-preflight.md b/packages/evalchemy-config/docs/debug-log-nupa-openai-preflight.md new file mode 100644 index 00000000..5c808be5 --- /dev/null +++ b/packages/evalchemy-config/docs/debug-log-nupa-openai-preflight.md @@ -0,0 +1,35 @@ +# Debugging log for NUPA OpenAI preflight + +Allow native benchmarks to use the supported `openai-chat-completions` adapter +without requiring a tokenizer that the adapter intentionally does not provide. + +## Initial status + +The NUPA debug run fails before transport because endpoint preflight calls +`apply_chat_template` on a `None` tokenizer. + +## Hypothesis 1 + +The OpenAI chat adapter defaults to no tokenizer, but bounded native generation +unconditionally tokenizes its payload during context preflight. + +## Changes to make + +Add a regression test requiring preflight to preserve generation arguments and +skip token counts when no tokenizer is configured. + +## Results + +Inspection confirmed `OpenAIChatCompletion(tokenizer_backend=None)` passes a +`None` tokenizer to `preflight_endpoint_generation`. + +The regression test failed with the reported `NoneType.apply_chat_template` +error. Preflight now treats a missing tokenizer like a missing context limit: +it preserves the requested generation arguments and skips local token counting. + +## End-to-end check + +A 40-example NUPA run completed through `openai-chat-completions` after the +preflight change. It produced 40 model responses without infrastructure-error +markers. This run used a limited staging dataset and is evidence for integration +behavior, not benchmark performance. diff --git a/packages/evalchemy-config/src/evalchemy_config/limits.py b/packages/evalchemy-config/src/evalchemy_config/limits.py index 97501aae..8a48305e 100644 --- a/packages/evalchemy-config/src/evalchemy_config/limits.py +++ b/packages/evalchemy-config/src/evalchemy_config/limits.py @@ -12,9 +12,9 @@ from __future__ import annotations -from dataclasses import dataclass import copy import json +from dataclasses import dataclass from typing import Any, Iterable, Mapping, Optional, Sequence MAX_OUTPUT_ALIASES = ("max_tokens", "max_new_tokens", "max_gen_toks") @@ -111,7 +111,7 @@ def preflight_endpoint_generation( shares one endpoint generation kwargs dictionary, so it uses the largest rendered prompt in that batch. """ - if context_length is None or gen_kwargs is None: + if tokenizer is None or context_length is None or gen_kwargs is None: return (dict(gen_kwargs) if gen_kwargs is not None else None, None, None) present = [(key, gen_kwargs[key]) for key in MAX_OUTPUT_ALIASES if key in gen_kwargs] if not present: diff --git a/pyproject.toml b/pyproject.toml index 7344c091..d4ef3458 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -204,6 +204,7 @@ livecodebenchv5 = [] livecodebenchv5-official = [] math500 = [] mmlupro = [] +nupa = ["ijson"] # OlympiadBench grades via lm-eval's hendrycks_math boxed-answer utils (in the lean base # via lm-eval[math]). The legacy subset is local; the pinned full text-only task loads # through datasets, also in the lean base, so neither needs an extra dependency. @@ -214,7 +215,7 @@ olympiadbenchfull = [] # (uv expands `evalchemy[...]` against this project's own extras), the same shape lm-eval # ships its `tasks` union in. `uv sync --extra benchmarks` gives you the whole tree. benchmarks = [ - "evalchemy[aime24,aime25,aiw,amc23,bigcodebench,codeelo,codeforces,cruxeval,financebench,gpqadiamond,gsm8kperturbed,hle,hmmt,humaneval,humanevalplus,ifbench,ifeval,jeebench,livebench,livecodebench,livecodebenchv5,livecodebenchv5-official,math500,mbpp,mbppplus,mmlupro,mtbench,mixeval,multiple,olympiadbench,olympiadbenchfull,repobench,swebench,wildbench,alpaca-eval,zeroeval]", + "evalchemy[aime24,aime25,aiw,amc23,bigcodebench,codeelo,codeforces,cruxeval,financebench,gpqadiamond,gsm8kperturbed,hle,hmmt,humaneval,humanevalplus,ifbench,ifeval,jeebench,livebench,livecodebench,livecodebenchv5,livecodebenchv5-official,math500,mbpp,mbppplus,mmlupro,mtbench,mixeval,multiple,nupa,olympiadbench,olympiadbenchfull,repobench,swebench,wildbench,alpaca-eval,zeroeval]", ] # --- Marin `marin-serve` (Iris) provider for the runner (eval/serve_eval/) ------- diff --git a/tests/nupa/test_data_prep.py b/tests/nupa/test_data_prep.py new file mode 100644 index 00000000..38296e40 --- /dev/null +++ b/tests/nupa/test_data_prep.py @@ -0,0 +1,46 @@ +import json + +import pytest + +from eval.chat_benchmarks.NUPA.data_prep.flatten_hf_dataset import _provenance, convert_file + + +def test_convert_file_streams_original_schema_and_limits_each_group(tmp_path): + source = tmp_path / "test.json" + source.write_text( + json.dumps( + { + "add_Integer_Integer_Integer": { + "3": [ + "Directly return an integer. Add: 830 + 70 = 900", + "Directly return an integer. Add: 98 + 150 = 248", + ], + "4": ["Directly return an integer. Add: 1000 + 1 = 1001"], + } + } + ) + ) + output = tmp_path / "flattened.jsonl" + + count = convert_file(source, output, split="test", limit_per_task_digit=1) + + records = [json.loads(line) for line in output.read_text().splitlines()] + assert count == 2 + assert [(record["digit"], record["answer"]) for record in records] == [(3, "900"), (4, "1001")] + assert all(record["task_name"] == "add_Integer_Integer_Integer" for record in records) + + +def test_convert_file_rejects_nonpositive_limit(tmp_path): + with pytest.raises(ValueError, match="must be positive"): + convert_file(tmp_path / "unused.json", tmp_path / "unused.jsonl", split="test", limit_per_task_digit=0) + + +def test_dataset_card_records_schema_source_revision_and_reproduction_command(): + card = _provenance("HaotongYang/NUPA_text", "source-sha", "default", "test") + normalized_card = " ".join(card.split()) + + assert "Source revision: `source-sha`" in card + assert "`task_name`: original NUPA task-family" in card + assert "--revision source-sha" in card + assert "--limit-per-task-digit" in card + assert "Do not use a limited conversion to report benchmark performance" in normalized_card diff --git a/tests/nupa/test_nupa_benchmark.py b/tests/nupa/test_nupa_benchmark.py new file mode 100644 index 00000000..76e2a663 --- /dev/null +++ b/tests/nupa/test_nupa_benchmark.py @@ -0,0 +1,77 @@ +from eval.chat_benchmarks.NUPA.eval_instruct import NUPABenchmark, flatten_nupa_row, split_prompt_answer +from eval.task import TaskManager + + +def test_split_prompt_answer_keeps_equals_in_prompt(): + prompt, answer = split_prompt_answer("Get the maximal number: 9.11 and 9.9 = 9.9") + assert prompt == "Get the maximal number: 9.11 and 9.9 =" + assert answer == "9.9" + + +def test_flatten_nupa_row_adds_metadata(): + rows = flatten_nupa_row( + { + "max_Float_Float_Float": { + "3": [ + "Directly return the answer as a float without any comma separator, like 10.4 . " + "Get the maximal number: 9.11 and 9.9 = 9.9" + ] + } + }, + split="test", + ) + assert rows == [ + { + "id": "test:max_Float_Float_Float:3:000000", + "task_name": "max_Float_Float_Float", + "operation": "max", + "answer_format": "Float", + "digit": 3, + "length_bucket": "S", + "prompt": "Directly return the answer as a float without any comma separator, like 10.4 . " + "Get the maximal number: 9.11 and 9.9 =", + "answer": "9.9", + } + ] + + +def test_flatten_nupa_row_maps_scalar_int_output_to_integer_format(): + rows = flatten_nupa_row( + {"get_digit_Integer_int_int": {"3": ["Get digit: 123 and 2 = 2"]}}, + split="test", + ) + + assert rows[0]["answer_format"] == "Integer" + + +def test_nupa_benchmark_aggregates_overall_task_and_bucket_metrics(): + benchmark = NUPABenchmark(debug=True) + results = { + "examples": [ + { + "task_name": "max_Float_Float_Float", + "length_bucket": "S", + "answer_format": "Float", + "answer": "9.9", + "output": "9.9", + }, + { + "task_name": "max_Float_Float_Float", + "length_bucket": "S", + "answer_format": "Float", + "answer": "9.9", + "output": "9.11", + }, + ] + } + metrics = benchmark.evaluate_responses(results) + assert metrics["exact_match"] == 0.5 + assert metrics["task:max_Float_Float_Float/exact_match"] == 0.5 + assert metrics["bucket:S/exact_match"] == 0.5 + assert metrics["task:max_Float_Float_Float/bucket:S/exact_match"] == 0.5 + assert metrics["dataset_num_samples"] == 2.0 + + +def test_task_manager_loads_nupa_native_benchmark(): + task_manager = TaskManager(task_list=["NUPA"]) + assert task_manager.is_valid_task("NUPA") diff --git a/tests/nupa/test_scorer.py b/tests/nupa/test_scorer.py new file mode 100644 index 00000000..467927ce --- /dev/null +++ b/tests/nupa/test_scorer.py @@ -0,0 +1,78 @@ +from eval.chat_benchmarks.NUPA.scorer import ( + FLOAT, + FRACTION, + INTEGER, + SCIENTIFIC, + extract_answer, + length_bucket, + normalize_answer, + score_prediction, +) + + +def test_extract_answer_accepts_official_api_answer_marker(): + assert extract_answer("The answer is 9.9, because 9.9 is larger.", FLOAT) == "9.9" + + +def test_extract_answer_does_not_search_explanatory_prose(): + assert extract_answer("I think the answer is 9.9", FLOAT) is None + + +def test_exact_match_float_nupa_trap(): + score = score_prediction("9.9", "9.9", FLOAT) + assert score.exact_match == 1.0 + assert score.digit_match == 1.0 + assert score.dlength == 0.0 + + +def test_exact_match_rejects_9_11_for_9_9(): + score = score_prediction("9.11", "9.9", FLOAT) + assert score.exact_match == 0.0 + # integer part matches; first decimal digit does not. + assert score.digit_match == 0.5 + assert score.dlength == 1.0 + + +def test_exact_match_preserves_leading_zeroes_and_does_not_ignore_commas(): + assert score_prediction("001234", "1234", INTEGER).exact_match == 0.0 + assert score_prediction("1,234", "1234", INTEGER).exact_match == 0.0 + + +def test_exact_match_preserves_float_representation(): + assert score_prediction("9.90", "9.9", FLOAT).exact_match == 0.0 + assert score_prediction("9", "9.0", FLOAT).format_valid == 0.0 + + +def test_fraction_scoring_handles_components(): + score = score_prediction("3/2", "3/2", FRACTION) + assert score.exact_match == 1.0 + assert score.digit_match == 1.0 + + +def test_scientific_notation_normalization(): + assert normalize_answer("05.040e+04", SCIENTIFIC) == "05.040e04" + score = score_prediction("5.04e4", "5.04e4", SCIENTIFIC) + assert score.exact_match == 1.0 + + +def test_invalid_format_is_not_format_valid(): + score = score_prediction("one half", "1/2", FRACTION) + assert score.exact_match == 0.0 + assert score.digit_match == 0.0 + assert score.format_valid == 0.0 + + +def test_dlength_compares_total_digit_count(): + score = score_prediction("123/456", "12/3456", FRACTION) + assert score.dlength == 0.0 + + +def test_length_bucket_boundaries(): + assert length_bucket(4, max_digit=20) == "S" + assert length_bucket(8, max_digit=20) == "M" + assert length_bucket(14, max_digit=20) == "L" + assert length_bucket(15, max_digit=20) == "XL" + assert length_bucket(10, max_digit=100) == "S" + assert length_bucket(20, max_digit=100) == "M" + assert length_bucket(60, max_digit=100) == "L" + assert length_bucket(61, max_digit=100) == "XL" diff --git a/tests/test_evaluation_limits.py b/tests/test_evaluation_limits.py index 4b26a160..26e6e071 100644 --- a/tests/test_evaluation_limits.py +++ b/tests/test_evaluation_limits.py @@ -1,7 +1,8 @@ -from types import SimpleNamespace from pathlib import Path +from types import SimpleNamespace import pytest +from evalchemy_config import EvaluationConfig from lm_eval.api.instance import Instance from eval.limits import ( @@ -13,7 +14,6 @@ safe_generation_cap, ) from eval.task import BaseBenchmark -from evalchemy_config import EvaluationConfig def _args(**overrides): @@ -165,6 +165,21 @@ def test_endpoint_preflight_uses_the_chat_template_and_is_a_noop_without_context assert cap is None +def test_endpoint_preflight_is_a_noop_when_api_adapter_has_no_tokenizer(): + messages = [{"role": "user", "content": "one two"}] + + kwargs, prompt_tokens, cap = preflight_endpoint_generation( + tokenizer=None, + payloads=[messages], + gen_kwargs={"max_tokens": 128}, + context_length=2048, + ) + + assert kwargs == {"max_tokens": 128} + assert prompt_tokens is None + assert cap is None + + def test_every_custom_benchmark_routes_generation_through_base_limit_guard(): """All chat benchmarks must reach ``BaseBenchmark.compute`` before inference. diff --git a/uv.lock b/uv.lock index 7fd82c1c..444448e0 100644 --- a/uv.lock +++ b/uv.lock @@ -1052,6 +1052,7 @@ benchmarks = [ { name = "fuzzywuzzy" }, { name = "google-generativeai" }, { name = "immutabledict" }, + { name = "ijson" }, { name = "jsonlines" }, { name = "langdetect" }, { name = "lark" }, @@ -1125,6 +1126,9 @@ mixeval = [ { name = "nltk" }, { name = "prettytable" }, ] +nupa = [ + { name = "ijson" }, +] mtbench = [ { name = "anthropic" }, { name = "fschat" }, @@ -1186,7 +1190,7 @@ requires-dist = [ { name = "cohere", marker = "extra == 'zeroeval'" }, { name = "datasets" }, { name = "emoji", marker = "extra == 'ifbench'" }, - { name = "evalchemy", extras = ["aime24", "aime25", "aiw", "amc23", "bigcodebench", "codeelo", "codeforces", "cruxeval", "financebench", "gpqadiamond", "gsm8kperturbed", "hle", "hmmt", "humaneval", "humanevalplus", "ifbench", "ifeval", "jeebench", "livebench", "livecodebench", "livecodebenchv5", "livecodebenchv5-official", "math500", "mbpp", "mbppplus", "mmlupro", "mtbench", "mixeval", "multiple", "olympiadbench", "olympiadbenchfull", "repobench", "swebench", "wildbench", "alpaca-eval", "zeroeval"], marker = "extra == 'benchmarks'" }, + { name = "evalchemy", extras = ["aime24", "aime25", "aiw", "amc23", "bigcodebench", "codeelo", "codeforces", "cruxeval", "financebench", "gpqadiamond", "gsm8kperturbed", "hle", "hmmt", "humaneval", "humanevalplus", "ifbench", "ifeval", "jeebench", "livebench", "livecodebench", "livecodebenchv5", "livecodebenchv5-official", "math500", "mbpp", "mbppplus", "mmlupro", "mtbench", "mixeval", "multiple", "nupa", "olympiadbench", "olympiadbenchfull", "repobench", "swebench", "wildbench", "alpaca-eval", "zeroeval"], marker = "extra == 'benchmarks'" }, { name = "fire", marker = "extra == 'bigcodebench'" }, { name = "fire", marker = "extra == 'humaneval'" }, { name = "fire", marker = "extra == 'humanevalplus'" }, @@ -1201,6 +1205,7 @@ requires-dist = [ { name = "huggingface-hub" }, { name = "immutabledict", marker = "extra == 'ifeval'" }, { name = "immutabledict", marker = "extra == 'livebench'" }, + { name = "ijson", marker = "extra == 'nupa'" }, { name = "isort", marker = "extra == 'dev'" }, { name = "jsonlines", marker = "extra == 'wildbench'" }, { name = "langdetect", marker = "extra == 'ifeval'" }, @@ -1251,7 +1256,7 @@ requires-dist = [ { name = "tree-sitter", marker = "extra == 'multiple'" }, { name = "tree-sitter-python", marker = "extra == 'bigcodebench'" }, ] -provides-extras = ["dev", "serve-eval", "database", "bigcodebench", "cruxeval", "humaneval", "humanevalplus", "ifeval", "ifbench", "livebench", "mbpp", "mbppplus", "mtbench", "mixeval", "multiple", "repobench", "swebench", "wildbench", "alpaca-eval", "zeroeval", "aime24", "aime25", "aiw", "amc23", "codeelo", "codeforces", "financebench", "gpqadiamond", "gsm8kperturbed", "hle", "hmmt", "jeebench", "livecodebench", "livecodebenchv5", "livecodebenchv5-official", "math500", "mmlupro", "olympiadbench", "olympiadbenchfull", "benchmarks", "vllm"] +provides-extras = ["dev", "serve-eval", "database", "bigcodebench", "cruxeval", "humaneval", "humanevalplus", "ifeval", "ifbench", "livebench", "mbpp", "mbppplus", "mtbench", "mixeval", "multiple", "repobench", "swebench", "wildbench", "alpaca-eval", "zeroeval", "aime24", "aime25", "aiw", "amc23", "codeelo", "codeforces", "financebench", "gpqadiamond", "gsm8kperturbed", "hle", "hmmt", "jeebench", "livecodebench", "livecodebenchv5", "livecodebenchv5-official", "math500", "mmlupro", "nupa", "olympiadbench", "olympiadbenchfull", "benchmarks", "vllm"] [[package]] name = "evaluate"