From 7c3bc3f9357c1c186b2c059714bcb8c714330855 Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 9 Jan 2025 12:01:44 -0800 Subject: [PATCH 1/5] grok sampler --- sampler/grok_sampler.py | 59 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 sampler/grok_sampler.py diff --git a/sampler/grok_sampler.py b/sampler/grok_sampler.py new file mode 100644 index 00000000..2d04e9ef --- /dev/null +++ b/sampler/grok_sampler.py @@ -0,0 +1,59 @@ +import os +import time +from typing import Any +from openai import OpenAI + +from ..types import MessageList, SamplerBase + +GROK_SYSTEM_MESSAGE = "You are Grok 2, an AI built by xAI, designed to answer almost any question with an outside perspective on humanity, aiming for maximum helpfulness." + + +class GrokSampler(SamplerBase): + """ + Sample from X.AI's Grok model via their API + """ + + def __init__( + self, + model: str = "grok-2-1212", + system_message: str | None = None, + temperature: float = 0.5, + max_tokens: int = 1024, + ): + self.api_key_name = "XAI_API_KEY" + self.client = OpenAI( + api_key=os.environ.get("XAI_API_KEY"), + base_url="https://api.x.ai/v1", + ) + self.model = model + self.system_message = system_message + self.temperature = temperature + self.max_tokens = max_tokens + + def _pack_message(self, role: str, content: Any): + return {"role": str(role), "content": content} + + def __call__(self, message_list: MessageList) -> str: + if self.system_message: + message_list = [ + self._pack_message("system", self.system_message) + ] + message_list + trial = 0 + while True: + try: + response = self.client.chat.completions.create( + model=self.model, + messages=message_list, + temperature=self.temperature, + max_tokens=self.max_tokens, + ) + return response.choices[0].message.content + except Exception as e: + exception_backoff = 2**trial # exponential back off + print( + f"Rate limit exception so wait and retry {trial} after {exception_backoff} sec", + e, + ) + time.sleep(exception_backoff) + trial += 1 + # unknown error shall throw exception From ea8dd0d959efd6bf5ea33bd63c85c9232f662b44 Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 9 Jan 2025 12:04:40 -0800 Subject: [PATCH 2/5] added sampler to evals --- sampler/grok_sampler.py | 2 +- simple_evals.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/sampler/grok_sampler.py b/sampler/grok_sampler.py index 2d04e9ef..603f89d1 100644 --- a/sampler/grok_sampler.py +++ b/sampler/grok_sampler.py @@ -8,7 +8,7 @@ GROK_SYSTEM_MESSAGE = "You are Grok 2, an AI built by xAI, designed to answer almost any question with an outside perspective on humanity, aiming for maximum helpfulness." -class GrokSampler(SamplerBase): +class GrokCompletionSampler(SamplerBase): """ Sample from X.AI's Grok model via their API """ diff --git a/simple_evals.py b/simple_evals.py index e3debd3b..476a1d77 100644 --- a/simple_evals.py +++ b/simple_evals.py @@ -16,6 +16,7 @@ ) from .sampler.o1_chat_completion_sampler import O1ChatCompletionSampler from .sampler.claude_sampler import ClaudeCompletionSampler, CLAUDE_SYSTEM_MESSAGE_LMSYS +from .sampler.grok_sampler import GrokCompletionSampler, GROK_SYSTEM_MESSAGE def main(): @@ -47,7 +48,7 @@ def main(): ), "o1": O1ChatCompletionSampler( model="o1", - ), + ), "o1-preview": O1ChatCompletionSampler( model="o1-preview", ), @@ -82,6 +83,11 @@ def main(): model="claude-3-opus-20240229", system_message=CLAUDE_SYSTEM_MESSAGE_LMSYS, ), + # grok models: + "grok-2-1212": GrokCompletionSampler( + model="grok-2-1212", + system_message=GROK_SYSTEM_MESSAGE, + ), } if args.list_models: From 42f52a65748acf7b083c280c06a6dcb424afb7b7 Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 9 Jan 2025 12:08:57 -0800 Subject: [PATCH 3/5] updated README --- README.md | 89 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 0a0ac114..69193c49 100644 --- a/README.md +++ b/README.md @@ -1,45 +1,47 @@ # Overview + This repository contains a lightweight library for evaluating language models. We are open sourcing it so we can be transparent about the accuracy numbers we're publishing alongside our latest models. ## Benchmark Results -| Model | Prompt | MMLU | GPQA | MATH | HumanEval | MGSM[^5] | DROP[^5]
(F1, 3-shot) | SimpleQA -|:----------------------------:|:-------------:|:------:|:------:|:------:|:---------:|:------:|:--------------------------:|:---------:| -| **o1** | | | | MATH-500[^6] | | | | -| o1 | n/a[^7] | 91.8 | 75.7 | 96.4 | n/a | 89.3 | 90.2 | 42.6 -| o1-preview | n/a | 90.8 | 73.3 | 85.5 | 92.4 | 90.8 | 74.8 | 42.4 | -| o1-mini | n/a | 85.2 | 60.0 | 90.0 | 92.4 | 89.9 | 83.9 | 7.6 | -| **GPT-4o** | | | | | | | | -| gpt-4o-2024-11-20 | assistant | 85.7 | 46.0 | 68.5 | 90.2 | 90.3 | 81.5 | 38.8 | -| gpt-4o-2024-08-06 | assistant[^2] | 88.7 | 53.1 | 75.9 | 90.2 | 90.0 | 79.8 | 40.1 | -| gpt-4o-2024-05-13 | assistant | 87.2 | 49.9 | 76.6 | 91.0 | 89.9 | 83.7 | 39.0 | -| gpt-4o-mini-2024-07-18 | assistant | 82.0 | 40.2 | 70.2 | 87.2 | 87.0 | 79.7 | 9.5 | -| **GPT-4 Turbo and GPT-4** | | | | | | | | -| gpt-4-turbo-2024-04-09 | assistant | 86.7 | 49.3 | 73.4 | 88.2 | 89.6 | 86.0 | 24.2 | -| gpt-4-0125-preview | assistant | 85.4 | 41.4 | 64.5 | 86.6 | 85.1 | 81.5 | n/a -| gpt-4-1106-preview | assistant | 84.7 | 42.5 | 64.3 | 83.7 | 87.1 | 83.2 | n/a -| **Other Models (Reported)** | | | | | | | | -| [Claude 3.5 Sonnet](https://www.anthropic.com/news/claude-3-5-sonnet) | unknown | 88.3 | 59.4 | 71.1 | 92.0 | **`91.6`** | **`87.1`** | 28.9 | -| [Claude 3 Opus](https://www.anthropic.com/news/claude-3-family) | unknown | 86.8 | 50.4 | 60.1 | 84.9 | 90.7 | 83.1 | 23.5 | -| [Llama 3.1 405b](https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md) | unknown | 88.6 | 50.7 | 73.8 | 89.0 | **`91.6`** | 84.8 | n/a -| [Llama 3.1 70b](https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md) | unknown | 82.0 | 41.7 | 68.0 | 80.5 | 86.9 | 79.6 | n/a -| [Llama 3.1 8b](https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md) | unknown | 68.4 | 30.4 | 51.9 | 72.6 | 68.9 | 59.5 | n/a -| [Grok 2](https://x.ai/blog/grok-2) | unknown | 87.5 | 56.0 | 76.1 | 88.4 | n/a | n/a | n/a -| [Grok 2 mini](https://x.ai/blog/grok-2) | unknown | 86.2 | 51.0 | 73.0 | 85.7 | n/a | n/a | n/a -| [Gemini 1.0 Ultra](https://goo.gle/GeminiV1-5) | unknown | 83.7 | n/a | 53.2 | 74.4 | 79.0 | 82.4 | n/a -| [Gemini 1.5 Pro](https://goo.gle/GeminiV1-5) | unknown | 81.9 | n/a | 58.5 | 71.9 | 88.7 | 78.9 | n/a -| [Gemini 1.5 Flash](https://goo.gle/GeminiV1-5) | unknown | 77.9 | 38.6 | 40.9 | 71.5 | 75.5 | 78.4 | n/a +| Model | Prompt | MMLU | GPQA | MATH | HumanEval | MGSM[^5] | DROP[^5]
(F1, 3-shot) | SimpleQA | +| :--------------------------------------------------------------------------------------------------: | :-----------: | :--: | :--: | :----------: | :-------: | :--------: | :----------------------: | :------: | +| **o1** | | | | MATH-500[^6] | | | | +| o1 | n/a[^7] | 91.8 | 75.7 | 96.4 | n/a | 89.3 | 90.2 | 42.6 | +| o1-preview | n/a | 90.8 | 73.3 | 85.5 | 92.4 | 90.8 | 74.8 | 42.4 | +| o1-mini | n/a | 85.2 | 60.0 | 90.0 | 92.4 | 89.9 | 83.9 | 7.6 | +| **GPT-4o** | | | | | | | | +| gpt-4o-2024-11-20 | assistant | 85.7 | 46.0 | 68.5 | 90.2 | 90.3 | 81.5 | 38.8 | +| gpt-4o-2024-08-06 | assistant[^2] | 88.7 | 53.1 | 75.9 | 90.2 | 90.0 | 79.8 | 40.1 | +| gpt-4o-2024-05-13 | assistant | 87.2 | 49.9 | 76.6 | 91.0 | 89.9 | 83.7 | 39.0 | +| gpt-4o-mini-2024-07-18 | assistant | 82.0 | 40.2 | 70.2 | 87.2 | 87.0 | 79.7 | 9.5 | +| **GPT-4 Turbo and GPT-4** | | | | | | | | +| gpt-4-turbo-2024-04-09 | assistant | 86.7 | 49.3 | 73.4 | 88.2 | 89.6 | 86.0 | 24.2 | +| gpt-4-0125-preview | assistant | 85.4 | 41.4 | 64.5 | 86.6 | 85.1 | 81.5 | n/a | +| gpt-4-1106-preview | assistant | 84.7 | 42.5 | 64.3 | 83.7 | 87.1 | 83.2 | n/a | +| **Other Models (Reported)** | | | | | | | | +| [Claude 3.5 Sonnet](https://www.anthropic.com/news/claude-3-5-sonnet) | unknown | 88.3 | 59.4 | 71.1 | 92.0 | **`91.6`** | **`87.1`** | 28.9 | +| [Claude 3 Opus](https://www.anthropic.com/news/claude-3-family) | unknown | 86.8 | 50.4 | 60.1 | 84.9 | 90.7 | 83.1 | 23.5 | +| [Llama 3.1 405b](https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md) | unknown | 88.6 | 50.7 | 73.8 | 89.0 | **`91.6`** | 84.8 | n/a | +| [Llama 3.1 70b](https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md) | unknown | 82.0 | 41.7 | 68.0 | 80.5 | 86.9 | 79.6 | n/a | +| [Llama 3.1 8b](https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md) | unknown | 68.4 | 30.4 | 51.9 | 72.6 | 68.9 | 59.5 | n/a | +| [Grok 2](https://x.ai/blog/grok-2) | unknown | 87.5 | 56.0 | 76.1 | 88.4 | n/a | n/a | n/a | +| [Grok 2 mini](https://x.ai/blog/grok-2) | unknown | 86.2 | 51.0 | 73.0 | 85.7 | n/a | n/a | n/a | +| [Gemini 1.0 Ultra](https://goo.gle/GeminiV1-5) | unknown | 83.7 | n/a | 53.2 | 74.4 | 79.0 | 82.4 | n/a | +| [Gemini 1.5 Pro](https://goo.gle/GeminiV1-5) | unknown | 81.9 | n/a | 58.5 | 71.9 | 88.7 | 78.9 | n/a | +| [Gemini 1.5 Flash](https://goo.gle/GeminiV1-5) | unknown | 77.9 | 38.6 | 40.9 | 71.5 | 75.5 | 78.4 | n/a | ## Background Evals are sensitive to prompting, and there's significant variation in the formulations used in recent publications and libraries. Some use few-shot prompts or role playing prompts ("You are an expert software programmer..."). -These approaches are carryovers from evaluating *base models* (rather than instruction/chat-tuned models) and from models that were worse at following instructions. +These approaches are carryovers from evaluating _base models_ (rather than instruction/chat-tuned models) and from models that were worse at following instructions. -For this library, we are emphasizing the *zero-shot, chain-of-thought* setting, with simple instructions like "Solve the following multiple choice problem". We believe that this prompting technique is a better reflection of the models' performance in realistic usage. +For this library, we are emphasizing the _zero-shot, chain-of-thought_ setting, with simple instructions like "Solve the following multiple choice problem". We believe that this prompting technique is a better reflection of the models' performance in realistic usage. **We will not be actively maintaining this repository and monitoring PRs and Issues.** In particular, we're not accepting new evals. Here are the changes we might accept. + - Bug fixes (hopefully not needed!) - Adding adapters for new models - Adding new rows to the table below with eval results, given new models and new system prompts. @@ -52,7 +54,7 @@ This repository currently contains the following evals: - MMLU: Measuring Massive Multitask Language Understanding, reference: https://arxiv.org/abs/2009.03300, https://github.com/hendrycks/test, [MIT License](https://github.com/hendrycks/test/blob/master/LICENSE) - MATH: Measuring Mathematical Problem Solving With the MATH Dataset, reference: https://arxiv.org/abs/2103.03874, https://github.com/hendrycks/math, [MIT License](https://github.com/idavidrein/gpqa/blob/main/LICENSE) -- GPQA: A Graduate-Level Google-Proof Q&A Benchmark, reference: https://arxiv.org/abs/2311.12022, https://github.com/idavidrein/gpqa/, [MIT License](https://github.com/idavidrein/gpqa/blob/main/LICENSE) +- GPQA: A Graduate-Level Google-Proof Q&A Benchmark, reference: https://arxiv.org/abs/2311.12022, https://github.com/idavidrein/gpqa/, [MIT License](https://github.com/idavidrein/gpqa/blob/main/LICENSE) - DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs, reference: https://arxiv.org/abs/1903.00161, https://allenai.org/data/drop, [Apache License 2.0](https://github.com/allenai/allennlp-models/blob/main/LICENSE) - MGSM: Multilingual Grade School Math Benchmark (MGSM), Language Models are Multilingual Chain-of-Thought Reasoners, reference: https://arxiv.org/abs/2210.03057, https://github.com/google-research/url-nlp, [Creative Commons Attribution 4.0 International Public License (CC-BY)](https://github.com/google-research/url-nlp/blob/main/LICENSE) - HumanEval: Evaluating Large Language Models Trained on Code, reference https://arxiv.org/abs/2107.03374, https://github.com/openai/human-eval, [MIT License](https://github.com/openai/human-eval/blob/master/LICENSE) @@ -71,42 +73,57 @@ Make sure to set the `*_API_KEY` environment variables before using these APIs. Due to the optional dependencies, we're not providing a unified setup mechanism. Instead, we're providing instructions for each eval and sampler. For [HumanEval](https://github.com/openai/human-eval/) (python programming) + ```bash git clone https://github.com/openai/human-eval pip install -e human-eval ``` For the [OpenAI API](https://pypi.org/project/openai/): + ```bash pip install openai ``` For the [Anthropic API](https://docs.anthropic.com/claude/docs/quickstart-guide): + ```bash pip install anthropic ``` +For the [xAI API](https://docs.x.ai/docs/overview): + +```bash +pip install openai +export XAI_API_KEY= +``` + ## Running the evals + ```bash python -m simple-evals.simple_evals --list-models ``` + This will list all the models that you can evaluate. To run the evaluations, you can use the following command: + ```bash python -m simple-evals.simple_evals --model --examples ``` + This will launch evaluations through the OpenAI API. ## Notes -[^1]:chatgpt system message: "You are ChatGPT, a large language model trained by OpenAI, based on the GPT-4 architecture.\nKnowledge cutoff: 2023-12\nCurrent date: 2024-04-01" -[^2]:assistant system message in [OpenAI API doc](https://platform.openai.com/docs/api-reference/introduction): "You are a helpful assistant." . -[^3]:claude-3 empty system message: suggested by Anthropic API doc, and we have done limited experiments due to [rate limit](https://docs.anthropic.com/claude/reference/rate-limits) issues, but we welcome PRs with alternative choices. -[^4]:claude-3 lmsys system message: system message in LMSYS [Fast-chat open source code](https://github.com/lm-sys/FastChat/blob/7899355ebe32117fdae83985cf8ee476d2f4243f/fastchat/conversation.py#L894): "The assistant is Claude, created by Anthropic. The current date is {{currentDateTime}}. Claude's knowledge base was last updated ... ". We have done limited experiments due to [rate limit](https://docs.anthropic.com/claude/reference/rate-limits) issues, but we welcome PRs with alternative choices. -[^5]:We believe these evals are saturated for our newer models, but are reporting them for completeness. -[^6]:For o1 models, we evaluate on [MATH-500](https://github.com/openai/prm800k/tree/main/prm800k/math_splits), which is a newer, IID version of MATH. -[^7]:o1 models do not support using a system prompt. +[^1]: chatgpt system message: "You are ChatGPT, a large language model trained by OpenAI, based on the GPT-4 architecture.\nKnowledge cutoff: 2023-12\nCurrent date: 2024-04-01" +[^2]: assistant system message in [OpenAI API doc](https://platform.openai.com/docs/api-reference/introduction): "You are a helpful assistant." . +[^3]: claude-3 empty system message: suggested by Anthropic API doc, and we have done limited experiments due to [rate limit](https://docs.anthropic.com/claude/reference/rate-limits) issues, but we welcome PRs with alternative choices. +[^4]: claude-3 lmsys system message: system message in LMSYS [Fast-chat open source code](https://github.com/lm-sys/FastChat/blob/7899355ebe32117fdae83985cf8ee476d2f4243f/fastchat/conversation.py#L894): "The assistant is Claude, created by Anthropic. The current date is {{currentDateTime}}. Claude's knowledge base was last updated ... ". We have done limited experiments due to [rate limit](https://docs.anthropic.com/claude/reference/rate-limits) issues, but we welcome PRs with alternative choices. +[^5]: We believe these evals are saturated for our newer models, but are reporting them for completeness. +[^6]: For o1 models, we evaluate on [MATH-500](https://github.com/openai/prm800k/tree/main/prm800k/math_splits), which is a newer, IID version of MATH. +[^7]: o1 models do not support using a system prompt. ## Legal Stuff + By contributing to evals, you are agreeing to make your evaluation logic and data under the same MIT license as this repository. You must have adequate rights to upload any data used in an eval. OpenAI reserves the right to use this data in future service improvements to our product. Contributions to OpenAI evals will be subject to our usual Usage Policies: https://platform.openai.com/docs/usage-policies. From 946d14a805ca10bd0bcd4f38ce223333cd369f59 Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 9 Jan 2025 12:15:27 -0800 Subject: [PATCH 4/5] fix README format --- README.md | 85 ++++++++++++++++++++++++------------------------------- 1 file changed, 37 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 69193c49..a625becc 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,45 @@ # Overview - This repository contains a lightweight library for evaluating language models. We are open sourcing it so we can be transparent about the accuracy numbers we're publishing alongside our latest models. ## Benchmark Results -| Model | Prompt | MMLU | GPQA | MATH | HumanEval | MGSM[^5] | DROP[^5]
(F1, 3-shot) | SimpleQA | -| :--------------------------------------------------------------------------------------------------: | :-----------: | :--: | :--: | :----------: | :-------: | :--------: | :----------------------: | :------: | -| **o1** | | | | MATH-500[^6] | | | | -| o1 | n/a[^7] | 91.8 | 75.7 | 96.4 | n/a | 89.3 | 90.2 | 42.6 | -| o1-preview | n/a | 90.8 | 73.3 | 85.5 | 92.4 | 90.8 | 74.8 | 42.4 | -| o1-mini | n/a | 85.2 | 60.0 | 90.0 | 92.4 | 89.9 | 83.9 | 7.6 | -| **GPT-4o** | | | | | | | | -| gpt-4o-2024-11-20 | assistant | 85.7 | 46.0 | 68.5 | 90.2 | 90.3 | 81.5 | 38.8 | -| gpt-4o-2024-08-06 | assistant[^2] | 88.7 | 53.1 | 75.9 | 90.2 | 90.0 | 79.8 | 40.1 | -| gpt-4o-2024-05-13 | assistant | 87.2 | 49.9 | 76.6 | 91.0 | 89.9 | 83.7 | 39.0 | -| gpt-4o-mini-2024-07-18 | assistant | 82.0 | 40.2 | 70.2 | 87.2 | 87.0 | 79.7 | 9.5 | -| **GPT-4 Turbo and GPT-4** | | | | | | | | -| gpt-4-turbo-2024-04-09 | assistant | 86.7 | 49.3 | 73.4 | 88.2 | 89.6 | 86.0 | 24.2 | -| gpt-4-0125-preview | assistant | 85.4 | 41.4 | 64.5 | 86.6 | 85.1 | 81.5 | n/a | -| gpt-4-1106-preview | assistant | 84.7 | 42.5 | 64.3 | 83.7 | 87.1 | 83.2 | n/a | -| **Other Models (Reported)** | | | | | | | | -| [Claude 3.5 Sonnet](https://www.anthropic.com/news/claude-3-5-sonnet) | unknown | 88.3 | 59.4 | 71.1 | 92.0 | **`91.6`** | **`87.1`** | 28.9 | -| [Claude 3 Opus](https://www.anthropic.com/news/claude-3-family) | unknown | 86.8 | 50.4 | 60.1 | 84.9 | 90.7 | 83.1 | 23.5 | -| [Llama 3.1 405b](https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md) | unknown | 88.6 | 50.7 | 73.8 | 89.0 | **`91.6`** | 84.8 | n/a | -| [Llama 3.1 70b](https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md) | unknown | 82.0 | 41.7 | 68.0 | 80.5 | 86.9 | 79.6 | n/a | -| [Llama 3.1 8b](https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md) | unknown | 68.4 | 30.4 | 51.9 | 72.6 | 68.9 | 59.5 | n/a | -| [Grok 2](https://x.ai/blog/grok-2) | unknown | 87.5 | 56.0 | 76.1 | 88.4 | n/a | n/a | n/a | -| [Grok 2 mini](https://x.ai/blog/grok-2) | unknown | 86.2 | 51.0 | 73.0 | 85.7 | n/a | n/a | n/a | -| [Gemini 1.0 Ultra](https://goo.gle/GeminiV1-5) | unknown | 83.7 | n/a | 53.2 | 74.4 | 79.0 | 82.4 | n/a | -| [Gemini 1.5 Pro](https://goo.gle/GeminiV1-5) | unknown | 81.9 | n/a | 58.5 | 71.9 | 88.7 | 78.9 | n/a | -| [Gemini 1.5 Flash](https://goo.gle/GeminiV1-5) | unknown | 77.9 | 38.6 | 40.9 | 71.5 | 75.5 | 78.4 | n/a | +| Model | Prompt | MMLU | GPQA | MATH | HumanEval | MGSM[^5] | DROP[^5]
(F1, 3-shot) | SimpleQA +|:----------------------------:|:-------------:|:------:|:------:|:------:|:---------:|:------:|:--------------------------:|:---------:| +| **o1** | | | | MATH-500[^6] | | | | +| o1 | n/a[^7] | 91.8 | 75.7 | 96.4 | n/a | 89.3 | 90.2 | 42.6 +| o1-preview | n/a | 90.8 | 73.3 | 85.5 | 92.4 | 90.8 | 74.8 | 42.4 | +| o1-mini | n/a | 85.2 | 60.0 | 90.0 | 92.4 | 89.9 | 83.9 | 7.6 | +| **GPT-4o** | | | | | | | | +| gpt-4o-2024-11-20 | assistant | 85.7 | 46.0 | 68.5 | 90.2 | 90.3 | 81.5 | 38.8 | +| gpt-4o-2024-08-06 | assistant[^2] | 88.7 | 53.1 | 75.9 | 90.2 | 90.0 | 79.8 | 40.1 | +| gpt-4o-2024-05-13 | assistant | 87.2 | 49.9 | 76.6 | 91.0 | 89.9 | 83.7 | 39.0 | +| gpt-4o-mini-2024-07-18 | assistant | 82.0 | 40.2 | 70.2 | 87.2 | 87.0 | 79.7 | 9.5 | +| **GPT-4 Turbo and GPT-4** | | | | | | | | +| gpt-4-turbo-2024-04-09 | assistant | 86.7 | 49.3 | 73.4 | 88.2 | 89.6 | 86.0 | 24.2 | +| gpt-4-0125-preview | assistant | 85.4 | 41.4 | 64.5 | 86.6 | 85.1 | 81.5 | n/a +| gpt-4-1106-preview | assistant | 84.7 | 42.5 | 64.3 | 83.7 | 87.1 | 83.2 | n/a +| **Other Models (Reported)** | | | | | | | | +| [Claude 3.5 Sonnet](https://www.anthropic.com/news/claude-3-5-sonnet) | unknown | 88.3 | 59.4 | 71.1 | 92.0 | **`91.6`** | **`87.1`** | 28.9 | +| [Claude 3 Opus](https://www.anthropic.com/news/claude-3-family) | unknown | 86.8 | 50.4 | 60.1 | 84.9 | 90.7 | 83.1 | 23.5 | +| [Llama 3.1 405b](https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md) | unknown | 88.6 | 50.7 | 73.8 | 89.0 | **`91.6`** | 84.8 | n/a +| [Llama 3.1 70b](https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md) | unknown | 82.0 | 41.7 | 68.0 | 80.5 | 86.9 | 79.6 | n/a +| [Llama 3.1 8b](https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md) | unknown | 68.4 | 30.4 | 51.9 | 72.6 | 68.9 | 59.5 | n/a +| [Grok 2](https://x.ai/blog/grok-2) | unknown | 87.5 | 56.0 | 76.1 | 88.4 | n/a | n/a | n/a +| [Grok 2 mini](https://x.ai/blog/grok-2) | unknown | 86.2 | 51.0 | 73.0 | 85.7 | n/a | n/a | n/a +| [Gemini 1.0 Ultra](https://goo.gle/GeminiV1-5) | unknown | 83.7 | n/a | 53.2 | 74.4 | 79.0 | 82.4 | n/a +| [Gemini 1.5 Pro](https://goo.gle/GeminiV1-5) | unknown | 81.9 | n/a | 58.5 | 71.9 | 88.7 | 78.9 | n/a +| [Gemini 1.5 Flash](https://goo.gle/GeminiV1-5) | unknown | 77.9 | 38.6 | 40.9 | 71.5 | 75.5 | 78.4 | n/a ## Background Evals are sensitive to prompting, and there's significant variation in the formulations used in recent publications and libraries. Some use few-shot prompts or role playing prompts ("You are an expert software programmer..."). -These approaches are carryovers from evaluating _base models_ (rather than instruction/chat-tuned models) and from models that were worse at following instructions. +These approaches are carryovers from evaluating *base models* (rather than instruction/chat-tuned models) and from models that were worse at following instructions. -For this library, we are emphasizing the _zero-shot, chain-of-thought_ setting, with simple instructions like "Solve the following multiple choice problem". We believe that this prompting technique is a better reflection of the models' performance in realistic usage. +For this library, we are emphasizing the *zero-shot, chain-of-thought* setting, with simple instructions like "Solve the following multiple choice problem". We believe that this prompting technique is a better reflection of the models' performance in realistic usage. **We will not be actively maintaining this repository and monitoring PRs and Issues.** In particular, we're not accepting new evals. Here are the changes we might accept. - - Bug fixes (hopefully not needed!) - Adding adapters for new models - Adding new rows to the table below with eval results, given new models and new system prompts. @@ -54,7 +52,7 @@ This repository currently contains the following evals: - MMLU: Measuring Massive Multitask Language Understanding, reference: https://arxiv.org/abs/2009.03300, https://github.com/hendrycks/test, [MIT License](https://github.com/hendrycks/test/blob/master/LICENSE) - MATH: Measuring Mathematical Problem Solving With the MATH Dataset, reference: https://arxiv.org/abs/2103.03874, https://github.com/hendrycks/math, [MIT License](https://github.com/idavidrein/gpqa/blob/main/LICENSE) -- GPQA: A Graduate-Level Google-Proof Q&A Benchmark, reference: https://arxiv.org/abs/2311.12022, https://github.com/idavidrein/gpqa/, [MIT License](https://github.com/idavidrein/gpqa/blob/main/LICENSE) +- GPQA: A Graduate-Level Google-Proof Q&A Benchmark, reference: https://arxiv.org/abs/2311.12022, https://github.com/idavidrein/gpqa/, [MIT License](https://github.com/idavidrein/gpqa/blob/main/LICENSE) - DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs, reference: https://arxiv.org/abs/1903.00161, https://allenai.org/data/drop, [Apache License 2.0](https://github.com/allenai/allennlp-models/blob/main/LICENSE) - MGSM: Multilingual Grade School Math Benchmark (MGSM), Language Models are Multilingual Chain-of-Thought Reasoners, reference: https://arxiv.org/abs/2210.03057, https://github.com/google-research/url-nlp, [Creative Commons Attribution 4.0 International Public License (CC-BY)](https://github.com/google-research/url-nlp/blob/main/LICENSE) - HumanEval: Evaluating Large Language Models Trained on Code, reference https://arxiv.org/abs/2107.03374, https://github.com/openai/human-eval, [MIT License](https://github.com/openai/human-eval/blob/master/LICENSE) @@ -73,57 +71,48 @@ Make sure to set the `*_API_KEY` environment variables before using these APIs. Due to the optional dependencies, we're not providing a unified setup mechanism. Instead, we're providing instructions for each eval and sampler. For [HumanEval](https://github.com/openai/human-eval/) (python programming) - ```bash git clone https://github.com/openai/human-eval pip install -e human-eval ``` For the [OpenAI API](https://pypi.org/project/openai/): - ```bash pip install openai ``` For the [Anthropic API](https://docs.anthropic.com/claude/docs/quickstart-guide): - ```bash pip install anthropic ``` For the [xAI API](https://docs.x.ai/docs/overview): - ```bash pip install openai export XAI_API_KEY= ``` ## Running the evals - ```bash python -m simple-evals.simple_evals --list-models ``` - This will list all the models that you can evaluate. To run the evaluations, you can use the following command: - ```bash python -m simple-evals.simple_evals --model --examples ``` - This will launch evaluations through the OpenAI API. ## Notes -[^1]: chatgpt system message: "You are ChatGPT, a large language model trained by OpenAI, based on the GPT-4 architecture.\nKnowledge cutoff: 2023-12\nCurrent date: 2024-04-01" -[^2]: assistant system message in [OpenAI API doc](https://platform.openai.com/docs/api-reference/introduction): "You are a helpful assistant." . -[^3]: claude-3 empty system message: suggested by Anthropic API doc, and we have done limited experiments due to [rate limit](https://docs.anthropic.com/claude/reference/rate-limits) issues, but we welcome PRs with alternative choices. -[^4]: claude-3 lmsys system message: system message in LMSYS [Fast-chat open source code](https://github.com/lm-sys/FastChat/blob/7899355ebe32117fdae83985cf8ee476d2f4243f/fastchat/conversation.py#L894): "The assistant is Claude, created by Anthropic. The current date is {{currentDateTime}}. Claude's knowledge base was last updated ... ". We have done limited experiments due to [rate limit](https://docs.anthropic.com/claude/reference/rate-limits) issues, but we welcome PRs with alternative choices. -[^5]: We believe these evals are saturated for our newer models, but are reporting them for completeness. -[^6]: For o1 models, we evaluate on [MATH-500](https://github.com/openai/prm800k/tree/main/prm800k/math_splits), which is a newer, IID version of MATH. -[^7]: o1 models do not support using a system prompt. +[^1]:chatgpt system message: "You are ChatGPT, a large language model trained by OpenAI, based on the GPT-4 architecture.\nKnowledge cutoff: 2023-12\nCurrent date: 2024-04-01" +[^2]:assistant system message in [OpenAI API doc](https://platform.openai.com/docs/api-reference/introduction): "You are a helpful assistant." . +[^3]:claude-3 empty system message: suggested by Anthropic API doc, and we have done limited experiments due to [rate limit](https://docs.anthropic.com/claude/reference/rate-limits) issues, but we welcome PRs with alternative choices. +[^4]:claude-3 lmsys system message: system message in LMSYS [Fast-chat open source code](https://github.com/lm-sys/FastChat/blob/7899355ebe32117fdae83985cf8ee476d2f4243f/fastchat/conversation.py#L894): "The assistant is Claude, created by Anthropic. The current date is {{currentDateTime}}. Claude's knowledge base was last updated ... ". We have done limited experiments due to [rate limit](https://docs.anthropic.com/claude/reference/rate-limits) issues, but we welcome PRs with alternative choices. +[^5]:We believe these evals are saturated for our newer models, but are reporting them for completeness. +[^6]:For o1 models, we evaluate on [MATH-500](https://github.com/openai/prm800k/tree/main/prm800k/math_splits), which is a newer, IID version of MATH. +[^7]:o1 models do not support using a system prompt. ## Legal Stuff - -By contributing to evals, you are agreeing to make your evaluation logic and data under the same MIT license as this repository. You must have adequate rights to upload any data used in an eval. OpenAI reserves the right to use this data in future service improvements to our product. Contributions to OpenAI evals will be subject to our usual Usage Policies: https://platform.openai.com/docs/usage-policies. +By contributing to evals, you are agreeing to make your evaluation logic and data under the same MIT license as this repository. You must have adequate rights to upload any data used in an eval. OpenAI reserves the right to use this data in future service improvements to our product. Contributions to OpenAI evals will be subject to our usual Usage Policies: https://platform.openai.com/docs/usage-policies. \ No newline at end of file From 8a87aa948b6b30b52312d790225854fd386f0b68 Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 9 Jan 2025 12:22:11 -0800 Subject: [PATCH 5/5] fix README format --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a625becc..12b31125 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,9 @@ pip install anthropic For the [xAI API](https://docs.x.ai/docs/overview): ```bash pip install openai -export XAI_API_KEY= + +# Create a .env file and add your XAI API key +echo "XAI_API_KEY=" > .env ``` ## Running the evals @@ -115,4 +117,4 @@ This will launch evaluations through the OpenAI API. [^7]:o1 models do not support using a system prompt. ## Legal Stuff -By contributing to evals, you are agreeing to make your evaluation logic and data under the same MIT license as this repository. You must have adequate rights to upload any data used in an eval. OpenAI reserves the right to use this data in future service improvements to our product. Contributions to OpenAI evals will be subject to our usual Usage Policies: https://platform.openai.com/docs/usage-policies. \ No newline at end of file +By contributing to evals, you are agreeing to make your evaluation logic and data under the same MIT license as this repository. You must have adequate rights to upload any data used in an eval. OpenAI reserves the right to use this data in future service improvements to our product. Contributions to OpenAI evals will be subject to our usual Usage Policies: https://platform.openai.com/docs/usage-policies.