diff --git a/auto_round/cli/main.py b/auto_round/cli/main.py index e99595119e..a20e11612f 100644 --- a/auto_round/cli/main.py +++ b/auto_round/cli/main.py @@ -444,6 +444,9 @@ def run_eval(argv=None): trust_remote_code=not args.disable_trust_remote_code, eval_model_dtype=args.eval_model_dtype, add_bos_token=args.add_bos_token, + num_fewshot=args.num_fewshot, + gen_kwargs=args.eval_gen_kwargs, + fewshot_as_multiturn=args.fewshot_as_multiturn, ) else: eval(args) diff --git a/auto_round/cli/parser.py b/auto_round/cli/parser.py index 31001eab1e..198cd57859 100644 --- a/auto_round/cli/parser.py +++ b/auto_round/cli/parser.py @@ -207,6 +207,16 @@ def build_quantize_parser(*, prog: str = "auto_round quantize") -> argparse.Argu ev.add_argument( "--limit", type=float, default=None, metavar="N|0=0.4.2", "lm-eval is required for evaluation, please install it with `pip install 'lm-eval>=0.4.2'`" @@ -404,7 +426,17 @@ def eval_task_by_task( add_bos_token=add_bos_token, ) - _evaluate_tasks_with_retry(tasks, hflm, device_str, batch_size, limit, retry_times) + _evaluate_tasks_with_retry( + tasks, + hflm, + device_str, + batch_size, + limit, + retry_times, + num_fewshot=num_fewshot, + gen_kwargs=gen_kwargs, + fewshot_as_multiturn=fewshot_as_multiturn, + ) def _load_gguf_model_if_needed(model_path, eval_model_dtype=None): @@ -459,7 +491,35 @@ def _load_gguf_model_if_needed(model_path, eval_model_dtype=None): return model, tokenizer, is_gguf_file, gguf_file -def _evaluate_tasks_with_retry(tasks, hflm, device_str, batch_size, limit, retry_times): +def _get_lm_eval_task_manager(tasks): + """Use exact installed task dirs when possible to avoid scanning all lm-eval tasks.""" + try: + import lm_eval # pylint: disable=E0401 + from lm_eval.tasks import TaskManager # pylint: disable=E0401 + + tasks_root = os.path.join(os.path.dirname(lm_eval.__file__), "tasks") + task_paths = [] + for task in tasks: + task_path = os.path.join(tasks_root, task) + if not os.path.isdir(task_path): + return None + task_paths.append(task_path) + return TaskManager(include_defaults=False, include_path=task_paths) + except Exception: + return None + + +def _evaluate_tasks_with_retry( + tasks, + hflm, + device_str, + batch_size, + limit, + retry_times, + num_fewshot=None, + gen_kwargs=None, + fewshot_as_multiturn=False, +): """Evaluate tasks with automatic retry on OOM errors. Args: @@ -487,18 +547,33 @@ def _evaluate_tasks_with_retry(tasks, hflm, device_str, batch_size, limit, retry res_all = {} res_keys = ["results", "versions", "n-shot", "higher_is_better"] st = time.time() + task_manager = _get_lm_eval_task_manager(tasks) for task in tasks: current_retry_times = retry_times res = None + last_error = None while current_retry_times: try: res = lm_eval.simple_evaluate( - model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size, limit=limit + model=hflm, + model_args=None, + device=device_str, + tasks=task, + batch_size=batch_size, + limit=limit, + num_fewshot=num_fewshot, + gen_kwargs=gen_kwargs, + task_manager=task_manager, + fewshot_as_multiturn=fewshot_as_multiturn, ) break except Exception as e: + last_error = e cuda_error_msg = traceback.format_exc() + if "out of memory" not in cuda_error_msg.lower(): + logger.error(cuda_error_msg) + raise try: ori_batch_sizes = hflm.batch_sizes or {"0": 64} if not hflm.batch_sizes: @@ -508,20 +583,31 @@ def _evaluate_tasks_with_retry(tasks, hflm, device_str, batch_size, limit, retry hflm.batch_sizes[k] = max(v // 2, 1) logger.warning(f"Out of memory, reset batch_size to {hflm.batch_sizes} and re-try.") res = lm_eval.simple_evaluate( - model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1, limit=limit + model=hflm, + model_args=None, + device=device_str, + tasks=task, + batch_size=1, + limit=limit, + num_fewshot=num_fewshot, + gen_kwargs=gen_kwargs, + task_manager=task_manager, + fewshot_as_multiturn=fewshot_as_multiturn, ) hflm.batch_sizes = ori_batch_sizes except Exception as e: + last_error = e traceback.print_exc() res = None except Exception as e: + last_error = e logger.error(cuda_error_msg) traceback.print_exc() res = None current_retry_times -= 1 if res is None: - raise RuntimeError(f"Failed to evaluate task '{task}' after {retry_times} attempts") + raise RuntimeError(f"Failed to evaluate task '{task}' after {retry_times} attempts") from last_error if not res_all: res_all = res else: diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.py index 760c3acca0..c01e5feb72 100644 --- a/auto_round/eval/evaluation.py +++ b/auto_round/eval/evaluation.py @@ -310,6 +310,9 @@ def evaluate_with_model_instance(model, tokenizer, device_str, args): batch_size=args.eval_bs, eval_model_dtype=get_model_dtype(args.eval_model_dtype, "auto"), add_bos_token=args.add_bos_token, + num_fewshot=getattr(args, "num_fewshot", None), + gen_kwargs=getattr(args, "eval_gen_kwargs", None), + fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False), ) else: # Batch evaluation @@ -328,6 +331,9 @@ def evaluate_with_model_instance(model, tokenizer, device_str, args): device=device_str, eval_model_dtype=get_model_dtype(args.eval_model_dtype, "auto"), add_bos_token=args.add_bos_token, + num_fewshot=getattr(args, "num_fewshot", None), + gen_kwargs=getattr(args, "eval_gen_kwargs", None), + fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False), ) print(make_table(res)) print("evaluation running time=%ds" % (time.time() - st)) @@ -366,6 +372,9 @@ def evaluate_with_model_path(eval_folder, device_str, autoround, args): eval_model_dtype=get_model_dtype(args.eval_model_dtype, "auto"), mllm=getattr(autoround, "mllm", False), add_bos_token=args.add_bos_token, + num_fewshot=getattr(args, "num_fewshot", None), + gen_kwargs=getattr(args, "eval_gen_kwargs", None), + fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False), ) else: # Batch evaluation @@ -398,6 +407,9 @@ def evaluate_with_model_path(eval_folder, device_str, autoround, args): device=device_str, batch_size=eval_bs, limit=args.limit, + num_fewshot=getattr(args, "num_fewshot", None), + gen_kwargs=getattr(args, "eval_gen_kwargs", None), + fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False), ) print(make_table(res)) print("evaluation running time=%ds" % (time.time() - st)) @@ -468,6 +480,9 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, args): vllm_args.disable_trust_remote_code = getattr(args, "disable_trust_remote_code", False) vllm_args.add_bos_token = getattr(args, "add_bos_token", False) vllm_args.seed = getattr(args, "seed", 42) + vllm_args.num_fewshot = getattr(args, "num_fewshot", None) + vllm_args.eval_gen_kwargs = getattr(args, "eval_gen_kwargs", None) + vllm_args.fewshot_as_multiturn = getattr(args, "fewshot_as_multiturn", False) # VLLM-specific parameters vllm_args.vllm_args = getattr(args, "vllm_args", None) eval_with_vllm(vllm_args) diff --git a/docs/step_by_step.md b/docs/step_by_step.md index f104663823..f5fea9bda5 100644 --- a/docs/step_by_step.md +++ b/docs/step_by_step.md @@ -1093,6 +1093,7 @@ CUDA_VISIBLE_DEVICES=0,1 auto-round "your_model_path" --eval --tasks lambada_ope - Use the `--eval` flag to evaluate models directly. This supports both original and quantized models. - The `--eval_task_by_task` option helps handle task failures by evaluating tasks sequentially. This only applies to the HF backend. +- Use `--num_fewshot`, `--eval_gen_kwargs`, and `--fewshot_as_multiturn` to pass few-shot and generation options through to lm-eval. - When multiple formats are exported, the last format in the list will be used for evaluation. - For vLLM backend, you can use `--device 0,1,2` to specify GPU devices. This will automatically set `CUDA_VISIBLE_DEVICES` and configure `tensor_parallel_size` based on the number of devices. Alternatively, you can manually set these via environment variables and `--vllm_args`. diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md index 77b5eedfa3..e553e3b6e0 100644 --- a/docs/step_by_step_CN.md +++ b/docs/step_by_step_CN.md @@ -1060,6 +1060,7 @@ CUDA_VISIBLE_DEVICES=0,1 auto-round "your_model_path" --eval --tasks lambada_ope - 对于原始模型和量化后的模型，都支持用 `--eval` 参数直接评估。 - 为应对部分任务运行失败的情况，可使用 `--eval_task_by_task` 参数，按顺序执行评测任务（该参数目前只适用于 HF 后端）。 +- 可使用 `--num_fewshot`、`--eval_gen_kwargs` 和 `--fewshot_as_multiturn` 将 few-shot 与生成参数传递给 lm-eval。 - 若导出了多种格式，会自动选用列表中的**最后一种格式**的模型评估。 - 对于 vLLM 后端，可通过 `--device 0,1,2` 指定 GPU 设备。该参数会自动设置 `CUDA_VISIBLE_DEVICES`，并根据设备数量配置 `tensor_parallel_size` 。此外，也支持通过环境变量和 `--vllm_args` 参数进行手动设置。 diff --git a/test/unit/test_cpu/advanced/test_evaluation_functions.py b/test/unit/test_cpu/advanced/test_evaluation_functions.py index d6476ec217..e195ada392 100644 --- a/test/unit/test_cpu/advanced/test_evaluation_functions.py +++ b/test/unit/test_cpu/advanced/test_evaluation_functions.py @@ -16,14 +16,14 @@ CPU tests for evaluation utility functions. Lightweight tests focusing on key utility functions without heavy model loading. -Run with: pytest test/test_cpu/advanced/test_evaluation_functions.py +Run with: pytest test/unit/test_cpu/advanced/test_evaluation_functions.py """ -import os +import argparse +import sys +from types import ModuleType, SimpleNamespace from unittest.mock import MagicMock, patch -import pytest - class TestSelectGgufEvalFile: """Test GGUF text file selection for evaluation.""" @@ -138,11 +138,178 @@ def test_load_gguf_model_non_gguf_string_path(self): assert is_gguf is False assert gguf_file is None - def test_load_gguf_model_non_string_model(self, tiny_opt_model_path): + def test_load_gguf_model_non_string_model(self): """Test with model object (not a string path).""" from auto_round.eval.eval_cli import _load_gguf_model_if_needed - model, tokenizer, is_gguf, gguf_file = _load_gguf_model_if_needed(tiny_opt_model_path) + model_obj = object() + model, tokenizer, is_gguf, gguf_file = _load_gguf_model_if_needed(model_obj) + assert model is model_obj assert tokenizer is None assert is_gguf is False assert gguf_file is None + + +def _make_eval_args(**overrides): + defaults = dict( + model="test-model", + model_name="test-model", + mllm=False, + device_map="0", + tasks="lambada_openai", + disable_trust_remote_code=False, + seed=42, + eval_bs=2, + eval_task_by_task=False, + eval_model_dtype=None, + limit=5, + num_fewshot=3, + eval_gen_kwargs="temperature=0.1,top_p=0.9", + fewshot_as_multiturn=True, + eval_backend="hf", + add_bos_token=False, + vllm_args=None, + ) + defaults.update(overrides) + return argparse.Namespace(**defaults) + + +def _fake_lm_eval_modules(simple_evaluate_impl): + root_module = ModuleType("lm_eval") + utils_module = ModuleType("lm_eval.utils") + utils_module.make_table = lambda result: "table" + + evaluator_module = ModuleType("lm_eval.evaluator") + evaluator_module.simple_evaluate = simple_evaluate_impl + + models_module = ModuleType("lm_eval.models") + vllm_module = ModuleType("lm_eval.models.vllm_causallms") + vllm_module.VLLM = MagicMock(side_effect=lambda **kwargs: SimpleNamespace(kwargs=kwargs)) + + vllm_vlm_module = ModuleType("lm_eval.models.vllm_vlms") + vllm_vlm_module.VLLM_VLM = MagicMock(side_effect=lambda **kwargs: SimpleNamespace(kwargs=kwargs)) + + root_module.evaluator = evaluator_module + root_module.utils = utils_module + root_module.models = models_module + models_module.vllm_causallms = vllm_module + models_module.vllm_vlms = vllm_vlm_module + + return { + "lm_eval": root_module, + "lm_eval.utils": utils_module, + "lm_eval.models": models_module, + "lm_eval.evaluator": evaluator_module, + "lm_eval.models.vllm_causallms": vllm_module, + "lm_eval.models.vllm_vlms": vllm_vlm_module, + } + + +class TestEvalArgumentForwarding: + def test_run_eval_task_by_task_forwards_eval_generation_arguments(self): + from auto_round.cli.main import run_eval + + args = _make_eval_args(eval_task_by_task=True) + + with patch("auto_round.cli.main.setup_eval_parser", return_value=args), patch( + "auto_round.utils.is_gguf_model", return_value=False + ), patch("auto_round.utils.is_mllm_model", return_value=False), patch( + "auto_round.eval.eval_cli.eval_task_by_task" + ) as mock_eval_task_by_task: + run_eval([]) + + _, kwargs = mock_eval_task_by_task.call_args + assert kwargs["num_fewshot"] == args.num_fewshot + assert kwargs["gen_kwargs"] == args.eval_gen_kwargs + assert kwargs["fewshot_as_multiturn"] == args.fewshot_as_multiturn + + def test_eval_hf_batch_forwards_eval_generation_arguments(self): + from auto_round.eval.eval_cli import eval + + args = _make_eval_args() + result = {"results": {"lambada_openai": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}} + + with patch("auto_round.eval.eval_cli.require_version"), patch( + "auto_round.eval.eval_cli.is_diffusion_model", return_value=False + ), patch( + "auto_round.eval.eval_cli._eval_init", return_value=(["lambada_openai"], "pretrained=test-model", "cpu") + ), patch( + "auto_round.eval.eval_cli._load_gguf_model_if_needed", return_value=(None, None, False, None) + ), patch( + "auto_round.eval.evaluation.simple_evaluate", return_value=result + ) as mock_simple_evaluate, patch( + "builtins.print" + ), patch.dict( + sys.modules, _fake_lm_eval_modules(MagicMock(return_value=result)) + ): + eval(args) + + _, kwargs = mock_simple_evaluate.call_args + assert kwargs["num_fewshot"] == args.num_fewshot + assert kwargs["gen_kwargs"] == args.eval_gen_kwargs + assert kwargs["fewshot_as_multiturn"] == args.fewshot_as_multiturn + + def test_eval_gguf_batch_forwards_eval_generation_arguments(self): + from auto_round.eval.eval_cli import eval + + args = _make_eval_args() + result = {"results": {"lambada_openai": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}} + + with patch("auto_round.eval.eval_cli.require_version"), patch( + "auto_round.eval.eval_cli.is_diffusion_model", return_value=False + ), patch( + "auto_round.eval.eval_cli._eval_init", return_value=(["lambada_openai"], "pretrained=test-model", "cpu") + ), patch( + "auto_round.eval.eval_cli._load_gguf_model_if_needed", + return_value=(object(), object(), True, "model.gguf"), + ), patch( + "auto_round.eval.evaluation.simple_evaluate_user_model", return_value=result + ) as mock_simple_evaluate_user_model, patch( + "builtins.print" + ), patch.dict( + sys.modules, _fake_lm_eval_modules(MagicMock(return_value=result)) + ): + eval(args) + + _, kwargs = mock_simple_evaluate_user_model.call_args + assert kwargs["num_fewshot"] == args.num_fewshot + assert kwargs["gen_kwargs"] == args.eval_gen_kwargs + assert kwargs["fewshot_as_multiturn"] == args.fewshot_as_multiturn + + def test_eval_with_vllm_forwards_eval_generation_arguments(self): + from auto_round.eval.eval_cli import eval_with_vllm + + args = _make_eval_args() + evaluator_simple_evaluate = MagicMock( + return_value={"results": {"lambada_openai": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}} + ) + + with patch("auto_round.eval.eval_cli.get_device_and_parallelism", return_value=("cuda:0", False)), patch( + "auto_round.eval.eval_cli.get_major_device", return_value="cuda" + ), patch("auto_round.eval.eval_cli.get_model_dtype", return_value="float16"), patch( + "builtins.print" + ), patch.dict( + sys.modules, _fake_lm_eval_modules(evaluator_simple_evaluate) + ): + eval_with_vllm(args) + + _, kwargs = evaluator_simple_evaluate.call_args + assert kwargs["num_fewshot"] == args.num_fewshot + assert kwargs["gen_kwargs"] == args.eval_gen_kwargs + assert kwargs["fewshot_as_multiturn"] == args.fewshot_as_multiturn + + def test_run_model_evaluation_vllm_forwards_eval_generation_arguments(self, tmp_path): + from auto_round.eval.evaluation import run_model_evaluation + + args = _make_eval_args(eval_backend="vllm") + autoround = SimpleNamespace() + + with patch("auto_round.utils.model.detect_model_type", return_value="llm"), patch( + "auto_round.utils.device_manager.get_device_and_parallelism", return_value=("cpu", False) + ), patch("auto_round.eval.eval_cli.eval_with_vllm") as mock_eval_with_vllm: + run_model_evaluation(None, None, autoround, str(tmp_path), ["auto_round"], args) + + forwarded_args = mock_eval_with_vllm.call_args.args[0] + assert forwarded_args.num_fewshot == args.num_fewshot + assert forwarded_args.eval_gen_kwargs == args.eval_gen_kwargs + assert forwarded_args.fewshot_as_multiturn == args.fewshot_as_multiturn diff --git a/test/unit/test_cpu/eval/test_eval_cli.py b/test/unit/test_cpu/eval/test_eval_cli.py index f21a0fd37b..30fbaf995f 100644 --- a/test/unit/test_cpu/eval/test_eval_cli.py +++ b/test/unit/test_cpu/eval/test_eval_cli.py @@ -131,6 +131,21 @@ def test_vllm_args_are_accepted(self): args = parser.parse_args(["--vllm_args", "tensor_parallel_size=2,gpu_memory_utilization=0.9"]) assert args.vllm_args == "tensor_parallel_size=2,gpu_memory_utilization=0.9" + def test_fewshot_and_generation_args_are_accepted(self): + parser = eval_cli.EvalArgumentParser() + args = parser.parse_args( + [ + "--num-fewshot", + "3", + "--eval-gen-kwargs", + "temperature=0.1", + "--fewshot-as-multiturn", + ] + ) + assert args.num_fewshot == 3 + assert args.eval_gen_kwargs == "temperature=0.1" + assert args.fewshot_as_multiturn is True + class TestEvalInit: """Tests for `_eval_init` task normalization, device resolution, and dtype.""" @@ -584,7 +599,7 @@ def test_oom_retry_reduces_batch_size(self, monkeypatch): def fake_simple_evaluate(**kwargs): calls.append(kwargs.get("batch_size")) if len(calls) <= 2 and calls[-1] == 8: - raise RuntimeError("oom") + raise RuntimeError("CUDA out of memory") return fake_res monkeypatch.setattr("lm_eval.simple_evaluate", fake_simple_evaluate) @@ -601,13 +616,19 @@ def fake_simple_evaluate(**kwargs): assert calls == [8, 1] - def test_exhausted_retries_raises_runtime_error(self, monkeypatch): + def test_non_oom_error_is_raised_without_retry(self, monkeypatch): + calls = [] + + def fake_simple_evaluate(**kwargs): + calls.append(kwargs) + raise RuntimeError("permanent failure") + monkeypatch.setattr( "lm_eval.simple_evaluate", - lambda **kwargs: (_ for _ in ()).throw(RuntimeError("permanent failure")), + fake_simple_evaluate, ) - with pytest.raises(RuntimeError, match="Failed to evaluate task 'bad-task'"): + with pytest.raises(RuntimeError, match="permanent failure"): eval_cli._evaluate_tasks_with_retry( tasks=["bad-task"], hflm=object(), @@ -617,6 +638,8 @@ def test_exhausted_retries_raises_runtime_error(self, monkeypatch): retry_times=2, ) + assert len(calls) == 1 + def test_multiple_tasks_are_aggregated(self, monkeypatch): fake_hflm = object() res_a = {