feat(eval): add model cache controls

docs/commands/eval.md

@@ -141,6 +141,20 @@ Evaluate a composite model from pre-exported ONNX files. Some tasks (e.g., `imag
 $ winml eval -m encoder=encoder.onnx -m decoder=decoder.onnx --model-id microsoft/trocr-base-printed
 ```
 
+## Model build cache
+
+Evaluation reuses persistent model build artifacts by default. Pass
+`--no-use-cache` for a fresh build in a temporary directory, or `--rebuild` for
+a fresh build that replaces the persistent cache entry.
+
+For a pre-built ONNX input, cache controls apply only when
+`--no-skip-build` is set. Cache controls are ignored when no model build runs,
+including two-ONNX comparisons; the CLI warns when an explicit cache control
+has no effect. GenAI's runtime `_compiled/` artifacts are a separate cache and
+are not currently governed by these model build cache controls. Explicit build
+or cache controls on a GenAI bundle produce a warning that distinguishes the
+model-build pipeline from the runtime compilation cache.
+
 ## Common pitfalls
 
 - **ONNX file without `--model-id` fails.** When `-m` is a `.onnx` path, `--model-id` is mandatory. Without it the command cannot resolve the preprocessor or label vocabulary and will exit with a usage error.

src/winml/modelkit/commands/eval.py

@@ -9,6 +9,7 @@
 
 import json
 import logging
+from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -209,6 +210,7 @@
         "--model-id / --task are not required in this mode."
     ),
 )
+@cli_utils.cache_options()
 @cli_utils.skip_build_option()
 @cli_utils.format_option()
 @cli_utils.build_config_option()
@@ -253,6 +255,8 @@ def eval(
     input_data: str | None,
     reference: str | None,
     config_file: Path | None,
+    use_cache: bool,
+    rebuild: bool,
     skip_build: bool,
 ) -> None:
     r"""Evaluate a model for a task.
@@ -296,7 +300,7 @@ def eval(
     from ..eval import evaluate
 
     # ── 1. Build config: defaults ← config file ← CLI ──
-    cfg = _build_eval_config(ctx, config_file, column, label_mapping_path)
+    cfg, config_fields = _build_eval_config(ctx, config_file, column, label_mapping_path)
 
     if cfg.input_data is not None and cfg.mode != "compare":
         raise click.UsageError("--input-data is only valid with --mode compare.")
@@ -334,26 +338,12 @@ def eval(
             "comes from the leading axis of the provided tensors."
         )
 
-    # The build-pipeline flags only take effect when eval rebuilds the model.
-    # With a pre-built ONNX path and skip_build (the default), they are no-ops
-    # forwarded to from_onnx, so warn the user that they were ignored — mirrors
-    # the --precision warning above. Shared with perf via utils/cli.py.
-    build_flags_warning = cli_utils.ignored_build_flags_warning(
-        skip_build_onnx=cfg.model_path is not None and cfg.skip_build,
-        quant=cfg.quant,
-        optimize=cfg.optimize,
-        analyze=cfg.analyze,
-        max_optim_iterations=cfg.max_optim_iterations,
-    )
-    if build_flags_warning:
-        logger.warning(build_flags_warning)
-
-    logger.debug("Effective eval config: %s", cfg.to_dict())
-
     json_mode = output_format == "json"
 
     # ── 3. Evaluate ──
     try:
+        _warn_ignored_model_build_controls(ctx, cfg, config_fields)
+        logger.debug("Effective eval config: %s", cfg.to_dict())
         result = evaluate(cfg)
         _write_and_display(result, cfg.output_path, json_mode=json_mode)
     except Exception as e:
@@ -367,12 +357,15 @@ def _build_eval_config(
     config_file: Path | None,
     column: tuple[str, ...],
     label_mapping_path: Path | None,
-) -> WinMLEvaluationConfig:
+) -> tuple[WinMLEvaluationConfig, set[str]]:
     """Build a WinMLEvaluationConfig with precedence: defaults ← config file ← CLI.
 
     Reads raw JSON for config-file values so only explicitly-present keys
     are applied (avoids overriding with dataclass defaults).
     Uses ``collect_cli_overrides`` for automatic CLI-to-field mapping.
+
+    Returns the resolved config and the field names explicitly present in the
+    config file.
     """
     from ..eval import DatasetConfig, WinMLEvaluationConfig
     from ..utils.config_utils import merge_config
@@ -386,6 +379,7 @@ def _build_eval_config(
     eval_kwargs = cli_utils.collect_cli_overrides(ctx, WinMLEvaluationConfig)
     dataset_kwargs = cli_utils.collect_cli_overrides(ctx, DatasetConfig)
     cfg = WinMLEvaluationConfig(dataset=DatasetConfig(**dataset_kwargs), **eval_kwargs)
+    config_fields: set[str] = set()
 
     # ── Config file layer (only explicitly-present keys) ──
     if config_file is not None:
@@ -404,6 +398,7 @@ def _build_eval_config(
         # Eval section overrides loader/compile fallbacks
         eval_data = raw.get("eval")
         if eval_data:
+            config_fields.update(eval_data)
             cfg = merge_config(cfg, eval_data)
 
     # ── CLI layer (highest priority, auto-mapped via metadata) ──
@@ -432,7 +427,89 @@ def _build_eval_config(
     if overrides:
         cfg = merge_config(cfg, overrides)
 
-    return cfg
+    return cfg, config_fields
+
+
+@dataclass(frozen=True)
+class _ModelBuildBypass:
+    reason: str
+    build_explanation: str = "no build runs"
+    cache_explanation: str = "no build runs"
+
+
+def _model_build_bypass(cfg: WinMLEvaluationConfig) -> _ModelBuildBypass | None:
+    """Describe a selected loader that bypasses the model-build pipeline."""
+    from ..eval.evaluate import _ModelLoaderKind, _select_model_loader
+
+    loader = _select_model_loader(cfg)
+    if loader is _ModelLoaderKind.GENAI:
+        return _ModelBuildBypass(
+            reason="GenAI bundles",
+            build_explanation="no model build pipeline runs",
+            cache_explanation=(
+                "model build cache controls do not govern the GenAI runtime _compiled/ cache"
+            ),
+        )
+    if loader is _ModelLoaderKind.DIRECT_ONNX_COMPARE:
+        return _ModelBuildBypass("two-ONNX comparisons")
+    if loader is _ModelLoaderKind.EVALUATOR_MANAGED:
+        return _ModelBuildBypass("evaluator-managed composite inputs")
+    if loader is _ModelLoaderKind.ONNX and cfg.skip_build:
+        return _ModelBuildBypass("pre-built ONNX inputs")
+    return None
+
+
+def _warn_ignored_model_build_controls(
+    ctx: click.Context,
+    cfg: WinMLEvaluationConfig,
+    config_fields: set[str],
+) -> None:
+    """Warn when explicit model-build controls cannot affect the selected loader."""
+    _resolve_model_loader_task(cfg)
+    bypass = _model_build_bypass(cfg)
+    build_runs = bypass is None
+    reason = bypass.reason if bypass is not None else None
+
+    build_flags_warning = cli_utils.ignored_build_flags_warning(
+        build_runs=build_runs,
+        quant=cfg.quant,
+        optimize=cfg.optimize,
+        analyze=cfg.analyze,
+        max_optim_iterations=cfg.max_optim_iterations,
+        reason=reason,
+        rebuild_hint=("--no-skip-build" if reason == "pre-built ONNX inputs" else None),
+        explanation=bypass.build_explanation if bypass is not None else None,
+    )
+    if build_flags_warning:
+        logger.warning(build_flags_warning)
+
+    cache_flags_warning = cli_utils.ignored_cache_flags_warning(
+        build_runs=build_runs,
+        use_cache=cfg.use_cache,
+        rebuild=cfg.rebuild,
+        use_cache_was_set=cli_utils.is_cli_provided(ctx, "use_cache"),
+        rebuild_was_set=cli_utils.is_cli_provided(ctx, "rebuild"),
+        use_cache_source=("--config" if "use_cache" in config_fields else None),
+        rebuild_source=("--config" if "rebuild" in config_fields else None),
+        reason=reason,
+        explanation=bypass.cache_explanation if bypass is not None else None,
+    )
+    if cache_flags_warning:
+        logger.warning(cache_flags_warning)
+
+
+def _resolve_model_loader_task(cfg: WinMLEvaluationConfig) -> None:
+    """Resolve an omitted task when it can change the selected model loader."""
+    if cfg.task is not None or cfg.reference_path is not None:
+        return
+    if not isinstance(cfg.model_path, dict) and not (
+        isinstance(cfg.model_path, str) and Path(cfg.model_path).is_dir()
+    ):
+        return
+
+    from ..eval.evaluate import _infer_task
+
+    cfg.task = _infer_task(cfg)
 
 
 def _resolve_model(

src/winml/modelkit/commands/perf.py

@@ -3126,11 +3126,13 @@ def perf(
             # build is skipped (the default). Warn so the silent no-op is visible
             # — shared detection with eval via utils/cli.py.
             build_flags_warning = cli_utils.ignored_build_flags_warning(
-                skip_build_onnx=skip_build,
+                build_runs=not skip_build,
                 quant=quant,
                 optimize=optimize,
                 analyze=analyze,
                 max_optim_iterations=max_optim_iterations,
+                reason="pre-built ONNX inputs",
+                rebuild_hint="--no-skip-build",
             )
             if build_flags_warning:
                 console.print(f"[yellow]Warning:[/yellow] {build_flags_warning}")

src/winml/modelkit/eval/config.py

@@ -172,6 +172,8 @@ class WinMLEvaluationConfig:
     output_path: Path | None = field(default=None, metadata={"cli_name": "output"})
     mode: EvalMode = "onnx"
     skip_build: bool = True
+    use_cache: bool = True
+    rebuild: bool = False
     _auto_device_selected: bool = field(default=False, repr=False, compare=False, kw_only=True)
 
     def to_dict(self) -> dict:
@@ -214,6 +216,8 @@ def to_dict(self) -> dict:
         if self.mode != "onnx":
             result["mode"] = self.mode
         result["skip_build"] = self.skip_build
+        result["use_cache"] = self.use_cache
+        result["rebuild"] = self.rebuild
         return result
 
     @classmethod
@@ -253,4 +257,6 @@ def from_dict(cls, data: dict) -> WinMLEvaluationConfig:
             output_path=(Path(data["output_path"]) if data.get("output_path") else None),
             mode=data.get("mode", "onnx"),
             skip_build=data.get("skip_build", True),
+            use_cache=data.get("use_cache", True),
+            rebuild=data.get("rebuild", False),
         )

src/winml/modelkit/eval/evaluate.py

@@ -11,6 +11,7 @@
 import logging
 from copy import deepcopy
 from dataclasses import dataclass, field, replace
+from enum import Enum, auto
 from typing import TYPE_CHECKING, Any, cast
 
 from rich.console import Console
@@ -28,6 +29,28 @@
 
 logger = logging.getLogger(__name__)
 
+
+class _ModelLoaderKind(Enum):
+    GENAI = auto()
+    DIRECT_ONNX_COMPARE = auto()
+    EVALUATOR_MANAGED = auto()
+    ONNX = auto()
+    PRETRAINED = auto()
+
+
+def _select_model_loader(config: WinMLEvaluationConfig) -> _ModelLoaderKind:
+    """Select the model-loading path shared by loading and CLI diagnostics."""
+    if config.task == "text-generation":
+        return _ModelLoaderKind.GENAI
+    if config.mode == "compare" and config.reference_path is not None:
+        return _ModelLoaderKind.DIRECT_ONNX_COMPARE
+    if isinstance(config.model_path, dict) and config.task == "mask-generation":
+        return _ModelLoaderKind.EVALUATOR_MANAGED
+    if config.model_path is not None:
+        return _ModelLoaderKind.ONNX
+    return _ModelLoaderKind.PRETRAINED
+
+
 # Map task -> "module_path:ClassName"; modules are imported lazily by
 # get_evaluator_class() to improve command latency.
 # Keep the key/value-per-line layout: collapsing each entry onto one line (the
@@ -256,15 +279,15 @@ def _load_model(
     from ..session import EPDeviceTarget, WinMLEPRegistry, resolve_device
     from ..utils import cli as cli_utils
 
-    if config.task == "text-generation":
+    loader = _select_model_loader(config)
+    if loader is _ModelLoaderKind.GENAI:
         return _load_genai_causal_lm(config)
 
     # Two-ONNX compare: the evaluator builds both raw ORT sessions directly from
     # config.model_path / config.reference_path — no WinMLAutoModel / HF config.
-    if config.mode == "compare" and config.reference_path is not None:
+    if loader is _ModelLoaderKind.DIRECT_ONNX_COMPARE:
         return None
 
-
     quant_override: Any = None
     if not config.quant:
         from ..config import WinMLBuildConfig
@@ -277,11 +300,15 @@ def _load_model(
         analyze=config.analyze,
         max_optim_iterations=config.max_optim_iterations,
     )
+    cache_kwargs = cli_utils.cache_extra_kwargs(
+        use_cache=config.use_cache,
+        rebuild=config.rebuild,
+    )
 
     if config.model_id is None:
         raise ValueError("model_id is required.")
 
-    if isinstance(config.model_path, dict) and config.task == "mask-generation":
+    if loader is _ModelLoaderKind.EVALUATOR_MANAGED:
         # Evaluator-driven session loading; skip WinMLAutoModel entirely.
         return None
 
@@ -295,7 +322,7 @@ def _load_model(
     from onnxruntime.capi.onnxruntime_pybind11_state import RuntimeException
 
     try:
-        if config.model_path is not None:
+        if loader is _ModelLoaderKind.ONNX:
             # Pre-built ONNX: precision is already baked into the model and is
             # ignored here (mirrors winml perf's ONNX path).
             from transformers import AutoConfig
@@ -312,6 +339,7 @@ def _load_model(
                 skip_build=config.skip_build,
                 config=quant_override,
                 hf_config=hf_config,
+                **cache_kwargs,
                 **pipeline_kwargs,
             )
             model.config = hf_config
@@ -341,6 +369,7 @@ def _load_model(
             allow_unsupported_nodes=config.allow_unsupported_nodes,
             config=build_override,
             shape_config=config.shape_config,
+            **cache_kwargs,
             **pipeline_kwargs,
         )
     except RuntimeException as error:
@@ -386,8 +415,7 @@ def _load_genai_causal_lm(config: WinMLEvaluationConfig) -> WinMLGenaiCausalLM:
     bundle_path = config.model_path
     if not bundle_path or isinstance(bundle_path, dict):
         raise ValueError(
-            "text-generation evaluation requires a genai bundle *directory* via "
-            "-m <bundle_dir>."
+            "text-generation evaluation requires a genai bundle *directory* via -m <bundle_dir>."
         )
 
     bundle_dir = Path(bundle_path).expanduser()
@@ -420,19 +448,7 @@ def _resolve_task(config: WinMLEvaluationConfig) -> str:
     console = Console()
     console.print("[bold]Resolving task...[/bold]")
 
-    if config.task is not None:
-        task = config.task
-    else:
-        if config.model_id is None:
-            raise ValueError("Cannot infer task without model_id. Provide --task.")
-
-        from transformers import AutoConfig
-
-        from ..loader import load_hf_config
-        from ..loader.resolution import resolve_task
-
-        hf_config = load_hf_config(AutoConfig, config.model_id)
-        task = resolve_task(hf_config).task
+    task = config.task if config.task is not None else _infer_task(config)
 
     console.print(f"[dim]Use[/dim] {task} [dim]to evaluate[/dim]")
 
@@ -442,6 +458,20 @@ def _resolve_task(config: WinMLEvaluationConfig) -> str:
     return task
 
 
+def _infer_task(config: WinMLEvaluationConfig) -> str:
+    """Infer the evaluation task from the model's Hugging Face config."""
+    if config.model_id is None:
+        raise ValueError("Cannot infer task without model_id. Provide --task.")
+
+    from transformers import AutoConfig
+
+    from ..loader import load_hf_config
+    from ..loader.resolution import resolve_task
+
+    hf_config = load_hf_config(AutoConfig, config.model_id)
+    return resolve_task(hf_config).task
+
+
 def evaluate(config: WinMLEvaluationConfig) -> EvalResult:
     """Run model evaluation.