From 39e54eee6c8c05d1ee9d4bcc19bd965a9b2862fe Mon Sep 17 00:00:00 2001 From: Hualiang Xie Date: Tue, 4 Aug 2026 14:20:22 +0800 Subject: [PATCH] feat(perf): add native Hugging Face benchmark Benchmark original PyTorch checkpoints with --no-export and map GPU execution to CUDA while preserving the existing ONNX path. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/commands/perf.md | 16 +- docs/concepts/perf-and-monitoring.md | 28 + src/winml/modelkit/commands/perf.py | 1005 ++++++++++++++++- src/winml/modelkit/loader/hf.py | 32 +- src/winml/modelkit/session/monitor/_pdh.py | 36 +- .../modelkit/session/monitor/hw_monitor.py | 14 +- tests/e2e/test_perf_e2e.py | 244 ++++ tests/unit/commands/test_perf_pytorch.py | 738 ++++++++++++ tests/unit/loader/test_load_hf_model.py | 46 + tests/unit/session/test_ep_monitor.py | 97 ++ 10 files changed, 2236 insertions(+), 20 deletions(-) create mode 100644 tests/unit/commands/test_perf_pytorch.py diff --git a/docs/commands/perf.md b/docs/commands/perf.md index 492261bdf..067788856 100644 --- a/docs/commands/perf.md +++ b/docs/commands/perf.md @@ -21,7 +21,7 @@ $ winml perf [options] | `--task` | | `TEXT` | auto-detected | Explicit task override (e.g., `image-classification`). Inferred from the model if omitted. | | `--iterations` | | `INTEGER` | `100` | Number of timed inference iterations used to compute statistics. | | `--warmup` | | `INTEGER` | `10` | Number of warm-up iterations run before timing begins; excluded from statistics. | -| `--device` | `-d` | `auto\|cpu\|gpu\|npu` | `auto` | Device to run the benchmark on. `auto` selects the highest-priority available device. | +| `--device` | `-d` | `auto\|cpu\|gpu\|npu` | `auto` | Device to run the benchmark on. `auto` selects the highest-priority available device. With `--no-export`, only `auto`, `cpu`, and `gpu` apply; `gpu` maps to `torch.device("cuda")`, while `auto` selects CUDA when available and CPU otherwise. | | `--precision` | | `TEXT` | `auto` | Precision mode applied during model build: `auto`, `fp32`, `fp16`, `int8`, `int16`, or compound forms such as `w8a16`. | | `--ep` | | `TEXT` | — | Force a specific execution provider (e.g., `qnn`, `dml`, `vitisai`, `openvino`, `cpu`). Overrides the device-to-provider mapping. | | `--ep-options` | | `KEY=VALUE` (multiple) | — | Runtime EP provider option forwarded to the inference session (e.g., `--ep-options htp_performance_mode=burst`). Repeatable. Applies to both HuggingFace model IDs and ONNX file inputs. When detail op-tracing automatically compiles a raw ONNX model, these options are also applied to that compilation. | @@ -32,6 +32,7 @@ $ winml perf [options] | `--input-specs` | | `PATH` | — | JSON input tensor specs to merge into the Hugging Face export config before benchmarking. Symbolic string dimensions infer dynamic axes. Ignored for pre-exported ONNX files and in `--module` mode. | | `--export-config` | | `PATH` | — | JSON ONNX export config overrides to apply when `perf` builds a Hugging Face model before benchmarking. Ignored for pre-exported ONNX files and in `--module` mode. | | `--dynamic-axes` | | `PATH` | — | JSON dynamic axes mapping for Hugging Face ONNX export, for example `{"input_ids": {"0": "batch", "1": "sequence"}}`. Ignored for pre-exported ONNX files and in `--module` mode. | +| `--export/--no-export` | | flag | `true` | Export a Hugging Face model to ONNX before benchmarking. `--no-export` instead times the original PyTorch model's raw forward pass. | | `--quantize/--no-quantize` | | flag | `true` | Run quantization during model build (use `--no-quantize` to skip it). Useful for measuring the fp32 baseline. | | `--rebuild/--no-rebuild` | | flag | `false` | Force model rebuild even if a cached artifact already exists. | | `--ignore-cache/--no-ignore-cache` | | flag | `false` | Build from scratch in a temporary folder and discard the artifact after benchmarking. Implies `--rebuild`. | @@ -46,7 +47,11 @@ $ winml perf [options] ## How it works -`winml perf` loads the model through `WinMLAutoModel` — accepting both HuggingFace IDs and local ONNX files — then generates random input tensors from the model's I/O configuration. It runs the specified number of warm-up iterations (excluded from statistics) followed by the timed iterations, collecting per-sample latency. The final report includes mean, min, max, P50, P90, P95, P99, standard deviation, and throughput in samples per second. When `--monitor` is active, a hardware polling loop runs in parallel and records NPU / GPU utilization, CPU usage, and device memory alongside the timing data. +By default, `winml perf` loads the model through `WinMLAutoModel` — accepting both HuggingFace IDs and local ONNX files — then generates random input tensors from the model's I/O configuration. It runs the specified number of warm-up iterations (excluded from statistics) followed by the timed iterations, collecting per-sample latency. The final report includes mean, min, max, P50, P90, P95, P99, standard deviation, and throughput in samples per second. When `--monitor` is active, a hardware polling loop runs in parallel and records NPU / GPU utilization, CPU usage, and device memory alongside the timing data. + +With `--no-export`, `perf` loads the task-resolved pretrained Hugging Face module in eval mode and times `model(**inputs)` under `torch.inference_mode()`. Preprocessing and postprocessing are outside the timing boundary. CUDA runs synchronize immediately before and after each timed forward pass so reported latency includes asynchronous GPU work. The checkpoint's parameter dtype is preserved, and JSON reports identify this path with `"backend": "pytorch"` and `"ep": null`. + +The native PyTorch path is limited to complete Hugging Face models. It does not accept ONNX files, `--runtime winml-genai`, `--module`, `--submodel`, NPU devices, execution providers, op tracing, or ONNX build/export controls such as `--precision`, quantization, optimization, compilation, cache controls, and export overrides. Iteration, warm-up, duration, batch, shape, `.npz` input, monitoring, memory, and output options remain available. ## Examples @@ -78,6 +83,12 @@ Benchmark a pre-exported ONNX file on CPU with more iterations: $ winml perf -m model.onnx --device cpu --iterations 500 ``` +Benchmark the original Hugging Face PyTorch model on CUDA without exporting: + +```bash +$ winml perf -m microsoft/resnet-50 --no-export --device gpu +``` + Benchmark a text model with an explicit task, targeting the NPU: ```bash @@ -136,6 +147,7 @@ and logs a warning. ## Common pitfalls - **Warm-up too low on NPU.** The first several inferences on an NPU EP can be significantly slower due to kernel compilation and caching. The default of 10 warm-up iterations is usually enough for vision models, but transformer models with many operators may need `--warmup 30` or higher to reach steady-state latency. +- **CUDA unavailable with `--no-export --device gpu`.** This mode requires a CUDA-enabled PyTorch installation and an available CUDA device. Use `--device auto` to fall back to CPU automatically. - **Hidden third-party diagnostics.** Normal `winml perf` output suppresses noisy native warning-level diagnostics and Hugging Face download/progress chatter so benchmark results stay readable. Use `-v`/`-vv` or set `WINMLCLI_SHOW_ALL_WARNINGS=1` to show those warnings when debugging provider or Hub issues. - **`--input-data` keys must match; dtypes are cast.** The `.npz` keys must equal the model's input names — a missing or unexpected key is a hard error (typo protection). Array dtypes are cast to the model's expected dtype with a warning (matching normal inference), so you don't have to hand-match widths. `.npy` files are not supported — save named arrays as `.npz`. When `--input-data` is set, `--batch-size` and `--shape-config` are ignored (the tensors define their own shapes). It is also rejected for `--module` mode, `--runtime winml-genai`, and composite (dual-encoder) models such as CLIP/SigLIP, where each sub-model has its own inputs that a single `.npz` cannot address. - **Real data only binds if the export kept axes dynamic.** When `-m` is a HuggingFace model ID, `perf` exports it with default shapes (because `--shape-config`/`--batch-size` are ignored under `--input-data`). If that export baked in static shapes, ORT will reject differently-shaped `--input-data`. Use `--dynamic-axes`/symbolic `--input-specs` for the Hugging Face build, or point `-m` at an ONNX file that already has dynamic axes. diff --git a/docs/concepts/perf-and-monitoring.md b/docs/concepts/perf-and-monitoring.md index 3a30ae92a..a5d5b4c62 100644 --- a/docs/concepts/perf-and-monitoring.md +++ b/docs/concepts/perf-and-monitoring.md @@ -43,6 +43,33 @@ Key parameters: | `--precision` | Precision mode: `auto`, `fp32`, `fp16`, `int8`, `int16`, or `w{x}a{y}` | `auto` | | `--quantize/--no-quantize` | Include quantization during model build | `--quantize` | | `--skip-build/--no-skip-build` | Skip the build pipeline for ONNX inputs | `--skip-build` | +| `--export/--no-export` | Export a Hugging Face model to ONNX or benchmark its original PyTorch forward pass | `--export` | + +### Native PyTorch baseline + +Use `--no-export` to measure a Hugging Face checkpoint before ONNX conversion: + +``` +winml perf -m microsoft/resnet-50 --no-export --device gpu +``` + +This path loads the task-resolved pretrained module, preserves its parameter +dtype, and times only `model(**inputs)` under `torch.inference_mode()`. +Tokenization, image preprocessing, and output postprocessing are excluded, so +the result remains comparable to the normal session-level ONNX timing boundary. +CUDA is synchronized before and after every timed forward pass. +On Windows, hardware monitoring binds PDH counters to the CUDA device's adapter +LUID. If that identity cannot be resolved, `perf` warns and records CPU/RAM only +rather than attributing another GPU's activity to the benchmark. + +For `--no-export`, `--device gpu` means `torch.device("cuda")`; +`--device cpu` uses PyTorch CPU; and `--device auto` chooses CUDA when available, +otherwise CPU. NPU and execution-provider options do not apply. The initial +native path supports complete Hugging Face models only, not ONNX files, GenAI +bundles, `--module`, or `--submodel`. ONNX build, export, precision, cache, +compile, and op-tracing flags are rejected instead of being silently ignored. +Batch size, shape overrides, `.npz` inputs, duration, monitoring, memory, and +report options continue to work. ### Output format @@ -52,6 +79,7 @@ Add `-f json` to emit structured JSON to stdout, suitable for CI pipelines or au { "benchmark_info": { "model_id": "bert-tiny.onnx", + "backend": "winml", "task": "auto-detected", "device": "cpu", "ep": "CPUExecutionProvider", diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index b23a13842..977a75f60 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -15,6 +15,7 @@ from __future__ import annotations +import inspect import json import logging import math @@ -302,6 +303,7 @@ class BenchmarkConfig: """Configuration for benchmark execution.""" model_id: str + backend: Literal["winml", "pytorch"] = "winml" task: str | None = None submodel: str | None = None device: str = "auto" @@ -402,13 +404,14 @@ def to_dict(self) -> dict[str, Any]: result: dict[str, Any] = { "benchmark_info": { "model_id": self.config.model_id, + "backend": self.config.backend, "running_model_path": self.running_model_path, "task": self.actual_task, "device": self.actual_device, "ep": self.actual_ep, "ep_source": self.config.ep_source, "ep_options": self.config.ep_options, - "precision": self.config.precision, + "precision": (None if self.config.backend == "pytorch" else self.config.precision), # In duration mode the run isn't bounded by a fixed count, so # report the actual number of timed (post-warmup) samples rather # than the unused ``--iterations`` default. @@ -629,7 +632,7 @@ def load_input_data( def effective_batch_size( - inputs: dict[str, np.ndarray], + inputs: dict[str, Any], input_names: list[str], requested: int, ) -> int: @@ -1316,6 +1319,858 @@ def _collect_results(self, stats: PerfStats) -> BenchmarkResult: ) +class _PyTorchForwardRunner: + """Session-like adapter that records synchronized PyTorch forward latency.""" + + def __init__( + self, + model: Any, + stats: PerfStats, + synchronize: Any | None, + ) -> None: + self._model = model + self._stats = stats + self._synchronize = synchronize + self.output_metadata: list[tuple[str, list[int], str]] = [] + + def run(self, inputs: dict[str, Any]) -> None: + """Run one forward pass and capture output metadata outside the timed region.""" + if self._synchronize is not None: + self._synchronize() + + def forward() -> Any: + output = self._model(**inputs) + if self._synchronize is not None: + self._synchronize() + return output + + output = self._stats.record(forward) + if not self.output_metadata: + from ..inspect.module_io_capture import _extract_tensors + + self.output_metadata = [ + (name, list(tensor.shape), str(tensor.dtype).replace("torch.", "")) + for name, tensor in _extract_tensors(output) + ] + + +class PyTorchPerfBenchmark: + """Benchmark an original Hugging Face PyTorch model without ONNX export.""" + + def __init__(self, config: BenchmarkConfig) -> None: + self.config = config + self._model: Any | None = None + self._inputs: dict[str, Any] | None = None + self._io_config: dict[str, Any] | None = None + self._torch_device: Any | None = None + self._actual_device = "" + self._actual_task = "" + self._model_precision: str | None = None + self._effective_batch = config.batch_size + self._memory: dict[str, float] | None = None + self._hw_metrics: dict[str, Any] | None = None + self._input_specs: list[Any] = [] + self._output_names: list[str] = [] + self._native_dummy_inputs: dict[str, Any] = {} + self._main_input_name: str | None = None + + def close(self) -> None: + """Release model and input references, including cached CUDA allocations.""" + self._inputs = None + self._model = None + if self._torch_device is None or self._torch_device.type != "cuda": + return + + import gc + + import torch + + gc.collect() + torch.cuda.empty_cache() + + def run(self) -> BenchmarkResult: + """Load and benchmark the task-resolved pretrained PyTorch module.""" + import gc + + import torch + + self._resolve_device(torch) + gc.collect() + baseline = self._memory_snapshot(torch) if self.config.memory else None + + self._load_model(torch) + gc.collect() + after_model_load = self._memory_snapshot(torch) if self.config.memory else None + if self._torch_device.type == "cuda": + torch.cuda.reset_peak_memory_stats(self._torch_device) + + self._prepare_inputs(torch) + assert self._inputs is not None + assert self._io_config is not None + + _print_model_info( + self._io_config, + backend="pytorch", + task=self._actual_task, + req_device=self.config.device, + act_device=self._actual_device, + actual_shapes={name: tuple(value.shape) for name, value in self._inputs.items()}, + ) + + stats, output_metadata = self._run_benchmark(torch) + if self.config.memory: + gc.collect() + after_inference = self._memory_snapshot(torch, use_cuda_peak=True) + assert baseline is not None + assert after_model_load is not None + self._memory = self._build_memory_profile( + baseline, + after_model_load, + after_inference, + ) + + return self._collect_results(stats, output_metadata) + + def _resolve_device(self, torch: Any) -> None: + requested = self.config.device.lower() + if requested == "auto": + requested = "gpu" if torch.cuda.is_available() else "cpu" + if requested == "gpu": + if not torch.cuda.is_available(): + raise click.UsageError( + "--device gpu with --no-export requires a CUDA-enabled PyTorch " + "installation and an available CUDA device." + ) + self._torch_device = torch.device("cuda") + self._actual_device = "gpu" + return + if requested == "cpu": + self._torch_device = torch.device("cpu") + self._actual_device = "cpu" + return + raise click.UsageError( + f"--device {self.config.device} is not supported with --no-export; " + "use auto, cpu, or gpu." + ) + + def _load_model(self, torch: Any) -> None: + from ..config import generate_hf_build_config + from ..loader import composite_pipeline_tasks, load_hf_model + + model, hf_config, resolved_task = load_hf_model( + self.config.model_id, + task=self.config.task, + use_checkpoint_class=True, + torch_dtype="auto", + ) + assert self._torch_device is not None + self._model = model.to(self._torch_device).eval() + self._actual_task = resolved_task + self._model_precision = self._resolve_model_precision(torch) + main_input_name = getattr(model, "main_input_name", None) + self._main_input_name = main_input_name if isinstance(main_input_name, str) else None + self._native_dummy_inputs = self._get_native_dummy_inputs(torch) + + input_override = { + "export": { + "batch_size": self.config.batch_size, + } + } + model_type = getattr(hf_config, "model_type", "") + supplemental_tasks = ( + composite_pipeline_tasks(model_type.lower().replace("_", "-")) if model_type else [] + ) + tasks = list(dict.fromkeys([resolved_task, *supplemental_tasks])) + resolution_errors: list[Exception] = [] + for task in tasks: + try: + build_config = generate_hf_build_config( + model_id=self.config.model_id, + task=task, + override=input_override, + shape_config=self.config.shape_config, + device="cpu", + precision="auto", + no_compile=True, + ) + except (AttributeError, KeyError, ValueError) as exc: + resolution_errors.append(exc) + logger.warning( + "Could not resolve export-derived input specs for task '%s': %s", + task, + exc, + ) + continue + + configs = build_config if isinstance(build_config, list) else [build_config] + for config in configs: + export_config = config.export + if export_config is None: + continue + for spec in export_config.input_tensors or []: + self._append_input_spec(spec) + if not self._output_names: + self._output_names = export_config.get_output_names() + + self._merge_nested_config_inputs(hf_config) + self._prioritize_checkpoint_main_input(torch, hf_config) + if not self._input_specs and not self._native_dummy_inputs: + if resolution_errors: + raise ValueError( + "Could not resolve inputs for the Hugging Face model." + ) from resolution_errors[0] + raise ValueError( + "Could not resolve input tensor specifications for the Hugging Face model." + ) + if supplemental_tasks: + logger.warning( + "Merged compatible input specs from composite tasks for the full " + "PyTorch checkpoint: %s.", + ", ".join(supplemental_tasks), + ) + + def _append_input_spec(self, spec: Any) -> None: + if spec.name and all(existing.name != spec.name for existing in self._input_specs): + self._input_specs.append(spec) + + def _prioritize_checkpoint_main_input(self, torch: Any, hf_config: Any) -> None: + from ..export import InputTensorSpec + + name = self._main_input_name + if not name: + return + existing = next((spec for spec in self._input_specs if spec.name == name), None) + shape = self._infer_checkpoint_main_shape(torch, hf_config) + if shape is not None: + spec = InputTensorSpec( + name=name, + shape=shape, + dtype=existing.dtype if existing is not None else "float32", + value_range=existing.value_range if existing is not None else None, + ) + elif existing is not None: + spec = existing + else: + return + self._input_specs = [ + spec, + *(candidate for candidate in self._input_specs if candidate.name != name), + ] + + def _infer_checkpoint_main_shape(self, torch: Any, hf_config: Any) -> tuple[int, ...] | None: + shape_config = self.config.shape_config or {} + image_size = getattr(hf_config, "image_size", None) + channels = getattr(hf_config, "num_channels", None) + if image_size is not None and channels is not None: + if isinstance(image_size, int): + height = width = image_size + elif isinstance(image_size, (list, tuple)) and len(image_size) == 2: + height, width = image_size + else: + return None + height = shape_config.get("height", height) + width = shape_config.get("width", width) + channels = shape_config.get("num_channels", channels) + dimensions = [self.config.batch_size] + frames = getattr(hf_config, "num_frames", None) + if frames is not None: + dimensions.append(frames) + dimensions.extend([channels, height, width]) + return self._validated_shape(dimensions) + + feature_size = getattr(hf_config, "num_mel_bins", None) + max_positions = getattr(hf_config, "max_source_positions", None) + if feature_size is not None and max_positions is not None: + feature_size = shape_config.get("feature_size", feature_size) + frames = shape_config.get("nb_max_frames") + if frames is None: + frames = max_positions * self._conv1d_downsample_factor(torch, feature_size) + return self._validated_shape([self.config.batch_size, feature_size, frames]) + return None + + @staticmethod + def _validated_shape(dimensions: list[Any]) -> tuple[int, ...]: + if any(isinstance(dim, bool) or not isinstance(dim, int) or dim <= 0 for dim in dimensions): + raise ValueError("Resolved checkpoint input dimensions must be positive integers.") + return tuple(dimensions) + + def _conv1d_downsample_factor(self, torch: Any, input_channels: int) -> int: + assert self._model is not None + channels = input_channels + factor = 1 + for module in self._model.modules(): + if not isinstance(module, torch.nn.Conv1d) or module.in_channels != channels: + continue + factor *= module.stride[0] + channels = module.out_channels + return factor + + def _merge_nested_config_inputs(self, hf_config: Any) -> None: + from transformers import PretrainedConfig + + from ..export import InputTensorSpec, resolve_io_specs + + pending = list(vars(hf_config).values()) + seen: set[int] = {id(hf_config)} + while pending: + value = pending.pop() + if isinstance(value, PretrainedConfig): + if id(value) in seen: + continue + seen.add(id(value)) + pending.extend(vars(value).values()) + model_type = getattr(value, "model_type", None) + if not model_type: + continue + try: + io_specs = resolve_io_specs( + model_type, + "feature-extraction", + value, + model_id=self.config.model_id, + batch_size=self.config.batch_size, + **(self.config.shape_config or {}), + ) + except (AttributeError, KeyError, ValueError) as exc: + logger.debug( + "Could not resolve nested input specs for model type '%s': %s", + model_type, + exc, + ) + continue + names = io_specs["input_names"] + shapes = io_specs["input_shapes"] + dtypes = io_specs["input_dtypes"] + value_ranges = io_specs.get("value_ranges", {}) + for index, name in enumerate(names): + self._append_input_spec( + InputTensorSpec( + name=name, + shape=tuple(shapes[index]), + dtype=dtypes[index], + value_range=value_ranges.get(name), + ) + ) + continue + if isinstance(value, dict): + pending.extend(value.values()) + elif isinstance(value, (list, tuple)): + pending.extend(value) + + def _resolve_model_precision(self, torch: Any) -> str | None: + assert self._model is not None + for parameter in self._model.parameters(): + if torch.is_floating_point(parameter): + return str(parameter.dtype).replace("torch.", "") + return None + + def _get_native_dummy_inputs(self, torch: Any) -> dict[str, Any]: + assert self._model is not None + parameters = inspect.signature(self._model.forward).parameters + dummy_inputs = getattr(self._model, "dummy_inputs", None) + if not isinstance(dummy_inputs, dict): + return {} + return { + name: self._prepare_native_dummy_input(value) + for name, value in dummy_inputs.items() + if name in parameters and isinstance(value, torch.Tensor) + } + + def _prepare_inputs(self, torch: Any) -> None: + specs = [] + for spec in self._input_specs: + if not spec.name or spec.shape is None: + continue + if self.config.input_data is None and spec.shape: + spec = replace( + spec, + shape=(self.config.batch_size, *spec.shape[1:]), + ) + specs.append(spec) + + generated = {spec.name: spec.to_tensor() for spec in specs if spec.name} + assert self._model is not None + parameters = inspect.signature(self._model.forward).parameters + keyword_parameters = { + name + for name, parameter in parameters.items() + if parameter.kind + in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY) + } + generated = {name: value for name, value in generated.items() if name in keyword_parameters} + + for name, value in self._native_dummy_inputs.items(): + if name in keyword_parameters and name not in generated: + generated[name] = value + + ordered_names = list( + dict.fromkeys( + [ + self._main_input_name, + *self._native_dummy_inputs, + *generated, + ] + ) + ) + generated = { + name: generated[name] + for name in ordered_names + if name is not None and name in generated + } + + missing = [ + name + for name, parameter in parameters.items() + if name not in generated + and parameter.default is inspect.Parameter.empty + and parameter.kind + in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY) + ] + if missing: + raise ValueError( + f"Could not generate required PyTorch forward inputs: {', '.join(missing)}." + ) + if not generated: + raise ValueError("No PyTorch forward inputs matched the resolved model.") + + io_config = { + "input_names": list(generated), + "input_shapes": [list(value.shape) for value in generated.values()], + "input_types": [ + self._numpy_compatible_dtype(str(value.dtype).replace("torch.", "")) + for value in generated.values() + ], + "output_names": self._output_names, + "output_shapes": [[] for _ in self._output_names], + "precision": self._model_precision, + } + + if self.config.input_data is not None: + numpy_inputs = load_input_data(self.config.input_data, io_config) + generated = {name: torch.from_numpy(value) for name, value in numpy_inputs.items()} + + assert self._torch_device is not None + model_dtype = None + if self._model_precision is not None: + model_dtype = getattr(torch, self._model_precision, None) + self._inputs = {} + for name, value in generated.items(): + if model_dtype is not None and torch.is_floating_point(value): + value = value.to(device=self._torch_device, dtype=model_dtype) + else: + value = value.to(device=self._torch_device) + self._inputs[name] = value + if self.config.input_data is None: + self._inputs = self._select_checkpoint_forward_inputs(torch, self._inputs) + + io_config["input_shapes"] = [list(value.shape) for value in self._inputs.values()] + io_config["input_names"] = list(self._inputs) + io_config["input_types"] = [ + str(value.dtype).replace("torch.", "") for value in self._inputs.values() + ] + self._io_config = io_config + self._effective_batch = effective_batch_size( + self._inputs, + io_config["input_names"], + self.config.batch_size, + ) + + def _select_checkpoint_forward_inputs( + self, + torch: Any, + inputs: dict[str, Any], + ) -> dict[str, Any]: + assert self._model is not None + main_name = self._main_input_name + if main_name is None: + return inputs + if main_name not in inputs: + raise ValueError( + f"Could not resolve the checkpoint's main PyTorch input '{main_name}'." + ) + + forward_errors = ( + AssertionError, + AttributeError, + IndexError, + KeyError, + RuntimeError, + TypeError, + ValueError, + ) + last_error: Exception | None = None + with torch.inference_mode(): + try: + output = self._model(**inputs) + except torch.cuda.OutOfMemoryError: + raise + except forward_errors as exc: + last_error = exc + else: + if self._output_depends_on_main_input(torch, inputs, output): + return inputs + + selected: dict[str, Any] = {} + found_valid = False + for name, value in inputs.items(): + selected[name] = value + try: + output = self._model(**selected) + except torch.cuda.OutOfMemoryError: + raise + except forward_errors as exc: + last_error = exc + if found_valid: + selected.pop(name) + continue + if self._output_depends_on_main_input(torch, selected, output): + found_valid = True + else: + selected.pop(name) + if found_valid: + return selected + raise ValueError( + "Could not assemble compatible inputs for the checkpoint's full PyTorch forward." + ) from last_error + + def _output_depends_on_main_input( + self, + torch: Any, + inputs: dict[str, Any], + output: Any, + ) -> bool: + assert self._model is not None + assert self._main_input_name is not None + without_main = { + name: value for name, value in inputs.items() if name != self._main_input_name + } + try: + comparison = self._model(**without_main) + except torch.cuda.OutOfMemoryError: + raise + except ( + AssertionError, + AttributeError, + IndexError, + KeyError, + RuntimeError, + TypeError, + ValueError, + ): + return True + + from ..inspect.module_io_capture import _extract_tensors + + tensors = list(_extract_tensors(output)) + comparison_tensors = list(_extract_tensors(comparison)) + metadata = [(name, tensor.shape, tensor.dtype) for name, tensor in tensors] + comparison_metadata = [ + (name, tensor.shape, tensor.dtype) for name, tensor in comparison_tensors + ] + if metadata != comparison_metadata or not tensors: + return True + return any( + not torch.equal(tensor, comparison_tensor) + for (_, tensor), (_, comparison_tensor) in zip( + tensors, + comparison_tensors, + strict=True, + ) + ) + + def _prepare_native_dummy_input(self, value: Any) -> Any: + """Apply supported semantic shape overrides to a model-provided tensor.""" + if value.ndim == 0: + return value + targets = {0: self.config.batch_size} + shape_config = self.config.shape_config or {} + if value.ndim == 2: + key = ( + "sequence_length" if "sequence_length" in shape_config else "audio_sequence_length" + ) + if key in shape_config: + targets[1] = self._shape_override_size(shape_config, key) + elif value.ndim == 3: + if "feature_size" in shape_config: + targets[1] = self._shape_override_size(shape_config, "feature_size") + if "nb_max_frames" in shape_config: + targets[2] = self._shape_override_size(shape_config, "nb_max_frames") + elif value.ndim >= 4: + if "num_channels" in shape_config: + targets[1] = self._shape_override_size(shape_config, "num_channels") + if "height" in shape_config: + targets[value.ndim - 2] = self._shape_override_size(shape_config, "height") + if "width" in shape_config: + targets[value.ndim - 1] = self._shape_override_size(shape_config, "width") + + for axis, size in targets.items(): + value = self._resize_tensor_dimension(value, axis, size) + return value + + @staticmethod + def _shape_override_size(shape_config: dict[str, Any], key: str) -> int: + value = shape_config[key] + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"Shape override '{key}' must be a positive integer.") + return value + + @staticmethod + def _resize_tensor_dimension(value: Any, axis: int, size: int) -> Any: + current = value.shape[axis] + if current == size: + return value + if current == 0: + raise ValueError("Cannot resize a dummy input with an empty dimension.") + repeats = [1] * value.ndim + repeats[axis] = (size + current - 1) // current + return value.repeat(*repeats).narrow(axis, 0, size).clone() + + @staticmethod + def _numpy_compatible_dtype(dtype: str) -> str: + return "float32" if dtype == "bfloat16" else dtype + + def _run_benchmark(self, torch: Any) -> tuple[PerfStats, list[tuple[str, list[int], str]]]: + from ..session.stats import PerfStats + + assert self._model is not None + assert self._inputs is not None + stats = PerfStats(warmup=self.config.warmup) + synchronize = torch.cuda.synchronize if self._torch_device.type == "cuda" else None + runner = _PyTorchForwardRunner(self._model, stats, synchronize) + total_iterations = self.config.warmup + self.config.iterations + + with torch.inference_mode(): + if self.config.monitor: + self._run_monitored(runner, stats, total_iterations) + else: + _run_simple_loop( + runner, + self._inputs, + total_iterations, + warmup=self.config.warmup, + duration_sec=self.config.duration, + ) + return stats, runner.output_metadata + + def _run_monitored( + self, + runner: _PyTorchForwardRunner, + stats: PerfStats, + total_iterations: int, + ) -> None: + from ..session.monitor.hw_monitor import HWMonitor + + assert self._inputs is not None + if not HWMonitor.is_available(): + SafeConsole(stderr=True).print( + "[yellow]Warning:[/yellow] HWMonitor unavailable on this system. " + "Running without hardware monitoring." + ) + _run_simple_loop( + runner, + self._inputs, + total_iterations, + warmup=self.config.warmup, + duration_sec=self.config.duration, + ) + return + + adapter_luid = self._cuda_adapter_luid() + monitor_device = self._monitor_device(adapter_luid) + hw_monitor = HWMonitor( + poll_interval_ms=_HW_POLL_INTERVAL_MS, + device=monitor_device, + adapter_luid=adapter_luid, + include_gpu_aggregate=not (self._actual_device == "gpu" and adapter_luid is None), + ) + with hw_monitor as hw: + _run_monitored_loop( + runner, + self._inputs, + stats, + hw, + total_iterations=total_iterations, + warmup=self.config.warmup, + model_id=self.config.model_id, + device=self._actual_device, + duration_sec=self.config.duration, + ) + self._hw_metrics = hw_monitor.to_dict() + + def _monitor_device(self, adapter_luid: str | None) -> str: + if self._actual_device != "gpu": + return self._actual_device + if adapter_luid is not None: + return "gpu" + SafeConsole(stderr=True).print( + "[yellow]Warning:[/yellow] Could not identify the CUDA adapter for exact " + "GPU monitoring. Collecting CPU/RAM metrics only." + ) + return "cpu" + + def _cuda_adapter_luid(self) -> str | None: + if self._actual_device != "gpu" or sys.platform != "win32": + return None + import torch + + cached = getattr(self, "_resolved_cuda_adapter_luid", "") + if cached is False: + return None + if isinstance(cached, str) and cached: + return cached + + import ctypes + + torch_dir = Path(torch.__file__).resolve().parent + candidates = [ + *sorted((torch_dir / "lib").glob("cudart64*.dll")), + *sorted((torch_dir.parent / "nvidia" / "cuda_runtime" / "bin").glob("cudart64*.dll")), + ] + library = None + for path in candidates: + try: + library = ctypes.WinDLL(str(path)) + break + except OSError: + continue + if library is None: + cuda_version = getattr(torch.version, "cuda", None) + if cuda_version: + major = cuda_version.split(".", maxsplit=1)[0] + names = [f"cudart64_{major}.dll", f"cudart64_{major}0.dll"] + for name in names: + try: + library = ctypes.WinDLL(name) + break + except OSError: + continue + if library is None or not hasattr(library, "cudaDeviceGetLuid"): + self._resolved_cuda_adapter_luid = False + return None + + get_luid = library.cudaDeviceGetLuid + get_luid.argtypes = [ + ctypes.POINTER(ctypes.c_char), + ctypes.POINTER(ctypes.c_uint), + ctypes.c_int, + ] + get_luid.restype = ctypes.c_int + raw_luid = (ctypes.c_char * 8)() + node_mask = ctypes.c_uint() + assert self._torch_device is not None + device_index = self._torch_device.index + if device_index is None: + device_index = torch.cuda.current_device() + status = get_luid(raw_luid, ctypes.byref(node_mask), device_index) + if status != 0: + self._resolved_cuda_adapter_luid = False + return None + luid = self._format_cuda_luid(bytes(raw_luid)) + self._resolved_cuda_adapter_luid = luid + return luid + + @staticmethod + def _format_cuda_luid(raw_luid: bytes) -> str: + if len(raw_luid) != 8: + raise ValueError("A CUDA adapter LUID must contain exactly 8 bytes.") + low = int.from_bytes(raw_luid[:4], byteorder="little", signed=False) + high = int.from_bytes(raw_luid[4:], byteorder="little", signed=False) + return f"0x{high:08X}_0x{low:08X}" + + def _memory_snapshot( + self, + torch: Any, + *, + use_cuda_peak: bool = False, + ) -> tuple[float, float, float]: + from ..session.monitor.memory_tracker import get_rss_mb + + local_mb = 0.0 + if self._torch_device.type == "cuda": + cuda_bytes = ( + torch.cuda.max_memory_allocated(self._torch_device) + if use_cuda_peak + else torch.cuda.memory_allocated(self._torch_device) + ) + local_mb = cuda_bytes / (1024 * 1024) + return get_rss_mb(), local_mb, 0.0 + + @staticmethod + def _build_memory_profile( + baseline: tuple[float, float, float], + after_model_load: tuple[float, float, float], + after_inference: tuple[float, float, float], + ) -> dict[str, float]: + rss_baseline, local_baseline, shared_baseline = baseline + rss_model, local_model, shared_model = after_model_load + rss_infer, local_infer, shared_infer = after_inference + return { + "rss_baseline_mb": round(rss_baseline, 2), + "rss_after_model_load_mb": round(rss_model, 2), + "rss_after_compile_mb": round(rss_model, 2), + "rss_after_inference_mb": round(rss_infer, 2), + "rss_model_load_delta_mb": round(rss_model - rss_baseline, 2), + "rss_inference_delta_mb": round(rss_infer - rss_model, 2), + "rss_total_delta_mb": round(rss_infer - rss_baseline, 2), + "vram_local_after_model_load_mb": round(local_model, 2), + "vram_local_after_inference_mb": round(local_infer, 2), + "vram_shared_after_inference_mb": round(shared_infer, 2), + "vram_local_model_load_delta_mb": round(local_model - local_baseline, 2), + "vram_local_inference_delta_mb": round(local_infer - local_model, 2), + "vram_local_total_delta_mb": round(local_infer - local_baseline, 2), + "vram_shared_model_load_delta_mb": round(shared_model - shared_baseline, 2), + "vram_shared_inference_delta_mb": round(shared_infer - shared_model, 2), + "vram_shared_total_delta_mb": round(shared_infer - shared_baseline, 2), + } + + def _collect_results( + self, + stats: PerfStats, + output_metadata: list[tuple[str, list[int], str]], + ) -> BenchmarkResult: + assert self._io_config is not None + mean_latency_sec = stats.mean_ms / 1000.0 + samples_per_sec = self._effective_batch / mean_latency_sec if mean_latency_sec > 0 else 0 + batches_per_sec = 1.0 / mean_latency_sec if mean_latency_sec > 0 else 0 + samples = stats.samples_ms + warmup_samples = stats.all_samples_ms[: self.config.warmup] + + output_names = [name for name, _, _ in output_metadata] + output_shapes = [shape for _, shape, _ in output_metadata] + if not output_names: + output_names = self._io_config["output_names"] + output_shapes = self._io_config["output_shapes"] + + return BenchmarkResult( + config=self.config, + input_names=self._io_config["input_names"], + input_shapes=self._io_config["input_shapes"], + input_types=self._io_config["input_types"], + output_names=output_names, + output_shapes=output_shapes, + model_precision=self._model_precision, + mean_ms=stats.mean_ms, + min_ms=stats.min_ms, + max_ms=stats.max_ms, + p50_ms=stats.p50_ms, + p90_ms=stats.p90_ms, + p95_ms=stats.p95_ms, + p99_ms=stats.p99_ms, + std_ms=float(np.std(samples)) if samples else 0.0, + warmup_mean_ms=float(np.mean(warmup_samples)) if warmup_samples else 0.0, + raw_samples_ms=samples, + samples_per_sec=samples_per_sec, + batches_per_sec=batches_per_sec, + effective_batch_size=self._effective_batch, + actual_device=self._actual_device, + actual_task=self._actual_task, + actual_ep=None, + running_model_path="", + hw_monitor=self._hw_metrics, + memory_profile=self._memory, + ) + + # ============================================================================= # Per-Module Perf # ============================================================================= @@ -2096,6 +2951,95 @@ def _run_simple_loop( next_log += log_step +_NO_EXPORT_INCOMPATIBLE_OPTIONS: dict[str, str] = { + "prompt": "--prompt", + "apply_template": "--apply-template/--no-apply-template", + "max_new_tokens": "--max-new-tokens", + "compile_timeout": "--compile-timeout", + "precision": "--precision", + "ep": "--ep", + "ep_options": "--ep-options", + "input_specs": "--input-specs", + "export_config": "--export-config", + "dynamic_axes": "--dynamic-axes", + "quant": "--quant/--no-quant", + "optimize": "--optimize/--no-optimize", + "analyze": "--analyze/--no-analyze", + "max_optim_iterations": "--max-optim-iterations", + "rebuild": "--rebuild/--no-rebuild", + "ignore_cache": "--ignore-cache/--no-ignore-cache", + "skip_build": "--skip-build/--no-skip-build", + "no_compile": "--compile/--no-compile", + "allow_unsupported_nodes": "--allow-unsupported-nodes", + "module_class": "--module", + "submodel": "--submodel", + "op_tracing": "--op-tracing", + "top_k": "--top-k", + "compare_devices": "--compare-devices", + "config_file": "--config", +} + + +def _validate_no_export_options( + ctx: click.Context, + *, + runtime: RuntimeName, + device: str, +) -> None: + """Reject options whose semantics depend on ONNX export or an ORT session.""" + if runtime != "winml": + raise click.UsageError("--no-export is only supported with --runtime winml.") + if device.lower() not in ("auto", "cpu", "gpu"): + raise click.UsageError( + f"--device {device} is not supported with --no-export; use auto, cpu, or gpu." + ) + + incompatible = [ + flag + for param_name, flag in _NO_EXPORT_INCOMPATIBLE_OPTIONS.items() + if cli_utils.is_cli_provided(ctx, param_name) + ] + if incompatible: + raise click.UsageError( + f"--no-export cannot be combined with incompatible options: {', '.join(incompatible)}." + ) + + +def _run_pytorch_perf_command( + config: BenchmarkConfig, + *, + console: SafeConsole, + json_mode: bool, + verbose: int, + quiet: bool, +) -> None: + """Run the native PyTorch path with the standard report/error contract.""" + benchmark = PyTorchPerfBenchmark(config) + try: + console.print(f"[dim]Loading PyTorch model:[/dim] {config.model_id}") + with ( + suppress_huggingface_warning_logs(verbosity=verbose, quiet=quiet), + suppress_third_party_progress(verbosity=verbose, quiet=quiet), + ): + result = benchmark.run() + + if json_mode: + click.echo(json.dumps(result.to_dict(), indent=2)) + else: + display_console_report(result, console) + assert config.output_path is not None + write_json_report(result, config.output_path) + console.print(f"[green]Results saved to:[/green] {config.output_path}") + except click.ClickException: + raise + except Exception as e: + if verbose: + logger.exception("PyTorch benchmark failed") + raise click.ClickException(f"Benchmark failed: {e}") from e + finally: + benchmark.close() + + # ============================================================================= # CLI Command # ============================================================================= @@ -2560,6 +3504,14 @@ def _validate_duration( '(e.g., {"input_ids": {"0": "batch", "1": "sequence"}}).' ) ) +@click.option( + "--export/--no-export", + "export_model", + default=True, + show_default=True, + help="Export Hugging Face models to ONNX before benchmarking. Use --no-export " + "to benchmark the original PyTorch model forward pass.", +) @cli_utils.quant_option(optional_message="Applied during model build.") @cli_utils.optimize_option(optional_message="Applied during model build.") @cli_utils.analyze_option(optional_message="Applied during model build.") @@ -2660,6 +3612,7 @@ def perf( input_specs: Path | None, export_config: Path | None, dynamic_axes: Path | None, + export_model: bool, quant: bool, optimize: bool, analyze: bool, @@ -2685,10 +3638,11 @@ def perf( Measures latency and throughput using random input data generated from the model's I/O configuration. - Accepts both HuggingFace model IDs and local .onnx files. Both flow + Accepts both HuggingFace model IDs and local .onnx files. By default both flow through the same PerfBenchmark pipeline (optimize → [quantize] → [compile] minus export for ONNX inputs), so latency numbers are directly comparable - between the two inputs. + between the two inputs. Use --no-export with a Hugging Face model to benchmark + its original PyTorch forward pass instead. \b Examples: @@ -2704,6 +3658,9 @@ def perf( # Text model with explicit task winml perf -m bert-base-uncased --task text-classification + # Original Hugging Face PyTorch model on CUDA + winml perf -m microsoft/resnet-50 --no-export --device gpu + # Pass runtime EP provider options (repeatable) winml perf -m model.onnx --device npu --ep-options htp_performance_mode=burst @@ -2724,6 +3681,13 @@ def perf( verbose, quiet = cli_utils.resolve_verbosity(ctx, verbose, quiet) configure_logging(verbosity=verbose, quiet=quiet) + if not export_model: + _validate_no_export_options( + ctx, + runtime=runtime, + device=device, + ) + # Hub-hosted ONNX (e.g. ``onnx-community/sam3-tracker-ONNX/onnx/...``) # is downloaded once and treated as a local .onnx path thereafter. # Must run BEFORE the ``Path(hf_model).suffix == ".onnx"`` check below @@ -2832,6 +3796,11 @@ def perf( is_onnx = model_input.kind is ModelInputKind.ONNX_FILE if is_onnx and model_input.local_path and not Path(model_input.local_path).exists(): raise click.UsageError(f"ONNX file not found: {hf_model}") + if not export_model and is_onnx: + raise click.UsageError( + "--no-export requires a Hugging Face model ID or model directory, " + "not a pre-exported ONNX model." + ) # --ep is parsed by EpAtSourceParamType at click parse time into an # ``(ep, source)`` tuple; the config-file merge above normalizes to the @@ -3030,6 +3999,31 @@ def perf( # Refuse to clobber an existing report unless the user opted in. cli_utils.guard_output(output, overwrite) + if not export_model: + config = BenchmarkConfig( + model_id=hf_model, + backend="pytorch", + task=task, + device=device.lower(), + iterations=iterations, + warmup=warmup, + duration=duration, + batch_size=batch_size, + output_path=output, + monitor=monitor, + memory=memory, + shape_config=shape_config, + input_data=input_data, + ) + _run_pytorch_perf_command( + config, + console=console, + json_mode=json_mode, + verbose=verbose, + quiet=quiet, + ) + return + compile_ep_options = None from ..session import short_ep_name @@ -3318,6 +4312,7 @@ def _format_input_shape(shape: list, actual: tuple | None) -> str: def _print_model_info( io_config: dict, *, + backend: str | None = None, task: str | None = None, req_device: str = "auto", act_device: str = "auto", @@ -3327,6 +4322,8 @@ def _print_model_info( """Print model I/O metadata before the benchmark starts.""" console = SafeConsole(stderr=True) console.print() + if backend: + console.print(f"[dim]Backend:[/dim] {backend}") device_line = _device_string(req_device, act_device, ep_name) console.print(f"[dim]Device:[/dim] {device_line}") if task: diff --git a/src/winml/modelkit/loader/hf.py b/src/winml/modelkit/loader/hf.py index 131dc1179..6f1f7491c 100644 --- a/src/winml/modelkit/loader/hf.py +++ b/src/winml/modelkit/loader/hf.py @@ -145,6 +145,9 @@ def load_hf_model( trust_remote_code: bool = False, hf_config: PretrainedConfig | None = None, model_type: str | None = None, + *, + use_checkpoint_class: bool = False, + torch_dtype: Any | None = None, ) -> tuple[nn.Module, PretrainedConfig, str]: """Load, detect task, and prepare HuggingFace model. @@ -172,6 +175,12 @@ def load_hf_model( hf_config: Optional pre-loaded HF config. When supplied, the ``AutoConfig.from_pretrained`` round-trip is skipped — same dedup pattern as ``resolve_loader_config(hf_config=...)`` from PR #719. + use_checkpoint_class: Load the architecture declared by the checkpoint + instead of a WinML task-specific export wrapper. Falls back to the + task-resolved class when the declared architecture is unavailable + from transformers (for example, a remote-code model). + torch_dtype: Optional dtype policy forwarded to ``from_pretrained``. + Pass ``"auto"`` to preserve the checkpoint's stored dtype. Returns: Tuple of (model, hf_config, task) @@ -263,6 +272,17 @@ def load_hf_model( raise ValueError( f"Cannot resolve task/model for {model_name_or_path}. Original error: {e}" ) from e + if use_checkpoint_class: + from .resolution import _resolve_model_class_from_config + + try: + resolved_class = _resolve_model_class_from_config(hf_config) + except ValueError: + logger.debug( + "Checkpoint architecture is not importable from transformers; " + "using the task-resolved model class %s", + resolved_class.__name__, + ) # [4] Model Instantiation logger.debug("Loading model with class: %s", resolved_class.__name__) @@ -279,11 +299,13 @@ def load_hf_model( if len(matching_subconfigs) == 1: model_config = cast("PretrainedConfig", matching_subconfigs[0]) - model = loader_cls.from_pretrained( - model_name_or_path, - trust_remote_code=trust_remote_code, - config=model_config, - ) + load_kwargs: dict[str, Any] = { + "trust_remote_code": trust_remote_code, + "config": model_config, + } + if torch_dtype is not None: + load_kwargs["torch_dtype"] = torch_dtype + model = loader_cls.from_pretrained(model_name_or_path, **load_kwargs) # [5] Export Preparation model.eval() diff --git a/src/winml/modelkit/session/monitor/_pdh.py b/src/winml/modelkit/session/monitor/_pdh.py index 8516148da..477bf7546 100644 --- a/src/winml/modelkit/session/monitor/_pdh.py +++ b/src/winml/modelkit/session/monitor/_pdh.py @@ -460,6 +460,8 @@ def __init__( poll_interval_ms: int = 200, device: str = "auto", ep_name: EPName | None = None, + adapter_luid: str | None = None, + include_gpu_aggregate: bool = True, ) -> None: device_norm = (device or "auto").lower() if device_norm not in _DEVICE_KINDS: @@ -469,6 +471,8 @@ def __init__( # Full ORT EP name (e.g. "QNNExecutionProvider") to disambiguate when # multiple EPs cover the same device type during LUID resolution. self._ep_name = ep_name + self._requested_adapter_luid = adapter_luid + self._include_gpu_aggregate = include_gpu_aggregate self._device_kind: str | None = None # resolved at start(): "npu" | "gpu" | None self._query: PdhQuery | None = None self._adapter_luid: str | None = None @@ -501,9 +505,16 @@ def start(self) -> None: or PDH fingerprinting (fallback). """ try: - self._adapter_luid, self._device_kind = self._resolve_adapter( - self._requested_device, self._ep_name - ) + if ( + self._requested_adapter_luid is not None + and self._requested_device in ACCELERATOR_DEVICE_TYPES + ): + self._adapter_luid = self._requested_adapter_luid + self._device_kind = self._requested_device + else: + self._adapter_luid, self._device_kind = self._resolve_adapter( + self._requested_device, self._ep_name + ) # Try to build the per-adapter query. If the resolved LUID is # missing from PDH enumeration or the engine type isn't present @@ -556,11 +567,15 @@ def start(self) -> None: # GPU adapters (multi-engine; max-aggregated). Independent of the # NPU — both can be present and monitored simultaneously. - self._gpu_luids = discover_gpu_luids() - if self._gpu_luids: - self._gpu_counter_names = add_gpu_engine_counters(self._query, self._gpu_luids) - else: - logger.info("No GPU found via PDH; monitoring CPU/RAM/NPU only") + if self._include_gpu_aggregate: + self._gpu_luids = discover_gpu_luids() + if self._gpu_luids: + self._gpu_counter_names = add_gpu_engine_counters( + self._query, + self._gpu_luids, + ) + else: + logger.info("No GPU found via PDH; monitoring CPU/RAM/NPU only") self._query.prime() @@ -582,6 +597,11 @@ def start(self) -> None: except (ImportError, RuntimeError) as exc: logger.warning("PDH monitoring unavailable: %s", exc) + if self._query is not None: + self._query.close() + self._query = None + self._adapter_luid = None + self._device_kind = None def stop(self) -> None: """Stop polling thread, capture final running_time, close query.""" diff --git a/src/winml/modelkit/session/monitor/hw_monitor.py b/src/winml/modelkit/session/monitor/hw_monitor.py index 227cdf1f6..e7d5dc6a1 100644 --- a/src/winml/modelkit/session/monitor/hw_monitor.py +++ b/src/winml/modelkit/session/monitor/hw_monitor.py @@ -66,6 +66,8 @@ def __init__( poll_interval_ms: int = 200, device: str = "auto", ep_name: EPName | None = None, + adapter_luid: str | None = None, + include_gpu_aggregate: bool = True, ) -> None: """Initialize the monitor. @@ -80,8 +82,18 @@ def __init__( metadata to resolve the same LUID the inference session will bind to — useful on hybrid systems where multiple adapters share a device type. + adapter_luid: Exact PDH-formatted adapter LUID to monitor. When + provided for an accelerator device, bypasses adapter discovery. + include_gpu_aggregate: Collect aggregate utilization across all + GPUs independently of the selected adapter. """ - self._pdh = PdhPoller(poll_interval_ms, device=device, ep_name=ep_name) + self._pdh = PdhPoller( + poll_interval_ms, + device=device, + ep_name=ep_name, + adapter_luid=adapter_luid, + include_gpu_aggregate=include_gpu_aggregate, + ) def __enter__(self) -> Self: """Start PDH background polling.""" diff --git a/tests/e2e/test_perf_e2e.py b/tests/e2e/test_perf_e2e.py index b9ec8f758..34122a40a 100644 --- a/tests/e2e/test_perf_e2e.py +++ b/tests/e2e/test_perf_e2e.py @@ -129,6 +129,7 @@ def _build_perf_args( input_data: Path | None = None, op_tracing: str | None = None, duration_overwrite: float | None = None, + no_export: bool = False, ) -> list[str]: """Build the argv list passed to the perf CLI. @@ -172,6 +173,8 @@ def _build_perf_args( args += ["--op-tracing", op_tracing] if duration_overwrite is not None: args += ["--duration", str(duration_overwrite)] + if no_export: + args.append("--no-export") return args @@ -836,6 +839,247 @@ class TestPerfHuggingFace: def model_arg(self) -> str: return "microsoft/resnet-50" + def test_no_export_benchmark_cpu(self, tmp_path: Path, model_arg: str): + """Benchmark the original Hugging Face PyTorch model on CPU.""" + output_file = tmp_path / "perf_hf_pytorch_cpu.json" + + result = CliRunner().invoke( + perf, + _build_perf_args( + model_arg=model_arg, + output_file=output_file, + device="cpu", + memory=False, + no_export=True, + ), + obj={}, + catch_exceptions=False, + ) + + assert result.exit_code == 0, f"perf failed (exit {result.exit_code}):\n{result.output}" + data = json.loads(output_file.read_text()) + assert data["benchmark_info"]["backend"] == "pytorch" + assert data["benchmark_info"]["device"] == "cpu" + assert data["benchmark_info"]["ep"] is None + assert data["benchmark_info"]["running_model_path"] == "" + assert data["latency_ms"]["mean"] > 0 + + def test_no_export_benchmark_cuda(self, tmp_path: Path, model_arg: str): + """Benchmark the original Hugging Face PyTorch model on CUDA when available.""" + import torch + + if not torch.cuda.is_available(): + pytest.skip("CUDA is not available") + + output_file = tmp_path / "perf_hf_pytorch_cuda.json" + result = CliRunner().invoke( + perf, + _build_perf_args( + model_arg=model_arg, + output_file=output_file, + device="gpu", + memory=False, + no_export=True, + ), + obj={}, + catch_exceptions=False, + ) + + assert result.exit_code == 0, f"perf failed (exit {result.exit_code}):\n{result.output}" + data = json.loads(output_file.read_text()) + assert data["benchmark_info"]["backend"] == "pytorch" + assert data["benchmark_info"]["device"] == "gpu" + assert data["benchmark_info"]["ep"] is None + assert data["latency_ms"]["mean"] > 0 + + def test_no_export_benchmark_decoder(self, tmp_path: Path): + """Benchmark a full decoder model without passing flattened ONNX cache inputs.""" + output_file = tmp_path / "perf_hf_pytorch_decoder.json" + shape_config = tmp_path / "shape.json" + shape_config.write_text(json.dumps({"sequence_length": 7})) + args = _build_perf_args( + model_arg="hf-internal-testing/tiny-random-T5ForConditionalGeneration", + output_file=output_file, + device="cpu", + memory=False, + no_export=True, + ) + args += ["--task", "text2text-generation", "--shape-config", str(shape_config)] + + result = CliRunner().invoke( + perf, + args, + obj={}, + catch_exceptions=False, + ) + + assert result.exit_code == 0, f"perf failed (exit {result.exit_code}):\n{result.output}" + data = json.loads(output_file.read_text()) + assert data["benchmark_info"]["backend"] == "pytorch" + assert data["benchmark_info"]["task"] == "text2text-generation" + assert "input_ids" in data["model_info"]["input_names"] + assert all("." not in name for name in data["model_info"]["input_names"]) + assert all(shape[1] == 7 for shape in data["model_info"]["input_shapes"] if len(shape) == 2) + assert data["latency_ms"]["mean"] > 0 + + def test_no_export_benchmark_multimodal(self, tmp_path: Path): + """Benchmark a full multimodal checkpoint with every compatible component input.""" + output_file = tmp_path / "perf_hf_pytorch_multimodal.json" + args = _build_perf_args( + model_arg="hf-internal-testing/tiny-random-CLIPModel", + output_file=output_file, + device="cpu", + memory=False, + no_export=True, + ) + + result = CliRunner().invoke( + perf, + args, + obj={}, + catch_exceptions=False, + ) + + assert result.exit_code == 0, f"perf failed (exit {result.exit_code}):\n{result.output}" + data = json.loads(output_file.read_text()) + assert data["benchmark_info"]["backend"] == "pytorch" + assert {"input_ids", "pixel_values"} <= set(data["model_info"]["input_names"]) + assert data["latency_ms"]["mean"] > 0 + + def test_no_export_benchmark_unregistered_multimodal(self, tmp_path: Path): + """Derive full-model inputs from nested configs without a composite registration.""" + from transformers import ( + BertConfig, + VisionTextDualEncoderConfig, + VisionTextDualEncoderModel, + ViTConfig, + ) + + model_dir = tmp_path / "dual-encoder" + config = VisionTextDualEncoderConfig.from_vision_text_configs( + ViTConfig( + image_size=16, + patch_size=4, + hidden_size=16, + num_hidden_layers=1, + num_attention_heads=2, + intermediate_size=32, + ), + BertConfig( + vocab_size=100, + hidden_size=16, + num_hidden_layers=1, + num_attention_heads=2, + intermediate_size=32, + max_position_embeddings=16, + ), + projection_dim=8, + ) + VisionTextDualEncoderModel(config).save_pretrained(model_dir) + output_file = tmp_path / "perf_hf_pytorch_unregistered_multimodal.json" + args = _build_perf_args( + model_arg=str(model_dir), + output_file=output_file, + device="cpu", + memory=False, + no_export=True, + ) + + result = CliRunner().invoke( + perf, + args, + obj={}, + catch_exceptions=False, + ) + + assert result.exit_code == 0, f"perf failed (exit {result.exit_code}):\n{result.output}" + data = json.loads(output_file.read_text()) + assert {"input_ids", "pixel_values"} <= set(data["model_info"]["input_names"]) + assert data["latency_ms"]["mean"] > 0 + + def test_no_export_benchmark_audio_checkpoint_shape(self, tmp_path: Path): + """Use the checkpoint encoder contract instead of an export-only audio shape.""" + from transformers import WhisperConfig, WhisperForConditionalGeneration + + model_dir = tmp_path / "whisper" + config = WhisperConfig( + vocab_size=100, + num_mel_bins=8, + d_model=16, + encoder_layers=1, + decoder_layers=1, + encoder_attention_heads=2, + decoder_attention_heads=2, + encoder_ffn_dim=32, + decoder_ffn_dim=32, + max_source_positions=30, + max_target_positions=16, + decoder_start_token_id=1, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + ) + WhisperForConditionalGeneration(config).save_pretrained(model_dir) + output_file = tmp_path / "perf_hf_pytorch_audio.json" + + result = CliRunner().invoke( + perf, + _build_perf_args( + model_arg=str(model_dir), + output_file=output_file, + device="cpu", + memory=False, + no_export=True, + ), + obj={}, + catch_exceptions=False, + ) + + assert result.exit_code == 0, f"perf failed (exit {result.exit_code}):\n{result.output}" + data = json.loads(output_file.read_text()) + main_index = data["model_info"]["input_names"].index("input_features") + assert data["model_info"]["input_shapes"][main_index] == [1, 8, 60] + assert data["latency_ms"]["mean"] > 0 + + def test_no_export_benchmark_video_checkpoint_shape(self, tmp_path: Path): + """Generate a native video tensor when the architecture has no exporter config.""" + from transformers import VideoMAEConfig, VideoMAEForVideoClassification + + model_dir = tmp_path / "videomae" + config = VideoMAEConfig( + image_size=16, + patch_size=4, + num_channels=3, + num_frames=4, + tubelet_size=2, + hidden_size=16, + num_hidden_layers=1, + num_attention_heads=2, + intermediate_size=32, + num_labels=2, + ) + VideoMAEForVideoClassification(config).save_pretrained(model_dir) + output_file = tmp_path / "perf_hf_pytorch_video.json" + + result = CliRunner().invoke( + perf, + _build_perf_args( + model_arg=str(model_dir), + output_file=output_file, + device="cpu", + memory=False, + no_export=True, + ), + obj={}, + catch_exceptions=False, + ) + + assert result.exit_code == 0, f"perf failed (exit {result.exit_code}):\n{result.output}" + data = json.loads(output_file.read_text()) + main_index = data["model_info"]["input_names"].index("pixel_values") + assert data["model_info"]["input_shapes"][main_index] == [1, 4, 3, 16, 16] + assert data["latency_ms"]["mean"] > 0 + @pytest.mark.parametrize("ep", CPU_EPS) def test_benchmark_ep_cpu(self, ep: str, tmp_path: Path, model_arg: str): """Benchmark with --ep .""" diff --git a/tests/unit/commands/test_perf_pytorch.py b/tests/unit/commands/test_perf_pytorch.py new file mode 100644 index 000000000..0b2566d56 --- /dev/null +++ b/tests/unit/commands/test_perf_pytorch.py @@ -0,0 +1,738 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- +"""Tests for native Hugging Face PyTorch benchmarking in ``winml perf``.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace +from unittest.mock import MagicMock + +import numpy as np +import pytest +import torch +from click.testing import CliRunner + +import winml.modelkit.commands.perf as perf_module +from winml.modelkit.commands.perf import ( + BenchmarkConfig, + BenchmarkResult, + PyTorchPerfBenchmark, + _PyTorchForwardRunner, + perf, +) +from winml.modelkit.export import InputTensorSpec, OutputTensorSpec, WinMLExportConfig +from winml.modelkit.session.stats import PerfStats + + +class _RecordingModel(torch.nn.Module): + def __init__(self, *, dtype: torch.dtype = torch.float32) -> None: + super().__init__() + self.projection = torch.nn.Linear(4, 2, dtype=dtype) + self.inference_modes: list[bool] = [] + self.input_devices: list[torch.device] = [] + self.input_dtypes: list[torch.dtype] = [] + self.input_shapes: list[tuple[int, ...]] = [] + + def forward(self, pixel_values: torch.Tensor) -> dict[str, torch.Tensor]: + self.inference_modes.append(torch.is_inference_mode_enabled()) + self.input_devices.append(pixel_values.device) + self.input_dtypes.append(pixel_values.dtype) + self.input_shapes.append(tuple(pixel_values.shape)) + return {"logits": self.projection(pixel_values)} + + +class _DecoderModel(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.embedding = torch.nn.Embedding(16, 4) + self.dummy_inputs = { + "input_ids": torch.ones((3, 5), dtype=torch.int64), + } + self.seen_inputs: list[dict[str, torch.Tensor]] = [] + + def forward( + self, + input_ids: torch.Tensor, + attention_mask: torch.Tensor | None = None, + ) -> dict[str, torch.Tensor]: + inputs = {"input_ids": input_ids} + if attention_mask is not None: + inputs["attention_mask"] = attention_mask + self.seen_inputs.append(inputs) + return {"logits": self.embedding(input_ids)} + + +class _MultimodalModel(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.scale = torch.nn.Parameter(torch.ones(())) + self.dummy_inputs = {"input_ids": torch.ones((3, 5), dtype=torch.int64)} + self.seen_inputs: list[set[str]] = [] + + def forward( + self, + input_ids: torch.Tensor | None = None, + pixel_values: torch.Tensor | None = None, + ) -> dict[str, torch.Tensor]: + if input_ids is None or pixel_values is None: + raise ValueError("both modalities are required") + self.seen_inputs.append({"input_ids", "pixel_values"}) + return {"logits": (input_ids.float().mean() + pixel_values.mean()) * self.scale} + + +class _OptionalMultimodalModel(torch.nn.Module): + main_input_name = "input_ids" + + def __init__(self) -> None: + super().__init__() + self.scale = torch.nn.Parameter(torch.ones(())) + + def forward( + self, + input_ids: torch.Tensor | None = None, + pixel_values: torch.Tensor | None = None, + ) -> dict[str, torch.Tensor]: + outputs = {} + if input_ids is not None: + outputs["text"] = input_ids.float() * self.scale + if pixel_values is not None: + outputs["image"] = pixel_values * self.scale + if not outputs: + raise ValueError("at least one modality is required") + return outputs + + +class _Seq2SeqDummyModel(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.embedding = torch.nn.Embedding(16, 4) + self.dummy_inputs = { + "input_ids": torch.ones((3, 5), dtype=torch.int64), + "attention_mask": torch.ones((3, 5), dtype=torch.int64), + } + self.seen_shapes: list[dict[str, tuple[int, ...]]] = [] + + def forward( + self, + input_ids: torch.Tensor, + attention_mask: torch.Tensor, + ) -> dict[str, torch.Tensor]: + self.seen_shapes.append( + { + "input_ids": tuple(input_ids.shape), + "attention_mask": tuple(attention_mask.shape), + } + ) + return {"logits": self.embedding(input_ids)} + + +class _AlternativeInputModel(torch.nn.Module): + main_input_name = "raw_input" + + def __init__(self) -> None: + super().__init__() + self.scale = torch.nn.Parameter(torch.ones(())) + self.paths: list[str] = [] + + def forward( + self, + raw_input: torch.Tensor | None = None, + precomputed_input: torch.Tensor | None = None, + ) -> dict[str, torch.Tensor]: + if precomputed_input is not None: + self.paths.append("precomputed") + return {"output": precomputed_input * self.scale} + if raw_input is None: + raise ValueError("raw input is required") + self.paths.append("raw") + return {"output": raw_input * self.scale} + + +def _fake_build_config() -> SimpleNamespace: + return SimpleNamespace( + loader=SimpleNamespace( + task="image-classification", + model_class="AutoModelForImageClassification", + trust_remote_code=False, + ), + export=WinMLExportConfig( + input_tensors=[ + InputTensorSpec( + name="pixel_values", + dtype="float32", + shape=(1, 4), + ) + ], + output_tensors=[OutputTensorSpec(name="logits")], + ), + ) + + +def _patch_hf_loading( + monkeypatch: pytest.MonkeyPatch, + model: torch.nn.Module, +) -> tuple[MagicMock, MagicMock]: + generate = MagicMock(return_value=_fake_build_config()) + load = MagicMock(return_value=(model, SimpleNamespace(), "image-classification")) + monkeypatch.setattr("winml.modelkit.config.generate_hf_build_config", generate) + monkeypatch.setattr("winml.modelkit.loader.load_hf_model", load) + return generate, load + + +class TestNoExportCli: + def test_help_shows_export_pair(self) -> None: + result = CliRunner().invoke(perf, ["--help"]) + + assert result.exit_code == 0 + assert "--export" in result.output + assert "--no-export" in result.output + + def test_no_export_dispatches_pytorch_backend( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path, + ) -> None: + captured: dict[str, object] = {} + + def run(config: BenchmarkConfig, **_: object) -> None: + captured["config"] = config + + monkeypatch.setattr(perf_module, "_run_pytorch_perf_command", run) + output = tmp_path / "result.json" + + result = CliRunner().invoke( + perf, + [ + "-m", + "fake/model", + "--no-export", + "--device", + "cpu", + "--iterations", + "2", + "-o", + str(output), + ], + obj={}, + ) + + assert result.exit_code == 0, result.output + config = captured["config"] + assert isinstance(config, BenchmarkConfig) + assert config.backend == "pytorch" + assert config.device == "cpu" + assert config.iterations == 2 + + @pytest.mark.parametrize( + "args, expected_flag", + [ + (["--ep", "cpu"], "--ep"), + (["--precision", "fp16"], "--precision"), + (["--prompt", "hello"], "--prompt"), + (["--max-new-tokens", "4"], "--max-new-tokens"), + (["--no-quant"], "--quant/--no-quant"), + (["--compile"], "--compile/--no-compile"), + (["--module", "Linear"], "--module"), + (["--submodel", "encoder"], "--submodel"), + (["--op-tracing", "basic"], "--op-tracing"), + ], + ) + def test_rejects_onnx_only_options( + self, + args: list[str], + expected_flag: str, + ) -> None: + result = CliRunner().invoke( + perf, + ["-m", "fake/model", "--no-export", *args], + obj={}, + ) + + assert result.exit_code == 2 + assert expected_flag in result.output + + def test_rejects_onnx_input(self, tmp_path) -> None: + model_path = tmp_path / "model.onnx" + model_path.write_bytes(b"not used") + + result = CliRunner().invoke( + perf, + ["-m", str(model_path), "--no-export"], + obj={}, + ) + + assert result.exit_code == 2 + assert "requires a Hugging Face model" in result.output + + def test_rejects_genai_runtime(self) -> None: + result = CliRunner().invoke( + perf, + [ + "-m", + "fake/model", + "--no-export", + "--runtime", + "winml-genai", + ], + obj={}, + ) + + assert result.exit_code == 2 + assert "only supported with --runtime winml" in result.output + + def test_rejects_npu_device(self) -> None: + result = CliRunner().invoke( + perf, + ["-m", "fake/model", "--no-export", "--device", "npu"], + obj={}, + ) + + assert result.exit_code == 2 + assert "use auto, cpu, or gpu" in result.output + + def test_gpu_requires_cuda( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path, + ) -> None: + monkeypatch.setattr(torch.cuda, "is_available", lambda: False) + + result = CliRunner().invoke( + perf, + [ + "-m", + "fake/model", + "--no-export", + "--device", + "gpu", + "-o", + str(tmp_path / "result.json"), + ], + obj={}, + ) + + assert result.exit_code == 2 + assert "requires a CUDA-enabled PyTorch" in result.output + + +class TestPyTorchPerfBenchmark: + def test_runs_raw_forward_on_cpu( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + model = _RecordingModel(dtype=torch.float64) + generate, load = _patch_hf_loading(monkeypatch, model) + benchmark = PyTorchPerfBenchmark( + BenchmarkConfig( + model_id="fake/model", + backend="pytorch", + device="cpu", + iterations=2, + warmup=1, + batch_size=3, + memory=False, + ) + ) + + result = benchmark.run() + + assert len(model.inference_modes) == 3 + assert all(model.inference_modes) + assert model.input_devices == [torch.device("cpu")] * 3 + assert model.input_dtypes == [torch.float64] * 3 + assert model.input_shapes == [(3, 4)] * 3 + assert result.config.backend == "pytorch" + assert result.actual_device == "cpu" + assert result.actual_ep is None + assert result.actual_task == "image-classification" + assert result.model_precision == "float64" + assert result.input_shapes == [[3, 4]] + assert result.input_types == ["float64"] + assert result.output_names == ["logits"] + assert result.output_shapes == [[3, 2]] + assert len(result.raw_samples_ms) == 2 + assert result.mean_ms > 0 + assert result.effective_batch_size == 3 + generate.assert_called_once() + load.assert_called_once_with( + "fake/model", + task=None, + use_checkpoint_class=True, + torch_dtype="auto", + ) + + def test_uses_native_dummy_input_for_flattened_export_inputs( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + model = _DecoderModel() + build_config = _fake_build_config() + build_config.loader.task = "text-generation" + build_config.export.input_tensors = [ + InputTensorSpec(name="past_key_values.0.key", dtype="float32", shape=(1, 2, 4, 2)), + ] + generate = MagicMock(return_value=build_config) + load = MagicMock(return_value=(model, SimpleNamespace(), "text-generation")) + monkeypatch.setattr("winml.modelkit.config.generate_hf_build_config", generate) + monkeypatch.setattr("winml.modelkit.loader.load_hf_model", load) + benchmark = PyTorchPerfBenchmark( + BenchmarkConfig( + model_id="fake/decoder", + backend="pytorch", + device="cpu", + iterations=1, + warmup=0, + batch_size=2, + memory=False, + ) + ) + + result = benchmark.run() + + assert list(model.seen_inputs) == [{"input_ids": model.seen_inputs[0]["input_ids"]}] + assert tuple(model.seen_inputs[0]["input_ids"].shape) == (2, 5) + assert result.input_names == ["input_ids"] + + def test_uses_native_dummy_input_when_export_specs_cannot_resolve( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + model = _DecoderModel() + generate = MagicMock(side_effect=AttributeError("missing normalized attribute")) + load = MagicMock(return_value=(model, SimpleNamespace(), "text-generation")) + monkeypatch.setattr("winml.modelkit.config.generate_hf_build_config", generate) + monkeypatch.setattr("winml.modelkit.loader.load_hf_model", load) + benchmark = PyTorchPerfBenchmark( + BenchmarkConfig( + model_id="fake/decoder", + backend="pytorch", + device="cpu", + iterations=1, + warmup=0, + memory=False, + ) + ) + + result = benchmark.run() + + assert result.input_names == ["input_ids"] + assert len(result.raw_samples_ms) == 1 + + def test_merges_composite_inputs_for_full_checkpoint( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + model = _MultimodalModel() + text_config = _fake_build_config() + text_config.loader.task = "feature-extraction" + text_config.export.input_tensors = [ + InputTensorSpec(name="input_ids", dtype="int64", shape=(1, 5)), + ] + full_config = _fake_build_config() + full_config.loader.task = "zero-shot-image-classification" + full_config.export.input_tensors = [ + InputTensorSpec(name="input_ids", dtype="int64", shape=(1, 5)), + InputTensorSpec(name="pixel_values", dtype="float32", shape=(1, 3, 4, 4)), + ] + generate = MagicMock(side_effect=[text_config, full_config]) + load = MagicMock( + return_value=(model, SimpleNamespace(model_type="multimodal"), "feature-extraction") + ) + monkeypatch.setattr("winml.modelkit.config.generate_hf_build_config", generate) + monkeypatch.setattr("winml.modelkit.loader.load_hf_model", load) + monkeypatch.setattr( + "winml.modelkit.loader.composite_pipeline_tasks", + lambda _model_type: ["zero-shot-image-classification"], + ) + benchmark = PyTorchPerfBenchmark( + BenchmarkConfig( + model_id="fake/multimodal", + backend="pytorch", + device="cpu", + iterations=1, + warmup=0, + memory=False, + ) + ) + + result = benchmark.run() + + assert model.seen_inputs == [{"input_ids", "pixel_values"}] + assert result.input_names == ["input_ids", "pixel_values"] + + def test_merges_inputs_from_unregistered_nested_configs( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + from transformers import PretrainedConfig + + model = _MultimodalModel() + parent_config = PretrainedConfig() + parent_config.model_type = "multimodal" + parent_config.text_config = PretrainedConfig() + parent_config.text_config.model_type = "text-encoder" + parent_config.vision_config = PretrainedConfig() + parent_config.vision_config.model_type = "vision-encoder" + generate = MagicMock(side_effect=ValueError("no top-level exporter")) + load = MagicMock(return_value=(model, parent_config, "feature-extraction")) + + def resolve_specs(model_type: str, *_args: object, **_kwargs: object) -> dict[str, object]: + if model_type == "text-encoder": + return { + "input_names": ["input_ids"], + "input_shapes": [(1, 5)], + "input_dtypes": ["int64"], + "value_ranges": {}, + } + return { + "input_names": ["pixel_values"], + "input_shapes": [(1, 3, 4, 4)], + "input_dtypes": ["float32"], + "value_ranges": {}, + } + + monkeypatch.setattr("winml.modelkit.config.generate_hf_build_config", generate) + monkeypatch.setattr("winml.modelkit.loader.load_hf_model", load) + monkeypatch.setattr("winml.modelkit.loader.composite_pipeline_tasks", lambda _: []) + monkeypatch.setattr("winml.modelkit.export.resolve_io_specs", resolve_specs) + benchmark = PyTorchPerfBenchmark( + BenchmarkConfig( + model_id="fake/multimodal", + backend="pytorch", + device="cpu", + iterations=1, + warmup=0, + memory=False, + ) + ) + + result = benchmark.run() + + assert model.seen_inputs == [{"input_ids", "pixel_values"}] + assert set(result.input_names) == {"input_ids", "pixel_values"} + + def test_shape_config_resizes_all_native_sequence_inputs( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + model = _Seq2SeqDummyModel() + generate = MagicMock(side_effect=ValueError("no export specs")) + load = MagicMock( + return_value=(model, SimpleNamespace(model_type="decoder"), "text2text-generation") + ) + monkeypatch.setattr("winml.modelkit.config.generate_hf_build_config", generate) + monkeypatch.setattr("winml.modelkit.loader.load_hf_model", load) + monkeypatch.setattr("winml.modelkit.loader.composite_pipeline_tasks", lambda _: []) + benchmark = PyTorchPerfBenchmark( + BenchmarkConfig( + model_id="fake/decoder", + backend="pytorch", + device="cpu", + iterations=1, + warmup=0, + batch_size=2, + shape_config={"sequence_length": 7}, + memory=False, + ) + ) + + result = benchmark.run() + + assert model.seen_shapes == [ + { + "input_ids": (2, 7), + "attention_mask": (2, 7), + } + ] + assert result.input_shapes == [[2, 7], [2, 7]] + + def test_preflight_keeps_checkpoint_main_input_path( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + model = _AlternativeInputModel() + build_config = _fake_build_config() + build_config.export.input_tensors = [ + InputTensorSpec(name="precomputed_input", dtype="float32", shape=(1, 4)), + InputTensorSpec(name="raw_input", dtype="float32", shape=(1, 4)), + ] + generate = MagicMock(return_value=build_config) + load = MagicMock( + return_value=(model, SimpleNamespace(model_type="alternative"), "feature-extraction") + ) + monkeypatch.setattr("winml.modelkit.config.generate_hf_build_config", generate) + monkeypatch.setattr("winml.modelkit.loader.load_hf_model", load) + monkeypatch.setattr("winml.modelkit.loader.composite_pipeline_tasks", lambda _: []) + benchmark = PyTorchPerfBenchmark( + BenchmarkConfig( + model_id="fake/alternative", + backend="pytorch", + device="cpu", + iterations=1, + warmup=0, + memory=False, + ) + ) + + result = benchmark.run() + + assert model.paths[-1] == "raw" + assert result.input_names == ["raw_input"] + + def test_preflight_keeps_all_independent_modalities(self) -> None: + benchmark = PyTorchPerfBenchmark( + BenchmarkConfig(model_id="fake/multimodal", backend="pytorch") + ) + benchmark._model = _OptionalMultimodalModel() + benchmark._main_input_name = "input_ids" + inputs = { + "input_ids": torch.ones((1, 4), dtype=torch.int64), + "pixel_values": torch.ones((1, 3, 4, 4)), + } + + selected = benchmark._select_checkpoint_forward_inputs(torch, inputs) + + assert list(selected) == ["input_ids", "pixel_values"] + assert selected["input_ids"] is inputs["input_ids"] + assert selected["pixel_values"] is inputs["pixel_values"] + + def test_input_data_owns_batch_size( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path, + ) -> None: + model = _RecordingModel() + _patch_hf_loading(monkeypatch, model) + input_path = tmp_path / "inputs.npz" + np.savez(input_path, pixel_values=np.ones((2, 4), dtype=np.float32)) + benchmark = PyTorchPerfBenchmark( + BenchmarkConfig( + model_id="fake/model", + backend="pytorch", + device="cpu", + iterations=1, + warmup=0, + batch_size=9, + input_data=input_path, + memory=False, + ) + ) + + result = benchmark.run() + + assert model.input_shapes == [(2, 4)] + assert result.effective_batch_size == 2 + assert result.input_shapes == [[2, 4]] + + def test_auto_uses_cpu_without_cuda( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setattr(torch.cuda, "is_available", lambda: False) + benchmark = PyTorchPerfBenchmark( + BenchmarkConfig(model_id="fake/model", backend="pytorch", device="auto") + ) + + benchmark._resolve_device(torch) + + assert benchmark._actual_device == "cpu" + assert benchmark._torch_device == torch.device("cpu") + + def test_duration_runs_until_budget( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + model = _RecordingModel() + _patch_hf_loading(monkeypatch, model) + benchmark = PyTorchPerfBenchmark( + BenchmarkConfig( + model_id="fake/model", + backend="pytorch", + device="cpu", + duration=0.001, + warmup=0, + memory=False, + ) + ) + + result = benchmark.run() + + assert result.raw_samples_ms + assert result.to_dict()["benchmark_info"]["iterations"] == len(result.raw_samples_ms) + + def test_monitor_falls_back_when_hw_monitor_is_unavailable( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + from winml.modelkit.session.monitor.hw_monitor import HWMonitor + + model = _RecordingModel() + _patch_hf_loading(monkeypatch, model) + monkeypatch.setattr(HWMonitor, "is_available", lambda: False) + benchmark = PyTorchPerfBenchmark( + BenchmarkConfig( + model_id="fake/model", + backend="pytorch", + device="cpu", + iterations=1, + warmup=0, + monitor=True, + memory=False, + ) + ) + + result = benchmark.run() + + assert len(result.raw_samples_ms) == 1 + assert result.hw_monitor is None + + def test_memory_profile_uses_model_load_phase_names(self) -> None: + profile = PyTorchPerfBenchmark._build_memory_profile( + (100.0, 10.0, 0.0), + (140.0, 30.0, 0.0), + (150.0, 35.0, 0.0), + ) + + assert profile["rss_after_model_load_mb"] == 140.0 + assert profile["rss_model_load_delta_mb"] == 40.0 + assert profile["vram_local_model_load_delta_mb"] == 20.0 + assert profile["vram_local_inference_delta_mb"] == 5.0 + + +class TestPyTorchTiming: + def test_cuda_runner_synchronizes_before_and_after_forward(self) -> None: + model = MagicMock(return_value={"logits": torch.ones(1, 2)}) + synchronize = MagicMock() + stats = PerfStats() + runner = _PyTorchForwardRunner(model, stats, synchronize) + inputs = {"pixel_values": torch.ones(1, 4)} + + runner.run(inputs) + + assert synchronize.call_count == 2 + model.assert_called_once_with(**inputs) + assert len(stats.samples_ms) == 1 + assert runner.output_metadata == [("logits", [1, 2], "float32")] + + def test_cuda_luid_uses_windows_pdh_format(self) -> None: + raw_luid = bytes.fromhex("0200000001000000") + + assert PyTorchPerfBenchmark._format_cuda_luid(raw_luid) == "0x00000001_0x00000002" + + +def test_report_identifies_pytorch_backend() -> None: + result = BenchmarkResult( + config=BenchmarkConfig(model_id="fake/model", backend="pytorch"), + actual_device="gpu", + ) + + report = result.to_dict() + + assert report["benchmark_info"]["backend"] == "pytorch" + assert report["benchmark_info"]["ep"] is None + assert report["benchmark_info"]["precision"] is None + assert report["benchmark_info"]["running_model_path"] == "" + json.dumps(report) diff --git a/tests/unit/loader/test_load_hf_model.py b/tests/unit/loader/test_load_hf_model.py index 1c3f4066f..cbe0a4766 100644 --- a/tests/unit/loader/test_load_hf_model.py +++ b/tests/unit/loader/test_load_hf_model.py @@ -195,6 +195,52 @@ def mock_resolve(config, *, task=None, model_class=None, model_type_override=Non assert call["task"] is None # Auto-detect assert call["model_class"] is None + def test_checkpoint_class_and_dtype_override_export_wrapper(self, monkeypatch): + """Native consumers can load the checkpoint architecture and stored dtype.""" + from types import SimpleNamespace + from unittest.mock import MagicMock + + import winml.modelkit.loader.resolution as resolution_module + + checkpoint_class = MagicMock() + checkpoint_class.__name__ = "CheckpointModel" + checkpoint_class.config_class = None + checkpoint_model = MagicMock() + checkpoint_class.from_pretrained.return_value = checkpoint_model + export_wrapper = MagicMock() + export_wrapper.__name__ = "ExportWrapper" + config = SimpleNamespace(model_type="unit", architectures=["CheckpointModel"]) + + monkeypatch.setattr( + resolution_module, + "resolve_task", + lambda *_a, **_kw: SimpleNamespace( + task="text-generation", + model_class=export_wrapper, + ), + ) + monkeypatch.setattr( + resolution_module, + "_resolve_model_class_from_config", + lambda _config: checkpoint_class, + ) + + model, _, task = load_hf_model( + "fake/model", + hf_config=config, + use_checkpoint_class=True, + torch_dtype="auto", + ) + + assert model is checkpoint_model + assert task == "text-generation" + checkpoint_class.from_pretrained.assert_called_once_with( + "fake/model", + trust_remote_code=False, + config=config, + torch_dtype="auto", + ) + def test_bert_tiny_uses_model_specific_default_task(self, monkeypatch): """bert-tiny should use model-specific default task when task is omitted.""" from unittest.mock import MagicMock diff --git a/tests/unit/session/test_ep_monitor.py b/tests/unit/session/test_ep_monitor.py index e4fce6a07..91bb0f05b 100644 --- a/tests/unit/session/test_ep_monitor.py +++ b/tests/unit/session/test_ep_monitor.py @@ -1064,6 +1064,103 @@ def test_gpu_device_uses_3d_engine_query(self): assert poller.device_kind == "gpu" assert poller.adapter_luid == "0x00000000_0xDEADBEEF" + def test_explicit_adapter_luid_bypasses_discovery(self): + """A runtime-provided adapter identity must bind the monitor exactly.""" + from winml.modelkit.session.monitor._pdh import PdhPoller + + fake_query = type( + "Q", + (), + { + "open": lambda self: None, + "add_counter": lambda self, *a, **k: True, + "prime": lambda self: None, + "collect": lambda self, **k: {}, + "_collect_once": lambda self: {}, + "close": lambda self: None, + "counter_names": [], + }, + )() + + with ( + patch("winml.modelkit.session.monitor._pdh.resolve_adapter_luid") as mock_resolve, + patch( + "winml.modelkit.session.monitor._pdh.build_gpu_query", + return_value=fake_query, + ) as mock_build_gpu, + ): + poller = PdhPoller( + poll_interval_ms=50, + device="gpu", + adapter_luid="0x00000001_0x00000002", + ) + poller.start() + poller.stop() + + mock_resolve.assert_not_called() + mock_build_gpu.assert_called_once_with("0x00000001_0x00000002") + assert poller.adapter_luid == "0x00000001_0x00000002" + + def test_startup_failure_clears_explicit_adapter_identity(self): + """Failed PDH startup must not report telemetry for an unmonitored adapter.""" + from winml.modelkit.session.monitor._pdh import PdhPoller + + fake_query = MagicMock() + with ( + patch( + "winml.modelkit.session.monitor._pdh.build_gpu_query", + return_value=fake_query, + ), + patch( + "winml.modelkit.session.monitor._pdh.discover_gpu_luids", + side_effect=RuntimeError("PDH unavailable"), + ), + ): + poller = PdhPoller( + poll_interval_ms=50, + device="gpu", + adapter_luid="0x00000001_0x00000002", + ) + poller.start() + + fake_query.close.assert_called_once() + assert poller.adapter_luid is None + assert poller.device_kind is None + assert poller.is_active is False + + def test_gpu_aggregate_can_be_disabled(self): + """CPU/RAM-only fallback must not discover or poll unrelated GPUs.""" + from winml.modelkit.session.monitor._pdh import PdhPoller + + fake_query = type( + "Q", + (), + { + "open": lambda self: None, + "add_counter": lambda self, *a, **k: True, + "prime": lambda self: None, + "collect": lambda self, **k: {}, + "_collect_once": lambda self: {}, + "close": lambda self: None, + "counter_names": [], + }, + )() + with ( + patch("winml.modelkit.session.monitor._pdh.PdhQuery", return_value=fake_query), + patch("winml.modelkit.session.monitor._pdh.discover_gpu_luids") as mock_discover_gpu, + ): + poller = PdhPoller( + poll_interval_ms=50, + device="cpu", + include_gpu_aggregate=False, + ) + poller.start() + poller.stop() + + mock_discover_gpu.assert_not_called() + assert poller.gpu_luids == [] + assert poller.gpu_sample_count == 0 + def test_auto_prefers_npu_then_gpu(self): """device='auto' must probe NPU first, then GPU.""" from winml.modelkit.session.monitor._pdh import PdhPoller