diff --git a/.pipelines/modelkit-official-build.yml b/.pipelines/modelkit-official-build.yml index 2b7ca7d89..279503819 100644 --- a/.pipelines/modelkit-official-build.yml +++ b/.pipelines/modelkit-official-build.yml @@ -127,7 +127,14 @@ extends: artifactFeeds: 'windows.ai.toolkit/Modelkit' displayName: 'Authenticate pip with Azure Artifacts' - - script: python -m pip install --upgrade build twine packaging + # PINNED ON PURPOSE — do NOT relax to unpinned/--upgrade. + # An unpinned toolchain lets a third-party release break the + # release build on publish day. build 1.5.1 did exactly that: + # its isolated-env pip invocation fails inside this container + # with "FileNotFoundError: [WinError 2]" while resolving the + # user cache dir from the registry. These are the versions that + # produced the v0.2.0 release. Bump deliberately, never silently. + - script: python -m pip install build==1.5.0 twine==6.2.0 packaging==26.2 displayName: 'Install build tools' # Build sdist BEFORE iKey injection so the source archive diff --git a/CHANGELOG.md b/CHANGELOG.md index 68fb2f1f5..2f09a2913 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,65 @@ All notable changes to this project are documented in this file. The format is loosely based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## WinML CLI v0.3.0 + +This cycle expands **model preparation and evaluation** across the CLI: precision-driven quantization, composite-model and dynamic-axis workflows, real-input perf/eval, optimization previews, and opt-in Dynamo export. It also introduces one-command Qwen3 onnxruntime-genai bundles, GenAI benchmarking, broader model recipes, and more reliable EP discovery, compilation, and monitoring. See the behavior changes below. + +### ⚠️ Behavior changes + +- **Output-producing commands** now refuse to replace existing files or non-empty directories unless `--overwrite` is passed; `winml build` retains its existing incremental `--rebuild` behavior (#970). +- `winml quantize` renames `--model-name` to `--model-id`, including the corresponding quantization-config field (#984). +- **Compile configuration** no longer silently defaults a missing execution provider to QNN; incomplete configurations now fail validation instead (#1026). +- `winml inspect` / `winml perf` hide third-party and native warning noise by default; use `-v`, `-vv`, or `WINMLCLI_SHOW_ALL_WARNINGS=1` to restore diagnostics (#1232, #1246). + +### ✨ Improvements + +- **Quantization** — `--precision` selects FP16 conversion, RTN INT4, static QDQ, or calibration-free dynamic INT8; `winml quantize` can compose multiple precision passes such as INT4 followed by FP16 (#872, #985, #1047). +- `winml build` — `--export-type optimized` produces a complete Qwen3 onnxruntime-genai NPU/QNN bundle, including prefill/decode, embeddings, LM head, tokenizer, and manifest files (#836, #996, #1008, #1081, #1104). +- `winml perf --runtime winml-genai` — benchmarks prebuilt or automatically cached GenAI bundles with TTFT, token throughput, prompt-template controls, EP overrides, and isolated pre-compilation (#1015, #1042, #1046, #1054, #1109). +- **Composite models** — `export` and `build` automatically fan out pipeline components; `export` / `build` / `perf` support `--submodel`, and explicit composite tasks such as summarization and translation are accepted (#1031, #1037, #1058, #1071, #1089). +- **Export controls** — dynamic axes and symbolic input dimensions are supported while static TorchScript export remains the default; `build`, `config`, `perf`, and `eval` expose matching shape/input/export overrides (#1074, #1083, #1106, #1141, #1156, #1188). +- `winml perf` — real `.npz` inputs, time-budgeted `--duration` runs, cached per-module builds, actual dynamic dimensions, and QNN profiler ONNX metrics (#1004, #1055, #1066, #1102, #1168). +- **QNN op tracing** — per-model tracing can be enabled automatically; basic traces exclude warmup samples and null fields, while detail tracing accepts compile EP options and auto-compiles raw ONNX inputs when required (#1006, #1032, #1249, #1252). +- `winml eval --mode compare` — compares two ONNX models directly or uses real `.npz` samples against a Hugging Face reference; Qwen3 adds perplexity evaluation (#1139, #1209, #1221). +- `winml optimize` / `winml analyze` — `--check-optim` previews applicable rewrites and verifies their produced operators against the target EP; new rewrites cover static Split-to-Slice and Conv affine/BatchNormalization folding (#1142, #1167, #1171, #1238, #1257). +- **EP discovery and monitoring** — registration is isolated and failures are structured, startup remains lazy, op-tracing dispatch is unified, and provider-download progress is restored (#1019, #1239). +- **CLI quality of life** — EP/device and pipeline-stage flags are consistent across commands; `--no-color` disables ANSI output for one invocation (#923, #978, #992). +- **Hub-hosted ONNX** — commands accept `//.onnx` references from Hugging Face Hub, enabling SAM 3 encoder/decoder workflows (#582). +- **Keypoint detection** — ViTPose supports `config`, `build`, and `perf`, plus COCO OKS-AP evaluation (#905, #949). +- **Vision and document recipes** — refreshed coverage adds DINOv2, SwinV2, OWL-ViT/OWLv2, BEiT, SegFormer, YOLOS, ViTPose/SynthPose, LayoutLM/LayoutLMv3, and document/question-answering models (#925, #1064, #1088, #1093, #1100, #1101, #1123, #1125, #1145, #1155, #1173, #1174, #1178, #1187, #1201, #1202, #1205, #1208). +- **Language recipes** — expanded BART, BERT, DeBERTa, DistilBERT, KoELECTRA, MiniLM, MPNet, Marian/OPUS, GTE reranker, feature-extraction, and entity-linking coverage (#1068, #1080, #1112, #1115, #1116, #1117, #1118, #1120, #1121, #1124, #1134, #1143, #1144, #1153, #1169, #1170, #1179, #1200, #1214). +- **Audio recipes** — expanded Wav2Vec2, HuBERT, AST, MMS, language/gender/music classification, forced alignment, and multilingual ASR coverage (#1094, #1095, #1114, #1131, #1148, #1154, #1176, #1186, #1206, #1207, #1211, #1225). + +### 🐛 Fixes + +- **`winml perf`** — throughput uses the batch size actually executed; analyzer EP resolution and op-trace paths now match the runtime target (#930, #941, #1000). +- **`winml build`** — honors explicit `--ep`, supports non-compiling cross-target builds, keeps ONNX caches distinct by resolved path and configuration, preserves configured model classes, reports disk-full failures clearly, and keeps CPU/GPU automatic precision at FP32 (#856, #947, #987, #997, #998). +- **GenAI and composite export** — fixed component export/build failures, compile fallback paths, accelerator selection, isolated EPContext preparation, and final Qwen3 bundle assembly (#1037, #1051, #1103, #1138, #1248). +- **Task and model resolution** — reconciled the task registry, corrected model-specific task listings and Hub `pipeline_tag` fallback, accepted composite tasks, and resolved CTC-based ASR model classes correctly (#724, #986, #1070, #1071, #1113, #1154). +- **Depth and keypoint evaluation** — fixed inference-time evaluator failures (#1023). +- **Analyzer and optimizer rules** — corrected coverage counting, aligned pattern checks with node support, consolidated recommendation metadata, and fixed dtype constraints and unknown-pattern handling (#922, #1020, #1063, #1130, #1162). +- **EP / device resolution** — WindowsML catalog providers register correctly; device listings retain hardware details without duplicate aliases; analyzer auto-selection prefers the strongest exact target; CPU bridge providers resolve safely; invalid EP/device pairs fail early (#1076, #1220, #1227, #1228, #1231, #1237). +- **Native EP execution** — hardened spawned-provider progress, prevented compiler-output deadlocks, released native sessions before process exit, and replaced pipe-backed warning capture with a bounded file spool to avoid EP compiler hangs (#1017, #1223, #1230, #1266, release cherry-pick #1267). +- **Export and quantization** — standalone quantization suppresses duplicate ORT preprocessing warnings; decoder KV-cache dimensions survive tracing; large external-data models can convert to FP16; and EPs that quantize internally no longer receive redundant WinML quantization (#956, #1176, #1235, #1242). +- **QNN evaluation and tracing** — repaired evaluation failures, detail-trace DLL/summary handling, and compile-time provider options (#1247, #1249). +- `winml eval` — default text datasets and sentiment recipes use fully qualified Hugging Face dataset IDs (#1262, release cherry-pick #1263). +- **CLI help** — `winml --help` shows the correct `sys` summary and concise, untruncated `build` / `quantize` descriptions (#1254). +- **Telemetry** — local `Path` model references no longer cause successful commands to fail during telemetry scrubbing (#1273). + +### 🔧 Internals & CI + +- **Release pipelines** — E2E aligns ModelKitArtifacts with the matching release branch, stable GitHub releases receive CHANGELOG notes and “Latest” status, and the official-build toolchain is pinned for reproducibility (#940, #967, #1268). +- **Evaluation CI** — recipe-driven build/eval supports per-EP matrices, pre-exported ONNX, reliable resume behavior, actual applied-precision reporting, unquantized-track EPs, and broader MIGraphX/TensorRT RTX coverage (#845, #902, #1009, #1039, #1086, #1160, #1163, #1226, #1243, #1286). +- **Telemetry** — action events record scrubbed model identifiers, while error events retain scrubbed root-cause details for diagnosis (#1108, #1111). +- **Development environment** — expanded type checking, added a tracked `uv` lockfile, selected CPU-only PyTorch wheels, and consolidated development dependencies (#932, #957, #1105, #1251, #1255). +- **Documentation publishing** — added and published the version-stamped model accuracy report from the current documentation site (#974, #975, #979, #1203). + +### 📦 Assets + +- `winml_cli-0.3.0-py3-none-any.whl` +- `rules-v0.3.0.zip` + ## WinML CLI v0.2.0 This cycle unifies **task detection** across the CLI (modality- and architecture-aware) and expands the eval and perf surfaces — new depth-estimation and tensor-similarity evaluators, a full SA eval pipeline with an HTML report, `winml perf --memory` / `--ep-options`, and `--format json` on `eval` / `analyze` / `perf`. `winml compile` gains a multi-model shared EP context, `winml build` gains `--precision`, and timm image-classification is supported. See the behavior changes below. diff --git a/pyproject.toml b/pyproject.toml index 65a07ab62..c1becfbb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ requires = [ "setuptools>=61", "wheel" ] [project] name = "winml-cli" -version = "0.2.0" +version = "0.3.0" description = "Accelerate Model Deployment on WinML" readme = "README.md" keywords = [ "onnx", "winml" ] diff --git a/src/winml/modelkit/telemetry/utils.py b/src/winml/modelkit/telemetry/utils.py index c2a1a6ab0..f90e29bac 100644 --- a/src/winml/modelkit/telemetry/utils.py +++ b/src/winml/modelkit/telemetry/utils.py @@ -171,7 +171,9 @@ def _model_ref_local_marker(value: str) -> str: return f"" if ext else "" -def _scrub_model_ref(value: str | tuple[str, ...] | None) -> str | None: +def _scrub_model_ref( + value: str | os.PathLike[str] | tuple[str | os.PathLike[str], ...] | None, +) -> str | None: """Classify a ``-m/--model`` reference for telemetry. Clean HuggingFace Hub ids — one or two Hub-charset segments with at most @@ -185,6 +187,7 @@ def _scrub_model_ref(value: str | tuple[str, ...] | None) -> str | None: value = value[0] if value else None if not value: return None + value = os.fspath(value) normalized = value.replace("\\", "/") if re.match(r"^[A-Za-z]:[\\/]", value) or normalized.startswith("/"): return _model_ref_local_marker(value) diff --git a/tests/e2e/test_perf_e2e.py b/tests/e2e/test_perf_e2e.py index b9ec8f758..1a189f265 100644 --- a/tests/e2e/test_perf_e2e.py +++ b/tests/e2e/test_perf_e2e.py @@ -626,12 +626,12 @@ def test_benchmark_ep_device_gpu(self, ep: str, tmp_path: Path, gpu_model_arg: s assert output_file.exists() data = json.loads(output_file.read_text()) - # TODO openvino gpu could not emit valid pdh counter + # TODO OpenVINO GPU and TensorRT RTX do not emit reliable PDH utilization counters. _assert_monitor_result( data, device="gpu", ep=EP_ALIASES[ep], - require_utilization=ep != "openvino", + require_utilization=ep not in ("nv_tensorrt_rtx", "openvino"), ) @pytest.mark.parametrize("ep", NPU_EPS) @@ -886,8 +886,12 @@ def test_benchmark_ep_gpu(self, ep: str, tmp_path: Path, model_arg: str): assert output_file.exists() data = json.loads(output_file.read_text()) assert data["benchmark_info"]["ep"] == EP_ALIASES[ep] - # TODO openvino gpu could not emit valid pdh counter - _assert_monitor_result(data, device="gpu", require_utilization=ep != "openvino") + # TODO OpenVINO GPU and TensorRT RTX do not emit reliable PDH utilization counters. + _assert_monitor_result( + data, + device="gpu", + require_utilization=ep not in ("nv_tensorrt_rtx", "openvino"), + ) @pytest.mark.parametrize("ep", NPU_EPS) def test_benchmark_ep_npu(self, ep: str, tmp_path: Path, model_arg: str): diff --git a/tests/unit/telemetry/test_click_group.py b/tests/unit/telemetry/test_click_group.py index 8f2dfb397..d19b2277c 100644 --- a/tests/unit/telemetry/test_click_group.py +++ b/tests/unit/telemetry/test_click_group.py @@ -6,6 +6,7 @@ """Tests for ``ActionGroup`` — the Click ``Group`` subclass that auto-instruments every registered subcommand with WinML CLI telemetry.""" +from pathlib import Path from unittest.mock import MagicMock import click @@ -293,6 +294,29 @@ def perf(model): assert dict(action_record.attributes)["model_id"] == expected_model_id +def test_action_accepts_path_typed_model(enabled_telemetry, tmp_path): + @click.group(cls=ActionGroup) + def cli(): + pass + + @cli.command() + @click.option("-m", "--model", type=click.Path(exists=True, path_type=Path)) + def analyze(model): + (tmp_path / "analysis.json").write_text("{}") + + model_path = tmp_path / "model.onnx" + model_path.write_bytes(b"") + telemetry = Telemetry.get_or_init() + mock_logger = _with_mock_logger(telemetry) + + result = CliRunner().invoke(cli, ["analyze", "-m", str(model_path)]) + + assert (tmp_path / "analysis.json").exists() + assert result.exit_code == 0 + action_record = mock_logger.emit.call_args_list[1].args[0] + assert dict(action_record.attributes)["model_id"] == "" + + def test_action_prefers_model_id_param(enabled_telemetry): """When a command exposes ``--model-id`` (eval/quantize), that clean HF id is recorded directly, bypassing the scrubbed ``-m`` value.""" diff --git a/uv.lock b/uv.lock index 8a529bff1..aaa2643bd 100644 --- a/uv.lock +++ b/uv.lock @@ -3219,7 +3219,7 @@ wheels = [ [[package]] name = "winml-cli" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "click", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },