From e1d6f8748b8afe36f123496c303494ce2ccd9ab2 Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Fri, 31 Jul 2026 14:15:29 +0800 Subject: [PATCH 1/6] fix(datasets): use fully qualified HF dataset ids (#1262) [cherry-pick to v0.3.0] (#1263) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cherry-picks #1262 into the v0.3.0 release branch. #1262 landed on main after `release/v0.3.0` was cut, so it is not in the release branch. Its parent commit is the release branch tip (`16152fd4`), so this is a clean pick with no conflicts — the diff here is byte-identical to #1262. **Original:** fix(datasets): use fully qualified HF dataset ids (#1262) by @zhenchaoni Co-authored-by: Zhenchao Ni Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../text-classification_fp16_config.json | 2 +- .../text-classification_w8a16_config.json | 2 +- scripts/e2e_eval/cache/baseline_cache.json | 8 ++++---- scripts/e2e_eval/testsets/models_with_acc.json | 2 +- src/winml/modelkit/commands/eval.py | 2 +- src/winml/modelkit/datasets/text.py | 11 ++++++++--- src/winml/modelkit/eval/config.py | 2 +- .../integration/datasets/test_text_classification.py | 10 +++++----- tests/unit/eval/test_eval.py | 6 +++--- 9 files changed, 25 insertions(+), 20 deletions(-) diff --git a/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_fp16_config.json b/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_fp16_config.json index 8de0a638d..683080239 100644 --- a/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_fp16_config.json +++ b/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_fp16_config.json @@ -53,7 +53,7 @@ "eval": { "task": "text-classification", "dataset": { - "path": "tweet_eval", + "path": "cardiffnlp/tweet_eval", "name": "sentiment", "samples": 100, "columns_mapping": { diff --git a/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_w8a16_config.json b/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_w8a16_config.json index eeb1fff26..cd55911db 100644 --- a/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_w8a16_config.json +++ b/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_w8a16_config.json @@ -70,7 +70,7 @@ "eval": { "task": "text-classification", "dataset": { - "path": "tweet_eval", + "path": "cardiffnlp/tweet_eval", "name": "sentiment", "samples": 1000, "columns_mapping": { diff --git a/scripts/e2e_eval/cache/baseline_cache.json b/scripts/e2e_eval/cache/baseline_cache.json index 8acbbe305..7590060d4 100644 --- a/scripts/e2e_eval/cache/baseline_cache.json +++ b/scripts/e2e_eval/cache/baseline_cache.json @@ -9,7 +9,7 @@ "elapsed": 59.2, "command": "python.exe run_pytorch_baseline.py --model finbert --task text-classification --device cpu --num-samples 1000 --dataset finbert_dataset --split val" }, - "cardiffnlp/twitter-roberta-base-sentiment-latest|text-classification|tweet_eval|sentiment||1000": { + "cardiffnlp/twitter-roberta-base-sentiment-latest|text-classification|cardiffnlp/tweet_eval|sentiment||1000": { "status": "PASS", "metric": { "metric": "accuracy", @@ -17,7 +17,7 @@ "num_samples": 1000 }, "elapsed": 91.5, - "command": "python.exe run_pytorch_baseline.py --model twitter-roberta-base-sentiment-latest --task text-classification --device cpu --num-samples 1000 --dataset tweet_eval --dataset-config sentiment --columns-mapping {\"input_column\": \"text\"}" + "command": "python.exe run_pytorch_baseline.py --model twitter-roberta-base-sentiment-latest --task text-classification --device cpu --num-samples 1000 --dataset cardiffnlp/tweet_eval --dataset-config sentiment --columns-mapping {\"input_column\": \"text\"}" }, "distilbert/distilbert-base-uncased-finetuned-sst-2-english|text-classification|nyu-mll/glue|sst2||1000": { "status": "PASS", @@ -959,7 +959,7 @@ "elapsed": 52.0, "command": "python.exe run_pytorch_baseline.py --model finbert --task text-classification --device cpu --num-samples 100 --dataset finbert_dataset --split val --winml-metric-key accuracy" }, - "cardiffnlp/twitter-roberta-base-sentiment-latest|text-classification|tweet_eval|sentiment||100": { + "cardiffnlp/twitter-roberta-base-sentiment-latest|text-classification|cardiffnlp/tweet_eval|sentiment||100": { "status": "PASS", "metric": { "metric": "accuracy", @@ -967,7 +967,7 @@ "num_samples": 100 }, "elapsed": 54.9, - "command": "python.exe run_pytorch_baseline.py --model twitter-roberta-base-sentiment-latest --task text-classification --device cpu --num-samples 100 --dataset tweet_eval --dataset-config sentiment --columns-mapping {\"input_column\": \"text\"} --winml-metric-key accuracy" + "command": "python.exe run_pytorch_baseline.py --model twitter-roberta-base-sentiment-latest --task text-classification --device cpu --num-samples 100 --dataset cardiffnlp/tweet_eval --dataset-config sentiment --columns-mapping {\"input_column\": \"text\"} --winml-metric-key accuracy" }, "distilbert/distilbert-base-uncased-finetuned-sst-2-english|text-classification|nyu-mll/glue|sst2||100": { "status": "PASS", diff --git a/scripts/e2e_eval/testsets/models_with_acc.json b/scripts/e2e_eval/testsets/models_with_acc.json index e21423161..03a136206 100644 --- a/scripts/e2e_eval/testsets/models_with_acc.json +++ b/scripts/e2e_eval/testsets/models_with_acc.json @@ -18,7 +18,7 @@ "group": "Top200", "priority": "P1", "dataset_config": { - "path": "tweet_eval", + "path": "cardiffnlp/tweet_eval", "name": "sentiment", "metric": "accuracy", "columns_mapping": { diff --git a/src/winml/modelkit/commands/eval.py b/src/winml/modelkit/commands/eval.py index 07f1af61c..68d0c3e1c 100644 --- a/src/winml/modelkit/commands/eval.py +++ b/src/winml/modelkit/commands/eval.py @@ -45,7 +45,7 @@ "dataset_path", type=str, default=None, - help="HF dataset path (e.g. 'imagenet-1k', 'glue'). " + help="HF dataset path (e.g. 'imagenet-1k', 'nyu-mll/glue'). " "If omitted, uses a default dataset for the task.", ) @click.option( diff --git a/src/winml/modelkit/datasets/text.py b/src/winml/modelkit/datasets/text.py index e44f115e2..f3f709d69 100644 --- a/src/winml/modelkit/datasets/text.py +++ b/src/winml/modelkit/datasets/text.py @@ -27,6 +27,11 @@ logger = logging.getLogger(__name__) +# HF requires fully qualified repository ids ("namespace/name"); the legacy +# canonical alias "glue" is no longer resolvable. +DEFAULT_TEXT_DATASET = "nyu-mll/glue" +DEFAULT_TEXT_DATASET_SUBSET = "mrpc" + class TextDataset(BaseTaskDataset): """Dataset for text tasks with universal tokenization. @@ -56,7 +61,7 @@ def __init__( Args: model_name: HuggingFace model identifier - dataset_name: Dataset name (default: glue) + dataset_name: Dataset name (default: nyu-mll/glue) max_samples: Maximum samples (None = use all) data_split: Dataset split (default: train) max_length: Sequence length (default: from io_config or 128) @@ -85,8 +90,8 @@ def __init__( def _get_default_dataset(self) -> None: """Set default dataset if none specified.""" if self._dataset_name is None: - self._dataset_name = "glue" - self._config["subset"] = self._config.get("subset", "mrpc") + self._dataset_name = DEFAULT_TEXT_DATASET + self._config["subset"] = self._config.get("subset", DEFAULT_TEXT_DATASET_SUBSET) self._data_split = self._data_split or "train" def _resolve_max_length(self) -> None: diff --git a/src/winml/modelkit/eval/config.py b/src/winml/modelkit/eval/config.py index 1717ccdb5..03944010a 100644 --- a/src/winml/modelkit/eval/config.py +++ b/src/winml/modelkit/eval/config.py @@ -20,7 +20,7 @@ class DatasetConfig: """Dataset configuration, aligned with HF load_dataset() API. Attributes: - path: HF dataset path (e.g., "imagenet-1k", "glue"). + path: HF dataset path (e.g., "imagenet-1k", "nyu-mll/glue"). name: Config name for multi-config datasets (e.g., "mrpc"). split: Dataset split. samples: Number of samples to evaluate. diff --git a/tests/integration/datasets/test_text_classification.py b/tests/integration/datasets/test_text_classification.py index 9559d4006..d1a291d63 100644 --- a/tests/integration/datasets/test_text_classification.py +++ b/tests/integration/datasets/test_text_classification.py @@ -44,7 +44,7 @@ def test_default_seq_len_constant(self): assert TextDataset.DEFAULT_SEQ_LEN == 128 def test_default_dataset_glue_mrpc(self): - """Test default dataset is glue/mrpc when none specified.""" + """Test default dataset is nyu-mll/glue with the mrpc subset.""" from winml.modelkit.datasets import TextDataset dataset = TextDataset( @@ -52,7 +52,7 @@ def test_default_dataset_glue_mrpc(self): max_samples=5, ) - assert dataset.dataset_name == "glue" + assert dataset.dataset_name == "nyu-mll/glue" assert dataset.data_split == "train" def test_explicit_dataset_name(self): @@ -61,13 +61,13 @@ def test_explicit_dataset_name(self): dataset = TextDataset( model_name="bert-base-uncased", - dataset_name="glue", + dataset_name="nyu-mll/glue", data_split="validation", max_samples=5, subset="sst2", ) - assert dataset.dataset_name == "glue" + assert dataset.dataset_name == "nyu-mll/glue" assert dataset.data_split == "validation" def test_max_samples_limits_dataset_size(self): @@ -274,7 +274,7 @@ def test_single_sentence_detection_sst2(self): # GLUE/SST2 is a single sentence task dataset = TextDataset( model_name="bert-base-uncased", - dataset_name="glue", + dataset_name="nyu-mll/glue", data_split="train", max_samples=5, subset="sst2", diff --git a/tests/unit/eval/test_eval.py b/tests/unit/eval/test_eval.py index 6c5bb469b..dbf93b72e 100644 --- a/tests/unit/eval/test_eval.py +++ b/tests/unit/eval/test_eval.py @@ -752,7 +752,7 @@ def _fake_compute(**kw): model_id="test/model", task="text-classification", dataset=DatasetConfig( - path="glue", + path="nyu-mll/glue", name="mrpc", columns_mapping={"input_column": "sentence1", "second_input_column": "sentence2"}, ), @@ -803,7 +803,7 @@ def test_sets_padding_for_text_model( config = WinMLEvaluationConfig( model_id="test/model", task="text-classification", - dataset=DatasetConfig(path="glue", name="mrpc"), + dataset=DatasetConfig(path="nyu-mll/glue", name="mrpc"), ) WinMLTextClassificationEvaluator(config, model).compute() @@ -847,7 +847,7 @@ def test_no_padding_without_tokenizer( config = WinMLEvaluationConfig( model_id="test/model", task="text-classification", - dataset=DatasetConfig(path="glue"), + dataset=DatasetConfig(path="nyu-mll/glue"), ) WinMLTextClassificationEvaluator(config, model).compute() From b470c218e6dffdcb10b34b67a50035de79224aa7 Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Mon, 3 Aug 2026 10:32:48 +0800 Subject: [PATCH 2/6] Spool native warnings without pipes during EP compilation (#1266) [cherry-pick to v0.3.0] (#1267) Cherry-picks #1266 into the v0.3.0 release branch. This replaces pipe-backed native warning capture with a temporary-file spool, avoiding EP compiler hangs while preserving warning filtering and bounded memory use. Original commit: `e27a0faf` by @KayMKM. Cherry-picked cleanly onto release tip `e1d6f874` as `ce11819c`. --- src/winml/modelkit/utils/native_stderr.py | 79 ++++++++------------- tests/unit/utils/test_native_stderr.py | 84 ++++++----------------- 2 files changed, 50 insertions(+), 113 deletions(-) diff --git a/src/winml/modelkit/utils/native_stderr.py b/src/winml/modelkit/utils/native_stderr.py index 598e03963..af723e1ef 100644 --- a/src/winml/modelkit/utils/native_stderr.py +++ b/src/winml/modelkit/utils/native_stderr.py @@ -10,7 +10,7 @@ * ``suppress_native_stderr`` - discard to devnull (startup noise) * ``capture_native_stderr`` - capture via pipe and re-log (compilation output) -* ``suppress_native_warnings`` - hide warning lines, replay everything else +* ``suppress_native_warnings`` - spool to disk, hide warnings, replay other lines The first two are no-ops on non-Windows. ``suppress_native_warnings`` works via fd 2 on all platforms and also keeps the Win32 stderr handle in sync on Windows. @@ -22,8 +22,9 @@ import os import re import sys +import tempfile import threading -from contextlib import contextmanager +from contextlib import ExitStack, contextmanager from pathlib import Path from typing import TYPE_CHECKING, cast @@ -198,19 +199,25 @@ def suppress_native_warnings( ``[E:...]``. This process-wide fd redirect is opt-in; CLI entry points pass ``enabled=True`` only around native-heavy work. ``-v`` / ``-vv`` or ``WINMLCLI_SHOW_ALL_WARNINGS=1`` leaves stderr untouched. + + Output is spooled to a temporary file and replayed only after fd 2 is + restored. A pipe is deliberately not used here: some native execution- + provider compilers change behavior or hang when stderr is a pipe, even when + that pipe is drained concurrently. A file also bounds Python memory usage + without imposing a finite producer buffer. """ if not enabled or _show_native_warnings_requested(): yield return - read_fd: int | None = None - write_fd: int | None = None old_fd: int | None = None + capture_stack = ExitStack() try: - read_fd, write_fd = os.pipe() + capture = capture_stack.enter_context( + tempfile.TemporaryFile(mode="w+b") # noqa: SIM115 - owned by ExitStack + ) except OSError: - _close_fd(read_fd) - _close_fd(write_fd) + capture_stack.close() logger.debug( "Native warning suppression setup failed; leaving stderr unchanged", exc_info=True, @@ -223,8 +230,7 @@ def suppress_native_warnings( try: old_fd = os.dup(2) except OSError: - _close_fd(read_fd) - _close_fd(write_fd) + capture_stack.close() logger.debug( "Native warning suppression setup failed; leaving stderr unchanged", exc_info=True, @@ -232,17 +238,10 @@ def suppress_native_warnings( redirect_failed = True if not redirect_failed: assert old_fd is not None - reader = threading.Thread( - target=_drain_filtered_native_stderr, - args=(read_fd, old_fd, preserve_unclassified), - name="suppress-native-warnings", - daemon=True, - ) try: - os.dup2(write_fd, 2) + os.dup2(capture.fileno(), 2) except OSError: - _close_fd(read_fd) - _close_fd(write_fd) + capture_stack.close() _close_fd(old_fd) logger.debug( "Native warning suppression redirect failed; leaving stderr unchanged", @@ -250,22 +249,26 @@ def suppress_native_warnings( ) redirect_failed = True if not redirect_failed: - _close_fd(write_fd) _set_win32_std_handle_to_current_fd(2) - reader.start() try: yield finally: _restore_redirected_fd(2, old_fd) _set_win32_std_handle_to_current_fd(2) _refresh_click_windows_console_stream(2) - reader.join(timeout=_NATIVE_READER_JOIN_TIMEOUT_SECONDS) - if reader.is_alive(): + try: + capture.flush() + capture.seek(0) + for line in capture: + if _should_preserve_native_line(line, preserve_unclassified): + _write_all(old_fd, line) + except OSError: logger.debug( - "Native warning suppression reader did not finish after stderr restore" + "Could not replay filtered native stderr", + exc_info=True, ) - else: - _close_fd(old_fd) + capture_stack.close() + _close_fd(old_fd) if redirect_failed: yield @@ -307,32 +310,6 @@ def _restore_redirected_fd(fd: int, old_fd: int | None) -> None: ) -def _drain_filtered_native_stderr( - read_fd: int, - target_fd: int, - preserve_unclassified: bool, -) -> None: - pending = b"" - try: - while chunk := os.read(read_fd, 4096): - pending += chunk - while b"\n" in pending: - line, pending = pending.split(b"\n", 1) - _write_non_warning_line(target_fd, line + b"\n", preserve_unclassified) - if pending: - _write_non_warning_line(target_fd, pending, preserve_unclassified) - except OSError: - # fd redirection is best-effort cleanup; restore path handles usability. - pass - finally: - os.close(read_fd) - - -def _write_non_warning_line(fd: int, line: bytes, preserve_unclassified: bool) -> None: - if _should_preserve_native_line(line, preserve_unclassified): - _write_all(fd, line) - - def _should_preserve_native_line(line: bytes, preserve_unclassified: bool) -> bool: match = _NATIVE_SEVERITY_TOKEN_RE.search(line) if match is not None: diff --git a/tests/unit/utils/test_native_stderr.py b/tests/unit/utils/test_native_stderr.py index 22cb2a56d..c1905cc70 100644 --- a/tests/unit/utils/test_native_stderr.py +++ b/tests/unit/utils/test_native_stderr.py @@ -243,6 +243,23 @@ def test_filters_native_warning_lines_and_preserves_errors(self, monkeypatch, ca assert "useful error" in stderr assert "plain diagnostic" in stderr + def test_uses_file_backed_capture_instead_of_pipe(self, monkeypatch): + monkeypatch.delenv("WINMLCLI_SHOW_ALL_WARNINGS", raising=False) + logging.getLogger().setLevel(logging.WARNING) + pipe_called = False + + def fail_pipe() -> tuple[int, int]: + nonlocal pipe_called + pipe_called = True + raise AssertionError("warning suppression must not create a pipe") + + monkeypatch.setattr(native_stderr_module.os, "pipe", fail_pipe) + + with native_stderr_module.suppress_native_warnings(enabled=True): + os.write(2, b"2026 [W:custom-native:, file.cc:1 WarningFunc] hidden\n") + + assert pipe_called is False + def test_filters_native_prefix_info_without_dropping_python_stderr(self, monkeypatch, capfd): monkeypatch.delenv("WINMLCLI_SHOW_ALL_WARNINGS", raising=False) logging.getLogger().setLevel(logging.WARNING) @@ -299,14 +316,14 @@ def test_show_all_warnings_env_leaves_native_warnings_visible(self, monkeypatch, assert "env warning" in capfd.readouterr().err - def test_pipe_setup_failure_does_not_abort_wrapped_code(self, monkeypatch): + def test_capture_setup_failure_does_not_abort_wrapped_code(self, monkeypatch): monkeypatch.delenv("WINMLCLI_SHOW_ALL_WARNINGS", raising=False) logging.getLogger().setLevel(logging.WARNING) - def fail_pipe() -> tuple[int, int]: + def fail_capture(*args: object, **kwargs: object): raise OSError(1, "Incorrect function") - monkeypatch.setattr(native_stderr_module.os, "pipe", fail_pipe) + monkeypatch.setattr(native_stderr_module.tempfile, "TemporaryFile", fail_capture) ran = False with native_stderr_module.suppress_native_warnings(enabled=True): @@ -314,37 +331,21 @@ def fail_pipe() -> tuple[int, int]: assert ran - def test_restore_failure_closes_redirected_fd_and_bounds_reader_join(self, monkeypatch): + def test_restore_failure_closes_redirected_fd(self, monkeypatch): monkeypatch.delenv("WINMLCLI_SHOW_ALL_WARNINGS", raising=False) logging.getLogger().setLevel(logging.WARNING) closed: list[int] = [] - join_timeouts: list[float | None] = [] dup2_calls = 0 - class FakeThread: - def __init__(self, *args: object, **kwargs: object) -> None: - pass - - def start(self) -> None: - pass - - def join(self, timeout: float | None = None) -> None: - join_timeouts.append(timeout) - - def is_alive(self) -> bool: - return False - def fake_dup2(src: int, dst: int) -> None: nonlocal dup2_calls dup2_calls += 1 if dup2_calls == 2: raise OSError(1, "restore failed") - monkeypatch.setattr(native_stderr_module.os, "pipe", lambda: (10, 11)) monkeypatch.setattr(native_stderr_module.os, "dup", lambda fd: 12) monkeypatch.setattr(native_stderr_module.os, "dup2", fake_dup2) monkeypatch.setattr(native_stderr_module.os, "close", lambda fd: closed.append(fd)) - monkeypatch.setattr(native_stderr_module.threading, "Thread", FakeThread) monkeypatch.setattr( native_stderr_module, "_set_win32_std_handle_to_current_fd", @@ -360,48 +361,7 @@ def fake_dup2(src: int, dst: int) -> None: pass assert 2 in closed - assert join_timeouts == [native_stderr_module._NATIVE_READER_JOIN_TIMEOUT_SECONDS] - - def test_reader_owned_old_fd_stays_open_when_reader_outlives_join(self, monkeypatch): - monkeypatch.delenv("WINMLCLI_SHOW_ALL_WARNINGS", raising=False) - logging.getLogger().setLevel(logging.WARNING) - closed: list[int] = [] - join_timeouts: list[float | None] = [] - - class FakeThread: - def __init__(self, *args: object, **kwargs: object) -> None: - pass - - def start(self) -> None: - pass - - def join(self, timeout: float | None = None) -> None: - join_timeouts.append(timeout) - - def is_alive(self) -> bool: - return True - - monkeypatch.setattr(native_stderr_module.os, "pipe", lambda: (10, 11)) - monkeypatch.setattr(native_stderr_module.os, "dup", lambda fd: 12) - monkeypatch.setattr(native_stderr_module.os, "dup2", lambda src, dst: None) - monkeypatch.setattr(native_stderr_module.os, "close", lambda fd: closed.append(fd)) - monkeypatch.setattr(native_stderr_module.threading, "Thread", FakeThread) - monkeypatch.setattr( - native_stderr_module, - "_set_win32_std_handle_to_current_fd", - lambda fd: None, - ) - monkeypatch.setattr( - native_stderr_module, - "_refresh_click_windows_console_stream", - lambda fd, handle=None: None, - ) - - with native_stderr_module.suppress_native_warnings(enabled=True): - pass - - assert join_timeouts == [native_stderr_module._NATIVE_READER_JOIN_TIMEOUT_SECONDS] - assert 12 not in closed + assert 12 in closed @pytest.mark.skipif(sys.platform != "win32", reason="Win32 only") def test_win32_std_handle_sync_failure_does_not_abort_wrapped_code(self, monkeypatch): From b538ce3878534b38e6269d98800ec9e1c862ed7f Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Mon, 3 Aug 2026 11:55:09 +0800 Subject: [PATCH 3/6] chore+ci: bump version to 0.3.0 and pin official build toolchain (#1268) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Release-window changes needed to build a correct v0.3.0 wheel. ### `chore(release): bump version to 0.3.0` (+ uv.lock sync) Bumps `pyproject.toml` (and `uv.lock`) to `0.3.0`. Version is exposed at runtime via `importlib.metadata`, so `pyproject.toml` is the only source. ### `ci(release): pin official build toolchain versions` `ModelKit Official Build` installed its toolchain with `pip install --upgrade build twine packaging` — completely unpinned. `build` 1.5.1 (released since v0.2.0) breaks the build inside the OneBranch container: ``` platformdirs\windows.py -> get_win_folder_from_registry winreg.QueryValueEx(key, CSIDL_LOCAL_APPDATA) -> FileNotFoundError: [WinError 2] The system cannot find the file specified ``` The sdist step fails and the wheel step is skipped, so the build produces nothing. Verified on run [153739315](https://dev.azure.com/microsoft/windows.ai.toolkit/_build/results?buildId=153739315) (failed, `build` 1.5.1) vs [150550609](https://dev.azure.com/microsoft/windows.ai.toolkit/_build/results?buildId=150550609) (v0.2.0, succeeded, `build` 1.5.0). `build` 1.5.0 and 1.5.1 have identical `_pip_env()`, so this is environment behavior in 1.5.1, not a change on our side. Pinning to the versions that produced v0.2.0 restores a green build and, more importantly, stops a third-party release from breaking the release build on publish day. Validated: official build [153740145](https://dev.azure.com/microsoft/windows.ai.toolkit/_build/results?buildId=153740145) succeeded on this branch — sdist, wheel, iKey verification, and PyPI validation all green. --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .pipelines/modelkit-official-build.yml | 9 ++++++++- pyproject.toml | 2 +- uv.lock | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.pipelines/modelkit-official-build.yml b/.pipelines/modelkit-official-build.yml index 2b7ca7d89..279503819 100644 --- a/.pipelines/modelkit-official-build.yml +++ b/.pipelines/modelkit-official-build.yml @@ -127,7 +127,14 @@ extends: artifactFeeds: 'windows.ai.toolkit/Modelkit' displayName: 'Authenticate pip with Azure Artifacts' - - script: python -m pip install --upgrade build twine packaging + # PINNED ON PURPOSE — do NOT relax to unpinned/--upgrade. + # An unpinned toolchain lets a third-party release break the + # release build on publish day. build 1.5.1 did exactly that: + # its isolated-env pip invocation fails inside this container + # with "FileNotFoundError: [WinError 2]" while resolving the + # user cache dir from the registry. These are the versions that + # produced the v0.2.0 release. Bump deliberately, never silently. + - script: python -m pip install build==1.5.0 twine==6.2.0 packaging==26.2 displayName: 'Install build tools' # Build sdist BEFORE iKey injection so the source archive diff --git a/pyproject.toml b/pyproject.toml index 65a07ab62..c1becfbb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ requires = [ "setuptools>=61", "wheel" ] [project] name = "winml-cli" -version = "0.2.0" +version = "0.3.0" description = "Accelerate Model Deployment on WinML" readme = "README.md" keywords = [ "onnx", "winml" ] diff --git a/uv.lock b/uv.lock index 482d47149..26d508307 100644 --- a/uv.lock +++ b/uv.lock @@ -3218,7 +3218,7 @@ wheels = [ [[package]] name = "winml-cli" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "click" }, From 4df4a043353167e38f17c83255af3fb8e31e9b19 Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Mon, 3 Aug 2026 16:46:46 +0800 Subject: [PATCH 4/6] Fix telemetry handling for local model paths (#1273) ## Summary - Normalize PathLike model references before telemetry string scrubbing. - Prevent successful commands using local model paths from exiting with a Path.replace TypeError during telemetry reporting. - Add regression coverage for Click options parsed with path_type=Path. - Cherry-picked as a single focused commit onto release/v0.3.0. ## Validation - uv run pytest tests/unit/telemetry/test_click_group.py tests/unit/telemetry/test_utils_scrubbing.py -q (63 passed) - uv run ruff check src/winml/modelkit/telemetry/utils.py tests/unit/telemetry/test_click_group.py --- src/winml/modelkit/telemetry/utils.py | 5 ++++- tests/unit/telemetry/test_click_group.py | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/winml/modelkit/telemetry/utils.py b/src/winml/modelkit/telemetry/utils.py index c2a1a6ab0..f90e29bac 100644 --- a/src/winml/modelkit/telemetry/utils.py +++ b/src/winml/modelkit/telemetry/utils.py @@ -171,7 +171,9 @@ def _model_ref_local_marker(value: str) -> str: return f"" if ext else "" -def _scrub_model_ref(value: str | tuple[str, ...] | None) -> str | None: +def _scrub_model_ref( + value: str | os.PathLike[str] | tuple[str | os.PathLike[str], ...] | None, +) -> str | None: """Classify a ``-m/--model`` reference for telemetry. Clean HuggingFace Hub ids — one or two Hub-charset segments with at most @@ -185,6 +187,7 @@ def _scrub_model_ref(value: str | tuple[str, ...] | None) -> str | None: value = value[0] if value else None if not value: return None + value = os.fspath(value) normalized = value.replace("\\", "/") if re.match(r"^[A-Za-z]:[\\/]", value) or normalized.startswith("/"): return _model_ref_local_marker(value) diff --git a/tests/unit/telemetry/test_click_group.py b/tests/unit/telemetry/test_click_group.py index 8f2dfb397..d19b2277c 100644 --- a/tests/unit/telemetry/test_click_group.py +++ b/tests/unit/telemetry/test_click_group.py @@ -6,6 +6,7 @@ """Tests for ``ActionGroup`` — the Click ``Group`` subclass that auto-instruments every registered subcommand with WinML CLI telemetry.""" +from pathlib import Path from unittest.mock import MagicMock import click @@ -293,6 +294,29 @@ def perf(model): assert dict(action_record.attributes)["model_id"] == expected_model_id +def test_action_accepts_path_typed_model(enabled_telemetry, tmp_path): + @click.group(cls=ActionGroup) + def cli(): + pass + + @cli.command() + @click.option("-m", "--model", type=click.Path(exists=True, path_type=Path)) + def analyze(model): + (tmp_path / "analysis.json").write_text("{}") + + model_path = tmp_path / "model.onnx" + model_path.write_bytes(b"") + telemetry = Telemetry.get_or_init() + mock_logger = _with_mock_logger(telemetry) + + result = CliRunner().invoke(cli, ["analyze", "-m", str(model_path)]) + + assert (tmp_path / "analysis.json").exists() + assert result.exit_code == 0 + action_record = mock_logger.emit.call_args_list[1].args[0] + assert dict(action_record.attributes)["model_id"] == "" + + def test_action_prefers_model_id_param(enabled_telemetry): """When a command exposes ``--model-id`` (eval/quantize), that clean HF id is recorded directly, bypassing the scrubbed ``-m`` value.""" From faee05fccc8bdf4fd578a30149dc11298ddf9560 Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Thu, 6 Aug 2026 16:09:03 +0800 Subject: [PATCH 5/6] test(perf): relax TensorRT RTX PDH assertion (#1286) ## Summary - Treat TensorRT RTX like OpenVINO when validating Windows PDH GPU utilization. - Keep the benchmark, execution provider, device, adapter LUID, latency, and monitor structure assertions intact. - Document the unreliable PDH utilization behavior with a TODO in both ONNX-direct and Hugging Face GPU cases. ## Validation - `uv run --no-sync pytest -m e2e -q "tests/e2e/test_perf_e2e.py::TestPerfONNXDirect::test_benchmark_ep_device_gpu[nv_tensorrt_rtx]" "tests/e2e/test_perf_e2e.py::TestPerfHuggingFace::test_benchmark_ep_gpu[nv_tensorrt_rtx]" --basetemp=temp/pytest_tmp/release-v030-trtrtx-monitor-fix` (2 passed) - `uv run --no-sync ruff check --fix tests/e2e/test_perf_e2e.py` --- tests/e2e/test_perf_e2e.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/e2e/test_perf_e2e.py b/tests/e2e/test_perf_e2e.py index b9ec8f758..1a189f265 100644 --- a/tests/e2e/test_perf_e2e.py +++ b/tests/e2e/test_perf_e2e.py @@ -626,12 +626,12 @@ def test_benchmark_ep_device_gpu(self, ep: str, tmp_path: Path, gpu_model_arg: s assert output_file.exists() data = json.loads(output_file.read_text()) - # TODO openvino gpu could not emit valid pdh counter + # TODO OpenVINO GPU and TensorRT RTX do not emit reliable PDH utilization counters. _assert_monitor_result( data, device="gpu", ep=EP_ALIASES[ep], - require_utilization=ep != "openvino", + require_utilization=ep not in ("nv_tensorrt_rtx", "openvino"), ) @pytest.mark.parametrize("ep", NPU_EPS) @@ -886,8 +886,12 @@ def test_benchmark_ep_gpu(self, ep: str, tmp_path: Path, model_arg: str): assert output_file.exists() data = json.loads(output_file.read_text()) assert data["benchmark_info"]["ep"] == EP_ALIASES[ep] - # TODO openvino gpu could not emit valid pdh counter - _assert_monitor_result(data, device="gpu", require_utilization=ep != "openvino") + # TODO OpenVINO GPU and TensorRT RTX do not emit reliable PDH utilization counters. + _assert_monitor_result( + data, + device="gpu", + require_utilization=ep not in ("nv_tensorrt_rtx", "openvino"), + ) @pytest.mark.parametrize("ep", NPU_EPS) def test_benchmark_ep_npu(self, ep: str, tmp_path: Path, model_arg: str): From 90dd75cbbf8a4c8ddd03f611431e676e01b59665 Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Thu, 6 Aug 2026 16:27:12 +0800 Subject: [PATCH 6/6] docs(changelog): add v0.3.0 release notes (#1287) Adds the v0.3.0 CHANGELOG entry, grouped by user-visible behavior changes, improvements, fixes, internals, and release assets. --- CHANGELOG.md | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68fb2f1f5..2f09a2913 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,65 @@ All notable changes to this project are documented in this file. The format is loosely based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## WinML CLI v0.3.0 + +This cycle expands **model preparation and evaluation** across the CLI: precision-driven quantization, composite-model and dynamic-axis workflows, real-input perf/eval, optimization previews, and opt-in Dynamo export. It also introduces one-command Qwen3 onnxruntime-genai bundles, GenAI benchmarking, broader model recipes, and more reliable EP discovery, compilation, and monitoring. See the behavior changes below. + +### ⚠️ Behavior changes + +- **Output-producing commands** now refuse to replace existing files or non-empty directories unless `--overwrite` is passed; `winml build` retains its existing incremental `--rebuild` behavior (#970). +- `winml quantize` renames `--model-name` to `--model-id`, including the corresponding quantization-config field (#984). +- **Compile configuration** no longer silently defaults a missing execution provider to QNN; incomplete configurations now fail validation instead (#1026). +- `winml inspect` / `winml perf` hide third-party and native warning noise by default; use `-v`, `-vv`, or `WINMLCLI_SHOW_ALL_WARNINGS=1` to restore diagnostics (#1232, #1246). + +### ✨ Improvements + +- **Quantization** — `--precision` selects FP16 conversion, RTN INT4, static QDQ, or calibration-free dynamic INT8; `winml quantize` can compose multiple precision passes such as INT4 followed by FP16 (#872, #985, #1047). +- `winml build` — `--export-type optimized` produces a complete Qwen3 onnxruntime-genai NPU/QNN bundle, including prefill/decode, embeddings, LM head, tokenizer, and manifest files (#836, #996, #1008, #1081, #1104). +- `winml perf --runtime winml-genai` — benchmarks prebuilt or automatically cached GenAI bundles with TTFT, token throughput, prompt-template controls, EP overrides, and isolated pre-compilation (#1015, #1042, #1046, #1054, #1109). +- **Composite models** — `export` and `build` automatically fan out pipeline components; `export` / `build` / `perf` support `--submodel`, and explicit composite tasks such as summarization and translation are accepted (#1031, #1037, #1058, #1071, #1089). +- **Export controls** — dynamic axes and symbolic input dimensions are supported while static TorchScript export remains the default; `build`, `config`, `perf`, and `eval` expose matching shape/input/export overrides (#1074, #1083, #1106, #1141, #1156, #1188). +- `winml perf` — real `.npz` inputs, time-budgeted `--duration` runs, cached per-module builds, actual dynamic dimensions, and QNN profiler ONNX metrics (#1004, #1055, #1066, #1102, #1168). +- **QNN op tracing** — per-model tracing can be enabled automatically; basic traces exclude warmup samples and null fields, while detail tracing accepts compile EP options and auto-compiles raw ONNX inputs when required (#1006, #1032, #1249, #1252). +- `winml eval --mode compare` — compares two ONNX models directly or uses real `.npz` samples against a Hugging Face reference; Qwen3 adds perplexity evaluation (#1139, #1209, #1221). +- `winml optimize` / `winml analyze` — `--check-optim` previews applicable rewrites and verifies their produced operators against the target EP; new rewrites cover static Split-to-Slice and Conv affine/BatchNormalization folding (#1142, #1167, #1171, #1238, #1257). +- **EP discovery and monitoring** — registration is isolated and failures are structured, startup remains lazy, op-tracing dispatch is unified, and provider-download progress is restored (#1019, #1239). +- **CLI quality of life** — EP/device and pipeline-stage flags are consistent across commands; `--no-color` disables ANSI output for one invocation (#923, #978, #992). +- **Hub-hosted ONNX** — commands accept `//.onnx` references from Hugging Face Hub, enabling SAM 3 encoder/decoder workflows (#582). +- **Keypoint detection** — ViTPose supports `config`, `build`, and `perf`, plus COCO OKS-AP evaluation (#905, #949). +- **Vision and document recipes** — refreshed coverage adds DINOv2, SwinV2, OWL-ViT/OWLv2, BEiT, SegFormer, YOLOS, ViTPose/SynthPose, LayoutLM/LayoutLMv3, and document/question-answering models (#925, #1064, #1088, #1093, #1100, #1101, #1123, #1125, #1145, #1155, #1173, #1174, #1178, #1187, #1201, #1202, #1205, #1208). +- **Language recipes** — expanded BART, BERT, DeBERTa, DistilBERT, KoELECTRA, MiniLM, MPNet, Marian/OPUS, GTE reranker, feature-extraction, and entity-linking coverage (#1068, #1080, #1112, #1115, #1116, #1117, #1118, #1120, #1121, #1124, #1134, #1143, #1144, #1153, #1169, #1170, #1179, #1200, #1214). +- **Audio recipes** — expanded Wav2Vec2, HuBERT, AST, MMS, language/gender/music classification, forced alignment, and multilingual ASR coverage (#1094, #1095, #1114, #1131, #1148, #1154, #1176, #1186, #1206, #1207, #1211, #1225). + +### 🐛 Fixes + +- **`winml perf`** — throughput uses the batch size actually executed; analyzer EP resolution and op-trace paths now match the runtime target (#930, #941, #1000). +- **`winml build`** — honors explicit `--ep`, supports non-compiling cross-target builds, keeps ONNX caches distinct by resolved path and configuration, preserves configured model classes, reports disk-full failures clearly, and keeps CPU/GPU automatic precision at FP32 (#856, #947, #987, #997, #998). +- **GenAI and composite export** — fixed component export/build failures, compile fallback paths, accelerator selection, isolated EPContext preparation, and final Qwen3 bundle assembly (#1037, #1051, #1103, #1138, #1248). +- **Task and model resolution** — reconciled the task registry, corrected model-specific task listings and Hub `pipeline_tag` fallback, accepted composite tasks, and resolved CTC-based ASR model classes correctly (#724, #986, #1070, #1071, #1113, #1154). +- **Depth and keypoint evaluation** — fixed inference-time evaluator failures (#1023). +- **Analyzer and optimizer rules** — corrected coverage counting, aligned pattern checks with node support, consolidated recommendation metadata, and fixed dtype constraints and unknown-pattern handling (#922, #1020, #1063, #1130, #1162). +- **EP / device resolution** — WindowsML catalog providers register correctly; device listings retain hardware details without duplicate aliases; analyzer auto-selection prefers the strongest exact target; CPU bridge providers resolve safely; invalid EP/device pairs fail early (#1076, #1220, #1227, #1228, #1231, #1237). +- **Native EP execution** — hardened spawned-provider progress, prevented compiler-output deadlocks, released native sessions before process exit, and replaced pipe-backed warning capture with a bounded file spool to avoid EP compiler hangs (#1017, #1223, #1230, #1266, release cherry-pick #1267). +- **Export and quantization** — standalone quantization suppresses duplicate ORT preprocessing warnings; decoder KV-cache dimensions survive tracing; large external-data models can convert to FP16; and EPs that quantize internally no longer receive redundant WinML quantization (#956, #1176, #1235, #1242). +- **QNN evaluation and tracing** — repaired evaluation failures, detail-trace DLL/summary handling, and compile-time provider options (#1247, #1249). +- `winml eval` — default text datasets and sentiment recipes use fully qualified Hugging Face dataset IDs (#1262, release cherry-pick #1263). +- **CLI help** — `winml --help` shows the correct `sys` summary and concise, untruncated `build` / `quantize` descriptions (#1254). +- **Telemetry** — local `Path` model references no longer cause successful commands to fail during telemetry scrubbing (#1273). + +### 🔧 Internals & CI + +- **Release pipelines** — E2E aligns ModelKitArtifacts with the matching release branch, stable GitHub releases receive CHANGELOG notes and “Latest” status, and the official-build toolchain is pinned for reproducibility (#940, #967, #1268). +- **Evaluation CI** — recipe-driven build/eval supports per-EP matrices, pre-exported ONNX, reliable resume behavior, actual applied-precision reporting, unquantized-track EPs, and broader MIGraphX/TensorRT RTX coverage (#845, #902, #1009, #1039, #1086, #1160, #1163, #1226, #1243, #1286). +- **Telemetry** — action events record scrubbed model identifiers, while error events retain scrubbed root-cause details for diagnosis (#1108, #1111). +- **Development environment** — expanded type checking, added a tracked `uv` lockfile, selected CPU-only PyTorch wheels, and consolidated development dependencies (#932, #957, #1105, #1251, #1255). +- **Documentation publishing** — added and published the version-stamped model accuracy report from the current documentation site (#974, #975, #979, #1203). + +### 📦 Assets + +- `winml_cli-0.3.0-py3-none-any.whl` +- `rules-v0.3.0.zip` + ## WinML CLI v0.2.0 This cycle unifies **task detection** across the CLI (modality- and architecture-aware) and expands the eval and perf surfaces — new depth-estimation and tensor-similarity evaluators, a full SA eval pipeline with an HTML report, `winml perf --memory` / `--ep-options`, and `--format json` on `eval` / `analyze` / `perf`. `winml compile` gains a multi-model shared EP context, `winml build` gains `--precision`, and timm image-classification is supported. See the behavior changes below.