From b49327069d66d03ecd7ca11f599acfea51fcd2da Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 23:35:01 +0000 Subject: [PATCH 1/4] Extend eval --speech-model to all SDK-supported models The SDK's SpeechModel enum stops at the legacy generation (best, nano, slam-1, universal); the current models (universal-3-pro, universal-2) are requested through its newer speech_models list parameter. Replace the option's type with a CLI-level enum covering the union and route each choice through the SDK parameter that accepts it. https://claude.ai/code/session_01Ra3Q7AeGdx6uFxTtc7dL1K --- aai_cli/commands/evaluate.py | 54 ++++++++++++++++--- .../test_cli_output_snapshots.ambr | 7 +-- tests/test_eval_command.py | 24 +++++++++ 3 files changed, 75 insertions(+), 10 deletions(-) diff --git a/aai_cli/commands/evaluate.py b/aai_cli/commands/evaluate.py index f4e5857b..0d006cda 100644 --- a/aai_cli/commands/evaluate.py +++ b/aai_cli/commands/evaluate.py @@ -9,6 +9,7 @@ from __future__ import annotations from dataclasses import dataclass +from enum import StrEnum import assemblyai as aai import typer @@ -21,6 +22,47 @@ app = typer.Typer() +class EvalSpeechModel(StrEnum): + """Every speech model the SDK can request, current generation first. + + The SDK's own ``SpeechModel`` enum stops at the legacy generation; the current + models are requested through its newer ``speech_models`` list parameter, so the + CLI choices are the union of both. + """ + + universal_3_pro = "universal-3-pro" + universal_2 = "universal-2" + slam_1 = "slam-1" + universal = "universal" + best = "best" + nano = "nano" + + +def _transcription_config( + speech_model: EvalSpeechModel | None, *, language_code: str | None, speaker_labels: bool +) -> aai.TranscriptionConfig: + """Route the model choice through the SDK parameter that accepts it. + + Legacy models go through ``speech_model`` (the SDK enum); current-generation + models only exist as ``speech_models`` list values. + """ + legacy = {model.value for model in aai.SpeechModel} + return aai.TranscriptionConfig( + speech_model=( + aai.SpeechModel(speech_model.value) + if speech_model is not None and speech_model.value in legacy + else None + ), + speech_models=( + [speech_model.value] + if speech_model is not None and speech_model.value not in legacy + else None + ), + language_code=language_code, + speaker_labels=speaker_labels or None, + ) + + def _pct(value: object) -> str: return f"{jsonshape.as_float(value):.2%}" @@ -62,7 +104,7 @@ def _score_item( def _payload( - label: str, speech_model: aai.SpeechModel | None, results: list[_ItemResult] + label: str, speech_model: EvalSpeechModel | None, results: list[_ItemResult] ) -> dict[str, object]: payload: dict[str, object] = { "dataset": label, @@ -123,7 +165,7 @@ def _render(payload: dict[str, object]) -> RenderableType: ("Score a model on 10 rows of an HF dataset", "assembly eval distil-whisper/meanwhile"), ( "Compare models on your own audio", - "assembly eval calls.csv --speech-model universal", + "assembly eval calls.csv --speech-model universal-3-pro", ), ( "Score diarization too (WER + DER)", @@ -163,7 +205,7 @@ def evaluate( text_column: str | None = typer.Option( None, "--text-column", help="Reference text column name (default: auto-detect)." ), - speech_model: aai.SpeechModel | None = typer.Option( + speech_model: EvalSpeechModel | None = typer.Option( None, "--speech-model", help="Speech model to evaluate." ), language_code: str | None = typer.Option( @@ -212,10 +254,8 @@ def body(state: AppState, json_mode: bool) -> None: with_speakers=speaker_labels, ) api_key = config.resolve_api_key(profile=state.profile) - transcription_config = aai.TranscriptionConfig( - speech_model=speech_model, - language_code=language_code, - speaker_labels=speaker_labels or None, + transcription_config = _transcription_config( + speech_model, language_code=language_code, speaker_labels=speaker_labels ) results: list[_ItemResult] = [] for index, item in enumerate(data.items, start=1): diff --git a/tests/__snapshots__/test_cli_output_snapshots.ambr b/tests/__snapshots__/test_cli_output_snapshots.ambr index 4c03e8c9..181a133c 100644 --- a/tests/__snapshots__/test_cli_output_snapshots.ambr +++ b/tests/__snapshots__/test_cli_output_snapshots.ambr @@ -277,8 +277,9 @@ │ --text-column TEXT Reference text column │ │ name (default: │ │ auto-detect). │ - │ --speech-model [best|nano|slam-1|univer Speech model to │ - │ sal] evaluate. │ + │ --speech-model [universal-3-pro|univers Speech model to │ + │ al-2|slam-1|universal|be evaluate. │ + │ st|nano] │ │ --language-code TEXT Force a language (e.g. │ │ en_us). │ │ --speaker-labels Diarize and also score │ @@ -302,7 +303,7 @@ Score a model on 10 rows of an HF dataset $ assembly eval distil-whisper/meanwhile Compare models on your own audio - $ assembly eval calls.csv --speech-model universal + $ assembly eval calls.csv --speech-model universal-3-pro Score diarization too (WER + DER) $ assembly eval agent-calls.jsonl --speaker-labels Pick a subset/split and more rows diff --git a/tests/test_eval_command.py b/tests/test_eval_command.py index bcef496c..9e316d3a 100644 --- a/tests/test_eval_command.py +++ b/tests/test_eval_command.py @@ -132,9 +132,33 @@ def test_speech_model_flag_reaches_config_and_output(tmp_path, mocker): assert _payload_of(result)["speech_model"] == "universal" tx_config = tx.call_args.kwargs["config"] assert tx_config.speech_model == aai.SpeechModel.universal + assert tx_config.speech_models is None # legacy models ride the enum parameter only assert tx_config.speaker_labels is None # not requested -> omitted, not False +@pytest.mark.parametrize("model", ["universal-3-pro", "universal-2"]) +def test_current_models_ride_the_speech_models_list(tmp_path, mocker, model): + _auth() + _write_wer_manifest(tmp_path) + tx = _mock_transcribe(mocker, [_transcript("hello there"), _transcript("goodbye now")]) + result = runner.invoke(app, ["eval", "manifest.csv", "--speech-model", model, "--json"]) + assert result.exit_code == 0 + assert _payload_of(result)["speech_model"] == model + tx_config = tx.call_args.kwargs["config"] + assert tx_config.speech_models == [model] + assert tx_config.speech_model is None # not in the SDK enum -> enum parameter omitted + + +def test_no_speech_model_leaves_both_model_parameters_unset(tmp_path, mocker): + _auth() + _write_wer_manifest(tmp_path) + tx = _mock_transcribe(mocker, [_transcript("hello there"), _transcript("goodbye now")]) + assert runner.invoke(app, ["eval", "manifest.csv"]).exit_code == 0 + tx_config = tx.call_args.kwargs["config"] + assert tx_config.speech_model is None + assert tx_config.speech_models is None + + def test_speech_model_named_in_human_header(tmp_path, mocker): _auth() _write_wer_manifest(tmp_path) From 0f22087b4c47ff40ebf9955fc77fe656f8155d55 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 23:40:38 +0000 Subject: [PATCH 2/4] Drop legacy models from eval --speech-model Only the current generation (universal-3-pro, universal-2) remains, so every choice rides the SDK's speech_models list parameter and the legacy-routing helper goes away. https://claude.ai/code/session_01Ra3Q7AeGdx6uFxTtc7dL1K --- aai_cli/commands/evaluate.py | 43 ++----------- .../test_cli_output_snapshots.ambr | 63 +++++++++---------- tests/test_eval_command.py | 35 ++++------- 3 files changed, 49 insertions(+), 92 deletions(-) diff --git a/aai_cli/commands/evaluate.py b/aai_cli/commands/evaluate.py index 0d006cda..409f82d4 100644 --- a/aai_cli/commands/evaluate.py +++ b/aai_cli/commands/evaluate.py @@ -23,44 +23,11 @@ class EvalSpeechModel(StrEnum): - """Every speech model the SDK can request, current generation first. - - The SDK's own ``SpeechModel`` enum stops at the legacy generation; the current - models are requested through its newer ``speech_models`` list parameter, so the - CLI choices are the union of both. - """ + """The current-generation models, requested via the SDK's ``speech_models`` + list parameter (its legacy ``SpeechModel`` enum predates them).""" universal_3_pro = "universal-3-pro" universal_2 = "universal-2" - slam_1 = "slam-1" - universal = "universal" - best = "best" - nano = "nano" - - -def _transcription_config( - speech_model: EvalSpeechModel | None, *, language_code: str | None, speaker_labels: bool -) -> aai.TranscriptionConfig: - """Route the model choice through the SDK parameter that accepts it. - - Legacy models go through ``speech_model`` (the SDK enum); current-generation - models only exist as ``speech_models`` list values. - """ - legacy = {model.value for model in aai.SpeechModel} - return aai.TranscriptionConfig( - speech_model=( - aai.SpeechModel(speech_model.value) - if speech_model is not None and speech_model.value in legacy - else None - ), - speech_models=( - [speech_model.value] - if speech_model is not None and speech_model.value not in legacy - else None - ), - language_code=language_code, - speaker_labels=speaker_labels or None, - ) def _pct(value: object) -> str: @@ -254,8 +221,10 @@ def body(state: AppState, json_mode: bool) -> None: with_speakers=speaker_labels, ) api_key = config.resolve_api_key(profile=state.profile) - transcription_config = _transcription_config( - speech_model, language_code=language_code, speaker_labels=speaker_labels + transcription_config = aai.TranscriptionConfig( + speech_models=[speech_model.value] if speech_model else None, + language_code=language_code, + speaker_labels=speaker_labels or None, ) results: list[_ItemResult] = [] for index, item in enumerate(data.items, start=1): diff --git a/tests/__snapshots__/test_cli_output_snapshots.ambr b/tests/__snapshots__/test_cli_output_snapshots.ambr index 181a133c..a64e25b7 100644 --- a/tests/__snapshots__/test_cli_output_snapshots.ambr +++ b/tests/__snapshots__/test_cli_output_snapshots.ambr @@ -264,39 +264,36 @@ │ [required] │ ╰──────────────────────────────────────────────────────────────────────────────╯ ╭─ Options ────────────────────────────────────────────────────────────────────╮ - │ --split TEXT Hugging Face split to │ - │ score (default: test). │ - │ --subset TEXT Hugging Face │ - │ config/subset name (e.g. │ - │ a language). │ - │ --limit INTEGER RANGE Rows to evaluate │ - │ [1<=x<=100] (1-100). │ - │ [default: 10] │ - │ --audio-column TEXT Audio column name │ - │ (default: auto-detect). │ - │ --text-column TEXT Reference text column │ - │ name (default: │ - │ auto-detect). │ - │ --speech-model [universal-3-pro|univers Speech model to │ - │ al-2|slam-1|universal|be evaluate. │ - │ st|nano] │ - │ --language-code TEXT Force a language (e.g. │ - │ en_us). │ - │ --speaker-labels Diarize and also score │ - │ DER against the │ - │ dataset's reference │ - │ speaker turns │ - │ (speakers/timestamps_st… │ - │ columns, in seconds). │ - │ --collar FLOAT RANGE [x>=0.0] DER forgiveness │ - │ (seconds) around each │ - │ reference turn boundary. │ - │ [default: 1.0] │ - │ --json -j Output the rows and │ - │ summary as one JSON │ - │ object. │ - │ --help Show this message and │ - │ exit. │ + │ --split TEXT Hugging Face split to │ + │ score (default: test). │ + │ --subset TEXT Hugging Face │ + │ config/subset name (e.g. │ + │ a language). │ + │ --limit INTEGER RANGE Rows to evaluate (1-100). │ + │ [1<=x<=100] [default: 10] │ + │ --audio-column TEXT Audio column name │ + │ (default: auto-detect). │ + │ --text-column TEXT Reference text column │ + │ name (default: │ + │ auto-detect). │ + │ --speech-model [universal-3-pro|univer Speech model to evaluate. │ + │ sal-2] │ + │ --language-code TEXT Force a language (e.g. │ + │ en_us). │ + │ --speaker-labels Diarize and also score │ + │ DER against the dataset's │ + │ reference speaker turns │ + │ (speakers/timestamps_sta… │ + │ columns, in seconds). │ + │ --collar FLOAT RANGE [x>=0.0] DER forgiveness (seconds) │ + │ around each reference │ + │ turn boundary. │ + │ [default: 1.0] │ + │ --json -j Output the rows and │ + │ summary as one JSON │ + │ object. │ + │ --help Show this message and │ + │ exit. │ ╰──────────────────────────────────────────────────────────────────────────────╯ Examples diff --git a/tests/test_eval_command.py b/tests/test_eval_command.py index 9e316d3a..473aca97 100644 --- a/tests/test_eval_command.py +++ b/tests/test_eval_command.py @@ -9,7 +9,6 @@ import json from types import SimpleNamespace -import assemblyai as aai import pytest from typer.testing import CliRunner @@ -123,21 +122,8 @@ def test_json_payload_shape(tmp_path, mocker): assert payload["rows"][1] == {"item": "b.wav", "words": 2, "errors": 1, "wer": 0.5} -def test_speech_model_flag_reaches_config_and_output(tmp_path, mocker): - _auth() - _write_wer_manifest(tmp_path) - tx = _mock_transcribe(mocker, [_transcript("hello there"), _transcript("goodbye now")]) - result = runner.invoke(app, ["eval", "manifest.csv", "--speech-model", "universal", "--json"]) - assert result.exit_code == 0 - assert _payload_of(result)["speech_model"] == "universal" - tx_config = tx.call_args.kwargs["config"] - assert tx_config.speech_model == aai.SpeechModel.universal - assert tx_config.speech_models is None # legacy models ride the enum parameter only - assert tx_config.speaker_labels is None # not requested -> omitted, not False - - @pytest.mark.parametrize("model", ["universal-3-pro", "universal-2"]) -def test_current_models_ride_the_speech_models_list(tmp_path, mocker, model): +def test_speech_model_flag_reaches_config_and_output(tmp_path, mocker, model): _auth() _write_wer_manifest(tmp_path) tx = _mock_transcribe(mocker, [_transcript("hello there"), _transcript("goodbye now")]) @@ -146,25 +132,30 @@ def test_current_models_ride_the_speech_models_list(tmp_path, mocker, model): assert _payload_of(result)["speech_model"] == model tx_config = tx.call_args.kwargs["config"] assert tx_config.speech_models == [model] - assert tx_config.speech_model is None # not in the SDK enum -> enum parameter omitted + assert tx_config.speaker_labels is None # not requested -> omitted, not False -def test_no_speech_model_leaves_both_model_parameters_unset(tmp_path, mocker): +def test_no_speech_model_leaves_speech_models_unset(tmp_path, mocker): _auth() _write_wer_manifest(tmp_path) tx = _mock_transcribe(mocker, [_transcript("hello there"), _transcript("goodbye now")]) assert runner.invoke(app, ["eval", "manifest.csv"]).exit_code == 0 - tx_config = tx.call_args.kwargs["config"] - assert tx_config.speech_model is None - assert tx_config.speech_models is None + assert tx.call_args.kwargs["config"].speech_models is None + + +@pytest.mark.parametrize("model", ["best", "nano", "slam-1", "universal"]) +def test_legacy_models_are_a_usage_error(model): + result = runner.invoke(app, ["eval", "manifest.csv", "--speech-model", model]) + assert result.exit_code == 2 + assert "--speech-model" in result.output def test_speech_model_named_in_human_header(tmp_path, mocker): _auth() _write_wer_manifest(tmp_path) _mock_transcribe(mocker, [_transcript("hello there"), _transcript("goodbye now")]) - result = runner.invoke(app, ["eval", "manifest.csv", "--speech-model", "universal"]) - assert "universal" in result.output + result = runner.invoke(app, ["eval", "manifest.csv", "--speech-model", "universal-3-pro"]) + assert "universal-3-pro" in result.output assert "default model" not in result.output From 4177466652354da873c06f7b8a4c0e22334664f7 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 23:46:28 +0000 Subject: [PATCH 3/4] Assert usage error on the color-free render CI forces color on (Rich under GITHUB_ACTIONS), interleaving style codes mid-message, so the literal substring check failed there while passing locally. Strip SGR codes first, matching test_help_rendering. https://claude.ai/code/session_01Ra3Q7AeGdx6uFxTtc7dL1K --- tests/test_eval_command.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_eval_command.py b/tests/test_eval_command.py index 473aca97..c1ac5596 100644 --- a/tests/test_eval_command.py +++ b/tests/test_eval_command.py @@ -7,6 +7,7 @@ import contextlib import dataclasses import json +import re from types import SimpleNamespace import pytest @@ -147,7 +148,10 @@ def test_no_speech_model_leaves_speech_models_unset(tmp_path, mocker): def test_legacy_models_are_a_usage_error(model): result = runner.invoke(app, ["eval", "manifest.csv", "--speech-model", model]) assert result.exit_code == 2 - assert "--speech-model" in result.output + # CI forces color on (Rich under GITHUB_ACTIONS), interleaving style codes + # mid-message, so assert on the color-free render (see test_help_rendering.py). + plain = re.sub(r"\x1b\[[0-9;]*m", "", result.output) + assert "Invalid value for '--speech-model'" in plain def test_speech_model_named_in_human_header(tmp_path, mocker): From f975c23d3dc9833b9693f89376b996b086dc4d4b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 23:58:53 +0000 Subject: [PATCH 4/4] Recommend the tiny-audio eval datasets in assembly eval help Swap the dataset suggestions in the docstring and examples for the ASR suite used by tiny-audio (librispeech, tedlium, earnings22, spgispeech, ami, gigaspeech, peoples_speech, common_voice_17_0, voxpopuli, switchboard, expresso, LoquaciousSet) plus talkbank/callhome for DER, with the --subset/--split/--audio-column flags each needs. https://claude.ai/code/session_01Ra3Q7AeGdx6uFxTtc7dL1K --- aai_cli/commands/evaluate.py | 26 ++++++++++++++----- .../test_cli_output_snapshots.ambr | 23 +++++++++++----- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/aai_cli/commands/evaluate.py b/aai_cli/commands/evaluate.py index 409f82d4..2aa82952 100644 --- a/aai_cli/commands/evaluate.py +++ b/aai_cli/commands/evaluate.py @@ -129,7 +129,10 @@ def _render(payload: dict[str, object]) -> RenderableType: rich_help_panel=help_panels.TRANSCRIPTION, epilog=examples_epilog( [ - ("Score a model on 10 rows of an HF dataset", "assembly eval distil-whisper/meanwhile"), + ( + "Score a model on 10 rows of an HF dataset", + "assembly eval sanchit-gandhi/tedlium-data", + ), ( "Compare models on your own audio", "assembly eval calls.csv --speech-model universal-3-pro", @@ -144,11 +147,11 @@ def _render(payload: dict[str, object]) -> RenderableType: ), ( "Evaluate non-English audio", - "assembly eval PolyAI/minds14 --subset fr-FR --language-code fr", + "assembly eval fixie-ai/common_voice_17_0 --subset fr --language-code fr", ), ( "DER on a Hugging Face diarization set", - "assembly eval diarizers-community/simsamu --speaker-labels", + "assembly eval talkbank/callhome --subset eng --speaker-labels", ), ] ), @@ -204,10 +207,19 @@ def evaluate( serves with audio + reference columns; gated ones need HF_TOKEN) or a local .csv/.jsonl manifest with audio + text columns. Hub sets to try: openslr/librispeech_asr (read English; subsets clean/other), - MLCommons/peoples_speech (real-world US English; subset clean), - distil-whisper/meanwhile (long-form English), PolyAI/minds14 (banking - calls in 14 locales; subsets like fr-FR), and diarizers-community/simsamu - (French dispatch calls with speaker turns, for --speaker-labels). + sanchit-gandhi/tedlium-data (TED talks), + sanchit-gandhi/earnings22_robust_split (earnings calls), + kensho/spgispeech (financial calls; subset test), + edinburghcstr/ami (meetings; subsets ihm/sdm), + fixie-ai/gigaspeech (--subset dev --split dev), + fixie-ai/peoples_speech (real-world US English; subset clean), + fixie-ai/common_voice_17_0 (99 locales; subsets like en/fr), + facebook/voxpopuli (parliament speech; subset en), + hhoangphuoc/switchboard (phone calls; --split validation), + ylacombe/expresso (expressive speech), + speechbrain/LoquaciousSet (--subset small --audio-column wav), and + talkbank/callhome (phone calls with speaker turns; --subset eng, for + --speaker-labels). """ def body(state: AppState, json_mode: bool) -> None: diff --git a/tests/__snapshots__/test_cli_output_snapshots.ambr b/tests/__snapshots__/test_cli_output_snapshots.ambr index a64e25b7..af7b6544 100644 --- a/tests/__snapshots__/test_cli_output_snapshots.ambr +++ b/tests/__snapshots__/test_cli_output_snapshots.ambr @@ -253,10 +253,19 @@ serves with audio + reference columns; gated ones need HF_TOKEN) or a local .csv/.jsonl manifest with audio + text columns. Hub sets to try: openslr/librispeech_asr (read English; subsets clean/other), - MLCommons/peoples_speech (real-world US English; subset clean), - distil-whisper/meanwhile (long-form English), PolyAI/minds14 (banking - calls in 14 locales; subsets like fr-FR), and diarizers-community/simsamu - (French dispatch calls with speaker turns, for --speaker-labels). + sanchit-gandhi/tedlium-data (TED talks), + sanchit-gandhi/earnings22_robust_split (earnings calls), + kensho/spgispeech (financial calls; subset test), + edinburghcstr/ami (meetings; subsets ihm/sdm), + fixie-ai/gigaspeech (--subset dev --split dev), + fixie-ai/peoples_speech (real-world US English; subset clean), + fixie-ai/common_voice_17_0 (99 locales; subsets like en/fr), + facebook/voxpopuli (parliament speech; subset en), + hhoangphuoc/switchboard (phone calls; --split validation), + ylacombe/expresso (expressive speech), + speechbrain/LoquaciousSet (--subset small --audio-column wav), and + talkbank/callhome (phone calls with speaker turns; --subset eng, for + --speaker-labels). ╭─ Arguments ──────────────────────────────────────────────────────────────────╮ │ * dataset TEXT Hugging Face dataset id, or a local .csv/.jsonl │ @@ -298,7 +307,7 @@ Examples Score a model on 10 rows of an HF dataset - $ assembly eval distil-whisper/meanwhile + $ assembly eval sanchit-gandhi/tedlium-data Compare models on your own audio $ assembly eval calls.csv --speech-model universal-3-pro Score diarization too (WER + DER) @@ -306,9 +315,9 @@ Pick a subset/split and more rows $ assembly eval openslr/librispeech_asr --subset clean --limit 50 Evaluate non-English audio - $ assembly eval PolyAI/minds14 --subset fr-FR --language-code fr + $ assembly eval fixie-ai/common_voice_17_0 --subset fr --language-code fr DER on a Hugging Face diarization set - $ assembly eval diarizers-community/simsamu --speaker-labels + $ assembly eval talkbank/callhome --subset eng --speaker-labels