From b0da1cdebd3ba5a4fda9309eeb52df3587431055 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 23:24:59 +0000 Subject: [PATCH] Replace the dead Common Voice eval example and expand `assembly eval` help MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mozilla pulled the Common Voice corpora off the Hugging Face Hub in October 2025 (now exclusively on Mozilla Data Collective), so the `mozilla-foundation/common_voice_17_0` repo is empty and the datasets server answers every request with HTTP 500 "The dataset is empty", breaking the subset/split example verbatim. Swap it for openslr/librispeech_asr --subset clean, verified end-to-end against the datasets-server rows API (ungated, parquet-backed, multiple subsets so --subset is genuinely required, auto-picked test split, auto-detected audio/text columns). Also expand the eval help with how scoring works (normalization, pooled summary) and a sampling of vetted datasets — peoples_speech, minds14, simsamu — plus two new examples covering non-English audio and HF diarization sets. https://claude.ai/code/session_01WpRHsDXCWYTffZ8eMGK6Er --- aai_cli/commands/evaluate.py | 29 +++++++++++++++---- .../test_cli_output_snapshots.ambr | 26 +++++++++++++---- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/aai_cli/commands/evaluate.py b/aai_cli/commands/evaluate.py index 804d2e04..f4e5857b 100644 --- a/aai_cli/commands/evaluate.py +++ b/aai_cli/commands/evaluate.py @@ -131,7 +131,15 @@ def _render(payload: dict[str, object]) -> RenderableType: ), ( "Pick a subset/split and more rows", - "assembly eval mozilla-foundation/common_voice_17_0 --subset en --limit 50", + "assembly eval openslr/librispeech_asr --subset clean --limit 50", + ), + ( + "Evaluate non-English audio", + "assembly eval PolyAI/minds14 --subset fr-FR --language-code fr", + ), + ( + "DER on a Hugging Face diarization set", + "assembly eval diarizers-community/simsamu --speaker-labels", ), ] ), @@ -176,10 +184,21 @@ def evaluate( ) -> None: """Transcribe an evaluation dataset and score WER against its reference texts. - Handy for picking a model: run once per --speech-model and compare. Datasets - come from the Hugging Face Hub (gated ones need HF_TOKEN) or a local .csv/.jsonl - manifest. --speaker-labels also scores diarization (DER) against reference - speaker turns. + Each row's audio is transcribed, then scored against the row's reference + text; both are normalized first (lowercased, punctuation stripped) so style + differences don't count as errors, and the summary pools total errors over + total reference words. Handy for picking a model: run once per + --speech-model and compare. --speaker-labels also diarizes and scores DER + against reference speaker turns. + + Datasets come from the Hugging Face Hub (any public dataset its viewer + serves with audio + reference columns; gated ones need HF_TOKEN) or a local + .csv/.jsonl manifest with audio + text columns. Hub sets to try: + openslr/librispeech_asr (read English; subsets clean/other), + MLCommons/peoples_speech (real-world US English; subset clean), + distil-whisper/meanwhile (long-form English), PolyAI/minds14 (banking + calls in 14 locales; subsets like fr-FR), and diarizers-community/simsamu + (French dispatch calls with speaker turns, for --speaker-labels). """ def body(state: AppState, json_mode: bool) -> None: diff --git a/tests/__snapshots__/test_cli_output_snapshots.ambr b/tests/__snapshots__/test_cli_output_snapshots.ambr index c3a472ee..4c03e8c9 100644 --- a/tests/__snapshots__/test_cli_output_snapshots.ambr +++ b/tests/__snapshots__/test_cli_output_snapshots.ambr @@ -242,11 +242,21 @@ Transcribe an evaluation dataset and score WER against its reference texts. - Handy for picking a model: run once per --speech-model and compare. Datasets - come from the Hugging Face Hub (gated ones need HF_TOKEN) or a local - .csv/.jsonl - manifest. --speaker-labels also scores diarization (DER) against reference - speaker turns. + Each row's audio is transcribed, then scored against the row's reference + text; both are normalized first (lowercased, punctuation stripped) so style + differences don't count as errors, and the summary pools total errors over + total reference words. Handy for picking a model: run once per + --speech-model and compare. --speaker-labels also diarizes and scores DER + against reference speaker turns. + + Datasets come from the Hugging Face Hub (any public dataset its viewer + serves with audio + reference columns; gated ones need HF_TOKEN) or a local + .csv/.jsonl manifest with audio + text columns. Hub sets to try: + openslr/librispeech_asr (read English; subsets clean/other), + MLCommons/peoples_speech (real-world US English; subset clean), + distil-whisper/meanwhile (long-form English), PolyAI/minds14 (banking + calls in 14 locales; subsets like fr-FR), and diarizers-community/simsamu + (French dispatch calls with speaker turns, for --speaker-labels). ╭─ Arguments ──────────────────────────────────────────────────────────────────╮ │ * dataset TEXT Hugging Face dataset id, or a local .csv/.jsonl │ @@ -296,7 +306,11 @@ Score diarization too (WER + DER) $ assembly eval agent-calls.jsonl --speaker-labels Pick a subset/split and more rows - $ assembly eval mozilla-foundation/common_voice_17_0 --subset en --limit 50 + $ assembly eval openslr/librispeech_asr --subset clean --limit 50 + Evaluate non-English audio + $ assembly eval PolyAI/minds14 --subset fr-FR --language-code fr + DER on a Hugging Face diarization set + $ assembly eval diarizers-community/simsamu --speaker-labels