From 57b701f6da7e9c08290cbcd97d0fac36c3e3eaa7 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 00:37:44 +0000 Subject: [PATCH] Improve assembly eval: benchmark aliases, --concurrency, DER breakdown Ports the straightforward wins from the tiny-audio eval framework: - Built-in benchmark aliases (its DATASET_REGISTRY): `assembly eval tedlium` now resolves the hub id plus the fiddly subset/split/ audio-column defaults each set needs; explicit flags still win. - `--concurrency N` (its --num-workers): fans transcription out across a thread pool using the transcribe-batch pattern, with the first worker error cancelling the not-yet-started items. Sequential with the per-item spinner stays the default. - DER component breakdown (missed / false alarm / confusion): der.py already computed the components; the summary line and JSON payload (`der_breakdown`) now surface them instead of discarding them. https://claude.ai/code/session_014egNbYWh1Z7pkdaUHHDJ15 --- README.md | 2 +- aai_cli/commands/evaluate.py | 131 +++++++++++++----- aai_cli/eval_data.py | 41 ++++++ .../test_cli_output_snapshots.ambr | 42 +++--- tests/test_eval_command.py | 101 +++++++++++++- tests/test_eval_data_hf.py | 91 ++++++++++++ 6 files changed, 351 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 7d45c621..3b35cebb 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ neither. - **Real-time streaming**: `assembly stream` transcribes the microphone, a file, or a URL live — on macOS it can capture system audio too. - **Voice agent**: `assembly agent` runs a full-duplex spoken conversation in your terminal (use headphones). - **LLM Gateway**: `assembly llm` prompts an LLM over a transcript, stdin, or a live stream (`assembly stream --llm "summarize as I talk"`). -- **Model evaluation**: `assembly eval` transcribes a Hugging Face dataset or a local `.csv`/`.jsonl` manifest and scores WER against its references (plus DER with `--speaker-labels`) — handy for picking a speech model. +- **Model evaluation**: `assembly eval` transcribes a Hugging Face dataset (with built-in aliases for common benchmarks: `assembly eval tedlium`) or a local `.csv`/`.jsonl` manifest and scores WER against its references (plus DER with `--speaker-labels`) — handy for picking a speech model. - **Starter apps**: `assembly init` scaffolds a self-contained FastAPI + HTML app (`audio-transcription`, `live-captions`, `voice-agent`). - **Code generation**: add `--show-code` to `transcribe`/`stream`/`agent` to print the equivalent Python SDK script instead of running. - **Account self-service**: `assembly keys` / `balance` / `usage` / `limits` / `sessions` / `audit` via browser login. diff --git a/aai_cli/commands/evaluate.py b/aai_cli/commands/evaluate.py index 2aa82952..54f7c263 100644 --- a/aai_cli/commands/evaluate.py +++ b/aai_cli/commands/evaluate.py @@ -8,6 +8,7 @@ from __future__ import annotations +from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass from enum import StrEnum @@ -70,6 +71,52 @@ def _score_item( return _ItemResult(row=row, words=words, speakers=speakers) +def _transcripts( + api_key: str, + items: list[eval_data.EvalItem], + *, + transcription_config: aai.TranscriptionConfig, + concurrency: int, + json_mode: bool, + quiet: bool, +) -> list[aai.Transcript]: + """Each item's transcript, in dataset order. + + Sequential by default, with a per-item spinner; ``--concurrency`` fans the + API calls out across a thread pool (the transcribe-batch pattern: the first + worker error drops the not-yet-started items and re-raises). + """ + if concurrency == 1: + transcripts: list[aai.Transcript] = [] + for index, item in enumerate(items, start=1): + with output.status( + f"[{index}/{len(items)}] Transcribing {item.item_id}…", + json_mode=json_mode, + quiet=quiet, + ): + transcripts.append( + client.transcribe(api_key, item.audio, config=transcription_config) + ) + return transcripts + with ( + output.status( + f"Transcribing {len(items)} items (concurrency {concurrency})…", + json_mode=json_mode, + quiet=quiet, + ), + ThreadPoolExecutor(max_workers=concurrency) as pool, + ): + futures = [ + pool.submit(client.transcribe, api_key, item.audio, config=transcription_config) + for item in items + ] + for future in as_completed(futures): + if (exc := future.exception()) is not None: + pool.shutdown(cancel_futures=True) + raise exc + return [future.result() for future in futures] + + def _payload( label: str, speech_model: EvalSpeechModel | None, results: list[_ItemResult] ) -> dict[str, object]: @@ -85,7 +132,13 @@ def _payload( payload.update({"words": total.words, "errors": total.errors, "wer": total.wer}) der_scores = [result.speakers for result in results if result.speakers is not None] if der_scores: - payload["der"] = der.pooled(der_scores).der + pooled = der.pooled(der_scores) + payload["der"] = pooled.der + payload["der_breakdown"] = { + "missed": pooled.missed / pooled.total, + "false_alarm": pooled.false_alarm / pooled.total, + "confusion": pooled.confusion / pooled.total, + } return payload @@ -98,7 +151,12 @@ def _summary(payload: dict[str, object]) -> str: f"WER {_pct(payload.get('wer'))} ({errors} {noun} / {payload.get('words')} words)" ) if "der" in payload: - parts.append(f"DER {_pct(payload.get('der'))}") + breakdown = jsonshape.as_mapping(payload.get("der_breakdown")) or {} + parts.append( + f"DER {_pct(payload.get('der'))} (missed {_pct(breakdown.get('missed'))} · " + f"false alarm {_pct(breakdown.get('false_alarm'))} · " + f"confusion {_pct(breakdown.get('confusion'))})" + ) return output.heading(" ".join(parts)) @@ -130,8 +188,8 @@ def _render(payload: dict[str, object]) -> RenderableType: epilog=examples_epilog( [ ( - "Score a model on 10 rows of an HF dataset", - "assembly eval sanchit-gandhi/tedlium-data", + "Score a model on 10 rows of a benchmark", + "assembly eval tedlium", ), ( "Compare models on your own audio", @@ -142,16 +200,16 @@ def _render(payload: dict[str, object]) -> RenderableType: "assembly eval agent-calls.jsonl --speaker-labels", ), ( - "Pick a subset/split and more rows", - "assembly eval openslr/librispeech_asr --subset clean --limit 50", + "More rows, transcribed four at a time", + "assembly eval librispeech --limit 50 --concurrency 4", ), ( "Evaluate non-English audio", - "assembly eval fixie-ai/common_voice_17_0 --subset fr --language-code fr", + "assembly eval commonvoice --subset fr --language-code fr", ), ( - "DER on a Hugging Face diarization set", - "assembly eval talkbank/callhome --subset eng --speaker-labels", + "DER on a diarization benchmark", + "assembly eval callhome --speaker-labels", ), ] ), @@ -192,6 +250,12 @@ def evaluate( min=0.0, help="DER forgiveness (seconds) around each reference turn boundary.", ), + concurrency: int = typer.Option( + 1, + "--concurrency", + min=1, + help="How many items to transcribe at once (sequential by default).", + ), json_out: bool = options.json_option("Output the rows and summary as one JSON object."), ) -> None: """Transcribe an evaluation dataset and score WER against its reference texts. @@ -204,21 +268,15 @@ def evaluate( against reference speaker turns. Datasets come from the Hugging Face Hub (any public dataset its viewer - serves with audio + reference columns; gated ones need HF_TOKEN) or a local - .csv/.jsonl manifest with audio + text columns. Hub sets to try: - openslr/librispeech_asr (read English; subsets clean/other), - sanchit-gandhi/tedlium-data (TED talks), - sanchit-gandhi/earnings22_robust_split (earnings calls), - kensho/spgispeech (financial calls; subset test), - edinburghcstr/ami (meetings; subsets ihm/sdm), - fixie-ai/gigaspeech (--subset dev --split dev), - fixie-ai/peoples_speech (real-world US English; subset clean), - fixie-ai/common_voice_17_0 (99 locales; subsets like en/fr), - facebook/voxpopuli (parliament speech; subset en), - hhoangphuoc/switchboard (phone calls; --split validation), - ylacombe/expresso (expressive speech), - speechbrain/LoquaciousSet (--subset small --audio-column wav), and - talkbank/callhome (phone calls with speaker turns; --subset eng, for + serves with audio + reference columns; gated ones need HF_TOKEN), a local + .csv/.jsonl manifest with audio + text columns, or a built-in benchmark + alias that fills in the right hub id, subset, split, and columns: + librispeech / librispeech-other (read English), tedlium (TED talks), + earnings22 (earnings calls), spgispeech (financial calls), ami / ami-sdm + (meetings), gigaspeech, peoples (real-world US English), commonvoice + (English; --subset fr etc. for its 98 other locales), voxpopuli + (parliament speech), switchboard (phone calls), expresso (expressive + speech), loquacious, and callhome (phone calls with speaker turns, for --speaker-labels). """ @@ -238,15 +296,22 @@ def body(state: AppState, json_mode: bool) -> None: language_code=language_code, speaker_labels=speaker_labels or None, ) - results: list[_ItemResult] = [] - for index, item in enumerate(data.items, start=1): - with output.status( - f"[{index}/{len(data.items)}] Transcribing {item.item_id}…", - json_mode=json_mode, - quiet=state.quiet, - ): - transcript = client.transcribe(api_key, item.audio, config=transcription_config) - results.append(_score_item(item, transcript, collar=collar)) + transcripts = _transcripts( + api_key, + data.items, + transcription_config=transcription_config, + concurrency=concurrency, + json_mode=json_mode, + quiet=state.quiet, + ) + results = [ + _score_item(item, transcript, collar=collar) + for item, transcript in zip( + data.items, + transcripts, + strict=True, # pragma: no mutate (defensive invariant; _transcripts returns one transcript per item) + ) + ] output.emit(_payload(data.label, speech_model, results), _render, json_mode=json_mode) run_command(ctx, body, json=json_out) diff --git a/aai_cli/eval_data.py b/aai_cli/eval_data.py index dde0be36..021e8072 100644 --- a/aai_cli/eval_data.py +++ b/aai_cli/eval_data.py @@ -9,6 +9,8 @@ fetched through the hub's datasets-server REST API — no heavyweight ``datasets`` dependency, and the audio arrives as hosted URLs the AssemblyAI API ingests directly. Gated/private datasets authenticate via ``HF_TOKEN``. + The common benchmarks also have short **aliases** (``ALIASES``) that fill in + the hub id plus the subset/split/audio-column defaults each set needs. With ``with_speakers`` (the ``--speaker-labels`` flag), rows must also carry diarization references as the parallel ``speakers`` / ``timestamps_start`` / @@ -47,6 +49,39 @@ _SPEAKER_COLUMNS = ("speakers", "timestamps_start", "timestamps_end") +@dataclass(frozen=True) +class Alias: + """A built-in benchmark alias: the hub dataset id plus the subset/split/ + audio-column defaults its layout needs (explicit flags still win).""" + + dataset: str + subset: str | None = None + split: str | None = None + audio_column: str | None = None + + +# The benchmarks the `assembly eval` help recommends, under short memorable names +# — each pins the hub id and the fiddly defaults its layout needs, so +# `assembly eval tedlium` just works. +ALIASES: dict[str, Alias] = { + "librispeech": Alias("openslr/librispeech_asr", subset="clean"), + "librispeech-other": Alias("openslr/librispeech_asr", subset="other"), + "tedlium": Alias("sanchit-gandhi/tedlium-data"), + "earnings22": Alias("sanchit-gandhi/earnings22_robust_split"), + "spgispeech": Alias("kensho/spgispeech", subset="test"), + "ami": Alias("edinburghcstr/ami", subset="ihm"), + "ami-sdm": Alias("edinburghcstr/ami", subset="sdm"), + "gigaspeech": Alias("fixie-ai/gigaspeech", subset="dev", split="dev"), + "peoples": Alias("fixie-ai/peoples_speech", subset="clean"), + "commonvoice": Alias("fixie-ai/common_voice_17_0", subset="en"), + "voxpopuli": Alias("facebook/voxpopuli", subset="en"), + "switchboard": Alias("hhoangphuoc/switchboard", split="validation"), + "expresso": Alias("ylacombe/expresso"), + "loquacious": Alias("speechbrain/LoquaciousSet", subset="small", audio_column="wav"), + "callhome": Alias("talkbank/callhome", subset="eng"), +} + + @dataclass(frozen=True) class EvalItem: """One evaluation row: an audio source (path or URL) plus its references — @@ -91,6 +126,12 @@ def load( limit=limit, with_speakers=with_speakers, ) + alias = ALIASES.get(dataset) + if alias is not None: + dataset = alias.dataset + subset = subset or alias.subset + split = split or alias.split + audio_column = audio_column or alias.audio_column return _load_hf( dataset, split=split, diff --git a/tests/__snapshots__/test_cli_output_snapshots.ambr b/tests/__snapshots__/test_cli_output_snapshots.ambr index af7b6544..7404c450 100644 --- a/tests/__snapshots__/test_cli_output_snapshots.ambr +++ b/tests/__snapshots__/test_cli_output_snapshots.ambr @@ -250,21 +250,15 @@ against reference speaker turns. Datasets come from the Hugging Face Hub (any public dataset its viewer - serves with audio + reference columns; gated ones need HF_TOKEN) or a local - .csv/.jsonl manifest with audio + text columns. Hub sets to try: - openslr/librispeech_asr (read English; subsets clean/other), - sanchit-gandhi/tedlium-data (TED talks), - sanchit-gandhi/earnings22_robust_split (earnings calls), - kensho/spgispeech (financial calls; subset test), - edinburghcstr/ami (meetings; subsets ihm/sdm), - fixie-ai/gigaspeech (--subset dev --split dev), - fixie-ai/peoples_speech (real-world US English; subset clean), - fixie-ai/common_voice_17_0 (99 locales; subsets like en/fr), - facebook/voxpopuli (parliament speech; subset en), - hhoangphuoc/switchboard (phone calls; --split validation), - ylacombe/expresso (expressive speech), - speechbrain/LoquaciousSet (--subset small --audio-column wav), and - talkbank/callhome (phone calls with speaker turns; --subset eng, for + serves with audio + reference columns; gated ones need HF_TOKEN), a local + .csv/.jsonl manifest with audio + text columns, or a built-in benchmark + alias that fills in the right hub id, subset, split, and columns: + librispeech / librispeech-other (read English), tedlium (TED talks), + earnings22 (earnings calls), spgispeech (financial calls), ami / ami-sdm + (meetings), gigaspeech, peoples (real-world US English), commonvoice + (English; --subset fr etc. for its 98 other locales), voxpopuli + (parliament speech), switchboard (phone calls), expresso (expressive + speech), loquacious, and callhome (phone calls with speaker turns, for --speaker-labels). ╭─ Arguments ──────────────────────────────────────────────────────────────────╮ @@ -298,6 +292,10 @@ │ around each reference │ │ turn boundary. │ │ [default: 1.0] │ + │ --concurrency INTEGER RANGE [x>=1] How many items to │ + │ transcribe at once │ + │ (sequential by default). │ + │ [default: 1] │ │ --json -j Output the rows and │ │ summary as one JSON │ │ object. │ @@ -306,18 +304,18 @@ ╰──────────────────────────────────────────────────────────────────────────────╯ Examples - Score a model on 10 rows of an HF dataset - $ assembly eval sanchit-gandhi/tedlium-data + Score a model on 10 rows of a benchmark + $ assembly eval tedlium Compare models on your own audio $ assembly eval calls.csv --speech-model universal-3-pro Score diarization too (WER + DER) $ assembly eval agent-calls.jsonl --speaker-labels - Pick a subset/split and more rows - $ assembly eval openslr/librispeech_asr --subset clean --limit 50 + More rows, transcribed four at a time + $ assembly eval librispeech --limit 50 --concurrency 4 Evaluate non-English audio - $ assembly eval fixie-ai/common_voice_17_0 --subset fr --language-code fr - DER on a Hugging Face diarization set - $ assembly eval talkbank/callhome --subset eng --speaker-labels + $ assembly eval commonvoice --subset fr --language-code fr + DER on a diarization benchmark + $ assembly eval callhome --speaker-labels diff --git a/tests/test_eval_command.py b/tests/test_eval_command.py index c1ac5596..f56d4456 100644 --- a/tests/test_eval_command.py +++ b/tests/test_eval_command.py @@ -8,12 +8,15 @@ import dataclasses import json import re +import threading +from pathlib import Path from types import SimpleNamespace import pytest from typer.testing import CliRunner from aai_cli import config, eval_data +from aai_cli.commands import evaluate from aai_cli.errors import APIError from aai_cli.main import app @@ -172,6 +175,10 @@ def test_speaker_labels_scores_der_only_when_dataset_has_no_text(tmp_path, mocke assert tx.call_args.kwargs["config"].speaker_labels is True assert "DER" in result.output assert "8.33%" in result.output # 1.5/18 under the default 1s collar + # The summary names each DER component; here the error is pure confusion. + assert "missed 0.00%" in result.output + assert "false alarm 0.00%" in result.output + assert "confusion 8.33%" in result.output assert "WER" not in result.output assert "WORDS" not in result.output @@ -185,6 +192,7 @@ def test_speaker_labels_with_text_scores_both_metrics(tmp_path, mocker): payload = _payload_of(result) assert payload["wer"] == 0.0 assert payload["der"] == 1.5 / 18 # default 1s collar + assert payload["der_breakdown"] == {"missed": 0.0, "false_alarm": 0.0, "confusion": 1.5 / 18} assert payload["rows"][0]["wer"] == 0.0 assert payload["rows"][0]["der"] == 1.5 / 18 @@ -214,7 +222,30 @@ def test_der_counts_leading_speech_the_model_missed(tmp_path, mocker): utterances = [SimpleNamespace(speaker="A", start=1000, end=10000)] _mock_transcribe(mocker, [_transcript("x", utterances)]) result = runner.invoke(app, ["eval", "m.jsonl", "--speaker-labels", "--collar", "0", "--json"]) - assert _payload_of(result)["der"] == 0.1 + payload = _payload_of(result) + assert payload["der"] == 0.1 + # All missed detection — distinguishes the breakdown keys from one another. + assert payload["der_breakdown"] == {"missed": 0.1, "false_alarm": 0.0, "confusion": 0.0} + + +def test_der_counts_trailing_hypothesis_speech_as_false_alarm(tmp_path, mocker): + _auth() + (tmp_path / "a.wav").write_bytes(b"fake-audio") + row = { + "audio": "a.wav", + "speakers": ["alice"], + "timestamps_start": [0.0], + "timestamps_end": [10.0], + } + (tmp_path / "m.jsonl").write_text(json.dumps(row), encoding="utf-8") + # The hypothesis keeps "speaking" 5s past the 10s reference: 5s of false + # alarm over 10s of reference speech, scored without collar forgiveness. + utterances = [SimpleNamespace(speaker="A", start=0, end=15000)] + _mock_transcribe(mocker, [_transcript("x", utterances)]) + result = runner.invoke(app, ["eval", "m.jsonl", "--speaker-labels", "--collar", "0", "--json"]) + payload = _payload_of(result) + assert payload["der"] == 0.5 + assert payload["der_breakdown"] == {"missed": 0.0, "false_alarm": 0.5, "confusion": 0.0} def _assign(obj, attribute, value): @@ -320,6 +351,74 @@ def fake_status(message, *, json_mode, quiet): assert seen == ["[1/2] Transcribing a.wav…", "[2/2] Transcribing b.wav…"] +def test_concurrency_runs_items_at_once_and_keeps_dataset_order(tmp_path, mocker): + # With concurrency 2 and two items, both transcriptions must be in flight at + # once (the barrier times out otherwise), and the rows stay in dataset order + # no matter which finishes first. + _auth() + _write_wer_manifest(tmp_path) + barrier = threading.Barrier(2, timeout=10) + texts = {"a.wav": "hello there", "b.wav": "goodbye cow"} + + def fake_transcribe(api_key, audio, *, config): + barrier.wait() + return _transcript(texts[Path(audio).name]) + + mocker.patch( + "aai_cli.commands.evaluate.client.transcribe", autospec=True, side_effect=fake_transcribe + ) + result = runner.invoke(app, ["eval", "manifest.csv", "--concurrency", "2", "--json"]) + assert result.exit_code == 0 + payload = _payload_of(result) + assert [row["item"] for row in payload["rows"]] == ["a.wav", "b.wav"] + assert payload["rows"][0]["wer"] == 0.0 + assert payload["rows"][1]["wer"] == 0.5 + + +def test_concurrency_shows_one_pooled_status(tmp_path, mocker, monkeypatch): + _auth() + _write_wer_manifest(tmp_path) + _mock_transcribe(mocker, [_transcript("hello there"), _transcript("goodbye now")]) + seen = [] + + @contextlib.contextmanager + def fake_status(message, *, json_mode, quiet): + seen.append(message) + yield + + monkeypatch.setattr("aai_cli.commands.evaluate.output.status", fake_status) + assert runner.invoke(app, ["eval", "manifest.csv", "--concurrency", "2"]).exit_code == 0 + assert seen == ["Transcribing 2 items (concurrency 2)…"] + + +def test_concurrent_failure_drops_queued_items_and_fails_cleanly(tmp_path, mocker, monkeypatch): + # The abort path must shut the pool down with cancel_futures=True — that's what + # keeps one failure from burning an API call per queued item. Asserted on the + # shutdown call because which queued futures actually get dropped is a race + # (the test_transcribe_batch.py pattern). + _auth() + _write_wer_manifest(tmp_path) + seen = {} + real = evaluate.ThreadPoolExecutor + + class Capture(real): + def shutdown(self, wait=True, *, cancel_futures=False): + seen.setdefault("cancel_futures", cancel_futures) # first call wins + super().shutdown(wait=wait, cancel_futures=cancel_futures) + + monkeypatch.setattr(evaluate, "ThreadPoolExecutor", Capture) + _mock_transcribe(mocker, [APIError("rate limited"), APIError("rate limited")]) + result = runner.invoke(app, ["eval", "manifest.csv", "--concurrency", "2"]) + assert result.exit_code == 1 + assert "rate limited" in result.output + assert seen["cancel_futures"] is True + + +def test_concurrency_below_one_is_a_usage_error(tmp_path): + result = runner.invoke(app, ["eval", "org/ds", "--concurrency", "0"]) + assert result.exit_code == 2 + + def test_api_error_mid_run_fails_cleanly(tmp_path, mocker): _auth() _write_wer_manifest(tmp_path) diff --git a/tests/test_eval_data_hf.py b/tests/test_eval_data_hf.py index 72101cae..f0c9b50c 100644 --- a/tests/test_eval_data_hf.py +++ b/tests/test_eval_data_hf.py @@ -5,6 +5,8 @@ test_eval_data_manifest.py. """ +import dataclasses + import httpx2 as httpx import pytest @@ -272,3 +274,92 @@ def test_hf_empty_reference_text_rejected(monkeypatch): with pytest.raises(UsageError) as exc: eval_data.load("org/ds", limit=1) assert "test[0]" in exc.value.message and "empty reference" in exc.value.message + + +# ----------------------------------------------------------- benchmark aliases + + +@pytest.mark.parametrize( + ("alias", "expected"), + [ + ("librispeech", eval_data.Alias("openslr/librispeech_asr", subset="clean")), + ("librispeech-other", eval_data.Alias("openslr/librispeech_asr", subset="other")), + ("tedlium", eval_data.Alias("sanchit-gandhi/tedlium-data")), + ("earnings22", eval_data.Alias("sanchit-gandhi/earnings22_robust_split")), + ("spgispeech", eval_data.Alias("kensho/spgispeech", subset="test")), + ("ami", eval_data.Alias("edinburghcstr/ami", subset="ihm")), + ("ami-sdm", eval_data.Alias("edinburghcstr/ami", subset="sdm")), + ("gigaspeech", eval_data.Alias("fixie-ai/gigaspeech", subset="dev", split="dev")), + ("peoples", eval_data.Alias("fixie-ai/peoples_speech", subset="clean")), + ("commonvoice", eval_data.Alias("fixie-ai/common_voice_17_0", subset="en")), + ("voxpopuli", eval_data.Alias("facebook/voxpopuli", subset="en")), + ("switchboard", eval_data.Alias("hhoangphuoc/switchboard", split="validation")), + ("expresso", eval_data.Alias("ylacombe/expresso")), + ( + "loquacious", + eval_data.Alias("speechbrain/LoquaciousSet", subset="small", audio_column="wav"), + ), + ("callhome", eval_data.Alias("talkbank/callhome", subset="eng")), + ], +) +def test_alias_table_pins_each_benchmark(alias, expected): + assert eval_data.ALIASES[alias] == expected + + +def test_alias_table_has_no_unpinned_entries(): + # Every entry is asserted exactly in the parametrized test above. + assert len(eval_data.ALIASES) == 15 + + +def _assign(obj, attribute, value): + setattr(obj, attribute, value) + + +def test_alias_entries_are_immutable(): + with pytest.raises(dataclasses.FrozenInstanceError): + _assign(eval_data.ALIASES["tedlium"], "dataset", "org/other") + + +def test_alias_expands_to_hub_id_and_defaults(monkeypatch): + seen = [] + splits = [{"config": "dev", "split": "dev"}, {"config": "dev", "split": "test"}] + _hf_handler(monkeypatch, splits=splits, rows=[_hf_row()], seen=seen) + data = eval_data.load("gigaspeech", limit=1) + assert data.label == "fixie-ai/gigaspeech · dev/dev" + assert seen[0].url.params["dataset"] == "fixie-ai/gigaspeech" + params = seen[1].url.params + assert params["config"] == "dev" + assert params["split"] == "dev" # the alias split beats the usual 'test' preference + + +def test_alias_audio_column_beats_autodetect(monkeypatch): + # The row also carries an 'audio' column (the auto-detect favorite); the + # alias's audio_column must still win. + row = _hf_row(wav=_audio_cell("https://hf.example/from-wav.wav")) + splits = [{"config": "small", "split": "test"}] + _hf_handler(monkeypatch, splits=splits, rows=[row]) + item = eval_data.load("loquacious", limit=1).items[0] + assert item.audio == "https://hf.example/from-wav.wav" + + +def test_explicit_audio_column_overrides_alias(monkeypatch): + row = _hf_row(wav=_audio_cell("https://hf.example/from-wav.wav")) + splits = [{"config": "small", "split": "test"}] + _hf_handler(monkeypatch, splits=splits, rows=[row]) + item = eval_data.load("loquacious", audio_column="audio", limit=1).items[0] + assert item.audio == "https://hf.example/audio/0.wav" + + +def test_explicit_subset_and_split_override_alias(monkeypatch): + seen = [] + splits = [ + {"config": "en", "split": "test"}, + {"config": "fr", "split": "test"}, + {"config": "fr", "split": "validation"}, + ] + _hf_handler(monkeypatch, splits=splits, rows=[_hf_row()], seen=seen) + eval_data.load("commonvoice", subset="fr", split="validation", limit=1) + params = seen[1].url.params + assert params["dataset"] == "fixie-ai/common_voice_17_0" + assert params["config"] == "fr" + assert params["split"] == "validation"