From 46f31ad83f99cc6f88892d5b14991000c00abf4c Mon Sep 17 00:00:00 2001 From: ipoloka Date: Thu, 26 Jun 2025 15:05:27 +0200 Subject: [PATCH 1/4] Modification of NOTSOFAR-1 dataprep for single channel condition --- chime_utils/dgen/notsofar1.py | 10 ++++------ chime_utils/dprep/lhotse.py | 1 - 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/chime_utils/dgen/notsofar1.py b/chime_utils/dgen/notsofar1.py index a1af6dd..536383a 100644 --- a/chime_utils/dgen/notsofar1.py +++ b/chime_utils/dgen/notsofar1.py @@ -130,7 +130,7 @@ def convert2chime( output_devices_info = os.path.join(output_root, "devices", c_split) os.makedirs(output_devices_info, exist_ok=True) - if c_split in ["train", "train_sc", "dev", "eval"]: + if c_split in ["train", "train_sc", "dev", "dev_sc", "eval", "eval_sc"]: # dump transcriptions output_txt_f = os.path.join(output_root, "transcriptions", c_split) os.makedirs(output_txt_f, exist_ok=True) @@ -171,7 +171,7 @@ def convert2chime( } devices_info[device_name] = d_type - if c_split not in ["train", "train_sc", "dev", "eval"]: + if c_split not in ["train", "train_sc", "dev", "dev_sc", "eval", "eval_sc"]: devices_info = dict(sorted(devices_info.items(), key=lambda x: x[0])) with open(os.path.join(output_devices_info, f"{session_name}.json"), "w") as f: json.dump(devices_info, f, indent=4) @@ -303,10 +303,8 @@ def gen_notsofar1( with open(uem_file, "w") as f: f.writelines(uem_data) - if dset_part not in ["train"]: - return # also prep sc data - uem_file_sc = os.path.join(output_dir, "uem", "train_sc", "all.uem") + uem_file_sc = os.path.join(output_dir, "uem", f"{dset_part}_sc", "all.uem") Path(uem_file_sc).parent.mkdir(parents=True, exist_ok=True) uem_data_sc = [] for device_j in device_jsons: @@ -336,7 +334,7 @@ def gen_notsofar1( sess_name = sess_map[f"{orig_sess_name}_{device_name}_sc"] convert2chime( - "train_sc", + f"{dset_part}_sc", device_folder, sess_name, spk_map, diff --git a/chime_utils/dprep/lhotse.py b/chime_utils/dprep/lhotse.py index f01c69c..37cbe9c 100644 --- a/chime_utils/dprep/lhotse.py +++ b/chime_utils/dprep/lhotse.py @@ -181,7 +181,6 @@ def prepare_notsofar1( ("train", "dev" and "eval"), and the value is Dicts with the keys 'recordings' and 'supervisions'. """ - assert mic in ["ihm", "mdm"], "mic must be one of 'ihm' or 'mdm'" manifests = prep_lhotse_shared( corpus_dir, output_dir, dset_part, mic, "notsofar1", json_dir, txt_norm ) From 7f205cb2b54167d28c201e86941783fa840240f4 Mon Sep 17 00:00:00 2001 From: Alexander Polok <61480290+Lakoc@users.noreply.github.com> Date: Tue, 23 Sep 2025 00:52:22 +0200 Subject: [PATCH 2/4] Update lhotse version requirement in requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c8f838f..feb8f8e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ meeteval[cli] @ git+https://github.com/fgnt/meeteval jiwer==3.0.0 pyannote.metrics==3.2.1 -lhotse==1.21.0 +lhotse>=1.21.0 click==8.1.7 torchaudio>=0.13.1 regex==2023.12.25 From 0149d80c2a73ae8c54f6815dfdf8162e2d56a650 Mon Sep 17 00:00:00 2001 From: ipoloka Date: Tue, 23 Sep 2025 11:08:02 +0200 Subject: [PATCH 3/4] Replaced azcopy (which caused an authentication error) with Hugging Face Hub for NOTSOFAR data download. --- chime_utils/dgen/azure_storage.py | 177 ------------------------------ chime_utils/dgen/notsofar1.py | 36 +++++- requirements.txt | 2 +- 3 files changed, 35 insertions(+), 180 deletions(-) delete mode 100644 chime_utils/dgen/azure_storage.py diff --git a/chime_utils/dgen/azure_storage.py b/chime_utils/dgen/azure_storage.py deleted file mode 100644 index 5b13c40..0000000 --- a/chime_utils/dgen/azure_storage.py +++ /dev/null @@ -1,177 +0,0 @@ -""" -Copyright (c) Microsoft Corporation. All rights reserved. - -This module contains functions to download data from Azure blob storage to the local machine. -from: https://github.com/microsoft/NOTSOFAR1-Challenge/blob/main/utils/azure_storage.py -LICENSE: https://github.com/microsoft/NOTSOFAR1-Challenge/blob/main/LICENSE -""" -import logging -import os -import shutil -import subprocess -import tempfile -import time -from pathlib import Path -from typing import Literal, Optional, Union - -NOTSOFAR_STORAGE_ACCOUNT_URL = "https://notsofarsa.blob.core.windows.net" - -_LOG = logging.getLogger("azure_storage") - - -def download_blob_container_dir( - azure_source_dir: str, - destination_dir: str, - container_name: str, - keep_structure: bool = False, - overwrite: bool = True, -) -> Optional[str]: - """ - Download a directory from the container to the given output directory - Args: - azure_source_dir: Azure blob directory to download from - destination_dir: path to destination directory to download to - container_name: Azure container name - keep_structure: whether to keep the Azure directory structure in the destination directory - overwrite: whether to override the output file if it already exists - (warning!: if true, will delete the entire destination_dir if it exists) - Returns: - a string indicates the output directory path, or None if the download failed - """ - local_output_dir = destination_dir - if keep_structure: - local_output_dir = ( - os.path.join(destination_dir, azure_source_dir) - .replace("\\", os.sep) - .replace("/", os.sep) - ) - - if os.path.exists(destination_dir) and not overwrite: - _LOG.info(f"{destination_dir} already exists, skipping download") - return local_output_dir - - _LOG.info( - f"downloading `{azure_source_dir}` from container `{container_name}` to `{local_output_dir}`" - ) - with tempfile.TemporaryDirectory() as temp_dir: - _LOG.info(f"downloading to temp dir first: {temp_dir}") - os.makedirs(temp_dir, exist_ok=True) - command = ( - f"az storage copy --recursive --only-show-errors " - f"--destination {temp_dir} " - f"--source {NOTSOFAR_STORAGE_ACCOUNT_URL}/{container_name} " - ) - - if azure_source_dir: - command += f'--include-path {azure_source_dir.rstrip("/")}' - try: - start_time = time.time() - _LOG.debug(f"command: {command}") - subprocess.run(command, shell=True, check=True) - _LOG.info( - f"download completed successfully, time: {time.time() - start_time:.0f} seconds" - ) - except subprocess.CalledProcessError as e: - _LOG.error( - f"failed to download `{azure_source_dir}` from `{container_name}` to `{local_output_dir}`: {e}" - ) - return None - - if os.path.exists(destination_dir) and overwrite: - _LOG.debug(f"Deleting existing destination dir: {destination_dir}") - shutil.rmtree(destination_dir) - - temp_output_dir = ( - os.path.join(temp_dir, container_name, azure_source_dir) - .replace("\\", os.sep) - .replace("/", os.sep) - ) - shutil.move(temp_output_dir, local_output_dir) - return local_output_dir - - -def download_meeting_subset( - subset_name: Literal["train_set", "dev_set", "eval_set"], - version: str, - destination_dir: Union[str, Path], - overwrite: bool = False, -) -> Optional[str]: - """ - Download a subset of the meeting dataset to the destination directory. - The subsets and versions available will be updated in: - https://www.chimechallenge.org/current/task2/index - - Args: - subset_name: name of split to download (dev_set / eval_set / train_set) - version: version to download (240103g / etc.). it's best to use the latest. - destination_dir: path to the directory where files will be downloaded. - overwrite: whether to override the output file if it already exists - (warning!: if true, will delete the entire destination_dir if it exists) - Returns: - a string indicates the output directory path, or None if the download failed - """ - container_name = "benchmark-datasets" - azure_dir = f"{subset_name}/{version}/MTG" - return download_blob_container_dir( - azure_source_dir=azure_dir, - destination_dir=destination_dir, - container_name=container_name, - overwrite=overwrite, - keep_structure=True, - ) - - -def download_simulated_subset( - version: str, - volume: Literal["200hrs", "1000hrs"], - subset_name: Literal["train", "val"], - destination_dir: str, - overwrite: bool = False, -) -> Optional[str]: - """ - Download the simulated dataset to the destination directory - Args: - version: version of the train data to download (v1 / v1.1 / v1.2 / v1.3 / etc.) - volume: volume of the train data to download (200hrs / 1000hrs) - subset_name: train data type to download (train / val) - destination_dir: path to the directory where files will be downloaded. - overwrite: whether to override the output file if it already exists - (warning!: if true, will delete the entire destination_dir if it exists) - Returns: - a string indicates the output directory path, or None if the download failed - """ - container_name = "css-datasets" - azure_dir = "/".join([version, volume, subset_name]) - return download_blob_container_dir( - azure_source_dir=azure_dir, - destination_dir=destination_dir, - container_name=container_name, - overwrite=overwrite, - keep_structure=True, - ) - - -def download_models( - destination_dir: str, pattern: Optional[str] = None, overwrite: bool = False -) -> Optional[str]: - """ - Download the models to the destination directory - Args: - destination_dir: path to destination directory to download the models to - pattern: pattern to match the models to download. - (e.g. 'notsofar/mc' will download all notsofar baseline mc models). - To review all available models, view the container using Azure CLI or Azure Storage Explorer. - overwrite: whether to override the output file if it already exists - (warning!: if true, will delete the entire destination_dir if it exists) - Returns: - a string indicates the output directory path, or None if the download failed - """ - container_name = "css-models" - azure_dir = f'{f"/{pattern}" if pattern is not None else ""}' - return download_blob_container_dir( - azure_source_dir=azure_dir, - destination_dir=destination_dir, - container_name=container_name, - overwrite=overwrite, - keep_structure=True, - ) diff --git a/chime_utils/dgen/notsofar1.py b/chime_utils/dgen/notsofar1.py index 536383a..b1ad405 100644 --- a/chime_utils/dgen/notsofar1.py +++ b/chime_utils/dgen/notsofar1.py @@ -5,12 +5,13 @@ import subprocess from copy import deepcopy from pathlib import Path +from typing import Optional, Union, Literal import soundfile as sf - -from chime_utils.dgen.azure_storage import download_meeting_subset from chime_utils.dgen.utils import get_mappings, symlink from chime_utils.text_norm import get_txt_norm +from huggingface_hub import snapshot_download + logging.basicConfig( format=( @@ -27,6 +28,37 @@ _check_version_exists_cache = None +def download_meeting_subset( + subset_name: Literal["train_set", "dev_set", "eval_set"], + version: str, + destination_dir: Union[str, Path], + overwrite: bool = False, +) -> Optional[str]: + """ + Download a subset of the meeting dataset to the destination directory. + The subsets and versions available will be updated in: + https://www.chimechallenge.org/current/task2/index + + Args: + subset_name: name of split to download (dev_set / eval_set / train_set) + version: version to download (240103g / etc.). it's best to use the latest. + destination_dir: path to the directory where files will be downloaded. + overwrite: whether to override the output file if it already exists + (warning!: if true, will delete the entire destination_dir if it exists) + Returns: + a string indicates the output directory path, or None if the download failed + """ + local_dir = snapshot_download( + repo_id="microsoft/NOTSOFAR", + repo_type="dataset", + local_dir=destination_dir, + force_download=overwrite, + allow_patterns=f"benchmark-datasets/{subset_name}/{version}/*" + ) + + return local_dir + + def check_version_exists(version): global _check_version_exists_cache if _check_version_exists_cache is None: diff --git a/requirements.txt b/requirements.txt index feb8f8e..514d3ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,4 @@ click==8.1.7 torchaudio>=0.13.1 regex==2023.12.25 more-itertools==10.2.0 -azure-cli==2.57.0 +huggingface-hub==0.35.0 \ No newline at end of file From fe2bf3f88260a009d023646e698e723b0afbab66 Mon Sep 17 00:00:00 2001 From: ipoloka Date: Tue, 23 Sep 2025 13:04:29 +0200 Subject: [PATCH 4/4] Fix of folder structure to be compatible with followup lhotse prep --- chime_utils/dgen/notsofar1.py | 40 +++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/chime_utils/dgen/notsofar1.py b/chime_utils/dgen/notsofar1.py index b1ad405..43ccc8b 100644 --- a/chime_utils/dgen/notsofar1.py +++ b/chime_utils/dgen/notsofar1.py @@ -12,7 +12,6 @@ from chime_utils.text_norm import get_txt_norm from huggingface_hub import snapshot_download - logging.basicConfig( format=( "%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d]" " %(message)s" @@ -24,15 +23,14 @@ NOTSOFAR1_FS = 16000 - _check_version_exists_cache = None def download_meeting_subset( - subset_name: Literal["train_set", "dev_set", "eval_set"], - version: str, - destination_dir: Union[str, Path], - overwrite: bool = False, + subset_name: Literal["train_set", "dev_set", "eval_set"], + version: str, + destination_dir: Union[str, Path], + overwrite: bool = False, ) -> Optional[str]: """ Download a subset of the meeting dataset to the destination directory. @@ -48,7 +46,7 @@ def download_meeting_subset( Returns: a string indicates the output directory path, or None if the download failed """ - local_dir = snapshot_download( + snapshot_download( repo_id="microsoft/NOTSOFAR", repo_type="dataset", local_dir=destination_dir, @@ -56,7 +54,13 @@ def download_meeting_subset( allow_patterns=f"benchmark-datasets/{subset_name}/{version}/*" ) - return local_dir + # Create a symlink to mimic folder structure expected by following data prep scripts + downloaded_path = Path(destination_dir) / "benchmark-datasets" / subset_name + symlink_path = Path(destination_dir) / subset_name + if not symlink_path.exists(): + symlink_path.symlink_to(downloaded_path, target_is_directory=True) + + return str(destination_dir) def check_version_exists(version): @@ -114,7 +118,7 @@ def download_notsofar1(download_dir, subset_name): def normalize_notsofar1_annotation( - transcriptions, session_name, txt_normalization, spk_map + transcriptions, session_name, txt_normalization, spk_map ): # Sam: this is FUGLY but works output = [] @@ -148,13 +152,13 @@ def normalize_notsofar1_annotation( def convert2chime( - c_split, - audio_dir, - session_name, - spk_map, - txt_normalization, - output_root, - is_sc=False, + c_split, + audio_dir, + session_name, + spk_map, + txt_normalization, + output_root, + is_sc=False, ): output_audio_f = os.path.join(output_root, "audio", c_split) os.makedirs(output_audio_f, exist_ok=True) @@ -173,7 +177,7 @@ def convert2chime( # load device info here we need it to get the speaker mapping with open( - os.path.join(Path(audio_dir).parent, "gt_meeting_metadata.json"), "r" + os.path.join(Path(audio_dir).parent, "gt_meeting_metadata.json"), "r" ) as f: metadata = json.load(f) @@ -252,7 +256,7 @@ def convert2chime( def gen_notsofar1( - output_dir, corpus_dir, download=False, dset_part="dev", challenge="chime8" + output_dir, corpus_dir, download=False, dset_part="dev", challenge="chime8" ): corpus_dir = Path(corpus_dir).resolve() # allow for relative path mapping = get_mappings(challenge)