diff --git a/chime_utils/dgen/azure_storage.py b/chime_utils/dgen/azure_storage.py deleted file mode 100644 index 5b13c40..0000000 --- a/chime_utils/dgen/azure_storage.py +++ /dev/null @@ -1,177 +0,0 @@ -""" -Copyright (c) Microsoft Corporation. All rights reserved. - -This module contains functions to download data from Azure blob storage to the local machine. -from: https://github.com/microsoft/NOTSOFAR1-Challenge/blob/main/utils/azure_storage.py -LICENSE: https://github.com/microsoft/NOTSOFAR1-Challenge/blob/main/LICENSE -""" -import logging -import os -import shutil -import subprocess -import tempfile -import time -from pathlib import Path -from typing import Literal, Optional, Union - -NOTSOFAR_STORAGE_ACCOUNT_URL = "https://notsofarsa.blob.core.windows.net" - -_LOG = logging.getLogger("azure_storage") - - -def download_blob_container_dir( - azure_source_dir: str, - destination_dir: str, - container_name: str, - keep_structure: bool = False, - overwrite: bool = True, -) -> Optional[str]: - """ - Download a directory from the container to the given output directory - Args: - azure_source_dir: Azure blob directory to download from - destination_dir: path to destination directory to download to - container_name: Azure container name - keep_structure: whether to keep the Azure directory structure in the destination directory - overwrite: whether to override the output file if it already exists - (warning!: if true, will delete the entire destination_dir if it exists) - Returns: - a string indicates the output directory path, or None if the download failed - """ - local_output_dir = destination_dir - if keep_structure: - local_output_dir = ( - os.path.join(destination_dir, azure_source_dir) - .replace("\\", os.sep) - .replace("/", os.sep) - ) - - if os.path.exists(destination_dir) and not overwrite: - _LOG.info(f"{destination_dir} already exists, skipping download") - return local_output_dir - - _LOG.info( - f"downloading `{azure_source_dir}` from container `{container_name}` to `{local_output_dir}`" - ) - with tempfile.TemporaryDirectory() as temp_dir: - _LOG.info(f"downloading to temp dir first: {temp_dir}") - os.makedirs(temp_dir, exist_ok=True) - command = ( - f"az storage copy --recursive --only-show-errors " - f"--destination {temp_dir} " - f"--source {NOTSOFAR_STORAGE_ACCOUNT_URL}/{container_name} " - ) - - if azure_source_dir: - command += f'--include-path {azure_source_dir.rstrip("/")}' - try: - start_time = time.time() - _LOG.debug(f"command: {command}") - subprocess.run(command, shell=True, check=True) - _LOG.info( - f"download completed successfully, time: {time.time() - start_time:.0f} seconds" - ) - except subprocess.CalledProcessError as e: - _LOG.error( - f"failed to download `{azure_source_dir}` from `{container_name}` to `{local_output_dir}`: {e}" - ) - return None - - if os.path.exists(destination_dir) and overwrite: - _LOG.debug(f"Deleting existing destination dir: {destination_dir}") - shutil.rmtree(destination_dir) - - temp_output_dir = ( - os.path.join(temp_dir, container_name, azure_source_dir) - .replace("\\", os.sep) - .replace("/", os.sep) - ) - shutil.move(temp_output_dir, local_output_dir) - return local_output_dir - - -def download_meeting_subset( - subset_name: Literal["train_set", "dev_set", "eval_set"], - version: str, - destination_dir: Union[str, Path], - overwrite: bool = False, -) -> Optional[str]: - """ - Download a subset of the meeting dataset to the destination directory. - The subsets and versions available will be updated in: - https://www.chimechallenge.org/current/task2/index - - Args: - subset_name: name of split to download (dev_set / eval_set / train_set) - version: version to download (240103g / etc.). it's best to use the latest. - destination_dir: path to the directory where files will be downloaded. - overwrite: whether to override the output file if it already exists - (warning!: if true, will delete the entire destination_dir if it exists) - Returns: - a string indicates the output directory path, or None if the download failed - """ - container_name = "benchmark-datasets" - azure_dir = f"{subset_name}/{version}/MTG" - return download_blob_container_dir( - azure_source_dir=azure_dir, - destination_dir=destination_dir, - container_name=container_name, - overwrite=overwrite, - keep_structure=True, - ) - - -def download_simulated_subset( - version: str, - volume: Literal["200hrs", "1000hrs"], - subset_name: Literal["train", "val"], - destination_dir: str, - overwrite: bool = False, -) -> Optional[str]: - """ - Download the simulated dataset to the destination directory - Args: - version: version of the train data to download (v1 / v1.1 / v1.2 / v1.3 / etc.) - volume: volume of the train data to download (200hrs / 1000hrs) - subset_name: train data type to download (train / val) - destination_dir: path to the directory where files will be downloaded. - overwrite: whether to override the output file if it already exists - (warning!: if true, will delete the entire destination_dir if it exists) - Returns: - a string indicates the output directory path, or None if the download failed - """ - container_name = "css-datasets" - azure_dir = "/".join([version, volume, subset_name]) - return download_blob_container_dir( - azure_source_dir=azure_dir, - destination_dir=destination_dir, - container_name=container_name, - overwrite=overwrite, - keep_structure=True, - ) - - -def download_models( - destination_dir: str, pattern: Optional[str] = None, overwrite: bool = False -) -> Optional[str]: - """ - Download the models to the destination directory - Args: - destination_dir: path to destination directory to download the models to - pattern: pattern to match the models to download. - (e.g. 'notsofar/mc' will download all notsofar baseline mc models). - To review all available models, view the container using Azure CLI or Azure Storage Explorer. - overwrite: whether to override the output file if it already exists - (warning!: if true, will delete the entire destination_dir if it exists) - Returns: - a string indicates the output directory path, or None if the download failed - """ - container_name = "css-models" - azure_dir = f'{f"/{pattern}" if pattern is not None else ""}' - return download_blob_container_dir( - azure_source_dir=azure_dir, - destination_dir=destination_dir, - container_name=container_name, - overwrite=overwrite, - keep_structure=True, - ) diff --git a/chime_utils/dgen/notsofar1.py b/chime_utils/dgen/notsofar1.py index a1af6dd..43ccc8b 100644 --- a/chime_utils/dgen/notsofar1.py +++ b/chime_utils/dgen/notsofar1.py @@ -5,12 +5,12 @@ import subprocess from copy import deepcopy from pathlib import Path +from typing import Optional, Union, Literal import soundfile as sf - -from chime_utils.dgen.azure_storage import download_meeting_subset from chime_utils.dgen.utils import get_mappings, symlink from chime_utils.text_norm import get_txt_norm +from huggingface_hub import snapshot_download logging.basicConfig( format=( @@ -23,10 +23,46 @@ NOTSOFAR1_FS = 16000 - _check_version_exists_cache = None +def download_meeting_subset( + subset_name: Literal["train_set", "dev_set", "eval_set"], + version: str, + destination_dir: Union[str, Path], + overwrite: bool = False, +) -> Optional[str]: + """ + Download a subset of the meeting dataset to the destination directory. + The subsets and versions available will be updated in: + https://www.chimechallenge.org/current/task2/index + + Args: + subset_name: name of split to download (dev_set / eval_set / train_set) + version: version to download (240103g / etc.). it's best to use the latest. + destination_dir: path to the directory where files will be downloaded. + overwrite: whether to override the output file if it already exists + (warning!: if true, will delete the entire destination_dir if it exists) + Returns: + a string indicates the output directory path, or None if the download failed + """ + snapshot_download( + repo_id="microsoft/NOTSOFAR", + repo_type="dataset", + local_dir=destination_dir, + force_download=overwrite, + allow_patterns=f"benchmark-datasets/{subset_name}/{version}/*" + ) + + # Create a symlink to mimic folder structure expected by following data prep scripts + downloaded_path = Path(destination_dir) / "benchmark-datasets" / subset_name + symlink_path = Path(destination_dir) / subset_name + if not symlink_path.exists(): + symlink_path.symlink_to(downloaded_path, target_is_directory=True) + + return str(destination_dir) + + def check_version_exists(version): global _check_version_exists_cache if _check_version_exists_cache is None: @@ -82,7 +118,7 @@ def download_notsofar1(download_dir, subset_name): def normalize_notsofar1_annotation( - transcriptions, session_name, txt_normalization, spk_map + transcriptions, session_name, txt_normalization, spk_map ): # Sam: this is FUGLY but works output = [] @@ -116,13 +152,13 @@ def normalize_notsofar1_annotation( def convert2chime( - c_split, - audio_dir, - session_name, - spk_map, - txt_normalization, - output_root, - is_sc=False, + c_split, + audio_dir, + session_name, + spk_map, + txt_normalization, + output_root, + is_sc=False, ): output_audio_f = os.path.join(output_root, "audio", c_split) os.makedirs(output_audio_f, exist_ok=True) @@ -130,7 +166,7 @@ def convert2chime( output_devices_info = os.path.join(output_root, "devices", c_split) os.makedirs(output_devices_info, exist_ok=True) - if c_split in ["train", "train_sc", "dev", "eval"]: + if c_split in ["train", "train_sc", "dev", "dev_sc", "eval", "eval_sc"]: # dump transcriptions output_txt_f = os.path.join(output_root, "transcriptions", c_split) os.makedirs(output_txt_f, exist_ok=True) @@ -141,7 +177,7 @@ def convert2chime( # load device info here we need it to get the speaker mapping with open( - os.path.join(Path(audio_dir).parent, "gt_meeting_metadata.json"), "r" + os.path.join(Path(audio_dir).parent, "gt_meeting_metadata.json"), "r" ) as f: metadata = json.load(f) @@ -171,7 +207,7 @@ def convert2chime( } devices_info[device_name] = d_type - if c_split not in ["train", "train_sc", "dev", "eval"]: + if c_split not in ["train", "train_sc", "dev", "dev_sc", "eval", "eval_sc"]: devices_info = dict(sorted(devices_info.items(), key=lambda x: x[0])) with open(os.path.join(output_devices_info, f"{session_name}.json"), "w") as f: json.dump(devices_info, f, indent=4) @@ -220,7 +256,7 @@ def convert2chime( def gen_notsofar1( - output_dir, corpus_dir, download=False, dset_part="dev", challenge="chime8" + output_dir, corpus_dir, download=False, dset_part="dev", challenge="chime8" ): corpus_dir = Path(corpus_dir).resolve() # allow for relative path mapping = get_mappings(challenge) @@ -303,10 +339,8 @@ def gen_notsofar1( with open(uem_file, "w") as f: f.writelines(uem_data) - if dset_part not in ["train"]: - return # also prep sc data - uem_file_sc = os.path.join(output_dir, "uem", "train_sc", "all.uem") + uem_file_sc = os.path.join(output_dir, "uem", f"{dset_part}_sc", "all.uem") Path(uem_file_sc).parent.mkdir(parents=True, exist_ok=True) uem_data_sc = [] for device_j in device_jsons: @@ -336,7 +370,7 @@ def gen_notsofar1( sess_name = sess_map[f"{orig_sess_name}_{device_name}_sc"] convert2chime( - "train_sc", + f"{dset_part}_sc", device_folder, sess_name, spk_map, diff --git a/chime_utils/dprep/lhotse.py b/chime_utils/dprep/lhotse.py index f01c69c..37cbe9c 100644 --- a/chime_utils/dprep/lhotse.py +++ b/chime_utils/dprep/lhotse.py @@ -181,7 +181,6 @@ def prepare_notsofar1( ("train", "dev" and "eval"), and the value is Dicts with the keys 'recordings' and 'supervisions'. """ - assert mic in ["ihm", "mdm"], "mic must be one of 'ihm' or 'mdm'" manifests = prep_lhotse_shared( corpus_dir, output_dir, dset_part, mic, "notsofar1", json_dir, txt_norm ) diff --git a/requirements.txt b/requirements.txt index c8f838f..514d3ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ meeteval[cli] @ git+https://github.com/fgnt/meeteval jiwer==3.0.0 pyannote.metrics==3.2.1 -lhotse==1.21.0 +lhotse>=1.21.0 click==8.1.7 torchaudio>=0.13.1 regex==2023.12.25 more-itertools==10.2.0 -azure-cli==2.57.0 +huggingface-hub==0.35.0 \ No newline at end of file