From 6a08f22f66ffe8c06ee9995118773d88c69f54f8 Mon Sep 17 00:00:00 2001 From: Domenic <67119745+ToastyDom@users.noreply.github.com> Date: Sat, 28 Feb 2026 15:06:21 +0100 Subject: [PATCH 1/3] Update 1_Feature_Extraction.ipynb --- notebooks/1_Feature_Extraction.ipynb | 191 ++++++++++++++------------- 1 file changed, 101 insertions(+), 90 deletions(-) diff --git a/notebooks/1_Feature_Extraction.ipynb b/notebooks/1_Feature_Extraction.ipynb index bbed5ba..a265698 100644 --- a/notebooks/1_Feature_Extraction.ipynb +++ b/notebooks/1_Feature_Extraction.ipynb @@ -21,15 +21,15 @@ }, { "cell_type": "code", + "execution_count": null, "id": "4ODrk2C7yMjr", "metadata": { "id": "4ODrk2C7yMjr" }, + "outputs": [], "source": [ "#!pip install -U git+https://github.com/cvai-roig-lab/Net2Brain" - ], - "outputs": [], - "execution_count": null + ] }, { "attachments": {}, @@ -87,28 +87,29 @@ }, { "cell_type": "code", + "execution_count": null, "id": "73903734", "metadata": {}, + "outputs": [], "source": [ "from net2brain.utils.download_datasets import DatasetBonnerPNAS2017\n", "from pprint import pprint\n", "\n", "paths_bonner = DatasetBonnerPNAS2017.load_dataset()\n", - "pprint(paths_bonner)\n" - ], - "outputs": [], - "execution_count": null + "pprint(paths_bonner)\n", + "\n" + ] }, { "cell_type": "code", + "execution_count": null, "id": "6bf25268", "metadata": {}, + "outputs": [], "source": [ "stimuli_path = paths_bonner[\"stimuli_path\"]\n", "roi_path = paths_bonner[\"roi_path\"]" - ], - "outputs": [], - "execution_count": null + ] }, { "attachments": {}, @@ -131,6 +132,7 @@ }, { "cell_type": "code", + "execution_count": null, "id": "83e48930-2eed-4551-8177-88dfe7a452d3", "metadata": { "colab": { @@ -153,12 +155,11 @@ "id": "83e48930-2eed-4551-8177-88dfe7a452d3", "outputId": "34a91416-caca-4b3d-9fa5-859d40ce99eb" }, + "outputs": [], "source": [ "from net2brain.feature_extraction import FeatureExtractor\n", "fx = FeatureExtractor(model='AlexNet', netset='Standard', device='cpu')" - ], - "outputs": [], - "execution_count": null + ] }, { "attachments": {}, @@ -183,16 +184,16 @@ }, { "cell_type": "code", + "execution_count": null, "id": "39faa0f0", "metadata": {}, + "outputs": [], "source": [ "from net2brain.feature_extraction import FeatureExtractor\n", "\n", "fx = FeatureExtractor(model='AlexNet', netset='Standard', device='cpu')\n", - "fx.extract(data_path=stimuli_path, save_path='AlexNet_Feat', consolidate_per_layer=False)" - ], - "outputs": [], - "execution_count": null + "fx.extract(data_path=stimuli_path, save_path='AlexNet_Feat', consolidate_per_layer=False, layers_to_extract=[\"AAAAA\"])" + ] }, { "attachments": {}, @@ -208,6 +209,7 @@ }, { "cell_type": "code", + "execution_count": null, "id": "426df22a-b261-4a7f-b94b-fd8cbdaeb030", "metadata": { "colab": { @@ -216,11 +218,10 @@ "id": "426df22a-b261-4a7f-b94b-fd8cbdaeb030", "outputId": "62aa778b-455f-46c4-da52-8c344296a3bd" }, + "outputs": [], "source": [ "fx.layers_to_extract" - ], - "outputs": [], - "execution_count": null + ] }, { "attachments": {}, @@ -233,13 +234,13 @@ }, { "cell_type": "code", + "execution_count": null, "id": "6619e401", "metadata": {}, + "outputs": [], "source": [ "fx.get_all_layers()" - ], - "outputs": [], - "execution_count": null + ] }, { "attachments": {}, @@ -279,15 +280,15 @@ }, { "cell_type": "code", + "execution_count": null, "id": "d71b19f31ec77c3", "metadata": {}, + "outputs": [], "source": [ "from net2brain.feature_extraction import FeatureExtractor\n", "fx = FeatureExtractor(model='AlexNet', netset='Standard', device='cpu')\n", "fx.extract(data_path=stimuli_path, save_path='AlexNet_Feat_dim_red', dim_reduction=\"srp\", n_components=50)" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -300,24 +301,27 @@ ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "bf2da7462edaa62d", + "metadata": {}, "source": [ "---\n", "\n", "---" - ], - "id": "bf2da7462edaa62d" + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "# Extracting Features from MMAction Video Models", - "id": "b1b9eb878a1049de" + "id": "b1b9eb878a1049de", + "metadata": {}, + "source": [ + "# Extracting Features from MMAction Video Models" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "5edfa1a943864267", + "metadata": {}, "source": [ "When extracting features from the video models available in the MMAction netset, there are some \n", "additional options to keep in mind.\n", @@ -327,42 +331,44 @@ "\n", "First, let's now switch to using a video neural dataset, such as the BoldMomentsDataset from \n", "[Lahner et al.](https://www.nature.com/articles/s41467-024-50310-3):" - ], - "id": "5edfa1a943864267" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "9022831101d648cd", + "metadata": {}, + "outputs": [], "source": [ "from net2brain.utils.download_datasets import DatasetBoldMoments\n", "paths_bmd = DatasetBoldMoments.load_dataset()\n", - "stimuli_path = paths_bmd[\"stimuli_path\"]\n", + "stimuli_path = paths_bmd[\"stimuli_train\"]\n", "roi_path = paths_bmd[\"roi_path\"]" - ], - "id": "9022831101d648cd" + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "And load a video model from the MMAction netset:", - "id": "1636488947f96d79" + "id": "1636488947f96d79", + "metadata": {}, + "source": [ + "And load a video model from the MMAction netset:" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "8ebbf87a8dbfec30", + "metadata": {}, + "outputs": [], "source": [ "from net2brain.feature_extraction import FeatureExtractor\n", "fx = FeatureExtractor(model='videomae_v2_B', netset='MMAction', device='cpu')" - ], - "id": "8ebbf87a8dbfec30" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "ff57e379f8e0051a", + "metadata": {}, "source": [ "Now, when extracting features you can decide whether and how to aggregate model features across \n", "time, using the keyword argument `agg_frames`. The options are 'all', 'within_clips', \n", @@ -377,48 +383,52 @@ "processing horizon), resulting in one feature vector per subclip. Shape (1, N_Clips, 1, C, H, W).\n", "- 'across_clips': Model feature timesteps are averaged across subclips, resulting in a single \n", "time-resolved feature vector per video. Shape (1, 1, T, C, H, W)." - ], - "id": "ff57e379f8e0051a" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "fx.extract(data_path=stimuli_path, save_path='VideoModel_TimeResolvedFeat', agg_frames=None) # Useful if you want to do temporally resolved analyses later on", - "id": "79f3a2f18ca4d683" + "id": "79f3a2f18ca4d683", + "metadata": {}, + "outputs": [], + "source": [ + "fx.extract(data_path=stimuli_path, save_path='VideoModel_TimeResolvedFeat', agg_frames=None) # Useful if you want to do temporally resolved analyses later on" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "2d2420bb34df7c9d", + "metadata": {}, "source": [ "---\n", "\n", "---" - ], - "id": "2d2420bb34df7c9d" + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "# Extracting Features from Large Language Models", - "id": "655dccf2d11c450b" + "id": "655dccf2d11c450b", + "metadata": {}, + "source": [ + "# Extracting Features from Large Language Models" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "46913ac7341fab59", + "metadata": {}, "source": [ "We have also added optionality to extract features from Large Language Models (LLMs) using .txt files. For this you just enter the path to your .txt files, in which each new line represents one new sentence.\n", "\n", "Since the feautures are saved per file, and since a .txt file might contain multiple sentences, you can `consolidate_per_txt_file()` in order to seperate each sentence into its own .npz file!" - ], - "id": "46913ac7341fab59" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "c3c07d7f04289033", + "metadata": {}, + "outputs": [], "source": [ "from net2brain.feature_extraction import FeatureExtractor\n", "\n", @@ -430,42 +440,44 @@ "extractor.extract(data_path=\"textinput_folder\", \n", " save_path=\"LLM_output\",\n", " consolidate_per_layer=True)\n" - ], - "id": "c3c07d7f04289033" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "c2c654d1d91d26f4", + "metadata": {}, "source": [ "- - -\n", "\n", "- - -" - ], - "id": "c2c654d1d91d26f4" + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "# Extracting Features from Audio Models\n", - "id": "1eda33624db9ac37" + "id": "1eda33624db9ac37", + "metadata": {}, + "source": [ + "# Extracting Features from Audio Models\n" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "d195329fd2b1d483", + "metadata": {}, "source": [ "You can also extract features from audio models using any kind of audio data e.g. *.wav* or *.mp3* files. For this you just enter the path to your audio files. Currently, we provide a selection of CNN based models and Transformer based models. You can find the available models in the taxonomy notebook. For each model group there are mutliple models available, that differ in many design choices like the time window used or the number of layers. Please refer to the paper of the model for more details:\n", "* CNN-based models: [PANNS](https://arxiv.org/pdf/1912.10211)\n", "* Transformer-based models: [AST](https://arxiv.org/pdf/2104.01778)\n", "\n", "In the future we will also add more models to this list." - ], - "id": "d195329fd2b1d483" + ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "id": "17abe1ce072db65", + "metadata": {}, + "outputs": [], "source": [ "from net2brain.feature_extraction import FeatureExtractor\n", "\n", @@ -476,18 +488,17 @@ "extractor.extract(data_path=\"../net2brain/tests/audios\",\n", " save_path=\"Audio_output\",\n", " consolidate_per_layer=True)" - ], - "id": "17abe1ce072db65" + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "6df000f0", + "metadata": {}, "source": [ "- - - \n", "\n", "- - -" - ], - "id": "6df000f0" + ] }, { "attachments": {}, @@ -511,7 +522,7 @@ "You can also incorporate your own custom model with __Net2Brain__. To do this, supply the `FeatureExtractor` with the following components:\n", "\n", "1. Your model\n", - "2. An existing netset to fall back to (e.g. Standard, Clip, Pyvideo) when loading the data and applying standard \n", + "2. An existing netset to fall back to (e.g. Standard, Clip, MMAction) when loading the data and applying standard \n", "functions.\n", "3. Optionally, your custom transform function (if not provided, standard ImageNet transformations will be used)\n", "4. Optionally your custom extraction function (if not provided, standard Torchextractor will be used)\n", @@ -521,6 +532,7 @@ }, { "cell_type": "code", + "execution_count": null, "id": "bfcb9939-b1b0-4109-9f1f-d09cc3c5fa11", "metadata": { "colab": { @@ -543,6 +555,7 @@ "id": "bfcb9939-b1b0-4109-9f1f-d09cc3c5fa11", "outputId": "9d488345-e7a2-430f-9dcf-83853823bcee" }, + "outputs": [], "source": [ "from torchvision import models\n", "\n", @@ -555,9 +568,7 @@ "\n", "# Run extractor\n", "fx.extract(data_path=stimuli_path, save_path='ResNet50_Feat', layers_to_extract=['layer1', 'layer2', 'layer3', 'layer4'])" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -569,8 +580,10 @@ }, { "cell_type": "code", + "execution_count": null, "id": "e5a8342b193f61f4", "metadata": {}, + "outputs": [], "source": [ "from torchvision import transforms as T\n", "from torchvision import transforms as trn\n", @@ -622,9 +635,7 @@ "\n", "# Run extractor\n", "fx.extract(stimuli_path, layers_to_extract=['layer1', 'layer2', 'layer3', 'layer4'])" - ], - "outputs": [], - "execution_count": null + ] } ], "metadata": { From a8fa87d8fbcc75bf157f1f06742611c1fc3dea75 Mon Sep 17 00:00:00 2001 From: Domenic <67119745+ToastyDom@users.noreply.github.com> Date: Sun, 1 Mar 2026 14:43:58 +0100 Subject: [PATCH 2/3] Update Algonauts --- README.md | 4 + net2brain/utils/download_datasets.py | 209 +++++++++++--------------- notebooks/0_Exploring_Net2Brain.ipynb | 150 +++++++++--------- 3 files changed, 160 insertions(+), 203 deletions(-) diff --git a/README.md b/README.md index e5c5d80..165aca5 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,10 @@ To delve deeper into the toolbox, check out our [ReadTheDocs](https://net2brain. # Updates +# Updates 02/26 +1. Improved Algonauts 2023 NSD datasets (`Algonauts23_Net2Brain` and `Algonauts23_shared_Net2Brain`): proper metadata files linking fMRI rows to image filenames, NSD IDs, COCO IDs and captions, explicit per-subject ROI availability, and recomputed RDMs for the shared split. + + # Updates 01/26 1. **New netset** `MMAction`, with support for 50 video models 2. Deprecation of the `pyvideo` netset, as its previously supported models are now fully covered by MMAction diff --git a/net2brain/utils/download_datasets.py b/net2brain/utils/download_datasets.py index 735ed86..bc23d40 100644 --- a/net2brain/utils/download_datasets.py +++ b/net2brain/utils/download_datasets.py @@ -27,8 +27,13 @@ def load_dataset(dataset_name, path=None): # Here you can still call the old function if you maintain it for backward compatibility or remove it after a transition period. def list_available_datasets(): - available_datasets = [cls.__name__ for cls in BaseDataset.__subclasses__()] - return available_datasets + def get_all_subclasses(cls): + subclasses = cls.__subclasses__() + for subclass in subclasses: + subclasses.extend(get_all_subclasses(subclass)) + return subclasses + + return [cls.__name__ for cls in get_all_subclasses(BaseDataset)] class BaseDataset: @@ -74,6 +79,7 @@ class DatasetBonnerPNAS2017(BaseDataset): def __init__(self, path=None): super().__init__(path) + self.source_path = self.dataset_folder def _load(self): self.download_and_extract_zip() @@ -92,6 +98,7 @@ class Dataset78images(BaseDataset): def __init__(self, path=None): super().__init__(path) + self.source_path = self.dataset_folder def _load(self): self.download_and_extract_zip() @@ -108,6 +115,7 @@ class Workhsop_Harry_Potter_Cognition(BaseDataset): def __init__(self, path=None): super().__init__(path) + self.source_path = self.dataset_folder def _load(self): self.download_and_extract_zip() @@ -125,6 +133,7 @@ class Dataset92images(BaseDataset): def __init__(self, path=None): super().__init__(path) + self.source_path = self.dataset_folder def _load(self): self.download_and_extract_zip() @@ -142,6 +151,7 @@ class DatasetBoldMoments(BaseDataset): def __init__(self, path=None): super().__init__(path) + self.source_path = self.dataset_folder def _load(self): self.download_and_extract_zip() @@ -163,6 +173,7 @@ class WorkshopCuttingGardens(BaseDataset): def __init__(self, path=None): super().__init__(path) + self.source_path = self.dataset_folder def _load(self): self.download_and_extract_zip() @@ -181,6 +192,7 @@ class Tutorial_LE_Results(BaseDataset): def __init__(self, path=None): super().__init__(path) + self.source_path = self.dataset_folder def _load(self): self.download_and_extract_zip() @@ -198,30 +210,36 @@ def _load(self): -class DatasetNSD_872(BaseDataset): - dataset_name = "NSD Dataset" +class DatasetAlgonauts_NSD_Shared(BaseDataset): + dataset_name = "Algonauts23_shared_Net2Brain" DATASET_URLS = { - dataset_name: "https://hessenbox-a10.rz.uni-frankfurt.de/dl/fiSExLpb1b84jTi4QYXH2n/NSD%20Dataset.zip" + dataset_name: "https://www.dropbox.com/scl/fi/y3jbpl8pixeewjjj7svbz/Algonauts23_shared_subset_Net2Brain.zip?rlkey=f2tmrxj2wapp87wb1eux5u6cc&st=gfb0pi82&dl=1" } + def __init__(self, path=None): super().__init__(path) - self.source_path = "NSD Dataset" + self.source_path = self.dataset_folder def _load(self): self.download_and_extract_zip() - # Dictionary to store folder names and their paths - folder_paths = {} - - # Iterate over items in the dataset folder - for item in os.listdir(self.dataset_folder): - item_path = os.path.join(self.dataset_folder, item) - # Check if the item is a directory - if os.path.isdir(item_path): - folder_paths[item] = item_path - - return folder_paths + root = self.dataset_folder + paths = { + "root": root, + "images": os.path.join(root, "images"), + "metadata": os.path.join(root, "metadata_shared.csv"), + "coco_csv": os.path.join(root, "coco.csv"), + "coco_images": os.path.join(root, "coco_images"), + "coco_masks": os.path.join(root, "coco_masks"), + } + for s in range(1, 9): + subj = f"subj0{s}" + paths[subj] = os.path.join(root, subj) + paths[f"{subj}_fmri"] = os.path.join(root, subj, "fmri") + paths[f"{subj}_rois"] = os.path.join(root, subj, "rois") + paths[f"{subj}_rdms"] = os.path.join(root, subj, "rdms") + return paths @@ -242,13 +260,13 @@ def NSDtoCOCO(self, nsd_id, coco_path=""): nsd_id_str = str(nsd_id).lstrip('0') # Convert the nsd_id in the DataFrame to string and strip leading zeros for matching - df['nsdId'] = df['nsdId'].astype(str).apply(lambda x: x.lstrip('0')) + df['nsd_id'] = df['nsd_id'].astype(str).apply(lambda x: x.lstrip('0')) # Find the COCO ID corresponding to the given NSD ID - if nsd_id_str not in df['nsdId'].values: + if nsd_id_str not in df['nsd_id'].values: raise DatasetError(f"NSD ID {nsd_id} not found in the dataset.") - return int(df[df['nsdId'] == nsd_id_str]['cocoId'].values[0]) # Assuming cocoId is numeric + return int(df[df['nsd_id'] == nsd_id_str]['coco_id'].values[0]) # Assuming coco_id is numeric def COCOtoNSD(self, coco_id, coco_path=""): @@ -261,13 +279,12 @@ def COCOtoNSD(self, coco_id, coco_path=""): raise DatasetError(f"The file at '{coco_path}' could not be found. Please ensure the file exists or specify the correct location using the 'coco_path' parameter. This file can be downloaded using 'DatasetNSD.load_dataset()'.") # Load the dataset - df = pd.read_csv(coco_path, dtype={'cocoId': str}) # Ensure cocoId is read as string + df = pd.read_csv(coco_path, dtype={'coco_id': str}) - # Find the NSD ID corresponding to the given COCO ID - if coco_id not in df['cocoId'].values: + if coco_id not in df['coco_id'].values: raise DatasetError(f"COCO ID {coco_id} not found in the dataset.") - return df[df['cocoId'] == coco_id]['nsdId'].values[0] # nsdId is returned as it is in the csv, potentially with leading zeros + return df[df['coco_id'] == coco_id]['nsd_id'].values[0] @@ -279,33 +296,27 @@ def Download_COCO_Images(self, nsd_image_folder, target_folder, NSD_path=""): coco_csv_path = os.path.join(NSD_path, "coco.csv") # Load the NSD to COCO mapping - df = pd.read_csv(coco_csv_path, dtype={'nsdId': str, 'cocoId': str, 'cocoSplit': str}) # Treat IDs as strings + df = pd.read_csv(coco_csv_path, dtype={'nsd_id': str, 'coco_id': str, 'coco_split': str}) - # Ensure target folder exists if not os.path.exists(target_folder): os.makedirs(target_folder) - # Preload COCO API instances for both splits train_annotations_path = os.path.join(NSD_path, "instances_train2017.json") val_annotations_path = os.path.join(NSD_path, "instances_val2017.json") coco_train = COCO(train_annotations_path) if os.path.exists(train_annotations_path) else None coco_val = COCO(val_annotations_path) if os.path.exists(val_annotations_path) else None - # Iterate through NSD image files and collect NSD IDs for filename in tqdm(os.listdir(nsd_image_folder)): if filename.endswith(('.png', '.jpg', '.jpeg')): - # Extract NSD ID from the filename nsd_id_match = re.search(r'nsd-(\d+)', filename) nsd_id = nsd_id_match.group(1).lstrip('0') if nsd_id_match else filename.split('.')[0].lstrip('0') - # Check if nsd_id exists in the DataFrame and get the corresponding coco_id and split - if nsd_id in df['nsdId'].values: - row = df[df['nsdId'] == nsd_id].iloc[0] - coco_id = int(row['cocoId']) # coco_id should be an integer for COCO API - coco_split = row['cocoSplit'] + if nsd_id in df['nsd_id'].values: + row = df[df['nsd_id'] == nsd_id].iloc[0] + coco_id = int(row['coco_id']) + coco_split = row['coco_split'] - # Select the correct COCO instance based on the split coco = coco_train if coco_split == 'train2017' else coco_val # Fetch image info and URL @@ -331,30 +342,25 @@ def Download_COCO_Segmentation_Masks(self, nsd_image_folder, target_folder, NSD_ coco_csv_path = os.path.join(NSD_path, "coco.csv") # Load the NSD to COCO mapping - df = pd.read_csv(coco_csv_path, dtype={'nsdId': str, 'cocoId': str, 'cocoSplit': str}) # Treat IDs as strings + df = pd.read_csv(coco_csv_path, dtype={'nsd_id': str, 'coco_id': str, 'coco_split': str}) - # Ensure target folder exists if not os.path.exists(target_folder): os.makedirs(target_folder) - # Preload COCO API instances for both splits train_annotations_path = os.path.join(NSD_path, "instances_train2017.json") val_annotations_path = os.path.join(NSD_path, "instances_val2017.json") coco_train = COCO(train_annotations_path) if os.path.exists(train_annotations_path) else None coco_val = COCO(val_annotations_path) if os.path.exists(val_annotations_path) else None - # Iterate through NSD image files and collect NSD IDs, handling both naming conventions for filename in tqdm(os.listdir(nsd_image_folder)): if filename.endswith(('.png', '.jpg', '.jpeg')): - # Extract NSD ID from the filename nsd_id_match = re.search(r'nsd-(\d+)', filename) nsd_id = nsd_id_match.group(1).lstrip('0') if nsd_id_match else filename.split('.')[0].lstrip('0') - # Convert NSD ID to COCO ID and fetch the split information - if nsd_id in df['nsdId'].values: - row = df[df['nsdId'] == nsd_id].iloc[0] - coco_id = int(row['cocoId']) - coco_split = row['cocoSplit'] + if nsd_id in df['nsd_id'].values: + row = df[df['nsd_id'] == nsd_id].iloc[0] + coco_id = int(row['coco_id']) + coco_split = row['coco_split'] coco = coco_train if coco_split == 'train2017' else coco_val @@ -398,30 +404,26 @@ def Download_COCO_Captions(self, nsd_image_folder, target_folder, NSD_path=""): raise DatasetError(f"Required files within '{NSD_path}' were not found. Please download the dataset using 'DatasetNSD.load_dataset()' and ensure the correct path is specified with 'NSD_path = path'.") # Load the NSD to COCO mapping - df = pd.read_csv(coco_csv_path, dtype={'nsdId': str, 'cocoId': str, 'cocoSplit': str}) # Ensure IDs and splits are read as strings - - # Ensure target folder exists + df = pd.read_csv(coco_csv_path, dtype={'nsd_id': str, 'coco_id': str, 'coco_split': str}) + if not os.path.exists(target_folder): os.makedirs(target_folder) - # Initialize COCO API for captions for both train and val splits coco_captions_train = COCO(train_captions_path) coco_captions_val = COCO(val_captions_path) - # Fetch and save the first available COCO caption corresponding to COCO IDs for filename in tqdm(os.listdir(nsd_image_folder)): if filename.endswith(('.png', '.jpg', '.jpeg')): - # Extract NSD ID from the filename nsd_id_match = re.search(r'nsd-(\d+)', filename) if nsd_id_match: - nsd_id = nsd_id_match.group(1).lstrip('0') # Remove leading zeros + nsd_id = nsd_id_match.group(1).lstrip('0') else: - nsd_id = filename.split('.')[0].lstrip('0') # Remove leading zeros + nsd_id = filename.split('.')[0].lstrip('0') - if nsd_id in df['nsdId'].values: - row = df[df['nsdId'] == nsd_id].iloc[0] - coco_id = row['cocoId'] - coco_split = row['cocoSplit'] + if nsd_id in df['nsd_id'].values: + row = df[df['nsd_id'] == nsd_id].iloc[0] + coco_id = row['coco_id'] + coco_split = row['coco_split'] coco_captions = coco_captions_train if coco_split == 'train2017' else coco_captions_val @@ -465,8 +467,8 @@ def Crop_COCO_to_NSD(self, source_folder, target_folder, coco_path=""): coco_id = int(filename.split('.')[0]) # Extracting COCO ID from filename # Find the corresponding row in the DataFrame - if coco_id in df['cocoId'].values: - row = df[df['cocoId'] == coco_id].iloc[0] + if coco_id in df['coco_id'].values: + row = df[df['coco_id'] == coco_id].iloc[0] crop_str = row['cropBox'] crop_values = eval(crop_str) # Convert string to tuple @@ -550,6 +552,7 @@ def Visualize(self, image_folder, mask_folder, image_id): colored_mask = Image.fromarray(colored_mask_array) # Create an overlay of the image with the colored mask + colored_mask = colored_mask.resize(original_image.size, Image.NEAREST) overlay = Image.blend(original_image.convert("RGBA"), colored_mask.convert("RGBA"), alpha=0.5) # Find unique classes in the mask (excluding background) @@ -595,23 +598,19 @@ def RenameToCOCO(self, folder, coco_path=""): raise FileNotFoundError(f"The file at '{coco_path}' could not be found. Please ensure the file exists or specify the correct location using the 'coco_path' parameter. This file can be downloaded using 'DatasetNSD.load_dataset()'.") # Load the NSD to COCO mapping - df = pd.read_csv(coco_path, dtype={'nsdId': str, 'cocoId': str}) # Ensure IDs are read as strings + df = pd.read_csv(coco_path, dtype={'nsd_id': str, 'coco_id': str}) - # Iterate through the files in the folder for filename in tqdm(os.listdir(folder)): basename, extension = os.path.splitext(filename) - # Extract NSD ID from the filename nsd_id_match = re.search(r'nsd-(\d+)', basename) if nsd_id_match: nsd_id = nsd_id_match.group(1) else: nsd_id = basename - # Check if the nsd_id is in the nsdId column - if nsd_id in df['nsdId'].values: - # Find the corresponding COCO ID - coco_id = df[df['nsdId'] == nsd_id]['cocoId'].values[0] + if nsd_id in df['nsd_id'].values: + coco_id = df[df['nsd_id'] == nsd_id]['coco_id'].values[0] # Rename the file new_filename = f"{coco_id}{extension}" @@ -627,16 +626,13 @@ def RenameToNSD(self, folder, coco_path=""): raise DatasetError(f"The file at '{coco_path}' could not be found. Please ensure the file exists or specify the correct location using the 'coco_path' parameter. This file can be downloaded using 'DatasetNSD.load_dataset()'.") # Load the COCO to NSD mapping - df = pd.read_csv(coco_path, dtype={'nsdId': str, 'cocoId': str}) # Ensure IDs are read as strings + df = pd.read_csv(coco_path, dtype={'nsd_id': str, 'coco_id': str}) - # Iterate through the files in the folder for filename in tqdm(os.listdir(folder)): basename, extension = os.path.splitext(filename) - # Check if the basename is in the cocoId column - if basename in df['cocoId'].values: - # Find the corresponding NSD ID - nsd_id = df[df['cocoId'] == basename]['nsdId'].values[0] + if basename in df['coco_id'].values: + nsd_id = df[df['coco_id'] == basename]['nsd_id'].values[0] # Format NSD ID with leading zeros for 10 thousands nsd_id_formatted = nsd_id.zfill(5) @@ -674,61 +670,35 @@ def RenameAlgonautsToNSD(self,folder_path): -class DatasetAlgonauts_NSD(DatasetNSD_872): - dataset_name = "Algonauts_NSD" +class DatasetAlgonauts_NSD(DatasetAlgonauts_NSD_Shared): + dataset_name = "Algonauts23_Net2Brain" DATASET_URLS = { - dataset_name: "https://hessenbox-a10.rz.uni-frankfurt.de/dl/fiR2qmmGe3cUPrYp2N9oet/Algonauts_NSD.zip" + dataset_name: "https://www.dropbox.com/scl/fi/lzxtzp1sb7ivh0esy2d2y/Algonauts23_Net2Brain.zip?rlkey=dpqstfzr0gr03cfwcc0ea1t5v&st=d2d8ba4l&dl=1" } - + def __init__(self, path=None): super().__init__(path) - self.source_path = "Algonauts_NSD" + self.source_path = self.dataset_folder def _load(self): self.download_and_extract_zip() - # Dictionary to store folder names and their paths - folder_paths = {} - - # Iterate over items in the dataset folder - for item in os.listdir(self.dataset_folder): - item_path = os.path.join(self.dataset_folder, item) - # Check if the item is a directory - if os.path.isdir(item_path): - folder_paths[item] = item_path - - # Specifically add the path to the training_images subfolder for each subject - training_images_path = os.path.join(item_path, "training_split", "training_images") - if os.path.exists(training_images_path): - folder_paths[f"{item}_images"] = training_images_path - - brain_data_path = os.path.join(item_path, "training_split", "training_rois") - if os.path.exists(training_images_path): - folder_paths[f"{item}_rois"] = brain_data_path + root = self.dataset_folder + paths = { + "root": root, + "metadata": os.path.join(root, "metadata.csv"), + "coco_csv": os.path.join(root, "coco.csv"), + } + for s in range(1, 9): + subj = f"subj0{s}" + subj_dir = os.path.join(root, subj) + paths[subj] = subj_dir + paths[f"{subj}_images"] = os.path.join(subj_dir, "training_split", "training_images") + paths[f"{subj}_test_images"] = os.path.join(subj_dir, "test_split", "test_images") + paths[f"{subj}_fmri"] = os.path.join(subj_dir, "training_split", "training_fmri") + paths[f"{subj}_rois"] = os.path.join(subj_dir, "training_split", "training_rois") + return paths - return folder_paths - - -class DatasetNSD_25(DatasetNSD_872): - dataset_name = "NSD_25_images" - DATASET_URLS = { - dataset_name: "https://hessenbox-a10.rz.uni-frankfurt.de/dl/fi6sDQMUTHQeuj91a47s2c/NSD_25_images.zip" - } - def __init__(self, path=None): - super().__init__(path) - self.source_path = "NSD_25_images" - - def _load(self): - self.download_and_extract_zip() - # Dictionary to store folder names and their paths - - stimuli_path = os.path.join(self.dataset_folder, "NSD_25_images") - captions_path = os.path.join(self.dataset_folder, "NSD_25_captions") - roi_path = os.path.join(self.dataset_folder, "NSD_25_fmri") - viz_path = os.path.join(self.dataset_folder, "NSD_25_roi_masks") - - return {"stimuli_path": stimuli_path, "roi_path": roi_path, "viz_path": viz_path, "captions_path": captions_path} - class DatasetThings_fMRI(BaseDataset): dataset_name = "Things_test" @@ -753,5 +723,4 @@ def _load(self): if os.path.isdir(item_path): folder_paths[item] = item_path - return folder_paths - + return folder_paths \ No newline at end of file diff --git a/notebooks/0_Exploring_Net2Brain.ipynb b/notebooks/0_Exploring_Net2Brain.ipynb index 23d86ab..35217a4 100644 --- a/notebooks/0_Exploring_Net2Brain.ipynb +++ b/notebooks/0_Exploring_Net2Brain.ipynb @@ -289,16 +289,14 @@ "source": [ "To quickly test the Net2Brain toolbox, we provide some datasets that you can download and play around with. To download them, you can use the following code.\n", "It will return a dictionary containing all available datapaths.\n", - "\n", - "\n", "The available datasets are:\n", "- The 78Images-Dataset from [Algonauts2019 Challenge Training Set A](http://algonauts.csail.mit.edu/2019/download.html)\n", "- The 92Images-Dataset from [Algonauts2019 Challenge Test Set](http://algonauts.csail.mit.edu/2019/download.html)\n", "- The bonnerpnas2017-Dataset from [Micheal F. Bonner et. al](https://www.pnas.org/doi/full/10.1073/pnas.1618228114)\n", - "- The NSD-Dataset (Algonauts Challenge) from [EJ Allen et. al](https://www.nature.com/articles/s41593-021-00962-x)\n", - "- A subset of the NSD-Dataset with the 872 images that all participants have seen from [EJ Allen et. al](https://www.nature.com/articles/s41593-021-00962-x)\n", + "- The Algonauts 2023 NSD Dataset (`Algonauts23_Net2Brain`) from [EJ Allen et. al](https://www.nature.com/articles/s41593-021-00962-x) — full dataset with all 8 subjects, per-subject fMRI, ROI extractions, and metadata\n", + "- The Algonauts 2023 NSD Shared Split (`Algonauts23_shared_Net2Brain`) from [EJ Allen et. al](https://www.nature.com/articles/s41593-021-00962-x) — 872 images seen by all subjects, with RDMs\n", "- The Things_test Dataset from the test split of the [original things fMRI data](https://elifesciences.org/articles/82580), which contains 12 trials for each image in the stimuli for 3 subjects\n", - "- The BoldMoment-Dataset from [Lahner et al.](https://www.nature.com/articles/s41467-024-50310-3), which provides fMRI responses to naturalistic video clips, covering a wide range of semantic categories to study how the brain processes dynamic visual scenes. " + "- The BoldMoment-Dataset from [Lahner et al.](https://www.nature.com/articles/s41467-024-50310-3), which provides fMRI responses to naturalistic video clips" ] }, { @@ -335,15 +333,15 @@ "metadata": {}, "outputs": [], "source": [ - "from net2brain.utils.download_datasets import DatasetNSD_872, Dataset78images, Dataset92images, DatasetBonnerPNAS2017, DatasetAlgonauts_NSD, DatasetThings_fMRI, DatasetBoldMoments\n", + "from net2brain.utils.download_datasets import DatasetAlgonauts_NSD_Shared, Dataset78images, Dataset92images, DatasetBonnerPNAS2017, DatasetAlgonauts_NSD, DatasetThings_fMRI, DatasetBoldMoments\n", "from pprint import pprint\n", "\n", - "paths_78 = Dataset78images.load_dataset()\n", - "paths_92 = Dataset92images.load_dataset()\n", - "paths_bonner = DatasetBonnerPNAS2017.load_dataset()\n", - "paths_NSD = DatasetNSD_872.load_dataset()\n", - "paths_Algonauts = DatasetAlgonauts_NSD.load_dataset()\n", - "paths_things = DatasetThings_fMRI.load_dataset()\n", + "paths_78 = Dataset78images.load_dataset()\n", + "paths_92 = Dataset92images.load_dataset()\n", + "paths_bonner = DatasetBonnerPNAS2017.load_dataset()\n", + "paths_NSD_shared = DatasetAlgonauts_NSD_Shared.load_dataset()\n", + "paths_Algonauts = DatasetAlgonauts_NSD.load_dataset()\n", + "paths_things = DatasetThings_fMRI.load_dataset()\n", "paths_BoldMoments = DatasetBoldMoments.load_dataset()" ] }, @@ -365,19 +363,16 @@ "print(\"Paths for 78images-Dataset:\")\n", "pprint(paths_78)\n", "print(\"\")\n", - "\n", "print(\"Paths for 92images-Dataset:\")\n", "pprint(paths_92)\n", "print(\"\")\n", - "\n", "print(\"Paths for BonnerPNAS2017-Dataset:\")\n", "pprint(paths_bonner)\n", "print(\"\")\n", - "\n", - "print(\"Paths for NSD-Dataset:\")\n", - "pprint(paths_NSD)\n", - "\n", - "print(\"Paths for Algonauts-Dataset:\")\n", + "print(\"Paths for Algonauts23_shared_Net2Brain (872 shared images):\")\n", + "pprint(paths_NSD_shared)\n", + "print(\"\")\n", + "print(\"Paths for Algonauts23_Net2Brain (full dataset):\")\n", "pprint(paths_Algonauts)" ] }, @@ -408,40 +403,21 @@ "id": "238ce364", "metadata": {}, "source": [ - "# Datasets with additional functionality (NSD & Algonauts)" - ] - }, - { - "cell_type": "markdown", - "id": "e96b8002", - "metadata": {}, - "source": [ - "## The NSD-Interface" - ] - }, - { - "cell_type": "markdown", - "id": "441bbc83", - "metadata": {}, - "source": [ - "The Natural Scenes Dataset (NSD) is a valuable resource that shares its roots with the COCO dataset, making it a rich dataset for visual neuroscience studies. Recognizing the potential of leveraging COCO's extensive annotations and images, we've developed a suite of functions within the `Net2Brain` toolkit to facilitate seamless interchange between NSD and COCO identifiers, images, segmentations and captions." - ] - }, - { - "cell_type": "markdown", - "id": "2b6d311a", - "metadata": {}, - "source": [ - "### Key Functions Overview\n", + "# Datasets with additional functionality (NSD & Algonauts)\n", + "## The NSD Interface\n", "\n", - "The DatasetNSD class offers a range of functions designed to bridge NSD and COCO, enhancing the utility of the NSD dataset for comprehensive visual studies:\n", + "The Natural Scenes Dataset (NSD) shares its roots with the COCO dataset, making it a rich resource for visual neuroscience. We provide a suite of functions to facilitate seamless interchange between NSD and COCO identifiers, images, segmentations, and captions.\n", "\n", + "Both `DatasetAlgonauts_NSD_shared` (shared images across participants split) and `DatasetAlgonauts_NSD` (full dataset) support these functions.\n", + "\n", + "### Key Functions Overview\n", "* **Image Downloads**: Access original COCO images directly from the NSD context.\n", "* **ID Conversion**: Switch between NSD and COCO identifiers.\n", "* **Segmentation Masks**: Obtain COCO segmentation masks corresponding to NSD images.\n", - "* **Caption Downloads**: Obtain COCO original caption to each downloaded image.\n", "* **Image and Mask Manipulation**: Crop and rename files for consistency with NSD conventions.\n", - "* **Visualization**: Display the images along with their segmentaion masks." + "* **Visualization**: Display images along with their segmentation masks.\n", + "\n", + "> **Note**: Captions are now included directly in `metadata.csv` and `metadata_shared.csv` — no download needed." ] }, { @@ -459,11 +435,11 @@ "metadata": {}, "outputs": [], "source": [ - "from net2brain.utils.download_datasets import DatasetNSD_872, DatasetAlgonauts_NSD\n", + "from net2brain.utils.download_datasets import DatasetAlgonauts_NSD_Shared, DatasetAlgonauts_NSD\n", "\n", - "# Load dataset paths and initialize the DatasetNSD object\n", - "NSD_dataset = DatasetNSD_872() # or use DatasetAlgonauts_NSD\n", - "paths = NSD_dataset.load_dataset(path=\"my_save_path\")\n", + "# Use shared split (872 images) or full Algonauts dataset\n", + "NSD_dataset = DatasetAlgonauts_NSD_Shared(path='your/path')\n", + "paths = NSD_dataset._load()\n", "print(paths)" ] }, @@ -502,9 +478,10 @@ "metadata": {}, "outputs": [], "source": [ - "# Download COCO images corresponding to NSD images\n", - "NSD_dataset.Download_COCO_Images(nsd_image_folder=\"NSD Dataset/NSD_872_images\",\n", - " target_folder=\"NSD Dataset/coco_images\")\n" + "NSD_dataset.Download_COCO_Images(\n", + " nsd_image_folder=paths[\"images\"],\n", + " target_folder=paths[\"coco_images\"]\n", + ")" ] }, { @@ -522,9 +499,9 @@ "metadata": {}, "outputs": [], "source": [ - "# Download COCO segmentation masks for NSD images\n", - "NSD_dataset.Download_COCO_Segmentation_Masks(nsd_image_folder=\"NSD Dataset/NSD_872_images\",\n", - " target_folder=\"NSD Dataset/coco_masks\")\n" + "NSD_dataset.Download_COCO_Segmentation_Masks(\n", + " nsd_image_folder=paths[\"images\"],\n", + " target_folder=paths[\"coco_masks\"]" ] }, { @@ -532,7 +509,8 @@ "id": "33da0f2b", "metadata": {}, "source": [ - "#### Downloading COCO Captions" + "### Loading Captions\n", + "Captions are now included directly in the metadata — no download needed." ] }, { @@ -542,71 +520,77 @@ "metadata": {}, "outputs": [], "source": [ - "# Download COCO images corresponding to NSD images\n", - "NSD_dataset.Download_COCO_Captions(nsd_image_folder=\"NSD Dataset/NSD_872_images\",\n", - " target_folder=\"NSD Dataset/coco_captions\")" + "import pandas as pd\n", + "\n", + "# For the shared split\n", + "meta = pd.read_csv(paths[\"metadata\"])\n", + "print(meta[[\"nsd_id\", \"coco_id\", \"caption_1\"]].head())\n", + "\n", + "# For the full dataset (per subject) - requires DatasetAlgonauts_NSD\n", + "# algonauts_dataset = DatasetAlgonauts_NSD(path='/your/path')\n", + "# algonauts_paths = algonauts_dataset._load()\n", + "# meta_subj = pd.read_csv(algonauts_paths[\"subj01\"] + \"/metadata_train.csv\")\n", + "# meta_root = pd.read_csv(algonauts_paths[\"metadata\"])\n", + "# meta_subj = meta_subj.merge(meta_root[[\"nsd_id\", \"caption_1\"]], on=\"nsd_id\")\n", + "# print(meta_subj[[\"filename\", \"nsd_id\", \"caption_1\"]].head())" ] }, { "cell_type": "markdown", - "id": "ecb00668", + "id": "aaed33a4", "metadata": {}, "source": [ - "#### Visualizing Images and Masks" + "#### Cropping COCO Images to NSD aspect ratio" ] }, { "cell_type": "code", "execution_count": null, - "id": "0e95f96d", + "id": "5710ff77", "metadata": {}, "outputs": [], "source": [ - "# Visualize an image and its segmentation mask side by side\n", - "NSD_dataset.Visualize(\"NSD Dataset/coco_images\", \"NSD Dataset/coco_masks\", \"03171\")\n" + "# Crop\n", + "NSD_dataset.Crop_COCO_to_NSD(source_folder=paths[\"coco_images\"], target_folder=paths[\"coco_images\"])\n", + "NSD_dataset.Crop_COCO_to_NSD(source_folder=paths[\"coco_masks\"], target_folder=paths[\"coco_masks\"])" ] }, { "cell_type": "markdown", - "id": "aaed33a4", + "id": "29cb55f6", "metadata": {}, "source": [ - "#### Cropping COCO Images to NSD aspect ratio" + "#### Rename for COCO Images for NSD compability:" ] }, { "cell_type": "code", "execution_count": null, - "id": "5710ff77", + "id": "110a3718", "metadata": {}, "outputs": [], "source": [ - "# Crop COCO images/masks to fit NSD dimensions\n", - "NSD_dataset.Crop_COCO_to_NSD(source_folder=\"NSD Dataset/coco_images\",\n", - " target_folder=\"NSD Dataset/coco_images\")\n", - "\n", - "NSD_dataset.Crop_COCO_to_NSD(source_folder=\"NSD Dataset/coco_masks\",\n", - " target_folder=\"NSD Dataset/coco_masks\")" + "# Rename\n", + "NSD_dataset.RenameToNSD(paths[\"coco_images\"])\n", + "NSD_dataset.RenameToNSD(paths[\"coco_masks\"])" ] }, { "cell_type": "markdown", - "id": "29cb55f6", + "id": "ecb00668", "metadata": {}, "source": [ - "#### Rename for COCO Images for NSD compability:" + "#### Visualizing Images and Masks" ] }, { "cell_type": "code", "execution_count": null, - "id": "110a3718", + "id": "0e95f96d", "metadata": {}, "outputs": [], "source": [ - "# Rename files for consistent NSD naming conventions\n", - "NSD_dataset.RenameToNSD(\"NSD Dataset/coco_images\")\n", - "NSD_dataset.RenameToNSD(\"NSD Dataset/coco_masks\")" + "NSD_dataset.Visualize(paths[\"coco_images\"], paths[\"coco_masks\"], \"02950\")" ] }, { @@ -625,7 +609,7 @@ "outputs": [], "source": [ "# Visualize an image and its segmentation mask side by side\n", - "NSD_dataset.Visualize(\"NSD Dataset/coco_images\", \"NSD Dataset/coco_masks\", \"02950\") ## Attention: These are NSD IDs now!\n" + "NSD_dataset.Visualize(paths[\"coco_images\"], paths[\"coco_masks\"], \"02950\") ## Attention: These are NSD IDs now!" ] }, { @@ -645,7 +629,7 @@ "metadata": {}, "outputs": [], "source": [ - "NSD_dataset.RenameAlgonautsToNSD(\"path/to/Algonauts\")" + "NSD_dataset.RenameAlgonautsToNSD(paths[\"images\"])" ] } ], From 2489cc4a304b064e80472735d98e1dd04b871f4c Mon Sep 17 00:00:00 2001 From: Domenic <67119745+ToastyDom@users.noreply.github.com> Date: Sun, 1 Mar 2026 22:50:41 +0100 Subject: [PATCH 3/3] Update download_datasets.py --- net2brain/utils/download_datasets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/net2brain/utils/download_datasets.py b/net2brain/utils/download_datasets.py index bc23d40..91e9fc2 100644 --- a/net2brain/utils/download_datasets.py +++ b/net2brain/utils/download_datasets.py @@ -213,7 +213,7 @@ def _load(self): class DatasetAlgonauts_NSD_Shared(BaseDataset): dataset_name = "Algonauts23_shared_Net2Brain" DATASET_URLS = { - dataset_name: "https://www.dropbox.com/scl/fi/y3jbpl8pixeewjjj7svbz/Algonauts23_shared_subset_Net2Brain.zip?rlkey=f2tmrxj2wapp87wb1eux5u6cc&st=gfb0pi82&dl=1" + dataset_name: "https://www.dropbox.com/scl/fi/kscwsiiq1b5gajm49qbz4/Algonauts23_shared_Net2Brain.zip?rlkey=t95xbguxx31phvam7o0fv0y9n&st=b4rtzg8e&dl=1" } @@ -673,7 +673,7 @@ def RenameAlgonautsToNSD(self,folder_path): class DatasetAlgonauts_NSD(DatasetAlgonauts_NSD_Shared): dataset_name = "Algonauts23_Net2Brain" DATASET_URLS = { - dataset_name: "https://www.dropbox.com/scl/fi/lzxtzp1sb7ivh0esy2d2y/Algonauts23_Net2Brain.zip?rlkey=dpqstfzr0gr03cfwcc0ea1t5v&st=d2d8ba4l&dl=1" + dataset_name: "https://www.dropbox.com/scl/fi/pe72bcq15aobzpjcbl7tq/Algonauts23_Net2Brain.zip?rlkey=6apwm62va8xb8z3en8u0j1r0h&st=2qco3jt7&dl=1" } def __init__(self, path=None):