From 6057bcc5d90715ecc4a2c4b3bd0e86ad1f401578 Mon Sep 17 00:00:00 2001 From: kinichen Date: Thu, 2 Jul 2026 15:23:22 -0400 Subject: [PATCH 1/2] Add performance filter for methods in heatmaps and enhance AS vs FS scatterplot --- HT_LLM/similarity/AS_FS_scatterplot.py | 59 ------ .../algorithm_feature_scatterplot.py | 178 ++++++++++++++++++ HT_LLM/similarity/algorithm_similarity.py | 49 +++-- .../filter_methods_by_performance.py | 116 ++++++++++++ .../similarity/heatmaps_feature_similarity.py | 101 +++++++--- 5 files changed, 408 insertions(+), 95 deletions(-) delete mode 100644 HT_LLM/similarity/AS_FS_scatterplot.py create mode 100644 HT_LLM/similarity/algorithm_feature_scatterplot.py create mode 100644 HT_LLM/similarity/filter_methods_by_performance.py diff --git a/HT_LLM/similarity/AS_FS_scatterplot.py b/HT_LLM/similarity/AS_FS_scatterplot.py deleted file mode 100644 index 218430c..0000000 --- a/HT_LLM/similarity/AS_FS_scatterplot.py +++ /dev/null @@ -1,59 +0,0 @@ -import matplotlib.pyplot as plt -import pandas as pd -import seaborn as sns - -algorithm_pairs = pd.read_csv( - "HT_LLM/similarity/algorithm_similarity_results/AS_BOO_weighted_jaccard_pairs.csv" -) - -feature_pairs = pd.read_csv("HT_LLM/similarity/feature_similarity_results/FS_pairs.csv") - -scatter_df = feature_pairs.merge( - algorithm_pairs[["pair_key", "algorithm_similarity"]], - on="pair_key", - how="inner", -) - -plt.figure(figsize=(7, 6)) - -ax = sns.scatterplot( - data=scatter_df, - x="algorithm_similarity", - y="feature_similarity", - alpha=0.7, -) - - -# Label certain points by their method pair name for identification -counter = 0 -for idx, row in scatter_df.iterrows(): - # Only label points satisfying this condition - if row["algorithm_similarity"] > 0.5: - ax.text( - row["algorithm_similarity"] + 0.01, # Add a slight x-offset manually - row["feature_similarity"] + 0.01, # Add a slight y-offset manually - row["pair_key"], # The labelled text is the method pair's name - color="red", # Highlight color - weight="bold", - ) - counter += 1 - -# Sanity checks for correct number of method pairs (not missing any) -print("Feature pairs:", len(feature_pairs)) -print("Algorithm pairs:", len(algorithm_pairs)) -print("Merged pairs:", len(scatter_df)) -print("Plotted points:", counter) - -plt.xlim(0, 1) -plt.ylim(-0.2, 1) -plt.axvline(x=0.5) -plt.axhline(y=0.5) - -plt.xlabel("Algorithm similarity") -plt.ylabel("dFC feature similarity") -plt.title("Algorithm similarity vs. dFC feature similarity for method pairs") - -plt.tight_layout() -plt.savefig("HT_LLM/similarity/algorithm_vs_feature_similarity_scatter.png", dpi=600) -plt.savefig("HT_LLM/similarity/algorithm_vs_feature_similarity_scatter.pdf") -plt.show() diff --git a/HT_LLM/similarity/algorithm_feature_scatterplot.py b/HT_LLM/similarity/algorithm_feature_scatterplot.py new file mode 100644 index 0000000..fc96207 --- /dev/null +++ b/HT_LLM/similarity/algorithm_feature_scatterplot.py @@ -0,0 +1,178 @@ +### Scatter plot of algorithm similarity vs. feature similarity, coloured by difference in performance, between method pairs ### + +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import seaborn as sns +from matplotlib.lines import Line2D + +threshold = 0.6 # minimum performance threshold for a method to be included +print(f"Method Filtering Threshold: {threshold}") +OUTPUT_DIR = ( + Path("HT_LLM/similarity/scatterplot_algorithm_feature_results") + / f"threshold_{int(threshold*100)}" +) +OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + +algorithm_pairs = pd.read_csv( + f"HT_LLM/similarity/algorithm_similarity_results/threshold_{int(threshold*100)}/AS_BOO_weighted_jaccard_pairs.csv" +) + +feature_pairs = pd.read_csv( + f"HT_LLM/similarity/feature_similarity_results/threshold_{int(threshold*100)}/FS_pairs.csv" +) + +scatter_df = feature_pairs.merge( + algorithm_pairs[["pair_key", "algorithm_similarity"]], + on="pair_key", + how="inner", +) + +### Colour by difference in performance between the two methods in each pair +# Note: only methods that passed the performance threshold were saved and loaded above, +# so performance difference is only coloured for those methods automatically. +performance_diff_table_path = ( + Path("sample_data") + / f"threshold_{int(threshold*100)}" + / "filtered_performance_differences.csv" +) + +pair_performance_table = pd.read_csv(performance_diff_table_path) + +scatter_df = scatter_df.merge( + pair_performance_table[ + [ + "pair_key", + "method_a_performance", + "method_b_performance", + "absolute_performance_difference", + ] + ], + on="pair_key", + how="left", +) + +# Use a separate cutoff for categorizing whether each method in a pair is high-performing. +# If this equals `threshold`, most pairs may be classified as high-high because the +# upstream files are already filtered using `threshold` +# Note, however, that upstream files are filtered for at least 1 experiment/task (more lenient), whereas +# this threshold is applied to the average performance across all experiments/tasks for each method. +high_performance_threshold = threshold + +scatter_df["performance_pair_type"] = np.select( + [ + (scatter_df["method_a_performance"] >= high_performance_threshold) + & (scatter_df["method_b_performance"] >= high_performance_threshold), + (scatter_df["method_a_performance"] >= high_performance_threshold) + | (scatter_df["method_b_performance"] >= high_performance_threshold), + ], + [ + "both high performance", + "mixed high/low performance", + ], + default="both low performance", +) + +print( + "Pairs missing performance info:", + scatter_df["absolute_performance_difference"].isna().sum(), +) + + +### Plot +fig, ax = plt.subplots(figsize=(12, 8)) + +performance_pair_palette = { + "both high performance": "#D0021B", + "mixed high/low performance": "#8E9AA8", + "both low performance": "#4A90E2", +} + +sns.scatterplot( + data=scatter_df, + x="algorithm_similarity", + y="feature_similarity", + hue="performance_pair_type", + size="absolute_performance_difference", + sizes=(30, 180), + palette=performance_pair_palette, + alpha=0.75, + ax=ax, +) + +# Label certain points by their method pair name for identification +high_as_threshold = 0.6 # For vertical line; Only label points with algorithm similarity above this threshold +high_fs_threshold = 0.6 # For horizontal line +counter = 0 +for idx, row in scatter_df.iterrows(): + # Only label points satisfying this condition + if row["algorithm_similarity"] > high_as_threshold: + ax.text( + row["algorithm_similarity"] + 0.005, # Add a slight x-offset manually + row["feature_similarity"] + 0.005, # Add a slight y-offset manually + row["pair_key"], # The labelled text is the method pair's name + color=performance_pair_palette[row["performance_pair_type"]], + weight="bold", + size=7, + ) + counter += 1 + +# Sanity checks for correct number of method pairs (not missing any) +print("Feature pairs:", len(feature_pairs)) +print("Algorithm pairs:", len(algorithm_pairs)) +print("Merged pairs:", len(scatter_df)) +print("Plotted points:", counter) + +ax.set_xlim(0, 1) +ax.set_ylim(-0.2, 1) +ax.axvline(x=high_as_threshold, color="grey", linestyle="--") +ax.axhline(y=high_fs_threshold, color="grey", linestyle="--") + +legend_handles, legend_labels = ax.get_legend_handles_labels() +size_title_idx = legend_labels.index("absolute_performance_difference") +for label_idx in range(size_title_idx + 1, len(legend_labels)): + try: + legend_labels[label_idx] = f"{float(legend_labels[label_idx]):.0%}" + except ValueError: + pass +legend_handles.insert(size_title_idx, Line2D([], [], linestyle="none")) +legend_labels.insert(size_title_idx, "") + +legend = ax.legend( + title="SVM Balanced Accuracy Performance\n", + handles=legend_handles, + labels=legend_labels, + loc="lower left", + bbox_to_anchor=(1.02, 0), + fontsize=8, + title_fontsize=9, +) +# legend.get_title().set_fontweight("bold") +for legend_text in legend.get_texts(): + if legend_text.get_text() in [ + "performance_pair_type", + "absolute_performance_difference", + ]: + legend_text.set_fontweight("bold") + +ax.minorticks_on() +ax.set_xlabel("Algorithm similarity") +ax.set_ylabel("dFC feature similarity") +ax.set_title( + "Algorithm Similarity vs. dFC Feature Similarity vs. Performance Difference for Method Pairs", + fontsize=12, + y=1.02, +) + +plt.tight_layout() +plt.savefig( + OUTPUT_DIR / "algorithm_vs_feature_similarity_scatter.png", + dpi=600, + bbox_inches="tight", +) +plt.savefig( + OUTPUT_DIR / "algorithm_vs_feature_similarity_scatter.pdf", bbox_inches="tight" +) +plt.show() diff --git a/HT_LLM/similarity/algorithm_similarity.py b/HT_LLM/similarity/algorithm_similarity.py index 3daf4e4..b7fc4d7 100644 --- a/HT_LLM/similarity/algorithm_similarity.py +++ b/HT_LLM/similarity/algorithm_similarity.py @@ -26,6 +26,7 @@ import csv import itertools import json +import os import re import sys from collections import Counter @@ -61,11 +62,29 @@ } NON_AIGM_COLOR = "darkorange" -DEFAULT_OUTPUT_DIR = "HT_LLM/similarity/algorithm_similarity_results" METRIC_NAME = "BOO_weighted_jaccard" EXCLUDED_METHOD_FILES = {"__init__.py", "base_dfc_method.py"} +# Filter methods in heatmap by performance threshold +threshold = 0.6 # must exist; else, run filter_methods_by_performance.py with the desired threshold first + +eligible_methods_path = os.path.join( + "sample_data", f"threshold_{int(threshold*100)}", "filtered_methods.npy" +) +eligible_methods = set( + np.load(eligible_methods_path, allow_pickle=True).astype(str).tolist() +) +print(f"Loaded {len(eligible_methods)} eligible methods") +print("Examples:", sorted(eligible_methods)[:10]) + +DEFAULT_OUTPUT_DIR = ( + Path("HT_LLM/similarity/algorithm_similarity_results") + / f"threshold_{int(threshold*100)}" +) +DEFAULT_OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + + def _build_import_map(tree): """Map local alias -> fully-qualified module/object path, from this file's imports.""" import_map = {} @@ -213,7 +232,7 @@ def _make_unique_labels(filepaths): def make_pair_key(method_a, method_b): """Stable key for joining method-pair outputs across scripts for AS vs FS scatterplot.""" - return "+".join(sorted([method_a, method_b])) + return " x ".join(sorted([method_a, method_b])) def _hierarchical_cluster_order(matrix, cluster_method="average"): @@ -239,6 +258,8 @@ def plot_similarity_heatmap( figsize=(10, 8), cluster=True, cluster_method="average", + vmin=None, + vmax=None, ): """Return a heatmap figure for a saved AS matrix. @@ -263,8 +284,8 @@ def plot_similarity_heatmap( xticklabels=labels, yticklabels=labels, cmap="viridis", - vmin=0.0, - vmax=1.0, + vmin=vmin, + vmax=vmax, ax=ax, ) @@ -377,16 +398,22 @@ def main(filepaths): operation_bags[label] = extract_operation_counts(path, verbose=True) print("\nPairwise Algorithm Similarity (weighted Jaccard over operation counts):") - names = list(operation_bags.keys()) + labels = list(operation_bags.keys()) + + # Filter methods based on performance threshold + keep_idx = [i for i, method in enumerate(labels) if method in eligible_methods] + filtered_methods = [labels[i] for i in keep_idx] # Initialize with zeros so the main diagonal stays 0.0 for simple visualization. - alg_sim = np.zeros((len(names), len(names)), dtype=float) + alg_sim = np.zeros((len(filtered_methods), len(filtered_methods)), dtype=float) pairwise_rows = [] - for i, j in itertools.combinations(range(len(names)), 2): # only off diagonal pairs - method_a = names[i] - method_b = names[j] + for i, j in itertools.combinations( + range(len(filtered_methods)), 2 + ): # only off diagonal pairs + method_a = filtered_methods[i] + method_b = filtered_methods[j] counts_a = operation_bags[method_a] counts_b = operation_bags[method_b] weighted_overlap, weighted_union, similarity = weighted_jaccard_similarity( @@ -404,8 +431,6 @@ def main(filepaths): "pair_key": make_pair_key(method_a, method_b), "method_a": method_a, "method_b": method_b, - "source_a": source_paths[i], - "source_b": source_paths[j], "algorithm_similarity": similarity, "weighted_overlap": weighted_overlap, "weighted_union": weighted_union, @@ -426,7 +451,7 @@ def main(filepaths): save_similarity_outputs( DEFAULT_OUTPUT_DIR, - names, + filtered_methods, alg_sim, pairwise_rows, ) diff --git a/HT_LLM/similarity/filter_methods_by_performance.py b/HT_LLM/similarity/filter_methods_by_performance.py new file mode 100644 index 0000000..71f8faf --- /dev/null +++ b/HT_LLM/similarity/filter_methods_by_performance.py @@ -0,0 +1,116 @@ +### NEW: To filter methods in heatmaps by performance threshold ### +# Build a set of dFC methods with performance >= threshold in at least one task. +# If several runs exist for the same (task, dFC method), keep the best run. + +import csv +import itertools +import os +from pathlib import Path + +import numpy as np + +threshold = 0.6 # minimum performance threshold for a method to be included + +performances = np.load("sample_data/ALL_ML_SCORES_real.npy", allow_pickle=True).item() +# print(performances.keys()) + +metric = "SVM balanced accuracy" # change to another performance key if needed +output_root = Path("sample_data") / f"threshold_{int(threshold*100)}" +output_root.mkdir(parents=True, exist_ok=True) +output_path = output_root / "filtered_methods.npy" + +tasks = np.asarray(performances["task"]) +methods = np.asarray(performances["dFC method"]) +scores = np.asarray(performances[metric], dtype=float) + +# If several runs exist for the same (task, dFC method), keep the best run's score. +best_score_by_task_method = {} +for task, method, score in zip(tasks, methods, scores): + if np.isnan(score): + continue + + key = (str(task), str(method)) # unique key for each (task, dFC method) combination + if key not in best_score_by_task_method or score > best_score_by_task_method[key]: + best_score_by_task_method[key] = float(score) + +# Keep methods that have at least one task with best-run score >= threshold. +eligible_methods = { + method + for (task, method), best_score in best_score_by_task_method.items() + if best_score >= threshold +} + +# Save as a sorted numpy array for easy loading in heatmap scripts. +eligible_methods_sorted = np.array(sorted(eligible_methods), dtype=object) +np.save(output_path, eligible_methods_sorted, allow_pickle=True) + +print(f"Metric: {metric}") +print(f"Threshold: {threshold}") +print(f"Eligible methods: {len(eligible_methods_sorted)}") +print(f"Saved to: {output_path}") +print(eligible_methods_sorted) + + +### Save difference in performance between filtered method pairs for later use in scatterplot colouring ### + +# For each filtered method, use its AVERAGE score across tasks after already taking the +# best run for each (task, dFC method) combination above. This creates one scalar +# performance per method, suitable for colouring task-agnostic FS vs AS pair points. +avg_score_by_method = {} +for (task, method), best_score in best_score_by_task_method.items(): + if method not in eligible_methods: + continue + + if method not in avg_score_by_method: + avg_score_by_method[method] = [] + avg_score_by_method[method].append(best_score) + +# Calculate the average score for each method +for method in avg_score_by_method: + avg_score_by_method[method] = np.mean(avg_score_by_method[method]) + + +def make_pair_key(method_a, method_b): + """Stable key matching heatmaps_feature_similarity.py pair keys.""" + return " x ".join(sorted([method_a, method_b])) + + +pair_performance_difference = {} +pair_rows = [] + +for method_a, method_b in itertools.combinations(eligible_methods_sorted.tolist(), 2): + method_a = str(method_a) + method_b = str(method_b) + method_a_score = avg_score_by_method[method_a] + method_b_score = avg_score_by_method[method_b] + signed_difference = method_a_score - method_b_score + absolute_difference = abs(signed_difference) + pair_key = make_pair_key(method_a, method_b) + + pair_performance_difference[pair_key] = absolute_difference + pair_rows.append( + { + "pair_key": pair_key, + "method_a": method_a, + "method_b": method_b, + "method_a_performance": method_a_score, + "method_b_performance": method_b_score, + "signed_performance_difference": signed_difference, + "absolute_performance_difference": absolute_difference, + "metric": metric, + "threshold": threshold, + } + ) + +pair_diff_path = output_root / "filtered_performance_differences.npy" +pair_diff_csv_path = pair_diff_path.with_suffix(".csv") + +np.save(pair_diff_path, pair_performance_difference, allow_pickle=True) +with open(pair_diff_csv_path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=list(pair_rows[0].keys())) + writer.writeheader() + writer.writerows(pair_rows) + +print(f"Saved pair performance differences to: {pair_diff_path}") +print(f"Saved pair performance differences table to: {pair_diff_csv_path}") +print(f"Method pairs saved: {len(pair_rows)}") diff --git a/HT_LLM/similarity/heatmaps_feature_similarity.py b/HT_LLM/similarity/heatmaps_feature_similarity.py index 3bd3df7..c28755b 100644 --- a/HT_LLM/similarity/heatmaps_feature_similarity.py +++ b/HT_LLM/similarity/heatmaps_feature_similarity.py @@ -16,11 +16,28 @@ from scipy.cluster.hierarchy import leaves_list, linkage from scipy.spatial.distance import squareform -DEFAULT_OUTPUT_DIR = Path("HT_LLM/similarity/feature_similarity_results") +# %% +# Filter methods in heatmaps by performance threshold +threshold = 0.6 # must exist; else, run filter_methods_by_performance.py with the desired threshold first + +eligible_methods_path = os.path.join( + "sample_data", f"threshold_{int(threshold*100)}", "filtered_methods.npy" +) +eligible_methods = set( + np.load(eligible_methods_path, allow_pickle=True).astype(str).tolist() +) -os.makedirs(DEFAULT_OUTPUT_DIR / "pdf", exist_ok=True) -os.makedirs(DEFAULT_OUTPUT_DIR / "png", exist_ok=True) +print(f"Loaded {len(eligible_methods)} eligible methods") +print("Examples:", sorted(eligible_methods)[:10]) +# %% +DEFAULT_OUTPUT_DIR = Path( + f"HT_LLM/similarity/feature_similarity_results/threshold_{int(threshold*100)}" +) +OUTPUT_DIR_PDF = DEFAULT_OUTPUT_DIR / "pdf" +OUTPUT_DIR_PNG = DEFAULT_OUTPUT_DIR / "png" +OUTPUT_DIR_PDF.mkdir(parents=True, exist_ok=True) +OUTPUT_DIR_PNG.mkdir(parents=True, exist_ok=True) NON_AIGM_SET = { "CAP", @@ -57,6 +74,7 @@ methods = [ method.MEASURE_NAME for method in measures ] # extract method names from the pydfc dfc_methods objects +methods = sorted(methods) # sort alphabetically for consistent ordering print("Example methods:", methods[:5]) matrix_ex = sim_ex["matrix"]["all"]["spearman"] # similarity matrix for all methods @@ -65,12 +83,13 @@ # %% ######### Helper functions to collect and aggregate similarity matrices based on filters -# for various levels (dataset, subject, session, run, task) ######### +# for various levels (dataset, subject, session, run, task), +# plot heatmaps, and save method pairs for scatterplot ######### def make_pair_key(method_a, method_b): """Stable key for joining method-pair outputs across scripts for AS vs FS scatterplot.""" - return "+".join(sorted([method_a, method_b])) + return " x ".join(sorted([method_a, method_b])) def collect_similarity_matrices( @@ -178,7 +197,7 @@ def save_feature_similarity_outputs( matrix = np.squeeze(matrix) method_names = list(method_names) - # Note: Saved matrix is in the original ethods order, not the reordered version for plotting + # Note: Saved matrix is in the original methods order, not the reordered version for plotting np.save(DEFAULT_OUTPUT_DIR / f"{output_name}_matrix.npy", matrix) np.save( DEFAULT_OUTPUT_DIR / f"{output_name}_method_names.npy", @@ -234,6 +253,8 @@ def plot_similarity_heatmap( figsize=(10, 8), cluster=False, cluster_method="average", + vmin=None, + vmax=None, ): method_names = list(method_names) matrix = np.squeeze(matrix) @@ -254,6 +275,7 @@ def plot_similarity_heatmap( # Highlight non-AIGM names in a different color highlight_color = NON_AIGM_COLOR highlight_method_names = NON_AIGM_SET + matrix = np.squeeze(matrix) # Optional hierarchical clustering to reorder methods based on similarity to each other if cluster: @@ -290,8 +312,8 @@ def plot_similarity_heatmap( xticklabels=method_names, yticklabels=method_names, cmap="viridis", - vmin=-0.2, - vmax=1.0, + vmin=vmin, + vmax=vmax, ) if aggregation_size is not None: @@ -320,8 +342,8 @@ def plot_similarity_heatmap( plt.tight_layout() - plt.savefig(f"{DEFAULT_OUTPUT_DIR}/pdf/{title}.pdf", bbox_inches="tight") - plt.savefig(f"{DEFAULT_OUTPUT_DIR}/png/{title}.png", dpi=600, bbox_inches="tight") + plt.savefig(f"{OUTPUT_DIR_PDF}/{title}.pdf", bbox_inches="tight") + plt.savefig(f"{OUTPUT_DIR_PNG}/{title}.png", dpi=600, bbox_inches="tight") plt.close() return plotted_methods_order @@ -380,16 +402,24 @@ def plot_similarity_heatmap( aggregated, aggregation_size = aggregate_similarity_matrices(matrices, aggregation="mean") -methods_order = plot_similarity_heatmap( - aggregated, - aggregation_size, +# Filter methods based on performance threshold +keep_idx = [i for i, method in enumerate(methods) if method in eligible_methods] +filtered_matrix = aggregated[np.ix_(keep_idx, keep_idx)] +filtered_methods = [methods[i] for i in keep_idx] + +filtered_methods_order = plot_similarity_heatmap( + matrix=filtered_matrix, + aggregation_size=aggregation_size, + method_names=filtered_methods, title="Mean dFC feature similarity between methods", cluster=True, + vmin=-0.2, + vmax=1.0, ) save_feature_similarity_outputs( - matrix=aggregated, - method_names=methods, # not reordered since aggregated did not go through hierarchical clustering in plotting function + matrix=filtered_matrix, + method_names=filtered_methods, # not reordered since aggregated did not go through hierarchical clustering in plotting function aggregation_size=aggregation_size, output_name="FS", aggregation="mean", @@ -407,12 +437,16 @@ def plot_similarity_heatmap( aggregated, aggregation_size = aggregate_similarity_matrices(matrices, aggregation="std") +# Filter methods based on performance threshold +keep_idx = [i for i, method in enumerate(methods) if method in eligible_methods] +filtered_std_matrix = aggregated[np.ix_(keep_idx, keep_idx)] + plot_similarity_heatmap( - aggregated, + filtered_std_matrix, aggregation_size, + method_names=filtered_methods, + ordered_method_names=filtered_methods_order, title="Standard deviation of dFC feature similarity between methods", - method_names=methods, - ordered_method_names=methods_order, ) @@ -432,11 +466,21 @@ def plot_task_similarity_heatmap_grid( ncols=3, tick_fontsize=3, title_fontsize=12, + vmin=None, + vmax=None, ): """Plot one heatmap per task using a shared method ordering and colorbar.""" ordered_method_names = list(ordered_method_names) original_method_names = list(original_method_names) + + # Filter methods based on performance threshold + keep_idx = [ + i for i, method in enumerate(original_method_names) if method in eligible_methods + ] + original_method_names = [original_method_names[i] for i in keep_idx] + + # Reorder based on same methods order from average heatmap order = [original_method_names.index(name) for name in ordered_method_names] if len(task_ids) > nrows * ncols: @@ -464,7 +508,12 @@ def plot_task_similarity_heatmap_grid( ) matrix = np.squeeze(aggregated) - matrix = matrix[np.ix_(order, order)] + matrix = matrix[ + np.ix_(keep_idx, keep_idx) + ] # Filter methods based on performance threshold first + matrix = matrix[ + np.ix_(order, order) + ] # Reorder based on same methods order from average heatmap sns.heatmap( matrix, @@ -472,8 +521,8 @@ def plot_task_similarity_heatmap_grid( xticklabels=ordered_method_names, yticklabels=ordered_method_names, cmap="viridis", - vmin=-0.2, - vmax=1.0, + vmin=vmin, + vmax=vmax, cbar=ax_idx == 0, cbar_ax=cbar_ax if ax_idx == 0 else None, square=True, @@ -512,8 +561,8 @@ def plot_task_similarity_heatmap_grid( hspace=0.35, ) - fig.savefig(f"{DEFAULT_OUTPUT_DIR}/pdf/{title}.pdf", bbox_inches="tight") - fig.savefig(f"{DEFAULT_OUTPUT_DIR}/png/{title}.png", dpi=600, bbox_inches="tight") + fig.savefig(f"{OUTPUT_DIR_PDF}/{title}.pdf", bbox_inches="tight") + fig.savefig(f"{OUTPUT_DIR_PNG}/{title}.png", dpi=600, bbox_inches="tight") plt.close() @@ -531,7 +580,11 @@ def plot_task_similarity_heatmap_grid( ) plot_task_similarity_heatmap_grid( - similarity=similarity, task_ids=task_ids, ordered_method_names=methods_order + similarity=similarity, + task_ids=task_ids, + ordered_method_names=filtered_methods_order, + vmin=-0.2, + vmax=1.0, ) From 037a52db6e0a26f05ea28391fd537daa3c08f77e Mon Sep 17 00:00:00 2001 From: kinichen Date: Thu, 2 Jul 2026 15:48:54 -0400 Subject: [PATCH 2/2] Pre-commit fix: Add performance filter for methods in heatmaps and enhance AS vs FS scatterplot --- HT_LLM/similarity/algorithm_feature_scatterplot.py | 8 ++++---- HT_LLM/similarity/algorithm_similarity.py | 4 ++-- HT_LLM/similarity/filter_methods_by_performance.py | 2 +- HT_LLM/similarity/heatmaps_feature_similarity.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/HT_LLM/similarity/algorithm_feature_scatterplot.py b/HT_LLM/similarity/algorithm_feature_scatterplot.py index fc96207..509f2a4 100644 --- a/HT_LLM/similarity/algorithm_feature_scatterplot.py +++ b/HT_LLM/similarity/algorithm_feature_scatterplot.py @@ -12,16 +12,16 @@ print(f"Method Filtering Threshold: {threshold}") OUTPUT_DIR = ( Path("HT_LLM/similarity/scatterplot_algorithm_feature_results") - / f"threshold_{int(threshold*100)}" + / f"threshold_{int(threshold * 100)}" ) OUTPUT_DIR.mkdir(parents=True, exist_ok=True) algorithm_pairs = pd.read_csv( - f"HT_LLM/similarity/algorithm_similarity_results/threshold_{int(threshold*100)}/AS_BOO_weighted_jaccard_pairs.csv" + f"HT_LLM/similarity/algorithm_similarity_results/threshold_{int(threshold * 100)}/AS_BOO_weighted_jaccard_pairs.csv" ) feature_pairs = pd.read_csv( - f"HT_LLM/similarity/feature_similarity_results/threshold_{int(threshold*100)}/FS_pairs.csv" + f"HT_LLM/similarity/feature_similarity_results/threshold_{int(threshold * 100)}/FS_pairs.csv" ) scatter_df = feature_pairs.merge( @@ -35,7 +35,7 @@ # so performance difference is only coloured for those methods automatically. performance_diff_table_path = ( Path("sample_data") - / f"threshold_{int(threshold*100)}" + / f"threshold_{int(threshold * 100)}" / "filtered_performance_differences.csv" ) diff --git a/HT_LLM/similarity/algorithm_similarity.py b/HT_LLM/similarity/algorithm_similarity.py index b7fc4d7..c93a3c4 100644 --- a/HT_LLM/similarity/algorithm_similarity.py +++ b/HT_LLM/similarity/algorithm_similarity.py @@ -70,7 +70,7 @@ threshold = 0.6 # must exist; else, run filter_methods_by_performance.py with the desired threshold first eligible_methods_path = os.path.join( - "sample_data", f"threshold_{int(threshold*100)}", "filtered_methods.npy" + "sample_data", f"threshold_{int(threshold * 100)}", "filtered_methods.npy" ) eligible_methods = set( np.load(eligible_methods_path, allow_pickle=True).astype(str).tolist() @@ -80,7 +80,7 @@ DEFAULT_OUTPUT_DIR = ( Path("HT_LLM/similarity/algorithm_similarity_results") - / f"threshold_{int(threshold*100)}" + / f"threshold_{int(threshold * 100)}" ) DEFAULT_OUTPUT_DIR.mkdir(parents=True, exist_ok=True) diff --git a/HT_LLM/similarity/filter_methods_by_performance.py b/HT_LLM/similarity/filter_methods_by_performance.py index 71f8faf..9815273 100644 --- a/HT_LLM/similarity/filter_methods_by_performance.py +++ b/HT_LLM/similarity/filter_methods_by_performance.py @@ -15,7 +15,7 @@ # print(performances.keys()) metric = "SVM balanced accuracy" # change to another performance key if needed -output_root = Path("sample_data") / f"threshold_{int(threshold*100)}" +output_root = Path("sample_data") / f"threshold_{int(threshold * 100)}" output_root.mkdir(parents=True, exist_ok=True) output_path = output_root / "filtered_methods.npy" diff --git a/HT_LLM/similarity/heatmaps_feature_similarity.py b/HT_LLM/similarity/heatmaps_feature_similarity.py index c28755b..8190e07 100644 --- a/HT_LLM/similarity/heatmaps_feature_similarity.py +++ b/HT_LLM/similarity/heatmaps_feature_similarity.py @@ -21,7 +21,7 @@ threshold = 0.6 # must exist; else, run filter_methods_by_performance.py with the desired threshold first eligible_methods_path = os.path.join( - "sample_data", f"threshold_{int(threshold*100)}", "filtered_methods.npy" + "sample_data", f"threshold_{int(threshold * 100)}", "filtered_methods.npy" ) eligible_methods = set( np.load(eligible_methods_path, allow_pickle=True).astype(str).tolist() @@ -32,7 +32,7 @@ # %% DEFAULT_OUTPUT_DIR = Path( - f"HT_LLM/similarity/feature_similarity_results/threshold_{int(threshold*100)}" + f"HT_LLM/similarity/feature_similarity_results/threshold_{int(threshold * 100)}" ) OUTPUT_DIR_PDF = DEFAULT_OUTPUT_DIR / "pdf" OUTPUT_DIR_PNG = DEFAULT_OUTPUT_DIR / "png"