diff --git a/DashAI/alembic/versions/0b08a41e4dbd_add_stories_to_local_explainer.py b/DashAI/alembic/versions/0b08a41e4dbd_add_stories_to_local_explainer.py new file mode 100644 index 000000000..010173bdc --- /dev/null +++ b/DashAI/alembic/versions/0b08a41e4dbd_add_stories_to_local_explainer.py @@ -0,0 +1,25 @@ +"""Add stories to local_explainer + +Revision ID: 0b08a41e4dbd +Revises: 6273b0d04af3 +Create Date: 2026-08-04 10:46:18.347215 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0b08a41e4dbd" +down_revision: Union[str, None] = "6273b0d04af3" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("local_explainer", sa.Column("stories", sa.JSON(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("local_explainer", "stories") diff --git a/DashAI/alembic/versions/6273b0d04af3_add_story_fields_to_global_explainer.py b/DashAI/alembic/versions/6273b0d04af3_add_story_fields_to_global_explainer.py new file mode 100644 index 000000000..1c0f1c2e9 --- /dev/null +++ b/DashAI/alembic/versions/6273b0d04af3_add_story_fields_to_global_explainer.py @@ -0,0 +1,33 @@ +"""Add story fields to global_explainer + +Revision ID: 6273b0d04af3 +Revises: c4e8a1d20f3b +Create Date: 2026-08-03 23:22:57.981298 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "6273b0d04af3" +down_revision: Union[str, None] = "c4e8a1d20f3b" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "global_explainer", + sa.Column("story", sa.String(), nullable=True), + ) + op.add_column( + "global_explainer", + sa.Column("story_huey_id", sa.String(), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("global_explainer", "story_huey_id") + op.drop_column("global_explainer", "story") diff --git a/DashAI/back/api/api_v1/endpoints/explainers.py b/DashAI/back/api/api_v1/endpoints/explainers.py index 617afa2bd..72e380cdf 100755 --- a/DashAI/back/api/api_v1/endpoints/explainers.py +++ b/DashAI/back/api/api_v1/endpoints/explainers.py @@ -25,6 +25,7 @@ if TYPE_CHECKING: from sqlalchemy.orm import sessionmaker + logging.basicConfig(level=logging.DEBUG) log = logging.getLogger(__name__) diff --git a/DashAI/back/dependencies/database/models.py b/DashAI/back/dependencies/database/models.py index c1160db89..6ba27b866 100644 --- a/DashAI/back/dependencies/database/models.py +++ b/DashAI/back/dependencies/database/models.py @@ -349,6 +349,8 @@ class GlobalExplainer(Base): explanation_path: Mapped[str] = mapped_column(String, nullable=True) plot_path: Mapped[str] = mapped_column(String, nullable=True) plot_overrides: Mapped[JSON] = mapped_column(JSON, nullable=True) + story: Mapped[str] = mapped_column(String, nullable=True) + story_huey_id: Mapped[str] = mapped_column(String, nullable=True) parameters: Mapped[JSON] = mapped_column(JSON) created: Mapped[DateTime] = mapped_column(DateTime, default=datetime.now) status: Mapped[Enum] = mapped_column( @@ -392,6 +394,7 @@ class LocalExplainer(Base): explanation_path: Mapped[str] = mapped_column(String, nullable=True) plots_path: Mapped[str] = mapped_column(String, nullable=True) plot_overrides: Mapped[JSON] = mapped_column(JSON, nullable=True) + stories: Mapped[JSON] = mapped_column(JSON, nullable=True) input_dataset_path: Mapped[str] = mapped_column(String, nullable=True) parameters: Mapped[JSON] = mapped_column(JSON) fit_parameters: Mapped[JSON] = mapped_column(JSON) diff --git a/DashAI/back/dependencies/registry/component_registry.py b/DashAI/back/dependencies/registry/component_registry.py index e97115ab6..4153c7871 100644 --- a/DashAI/back/dependencies/registry/component_registry.py +++ b/DashAI/back/dependencies/registry/component_registry.py @@ -171,6 +171,36 @@ def _get_base_type(self, new_component: type) -> str: return base_classes_cantidates[0].TYPE + @staticmethod + def _overrides_story(component: type) -> bool: + """Whether ``component`` provides its own ``story`` implementation. + + ``BaseGlobalExplainer``/``BaseLocalExplainer`` both declare a + ``story`` method that just raises ``NotImplementedError``, so + inheriting it unmodified means the explainer does not support story + generation. Walks the MRO (most derived first) to find which class + actually defines ``story``, rather than importing the base classes + directly (avoids a needless coupling to the explainability module). + + Parameters + ---------- + component : type + The component class to inspect. + + Returns + ------- + bool + ``True`` if some class more specific than the two base + explainer classes defines ``story``. + """ + for klass in component.__mro__: + if "story" in vars(klass): + return klass.__name__ not in ( + "BaseGlobalExplainer", + "BaseLocalExplainer", + ) + return False + @staticmethod @beartype def _collect_compatible_components(component: type) -> List[str]: @@ -251,6 +281,11 @@ def register_component(self, new_component: Type) -> None: "color": getattr(new_component, "COLOR", None), } + if hasattr(new_component, "story"): + new_register_component["supports_story"] = self._overrides_story( + new_component + ) + if base_type not in self._registry: self._registry[base_type] = {new_component.__name__: new_register_component} else: diff --git a/DashAI/back/explainability/explainers/contrastive_shap.py b/DashAI/back/explainability/explainers/contrastive_shap.py index 3c6e8243d..b671fe8c6 100644 --- a/DashAI/back/explainability/explainers/contrastive_shap.py +++ b/DashAI/back/explainability/explainers/contrastive_shap.py @@ -4,7 +4,6 @@ ArtifactGroup, GroupedArtifacts, PlotlyArtifact, - TextArtifact, ) from DashAI.back.core.schema_fields import ( BaseSchema, @@ -307,7 +306,8 @@ def explain_instance(self, instances): ------- dict Dictionary with, for each instance, the fact and foil classes and - the per-feature attribution difference (fact minus foil). + the per-feature attribution difference (fact minus foil). Also + cached on ``self.explanation`` for :meth:`story`. """ import numpy as np @@ -342,6 +342,7 @@ def explain_instance(self, instances): "delta_values": np.round(delta, 3).tolist(), } + self.explanation = explanation return explanation def _create_plot(self, data, fact_name, foil_name, fact_prob, foil_prob): @@ -398,8 +399,62 @@ def _create_plot(self, data, fact_name, foil_name, fact_prob, foil_prob): return fig + def _summarize_instance(self, instance: dict, metadata: dict) -> str: + """Build the "why P rather than Q" sentence for one explained instance. + + Computed directly from the explanation's own numbers (fact/foil + class, their probabilities, and the top contributing features by + absolute delta), independent of how :meth:`plot` renders them, so + :meth:`story` can call this without depending on a rendered artifact. + + Parameters + ---------- + instance : dict + One instance's entry from the explanation dict (i.e. + ``explanation[i]`` for some index ``i``, excluding the top-level + ``"metadata"`` entry). + metadata : dict + The explanation's ``"metadata"`` entry (``feature_names``, + ``target_names``). + + Returns + ------- + str + The contrastive summary sentence. + """ + import numpy as np + + feature_names = metadata["feature_names"] + target_names = metadata["target_names"] + + fact_class = instance["fact_class"] + foil_class = instance["foil_class"] + fact_name = target_names[fact_class] + foil_name = target_names[foil_class] + prediction = instance["model_prediction"] + fact_prob = float(np.round(prediction[fact_class], 3)) + foil_prob = float(np.round(prediction[foil_class], 3)) + + instance_values = instance["instance_values"] + delta = np.asarray(instance["delta_values"]) + # Stable sort to match plot()'s previous pandas-based tie-breaking: + # largest |delta| first among the top 3. + top_indices = np.argsort(np.abs(delta), kind="stable")[::-1][:3] + top_features = ", ".join( + f"{feature_names[j]}={instance_values[j]}" for j in top_indices + ) + + return ( + f"The model predicted {fact_name} (p={fact_prob}) rather than " + f"{foil_name} (p={foil_prob}) mainly because of: " + f"{top_features}." + ) + def plot(self, explanation: dict) -> List[GroupedArtifacts]: - """Render each instance as a contrastive bar plot plus a text summary. + """Render each instance as a contrastive bar plot. + + The narrative summary is not computed here: it is only built on + demand by :meth:`story`. Parameters ---------- @@ -410,7 +465,7 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: ------- List[GroupedArtifacts] A single grouped artifact with one group per explained instance, - each holding that instance's contrastive plot and text summary. + each holding that instance's contrastive plot. """ import numpy as np import pandas as pd @@ -449,21 +504,51 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: fig = self._create_plot(data, fact_name, foil_name, fact_prob, foil_prob) plot = PlotlyArtifact(payload=fig) - top = data.iloc[::-1].head(3) - top_features = ", ".join( - f"{feature}={value}" - for feature, value in zip( - top["features"].tolist(), - top["values"].tolist(), - strict=True, - ) - ) - summary = ( - f"The model predicted {fact_name} (p={fact_prob}) rather than " - f"{foil_name} (p={foil_prob}) mainly because of: " - f"{top_features}." - ) - text = TextArtifact(payload=summary) - groups.append(ArtifactGroup(title=title, artifacts=[plot, text])) + groups.append(ArtifactGroup(title=title, artifacts=[plot])) return [GroupedArtifacts(groups=groups)] + + def story(self, explainer_output, prediction_context): + """Build the "why P rather than Q" sentence from ``self.explanation``. + + Computed on demand, only when a story is requested: :meth:`plot` + never builds this narrative, so no cost is paid unless it is asked + for. + + Parameters + ---------- + explainer_output : GroupedArtifacts + The explained instance's group, as produced by :meth:`plot`. Only + its ``title`` (``"Instance {n}"``) is used, to recover which + entry of ``self.explanation`` this call is about. + prediction_context : DashAIDataset + Unused; the summary is built entirely from ``self.explanation``. + + Returns + ------- + str + The instance's contrastive summary sentence. + + Raises + ------ + ValueError + If the instance's title cannot be matched to an explained + instance, or ``self.explanation`` was not set before calling this + (see :meth:`explain_instance`). + """ + if not self.explanation: + raise ValueError( + "self.explanation must be set before calling story() " + "(see explain_instance)." + ) + + title = explainer_output.groups[0].title or "" + try: + index = int(title.rsplit(" ", 1)[-1]) - 1 + instance = self.explanation[index] + except (ValueError, KeyError) as e: + raise ValueError( + f"Could not match group title {title!r} to an explained instance." + ) from e + + return self._summarize_instance(instance, self.explanation["metadata"]) diff --git a/DashAI/back/explainability/explainers/dice_counterfactual.py b/DashAI/back/explainability/explainers/dice_counterfactual.py index 381fd6e6a..923fbecdf 100644 --- a/DashAI/back/explainability/explainers/dice_counterfactual.py +++ b/DashAI/back/explainability/explainers/dice_counterfactual.py @@ -5,7 +5,6 @@ GroupedArtifacts, TableArtifact, TablePayload, - TextArtifact, ) from DashAI.back.core.schema_fields import ( BaseSchema, @@ -373,10 +372,59 @@ def explain_instance(self, instances): "counterfactuals": counterfactuals, } + self.explanation = explanation return explanation + def _summarize_instance(self, instance: dict, metadata: dict) -> str: + """Build the DiCE counterfactual comparison sentence for one instance. + + Computed directly from the explanation's own numbers, independent of + how :meth:`plot` renders the comparison table, so :meth:`story` can + call this without depending on a rendered artifact. + + Parameters + ---------- + instance : dict + One instance's entry from the explanation dict. + metadata : dict + The explanation's ``"metadata"`` entry (``target_names``). + + Returns + ------- + str + The counterfactual comparison summary. + """ + import numpy as np + + target_names = metadata["target_names"] + predicted_class = instance["predicted_class"] + predicted_name = target_names[predicted_class] + predicted_prob = float( + np.round(instance["model_prediction"][predicted_class], 3) + ) + counterfactuals = instance["counterfactuals"] + + if not counterfactuals: + return ( + f"The model predicted {predicted_name} " + f"(p={predicted_prob}). DiCE could not generate " + "counterfactuals for this instance." + ) + + lines = [f"The model predicted {predicted_name} (p={predicted_prob})."] + for cf_idx, counterfactual in enumerate(counterfactuals): + cf_name = target_names[counterfactual["predicted_class"]] + changed = ", ".join(counterfactual["changed_features"]) or "nothing" + lines.append( + f"Counterfactual {cf_idx + 1}: changing {changed} yields {cf_name}." + ) + return "\n".join(lines) + def plot(self, explanation: dict) -> List[GroupedArtifacts]: - """Render each instance as a comparison table plus a text summary. + """Render each instance as a comparison table. + + The narrative summary is not computed here: it is only built on + demand by :meth:`story`. Parameters ---------- @@ -387,10 +435,8 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: ------- List[GroupedArtifacts] A single grouped artifact with one group per explained instance, - each holding that instance's comparison table and text summary. + each holding that instance's comparison table. """ - import numpy as np - exp = explanation.copy() metadata = exp.pop("metadata") feature_names = metadata["feature_names"] @@ -401,9 +447,6 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: instance = exp[i] predicted_class = instance["predicted_class"] predicted_name = target_names[predicted_class] - predicted_prob = float( - np.round(instance["model_prediction"][predicted_class], 3) - ) counterfactuals = instance["counterfactuals"] columns = ["Feature", "Instance"] + [ @@ -431,24 +474,51 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: table = TableArtifact( payload=TablePayload(columns=columns, rows=rows, highlight=highlight), ) - - if counterfactuals: - lines = [f"The model predicted {predicted_name} (p={predicted_prob})."] - for cf_idx, counterfactual in enumerate(counterfactuals): - cf_name = target_names[counterfactual["predicted_class"]] - changed = ", ".join(counterfactual["changed_features"]) or "nothing" - lines.append( - f"Counterfactual {cf_idx + 1}: changing {changed} " - f"yields {cf_name}." - ) - summary = "\n".join(lines) - else: - summary = ( - f"The model predicted {predicted_name} " - f"(p={predicted_prob}). DiCE could not generate " - "counterfactuals for this instance." - ) - text = TextArtifact(payload=summary) - groups.append(ArtifactGroup(title=title, artifacts=[table, text])) + groups.append(ArtifactGroup(title=title, artifacts=[table])) return [GroupedArtifacts(groups=groups)] + + def story(self, explainer_output, prediction_context): + """Build the DiCE counterfactual sentence from ``self.explanation``. + + Computed on demand, only when a story is requested: :meth:`plot` + never builds this narrative, so no cost is paid unless it is asked + for. + + Parameters + ---------- + explainer_output : GroupedArtifacts + The explained instance's group, as produced by :meth:`plot`. Only + its ``title`` (``"Instance {n}"``) is used, to recover which + entry of ``self.explanation`` this call is about. + prediction_context : DashAIDataset + Unused; the summary is built entirely from ``self.explanation``. + + Returns + ------- + str + The instance's counterfactual comparison summary. + + Raises + ------ + ValueError + If the instance's title cannot be matched to an explained + instance, or ``self.explanation`` was not set before calling this + (see :meth:`explain_instance`). + """ + if not self.explanation: + raise ValueError( + "self.explanation must be set before calling story() " + "(see explain_instance)." + ) + + title = explainer_output.groups[0].title or "" + try: + index = int(title.rsplit(" ", 1)[-1]) - 1 + instance = self.explanation[index] + except (ValueError, KeyError) as e: + raise ValueError( + f"Could not match group title {title!r} to an explained instance." + ) from e + + return self._summarize_instance(instance, self.explanation["metadata"]) diff --git a/DashAI/back/explainability/explainers/grad_cam.py b/DashAI/back/explainability/explainers/grad_cam.py index d22b95a30..b6b1c15dd 100644 --- a/DashAI/back/explainability/explainers/grad_cam.py +++ b/DashAI/back/explainability/explainers/grad_cam.py @@ -3,7 +3,6 @@ from DashAI.back.core.artifacts import ( ArtifactGroup, GroupedArtifacts, - TextArtifact, ) from DashAI.back.core.schema_fields import ( BaseSchema, @@ -241,10 +240,48 @@ def explain_instance(self, instances): "predicted_class": predicted_class, } + self.explanation = explanation return explanation + def _summarize_instance(self, instance: dict, metadata: dict) -> str: + """Build the heatmap description sentence for one explained image. + + Computed directly from the explanation's own numbers, independent of + how :meth:`plot` renders the heatmap overlay, so :meth:`story` can + call this without depending on a rendered artifact. + + Parameters + ---------- + instance : dict + One image's entry from the explanation dict. + metadata : dict + The explanation's ``"metadata"`` entry (``target_names``). + + Returns + ------- + str + The heatmap description sentence. + """ + import numpy as np + + target_names = metadata["target_names"] + predicted_class = instance["predicted_class"] + predicted_name = target_names[predicted_class] + predicted_prob = float( + np.round(instance["model_prediction"][predicted_class], 3) + ) + + return ( + f"The model predicted {predicted_name} " + f"(p={predicted_prob}). Highlighted regions are the " + "areas whose activations most supported this class." + ) + def plot(self, explanation: dict) -> List[GroupedArtifacts]: - """Render each image as a heatmap overlay plus a text summary. + """Render each image as a heatmap overlay. + + The narrative summary is not computed here: it is only built on + demand by :meth:`story`. Parameters ---------- @@ -255,7 +292,7 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: ------- List[GroupedArtifacts] A single grouped artifact with one group per explained image, each - holding that image's heatmap overlay and text summary. + holding that image's heatmap overlay. """ import numpy as np @@ -280,13 +317,51 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: overlay = heatmap_overlay_artifact( instance["image"], instance["heatmap"], title, subtitle ) - text = TextArtifact( - payload=( - f"The model predicted {predicted_name} " - f"(p={predicted_prob}). Highlighted regions are the " - "areas whose activations most supported this class." - ), - ) - groups.append(ArtifactGroup(title=title, artifacts=[overlay, text])) + groups.append(ArtifactGroup(title=title, artifacts=[overlay])) return [GroupedArtifacts(groups=groups)] + + def story(self, explainer_output, prediction_context): + """Build the heatmap description sentence from ``self.explanation``. + + Computed on demand, only when a story is requested: :meth:`plot` + never builds this narrative, so no cost is paid unless it is asked + for. + + Parameters + ---------- + explainer_output : GroupedArtifacts + The explained image's group, as produced by :meth:`plot`. Only + its ``title`` (``"Image {n}"``) is used, to recover which entry + of ``self.explanation`` this call is about. + prediction_context : DashAIDataset + Unused; the summary is built entirely from ``self.explanation``. + + Returns + ------- + str + The instance's heatmap description. + + Raises + ------ + ValueError + If the instance's title cannot be matched to an explained + instance, or ``self.explanation`` was not set before calling this + (see :meth:`explain_instance`). + """ + if not self.explanation: + raise ValueError( + "self.explanation must be set before calling story() " + "(see explain_instance)." + ) + + title = explainer_output.groups[0].title or "" + try: + index = int(title.rsplit(" ", 1)[-1]) - 1 + instance = self.explanation[index] + except (ValueError, KeyError) as e: + raise ValueError( + f"Could not match group title {title!r} to an explained instance." + ) from e + + return self._summarize_instance(instance, self.explanation["metadata"]) diff --git a/DashAI/back/explainability/explainers/nearest_counterfactual.py b/DashAI/back/explainability/explainers/nearest_counterfactual.py index 948d5b88b..ad1768408 100644 --- a/DashAI/back/explainability/explainers/nearest_counterfactual.py +++ b/DashAI/back/explainability/explainers/nearest_counterfactual.py @@ -5,7 +5,6 @@ GroupedArtifacts, TableArtifact, TablePayload, - TextArtifact, ) from DashAI.back.core.schema_fields import ( BaseSchema, @@ -334,10 +333,60 @@ def explain_instance(self, instances): "counterfactuals": counterfactuals, } + self.explanation = explanation return explanation + def _summarize_instance(self, instance: dict, metadata: dict) -> str: + """Build the counterfactual comparison sentence for one instance. + + Computed directly from the explanation's own numbers, independent of + how :meth:`plot` renders the comparison table, so :meth:`story` can + call this without depending on a rendered artifact. + + Parameters + ---------- + instance : dict + One instance's entry from the explanation dict. + metadata : dict + The explanation's ``"metadata"`` entry (``target_names``). + + Returns + ------- + str + The counterfactual comparison summary. + """ + import numpy as np + + target_names = metadata["target_names"] + predicted_class = instance["predicted_class"] + predicted_name = target_names[predicted_class] + predicted_prob = float( + np.round(instance["model_prediction"][predicted_class], 3) + ) + counterfactuals = instance["counterfactuals"] + + if not counterfactuals: + return ( + f"The model predicted {predicted_name} (p={predicted_prob}). " + "No counterfactual examples were found in the training data." + ) + + lines = [f"The model predicted {predicted_name} (p={predicted_prob})."] + for cf_idx, counterfactual in enumerate(counterfactuals): + cf_name = target_names[counterfactual["predicted_class"]] + changed = ", ".join(counterfactual["changed_features"]) or "nothing" + lines.append( + f"Counterfactual {cf_idx + 1}: changing {changed} " + f"yields {cf_name} " + f"(distance {counterfactual['distance']})." + ) + return "\n".join(lines) + def plot(self, explanation: dict) -> List[GroupedArtifacts]: - """Render each instance as a comparison table plus a text summary. + """Render each instance as a comparison table. + + The narrative summary is not computed here: it is only built on + demand by :meth:`story`. Parameters ---------- @@ -348,10 +397,8 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: ------- List[GroupedArtifacts] A single grouped artifact with one group per explained instance, - each holding that instance's comparison table and text summary. + each holding that instance's comparison table. """ - import numpy as np - exp = explanation.copy() metadata = exp.pop("metadata") feature_names = metadata["feature_names"] @@ -363,9 +410,6 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: instance_values = instance["instance_values"] predicted_class = instance["predicted_class"] predicted_name = target_names[predicted_class] - predicted_prob = float( - np.round(instance["model_prediction"][predicted_class], 3) - ) counterfactuals = instance["counterfactuals"] columns = ["Feature", "Instance"] + [ @@ -393,24 +437,51 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: table = TableArtifact( payload=TablePayload(columns=columns, rows=rows, highlight=highlight), ) - - if counterfactuals: - lines = [f"The model predicted {predicted_name} (p={predicted_prob})."] - for cf_idx, counterfactual in enumerate(counterfactuals): - cf_name = target_names[counterfactual["predicted_class"]] - changed = ", ".join(counterfactual["changed_features"]) or "nothing" - lines.append( - f"Counterfactual {cf_idx + 1}: changing {changed} " - f"yields {cf_name} " - f"(distance {counterfactual['distance']})." - ) - summary = "\n".join(lines) - else: - summary = ( - f"The model predicted {predicted_name} (p={predicted_prob}). " - "No counterfactual examples were found in the training data." - ) - text = TextArtifact(payload=summary) - groups.append(ArtifactGroup(title=title, artifacts=[table, text])) + groups.append(ArtifactGroup(title=title, artifacts=[table])) return [GroupedArtifacts(groups=groups)] + + def story(self, explainer_output, prediction_context): + """Build the counterfactual comparison sentence from ``self.explanation``. + + Computed on demand, only when a story is requested: :meth:`plot` + never builds this narrative, so no cost is paid unless it is asked + for. + + Parameters + ---------- + explainer_output : GroupedArtifacts + The explained instance's group, as produced by :meth:`plot`. Only + its ``title`` (``"Instance {n}"``) is used, to recover which + entry of ``self.explanation`` this call is about. + prediction_context : DashAIDataset + Unused; the summary is built entirely from ``self.explanation``. + + Returns + ------- + str + The instance's counterfactual comparison summary. + + Raises + ------ + ValueError + If the instance's title cannot be matched to an explained + instance, or ``self.explanation`` was not set before calling this + (see :meth:`explain_instance`). + """ + if not self.explanation: + raise ValueError( + "self.explanation must be set before calling story() " + "(see explain_instance)." + ) + + title = explainer_output.groups[0].title or "" + try: + index = int(title.rsplit(" ", 1)[-1]) - 1 + instance = self.explanation[index] + except (ValueError, KeyError) as e: + raise ValueError( + f"Could not match group title {title!r} to an explained instance." + ) from e + + return self._summarize_instance(instance, self.explanation["metadata"]) diff --git a/DashAI/back/explainability/explainers/occlusion_saliency.py b/DashAI/back/explainability/explainers/occlusion_saliency.py index 5baa6a18d..9d49961fe 100644 --- a/DashAI/back/explainability/explainers/occlusion_saliency.py +++ b/DashAI/back/explainability/explainers/occlusion_saliency.py @@ -3,7 +3,6 @@ from DashAI.back.core.artifacts import ( ArtifactGroup, GroupedArtifacts, - TextArtifact, ) from DashAI.back.core.schema_fields import ( BaseSchema, @@ -299,10 +298,48 @@ def explain_instance(self, instances): "predicted_class": predicted_class, } + self.explanation = explanation return explanation + def _summarize_instance(self, instance: dict, metadata: dict) -> str: + """Build the saliency description sentence for one explained image. + + Computed directly from the explanation's own numbers, independent of + how :meth:`plot` renders the saliency overlay, so :meth:`story` can + call this without depending on a rendered artifact. + + Parameters + ---------- + instance : dict + One image's entry from the explanation dict. + metadata : dict + The explanation's ``"metadata"`` entry (``target_names``). + + Returns + ------- + str + The saliency description sentence. + """ + import numpy as np + + target_names = metadata["target_names"] + predicted_class = instance["predicted_class"] + predicted_name = target_names[predicted_class] + predicted_prob = float( + np.round(instance["model_prediction"][predicted_class], 3) + ) + + return ( + f"The model predicted {predicted_name} " + f"(p={predicted_prob}). Highlighted regions are those " + "whose occlusion most lowered that probability." + ) + def plot(self, explanation: dict) -> List[GroupedArtifacts]: - """Render each image as a saliency overlay plus a text summary. + """Render each image as a saliency overlay. + + The narrative summary is not computed here: it is only built on + demand by :meth:`story`. Parameters ---------- @@ -313,7 +350,7 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: ------- List[GroupedArtifacts] A single grouped artifact with one group per explained image, each - holding that image's saliency overlay and text summary. + holding that image's saliency overlay. """ import numpy as np @@ -335,13 +372,51 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: overlay = heatmap_overlay_artifact( instance["image"], instance["heatmap"], title, subtitle ) - text = TextArtifact( - payload=( - f"The model predicted {predicted_name} " - f"(p={predicted_prob}). Highlighted regions are those " - "whose occlusion most lowered that probability." - ), - ) - groups.append(ArtifactGroup(title=title, artifacts=[overlay, text])) + groups.append(ArtifactGroup(title=title, artifacts=[overlay])) return [GroupedArtifacts(groups=groups)] + + def story(self, explainer_output, prediction_context): + """Build the saliency description sentence from ``self.explanation``. + + Computed on demand, only when a story is requested: :meth:`plot` + never builds this narrative, so no cost is paid unless it is asked + for. + + Parameters + ---------- + explainer_output : GroupedArtifacts + The explained image's group, as produced by :meth:`plot`. Only + its ``title`` (``"Image {n}"``) is used, to recover which entry + of ``self.explanation`` this call is about. + prediction_context : DashAIDataset + Unused; the summary is built entirely from ``self.explanation``. + + Returns + ------- + str + The instance's saliency description. + + Raises + ------ + ValueError + If the instance's title cannot be matched to an explained + instance, or ``self.explanation`` was not set before calling this + (see :meth:`explain_instance`). + """ + if not self.explanation: + raise ValueError( + "self.explanation must be set before calling story() " + "(see explain_instance)." + ) + + title = explainer_output.groups[0].title or "" + try: + index = int(title.rsplit(" ", 1)[-1]) - 1 + instance = self.explanation[index] + except (ValueError, KeyError) as e: + raise ValueError( + f"Could not match group title {title!r} to an explained instance." + ) from e + + return self._summarize_instance(instance, self.explanation["metadata"]) diff --git a/DashAI/back/explainability/explainers/permutation_feature_importance.py b/DashAI/back/explainability/explainers/permutation_feature_importance.py index 2bd6dc556..af7452d51 100644 --- a/DashAI/back/explainability/explainers/permutation_feature_importance.py +++ b/DashAI/back/explainability/explainers/permutation_feature_importance.py @@ -1,6 +1,7 @@ from typing import Dict, List, Union from DashAI.back.core.artifacts import ( + Artifact, ArtifactGroup, GroupedArtifacts, PlotlyArtifact, @@ -433,7 +434,8 @@ def explain(self, dataset): dict Dictionary with keys ``"features"`` (list of str), ``"importances_mean"`` (list of float, rounded to 3 dp), and - ``"importances_std"`` (list of float, rounded to 3 dp). + ``"importances_std"`` (list of float, rounded to 3 dp). Also + cached on ``self.explanation`` for :meth:`story`. """ # Lazy imports import numpy as np @@ -480,11 +482,13 @@ def explain(self, dataset): results = self._calculate_grouped_importance( X_df, y_df, feature_groups, max_samples ) - return { + explanation = { "features": results["features"], "importances_mean": np.round(results["importances_mean"], 3).tolist(), "importances_std": np.round(results["importances_std"], 3).tolist(), } + self.explanation = explanation + return explanation else: def patched_metric(y_true, y_pred_probas): @@ -519,11 +523,13 @@ def patched_metric(y_true, y_pred_probas): max_samples=max_samples, ) - return { + explanation = { "features": input_columns, "importances_mean": np.round(pfi["importances_mean"], 3).tolist(), "importances_std": np.round(pfi["importances_std"], 3).tolist(), } + self.explanation = explanation + return explanation def _create_plot(self, data) -> List[GroupedArtifacts]: """Build one selector over feature counts. @@ -589,3 +595,47 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: data = data.sort_values(by=["importances_mean"], ascending=True) return self._create_plot(data) + + def story(self, explainer_output: Union[Artifact, GroupedArtifacts]) -> str: + """Summarize the feature ranking in a deterministic natural-language story. + + Built directly from ``self.explanation`` (see :meth:`explain`), not + from ``explainer_output``: every feature count selector shows the + same ranking, so there is nothing instance-specific to pick out. + + Parameters + ---------- + explainer_output : Artifact or GroupedArtifacts + Unused; present only to satisfy the base class signature. + + Returns + ------- + str + A natural-language summary of which features the model relies on + most and least. + """ + features = self.explanation["features"] + means = self.explanation["importances_mean"] + + ranking = sorted( + zip(features, means, strict=True), key=lambda pair: pair[1], reverse=True + ) + + top = ranking[:3] + top_text = ", ".join(f"{name} ({value:+.3f})" for name, value in top) + least_name, least_value = ranking[-1] + + story = ( + f"The model relies most on {top_text}. " + f"{least_name} contributes the least to its predictions " + f"(importance={least_value:+.3f})." + ) + + if least_value <= 0: + story += ( + " A near-zero or negative importance means shuffling that " + "feature barely changed (or even improved) performance, so " + "the model may not need it." + ) + + return story diff --git a/DashAI/back/explainability/explainers/regression_kernel_shap.py b/DashAI/back/explainability/explainers/regression_kernel_shap.py index 63661beaf..4040d2913 100644 --- a/DashAI/back/explainability/explainers/regression_kernel_shap.py +++ b/DashAI/back/explainability/explainers/regression_kernel_shap.py @@ -4,7 +4,6 @@ ArtifactGroup, GroupedArtifacts, PlotlyArtifact, - TextArtifact, ) from DashAI.back.core.schema_fields import ( BaseSchema, @@ -251,10 +250,60 @@ def explain_instance(self, instances): "shap_values": np.round(contributions, 3).tolist(), } + self.explanation = explanation return explanation + def _summarize_instance( + self, instance: dict, metadata: dict, base_value: float + ) -> str: + """Build the SHAP contribution sentence for one explained instance. + + Computed directly from the explanation's own numbers (top features + by absolute SHAP value), independent of how :meth:`plot` renders + them, so :meth:`story` can call this without depending on a rendered + artifact. + + Parameters + ---------- + instance : dict + One instance's entry from the explanation dict. + metadata : dict + The explanation's ``"metadata"`` entry (``feature_names``, + ``output_column``). + base_value : float + The explanation's top-level ``"base_value"`` (SHAP baseline). + + Returns + ------- + str + The SHAP contribution summary sentence. + """ + import numpy as np + + feature_names = metadata["feature_names"] + output_column = metadata["output_column"] + prediction = instance["model_prediction"] + + instance_values = instance["instance_values"] + shap_values = np.asarray(instance["shap_values"]) + top_indices = np.argsort(np.abs(shap_values), kind="stable")[::-1][:3] + top_features = ", ".join( + f"{feature_names[j]}={instance_values[j]} ({shap_values[j]:+})" + for j in top_indices + ) + + delta = float(np.round(prediction - base_value, 3)) + return ( + f"The model predicted {output_column}={prediction}, " + f"{delta:+} from the baseline {base_value}. " + f"Main contributions: {top_features}." + ) + def plot(self, explanation: dict) -> List[GroupedArtifacts]: - """Render each instance as a SHAP bar plot plus a text summary. + """Render each instance as a SHAP bar plot. + + The narrative summary is not computed here: it is only built on + demand by :meth:`story`. Parameters ---------- @@ -265,9 +314,8 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: ------- List[GroupedArtifacts] A single grouped artifact with one group per explained instance, - each holding that instance's plotly plot and text summary. + each holding that instance's plotly plot. """ - import numpy as np import pandas as pd import plotly.graph_objs as go @@ -325,24 +373,53 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: title = f"Instance {int(i) + 1}" plot = PlotlyArtifact(payload=fig) + groups.append(ArtifactGroup(title=title, artifacts=[plot])) - top = data.iloc[::-1].head(3) - top_features = ", ".join( - f"{feature}={value} ({shap:+})" - for feature, value, shap in zip( - top["features"].tolist(), - top["values"].tolist(), - top["shap_values"].tolist(), - strict=True, - ) - ) - delta = float(np.round(prediction - base_value, 3)) - summary = ( - f"The model predicted {output_column}={prediction}, " - f"{delta:+} from the baseline {base_value}. " - f"Main contributions: {top_features}." + return [GroupedArtifacts(groups=groups)] + + def story(self, explainer_output, prediction_context): + """Build the SHAP contribution sentence from ``self.explanation``. + + Computed on demand, only when a story is requested: :meth:`plot` + never builds this narrative, so no cost is paid unless it is asked + for. + + Parameters + ---------- + explainer_output : GroupedArtifacts + The explained instance's group, as produced by :meth:`plot`. Only + its ``title`` (``"Instance {n}"``) is used, to recover which + entry of ``self.explanation`` this call is about. + prediction_context : DashAIDataset + Unused; the summary is built entirely from ``self.explanation``. + + Returns + ------- + str + The instance's SHAP contribution summary. + + Raises + ------ + ValueError + If the instance's title cannot be matched to an explained + instance, or ``self.explanation`` was not set before calling this + (see :meth:`explain_instance`). + """ + if not self.explanation: + raise ValueError( + "self.explanation must be set before calling story() " + "(see explain_instance)." ) - text = TextArtifact(payload=summary) - groups.append(ArtifactGroup(title=title, artifacts=[plot, text])) - return [GroupedArtifacts(groups=groups)] + title = explainer_output.groups[0].title or "" + try: + index = int(title.rsplit(" ", 1)[-1]) - 1 + instance = self.explanation[index] + except (ValueError, KeyError) as e: + raise ValueError( + f"Could not match group title {title!r} to an explained instance." + ) from e + + return self._summarize_instance( + instance, self.explanation["metadata"], self.explanation["base_value"] + ) diff --git a/DashAI/back/explainability/explainers/token_ablation.py b/DashAI/back/explainability/explainers/token_ablation.py index 401daf1c8..ab98fe984 100644 --- a/DashAI/back/explainability/explainers/token_ablation.py +++ b/DashAI/back/explainability/explainers/token_ablation.py @@ -4,7 +4,6 @@ ArtifactGroup, GroupedArtifacts, PlotlyArtifact, - TextArtifact, ) from DashAI.back.core.schema_fields import ( BaseSchema, @@ -269,10 +268,58 @@ def explain_instance(self, instances): "predicted_class": predicted_class, } + self.explanation = explanation return explanation + def _summarize_instance(self, instance: dict, metadata: dict) -> str: + """Build the token importance sentence for one explained instance. + + Computed directly from the explanation's own numbers (top tokens by + absolute importance), independent of how :meth:`plot` renders them, + so :meth:`story` can call this without depending on a rendered + artifact. + + Parameters + ---------- + instance : dict + One instance's entry from the explanation dict. + metadata : dict + The explanation's ``"metadata"`` entry (``target_names``). + + Returns + ------- + str + The token importance summary sentence. + """ + import numpy as np + + target_names = metadata["target_names"] + predicted_class = instance["predicted_class"] + predicted_name = target_names[predicted_class] + predicted_prob = float( + np.round(instance["model_prediction"][predicted_class], 3) + ) + + tokens = instance["tokens"] + importances = np.asarray(instance["token_importances"]) + if len(importances) > 0: + top_indices = np.argsort(np.abs(importances), kind="stable")[::-1][:3] + top_tokens = ", ".join( + f"'{tokens[j]} ({j})' ({importances[j]:+})" for j in top_indices + ) + else: + top_tokens = "none" + + return ( + f"The model predicted {predicted_name} (p={predicted_prob}). " + f"Most influential tokens: {top_tokens}." + ) + def plot(self, explanation: dict) -> List[GroupedArtifacts]: - """Render each instance as a token importance bar plot plus a summary. + """Render each instance as a token importance bar plot. + + The narrative summary is not computed here: it is only built on + demand by :meth:`story`. Parameters ---------- @@ -283,7 +330,7 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: ------- List[GroupedArtifacts] A single grouped artifact with one group per explained instance, - each holding that instance's token plot and text summary. + each holding that instance's token plot. """ import numpy as np import pandas as pd @@ -346,19 +393,51 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: title = f"Instance {int(i) + 1}" plot = PlotlyArtifact(payload=fig) + groups.append(ArtifactGroup(title=title, artifacts=[plot])) - top = data.iloc[::-1].head(3) - top_tokens = ", ".join( - f"'{token}' ({importance:+})" - for token, importance in zip( - top["tokens"].tolist(), top["importances"].tolist(), strict=True - ) - ) - summary = ( - f"The model predicted {predicted_name} (p={predicted_prob}). " - f"Most influential tokens: {top_tokens}." + return [GroupedArtifacts(groups=groups)] + + def story(self, explainer_output, prediction_context): + """Build the token importance sentence from ``self.explanation``. + + Computed on demand, only when a story is requested: :meth:`plot` + never builds this narrative, so no cost is paid unless it is asked + for. + + Parameters + ---------- + explainer_output : GroupedArtifacts + The explained instance's group, as produced by :meth:`plot`. Only + its ``title`` (``"Instance {n}"``) is used, to recover which + entry of ``self.explanation`` this call is about. + prediction_context : DashAIDataset + Unused; the summary is built entirely from ``self.explanation``. + + Returns + ------- + str + The instance's token importance summary. + + Raises + ------ + ValueError + If the instance's title cannot be matched to an explained + instance, or ``self.explanation`` was not set before calling this + (see :meth:`explain_instance`). + """ + if not self.explanation: + raise ValueError( + "self.explanation must be set before calling story() " + "(see explain_instance)." ) - text = TextArtifact(payload=summary) - groups.append(ArtifactGroup(title=title, artifacts=[plot, text])) - return [GroupedArtifacts(groups=groups)] + title = explainer_output.groups[0].title or "" + try: + index = int(title.rsplit(" ", 1)[-1]) - 1 + instance = self.explanation[index] + except (ValueError, KeyError) as e: + raise ValueError( + f"Could not match group title {title!r} to an explained instance." + ) from e + + return self._summarize_instance(instance, self.explanation["metadata"]) diff --git a/DashAI/back/job/explainer_job.py b/DashAI/back/job/explainer_job.py index 390b6e149..9d551a907 100644 --- a/DashAI/back/job/explainer_job.py +++ b/DashAI/back/job/explainer_job.py @@ -135,12 +135,24 @@ def _generate_global_explanation( with session_factory() as db: try: explanation = explainer.explain(dataset) - plot = normalize_artifacts(explainer.plot(explanation)) + raw_plot = explainer.plot(explanation) + plot = normalize_artifacts(raw_plot) except Exception as e: log.exception(e) raise JobError( "Failed to generate the explanation", ) from e + + # Deterministic stories are cheap (they only read the explanation + # already computed above) so they are generated unconditionally, + # not on demand. A story bug must never fail the explanation + # itself, so it is only logged, never re-raised. + story = None + if hasattr(explainer, "story") and raw_plot: + try: + story = explainer.story(raw_plot[0]) + except Exception as e: + log.exception(e) try: explanation_filename = f"global_explanation_{explainer_id}.pickle" explanation_path = os.path.join( @@ -163,6 +175,7 @@ def _generate_global_explanation( self.explainer_db.explanation_path = explanation_path self.explainer_db.plot_path = plot_path self.explainer_db.plot_overrides = None + self.explainer_db.story = story db.commit() except Exception as e: log.exception(e) @@ -186,7 +199,7 @@ def _generate_local_explanation( from datasets import DatasetDict from kink import di - from DashAI.back.core.artifacts import normalize_artifacts + from DashAI.back.core.artifacts import GroupedArtifacts, normalize_artifacts from DashAI.back.dataloaders.classes.dashai_dataset import ( load_dataset, prepare_for_model_session, @@ -335,14 +348,32 @@ def _generate_local_explanation( ) from e try: explanation = explainer.explain_instance(X) - plots = normalize_artifacts( - explainer.plot(explanation), create_grouped=True - ) + raw_plots = explainer.plot(explanation) + plots = normalize_artifacts(raw_plots, create_grouped=True) except Exception as e: log.exception(e) raise JobError( "Failed to generate the explanation", ) from e + + # Deterministic stories are cheap (they only read the explanation + # already computed above) so they are generated unconditionally, + # not on demand, one per explained instance. A story bug for one + # instance must never fail the explanation itself, nor block the + # other instances' stories. + stories = None + if hasattr(explainer, "story") and raw_plots: + stories = {} + for i, group in enumerate(raw_plots[0].groups): + try: + single_group = GroupedArtifacts(groups=[group]) + prediction_context = input_source.select([i]) + stories[str(i)] = explainer.story( + single_group, prediction_context + ) + except Exception as e: + log.exception(e) + stories = stories or None try: explanation_filename = f"local_explanation_{explainer_id}.pickle" explanation_path = os.path.join( @@ -366,6 +397,7 @@ def _generate_local_explanation( self.explainer_db.plots_path = plots_path self.explainer_db.input_dataset_path = input_dataset_path self.explainer_db.plot_overrides = None + self.explainer_db.stories = stories db.commit() except Exception as e: log.exception(e) diff --git a/DashAI/front/src/components/explainers/ExplainersCard.jsx b/DashAI/front/src/components/explainers/ExplainersCard.jsx index a9600660d..37d5c6500 100644 --- a/DashAI/front/src/components/explainers/ExplainersCard.jsx +++ b/DashAI/front/src/components/explainers/ExplainersCard.jsx @@ -14,6 +14,7 @@ import DeleteIcon from "@mui/icons-material/Delete"; import ZoomInIcon from "@mui/icons-material/ZoomIn"; import PropTypes from "prop-types"; import ExplainersPlot from "./ExplainersPlot"; +import ArtifactViewer from "../shared/ArtifactViewer"; import { useNavigate } from "react-router-dom"; import { deleteExplainer, @@ -35,6 +36,7 @@ export default function ExplainersCard({ onDelete, compact = false, displayName = null, + supportsStory = false, cacheEntry = null, onCacheUpdate = null, isHighlighted = false, @@ -160,21 +162,23 @@ export default function ExplainersCard({ ) : ( - - {/* Reserved slot for the future "generate story" action button. - Kept hidden until that feature lands. */} - + {scope === "global" && supportsStory && explainer.story && ( + + + + )} )} @@ -235,7 +239,11 @@ export default function ExplainersCard({ /> - + ); @@ -258,11 +266,13 @@ ExplainersCard.propTypes = { ), created: PropTypes.string, status: PropTypes.number, + story: PropTypes.string, }).isRequired, scope: PropTypes.string.isRequired, onDelete: PropTypes.func, compact: PropTypes.bool, displayName: PropTypes.string, + supportsStory: PropTypes.bool, cacheEntry: PropTypes.shape({ items: PropTypes.array, overriddenIndexes: PropTypes.arrayOf(PropTypes.number), diff --git a/DashAI/front/src/components/explainers/ExplainersPlot.jsx b/DashAI/front/src/components/explainers/ExplainersPlot.jsx index 99c8f0cec..067337a80 100644 --- a/DashAI/front/src/components/explainers/ExplainersPlot.jsx +++ b/DashAI/front/src/components/explainers/ExplainersPlot.jsx @@ -106,6 +106,24 @@ ArtifactBatch.propTypes = { siblingOffset: PropTypes.number, }; +/** + * Story for one explained instance (local explainers only). Generated + * automatically as part of the explainer job - this only displays it, the + * same box used for every other text artifact. + */ +function InstanceStoryBox({ story }) { + if (!story) return null; + return ( + + + + ); +} + +InstanceStoryBox.propTypes = { + story: PropTypes.string, +}; + /** * Render a GroupedArtifacts item: a selector listing every group, beside the * selected group's first artifact (with the rest stacked below). Holds its own @@ -115,6 +133,9 @@ ArtifactBatch.propTypes = { * explained rows dataset path so the picker shows the actual instance feature * values (the row index selects the group); global explainers omit it and get * a plain title list. + * + * `story` (optional) looks up the per-instance story text: only passed for + * local explainers whose explainer type supports it. */ function GroupedArtifactsView({ grouped, @@ -122,6 +143,7 @@ function GroupedArtifactsView({ datasetPath = null, selected: selectedProp = null, onSelect = null, + story = null, }) { const { t } = useTranslation(["explainers"]); const [localSelected, setLocalSelected] = useState(0); @@ -137,13 +159,25 @@ function GroupedArtifactsView({ ); const wide = Boolean(datasetPath); + // When the story box is shown, it takes over the space of any pre-existing + // text artifact (e.g. ContrastiveShap's plot already carries the same + // sentence as a caption) instead of showing both. Falls back to the + // unfiltered list if a group has nothing left after dropping its text + // artifact(s), so a text-only group never renders empty. + const dropCaption = Boolean(story); + const displayedArtifacts = (arts) => { + if (!dropCaption) return arts; + const withoutText = arts.filter((a) => a.type !== "text"); + return withoutText.length > 0 ? withoutText : arts; + }; + // Fullscreen navigation spans every group's artifacts (flattened), so the // viewer can page across groups even when each group has a single artifact. // The selected group's artifacts occupy the slice starting at `offset`. - const allArtifacts = groups.flatMap((g) => g.artifacts); + const allArtifacts = groups.flatMap((g) => displayedArtifacts(g.artifacts)); const offset = groups .slice(0, selected) - .reduce((n, g) => n + g.artifacts.length, 0); + .reduce((n, g) => n + displayedArtifacts(g.artifacts).length, 0); // Rendered directly (no height cap): ExplainerInstanceTable's root is // height:100%, so it fills the stretched batch cell and matches the height @@ -158,15 +192,20 @@ function GroupedArtifactsView({ ); return ( - + + + {story && } + ); } @@ -176,6 +215,9 @@ GroupedArtifactsView.propTypes = { datasetPath: PropTypes.string, selected: PropTypes.number, onSelect: PropTypes.func, + story: PropTypes.shape({ + getStory: PropTypes.func.isRequired, + }), }; /** @@ -184,7 +226,13 @@ GroupedArtifactsView.propTypes = { * width). `datasetPath` is forwarded to grouped items so local explainers get * the dataset row picker. */ -function renderItem(item, ctx, datasetPath = null, selection = null) { +function renderItem( + item, + ctx, + datasetPath = null, + selection = null, + story = null, +) { if (item.type === "grouped") { return ( ); } @@ -202,6 +251,7 @@ function renderItem(item, ctx, datasetPath = null, selection = null) { export default function ExplainersPlot({ explainer, scope, + supportsStory = false, onSaveOverride = null, onResetOverride = null, overriddenIndexes = [], @@ -216,6 +266,14 @@ export default function ExplainersPlot({ const isLocal = scope === "local"; const datasetPath = isLocal ? explainer.input_dataset_path : null; + const storyProps = + isLocal && supportsStory + ? { + getStory: (groupIndex) => + explainer.stories?.[String(groupIndex)] ?? null, + } + : null; + const getExplainerPlot = async () => { setLoading(true); try { @@ -280,18 +338,26 @@ export default function ExplainersPlot({ > {items.map((item, i) => ( - {renderItem(item, ctx, datasetPath, { - selected: cacheEntry ? (cacheEntry.selectedGroups?.[i] ?? 0) : null, - onSelect: onCacheUpdate - ? (value) => - onCacheUpdate({ - selectedGroups: { - ...(cacheEntry?.selectedGroups ?? {}), - [i]: value, - }, - }) - : null, - })} + {renderItem( + item, + ctx, + datasetPath, + { + selected: cacheEntry + ? (cacheEntry.selectedGroups?.[i] ?? 0) + : null, + onSelect: onCacheUpdate + ? (value) => + onCacheUpdate({ + selectedGroups: { + ...(cacheEntry?.selectedGroups ?? {}), + [i]: value, + }, + }) + : null, + }, + storyProps, + )} ))} @@ -301,10 +367,13 @@ export default function ExplainersPlot({ ExplainersPlot.propTypes = { explainer: PropTypes.shape({ id: PropTypes.number, + run_id: PropTypes.number, status: PropTypes.number, input_dataset_path: PropTypes.string, + stories: PropTypes.object, }).isRequired, scope: PropTypes.string.isRequired, + supportsStory: PropTypes.bool, onSaveOverride: PropTypes.func, onResetOverride: PropTypes.func, overriddenIndexes: PropTypes.arrayOf(PropTypes.number), diff --git a/DashAI/front/src/components/explainers/LazyExplainerCard.jsx b/DashAI/front/src/components/explainers/LazyExplainerCard.jsx index 0c7a03b60..53011b395 100644 --- a/DashAI/front/src/components/explainers/LazyExplainerCard.jsx +++ b/DashAI/front/src/components/explainers/LazyExplainerCard.jsx @@ -22,6 +22,7 @@ const LazyExplainerCard = React.memo(function LazyExplainerCard({ explainer, scope, displayName, + supportsStory, onDelete, cacheEntry, updateCacheEntry, @@ -80,6 +81,7 @@ const LazyExplainerCard = React.memo(function LazyExplainerCard({ explainer={explainer} scope={scope} displayName={displayName} + supportsStory={supportsStory} onDelete={onDelete} cacheEntry={cacheEntry} onCacheUpdate={onCacheUpdate} @@ -98,6 +100,7 @@ LazyExplainerCard.propTypes = { explainer: PropTypes.object.isRequired, scope: PropTypes.string.isRequired, displayName: PropTypes.string, + supportsStory: PropTypes.bool, onDelete: PropTypes.func, cacheEntry: PropTypes.object, updateCacheEntry: PropTypes.func.isRequired, diff --git a/DashAI/front/src/components/models/RunResults.jsx b/DashAI/front/src/components/models/RunResults.jsx index 1e32fc071..16bf48985 100644 --- a/DashAI/front/src/components/models/RunResults.jsx +++ b/DashAI/front/src/components/models/RunResults.jsx @@ -45,6 +45,7 @@ export default function RunResults({ highlightedExplainerKey, setHighlightedExplainerKey, explainerDisplayNames, + explainerSupportsStory, cardHeightsRef, getCacheEntry, updateCacheEntry, @@ -147,6 +148,7 @@ export default function RunResults({ fillHeight={fillHeight} scrollParent={explainerScrollParent} explainerDisplayNames={explainerDisplayNames} + explainerSupportsStory={explainerSupportsStory} cardHeightsRef={cardHeightsRef} getCacheEntry={getCacheEntry} updateCacheEntry={updateCacheEntry} diff --git a/DashAI/front/src/components/models/runResults/ExplainerResultsTab.jsx b/DashAI/front/src/components/models/runResults/ExplainerResultsTab.jsx index 48d5eede8..8806da522 100644 --- a/DashAI/front/src/components/models/runResults/ExplainerResultsTab.jsx +++ b/DashAI/front/src/components/models/runResults/ExplainerResultsTab.jsx @@ -26,6 +26,7 @@ export default function ExplainerResultsTab({ fillHeight, scrollParent, explainerDisplayNames, + explainerSupportsStory, cardHeightsRef, getCacheEntry, updateCacheEntry, @@ -141,6 +142,9 @@ export default function ExplainerResultsTab({ explainer={explainer} scope={explainerFilter} displayName={explainerDisplayNames[explainer.explainer_name]} + supportsStory={Boolean( + explainerSupportsStory[explainer.explainer_name], + )} onDelete={onDelete} cacheEntry={getCacheEntry( explainerCacheKey(explainerFilter, explainer), @@ -166,6 +170,9 @@ export default function ExplainerResultsTab({ explainer={explainer} scope={explainerFilter} displayName={explainerDisplayNames[explainer.explainer_name]} + supportsStory={Boolean( + explainerSupportsStory[explainer.explainer_name], + )} onDelete={onDelete} cacheEntry={getCacheEntry(key)} onCacheUpdate={(patch) => updateCacheEntry(key, patch)} @@ -190,6 +197,7 @@ ExplainerResultsTab.propTypes = { fillHeight: PropTypes.bool, scrollParent: PropTypes.instanceOf(Element), explainerDisplayNames: PropTypes.object.isRequired, + explainerSupportsStory: PropTypes.object.isRequired, cardHeightsRef: PropTypes.shape({ current: PropTypes.object }).isRequired, getCacheEntry: PropTypes.func.isRequired, updateCacheEntry: PropTypes.func.isRequired, diff --git a/DashAI/front/src/components/models/runResults/useRunResultsData.js b/DashAI/front/src/components/models/runResults/useRunResultsData.js index 4f69589bd..7e4c84a0f 100644 --- a/DashAI/front/src/components/models/runResults/useRunResultsData.js +++ b/DashAI/front/src/components/models/runResults/useRunResultsData.js @@ -48,6 +48,9 @@ export default function useRunResultsData({ // Explainer component name to display name, fetched once and shared so cards // do not each fetch it (a per card fetch shifted heights). const [explainerDisplayNames, setExplainerDisplayNames] = useState({}); + // Explainer component name to whether it implements story(), fetched once + // alongside the display names so cards know whether to show the button. + const [explainerSupportsStory, setExplainerSupportsStory] = useState({}); // Per card plot state (items, edits, selection), so the list can unmount // offscreen cards without refetching or losing edits. const [explainerCache, setExplainerCache] = useState({}); @@ -151,10 +154,13 @@ export default function useRunResultsData({ getComponents({ selectTypes: ["GlobalExplainer", "LocalExplainer"] }) .then((components) => { const names = {}; + const supportsStory = {}; components.forEach((component) => { names[component.name] = component.display_name || component.name; + supportsStory[component.name] = Boolean(component.supports_story); }); setExplainerDisplayNames(names); + setExplainerSupportsStory(supportsStory); }) .catch(() => {}); }, []); @@ -263,6 +269,7 @@ export default function useRunResultsData({ highlightedExplainerKey, setHighlightedExplainerKey, explainerDisplayNames, + explainerSupportsStory, cardHeightsRef, getCacheEntry, updateCacheEntry, diff --git a/DashAI/front/src/types/explainer.ts b/DashAI/front/src/types/explainer.ts index 589947600..4c5f1d2e9 100644 --- a/DashAI/front/src/types/explainer.ts +++ b/DashAI/front/src/types/explainer.ts @@ -10,4 +10,7 @@ export interface IExplainer { fit_parameters: object; created: Date; status: number; + story?: string | null; + story_huey_id?: string | null; + stories?: Record | null; } diff --git a/tests/back/api/test_explainer_story_job.py b/tests/back/api/test_explainer_story_job.py new file mode 100644 index 000000000..8ac350671 --- /dev/null +++ b/tests/back/api/test_explainer_story_job.py @@ -0,0 +1,534 @@ +import json + +import joblib +import pytest +from datasets import ClassLabel, Value +from fastapi.testclient import TestClient + +from DashAI.back.core.artifacts import ArtifactGroup, GroupedArtifacts, TextArtifact +from DashAI.back.dependencies.database.models import ( + Dataset, + GlobalExplainer, + LocalExplainer, + ModelSession, + Run, +) +from DashAI.back.dependencies.registry import ComponentRegistry +from DashAI.back.explainability.global_explainer import BaseGlobalExplainer +from DashAI.back.explainability.local_explainer import BaseLocalExplainer +from DashAI.back.job.explainer_job import ExplainerJob +from DashAI.back.models.base_model import BaseModel +from DashAI.back.tasks.base_task import BaseTask + +input_columns = ["SepalLengthCm", "SepalWidthCm", "PetalLengthCm", "PetalWidthCm"] +output_columns = ["Species"] +splits = json.dumps( + { + "train": 0.5, + "test": 0.2, + "validation": 0.3, + "is_random": True, + "has_changed": True, + "seed": 42, + "shuffle": True, + "stratify": False, + } +) + + +@pytest.fixture(scope="module", name="dataset_id") +def dataset_id(dataset_1: Dataset) -> int: + return dataset_1.id + + +class DummyTask(BaseTask): + name: str = "DummyTask" + + metadata: dict = { + "inputs_types": [ClassLabel, Value], + "outputs_types": [ClassLabel], + "inputs_cardinality": "n", + "outputs_cardinality": 1, + } + + def prepare_for_task(self, dataset, input_columns=None, output_columns=None): + return dataset + + +class DummyModel(BaseModel): + COMPATIBLE_COMPONENTS = ["DummyTask"] + + @classmethod + def get_schema(cls): + return {} + + def save(self, filename): + joblib.dump(self, filename) + + @staticmethod + def load(filename): + return DummyModel() + + def predict(self, x): + return {} + + def train(self, x_train, y_train, x_validation=None, y_validation=None): + return + + def prepare_dataset(self, dataset, is_fit=False): + return dataset + + def prepare_output(self, dataset, is_fit=False): + return dataset + + +class StoryableGlobalExplainer(BaseGlobalExplainer): + """Global explainer with a deterministic, testable story().""" + + COMPATIBLE_COMPONENTS = ["DummyTask"] + + def __init__(self, model: BaseModel) -> None: + self.model = model + self.explanation = None + + @classmethod + def get_schema(cls): + return {} + + def explain(self, dataset): + self.explanation = {"headline": "feature X matters most"} + return self.explanation + + def plot(self, explanation): + return [TextArtifact(payload="a plot summary")] + + def story(self, explainer_output): + headline = (self.explanation or {}).get("headline", "unknown") + return f"Story based on '{explainer_output.payload}': {headline}" + + +class StorylessGlobalExplainer(BaseGlobalExplainer): + """Global explainer that never defines story() at all.""" + + COMPATIBLE_COMPONENTS = ["DummyTask"] + + def __init__(self, model: BaseModel) -> None: + self.model = model + self.explanation = None + + @classmethod + def get_schema(cls): + return {} + + def explain(self, dataset): + return {"headline": "irrelevant"} + + def plot(self, explanation): + return [TextArtifact(payload="a plot summary")] + + +class BrokenStoryGlobalExplainer(BaseGlobalExplainer): + """Global explainer whose story() always raises. + + Used to prove a story bug never fails the explanation itself. + """ + + COMPATIBLE_COMPONENTS = ["DummyTask"] + + def __init__(self, model: BaseModel) -> None: + self.model = model + self.explanation = None + + @classmethod + def get_schema(cls): + return {} + + def explain(self, dataset): + return {"headline": "irrelevant"} + + def plot(self, explanation): + return [TextArtifact(payload="a plot summary")] + + def story(self, explainer_output): + raise RuntimeError("boom") + + +class StoryableLocalExplainer(BaseLocalExplainer): + """Local explainer with a deterministic, testable story() per instance.""" + + COMPATIBLE_COMPONENTS = ["DummyTask"] + + def __init__(self, model: BaseModel) -> None: + self.model = model + self.explanation = None + + @classmethod + def get_schema(cls): + return {} + + def fit(self, dataset, **kwargs): + return self + + def explain_instance(self, instances): + from DashAI.back.dataloaders.classes.dashai_dataset import to_dashai_dataset + + return {"n_instances": to_dashai_dataset(instances).num_rows} + + def plot(self, explanation): + groups = [ + ArtifactGroup( + title=f"Instance {i}", artifacts=[TextArtifact(payload=f"plot {i}")] + ) + for i in range(explanation["n_instances"]) + ] + return [GroupedArtifacts(groups=groups)] + + def story(self, explainer_output, prediction_context): + group = explainer_output.groups[0] + text = group.artifacts[0].payload + return f"Local story for '{text}', context rows={prediction_context.num_rows}" + + +class StorylessLocalExplainer(BaseLocalExplainer): + """Local explainer that never defines story() at all.""" + + COMPATIBLE_COMPONENTS = ["DummyTask"] + + def __init__(self, model: BaseModel) -> None: + self.model = model + self.explanation = None + + @classmethod + def get_schema(cls): + return {} + + def fit(self, dataset, **kwargs): + return self + + def explain_instance(self, instances): + return {} + + def plot(self, explanation): + return [TextArtifact(payload="a plot summary")] + + +class BrokenStoryLocalExplainer(BaseLocalExplainer): + """Local explainer whose story() always raises for every instance. + + Used to prove a story bug never fails the explanation itself, and does + not block other instances (there are none left to block here, but the + explanation must still finish successfully with stories left empty). + """ + + COMPATIBLE_COMPONENTS = ["DummyTask"] + + def __init__(self, model: BaseModel) -> None: + self.model = model + self.explanation = None + + @classmethod + def get_schema(cls): + return {} + + def fit(self, dataset, **kwargs): + return self + + def explain_instance(self, instances): + from DashAI.back.dataloaders.classes.dashai_dataset import to_dashai_dataset + + return {"n_instances": to_dashai_dataset(instances).num_rows} + + def plot(self, explanation): + groups = [ + ArtifactGroup( + title=f"Instance {i}", artifacts=[TextArtifact(payload=f"plot {i}")] + ) + for i in range(explanation["n_instances"]) + ] + return [GroupedArtifacts(groups=groups)] + + def story(self, explainer_output, prediction_context): + raise RuntimeError("boom") + + +@pytest.fixture(autouse=True, name="test_registry") +def setup_test_registry(client, monkeypatch: pytest.MonkeyPatch): + container = client.app.container + + test_registry = ComponentRegistry( + initial_components=[ + DummyTask, + DummyModel, + StoryableGlobalExplainer, + StorylessGlobalExplainer, + BrokenStoryGlobalExplainer, + StoryableLocalExplainer, + StorylessLocalExplainer, + BrokenStoryLocalExplainer, + ExplainerJob, + ] + ) + + monkeypatch.setitem( + container._services, + "component_registry", + test_registry, + ) + return test_registry + + +@pytest.fixture(scope="module", name="model_session_id") +def create_model_session(client: TestClient, dataset_id: int): + container = client.app.container + session_factory = container["session_factory"] + + with session_factory() as db: + model_session = ModelSession( + dataset_id=dataset_id, + name="DummyExperiment", + task_name="DummyTask", + input_columns=input_columns, + output_columns=output_columns, + splits=splits, + ) + db.add(model_session) + db.commit() + db.refresh(model_session) + + yield model_session.id + + db.delete(model_session) + db.commit() + db.close() + + +@pytest.fixture(scope="module", name="run_id") +def create_run_id(client: TestClient, model_session_id: int): + container = client.app.container + session_factory = container["session_factory"] + + with session_factory() as db: + run = Run( + model_session_id=model_session_id, + optimizer_name="OptunaOptimizer", + optimizer_parameters={ + "n_trials": 10, + "sampler": "TPESampler", + "pruner": "None", + }, + model_name="DummyModel", + parameters={}, + goal_metric="Accuracy", + name="Run", + split_indexes="""{ + "train_indexes": [0, 1, 2, 3, 4], + "test_indexes": [5, 6, 7, 8], + "val_indexes": [9, 10, 11, 12] + }""", + ) + db.add(run) + db.commit() + db.refresh(run) + + yield run.id + + db.delete(run) + db.commit() + db.close() + + +def _run_explainer_job(client: TestClient, explainer_id: int, scope: str): + response = client.post( + "/api/v1/job/", + data={ + "job_type": "ExplainerJob", + "kwargs": json.dumps( + {"explainer_id": explainer_id, "explainer_scope": scope} + ), + }, + ) + assert response.status_code == 201, response.text + + +def test_global_explainer_job_generates_story_automatically( + client: TestClient, run_id: int +): + # The story is generated as part of the normal explainer job - there is + # no separate job, endpoint or button to request it. + container = client.app.container + session_factory = container["session_factory"] + + with session_factory() as db: + explainer = GlobalExplainer( + run_id=run_id, + explainer_name="StoryableGlobalExplainer", + parameters={}, + ) + db.add(explainer) + db.commit() + db.refresh(explainer) + explainer_id = explainer.id + + _run_explainer_job(client, explainer_id, "global") + + with session_factory() as db: + explainer = db.get(GlobalExplainer, explainer_id) + assert explainer.plot_path, explainer + assert explainer.story == ( + "Story based on 'a plot summary': feature X matters most" + ) + + +def test_global_explainer_job_leaves_story_none_when_unsupported( + client: TestClient, run_id: int +): + container = client.app.container + session_factory = container["session_factory"] + + with session_factory() as db: + explainer = GlobalExplainer( + run_id=run_id, + explainer_name="StorylessGlobalExplainer", + parameters={}, + ) + db.add(explainer) + db.commit() + db.refresh(explainer) + explainer_id = explainer.id + + _run_explainer_job(client, explainer_id, "global") + + with session_factory() as db: + explainer = db.get(GlobalExplainer, explainer_id) + assert explainer.plot_path, explainer + assert explainer.story is None + + +def test_global_explainer_job_survives_a_broken_story(client: TestClient, run_id: int): + # A bug in story() must never fail the explanation itself. + container = client.app.container + session_factory = container["session_factory"] + + with session_factory() as db: + explainer = GlobalExplainer( + run_id=run_id, + explainer_name="BrokenStoryGlobalExplainer", + parameters={}, + ) + db.add(explainer) + db.commit() + db.refresh(explainer) + explainer_id = explainer.id + + _run_explainer_job(client, explainer_id, "global") + + with session_factory() as db: + explainer = db.get(GlobalExplainer, explainer_id) + assert explainer.plot_path, explainer + assert explainer.story is None + assert explainer.status.name == "FINISHED" + + +@pytest.fixture(scope="module", name="local_explainer_id") +def create_local_explainer(client: TestClient, run_id: int, dataset_id: int): + container = client.app.container + session_factory = container["session_factory"] + + with session_factory() as db: + local_explainer = LocalExplainer( + name="test_story_local", + run_id=run_id, + explainer_name="StoryableLocalExplainer", + dataset_id=dataset_id, + scope={"split": "test", "percentage": 100}, + parameters={}, + fit_parameters={}, + ) + db.add(local_explainer) + db.commit() + db.refresh(local_explainer) + + yield local_explainer.id + + db.delete(local_explainer) + db.commit() + db.close() + + +def test_local_explainer_job_generates_stories_automatically( + client: TestClient, local_explainer_id: int +): + # One story per explained instance, all generated automatically as part + # of the normal explainer job - no separate job/endpoint/button. + _run_explainer_job(client, local_explainer_id, "local") + + container = client.app.container + session_factory = container["session_factory"] + with session_factory() as db: + explainer = db.get(LocalExplainer, local_explainer_id) + assert explainer.plots_path, explainer + # The "test" split has 4 rows (indexes 5-8) at 100%. + assert explainer.stories == { + "0": "Local story for 'plot 0', context rows=1", + "1": "Local story for 'plot 1', context rows=1", + "2": "Local story for 'plot 2', context rows=1", + "3": "Local story for 'plot 3', context rows=1", + } + + +def test_local_explainer_job_leaves_stories_none_when_unsupported( + client: TestClient, run_id: int, dataset_id: int +): + container = client.app.container + session_factory = container["session_factory"] + + with session_factory() as db: + explainer = LocalExplainer( + run_id=run_id, + explainer_name="StorylessLocalExplainer", + dataset_id=dataset_id, + scope={"split": "test", "percentage": 100}, + parameters={}, + fit_parameters={}, + ) + db.add(explainer) + db.commit() + db.refresh(explainer) + explainer_id = explainer.id + + _run_explainer_job(client, explainer_id, "local") + + with session_factory() as db: + explainer = db.get(LocalExplainer, explainer_id) + assert explainer.plots_path, explainer + assert explainer.stories is None + + +def test_local_explainer_job_survives_a_broken_story( + client: TestClient, run_id: int, dataset_id: int +): + # A bug in one instance's story() must never fail the explanation. + container = client.app.container + session_factory = container["session_factory"] + + with session_factory() as db: + explainer = LocalExplainer( + run_id=run_id, + explainer_name="BrokenStoryLocalExplainer", + dataset_id=dataset_id, + scope={"split": "test", "percentage": 100}, + parameters={}, + fit_parameters={}, + ) + db.add(explainer) + db.commit() + db.refresh(explainer) + explainer_id = explainer.id + + _run_explainer_job(client, explainer_id, "local") + + with session_factory() as db: + explainer = db.get(LocalExplainer, explainer_id) + assert explainer.plots_path, explainer + assert explainer.stories is None + assert explainer.status.name == "FINISHED" diff --git a/tests/back/explainers/test_explainers.py b/tests/back/explainers/test_explainers.py index 2c854683a..dec0acefc 100644 --- a/tests/back/explainers/test_explainers.py +++ b/tests/back/explainers/test_explainers.py @@ -218,6 +218,34 @@ def test_permutation_feature_importance(trained_model: BaseModel, dataset: Datas assert len(values) == len(INPUT_COLUMNS) +def test_permutation_feature_importance_story( + trained_model: BaseModel, dataset: DatasetDict +): + explainer = PermutationFeatureImportance( + trained_model, + scoring="accuracy", + n_repeats=5, + random_state=0, + max_samples_fraction=1.0, + ) + explanation = explainer.explain(copy.deepcopy(dataset)) + plot = explainer.plot(explanation) + + story = explainer.story(plot[0]) + + assert isinstance(story, str) + ranking = sorted( + zip(explanation["features"], explanation["importances_mean"], strict=True), + key=lambda pair: pair[1], + reverse=True, + ) + most_important_feature = ranking[0][0] + least_important_feature = ranking[-1][0] + + assert most_important_feature in story + assert least_important_feature in story + + def plot(self, explanation: list[dict]): """Create explanation plots using plotly, tolerant to missing metadata.""" import numpy as np diff --git a/tests/back/explainers/test_image_explainers.py b/tests/back/explainers/test_image_explainers.py index 88bc341b1..2917165f0 100644 --- a/tests/back/explainers/test_image_explainers.py +++ b/tests/back/explainers/test_image_explainers.py @@ -122,7 +122,13 @@ def test_grad_cam(images, method): groups = plot[0].groups assert len(groups) == len(images) for group in groups: - assert [a.type for a in group.artifacts] == ["plotly", "text"] + assert [a.type for a in group.artifacts] == ["plotly"] + + from DashAI.back.core.artifacts import GroupedArtifacts + + single_group_output = GroupedArtifacts(groups=[groups[0]]) + story = explainer.story(single_group_output, images) + assert "predicted" in story def test_grad_cam_rejects_non_convolutional_models(images): @@ -146,7 +152,13 @@ def test_occlusion_saliency(images): groups = plot[0].groups assert len(groups) == len(images) for group in groups: - assert [a.type for a in group.artifacts] == ["plotly", "text"] + assert [a.type for a in group.artifacts] == ["plotly"] + + from DashAI.back.core.artifacts import GroupedArtifacts + + single_group_output = GroupedArtifacts(groups=[groups[0]]) + story = explainer.story(single_group_output, images) + assert "predicted" in story def test_occlusion_saliency_works_without_conv_layers(images): diff --git a/tests/back/explainers/test_lib_explainers.py b/tests/back/explainers/test_lib_explainers.py index acd8cf9de..f5d5d2cc0 100644 --- a/tests/back/explainers/test_lib_explainers.py +++ b/tests/back/explainers/test_lib_explainers.py @@ -141,7 +141,13 @@ def test_dice_counterfactual(trained_model, dataset): groups = plot[0].groups assert len(groups) == len(instance_keys) for group in groups: - assert [a.type for a in group.artifacts] == ["table", "text"] + assert [a.type for a in group.artifacts] == ["table"] + + from DashAI.back.core.artifacts import GroupedArtifacts + + single_group_output = GroupedArtifacts(groups=[groups[0]]) + story = explainer.story(single_group_output, instances) + assert "predicted" in story class DummyTextModel: diff --git a/tests/back/explainers/test_new_explainers.py b/tests/back/explainers/test_new_explainers.py index 775a59366..9f2e2b0cd 100644 --- a/tests/back/explainers/test_new_explainers.py +++ b/tests/back/explainers/test_new_explainers.py @@ -139,12 +139,12 @@ def test_nearest_counterfactual(trained_model, dataset): plot = explainer.plot(explanation) # A single grouped artifact with one group per instance, each holding a - # table and a text artifact. + # table. The narrative is not computed here: only story() builds it. assert len(plot) == 1 groups = plot[0].groups assert len(groups) == len(instance_keys) for group in groups: - assert [a.type for a in group.artifacts] == ["table", "text"] + assert [a.type for a in group.artifacts] == ["table"] first_table = groups[0].artifacts[0].payload # Feature rows plus the predicted class row. @@ -153,6 +153,12 @@ def test_nearest_counterfactual(trained_model, dataset): assert 0 <= cell.row < len(first_table.rows) assert 0 <= cell.column < len(first_table.columns) + from DashAI.back.core.artifacts import GroupedArtifacts + + single_group_output = GroupedArtifacts(groups=[groups[0]]) + story = explainer.story(single_group_output, instances) + assert "predicted" in story + def test_nearest_counterfactual_distance_l2(trained_model, dataset): x, _ = dataset @@ -202,8 +208,17 @@ def test_contrastive_shap(trained_model, dataset): assert len(plot) == 1 groups = plot[0].groups assert len(groups) == len(instance_keys) - assert [a.type for a in groups[0].artifacts] == ["plotly", "text"] - assert "rather than" in groups[0].artifacts[1].payload + # plot() never computes the narrative: only the chart is rendered eagerly. + assert [a.type for a in groups[0].artifacts] == ["plotly"] + + # story() is the only place the "why P rather than Q" sentence is built, + # and only when explicitly requested. + from DashAI.back.core.artifacts import GroupedArtifacts + + for group in groups: + single_group_output = GroupedArtifacts(groups=[group]) + story = explainer.story(single_group_output, instances) + assert "rather than" in story def test_contrastive_shap_fixed_foil(trained_model, dataset): diff --git a/tests/back/explainers/test_task_explainers.py b/tests/back/explainers/test_task_explainers.py index c656efc59..dcccfa1a5 100644 --- a/tests/back/explainers/test_task_explainers.py +++ b/tests/back/explainers/test_task_explainers.py @@ -173,8 +173,13 @@ def test_regression_kernel_shap(trained_regressor, regression_dataset): assert len(plot) == 1 groups = plot[0].groups assert len(groups) == len(instance_keys) - assert [a.type for a in groups[0].artifacts] == ["plotly", "text"] - assert "baseline" in groups[0].artifacts[1].payload + assert [a.type for a in groups[0].artifacts] == ["plotly"] + + from DashAI.back.core.artifacts import GroupedArtifacts + + single_group_output = GroupedArtifacts(groups=[groups[0]]) + story = explainer.story(single_group_output, instances) + assert "baseline" in story def test_regression_partial_dependence(trained_regressor, regression_dataset): @@ -276,8 +281,13 @@ def test_token_ablation_explains_influential_tokens(): assert len(plot) == 1 groups = plot[0].groups assert len(groups) == 2 - assert [a.type for a in groups[0].artifacts] == ["plotly", "text"] - assert "good" in groups[0].artifacts[1].payload + assert [a.type for a in groups[0].artifacts] == ["plotly"] + + from DashAI.back.core.artifacts import GroupedArtifacts + + single_group_output = GroupedArtifacts(groups=[groups[0]]) + story = explainer.story(single_group_output, instances) + assert "good" in story def test_token_ablation_ignores_tokenizer_columns():