From 032a32141f0b57496eb6fcd3c3633fa817bc2e52 Mon Sep 17 00:00:00 2001 From: Creylay Date: Mon, 3 Aug 2026 23:25:36 -0400 Subject: [PATCH 01/11] feat: add story fields to GlobalExplainer model --- ...f3_add_story_fields_to_global_explainer.py | 33 +++++++++++++++++++ DashAI/back/dependencies/database/models.py | 2 ++ 2 files changed, 35 insertions(+) create mode 100644 DashAI/alembic/versions/6273b0d04af3_add_story_fields_to_global_explainer.py diff --git a/DashAI/alembic/versions/6273b0d04af3_add_story_fields_to_global_explainer.py b/DashAI/alembic/versions/6273b0d04af3_add_story_fields_to_global_explainer.py new file mode 100644 index 000000000..1c0f1c2e9 --- /dev/null +++ b/DashAI/alembic/versions/6273b0d04af3_add_story_fields_to_global_explainer.py @@ -0,0 +1,33 @@ +"""Add story fields to global_explainer + +Revision ID: 6273b0d04af3 +Revises: c4e8a1d20f3b +Create Date: 2026-08-03 23:22:57.981298 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "6273b0d04af3" +down_revision: Union[str, None] = "c4e8a1d20f3b" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "global_explainer", + sa.Column("story", sa.String(), nullable=True), + ) + op.add_column( + "global_explainer", + sa.Column("story_huey_id", sa.String(), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("global_explainer", "story_huey_id") + op.drop_column("global_explainer", "story") diff --git a/DashAI/back/dependencies/database/models.py b/DashAI/back/dependencies/database/models.py index c1160db89..a49edb8bc 100644 --- a/DashAI/back/dependencies/database/models.py +++ b/DashAI/back/dependencies/database/models.py @@ -349,6 +349,8 @@ class GlobalExplainer(Base): explanation_path: Mapped[str] = mapped_column(String, nullable=True) plot_path: Mapped[str] = mapped_column(String, nullable=True) plot_overrides: Mapped[JSON] = mapped_column(JSON, nullable=True) + story: Mapped[str] = mapped_column(String, nullable=True) + story_huey_id: Mapped[str] = mapped_column(String, nullable=True) parameters: Mapped[JSON] = mapped_column(JSON) created: Mapped[DateTime] = mapped_column(DateTime, default=datetime.now) status: Mapped[Enum] = mapped_column( From 693b5b253861656225427045ff5c553ffb50dc30 Mon Sep 17 00:00:00 2001 From: Creylay Date: Mon, 3 Aug 2026 23:42:26 -0400 Subject: [PATCH 02/11] feat: implement ExplainerStoryJob and add related tests --- DashAI/back/api/api_v1/schemas/job_params.py | 1 + DashAI/back/initial_components.py | 2 + DashAI/back/job/explainer_story_job.py | 221 ++++++++++++++++ tests/back/api/test_explainer_story_job.py | 261 +++++++++++++++++++ 4 files changed, 485 insertions(+) create mode 100644 DashAI/back/job/explainer_story_job.py create mode 100644 tests/back/api/test_explainer_story_job.py diff --git a/DashAI/back/api/api_v1/schemas/job_params.py b/DashAI/back/api/api_v1/schemas/job_params.py index 167e54dab..aa37148e3 100644 --- a/DashAI/back/api/api_v1/schemas/job_params.py +++ b/DashAI/back/api/api_v1/schemas/job_params.py @@ -9,6 +9,7 @@ class JobParams(BaseModel): job_type: Literal[ "ModelJob", "ExplainerJob", + "ExplainerStoryJob", "PredictJob", "DatasetJob", "ExplorerJob", diff --git a/DashAI/back/initial_components.py b/DashAI/back/initial_components.py index c4de73388..45205894c 100644 --- a/DashAI/back/initial_components.py +++ b/DashAI/back/initial_components.py @@ -132,6 +132,7 @@ from DashAI.back.job.datafile_job import DatafileJob from DashAI.back.job.dataset_job import DatasetJob from DashAI.back.job.explainer_job import ExplainerJob +from DashAI.back.job.explainer_story_job import ExplainerStoryJob from DashAI.back.job.explorer_job import ExplorerJob from DashAI.back.job.generative_job import GenerativeJob from DashAI.back.job.model_job import ModelJob @@ -506,6 +507,7 @@ def get_initial_components(): ComponentDownloadJob, DatafileJob, ExplainerJob, + ExplainerStoryJob, ModelJob, ExplorerJob, PredictJob, diff --git a/DashAI/back/job/explainer_story_job.py b/DashAI/back/job/explainer_story_job.py new file mode 100644 index 000000000..adcdb0856 --- /dev/null +++ b/DashAI/back/job/explainer_story_job.py @@ -0,0 +1,221 @@ +import logging +import pickle +from typing import TYPE_CHECKING, Any, Dict + +from kink import inject +from sqlalchemy import exc + +from DashAI.back.core.artifacts import Artifact, GroupedArtifacts +from DashAI.back.dependencies.database.models import ( + GlobalExplainer, + LocalExplainer, + Run, +) +from DashAI.back.job.base_job import BaseJob, JobError + +if TYPE_CHECKING: + from sqlalchemy.orm import Session, sessionmaker + +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger(__name__) + + +class ExplainerStoryJob(BaseJob): + """Generate a natural-language story for an already-computed explanation. + + Unlike ``ExplainerJob``, this job never recomputes an explanation: it + reloads the explanation and plot artifacts already persisted on disk and + reinstantiates the explainer purely to call its ``story`` method, which + is the pattern that method's docstring is written around (see + ``BaseGlobalExplainer.story`` / ``BaseLocalExplainer.story``). + """ + + @inject + def set_status_as_delivered(self) -> None: + log.debug("Explainer story job marked as delivered") + + @inject + def set_status_as_error(self) -> None: + log.debug("Explainer story job failed") + + @inject + def get_job_name( + self, session_factory: "sessionmaker" = lambda di: di["session_factory"] + ) -> str: + """Get a descriptive name for the job.""" + explainer_id = self.kwargs.get("explainer_id") + explainer_scope = self.kwargs.get("explainer_scope", "") + + if not explainer_id: + return f"{explainer_scope.capitalize()} Story" + + try: + with session_factory() as db: + model_cls = ( + GlobalExplainer if explainer_scope == "global" else LocalExplainer + ) + explainer = db.get(model_cls, explainer_id) + if explainer and explainer.name: + return f"Story: {explainer.name}" + if explainer and explainer.explainer_name: + return f"Story: {explainer.explainer_name.split('.')[-1]}" + except Exception as e: + log.exception(f"Error getting job name: {e}") + + return f"{explainer_scope.capitalize()} Story ({explainer_id})" + + def _load_explainer(self, db: "Session", explainer_db, component_registry): + """Reinstantiate the explainer that produced an already-computed explanation. + + Skips ``fit`` and ``explain``/``explain_instance`` (the expensive + steps): the explanation is reloaded from its pickle instead of + recomputed. + """ + run: Run = db.get(Run, explainer_db.run_id) + if not run: + raise JobError(f"Run {explainer_db.run_id} does not exist in DB.") + + try: + model_class = component_registry[run.model_name]["class"] + model = model_class(**run.parameters) + trained_model = model.load(run.run_path) + except Exception as e: + log.exception(e) + raise JobError(f"Unable to load model for run {run.id}") from e + + try: + explainer_class = component_registry[explainer_db.explainer_name]["class"] + explainer = explainer_class(model=trained_model, **explainer_db.parameters) + except Exception as e: + log.exception(e) + raise JobError( + f"Unable to instantiate explainer {explainer_db.explainer_name}" + ) from e + + try: + with open(explainer_db.explanation_path, "rb") as file: + explainer.explanation = pickle.load(file) + except Exception as e: + log.exception(e) + raise JobError("Unable to load the saved explanation") from e + + return explainer + + @staticmethod + def _parse_artifact(item: Dict[str, Any]): + """Reconstruct an ``Artifact``/``GroupedArtifacts`` from its saved wire dict.""" + if item.get("type") == "grouped": + return GroupedArtifacts.model_validate(item) + return Artifact.from_dict(item) + + def _generate_global_story( + self, db: "Session", explainer_db, component_registry + ) -> str: + explainer = self._load_explainer(db, explainer_db, component_registry) + + artifact_index = self.kwargs.get("artifact_index", 0) + try: + with open(explainer_db.plot_path, "rb") as file: + plots = pickle.load(file) + explainer_output = self._parse_artifact(plots[artifact_index]) + except Exception as e: + log.exception(e) + raise JobError("Unable to load the saved plot artifacts") from e + + try: + return explainer.story(explainer_output) + except NotImplementedError as e: + raise JobError(str(e)) from e + except Exception as e: + log.exception(e) + raise JobError("Failed to generate the story") from e + + def _generate_local_story( + self, db: "Session", explainer_db, component_registry + ) -> str: + from DashAI.back.dataloaders.classes.dashai_dataset import load_dataset + + group_index = self.kwargs.get("group_index") + if group_index is None: + raise JobError("group_index is required to generate a local story") + + explainer = self._load_explainer(db, explainer_db, component_registry) + + artifact_index = self.kwargs.get("artifact_index", 0) + try: + with open(explainer_db.plots_path, "rb") as file: + plots = pickle.load(file) + full_grouped = GroupedArtifacts.model_validate(plots[artifact_index]) + group = full_grouped.groups[group_index] + explainer_output = GroupedArtifacts(groups=[group]) + except IndexError as e: + raise JobError(f"No explained instance at index {group_index}") from e + except Exception as e: + log.exception(e) + raise JobError("Unable to load the saved plot artifacts") from e + + try: + prediction_context = load_dataset( + f"{explainer_db.input_dataset_path}/dataset" + ).select([group_index]) + except Exception as e: + log.exception(e) + raise JobError("Unable to load the explained instance") from e + + try: + return explainer.story(explainer_output, prediction_context) + except NotImplementedError as e: + raise JobError(str(e)) from e + except Exception as e: + log.exception(e) + raise JobError("Failed to generate the story") from e + + @inject + def run(self) -> None: + from kink import di + + session_factory = di["session_factory"] + component_registry = di["component_registry"] + + explainer_id: int = self.kwargs["explainer_id"] + explainer_scope: str = self.kwargs["explainer_scope"] + + with session_factory() as db: + if explainer_scope == "global": + explainer_db: GlobalExplainer = db.get(GlobalExplainer, explainer_id) + elif explainer_scope == "local": + explainer_db: LocalExplainer = db.get(LocalExplainer, explainer_id) + else: + raise JobError(f"{explainer_scope} is an invalid explainer type") + + if not explainer_db: + raise JobError( + f"Explainer with id {explainer_id} does not exist in DB." + ) + if not explainer_db.explanation_path: + raise JobError( + "The explanation has not been computed yet for this explainer." + ) + + try: + if explainer_scope == "global": + story = self._generate_global_story( + db, explainer_db, component_registry + ) + explainer_db.story = story + else: + story = self._generate_local_story( + db, explainer_db, component_registry + ) + explainer_db.stories = { + **(explainer_db.stories or {}), + str(self.kwargs.get("group_index")): story, + } + db.commit() + except JobError: + db.rollback() + raise + except exc.SQLAlchemyError as e: + log.exception(e) + db.rollback() + raise JobError("Error saving the generated story") from e diff --git a/tests/back/api/test_explainer_story_job.py b/tests/back/api/test_explainer_story_job.py new file mode 100644 index 000000000..b5cbd60a1 --- /dev/null +++ b/tests/back/api/test_explainer_story_job.py @@ -0,0 +1,261 @@ +import json + +import joblib +import pytest +from datasets import ClassLabel, Value +from fastapi.testclient import TestClient + +from DashAI.back.core.artifacts import TextArtifact +from DashAI.back.dependencies.database.models import ( + Dataset, + GlobalExplainer, + ModelSession, + Run, +) +from DashAI.back.dependencies.registry import ComponentRegistry +from DashAI.back.explainability.global_explainer import BaseGlobalExplainer +from DashAI.back.job.explainer_job import ExplainerJob +from DashAI.back.job.explainer_story_job import ExplainerStoryJob +from DashAI.back.models.base_model import BaseModel +from DashAI.back.tasks.base_task import BaseTask + +input_columns = ["SepalLengthCm", "SepalWidthCm", "PetalLengthCm", "PetalWidthCm"] +output_columns = ["Species"] +splits = json.dumps( + { + "train": 0.5, + "test": 0.2, + "validation": 0.3, + "is_random": True, + "has_changed": True, + "seed": 42, + "shuffle": True, + "stratify": False, + } +) + + +@pytest.fixture(scope="module", name="dataset_id") +def dataset_id(dataset_1: Dataset) -> int: + return dataset_1.id + + +class DummyTask(BaseTask): + name: str = "DummyTask" + + metadata: dict = { + "inputs_types": [ClassLabel, Value], + "outputs_types": [ClassLabel], + "inputs_cardinality": "n", + "outputs_cardinality": 1, + } + + def prepare_for_task(self, dataset, input_columns=None, output_columns=None): + return dataset + + +class DummyModel(BaseModel): + COMPATIBLE_COMPONENTS = ["DummyTask"] + + @classmethod + def get_schema(cls): + return {} + + def save(self, filename): + joblib.dump(self, filename) + + @staticmethod + def load(filename): + return DummyModel() + + def predict(self, x): + return {} + + def train(self, x_train, y_train, x_validation=None, y_validation=None): + return + + def prepare_dataset(self, dataset, is_fit=False): + return dataset + + def prepare_output(self, dataset, is_fit=False): + return dataset + + +class StoryableGlobalExplainer(BaseGlobalExplainer): + """Global explainer with a deterministic, testable story().""" + + COMPATIBLE_COMPONENTS = ["DummyTask"] + + def __init__(self, model: BaseModel) -> None: + self.model = model + self.explanation = None + + @classmethod + def get_schema(cls): + return {} + + def explain(self, dataset): + return {"headline": "feature X matters most"} + + def plot(self, explanation): + return [TextArtifact(payload="a plot summary")] + + def story(self, explainer_output): + headline = (self.explanation or {}).get("headline", "unknown") + return f"Story based on '{explainer_output.payload}': {headline}" + + +@pytest.fixture(autouse=True, name="test_registry") +def setup_test_registry(client, monkeypatch: pytest.MonkeyPatch): + container = client.app.container + + test_registry = ComponentRegistry( + initial_components=[ + DummyTask, + DummyModel, + StoryableGlobalExplainer, + ExplainerJob, + ExplainerStoryJob, + ] + ) + + monkeypatch.setitem( + container._services, + "component_registry", + test_registry, + ) + return test_registry + + +@pytest.fixture(scope="module", name="model_session_id") +def create_model_session(client: TestClient, dataset_id: int): + container = client.app.container + session_factory = container["session_factory"] + + with session_factory() as db: + model_session = ModelSession( + dataset_id=dataset_id, + name="DummyExperiment", + task_name="DummyTask", + input_columns=input_columns, + output_columns=output_columns, + splits=splits, + ) + db.add(model_session) + db.commit() + db.refresh(model_session) + + yield model_session.id + + db.delete(model_session) + db.commit() + db.close() + + +@pytest.fixture(scope="module", name="run_id") +def create_run_id(client: TestClient, model_session_id: int): + container = client.app.container + session_factory = container["session_factory"] + + with session_factory() as db: + run = Run( + model_session_id=model_session_id, + optimizer_name="OptunaOptimizer", + optimizer_parameters={ + "n_trials": 10, + "sampler": "TPESampler", + "pruner": "None", + }, + model_name="DummyModel", + parameters={}, + goal_metric="Accuracy", + name="Run", + split_indexes="""{ + "train_indexes": [0, 1, 2, 3, 4], + "test_indexes": [5, 6, 7, 8], + "val_indexes": [9, 10, 11, 12] + }""", + ) + db.add(run) + db.commit() + db.refresh(run) + + yield run.id + + db.delete(run) + db.commit() + db.close() + + +@pytest.fixture(scope="module", name="global_explainer_id") +def create_global_explainer(client: TestClient, run_id: int): + container = client.app.container + session_factory = container["session_factory"] + + with session_factory() as db: + global_explainer = GlobalExplainer( + name="test_story_global", + run_id=run_id, + explainer_name="StoryableGlobalExplainer", + parameters={}, + ) + db.add(global_explainer) + db.commit() + db.refresh(global_explainer) + + yield global_explainer.id + + db.delete(global_explainer) + db.commit() + db.close() + + +def test_global_story_job(client: TestClient, global_explainer_id: int): + # First compute the explanation itself, same as any other explainer job. + response = client.post( + "/api/v1/job/", + data={ + "job_type": "ExplainerJob", + "kwargs": json.dumps( + { + "explainer_id": global_explainer_id, + "explainer_scope": "global", + } + ), + }, + ) + assert response.status_code == 201, response.text + + response = client.get(f"/api/v1/explainer/global/?run_id={global_explainer_id}") + # Sanity: the explanation finished (plot_path set) before generating a story. + explainers = response.json() + assert explainers[0]["plot_path"], explainers + + # Now request the story for the already-computed explanation. + response = client.post( + "/api/v1/job/", + data={ + "job_type": "ExplainerStoryJob", + "kwargs": json.dumps( + { + "explainer_id": global_explainer_id, + "explainer_scope": "global", + } + ), + }, + ) + assert response.status_code == 201, response.text + story_job_id = response.json()["id"] + + response = client.get(f"/api/v1/job/status/{story_job_id}") + assert response.status_code == 200, response.text + job_status = response.json() + assert job_status["status"] == "finished", job_status + + container = client.app.container + session_factory = container["session_factory"] + with session_factory() as db: + explainer = db.get(GlobalExplainer, global_explainer_id) + assert explainer.story == ( + "Story based on 'a plot summary': feature X matters most" + ) From 1be64d58ccbb237c5cb9b5f257910d60aeff42ab Mon Sep 17 00:00:00 2001 From: Creylay Date: Mon, 3 Aug 2026 23:54:32 -0400 Subject: [PATCH 03/11] feat: add endpoint for creating global explainer story and related tests --- .../back/api/api_v1/endpoints/explainers.py | 77 +++++++++++++++++++ tests/back/api/test_explainer_story_job.py | 41 +++++++--- 2 files changed, 107 insertions(+), 11 deletions(-) diff --git a/DashAI/back/api/api_v1/endpoints/explainers.py b/DashAI/back/api/api_v1/endpoints/explainers.py index 617afa2bd..f56f64829 100755 --- a/DashAI/back/api/api_v1/endpoints/explainers.py +++ b/DashAI/back/api/api_v1/endpoints/explainers.py @@ -25,6 +25,8 @@ if TYPE_CHECKING: from sqlalchemy.orm import sessionmaker + from DashAI.back.dependencies.job_queues import BaseJobQueue + logging.basicConfig(level=logging.DEBUG) log = logging.getLogger(__name__) @@ -274,6 +276,81 @@ async def get_global_explanation_plot( return _apply_overrides(normalize_artifacts(plot), plot_overrides) +@router.post("/global/{explainer_id}/story", status_code=status.HTTP_201_CREATED) +@inject +async def create_global_explainer_story( + explainer_id: int, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), + job_queue: "BaseJobQueue" = Depends(lambda: di["job_queue"]), +): + """Enqueue story generation for an already-computed global explanation. + + Parameters + ---------- + explainer_id : int + Id of the global explainer whose explanation the story narrates. + session_factory : Callable[..., ContextManager[Session]] + A factory that creates a context manager that handles a SQLAlchemy session. + The generated session can be used to access and query the database. + job_queue : BaseJobQueue + The job queue used to enqueue the story generation job. + + Returns + ------- + dict + ``{"id": job_id}``, the huey job id; poll it via + ``GET /job/status/{job_id}`` the same way as any other job, or read + the finished result back from ``GET /global/{explainer_id}``. + + Raises + ------ + HTTPException + If the explainer does not exist or its explanation has not finished + computing yet. + """ + from DashAI.back.job.base_job import JobError + from DashAI.back.job.explainer_story_job import ExplainerStoryJob + + with session_factory() as db: + try: + global_explainer = db.get(GlobalExplainer, explainer_id) + + if not global_explainer: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Explainer not found", + ) + + if global_explainer.status != ExplainerStatus.FINISHED: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Explanation not found", + ) + + job = ExplainerStoryJob(explainer_id=explainer_id, explainer_scope="global") + try: + job.set_status_as_delivered() + except JobError as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Story job not delivered", + ) from e + + job_id = job_queue.put(job).id + global_explainer.story = None + global_explainer.story_huey_id = job_id + db.commit() + + return {"id": job_id} + + except exc.SQLAlchemyError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + + @router.post("/global", status_code=status.HTTP_201_CREATED) @inject async def upload_global_explainer( diff --git a/tests/back/api/test_explainer_story_job.py b/tests/back/api/test_explainer_story_job.py index b5cbd60a1..ced417a56 100644 --- a/tests/back/api/test_explainer_story_job.py +++ b/tests/back/api/test_explainer_story_job.py @@ -231,18 +231,10 @@ def test_global_story_job(client: TestClient, global_explainer_id: int): explainers = response.json() assert explainers[0]["plot_path"], explainers - # Now request the story for the already-computed explanation. + # Now request the story for the already-computed explanation, through the + # dedicated endpoint (not the generic /job/ POST). response = client.post( - "/api/v1/job/", - data={ - "job_type": "ExplainerStoryJob", - "kwargs": json.dumps( - { - "explainer_id": global_explainer_id, - "explainer_scope": "global", - } - ), - }, + f"/api/v1/explainer/global/{global_explainer_id}/story", ) assert response.status_code == 201, response.text story_job_id = response.json()["id"] @@ -256,6 +248,33 @@ def test_global_story_job(client: TestClient, global_explainer_id: int): session_factory = container["session_factory"] with session_factory() as db: explainer = db.get(GlobalExplainer, global_explainer_id) + assert explainer.story_huey_id == story_job_id assert explainer.story == ( "Story based on 'a plot summary': feature X matters most" ) + + +def test_global_story_endpoint_requires_finished_explanation( + client: TestClient, run_id: int +): + # A fresh explainer that was never run still has status NOT_STARTED. + container = client.app.container + session_factory = container["session_factory"] + with session_factory() as db: + unstarted_explainer = GlobalExplainer( + run_id=run_id, + explainer_name="StoryableGlobalExplainer", + parameters={}, + ) + db.add(unstarted_explainer) + db.commit() + db.refresh(unstarted_explainer) + unstarted_id = unstarted_explainer.id + + response = client.post(f"/api/v1/explainer/global/{unstarted_id}/story") + assert response.status_code == 404, response.text + + +def test_global_story_endpoint_requires_existing_explainer(client: TestClient): + response = client.post("/api/v1/explainer/global/999999/story") + assert response.status_code == 404, response.text From 895e24ae3597c2bb66829c2963d7ba9e6b36886f Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 4 Aug 2026 10:45:05 -0400 Subject: [PATCH 04/11] feat: implement story generation for global explainers and update related components --- .../back/api/api_v1/endpoints/explainers.py | 15 +++ .../registry/component_registry.py | 35 ++++++ .../permutation_feature_importance.py | 45 ++++++++ DashAI/back/job/explainer_story_job.py | 12 +- DashAI/front/src/api/explainer.ts | 9 ++ .../components/explainers/ExplainersCard.jsx | 106 ++++++++++++++++-- .../explainers/LazyExplainerCard.jsx | 3 + .../src/components/models/RunResults.jsx | 2 + .../models/runResults/ExplainerResultsTab.jsx | 8 ++ .../models/runResults/useRunResultsData.js | 7 ++ DashAI/front/src/types/explainer.ts | 2 + .../src/utils/i18n/locales/de/explainers.json | 8 +- .../src/utils/i18n/locales/en/explainers.json | 8 +- .../src/utils/i18n/locales/es/explainers.json | 8 +- .../src/utils/i18n/locales/pt/explainers.json | 8 +- .../src/utils/i18n/locales/zh/explainers.json | 8 +- tests/back/api/test_explainer_story_job.py | 56 +++++++++ tests/back/explainers/test_explainers.py | 32 ++++++ 18 files changed, 351 insertions(+), 21 deletions(-) diff --git a/DashAI/back/api/api_v1/endpoints/explainers.py b/DashAI/back/api/api_v1/endpoints/explainers.py index f56f64829..77fe22979 100755 --- a/DashAI/back/api/api_v1/endpoints/explainers.py +++ b/DashAI/back/api/api_v1/endpoints/explainers.py @@ -282,6 +282,7 @@ async def create_global_explainer_story( explainer_id: int, session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), job_queue: "BaseJobQueue" = Depends(lambda: di["job_queue"]), + component_registry=Depends(lambda: di["component_registry"]), ): """Enqueue story generation for an already-computed global explanation. @@ -327,6 +328,20 @@ async def create_global_explainer_story( detail="Explanation not found", ) + explainer_component = None + if global_explainer.explainer_name in component_registry: + explainer_component = component_registry[ + global_explainer.explainer_name + ] + if not explainer_component or not explainer_component.get("supports_story"): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"{global_explainer.explainer_name} does not support " + "story generation." + ), + ) + job = ExplainerStoryJob(explainer_id=explainer_id, explainer_scope="global") try: job.set_status_as_delivered() diff --git a/DashAI/back/dependencies/registry/component_registry.py b/DashAI/back/dependencies/registry/component_registry.py index e97115ab6..4153c7871 100644 --- a/DashAI/back/dependencies/registry/component_registry.py +++ b/DashAI/back/dependencies/registry/component_registry.py @@ -171,6 +171,36 @@ def _get_base_type(self, new_component: type) -> str: return base_classes_cantidates[0].TYPE + @staticmethod + def _overrides_story(component: type) -> bool: + """Whether ``component`` provides its own ``story`` implementation. + + ``BaseGlobalExplainer``/``BaseLocalExplainer`` both declare a + ``story`` method that just raises ``NotImplementedError``, so + inheriting it unmodified means the explainer does not support story + generation. Walks the MRO (most derived first) to find which class + actually defines ``story``, rather than importing the base classes + directly (avoids a needless coupling to the explainability module). + + Parameters + ---------- + component : type + The component class to inspect. + + Returns + ------- + bool + ``True`` if some class more specific than the two base + explainer classes defines ``story``. + """ + for klass in component.__mro__: + if "story" in vars(klass): + return klass.__name__ not in ( + "BaseGlobalExplainer", + "BaseLocalExplainer", + ) + return False + @staticmethod @beartype def _collect_compatible_components(component: type) -> List[str]: @@ -251,6 +281,11 @@ def register_component(self, new_component: Type) -> None: "color": getattr(new_component, "COLOR", None), } + if hasattr(new_component, "story"): + new_register_component["supports_story"] = self._overrides_story( + new_component + ) + if base_type not in self._registry: self._registry[base_type] = {new_component.__name__: new_register_component} else: diff --git a/DashAI/back/explainability/explainers/permutation_feature_importance.py b/DashAI/back/explainability/explainers/permutation_feature_importance.py index 2bd6dc556..19f7a166b 100644 --- a/DashAI/back/explainability/explainers/permutation_feature_importance.py +++ b/DashAI/back/explainability/explainers/permutation_feature_importance.py @@ -1,6 +1,7 @@ from typing import Dict, List, Union from DashAI.back.core.artifacts import ( + Artifact, ArtifactGroup, GroupedArtifacts, PlotlyArtifact, @@ -589,3 +590,47 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: data = data.sort_values(by=["importances_mean"], ascending=True) return self._create_plot(data) + + def story(self, explainer_output: Union[Artifact, GroupedArtifacts]) -> str: + """Summarize the feature ranking in a deterministic natural-language story. + + Built directly from ``self.explanation`` (see :meth:`explain`), not + from ``explainer_output``: every feature count selector shows the + same ranking, so there is nothing instance-specific to pick out. + + Parameters + ---------- + explainer_output : Artifact or GroupedArtifacts + Unused; present only to satisfy the base class signature. + + Returns + ------- + str + A natural-language summary of which features the model relies on + most and least. + """ + features = self.explanation["features"] + means = self.explanation["importances_mean"] + + ranking = sorted( + zip(features, means, strict=True), key=lambda pair: pair[1], reverse=True + ) + + top = ranking[:3] + top_text = ", ".join(f"{name} ({value:+.3f})" for name, value in top) + least_name, least_value = ranking[-1] + + story = ( + f"The model relies most on {top_text}. " + f"{least_name} contributes the least to its predictions " + f"(importance={least_value:+.3f})." + ) + + if least_value <= 0: + story += ( + " A near-zero or negative importance means shuffling that " + "feature barely changed (or even improved) performance, so " + "the model may not need it." + ) + + return story diff --git a/DashAI/back/job/explainer_story_job.py b/DashAI/back/job/explainer_story_job.py index adcdb0856..c51ab59dc 100644 --- a/DashAI/back/job/explainer_story_job.py +++ b/DashAI/back/job/explainer_story_job.py @@ -112,6 +112,10 @@ def _generate_global_story( self, db: "Session", explainer_db, component_registry ) -> str: explainer = self._load_explainer(db, explainer_db, component_registry) + if not hasattr(explainer, "story"): + raise JobError( + f"{explainer_db.explainer_name} does not support story generation." + ) artifact_index = self.kwargs.get("artifact_index", 0) try: @@ -124,8 +128,6 @@ def _generate_global_story( try: return explainer.story(explainer_output) - except NotImplementedError as e: - raise JobError(str(e)) from e except Exception as e: log.exception(e) raise JobError("Failed to generate the story") from e @@ -140,6 +142,10 @@ def _generate_local_story( raise JobError("group_index is required to generate a local story") explainer = self._load_explainer(db, explainer_db, component_registry) + if not hasattr(explainer, "story"): + raise JobError( + f"{explainer_db.explainer_name} does not support story generation." + ) artifact_index = self.kwargs.get("artifact_index", 0) try: @@ -164,8 +170,6 @@ def _generate_local_story( try: return explainer.story(explainer_output, prediction_context) - except NotImplementedError as e: - raise JobError(str(e)) from e except Exception as e: log.exception(e) raise JobError("Failed to generate the story") from e diff --git a/DashAI/front/src/api/explainer.ts b/DashAI/front/src/api/explainer.ts index 1778f3f23..1426283cb 100644 --- a/DashAI/front/src/api/explainer.ts +++ b/DashAI/front/src/api/explainer.ts @@ -39,6 +39,15 @@ export const createGlobalExplainer = async ( return response.data; }; +export const createGlobalExplainerStory = async ( + explainerId: number, +): Promise<{ id: string }> => { + const response = await api.post<{ id: string }>( + `/v1/explainer/global/${explainerId}/story`, + ); + return response.data; +}; + export const createLocalExplainer = async ( runId: number, explainerName: string, diff --git a/DashAI/front/src/components/explainers/ExplainersCard.jsx b/DashAI/front/src/components/explainers/ExplainersCard.jsx index a9600660d..dcca4a0df 100644 --- a/DashAI/front/src/components/explainers/ExplainersCard.jsx +++ b/DashAI/front/src/components/explainers/ExplainersCard.jsx @@ -1,10 +1,11 @@ -import React, { useState } from "react"; +import React, { useEffect, useState } from "react"; import { Grid, Typography, IconButton, Paper, Box, + Button, CircularProgress, } from "@mui/material"; import { useTheme, alpha } from "@mui/material/styles"; @@ -12,6 +13,7 @@ import DeleteConfirmationModal from "../threeSectionLayout/DeleteConfirmationMod import RunStatusDot from "../shared/RunStatusDot"; import DeleteIcon from "@mui/icons-material/Delete"; import ZoomInIcon from "@mui/icons-material/ZoomIn"; +import AutoAwesomeIcon from "@mui/icons-material/AutoAwesome"; import PropTypes from "prop-types"; import ExplainersPlot from "./ExplainersPlot"; import { useNavigate } from "react-router-dom"; @@ -19,7 +21,10 @@ import { deleteExplainer, saveExplainerPlotOverride, resetExplainerPlotOverride, + createGlobalExplainerStory, + getExplainers, } from "../../api/explainer"; +import { startJobPolling } from "../../utils/jobPoller"; import { useTranslation } from "react-i18next"; const RUNNING_STATUSES = [1, 2]; // Delivered or Started @@ -35,6 +40,7 @@ export default function ExplainersCard({ onDelete, compact = false, displayName = null, + supportsStory = false, cacheEntry = null, onCacheUpdate = null, isHighlighted = false, @@ -48,6 +54,43 @@ export default function ExplainersCard({ const { t } = useTranslation(["explainers"]); const isRunning = RUNNING_STATUSES.includes(explainer.status); + // Story generation is a separate, on-demand job: kept as local state so a + // page that doesn't refetch the explainer list still reflects the result. + const [storyState, setStoryState] = useState({ + story: explainer.story || null, + status: "idle", // idle | loading | error + }); + + useEffect(() => { + setStoryState((prev) => + prev.status === "loading" + ? prev + : { ...prev, story: explainer.story || null }, + ); + }, [explainer.story]); + + const handleGenerateStory = async () => { + setStoryState({ story: null, status: "loading" }); + try { + const { id: jobId } = await createGlobalExplainerStory(explainer.id); + startJobPolling( + jobId, + async () => { + try { + const refreshed = await getExplainers(explainer.run_id, "global"); + const updated = refreshed.find((e) => e.id === explainer.id); + setStoryState({ story: updated?.story || null, status: "idle" }); + } catch { + setStoryState({ story: null, status: "error" }); + } + }, + () => setStoryState({ story: null, status: "error" }), + ); + } catch { + setStoryState({ story: null, status: "error" }); + } + }; + function plotName(name) { return name.match(/[A-Z][a-z]+|[0-9]+/g).join(" "); } @@ -160,12 +203,37 @@ export default function ExplainersCard({ ) : ( - - {/* Reserved slot for the future "generate story" action button. - Kept hidden until that feature lands. */} - + {scope === "global" && supportsStory && ( + + {storyState.status === "loading" ? ( + + + + {t("explainers:label.generatingStory")} + + + ) : ( + + )} + + )} + {storyState.story && ( + alpha(t.palette.primary.main, 0.04), + borderColor: theme.palette.ui.border, + }} + > + {storyState.story} + + )} + {storyState.status === "error" && ( + + {t("explainers:error.storyGenerationFailed")} + + )} )} @@ -258,11 +348,13 @@ ExplainersCard.propTypes = { ), created: PropTypes.string, status: PropTypes.number, + story: PropTypes.string, }).isRequired, scope: PropTypes.string.isRequired, onDelete: PropTypes.func, compact: PropTypes.bool, displayName: PropTypes.string, + supportsStory: PropTypes.bool, cacheEntry: PropTypes.shape({ items: PropTypes.array, overriddenIndexes: PropTypes.arrayOf(PropTypes.number), diff --git a/DashAI/front/src/components/explainers/LazyExplainerCard.jsx b/DashAI/front/src/components/explainers/LazyExplainerCard.jsx index 0c7a03b60..53011b395 100644 --- a/DashAI/front/src/components/explainers/LazyExplainerCard.jsx +++ b/DashAI/front/src/components/explainers/LazyExplainerCard.jsx @@ -22,6 +22,7 @@ const LazyExplainerCard = React.memo(function LazyExplainerCard({ explainer, scope, displayName, + supportsStory, onDelete, cacheEntry, updateCacheEntry, @@ -80,6 +81,7 @@ const LazyExplainerCard = React.memo(function LazyExplainerCard({ explainer={explainer} scope={scope} displayName={displayName} + supportsStory={supportsStory} onDelete={onDelete} cacheEntry={cacheEntry} onCacheUpdate={onCacheUpdate} @@ -98,6 +100,7 @@ LazyExplainerCard.propTypes = { explainer: PropTypes.object.isRequired, scope: PropTypes.string.isRequired, displayName: PropTypes.string, + supportsStory: PropTypes.bool, onDelete: PropTypes.func, cacheEntry: PropTypes.object, updateCacheEntry: PropTypes.func.isRequired, diff --git a/DashAI/front/src/components/models/RunResults.jsx b/DashAI/front/src/components/models/RunResults.jsx index 1e32fc071..16bf48985 100644 --- a/DashAI/front/src/components/models/RunResults.jsx +++ b/DashAI/front/src/components/models/RunResults.jsx @@ -45,6 +45,7 @@ export default function RunResults({ highlightedExplainerKey, setHighlightedExplainerKey, explainerDisplayNames, + explainerSupportsStory, cardHeightsRef, getCacheEntry, updateCacheEntry, @@ -147,6 +148,7 @@ export default function RunResults({ fillHeight={fillHeight} scrollParent={explainerScrollParent} explainerDisplayNames={explainerDisplayNames} + explainerSupportsStory={explainerSupportsStory} cardHeightsRef={cardHeightsRef} getCacheEntry={getCacheEntry} updateCacheEntry={updateCacheEntry} diff --git a/DashAI/front/src/components/models/runResults/ExplainerResultsTab.jsx b/DashAI/front/src/components/models/runResults/ExplainerResultsTab.jsx index 48d5eede8..8806da522 100644 --- a/DashAI/front/src/components/models/runResults/ExplainerResultsTab.jsx +++ b/DashAI/front/src/components/models/runResults/ExplainerResultsTab.jsx @@ -26,6 +26,7 @@ export default function ExplainerResultsTab({ fillHeight, scrollParent, explainerDisplayNames, + explainerSupportsStory, cardHeightsRef, getCacheEntry, updateCacheEntry, @@ -141,6 +142,9 @@ export default function ExplainerResultsTab({ explainer={explainer} scope={explainerFilter} displayName={explainerDisplayNames[explainer.explainer_name]} + supportsStory={Boolean( + explainerSupportsStory[explainer.explainer_name], + )} onDelete={onDelete} cacheEntry={getCacheEntry( explainerCacheKey(explainerFilter, explainer), @@ -166,6 +170,9 @@ export default function ExplainerResultsTab({ explainer={explainer} scope={explainerFilter} displayName={explainerDisplayNames[explainer.explainer_name]} + supportsStory={Boolean( + explainerSupportsStory[explainer.explainer_name], + )} onDelete={onDelete} cacheEntry={getCacheEntry(key)} onCacheUpdate={(patch) => updateCacheEntry(key, patch)} @@ -190,6 +197,7 @@ ExplainerResultsTab.propTypes = { fillHeight: PropTypes.bool, scrollParent: PropTypes.instanceOf(Element), explainerDisplayNames: PropTypes.object.isRequired, + explainerSupportsStory: PropTypes.object.isRequired, cardHeightsRef: PropTypes.shape({ current: PropTypes.object }).isRequired, getCacheEntry: PropTypes.func.isRequired, updateCacheEntry: PropTypes.func.isRequired, diff --git a/DashAI/front/src/components/models/runResults/useRunResultsData.js b/DashAI/front/src/components/models/runResults/useRunResultsData.js index 4f69589bd..7e4c84a0f 100644 --- a/DashAI/front/src/components/models/runResults/useRunResultsData.js +++ b/DashAI/front/src/components/models/runResults/useRunResultsData.js @@ -48,6 +48,9 @@ export default function useRunResultsData({ // Explainer component name to display name, fetched once and shared so cards // do not each fetch it (a per card fetch shifted heights). const [explainerDisplayNames, setExplainerDisplayNames] = useState({}); + // Explainer component name to whether it implements story(), fetched once + // alongside the display names so cards know whether to show the button. + const [explainerSupportsStory, setExplainerSupportsStory] = useState({}); // Per card plot state (items, edits, selection), so the list can unmount // offscreen cards without refetching or losing edits. const [explainerCache, setExplainerCache] = useState({}); @@ -151,10 +154,13 @@ export default function useRunResultsData({ getComponents({ selectTypes: ["GlobalExplainer", "LocalExplainer"] }) .then((components) => { const names = {}; + const supportsStory = {}; components.forEach((component) => { names[component.name] = component.display_name || component.name; + supportsStory[component.name] = Boolean(component.supports_story); }); setExplainerDisplayNames(names); + setExplainerSupportsStory(supportsStory); }) .catch(() => {}); }, []); @@ -263,6 +269,7 @@ export default function useRunResultsData({ highlightedExplainerKey, setHighlightedExplainerKey, explainerDisplayNames, + explainerSupportsStory, cardHeightsRef, getCacheEntry, updateCacheEntry, diff --git a/DashAI/front/src/types/explainer.ts b/DashAI/front/src/types/explainer.ts index 589947600..2cdd570e5 100644 --- a/DashAI/front/src/types/explainer.ts +++ b/DashAI/front/src/types/explainer.ts @@ -10,4 +10,6 @@ export interface IExplainer { fit_parameters: object; created: Date; status: number; + story?: string | null; + story_huey_id?: string | null; } diff --git a/DashAI/front/src/utils/i18n/locales/de/explainers.json b/DashAI/front/src/utils/i18n/locales/de/explainers.json index 85b9c49b0..a4dd47628 100644 --- a/DashAI/front/src/utils/i18n/locales/de/explainers.json +++ b/DashAI/front/src/utils/i18n/locales/de/explainers.json @@ -29,7 +29,8 @@ "nameTooShort": "Der Name des Erklärungsmodells muss mindestens 4 alphanumerische Zeichen enthalten.", "validateDataset": "Fehler beim Validieren des ausgewählten Datensatzes.", "noData": "Keine Daten verfügbar", - "nameAlreadyExists": "Name existiert bereits" + "nameAlreadyExists": "Name existiert bereits", + "storyGenerationFailed": "Die Story konnte nicht erstellt werden." }, "label": { "configureExplainer": "Erklärungsmodell konfigurieren", @@ -77,7 +78,10 @@ "rowModePercentage": "Prozent-Schieberegler", "rowModeManual": "Manuelle Auswahl", "shuffleRows": "Ausgewählte Zeilen mischen (Zufallsstichprobe)", - "rowsSelectedManually": "Manuell ausgewählte Zeilen: {{selected}} / {{total}}" + "rowsSelectedManually": "Manuell ausgewählte Zeilen: {{selected}} / {{total}}", + "generateStory": "Story generieren", + "regenerateStory": "Story neu generieren", + "generatingStory": "Story wird generiert..." }, "message": { "explainerJobCompleted": "Erklärungsmodell {{name}} erfolgreich abgeschlossen", diff --git a/DashAI/front/src/utils/i18n/locales/en/explainers.json b/DashAI/front/src/utils/i18n/locales/en/explainers.json index 00ada09bb..6da8ff251 100644 --- a/DashAI/front/src/utils/i18n/locales/en/explainers.json +++ b/DashAI/front/src/utils/i18n/locales/en/explainers.json @@ -29,7 +29,8 @@ "nameTooShort": "The explainer name must have at least 4 alphanumeric characters.", "validateDataset": "Error while trying to validate the selected dataset.", "noData": "No data available", - "nameAlreadyExists": "Name already exists" + "nameAlreadyExists": "Name already exists", + "storyGenerationFailed": "Failed to generate the story." }, "label": { "configureExplainer": "Configure your Explainer", @@ -77,7 +78,10 @@ "rowModePercentage": "Percentage slider", "rowModeManual": "Manual selection", "shuffleRows": "Shuffle selected rows (random sample)", - "rowsSelectedManually": "Rows selected manually: {{selected}} / {{total}}" + "rowsSelectedManually": "Rows selected manually: {{selected}} / {{total}}", + "generateStory": "Generate story", + "regenerateStory": "Regenerate story", + "generatingStory": "Generating story..." }, "message": { "explainerJobCompleted": "Explainer {{name}} completed successfully", diff --git a/DashAI/front/src/utils/i18n/locales/es/explainers.json b/DashAI/front/src/utils/i18n/locales/es/explainers.json index 47154b54f..5442d17bf 100644 --- a/DashAI/front/src/utils/i18n/locales/es/explainers.json +++ b/DashAI/front/src/utils/i18n/locales/es/explainers.json @@ -29,7 +29,8 @@ "nameTooShort": "El nombre del explicador debe tener al menos 4 caracteres alfanuméricos.", "validateDataset": "Error al intentar validar el dataset seleccionado.", "noData": "No hay datos disponibles", - "nameAlreadyExists": "El nombre ya existe" + "nameAlreadyExists": "El nombre ya existe", + "storyGenerationFailed": "No se pudo generar la historia." }, "label": { "configureExplainer": "Configure su Explicador", @@ -77,7 +78,10 @@ "rowModePercentage": "Control deslizante de porcentaje", "rowModeManual": "Selección manual", "shuffleRows": "Mezclar filas seleccionadas (muestra aleatoria)", - "rowsSelectedManually": "Filas seleccionadas manualmente: {{selected}} / {{total}}" + "rowsSelectedManually": "Filas seleccionadas manualmente: {{selected}} / {{total}}", + "generateStory": "Generar historia", + "regenerateStory": "Regenerar historia", + "generatingStory": "Generando historia..." }, "message": { "explainerJobCompleted": "Explicador {{name}} completado exitosamente", diff --git a/DashAI/front/src/utils/i18n/locales/pt/explainers.json b/DashAI/front/src/utils/i18n/locales/pt/explainers.json index 9c8a1c44c..4dfa73ac8 100644 --- a/DashAI/front/src/utils/i18n/locales/pt/explainers.json +++ b/DashAI/front/src/utils/i18n/locales/pt/explainers.json @@ -29,7 +29,8 @@ "nameTooShort": "O nome do explicador deve ter pelo menos 4 caracteres alfanuméricos.", "validateDataset": "Erro ao tentar validar o conjunto de dados selecionado.", "noData": "Sem dados disponíveis", - "nameAlreadyExists": "O nome já existe" + "nameAlreadyExists": "O nome já existe", + "storyGenerationFailed": "Falha ao gerar a história." }, "label": { "configureExplainer": "Configure seu Explicador", @@ -77,7 +78,10 @@ "rowModePercentage": "Controle deslizante de porcentagem", "rowModeManual": "Seleção manual", "shuffleRows": "Embaralhar linhas selecionadas (amostra aleatória)", - "rowsSelectedManually": "Linhas selecionadas manualmente: {{selected}} / {{total}}" + "rowsSelectedManually": "Linhas selecionadas manualmente: {{selected}} / {{total}}", + "generateStory": "Gerar história", + "regenerateStory": "Regenerar história", + "generatingStory": "Gerando história..." }, "message": { "explainerJobCompleted": "Explicador {{name}} concluído com sucesso", diff --git a/DashAI/front/src/utils/i18n/locales/zh/explainers.json b/DashAI/front/src/utils/i18n/locales/zh/explainers.json index 8939cd457..f789bae85 100644 --- a/DashAI/front/src/utils/i18n/locales/zh/explainers.json +++ b/DashAI/front/src/utils/i18n/locales/zh/explainers.json @@ -29,7 +29,8 @@ "nameTooShort": "解释器名称至少需要 4 个字母或数字字符。", "validateDataset": "验证所选数据集时出错。", "noData": "暂无数据", - "nameAlreadyExists": "名称已存在" + "nameAlreadyExists": "名称已存在", + "storyGenerationFailed": "生成故事失败。" }, "label": { "configureExplainer": "配置解释器", @@ -77,7 +78,10 @@ "rowModePercentage": "百分比滑块", "rowModeManual": "手动选择", "shuffleRows": "打乱选中的行（随机抽样）", - "rowsSelectedManually": "手动选择的行数：{{selected}} / {{total}}" + "rowsSelectedManually": "手动选择的行数：{{selected}} / {{total}}", + "generateStory": "生成故事", + "regenerateStory": "重新生成故事", + "generatingStory": "正在生成故事..." }, "message": { "explainerJobCompleted": "解释器 {{name}} 成功完成", diff --git a/tests/back/api/test_explainer_story_job.py b/tests/back/api/test_explainer_story_job.py index ced417a56..279cff386 100644 --- a/tests/back/api/test_explainer_story_job.py +++ b/tests/back/api/test_explainer_story_job.py @@ -105,6 +105,26 @@ def story(self, explainer_output): return f"Story based on '{explainer_output.payload}': {headline}" +class StorylessGlobalExplainer(BaseGlobalExplainer): + """Global explainer that never defines story() at all.""" + + COMPATIBLE_COMPONENTS = ["DummyTask"] + + def __init__(self, model: BaseModel) -> None: + self.model = model + self.explanation = None + + @classmethod + def get_schema(cls): + return {} + + def explain(self, dataset): + return {"headline": "irrelevant"} + + def plot(self, explanation): + return [TextArtifact(payload="a plot summary")] + + @pytest.fixture(autouse=True, name="test_registry") def setup_test_registry(client, monkeypatch: pytest.MonkeyPatch): container = client.app.container @@ -114,6 +134,7 @@ def setup_test_registry(client, monkeypatch: pytest.MonkeyPatch): DummyTask, DummyModel, StoryableGlobalExplainer, + StorylessGlobalExplainer, ExplainerJob, ExplainerStoryJob, ] @@ -278,3 +299,38 @@ def test_global_story_endpoint_requires_finished_explanation( def test_global_story_endpoint_requires_existing_explainer(client: TestClient): response = client.post("/api/v1/explainer/global/999999/story") assert response.status_code == 404, response.text + + +def test_global_story_endpoint_rejects_explainer_without_story( + client: TestClient, run_id: int +): + container = client.app.container + session_factory = container["session_factory"] + + with session_factory() as db: + storyless_explainer = GlobalExplainer( + run_id=run_id, + explainer_name="StorylessGlobalExplainer", + parameters={}, + ) + db.add(storyless_explainer) + db.commit() + db.refresh(storyless_explainer) + storyless_id = storyless_explainer.id + + response = client.post( + "/api/v1/job/", + data={ + "job_type": "ExplainerJob", + "kwargs": json.dumps( + { + "explainer_id": storyless_id, + "explainer_scope": "global", + } + ), + }, + ) + assert response.status_code == 201, response.text + + response = client.post(f"/api/v1/explainer/global/{storyless_id}/story") + assert response.status_code == 400, response.text diff --git a/tests/back/explainers/test_explainers.py b/tests/back/explainers/test_explainers.py index 2c854683a..4f6aa32c4 100644 --- a/tests/back/explainers/test_explainers.py +++ b/tests/back/explainers/test_explainers.py @@ -218,6 +218,38 @@ def test_permutation_feature_importance(trained_model: BaseModel, dataset: Datas assert len(values) == len(INPUT_COLUMNS) +def test_permutation_feature_importance_story( + trained_model: BaseModel, dataset: DatasetDict +): + explainer = PermutationFeatureImportance( + trained_model, + scoring="accuracy", + n_repeats=5, + random_state=0, + max_samples_fraction=1.0, + ) + explanation = explainer.explain(copy.deepcopy(dataset)) + plot = explainer.plot(explanation) + + # The job assigns this from the persisted pickle instead of recomputing + # explain(); story() must work from it, not from a fresh explain() call. + explainer.explanation = explanation + + story = explainer.story(plot[0]) + + assert isinstance(story, str) + ranking = sorted( + zip(explanation["features"], explanation["importances_mean"], strict=True), + key=lambda pair: pair[1], + reverse=True, + ) + most_important_feature = ranking[0][0] + least_important_feature = ranking[-1][0] + + assert most_important_feature in story + assert least_important_feature in story + + def plot(self, explanation: list[dict]): """Create explanation plots using plotly, tolerant to missing metadata.""" import numpy as np From 2bc934d01fae219873485d952cacc80ec5c0588f Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 4 Aug 2026 10:57:22 -0400 Subject: [PATCH 05/11] feat: add stories field to LocalExplainer model and create migration script --- ...a41e4dbd_add_stories_to_local_explainer.py | 25 +++++++++++++++++++ DashAI/back/dependencies/database/models.py | 1 + 2 files changed, 26 insertions(+) create mode 100644 DashAI/alembic/versions/0b08a41e4dbd_add_stories_to_local_explainer.py diff --git a/DashAI/alembic/versions/0b08a41e4dbd_add_stories_to_local_explainer.py b/DashAI/alembic/versions/0b08a41e4dbd_add_stories_to_local_explainer.py new file mode 100644 index 000000000..010173bdc --- /dev/null +++ b/DashAI/alembic/versions/0b08a41e4dbd_add_stories_to_local_explainer.py @@ -0,0 +1,25 @@ +"""Add stories to local_explainer + +Revision ID: 0b08a41e4dbd +Revises: 6273b0d04af3 +Create Date: 2026-08-04 10:46:18.347215 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0b08a41e4dbd" +down_revision: Union[str, None] = "6273b0d04af3" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("local_explainer", sa.Column("stories", sa.JSON(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("local_explainer", "stories") diff --git a/DashAI/back/dependencies/database/models.py b/DashAI/back/dependencies/database/models.py index a49edb8bc..6ba27b866 100644 --- a/DashAI/back/dependencies/database/models.py +++ b/DashAI/back/dependencies/database/models.py @@ -394,6 +394,7 @@ class LocalExplainer(Base): explanation_path: Mapped[str] = mapped_column(String, nullable=True) plots_path: Mapped[str] = mapped_column(String, nullable=True) plot_overrides: Mapped[JSON] = mapped_column(JSON, nullable=True) + stories: Mapped[JSON] = mapped_column(JSON, nullable=True) input_dataset_path: Mapped[str] = mapped_column(String, nullable=True) parameters: Mapped[JSON] = mapped_column(JSON) fit_parameters: Mapped[JSON] = mapped_column(JSON) From 238731dbb0df25bdf08a068005a26ddc201c3e97 Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 4 Aug 2026 11:48:33 -0400 Subject: [PATCH 06/11] feat: add endpoint for creating local explainer story and implement related request body --- .../back/api/api_v1/endpoints/explainers.py | 107 +++++++++ tests/back/api/test_explainer_story_job.py | 205 +++++++++++++++++- 2 files changed, 311 insertions(+), 1 deletion(-) diff --git a/DashAI/back/api/api_v1/endpoints/explainers.py b/DashAI/back/api/api_v1/endpoints/explainers.py index 77fe22979..44e0f1733 100755 --- a/DashAI/back/api/api_v1/endpoints/explainers.py +++ b/DashAI/back/api/api_v1/endpoints/explainers.py @@ -652,6 +652,113 @@ async def get_local_explanation_plot( ) +class LocalStoryBody(BaseModel): + """Request body for requesting a local explainer's story. + + Parameters + ---------- + group_index : int + Index of the explained instance (its position among the explanation's + groups) whose story is being requested. + """ + + group_index: int + + +@router.post("/local/{explainer_id}/story", status_code=status.HTTP_201_CREATED) +@inject +async def create_local_explainer_story( + explainer_id: int, + body: LocalStoryBody, + session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), + job_queue: "BaseJobQueue" = Depends(lambda: di["job_queue"]), + component_registry=Depends(lambda: di["component_registry"]), +): + """Enqueue story generation for one instance of a local explanation. + + Parameters + ---------- + explainer_id : int + Id of the local explainer whose explanation the story narrates. + body : LocalStoryBody + Carries ``group_index``, the explained instance to narrate. + session_factory : Callable[..., ContextManager[Session]] + A factory that creates a context manager that handles a SQLAlchemy session. + The generated session can be used to access and query the database. + job_queue : BaseJobQueue + The job queue used to enqueue the story generation job. + + Returns + ------- + dict + ``{"id": job_id}``, the huey job id; poll it via + ``GET /job/status/{job_id}`` the same way as any other job, or read + the finished result back from ``GET /local/{explainer_id}`` (the + ``stories`` field, keyed by ``str(group_index)``). + + Raises + ------ + HTTPException + If the explainer does not exist, its explanation has not finished + computing yet, or its explainer type does not support story + generation. + """ + from DashAI.back.job.base_job import JobError + from DashAI.back.job.explainer_story_job import ExplainerStoryJob + + with session_factory() as db: + try: + local_explainer = db.get(LocalExplainer, explainer_id) + + if not local_explainer: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Explainer not found", + ) + + if local_explainer.status != ExplainerStatus.FINISHED: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Explanation not found", + ) + + explainer_component = None + if local_explainer.explainer_name in component_registry: + explainer_component = component_registry[local_explainer.explainer_name] + if not explainer_component or not explainer_component.get("supports_story"): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"{local_explainer.explainer_name} does not support " + "story generation." + ), + ) + + job = ExplainerStoryJob( + explainer_id=explainer_id, + explainer_scope="local", + group_index=body.group_index, + ) + try: + job.set_status_as_delivered() + except JobError as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Story job not delivered", + ) from e + + job_id = job_queue.put(job).id + + return {"id": job_id} + + except exc.SQLAlchemyError as e: + log.exception(e) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal database error", + ) from e + + @router.put("/{scope}/plot/{explainer_id}/override") @inject async def save_plot_override( diff --git a/tests/back/api/test_explainer_story_job.py b/tests/back/api/test_explainer_story_job.py index 279cff386..581a86409 100644 --- a/tests/back/api/test_explainer_story_job.py +++ b/tests/back/api/test_explainer_story_job.py @@ -5,15 +5,17 @@ from datasets import ClassLabel, Value from fastapi.testclient import TestClient -from DashAI.back.core.artifacts import TextArtifact +from DashAI.back.core.artifacts import ArtifactGroup, GroupedArtifacts, TextArtifact from DashAI.back.dependencies.database.models import ( Dataset, GlobalExplainer, + LocalExplainer, ModelSession, Run, ) from DashAI.back.dependencies.registry import ComponentRegistry from DashAI.back.explainability.global_explainer import BaseGlobalExplainer +from DashAI.back.explainability.local_explainer import BaseLocalExplainer from DashAI.back.job.explainer_job import ExplainerJob from DashAI.back.job.explainer_story_job import ExplainerStoryJob from DashAI.back.models.base_model import BaseModel @@ -125,6 +127,65 @@ def plot(self, explanation): return [TextArtifact(payload="a plot summary")] +class StoryableLocalExplainer(BaseLocalExplainer): + """Local explainer with a deterministic, testable story() per instance.""" + + COMPATIBLE_COMPONENTS = ["DummyTask"] + + def __init__(self, model: BaseModel) -> None: + self.model = model + self.explanation = None + + @classmethod + def get_schema(cls): + return {} + + def fit(self, dataset, **kwargs): + return self + + def explain_instance(self, instances): + from DashAI.back.dataloaders.classes.dashai_dataset import to_dashai_dataset + + return {"n_instances": to_dashai_dataset(instances).num_rows} + + def plot(self, explanation): + groups = [ + ArtifactGroup( + title=f"Instance {i}", artifacts=[TextArtifact(payload=f"plot {i}")] + ) + for i in range(explanation["n_instances"]) + ] + return [GroupedArtifacts(groups=groups)] + + def story(self, explainer_output, prediction_context): + group = explainer_output.groups[0] + text = group.artifacts[0].payload + return f"Local story for '{text}', context rows={prediction_context.num_rows}" + + +class StorylessLocalExplainer(BaseLocalExplainer): + """Local explainer that never defines story() at all.""" + + COMPATIBLE_COMPONENTS = ["DummyTask"] + + def __init__(self, model: BaseModel) -> None: + self.model = model + self.explanation = None + + @classmethod + def get_schema(cls): + return {} + + def fit(self, dataset, **kwargs): + return self + + def explain_instance(self, instances): + return {} + + def plot(self, explanation): + return [TextArtifact(payload="a plot summary")] + + @pytest.fixture(autouse=True, name="test_registry") def setup_test_registry(client, monkeypatch: pytest.MonkeyPatch): container = client.app.container @@ -135,6 +196,8 @@ def setup_test_registry(client, monkeypatch: pytest.MonkeyPatch): DummyModel, StoryableGlobalExplainer, StorylessGlobalExplainer, + StoryableLocalExplainer, + StorylessLocalExplainer, ExplainerJob, ExplainerStoryJob, ] @@ -231,6 +294,32 @@ def create_global_explainer(client: TestClient, run_id: int): db.close() +@pytest.fixture(scope="module", name="local_explainer_id") +def create_local_explainer(client: TestClient, run_id: int, dataset_id: int): + container = client.app.container + session_factory = container["session_factory"] + + with session_factory() as db: + local_explainer = LocalExplainer( + name="test_story_local", + run_id=run_id, + explainer_name="StoryableLocalExplainer", + dataset_id=dataset_id, + scope={"split": "test", "percentage": 100}, + parameters={}, + fit_parameters={}, + ) + db.add(local_explainer) + db.commit() + db.refresh(local_explainer) + + yield local_explainer.id + + db.delete(local_explainer) + db.commit() + db.close() + + def test_global_story_job(client: TestClient, global_explainer_id: int): # First compute the explanation itself, same as any other explainer job. response = client.post( @@ -334,3 +423,117 @@ def test_global_story_endpoint_rejects_explainer_without_story( response = client.post(f"/api/v1/explainer/global/{storyless_id}/story") assert response.status_code == 400, response.text + + +def test_local_story_job(client: TestClient, local_explainer_id: int): + # Compute the local explanation first, same as any other local explainer. + response = client.post( + "/api/v1/job/", + data={ + "job_type": "ExplainerJob", + "kwargs": json.dumps( + { + "explainer_id": local_explainer_id, + "explainer_scope": "local", + } + ), + }, + ) + assert response.status_code == 201, response.text + + container = client.app.container + session_factory = container["session_factory"] + with session_factory() as db: + explainer = db.get(LocalExplainer, local_explainer_id) + assert explainer.plots_path, explainer + assert explainer.stories is None + + # Now request the story for instance 0 through the dedicated endpoint. + response = client.post( + f"/api/v1/explainer/local/{local_explainer_id}/story", + json={"group_index": 0}, + ) + assert response.status_code == 201, response.text + story_job_id = response.json()["id"] + + response = client.get(f"/api/v1/job/status/{story_job_id}") + assert response.status_code == 200, response.text + assert response.json()["status"] == "finished", response.json() + + with session_factory() as db: + explainer = db.get(LocalExplainer, local_explainer_id) + assert explainer.stories == {"0": "Local story for 'plot 0', context rows=1"} + + +def test_local_story_endpoint_requires_finished_explanation( + client: TestClient, run_id: int, dataset_id: int +): + container = client.app.container + session_factory = container["session_factory"] + with session_factory() as db: + unstarted_explainer = LocalExplainer( + run_id=run_id, + explainer_name="StoryableLocalExplainer", + dataset_id=dataset_id, + scope={"split": "test", "percentage": 100}, + parameters={}, + fit_parameters={}, + ) + db.add(unstarted_explainer) + db.commit() + db.refresh(unstarted_explainer) + unstarted_id = unstarted_explainer.id + + response = client.post( + f"/api/v1/explainer/local/{unstarted_id}/story", + json={"group_index": 0}, + ) + assert response.status_code == 404, response.text + + +def test_local_story_endpoint_requires_existing_explainer(client: TestClient): + response = client.post( + "/api/v1/explainer/local/999999/story", json={"group_index": 0} + ) + assert response.status_code == 404, response.text + + +def test_local_story_endpoint_rejects_explainer_without_story( + client: TestClient, run_id: int, dataset_id: int +): + container = client.app.container + session_factory = container["session_factory"] + + with session_factory() as db: + storyless_explainer = LocalExplainer( + run_id=run_id, + explainer_name="StorylessLocalExplainer", + dataset_id=dataset_id, + scope={"split": "test", "percentage": 100}, + parameters={}, + fit_parameters={}, + ) + db.add(storyless_explainer) + db.commit() + db.refresh(storyless_explainer) + storyless_id = storyless_explainer.id + + response = client.post( + "/api/v1/job/", + data={ + "job_type": "ExplainerJob", + "kwargs": json.dumps( + { + "explainer_id": storyless_id, + "explainer_scope": "local", + } + ), + }, + ) + assert response.status_code == 201, response.text + + response = client.post( + f"/api/v1/explainer/local/{storyless_id}/story", + json={"group_index": 0}, + ) + assert response.status_code == 400, response.text From 7fb355fe528ab7d6f268668779df13f2b79897b9 Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 4 Aug 2026 13:00:34 -0400 Subject: [PATCH 07/11] feat: enhance ContrastiveShap and PermutationFeatureImportance with story method and caching explanation --- .../explainers/contrastive_shap.py | 116 +++++++++++++++--- .../permutation_feature_importance.py | 11 +- tests/back/explainers/test_explainers.py | 4 - tests/back/explainers/test_new_explainers.py | 10 ++ 4 files changed, 119 insertions(+), 22 deletions(-) diff --git a/DashAI/back/explainability/explainers/contrastive_shap.py b/DashAI/back/explainability/explainers/contrastive_shap.py index 3c6e8243d..cbc523bfc 100644 --- a/DashAI/back/explainability/explainers/contrastive_shap.py +++ b/DashAI/back/explainability/explainers/contrastive_shap.py @@ -307,7 +307,8 @@ def explain_instance(self, instances): ------- dict Dictionary with, for each instance, the fact and foil classes and - the per-feature attribution difference (fact minus foil). + the per-feature attribution difference (fact minus foil). Also + cached on ``self.explanation`` for :meth:`story`. """ import numpy as np @@ -342,6 +343,7 @@ def explain_instance(self, instances): "delta_values": np.round(delta, 3).tolist(), } + self.explanation = explanation return explanation def _create_plot(self, data, fact_name, foil_name, fact_prob, foil_prob): @@ -398,6 +400,57 @@ def _create_plot(self, data, fact_name, foil_name, fact_prob, foil_prob): return fig + def _summarize_instance(self, instance: dict, metadata: dict) -> str: + """Build the "why P rather than Q" sentence for one explained instance. + + Computed directly from the explanation's own numbers (fact/foil + class, their probabilities, and the top contributing features by + absolute delta), independent of how :meth:`plot` renders them, so + :meth:`story` can call this without depending on a rendered artifact. + + Parameters + ---------- + instance : dict + One instance's entry from the explanation dict (i.e. + ``explanation[i]`` for some index ``i``, excluding the top-level + ``"metadata"`` entry). + metadata : dict + The explanation's ``"metadata"`` entry (``feature_names``, + ``target_names``). + + Returns + ------- + str + The contrastive summary sentence. + """ + import numpy as np + + feature_names = metadata["feature_names"] + target_names = metadata["target_names"] + + fact_class = instance["fact_class"] + foil_class = instance["foil_class"] + fact_name = target_names[fact_class] + foil_name = target_names[foil_class] + prediction = instance["model_prediction"] + fact_prob = float(np.round(prediction[fact_class], 3)) + foil_prob = float(np.round(prediction[foil_class], 3)) + + instance_values = instance["instance_values"] + delta = np.asarray(instance["delta_values"]) + # Stable sort to match plot()'s previous pandas-based tie-breaking: + # largest |delta| first among the top 3. + top_indices = np.argsort(np.abs(delta), kind="stable")[::-1][:3] + top_features = ", ".join( + f"{feature_names[j]}={instance_values[j]}" for j in top_indices + ) + + return ( + f"The model predicted {fact_name} (p={fact_prob}) rather than " + f"{foil_name} (p={foil_prob}) mainly because of: " + f"{top_features}." + ) + def plot(self, explanation: dict) -> List[GroupedArtifacts]: """Render each instance as a contrastive bar plot plus a text summary. @@ -449,21 +502,54 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: fig = self._create_plot(data, fact_name, foil_name, fact_prob, foil_prob) plot = PlotlyArtifact(payload=fig) - top = data.iloc[::-1].head(3) - top_features = ", ".join( - f"{feature}={value}" - for feature, value in zip( - top["features"].tolist(), - top["values"].tolist(), - strict=True, - ) - ) - summary = ( - f"The model predicted {fact_name} (p={fact_prob}) rather than " - f"{foil_name} (p={foil_prob}) mainly because of: " - f"{top_features}." - ) + summary = self._summarize_instance(instance, metadata) text = TextArtifact(payload=summary) groups.append(ArtifactGroup(title=title, artifacts=[plot, text])) return [GroupedArtifacts(groups=groups)] + + def story(self, explainer_output, prediction_context): + """Rebuild the contrastive summary from ``self.explanation``. + + Builds the same "why P rather than Q" sentence :meth:`plot` writes + into each instance's :class:`TextArtifact`, but from the explanation's + own numbers rather than by reading that already-rendered text, so it + keeps working even if :meth:`plot`'s artifact layout changes. + + Parameters + ---------- + explainer_output : GroupedArtifacts + The explained instance's group, as produced by :meth:`plot`. Only + its ``title`` (``"Instance {n}"``) is used, to recover which + entry of ``self.explanation`` this call is about. + prediction_context : DashAIDataset + Unused; the summary is built entirely from ``self.explanation``. + + Returns + ------- + str + The instance's contrastive summary sentence. + + Raises + ------ + ValueError + If the instance's title cannot be matched to an explained + instance, or ``self.explanation`` was not set before calling this + (see :meth:`explain_instance`). + """ + if not self.explanation: + raise ValueError( + "self.explanation must be set before calling story() " + "(see explain_instance)." + ) + + title = explainer_output.groups[0].title or "" + try: + index = int(title.rsplit(" ", 1)[-1]) - 1 + instance = self.explanation[index] + except (ValueError, KeyError) as e: + raise ValueError( + f"Could not match group title {title!r} to an explained instance." + ) from e + + return self._summarize_instance(instance, self.explanation["metadata"]) diff --git a/DashAI/back/explainability/explainers/permutation_feature_importance.py b/DashAI/back/explainability/explainers/permutation_feature_importance.py index 19f7a166b..af7452d51 100644 --- a/DashAI/back/explainability/explainers/permutation_feature_importance.py +++ b/DashAI/back/explainability/explainers/permutation_feature_importance.py @@ -434,7 +434,8 @@ def explain(self, dataset): dict Dictionary with keys ``"features"`` (list of str), ``"importances_mean"`` (list of float, rounded to 3 dp), and - ``"importances_std"`` (list of float, rounded to 3 dp). + ``"importances_std"`` (list of float, rounded to 3 dp). Also + cached on ``self.explanation`` for :meth:`story`. """ # Lazy imports import numpy as np @@ -481,11 +482,13 @@ def explain(self, dataset): results = self._calculate_grouped_importance( X_df, y_df, feature_groups, max_samples ) - return { + explanation = { "features": results["features"], "importances_mean": np.round(results["importances_mean"], 3).tolist(), "importances_std": np.round(results["importances_std"], 3).tolist(), } + self.explanation = explanation + return explanation else: def patched_metric(y_true, y_pred_probas): @@ -520,11 +523,13 @@ def patched_metric(y_true, y_pred_probas): max_samples=max_samples, ) - return { + explanation = { "features": input_columns, "importances_mean": np.round(pfi["importances_mean"], 3).tolist(), "importances_std": np.round(pfi["importances_std"], 3).tolist(), } + self.explanation = explanation + return explanation def _create_plot(self, data) -> List[GroupedArtifacts]: """Build one selector over feature counts. diff --git a/tests/back/explainers/test_explainers.py b/tests/back/explainers/test_explainers.py index 4f6aa32c4..dec0acefc 100644 --- a/tests/back/explainers/test_explainers.py +++ b/tests/back/explainers/test_explainers.py @@ -231,10 +231,6 @@ def test_permutation_feature_importance_story( explanation = explainer.explain(copy.deepcopy(dataset)) plot = explainer.plot(explanation) - # The job assigns this from the persisted pickle instead of recomputing - # explain(); story() must work from it, not from a fresh explain() call. - explainer.explanation = explanation - story = explainer.story(plot[0]) assert isinstance(story, str) diff --git a/tests/back/explainers/test_new_explainers.py b/tests/back/explainers/test_new_explainers.py index 775a59366..73716031d 100644 --- a/tests/back/explainers/test_new_explainers.py +++ b/tests/back/explainers/test_new_explainers.py @@ -205,6 +205,16 @@ def test_contrastive_shap(trained_model, dataset): assert [a.type for a in groups[0].artifacts] == ["plotly", "text"] assert "rather than" in groups[0].artifacts[1].payload + # story() must reuse plot()'s own summary text, not build a new one. + from DashAI.back.core.artifacts import GroupedArtifacts + + for group in groups: + text_artifact = next(a for a in group.artifacts if a.type == "text") + single_group_output = GroupedArtifacts(groups=[group]) + assert explainer.story(single_group_output, instances) == ( + text_artifact.payload + ) + def test_contrastive_shap_fixed_foil(trained_model, dataset): x, _ = dataset From ad8b6d440ef5f57eabf0a38288d3bca4aa7d2a17 Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 4 Aug 2026 15:49:15 -0400 Subject: [PATCH 08/11] Refactor explainers to support narrative summaries on demand - Introduced `_summarize_instance` methods in GradCam, NearestCounterfactual, OcclusionSaliency, RegressionKernelShap, and TokenAblation classes to generate descriptive summaries based on explanation data. - Updated `plot` methods in these classes to remove narrative summary generation, delegating it to a new `story` method that retrieves the summary when requested. - Added `createLocalExplainerStory` API endpoint to facilitate story generation for local explainers. - Enhanced ExplainersCard and ExplainersPlot components to manage story generation and display results using ArtifactViewer. - Updated TypeScript types to include optional `stories` property for explainers. --- .../explainers/contrastive_shap.py | 21 +- .../explainers/dice_counterfactual.py | 124 +++++++--- .../explainability/explainers/grad_cam.py | 97 +++++++- .../explainers/nearest_counterfactual.py | 125 +++++++--- .../explainers/occlusion_saliency.py | 97 +++++++- .../explainers/regression_kernel_shap.py | 121 ++++++++-- .../explainers/token_ablation.py | 111 +++++++-- DashAI/front/src/api/explainer.ts | 11 + .../components/explainers/ExplainersCard.jsx | 107 ++++----- .../components/explainers/ExplainersPlot.jsx | 220 +++++++++++++++--- DashAI/front/src/types/explainer.ts | 1 + 11 files changed, 831 insertions(+), 204 deletions(-) diff --git a/DashAI/back/explainability/explainers/contrastive_shap.py b/DashAI/back/explainability/explainers/contrastive_shap.py index cbc523bfc..b671fe8c6 100644 --- a/DashAI/back/explainability/explainers/contrastive_shap.py +++ b/DashAI/back/explainability/explainers/contrastive_shap.py @@ -4,7 +4,6 @@ ArtifactGroup, GroupedArtifacts, PlotlyArtifact, - TextArtifact, ) from DashAI.back.core.schema_fields import ( BaseSchema, @@ -452,7 +451,10 @@ def _summarize_instance(self, instance: dict, metadata: dict) -> str: ) def plot(self, explanation: dict) -> List[GroupedArtifacts]: - """Render each instance as a contrastive bar plot plus a text summary. + """Render each instance as a contrastive bar plot. + + The narrative summary is not computed here: it is only built on + demand by :meth:`story`. Parameters ---------- @@ -463,7 +465,7 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: ------- List[GroupedArtifacts] A single grouped artifact with one group per explained instance, - each holding that instance's contrastive plot and text summary. + each holding that instance's contrastive plot. """ import numpy as np import pandas as pd @@ -502,19 +504,16 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: fig = self._create_plot(data, fact_name, foil_name, fact_prob, foil_prob) plot = PlotlyArtifact(payload=fig) - summary = self._summarize_instance(instance, metadata) - text = TextArtifact(payload=summary) - groups.append(ArtifactGroup(title=title, artifacts=[plot, text])) + groups.append(ArtifactGroup(title=title, artifacts=[plot])) return [GroupedArtifacts(groups=groups)] def story(self, explainer_output, prediction_context): - """Rebuild the contrastive summary from ``self.explanation``. + """Build the "why P rather than Q" sentence from ``self.explanation``. - Builds the same "why P rather than Q" sentence :meth:`plot` writes - into each instance's :class:`TextArtifact`, but from the explanation's - own numbers rather than by reading that already-rendered text, so it - keeps working even if :meth:`plot`'s artifact layout changes. + Computed on demand, only when a story is requested: :meth:`plot` + never builds this narrative, so no cost is paid unless it is asked + for. Parameters ---------- diff --git a/DashAI/back/explainability/explainers/dice_counterfactual.py b/DashAI/back/explainability/explainers/dice_counterfactual.py index 381fd6e6a..923fbecdf 100644 --- a/DashAI/back/explainability/explainers/dice_counterfactual.py +++ b/DashAI/back/explainability/explainers/dice_counterfactual.py @@ -5,7 +5,6 @@ GroupedArtifacts, TableArtifact, TablePayload, - TextArtifact, ) from DashAI.back.core.schema_fields import ( BaseSchema, @@ -373,10 +372,59 @@ def explain_instance(self, instances): "counterfactuals": counterfactuals, } + self.explanation = explanation return explanation + def _summarize_instance(self, instance: dict, metadata: dict) -> str: + """Build the DiCE counterfactual comparison sentence for one instance. + + Computed directly from the explanation's own numbers, independent of + how :meth:`plot` renders the comparison table, so :meth:`story` can + call this without depending on a rendered artifact. + + Parameters + ---------- + instance : dict + One instance's entry from the explanation dict. + metadata : dict + The explanation's ``"metadata"`` entry (``target_names``). + + Returns + ------- + str + The counterfactual comparison summary. + """ + import numpy as np + + target_names = metadata["target_names"] + predicted_class = instance["predicted_class"] + predicted_name = target_names[predicted_class] + predicted_prob = float( + np.round(instance["model_prediction"][predicted_class], 3) + ) + counterfactuals = instance["counterfactuals"] + + if not counterfactuals: + return ( + f"The model predicted {predicted_name} " + f"(p={predicted_prob}). DiCE could not generate " + "counterfactuals for this instance." + ) + + lines = [f"The model predicted {predicted_name} (p={predicted_prob})."] + for cf_idx, counterfactual in enumerate(counterfactuals): + cf_name = target_names[counterfactual["predicted_class"]] + changed = ", ".join(counterfactual["changed_features"]) or "nothing" + lines.append( + f"Counterfactual {cf_idx + 1}: changing {changed} yields {cf_name}." + ) + return "\n".join(lines) + def plot(self, explanation: dict) -> List[GroupedArtifacts]: - """Render each instance as a comparison table plus a text summary. + """Render each instance as a comparison table. + + The narrative summary is not computed here: it is only built on + demand by :meth:`story`. Parameters ---------- @@ -387,10 +435,8 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: ------- List[GroupedArtifacts] A single grouped artifact with one group per explained instance, - each holding that instance's comparison table and text summary. + each holding that instance's comparison table. """ - import numpy as np - exp = explanation.copy() metadata = exp.pop("metadata") feature_names = metadata["feature_names"] @@ -401,9 +447,6 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: instance = exp[i] predicted_class = instance["predicted_class"] predicted_name = target_names[predicted_class] - predicted_prob = float( - np.round(instance["model_prediction"][predicted_class], 3) - ) counterfactuals = instance["counterfactuals"] columns = ["Feature", "Instance"] + [ @@ -431,24 +474,51 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: table = TableArtifact( payload=TablePayload(columns=columns, rows=rows, highlight=highlight), ) - - if counterfactuals: - lines = [f"The model predicted {predicted_name} (p={predicted_prob})."] - for cf_idx, counterfactual in enumerate(counterfactuals): - cf_name = target_names[counterfactual["predicted_class"]] - changed = ", ".join(counterfactual["changed_features"]) or "nothing" - lines.append( - f"Counterfactual {cf_idx + 1}: changing {changed} " - f"yields {cf_name}." - ) - summary = "\n".join(lines) - else: - summary = ( - f"The model predicted {predicted_name} " - f"(p={predicted_prob}). DiCE could not generate " - "counterfactuals for this instance." - ) - text = TextArtifact(payload=summary) - groups.append(ArtifactGroup(title=title, artifacts=[table, text])) + groups.append(ArtifactGroup(title=title, artifacts=[table])) return [GroupedArtifacts(groups=groups)] + + def story(self, explainer_output, prediction_context): + """Build the DiCE counterfactual sentence from ``self.explanation``. + + Computed on demand, only when a story is requested: :meth:`plot` + never builds this narrative, so no cost is paid unless it is asked + for. + + Parameters + ---------- + explainer_output : GroupedArtifacts + The explained instance's group, as produced by :meth:`plot`. Only + its ``title`` (``"Instance {n}"``) is used, to recover which + entry of ``self.explanation`` this call is about. + prediction_context : DashAIDataset + Unused; the summary is built entirely from ``self.explanation``. + + Returns + ------- + str + The instance's counterfactual comparison summary. + + Raises + ------ + ValueError + If the instance's title cannot be matched to an explained + instance, or ``self.explanation`` was not set before calling this + (see :meth:`explain_instance`). + """ + if not self.explanation: + raise ValueError( + "self.explanation must be set before calling story() " + "(see explain_instance)." + ) + + title = explainer_output.groups[0].title or "" + try: + index = int(title.rsplit(" ", 1)[-1]) - 1 + instance = self.explanation[index] + except (ValueError, KeyError) as e: + raise ValueError( + f"Could not match group title {title!r} to an explained instance." + ) from e + + return self._summarize_instance(instance, self.explanation["metadata"]) diff --git a/DashAI/back/explainability/explainers/grad_cam.py b/DashAI/back/explainability/explainers/grad_cam.py index d22b95a30..b6b1c15dd 100644 --- a/DashAI/back/explainability/explainers/grad_cam.py +++ b/DashAI/back/explainability/explainers/grad_cam.py @@ -3,7 +3,6 @@ from DashAI.back.core.artifacts import ( ArtifactGroup, GroupedArtifacts, - TextArtifact, ) from DashAI.back.core.schema_fields import ( BaseSchema, @@ -241,10 +240,48 @@ def explain_instance(self, instances): "predicted_class": predicted_class, } + self.explanation = explanation return explanation + def _summarize_instance(self, instance: dict, metadata: dict) -> str: + """Build the heatmap description sentence for one explained image. + + Computed directly from the explanation's own numbers, independent of + how :meth:`plot` renders the heatmap overlay, so :meth:`story` can + call this without depending on a rendered artifact. + + Parameters + ---------- + instance : dict + One image's entry from the explanation dict. + metadata : dict + The explanation's ``"metadata"`` entry (``target_names``). + + Returns + ------- + str + The heatmap description sentence. + """ + import numpy as np + + target_names = metadata["target_names"] + predicted_class = instance["predicted_class"] + predicted_name = target_names[predicted_class] + predicted_prob = float( + np.round(instance["model_prediction"][predicted_class], 3) + ) + + return ( + f"The model predicted {predicted_name} " + f"(p={predicted_prob}). Highlighted regions are the " + "areas whose activations most supported this class." + ) + def plot(self, explanation: dict) -> List[GroupedArtifacts]: - """Render each image as a heatmap overlay plus a text summary. + """Render each image as a heatmap overlay. + + The narrative summary is not computed here: it is only built on + demand by :meth:`story`. Parameters ---------- @@ -255,7 +292,7 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: ------- List[GroupedArtifacts] A single grouped artifact with one group per explained image, each - holding that image's heatmap overlay and text summary. + holding that image's heatmap overlay. """ import numpy as np @@ -280,13 +317,51 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: overlay = heatmap_overlay_artifact( instance["image"], instance["heatmap"], title, subtitle ) - text = TextArtifact( - payload=( - f"The model predicted {predicted_name} " - f"(p={predicted_prob}). Highlighted regions are the " - "areas whose activations most supported this class." - ), - ) - groups.append(ArtifactGroup(title=title, artifacts=[overlay, text])) + groups.append(ArtifactGroup(title=title, artifacts=[overlay])) return [GroupedArtifacts(groups=groups)] + + def story(self, explainer_output, prediction_context): + """Build the heatmap description sentence from ``self.explanation``. + + Computed on demand, only when a story is requested: :meth:`plot` + never builds this narrative, so no cost is paid unless it is asked + for. + + Parameters + ---------- + explainer_output : GroupedArtifacts + The explained image's group, as produced by :meth:`plot`. Only + its ``title`` (``"Image {n}"``) is used, to recover which entry + of ``self.explanation`` this call is about. + prediction_context : DashAIDataset + Unused; the summary is built entirely from ``self.explanation``. + + Returns + ------- + str + The instance's heatmap description. + + Raises + ------ + ValueError + If the instance's title cannot be matched to an explained + instance, or ``self.explanation`` was not set before calling this + (see :meth:`explain_instance`). + """ + if not self.explanation: + raise ValueError( + "self.explanation must be set before calling story() " + "(see explain_instance)." + ) + + title = explainer_output.groups[0].title or "" + try: + index = int(title.rsplit(" ", 1)[-1]) - 1 + instance = self.explanation[index] + except (ValueError, KeyError) as e: + raise ValueError( + f"Could not match group title {title!r} to an explained instance." + ) from e + + return self._summarize_instance(instance, self.explanation["metadata"]) diff --git a/DashAI/back/explainability/explainers/nearest_counterfactual.py b/DashAI/back/explainability/explainers/nearest_counterfactual.py index 948d5b88b..ad1768408 100644 --- a/DashAI/back/explainability/explainers/nearest_counterfactual.py +++ b/DashAI/back/explainability/explainers/nearest_counterfactual.py @@ -5,7 +5,6 @@ GroupedArtifacts, TableArtifact, TablePayload, - TextArtifact, ) from DashAI.back.core.schema_fields import ( BaseSchema, @@ -334,10 +333,60 @@ def explain_instance(self, instances): "counterfactuals": counterfactuals, } + self.explanation = explanation return explanation + def _summarize_instance(self, instance: dict, metadata: dict) -> str: + """Build the counterfactual comparison sentence for one instance. + + Computed directly from the explanation's own numbers, independent of + how :meth:`plot` renders the comparison table, so :meth:`story` can + call this without depending on a rendered artifact. + + Parameters + ---------- + instance : dict + One instance's entry from the explanation dict. + metadata : dict + The explanation's ``"metadata"`` entry (``target_names``). + + Returns + ------- + str + The counterfactual comparison summary. + """ + import numpy as np + + target_names = metadata["target_names"] + predicted_class = instance["predicted_class"] + predicted_name = target_names[predicted_class] + predicted_prob = float( + np.round(instance["model_prediction"][predicted_class], 3) + ) + counterfactuals = instance["counterfactuals"] + + if not counterfactuals: + return ( + f"The model predicted {predicted_name} (p={predicted_prob}). " + "No counterfactual examples were found in the training data." + ) + + lines = [f"The model predicted {predicted_name} (p={predicted_prob})."] + for cf_idx, counterfactual in enumerate(counterfactuals): + cf_name = target_names[counterfactual["predicted_class"]] + changed = ", ".join(counterfactual["changed_features"]) or "nothing" + lines.append( + f"Counterfactual {cf_idx + 1}: changing {changed} " + f"yields {cf_name} " + f"(distance {counterfactual['distance']})." + ) + return "\n".join(lines) + def plot(self, explanation: dict) -> List[GroupedArtifacts]: - """Render each instance as a comparison table plus a text summary. + """Render each instance as a comparison table. + + The narrative summary is not computed here: it is only built on + demand by :meth:`story`. Parameters ---------- @@ -348,10 +397,8 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: ------- List[GroupedArtifacts] A single grouped artifact with one group per explained instance, - each holding that instance's comparison table and text summary. + each holding that instance's comparison table. """ - import numpy as np - exp = explanation.copy() metadata = exp.pop("metadata") feature_names = metadata["feature_names"] @@ -363,9 +410,6 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: instance_values = instance["instance_values"] predicted_class = instance["predicted_class"] predicted_name = target_names[predicted_class] - predicted_prob = float( - np.round(instance["model_prediction"][predicted_class], 3) - ) counterfactuals = instance["counterfactuals"] columns = ["Feature", "Instance"] + [ @@ -393,24 +437,51 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: table = TableArtifact( payload=TablePayload(columns=columns, rows=rows, highlight=highlight), ) - - if counterfactuals: - lines = [f"The model predicted {predicted_name} (p={predicted_prob})."] - for cf_idx, counterfactual in enumerate(counterfactuals): - cf_name = target_names[counterfactual["predicted_class"]] - changed = ", ".join(counterfactual["changed_features"]) or "nothing" - lines.append( - f"Counterfactual {cf_idx + 1}: changing {changed} " - f"yields {cf_name} " - f"(distance {counterfactual['distance']})." - ) - summary = "\n".join(lines) - else: - summary = ( - f"The model predicted {predicted_name} (p={predicted_prob}). " - "No counterfactual examples were found in the training data." - ) - text = TextArtifact(payload=summary) - groups.append(ArtifactGroup(title=title, artifacts=[table, text])) + groups.append(ArtifactGroup(title=title, artifacts=[table])) return [GroupedArtifacts(groups=groups)] + + def story(self, explainer_output, prediction_context): + """Build the counterfactual comparison sentence from ``self.explanation``. + + Computed on demand, only when a story is requested: :meth:`plot` + never builds this narrative, so no cost is paid unless it is asked + for. + + Parameters + ---------- + explainer_output : GroupedArtifacts + The explained instance's group, as produced by :meth:`plot`. Only + its ``title`` (``"Instance {n}"``) is used, to recover which + entry of ``self.explanation`` this call is about. + prediction_context : DashAIDataset + Unused; the summary is built entirely from ``self.explanation``. + + Returns + ------- + str + The instance's counterfactual comparison summary. + + Raises + ------ + ValueError + If the instance's title cannot be matched to an explained + instance, or ``self.explanation`` was not set before calling this + (see :meth:`explain_instance`). + """ + if not self.explanation: + raise ValueError( + "self.explanation must be set before calling story() " + "(see explain_instance)." + ) + + title = explainer_output.groups[0].title or "" + try: + index = int(title.rsplit(" ", 1)[-1]) - 1 + instance = self.explanation[index] + except (ValueError, KeyError) as e: + raise ValueError( + f"Could not match group title {title!r} to an explained instance." + ) from e + + return self._summarize_instance(instance, self.explanation["metadata"]) diff --git a/DashAI/back/explainability/explainers/occlusion_saliency.py b/DashAI/back/explainability/explainers/occlusion_saliency.py index 5baa6a18d..9d49961fe 100644 --- a/DashAI/back/explainability/explainers/occlusion_saliency.py +++ b/DashAI/back/explainability/explainers/occlusion_saliency.py @@ -3,7 +3,6 @@ from DashAI.back.core.artifacts import ( ArtifactGroup, GroupedArtifacts, - TextArtifact, ) from DashAI.back.core.schema_fields import ( BaseSchema, @@ -299,10 +298,48 @@ def explain_instance(self, instances): "predicted_class": predicted_class, } + self.explanation = explanation return explanation + def _summarize_instance(self, instance: dict, metadata: dict) -> str: + """Build the saliency description sentence for one explained image. + + Computed directly from the explanation's own numbers, independent of + how :meth:`plot` renders the saliency overlay, so :meth:`story` can + call this without depending on a rendered artifact. + + Parameters + ---------- + instance : dict + One image's entry from the explanation dict. + metadata : dict + The explanation's ``"metadata"`` entry (``target_names``). + + Returns + ------- + str + The saliency description sentence. + """ + import numpy as np + + target_names = metadata["target_names"] + predicted_class = instance["predicted_class"] + predicted_name = target_names[predicted_class] + predicted_prob = float( + np.round(instance["model_prediction"][predicted_class], 3) + ) + + return ( + f"The model predicted {predicted_name} " + f"(p={predicted_prob}). Highlighted regions are those " + "whose occlusion most lowered that probability." + ) + def plot(self, explanation: dict) -> List[GroupedArtifacts]: - """Render each image as a saliency overlay plus a text summary. + """Render each image as a saliency overlay. + + The narrative summary is not computed here: it is only built on + demand by :meth:`story`. Parameters ---------- @@ -313,7 +350,7 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: ------- List[GroupedArtifacts] A single grouped artifact with one group per explained image, each - holding that image's saliency overlay and text summary. + holding that image's saliency overlay. """ import numpy as np @@ -335,13 +372,51 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: overlay = heatmap_overlay_artifact( instance["image"], instance["heatmap"], title, subtitle ) - text = TextArtifact( - payload=( - f"The model predicted {predicted_name} " - f"(p={predicted_prob}). Highlighted regions are those " - "whose occlusion most lowered that probability." - ), - ) - groups.append(ArtifactGroup(title=title, artifacts=[overlay, text])) + groups.append(ArtifactGroup(title=title, artifacts=[overlay])) return [GroupedArtifacts(groups=groups)] + + def story(self, explainer_output, prediction_context): + """Build the saliency description sentence from ``self.explanation``. + + Computed on demand, only when a story is requested: :meth:`plot` + never builds this narrative, so no cost is paid unless it is asked + for. + + Parameters + ---------- + explainer_output : GroupedArtifacts + The explained image's group, as produced by :meth:`plot`. Only + its ``title`` (``"Image {n}"``) is used, to recover which entry + of ``self.explanation`` this call is about. + prediction_context : DashAIDataset + Unused; the summary is built entirely from ``self.explanation``. + + Returns + ------- + str + The instance's saliency description. + + Raises + ------ + ValueError + If the instance's title cannot be matched to an explained + instance, or ``self.explanation`` was not set before calling this + (see :meth:`explain_instance`). + """ + if not self.explanation: + raise ValueError( + "self.explanation must be set before calling story() " + "(see explain_instance)." + ) + + title = explainer_output.groups[0].title or "" + try: + index = int(title.rsplit(" ", 1)[-1]) - 1 + instance = self.explanation[index] + except (ValueError, KeyError) as e: + raise ValueError( + f"Could not match group title {title!r} to an explained instance." + ) from e + + return self._summarize_instance(instance, self.explanation["metadata"]) diff --git a/DashAI/back/explainability/explainers/regression_kernel_shap.py b/DashAI/back/explainability/explainers/regression_kernel_shap.py index 63661beaf..4040d2913 100644 --- a/DashAI/back/explainability/explainers/regression_kernel_shap.py +++ b/DashAI/back/explainability/explainers/regression_kernel_shap.py @@ -4,7 +4,6 @@ ArtifactGroup, GroupedArtifacts, PlotlyArtifact, - TextArtifact, ) from DashAI.back.core.schema_fields import ( BaseSchema, @@ -251,10 +250,60 @@ def explain_instance(self, instances): "shap_values": np.round(contributions, 3).tolist(), } + self.explanation = explanation return explanation + def _summarize_instance( + self, instance: dict, metadata: dict, base_value: float + ) -> str: + """Build the SHAP contribution sentence for one explained instance. + + Computed directly from the explanation's own numbers (top features + by absolute SHAP value), independent of how :meth:`plot` renders + them, so :meth:`story` can call this without depending on a rendered + artifact. + + Parameters + ---------- + instance : dict + One instance's entry from the explanation dict. + metadata : dict + The explanation's ``"metadata"`` entry (``feature_names``, + ``output_column``). + base_value : float + The explanation's top-level ``"base_value"`` (SHAP baseline). + + Returns + ------- + str + The SHAP contribution summary sentence. + """ + import numpy as np + + feature_names = metadata["feature_names"] + output_column = metadata["output_column"] + prediction = instance["model_prediction"] + + instance_values = instance["instance_values"] + shap_values = np.asarray(instance["shap_values"]) + top_indices = np.argsort(np.abs(shap_values), kind="stable")[::-1][:3] + top_features = ", ".join( + f"{feature_names[j]}={instance_values[j]} ({shap_values[j]:+})" + for j in top_indices + ) + + delta = float(np.round(prediction - base_value, 3)) + return ( + f"The model predicted {output_column}={prediction}, " + f"{delta:+} from the baseline {base_value}. " + f"Main contributions: {top_features}." + ) + def plot(self, explanation: dict) -> List[GroupedArtifacts]: - """Render each instance as a SHAP bar plot plus a text summary. + """Render each instance as a SHAP bar plot. + + The narrative summary is not computed here: it is only built on + demand by :meth:`story`. Parameters ---------- @@ -265,9 +314,8 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: ------- List[GroupedArtifacts] A single grouped artifact with one group per explained instance, - each holding that instance's plotly plot and text summary. + each holding that instance's plotly plot. """ - import numpy as np import pandas as pd import plotly.graph_objs as go @@ -325,24 +373,53 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: title = f"Instance {int(i) + 1}" plot = PlotlyArtifact(payload=fig) + groups.append(ArtifactGroup(title=title, artifacts=[plot])) - top = data.iloc[::-1].head(3) - top_features = ", ".join( - f"{feature}={value} ({shap:+})" - for feature, value, shap in zip( - top["features"].tolist(), - top["values"].tolist(), - top["shap_values"].tolist(), - strict=True, - ) - ) - delta = float(np.round(prediction - base_value, 3)) - summary = ( - f"The model predicted {output_column}={prediction}, " - f"{delta:+} from the baseline {base_value}. " - f"Main contributions: {top_features}." + return [GroupedArtifacts(groups=groups)] + + def story(self, explainer_output, prediction_context): + """Build the SHAP contribution sentence from ``self.explanation``. + + Computed on demand, only when a story is requested: :meth:`plot` + never builds this narrative, so no cost is paid unless it is asked + for. + + Parameters + ---------- + explainer_output : GroupedArtifacts + The explained instance's group, as produced by :meth:`plot`. Only + its ``title`` (``"Instance {n}"``) is used, to recover which + entry of ``self.explanation`` this call is about. + prediction_context : DashAIDataset + Unused; the summary is built entirely from ``self.explanation``. + + Returns + ------- + str + The instance's SHAP contribution summary. + + Raises + ------ + ValueError + If the instance's title cannot be matched to an explained + instance, or ``self.explanation`` was not set before calling this + (see :meth:`explain_instance`). + """ + if not self.explanation: + raise ValueError( + "self.explanation must be set before calling story() " + "(see explain_instance)." ) - text = TextArtifact(payload=summary) - groups.append(ArtifactGroup(title=title, artifacts=[plot, text])) - return [GroupedArtifacts(groups=groups)] + title = explainer_output.groups[0].title or "" + try: + index = int(title.rsplit(" ", 1)[-1]) - 1 + instance = self.explanation[index] + except (ValueError, KeyError) as e: + raise ValueError( + f"Could not match group title {title!r} to an explained instance." + ) from e + + return self._summarize_instance( + instance, self.explanation["metadata"], self.explanation["base_value"] + ) diff --git a/DashAI/back/explainability/explainers/token_ablation.py b/DashAI/back/explainability/explainers/token_ablation.py index 401daf1c8..ab98fe984 100644 --- a/DashAI/back/explainability/explainers/token_ablation.py +++ b/DashAI/back/explainability/explainers/token_ablation.py @@ -4,7 +4,6 @@ ArtifactGroup, GroupedArtifacts, PlotlyArtifact, - TextArtifact, ) from DashAI.back.core.schema_fields import ( BaseSchema, @@ -269,10 +268,58 @@ def explain_instance(self, instances): "predicted_class": predicted_class, } + self.explanation = explanation return explanation + def _summarize_instance(self, instance: dict, metadata: dict) -> str: + """Build the token importance sentence for one explained instance. + + Computed directly from the explanation's own numbers (top tokens by + absolute importance), independent of how :meth:`plot` renders them, + so :meth:`story` can call this without depending on a rendered + artifact. + + Parameters + ---------- + instance : dict + One instance's entry from the explanation dict. + metadata : dict + The explanation's ``"metadata"`` entry (``target_names``). + + Returns + ------- + str + The token importance summary sentence. + """ + import numpy as np + + target_names = metadata["target_names"] + predicted_class = instance["predicted_class"] + predicted_name = target_names[predicted_class] + predicted_prob = float( + np.round(instance["model_prediction"][predicted_class], 3) + ) + + tokens = instance["tokens"] + importances = np.asarray(instance["token_importances"]) + if len(importances) > 0: + top_indices = np.argsort(np.abs(importances), kind="stable")[::-1][:3] + top_tokens = ", ".join( + f"'{tokens[j]} ({j})' ({importances[j]:+})" for j in top_indices + ) + else: + top_tokens = "none" + + return ( + f"The model predicted {predicted_name} (p={predicted_prob}). " + f"Most influential tokens: {top_tokens}." + ) + def plot(self, explanation: dict) -> List[GroupedArtifacts]: - """Render each instance as a token importance bar plot plus a summary. + """Render each instance as a token importance bar plot. + + The narrative summary is not computed here: it is only built on + demand by :meth:`story`. Parameters ---------- @@ -283,7 +330,7 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: ------- List[GroupedArtifacts] A single grouped artifact with one group per explained instance, - each holding that instance's token plot and text summary. + each holding that instance's token plot. """ import numpy as np import pandas as pd @@ -346,19 +393,51 @@ def plot(self, explanation: dict) -> List[GroupedArtifacts]: title = f"Instance {int(i) + 1}" plot = PlotlyArtifact(payload=fig) + groups.append(ArtifactGroup(title=title, artifacts=[plot])) - top = data.iloc[::-1].head(3) - top_tokens = ", ".join( - f"'{token}' ({importance:+})" - for token, importance in zip( - top["tokens"].tolist(), top["importances"].tolist(), strict=True - ) - ) - summary = ( - f"The model predicted {predicted_name} (p={predicted_prob}). " - f"Most influential tokens: {top_tokens}." + return [GroupedArtifacts(groups=groups)] + + def story(self, explainer_output, prediction_context): + """Build the token importance sentence from ``self.explanation``. + + Computed on demand, only when a story is requested: :meth:`plot` + never builds this narrative, so no cost is paid unless it is asked + for. + + Parameters + ---------- + explainer_output : GroupedArtifacts + The explained instance's group, as produced by :meth:`plot`. Only + its ``title`` (``"Instance {n}"``) is used, to recover which + entry of ``self.explanation`` this call is about. + prediction_context : DashAIDataset + Unused; the summary is built entirely from ``self.explanation``. + + Returns + ------- + str + The instance's token importance summary. + + Raises + ------ + ValueError + If the instance's title cannot be matched to an explained + instance, or ``self.explanation`` was not set before calling this + (see :meth:`explain_instance`). + """ + if not self.explanation: + raise ValueError( + "self.explanation must be set before calling story() " + "(see explain_instance)." ) - text = TextArtifact(payload=summary) - groups.append(ArtifactGroup(title=title, artifacts=[plot, text])) - return [GroupedArtifacts(groups=groups)] + title = explainer_output.groups[0].title or "" + try: + index = int(title.rsplit(" ", 1)[-1]) - 1 + instance = self.explanation[index] + except (ValueError, KeyError) as e: + raise ValueError( + f"Could not match group title {title!r} to an explained instance." + ) from e + + return self._summarize_instance(instance, self.explanation["metadata"]) diff --git a/DashAI/front/src/api/explainer.ts b/DashAI/front/src/api/explainer.ts index 1426283cb..d1507d951 100644 --- a/DashAI/front/src/api/explainer.ts +++ b/DashAI/front/src/api/explainer.ts @@ -48,6 +48,17 @@ export const createGlobalExplainerStory = async ( return response.data; }; +export const createLocalExplainerStory = async ( + explainerId: number, + groupIndex: number, +): Promise<{ id: string }> => { + const response = await api.post<{ id: string }>( + `/v1/explainer/local/${explainerId}/story`, + { group_index: groupIndex }, + ); + return response.data; +}; + export const createLocalExplainer = async ( runId: number, explainerName: string, diff --git a/DashAI/front/src/components/explainers/ExplainersCard.jsx b/DashAI/front/src/components/explainers/ExplainersCard.jsx index dcca4a0df..14384673c 100644 --- a/DashAI/front/src/components/explainers/ExplainersCard.jsx +++ b/DashAI/front/src/components/explainers/ExplainersCard.jsx @@ -16,6 +16,7 @@ import ZoomInIcon from "@mui/icons-material/ZoomIn"; import AutoAwesomeIcon from "@mui/icons-material/AutoAwesome"; import PropTypes from "prop-types"; import ExplainersPlot from "./ExplainersPlot"; +import ArtifactViewer from "../shared/ArtifactViewer"; import { useNavigate } from "react-router-dom"; import { deleteExplainer, @@ -203,67 +204,63 @@ export default function ExplainersCard({ ) : ( - {scope === "global" && supportsStory && ( - - {storyState.status === "loading" ? ( - - - - {t("explainers:label.generatingStory")} - - - ) : ( - - )} - - )} - {storyState.story && ( - alpha(t.palette.primary.main, 0.04), - borderColor: theme.palette.ui.border, - }} - > - {storyState.story} - - )} - {storyState.status === "error" && ( - - {t("explainers:error.storyGenerationFailed")} - + {scope === "global" && supportsStory && ( + <> + + {storyState.status === "loading" ? ( + + + + {t("explainers:label.generatingStory")} + + + ) : ( + + )} + + {storyState.story && ( + + )} + {storyState.status === "error" && ( + + {t("explainers:error.storyGenerationFailed")} + + )} + )} )} @@ -325,7 +322,11 @@ export default function ExplainersCard({ /> - + ); diff --git a/DashAI/front/src/components/explainers/ExplainersPlot.jsx b/DashAI/front/src/components/explainers/ExplainersPlot.jsx index 99c8f0cec..d02534874 100644 --- a/DashAI/front/src/components/explainers/ExplainersPlot.jsx +++ b/DashAI/front/src/components/explainers/ExplainersPlot.jsx @@ -1,9 +1,15 @@ import { React, useEffect, useState } from "react"; -import { CircularProgress, Box } from "@mui/material"; +import { CircularProgress, Box, Button, Typography } from "@mui/material"; +import AutoAwesomeIcon from "@mui/icons-material/AutoAwesome"; import PropTypes from "prop-types"; import { useSnackbar } from "notistack"; -import { getExplainerPlot as getExplainerPlotRequest } from "../../api/explainer"; +import { + getExplainerPlot as getExplainerPlotRequest, + createLocalExplainerStory, + getExplainers, +} from "../../api/explainer"; +import { startJobPolling } from "../../utils/jobPoller"; import { useTranslation } from "react-i18next"; import ArtifactViewer from "../shared/ArtifactViewer"; import ExplainerInstanceTable from "./ExplainerInstanceTable"; @@ -106,6 +112,60 @@ ArtifactBatch.propTypes = { siblingOffset: PropTypes.number, }; +/** + * Story trigger + result for one explained instance (local explainers only). + * `groupIndex` identifies the instance within the explainer's `stories` map. + */ +function InstanceStoryBox({ groupIndex, story, status, onGenerate }) { + const { t } = useTranslation(["explainers"]); + + return ( + + + {status === "loading" ? ( + + + + {t("explainers:label.generatingStory")} + + + ) : ( + + )} + + {/* Same box used for every other text artifact (ArtifactViewer over a + "text" artifact), so a generated story looks identical to the + caption explainers used to render automatically in plot(). */} + {story && } + {status === "error" && ( + + {t("explainers:error.storyGenerationFailed")} + + )} + + ); +} + +InstanceStoryBox.propTypes = { + groupIndex: PropTypes.number.isRequired, + story: PropTypes.string, + status: PropTypes.string, + onGenerate: PropTypes.func.isRequired, +}; + /** * Render a GroupedArtifacts item: a selector listing every group, beside the * selected group's first artifact (with the rest stacked below). Holds its own @@ -115,6 +175,10 @@ ArtifactBatch.propTypes = { * explained rows dataset path so the picker shows the actual instance feature * values (the row index selects the group); global explainers omit it and get * a plain title list. + * + * `story` (optional) wires up the per-instance "generate story" action for + * the selected group: only passed for local explainers whose explainer type + * supports it. */ function GroupedArtifactsView({ grouped, @@ -122,6 +186,7 @@ function GroupedArtifactsView({ datasetPath = null, selected: selectedProp = null, onSelect = null, + story = null, }) { const { t } = useTranslation(["explainers"]); const [localSelected, setLocalSelected] = useState(0); @@ -137,13 +202,25 @@ function GroupedArtifactsView({ ); const wide = Boolean(datasetPath); + // When the story box is shown, it takes over the space of any pre-existing + // text artifact (e.g. ContrastiveShap's plot already carries the same + // sentence as a caption) instead of showing both. Falls back to the + // unfiltered list if a group has nothing left after dropping its text + // artifact(s), so a text-only group never renders empty. + const dropCaption = Boolean(story); + const displayedArtifacts = (arts) => { + if (!dropCaption) return arts; + const withoutText = arts.filter((a) => a.type !== "text"); + return withoutText.length > 0 ? withoutText : arts; + }; + // Fullscreen navigation spans every group's artifacts (flattened), so the // viewer can page across groups even when each group has a single artifact. // The selected group's artifacts occupy the slice starting at `offset`. - const allArtifacts = groups.flatMap((g) => g.artifacts); + const allArtifacts = groups.flatMap((g) => displayedArtifacts(g.artifacts)); const offset = groups .slice(0, selected) - .reduce((n, g) => n + g.artifacts.length, 0); + .reduce((n, g) => n + displayedArtifacts(g.artifacts).length, 0); // Rendered directly (no height cap): ExplainerInstanceTable's root is // height:100%, so it fills the stretched batch cell and matches the height @@ -158,15 +235,27 @@ function GroupedArtifactsView({ ); return ( - + + + {story && ( + + )} + ); } @@ -176,6 +265,11 @@ GroupedArtifactsView.propTypes = { datasetPath: PropTypes.string, selected: PropTypes.number, onSelect: PropTypes.func, + story: PropTypes.shape({ + getStory: PropTypes.func.isRequired, + getStatus: PropTypes.func.isRequired, + onGenerate: PropTypes.func.isRequired, + }), }; /** @@ -184,7 +278,13 @@ GroupedArtifactsView.propTypes = { * width). `datasetPath` is forwarded to grouped items so local explainers get * the dataset row picker. */ -function renderItem(item, ctx, datasetPath = null, selection = null) { +function renderItem( + item, + ctx, + datasetPath = null, + selection = null, + story = null, +) { if (item.type === "grouped") { return ( ); } @@ -202,6 +303,7 @@ function renderItem(item, ctx, datasetPath = null, selection = null) { export default function ExplainersPlot({ explainer, scope, + supportsStory = false, onSaveOverride = null, onResetOverride = null, overriddenIndexes = [], @@ -216,6 +318,61 @@ export default function ExplainersPlot({ const isLocal = scope === "local"; const datasetPath = isLocal ? explainer.input_dataset_path : null; + // Per-instance story text/status, keyed by group index (as a string, to + // match how `stories` is keyed on the explainer). Overrides + // explainer.stories once (re)generated, since the parent list may not + // refetch immediately. + const [localStories, setLocalStories] = useState({}); + const [storyStatus, setStoryStatus] = useState({}); + + const handleGenerateStory = async (groupIndex) => { + const key = String(groupIndex); + setStoryStatus((prev) => ({ ...prev, [key]: "loading" })); + try { + const { id: jobId } = await createLocalExplainerStory( + explainer.id, + groupIndex, + ); + startJobPolling( + jobId, + async () => { + try { + const refreshed = await getExplainers(explainer.run_id, "local"); + const updated = refreshed.find((e) => e.id === explainer.id); + setLocalStories((prev) => ({ + ...prev, + [key]: updated?.stories?.[key] ?? null, + })); + setStoryStatus((prev) => { + const next = { ...prev }; + delete next[key]; + return next; + }); + } catch { + setStoryStatus((prev) => ({ ...prev, [key]: "error" })); + } + }, + () => setStoryStatus((prev) => ({ ...prev, [key]: "error" })), + ); + } catch { + setStoryStatus((prev) => ({ ...prev, [key]: "error" })); + } + }; + + const storyProps = + isLocal && supportsStory + ? { + getStory: (groupIndex) => { + const key = String(groupIndex); + return key in localStories + ? localStories[key] + : (explainer.stories?.[key] ?? null); + }, + getStatus: (groupIndex) => storyStatus[String(groupIndex)], + onGenerate: handleGenerateStory, + } + : null; + const getExplainerPlot = async () => { setLoading(true); try { @@ -280,18 +437,26 @@ export default function ExplainersPlot({ > {items.map((item, i) => ( - {renderItem(item, ctx, datasetPath, { - selected: cacheEntry ? (cacheEntry.selectedGroups?.[i] ?? 0) : null, - onSelect: onCacheUpdate - ? (value) => - onCacheUpdate({ - selectedGroups: { - ...(cacheEntry?.selectedGroups ?? {}), - [i]: value, - }, - }) - : null, - })} + {renderItem( + item, + ctx, + datasetPath, + { + selected: cacheEntry + ? (cacheEntry.selectedGroups?.[i] ?? 0) + : null, + onSelect: onCacheUpdate + ? (value) => + onCacheUpdate({ + selectedGroups: { + ...(cacheEntry?.selectedGroups ?? {}), + [i]: value, + }, + }) + : null, + }, + storyProps, + )} ))} @@ -301,10 +466,13 @@ export default function ExplainersPlot({ ExplainersPlot.propTypes = { explainer: PropTypes.shape({ id: PropTypes.number, + run_id: PropTypes.number, status: PropTypes.number, input_dataset_path: PropTypes.string, + stories: PropTypes.object, }).isRequired, scope: PropTypes.string.isRequired, + supportsStory: PropTypes.bool, onSaveOverride: PropTypes.func, onResetOverride: PropTypes.func, overriddenIndexes: PropTypes.arrayOf(PropTypes.number), diff --git a/DashAI/front/src/types/explainer.ts b/DashAI/front/src/types/explainer.ts index 2cdd570e5..4c5f1d2e9 100644 --- a/DashAI/front/src/types/explainer.ts +++ b/DashAI/front/src/types/explainer.ts @@ -12,4 +12,5 @@ export interface IExplainer { status: number; story?: string | null; story_huey_id?: string | null; + stories?: Record | null; } From 58eec3869400ea782e451cb914c0c9ca18573001 Mon Sep 17 00:00:00 2001 From: Creylay Date: Tue, 4 Aug 2026 16:33:43 -0400 Subject: [PATCH 09/11] feat: remove unused AutoAwesomeIcon imports from ExplainersCard and ExplainersPlot components --- DashAI/front/src/components/explainers/ExplainersCard.jsx | 2 -- DashAI/front/src/components/explainers/ExplainersPlot.jsx | 2 -- 2 files changed, 4 deletions(-) diff --git a/DashAI/front/src/components/explainers/ExplainersCard.jsx b/DashAI/front/src/components/explainers/ExplainersCard.jsx index 14384673c..3936ee0d4 100644 --- a/DashAI/front/src/components/explainers/ExplainersCard.jsx +++ b/DashAI/front/src/components/explainers/ExplainersCard.jsx @@ -13,7 +13,6 @@ import DeleteConfirmationModal from "../threeSectionLayout/DeleteConfirmationMod import RunStatusDot from "../shared/RunStatusDot"; import DeleteIcon from "@mui/icons-material/Delete"; import ZoomInIcon from "@mui/icons-material/ZoomIn"; -import AutoAwesomeIcon from "@mui/icons-material/AutoAwesome"; import PropTypes from "prop-types"; import ExplainersPlot from "./ExplainersPlot"; import ArtifactViewer from "../shared/ArtifactViewer"; @@ -237,7 +236,6 @@ export default function ExplainersCard({ - )} - - {storyState.story && ( - - )} - {storyState.status === "error" && ( - - {t("explainers:error.storyGenerationFailed")} - - )} - + {scope === "global" && supportsStory && explainer.story && ( + + + )} )} diff --git a/DashAI/front/src/components/explainers/ExplainersPlot.jsx b/DashAI/front/src/components/explainers/ExplainersPlot.jsx index b350f8e02..067337a80 100644 --- a/DashAI/front/src/components/explainers/ExplainersPlot.jsx +++ b/DashAI/front/src/components/explainers/ExplainersPlot.jsx @@ -1,14 +1,9 @@ import { React, useEffect, useState } from "react"; -import { CircularProgress, Box, Button, Typography } from "@mui/material"; +import { CircularProgress, Box } from "@mui/material"; import PropTypes from "prop-types"; import { useSnackbar } from "notistack"; -import { - getExplainerPlot as getExplainerPlotRequest, - createLocalExplainerStory, - getExplainers, -} from "../../api/explainer"; -import { startJobPolling } from "../../utils/jobPoller"; +import { getExplainerPlot as getExplainerPlotRequest } from "../../api/explainer"; import { useTranslation } from "react-i18next"; import ArtifactViewer from "../shared/ArtifactViewer"; import ExplainerInstanceTable from "./ExplainerInstanceTable"; @@ -112,56 +107,21 @@ ArtifactBatch.propTypes = { }; /** - * Story trigger + result for one explained instance (local explainers only). - * `groupIndex` identifies the instance within the explainer's `stories` map. + * Story for one explained instance (local explainers only). Generated + * automatically as part of the explainer job - this only displays it, the + * same box used for every other text artifact. */ -function InstanceStoryBox({ groupIndex, story, status, onGenerate }) { - const { t } = useTranslation(["explainers"]); - +function InstanceStoryBox({ story }) { + if (!story) return null; return ( - - {status === "loading" ? ( - - - - {t("explainers:label.generatingStory")} - - - ) : ( - - )} - - {/* Same box used for every other text artifact (ArtifactViewer over a - "text" artifact), so a generated story looks identical to the - caption explainers used to render automatically in plot(). */} - {story && } - {status === "error" && ( - - {t("explainers:error.storyGenerationFailed")} - - )} + ); } InstanceStoryBox.propTypes = { - groupIndex: PropTypes.number.isRequired, story: PropTypes.string, - status: PropTypes.string, - onGenerate: PropTypes.func.isRequired, }; /** @@ -174,9 +134,8 @@ InstanceStoryBox.propTypes = { * values (the row index selects the group); global explainers omit it and get * a plain title list. * - * `story` (optional) wires up the per-instance "generate story" action for - * the selected group: only passed for local explainers whose explainer type - * supports it. + * `story` (optional) looks up the per-instance story text: only passed for + * local explainers whose explainer type supports it. */ function GroupedArtifactsView({ grouped, @@ -245,14 +204,7 @@ function GroupedArtifactsView({ leadingFlex={wide ? "0 0 46%" : "0 0 25%"} leadingMinWidth={wide ? 320 : 220} /> - {story && ( - - )} + {story && } ); } @@ -265,8 +217,6 @@ GroupedArtifactsView.propTypes = { onSelect: PropTypes.func, story: PropTypes.shape({ getStory: PropTypes.func.isRequired, - getStatus: PropTypes.func.isRequired, - onGenerate: PropTypes.func.isRequired, }), }; @@ -316,58 +266,11 @@ export default function ExplainersPlot({ const isLocal = scope === "local"; const datasetPath = isLocal ? explainer.input_dataset_path : null; - // Per-instance story text/status, keyed by group index (as a string, to - // match how `stories` is keyed on the explainer). Overrides - // explainer.stories once (re)generated, since the parent list may not - // refetch immediately. - const [localStories, setLocalStories] = useState({}); - const [storyStatus, setStoryStatus] = useState({}); - - const handleGenerateStory = async (groupIndex) => { - const key = String(groupIndex); - setStoryStatus((prev) => ({ ...prev, [key]: "loading" })); - try { - const { id: jobId } = await createLocalExplainerStory( - explainer.id, - groupIndex, - ); - startJobPolling( - jobId, - async () => { - try { - const refreshed = await getExplainers(explainer.run_id, "local"); - const updated = refreshed.find((e) => e.id === explainer.id); - setLocalStories((prev) => ({ - ...prev, - [key]: updated?.stories?.[key] ?? null, - })); - setStoryStatus((prev) => { - const next = { ...prev }; - delete next[key]; - return next; - }); - } catch { - setStoryStatus((prev) => ({ ...prev, [key]: "error" })); - } - }, - () => setStoryStatus((prev) => ({ ...prev, [key]: "error" })), - ); - } catch { - setStoryStatus((prev) => ({ ...prev, [key]: "error" })); - } - }; - const storyProps = isLocal && supportsStory ? { - getStory: (groupIndex) => { - const key = String(groupIndex); - return key in localStories - ? localStories[key] - : (explainer.stories?.[key] ?? null); - }, - getStatus: (groupIndex) => storyStatus[String(groupIndex)], - onGenerate: handleGenerateStory, + getStory: (groupIndex) => + explainer.stories?.[String(groupIndex)] ?? null, } : null; diff --git a/DashAI/front/src/utils/i18n/locales/de/explainers.json b/DashAI/front/src/utils/i18n/locales/de/explainers.json index a4dd47628..85b9c49b0 100644 --- a/DashAI/front/src/utils/i18n/locales/de/explainers.json +++ b/DashAI/front/src/utils/i18n/locales/de/explainers.json @@ -29,8 +29,7 @@ "nameTooShort": "Der Name des Erklärungsmodells muss mindestens 4 alphanumerische Zeichen enthalten.", "validateDataset": "Fehler beim Validieren des ausgewählten Datensatzes.", "noData": "Keine Daten verfügbar", - "nameAlreadyExists": "Name existiert bereits", - "storyGenerationFailed": "Die Story konnte nicht erstellt werden." + "nameAlreadyExists": "Name existiert bereits" }, "label": { "configureExplainer": "Erklärungsmodell konfigurieren", @@ -78,10 +77,7 @@ "rowModePercentage": "Prozent-Schieberegler", "rowModeManual": "Manuelle Auswahl", "shuffleRows": "Ausgewählte Zeilen mischen (Zufallsstichprobe)", - "rowsSelectedManually": "Manuell ausgewählte Zeilen: {{selected}} / {{total}}", - "generateStory": "Story generieren", - "regenerateStory": "Story neu generieren", - "generatingStory": "Story wird generiert..." + "rowsSelectedManually": "Manuell ausgewählte Zeilen: {{selected}} / {{total}}" }, "message": { "explainerJobCompleted": "Erklärungsmodell {{name}} erfolgreich abgeschlossen", diff --git a/DashAI/front/src/utils/i18n/locales/en/explainers.json b/DashAI/front/src/utils/i18n/locales/en/explainers.json index 6da8ff251..00ada09bb 100644 --- a/DashAI/front/src/utils/i18n/locales/en/explainers.json +++ b/DashAI/front/src/utils/i18n/locales/en/explainers.json @@ -29,8 +29,7 @@ "nameTooShort": "The explainer name must have at least 4 alphanumeric characters.", "validateDataset": "Error while trying to validate the selected dataset.", "noData": "No data available", - "nameAlreadyExists": "Name already exists", - "storyGenerationFailed": "Failed to generate the story." + "nameAlreadyExists": "Name already exists" }, "label": { "configureExplainer": "Configure your Explainer", @@ -78,10 +77,7 @@ "rowModePercentage": "Percentage slider", "rowModeManual": "Manual selection", "shuffleRows": "Shuffle selected rows (random sample)", - "rowsSelectedManually": "Rows selected manually: {{selected}} / {{total}}", - "generateStory": "Generate story", - "regenerateStory": "Regenerate story", - "generatingStory": "Generating story..." + "rowsSelectedManually": "Rows selected manually: {{selected}} / {{total}}" }, "message": { "explainerJobCompleted": "Explainer {{name}} completed successfully", diff --git a/DashAI/front/src/utils/i18n/locales/es/explainers.json b/DashAI/front/src/utils/i18n/locales/es/explainers.json index 5442d17bf..47154b54f 100644 --- a/DashAI/front/src/utils/i18n/locales/es/explainers.json +++ b/DashAI/front/src/utils/i18n/locales/es/explainers.json @@ -29,8 +29,7 @@ "nameTooShort": "El nombre del explicador debe tener al menos 4 caracteres alfanuméricos.", "validateDataset": "Error al intentar validar el dataset seleccionado.", "noData": "No hay datos disponibles", - "nameAlreadyExists": "El nombre ya existe", - "storyGenerationFailed": "No se pudo generar la historia." + "nameAlreadyExists": "El nombre ya existe" }, "label": { "configureExplainer": "Configure su Explicador", @@ -78,10 +77,7 @@ "rowModePercentage": "Control deslizante de porcentaje", "rowModeManual": "Selección manual", "shuffleRows": "Mezclar filas seleccionadas (muestra aleatoria)", - "rowsSelectedManually": "Filas seleccionadas manualmente: {{selected}} / {{total}}", - "generateStory": "Generar historia", - "regenerateStory": "Regenerar historia", - "generatingStory": "Generando historia..." + "rowsSelectedManually": "Filas seleccionadas manualmente: {{selected}} / {{total}}" }, "message": { "explainerJobCompleted": "Explicador {{name}} completado exitosamente", diff --git a/DashAI/front/src/utils/i18n/locales/pt/explainers.json b/DashAI/front/src/utils/i18n/locales/pt/explainers.json index 4dfa73ac8..9c8a1c44c 100644 --- a/DashAI/front/src/utils/i18n/locales/pt/explainers.json +++ b/DashAI/front/src/utils/i18n/locales/pt/explainers.json @@ -29,8 +29,7 @@ "nameTooShort": "O nome do explicador deve ter pelo menos 4 caracteres alfanuméricos.", "validateDataset": "Erro ao tentar validar o conjunto de dados selecionado.", "noData": "Sem dados disponíveis", - "nameAlreadyExists": "O nome já existe", - "storyGenerationFailed": "Falha ao gerar a história." + "nameAlreadyExists": "O nome já existe" }, "label": { "configureExplainer": "Configure seu Explicador", @@ -78,10 +77,7 @@ "rowModePercentage": "Controle deslizante de porcentagem", "rowModeManual": "Seleção manual", "shuffleRows": "Embaralhar linhas selecionadas (amostra aleatória)", - "rowsSelectedManually": "Linhas selecionadas manualmente: {{selected}} / {{total}}", - "generateStory": "Gerar história", - "regenerateStory": "Regenerar história", - "generatingStory": "Gerando história..." + "rowsSelectedManually": "Linhas selecionadas manualmente: {{selected}} / {{total}}" }, "message": { "explainerJobCompleted": "Explicador {{name}} concluído com sucesso", diff --git a/DashAI/front/src/utils/i18n/locales/zh/explainers.json b/DashAI/front/src/utils/i18n/locales/zh/explainers.json index f789bae85..8939cd457 100644 --- a/DashAI/front/src/utils/i18n/locales/zh/explainers.json +++ b/DashAI/front/src/utils/i18n/locales/zh/explainers.json @@ -29,8 +29,7 @@ "nameTooShort": "解释器名称至少需要 4 个字母或数字字符。", "validateDataset": "验证所选数据集时出错。", "noData": "暂无数据", - "nameAlreadyExists": "名称已存在", - "storyGenerationFailed": "生成故事失败。" + "nameAlreadyExists": "名称已存在" }, "label": { "configureExplainer": "配置解释器", @@ -78,10 +77,7 @@ "rowModePercentage": "百分比滑块", "rowModeManual": "手动选择", "shuffleRows": "打乱选中的行（随机抽样）", - "rowsSelectedManually": "手动选择的行数：{{selected}} / {{total}}", - "generateStory": "生成故事", - "regenerateStory": "重新生成故事", - "generatingStory": "正在生成故事..." + "rowsSelectedManually": "手动选择的行数：{{selected}} / {{total}}" }, "message": { "explainerJobCompleted": "解释器 {{name}} 成功完成", diff --git a/tests/back/api/test_explainer_story_job.py b/tests/back/api/test_explainer_story_job.py index 581a86409..8ac350671 100644 --- a/tests/back/api/test_explainer_story_job.py +++ b/tests/back/api/test_explainer_story_job.py @@ -17,7 +17,6 @@ from DashAI.back.explainability.global_explainer import BaseGlobalExplainer from DashAI.back.explainability.local_explainer import BaseLocalExplainer from DashAI.back.job.explainer_job import ExplainerJob -from DashAI.back.job.explainer_story_job import ExplainerStoryJob from DashAI.back.models.base_model import BaseModel from DashAI.back.tasks.base_task import BaseTask @@ -97,7 +96,8 @@ def get_schema(cls): return {} def explain(self, dataset): - return {"headline": "feature X matters most"} + self.explanation = {"headline": "feature X matters most"} + return self.explanation def plot(self, explanation): return [TextArtifact(payload="a plot summary")] @@ -127,6 +127,32 @@ def plot(self, explanation): return [TextArtifact(payload="a plot summary")] +class BrokenStoryGlobalExplainer(BaseGlobalExplainer): + """Global explainer whose story() always raises. + + Used to prove a story bug never fails the explanation itself. + """ + + COMPATIBLE_COMPONENTS = ["DummyTask"] + + def __init__(self, model: BaseModel) -> None: + self.model = model + self.explanation = None + + @classmethod + def get_schema(cls): + return {} + + def explain(self, dataset): + return {"headline": "irrelevant"} + + def plot(self, explanation): + return [TextArtifact(payload="a plot summary")] + + def story(self, explainer_output): + raise RuntimeError("boom") + + class StoryableLocalExplainer(BaseLocalExplainer): """Local explainer with a deterministic, testable story() per instance.""" @@ -186,6 +212,45 @@ def plot(self, explanation): return [TextArtifact(payload="a plot summary")] +class BrokenStoryLocalExplainer(BaseLocalExplainer): + """Local explainer whose story() always raises for every instance. + + Used to prove a story bug never fails the explanation itself, and does + not block other instances (there are none left to block here, but the + explanation must still finish successfully with stories left empty). + """ + + COMPATIBLE_COMPONENTS = ["DummyTask"] + + def __init__(self, model: BaseModel) -> None: + self.model = model + self.explanation = None + + @classmethod + def get_schema(cls): + return {} + + def fit(self, dataset, **kwargs): + return self + + def explain_instance(self, instances): + from DashAI.back.dataloaders.classes.dashai_dataset import to_dashai_dataset + + return {"n_instances": to_dashai_dataset(instances).num_rows} + + def plot(self, explanation): + groups = [ + ArtifactGroup( + title=f"Instance {i}", artifacts=[TextArtifact(payload=f"plot {i}")] + ) + for i in range(explanation["n_instances"]) + ] + return [GroupedArtifacts(groups=groups)] + + def story(self, explainer_output, prediction_context): + raise RuntimeError("boom") + + @pytest.fixture(autouse=True, name="test_registry") def setup_test_registry(client, monkeypatch: pytest.MonkeyPatch): container = client.app.container @@ -196,10 +261,11 @@ def setup_test_registry(client, monkeypatch: pytest.MonkeyPatch): DummyModel, StoryableGlobalExplainer, StorylessGlobalExplainer, + BrokenStoryGlobalExplainer, StoryableLocalExplainer, StorylessLocalExplainer, + BrokenStoryLocalExplainer, ExplainerJob, - ExplainerStoryJob, ] ) @@ -271,269 +337,198 @@ def create_run_id(client: TestClient, model_session_id: int): db.close() -@pytest.fixture(scope="module", name="global_explainer_id") -def create_global_explainer(client: TestClient, run_id: int): +def _run_explainer_job(client: TestClient, explainer_id: int, scope: str): + response = client.post( + "/api/v1/job/", + data={ + "job_type": "ExplainerJob", + "kwargs": json.dumps( + {"explainer_id": explainer_id, "explainer_scope": scope} + ), + }, + ) + assert response.status_code == 201, response.text + + +def test_global_explainer_job_generates_story_automatically( + client: TestClient, run_id: int +): + # The story is generated as part of the normal explainer job - there is + # no separate job, endpoint or button to request it. container = client.app.container session_factory = container["session_factory"] with session_factory() as db: - global_explainer = GlobalExplainer( - name="test_story_global", + explainer = GlobalExplainer( run_id=run_id, explainer_name="StoryableGlobalExplainer", parameters={}, ) - db.add(global_explainer) + db.add(explainer) db.commit() - db.refresh(global_explainer) + db.refresh(explainer) + explainer_id = explainer.id - yield global_explainer.id + _run_explainer_job(client, explainer_id, "global") - db.delete(global_explainer) - db.commit() - db.close() + with session_factory() as db: + explainer = db.get(GlobalExplainer, explainer_id) + assert explainer.plot_path, explainer + assert explainer.story == ( + "Story based on 'a plot summary': feature X matters most" + ) -@pytest.fixture(scope="module", name="local_explainer_id") -def create_local_explainer(client: TestClient, run_id: int, dataset_id: int): +def test_global_explainer_job_leaves_story_none_when_unsupported( + client: TestClient, run_id: int +): container = client.app.container session_factory = container["session_factory"] with session_factory() as db: - local_explainer = LocalExplainer( - name="test_story_local", + explainer = GlobalExplainer( run_id=run_id, - explainer_name="StoryableLocalExplainer", - dataset_id=dataset_id, - scope={"split": "test", "percentage": 100}, + explainer_name="StorylessGlobalExplainer", parameters={}, - fit_parameters={}, ) - db.add(local_explainer) + db.add(explainer) db.commit() - db.refresh(local_explainer) - - yield local_explainer.id + db.refresh(explainer) + explainer_id = explainer.id - db.delete(local_explainer) - db.commit() - db.close() + _run_explainer_job(client, explainer_id, "global") - -def test_global_story_job(client: TestClient, global_explainer_id: int): - # First compute the explanation itself, same as any other explainer job. - response = client.post( - "/api/v1/job/", - data={ - "job_type": "ExplainerJob", - "kwargs": json.dumps( - { - "explainer_id": global_explainer_id, - "explainer_scope": "global", - } - ), - }, - ) - assert response.status_code == 201, response.text - - response = client.get(f"/api/v1/explainer/global/?run_id={global_explainer_id}") - # Sanity: the explanation finished (plot_path set) before generating a story. - explainers = response.json() - assert explainers[0]["plot_path"], explainers - - # Now request the story for the already-computed explanation, through the - # dedicated endpoint (not the generic /job/ POST). - response = client.post( - f"/api/v1/explainer/global/{global_explainer_id}/story", - ) - assert response.status_code == 201, response.text - story_job_id = response.json()["id"] - - response = client.get(f"/api/v1/job/status/{story_job_id}") - assert response.status_code == 200, response.text - job_status = response.json() - assert job_status["status"] == "finished", job_status - - container = client.app.container - session_factory = container["session_factory"] with session_factory() as db: - explainer = db.get(GlobalExplainer, global_explainer_id) - assert explainer.story_huey_id == story_job_id - assert explainer.story == ( - "Story based on 'a plot summary': feature X matters most" - ) + explainer = db.get(GlobalExplainer, explainer_id) + assert explainer.plot_path, explainer + assert explainer.story is None -def test_global_story_endpoint_requires_finished_explanation( - client: TestClient, run_id: int -): - # A fresh explainer that was never run still has status NOT_STARTED. +def test_global_explainer_job_survives_a_broken_story(client: TestClient, run_id: int): + # A bug in story() must never fail the explanation itself. container = client.app.container session_factory = container["session_factory"] + with session_factory() as db: - unstarted_explainer = GlobalExplainer( + explainer = GlobalExplainer( run_id=run_id, - explainer_name="StoryableGlobalExplainer", + explainer_name="BrokenStoryGlobalExplainer", parameters={}, ) - db.add(unstarted_explainer) + db.add(explainer) db.commit() - db.refresh(unstarted_explainer) - unstarted_id = unstarted_explainer.id + db.refresh(explainer) + explainer_id = explainer.id - response = client.post(f"/api/v1/explainer/global/{unstarted_id}/story") - assert response.status_code == 404, response.text + _run_explainer_job(client, explainer_id, "global") - -def test_global_story_endpoint_requires_existing_explainer(client: TestClient): - response = client.post("/api/v1/explainer/global/999999/story") - assert response.status_code == 404, response.text + with session_factory() as db: + explainer = db.get(GlobalExplainer, explainer_id) + assert explainer.plot_path, explainer + assert explainer.story is None + assert explainer.status.name == "FINISHED" -def test_global_story_endpoint_rejects_explainer_without_story( - client: TestClient, run_id: int -): +@pytest.fixture(scope="module", name="local_explainer_id") +def create_local_explainer(client: TestClient, run_id: int, dataset_id: int): container = client.app.container session_factory = container["session_factory"] with session_factory() as db: - storyless_explainer = GlobalExplainer( + local_explainer = LocalExplainer( + name="test_story_local", run_id=run_id, - explainer_name="StorylessGlobalExplainer", + explainer_name="StoryableLocalExplainer", + dataset_id=dataset_id, + scope={"split": "test", "percentage": 100}, parameters={}, + fit_parameters={}, ) - db.add(storyless_explainer) + db.add(local_explainer) db.commit() - db.refresh(storyless_explainer) - storyless_id = storyless_explainer.id + db.refresh(local_explainer) - response = client.post( - "/api/v1/job/", - data={ - "job_type": "ExplainerJob", - "kwargs": json.dumps( - { - "explainer_id": storyless_id, - "explainer_scope": "global", - } - ), - }, - ) - assert response.status_code == 201, response.text + yield local_explainer.id - response = client.post(f"/api/v1/explainer/global/{storyless_id}/story") - assert response.status_code == 400, response.text + db.delete(local_explainer) + db.commit() + db.close() -def test_local_story_job(client: TestClient, local_explainer_id: int): - # Compute the local explanation first, same as any other local explainer. - response = client.post( - "/api/v1/job/", - data={ - "job_type": "ExplainerJob", - "kwargs": json.dumps( - { - "explainer_id": local_explainer_id, - "explainer_scope": "local", - } - ), - }, - ) - assert response.status_code == 201, response.text +def test_local_explainer_job_generates_stories_automatically( + client: TestClient, local_explainer_id: int +): + # One story per explained instance, all generated automatically as part + # of the normal explainer job - no separate job/endpoint/button. + _run_explainer_job(client, local_explainer_id, "local") container = client.app.container session_factory = container["session_factory"] with session_factory() as db: explainer = db.get(LocalExplainer, local_explainer_id) assert explainer.plots_path, explainer - assert explainer.stories is None - - # Now request the story for instance 0 through the dedicated endpoint. - response = client.post( - f"/api/v1/explainer/local/{local_explainer_id}/story", - json={"group_index": 0}, - ) - assert response.status_code == 201, response.text - story_job_id = response.json()["id"] - - response = client.get(f"/api/v1/job/status/{story_job_id}") - assert response.status_code == 200, response.text - assert response.json()["status"] == "finished", response.json() - - with session_factory() as db: - explainer = db.get(LocalExplainer, local_explainer_id) - assert explainer.stories == {"0": "Local story for 'plot 0', context rows=1"} + # The "test" split has 4 rows (indexes 5-8) at 100%. + assert explainer.stories == { + "0": "Local story for 'plot 0', context rows=1", + "1": "Local story for 'plot 1', context rows=1", + "2": "Local story for 'plot 2', context rows=1", + "3": "Local story for 'plot 3', context rows=1", + } -def test_local_story_endpoint_requires_finished_explanation( +def test_local_explainer_job_leaves_stories_none_when_unsupported( client: TestClient, run_id: int, dataset_id: int ): container = client.app.container session_factory = container["session_factory"] + with session_factory() as db: - unstarted_explainer = LocalExplainer( + explainer = LocalExplainer( run_id=run_id, - explainer_name="StoryableLocalExplainer", + explainer_name="StorylessLocalExplainer", dataset_id=dataset_id, scope={"split": "test", "percentage": 100}, parameters={}, fit_parameters={}, ) - db.add(unstarted_explainer) + db.add(explainer) db.commit() - db.refresh(unstarted_explainer) - unstarted_id = unstarted_explainer.id - - response = client.post( - f"/api/v1/explainer/local/{unstarted_id}/story", - json={"group_index": 0}, - ) - assert response.status_code == 404, response.text + db.refresh(explainer) + explainer_id = explainer.id + _run_explainer_job(client, explainer_id, "local") -def test_local_story_endpoint_requires_existing_explainer(client: TestClient): - response = client.post( - "/api/v1/explainer/local/999999/story", json={"group_index": 0} - ) - assert response.status_code == 404, response.text + with session_factory() as db: + explainer = db.get(LocalExplainer, explainer_id) + assert explainer.plots_path, explainer + assert explainer.stories is None -def test_local_story_endpoint_rejects_explainer_without_story( +def test_local_explainer_job_survives_a_broken_story( client: TestClient, run_id: int, dataset_id: int ): + # A bug in one instance's story() must never fail the explanation. container = client.app.container session_factory = container["session_factory"] with session_factory() as db: - storyless_explainer = LocalExplainer( + explainer = LocalExplainer( run_id=run_id, - explainer_name="StorylessLocalExplainer", + explainer_name="BrokenStoryLocalExplainer", dataset_id=dataset_id, scope={"split": "test", "percentage": 100}, parameters={}, fit_parameters={}, ) - db.add(storyless_explainer) + db.add(explainer) db.commit() - db.refresh(storyless_explainer) - storyless_id = storyless_explainer.id + db.refresh(explainer) + explainer_id = explainer.id - response = client.post( - "/api/v1/job/", - data={ - "job_type": "ExplainerJob", - "kwargs": json.dumps( - { - "explainer_id": storyless_id, - "explainer_scope": "local", - } - ), - }, - ) - assert response.status_code == 201, response.text + _run_explainer_job(client, explainer_id, "local") - response = client.post( - f"/api/v1/explainer/local/{storyless_id}/story", - json={"group_index": 0}, - ) - assert response.status_code == 400, response.text + with session_factory() as db: + explainer = db.get(LocalExplainer, explainer_id) + assert explainer.plots_path, explainer + assert explainer.stories is None + assert explainer.status.name == "FINISHED"