Interactive UI + adaptive learning + best-iter composite scoring

.gitignore

@@ -28,12 +28,21 @@ __pycache__/
 *.docx
 *.zip
 *.jpg
+*.png
 *.html
+!ui/templates/*.html
 ~$*
 
 # Pipeline outputs (regenerated by run_pipeline.py)
 outputs/
 data/processed/
 
+# Re-uploaded datasets (each ~350 MB; data/raw/ has the canonical copy)
+data/uploads/
+
+# Demo recordings (generated locally)
+recordings/
+*.webm
+
 # Project docs not for repo
 CLAUDE.md

agents/clusterer.py

@@ -212,6 +212,7 @@ def run(
         iteration: int = 1,
         config_override: dict | None = None,
         min_silhouette: float | None = None,
+        silhouette_target: float | None = None,
     ) -> ClusteringResult:
         """
         Parameters
@@ -230,6 +231,12 @@ def run(
         # Merge per-iteration config overrides over base config
         cfg = {**self.config, **(config_override or {})}
         _min_sil = min_silhouette if min_silhouette is not None else 0.05
+        # silhouette_target is the orchestrator-level pass bar (dynamic — starts
+        # at config.silhouette_target=0.5, relaxes -0.1 after each 3 consecutive
+        # misses). The clusterer uses it to decide success vs warning so the agent
+        # report aligns with what the orchestrator will do next.
+        _target = silhouette_target if silhouette_target is not None \
+            else float(cfg.get('silhouette_target', 0.5))
 
         print(f'\n[Clusterer] Iteration {iteration}')
         if feedback:
@@ -561,17 +568,20 @@ def run(
             "weak but usable" if sil >= 0.10 else
             "low (typical for transaction ratio features)"
         )
-        status = "success" if sil >= 0.25 else "warning"
+        # Success/warning is decided against the DYNAMIC orchestrator target,
+        # not a hardcoded threshold — so the chip in the UI matches the actual
+        # pass bar the orchestrator will enforce next.
+        status = "success" if sil >= _target else "warning"
         issues = []
-        if sil < 0.15:
+        if sil < _target:
             issues.append(
-                f"Silhouette={sil:.3f} < 0.15 — clusters overlap; interpretability may be low. "
-                "Consider reviewing feature selection or k."
+                f"Silhouette={sil:.3f} < target {_target:.2f} — orchestrator will "
+                "reselect features (or escalate after 3 consecutive misses)."
             )
         elif sil < 0.25:
             issues.append(
-                f"Silhouette={sil:.3f} < 0.25 — clusters may overlap slightly. "
-                "Consider different k or algorithm."
+                f"Silhouette={sil:.3f} < 0.25 — meets dynamic target {_target:.2f} "
+                "but clusters may still overlap; consider different k or algorithm."
             )
 
         if self.bus:
@@ -599,6 +609,7 @@ def run(
                     "k_selected": n_clusters,
                     "n_leaf_clusters": n_leaf,
                     "silhouette": round(sil, 4),
+                    "silhouette_target": round(_target, 3),
                     "k_scores": {str(k): v for k, v in k_scores.items()},
                 },
                 recommendation="proceed" if not issues else "retry",

agents/persona_namer.py

@@ -237,6 +237,24 @@ def run(
 
         prompt = build_all_clusters_prompt(profiles, lineage, tone_instr)
 
+        # ── Adaptive learning: prepend persistent user feedback ────────────
+        # Every change the user made in the interactive UI was logged to
+        # outputs/user_feedback_log.jsonl. We surface high-/medium-priority
+        # entries here so the Decision Maker honours the user's past
+        # preferences on every future run.
+        try:
+            from ui.feedback_store import build_preferences_block
+            prefs_block = build_preferences_block(
+                types=('manual_override', 'naming_hint', 'global_rule', 'merge'),
+            )
+            if prefs_block:
+                prompt = prefs_block + '\n' + prompt
+                print(f'  [PersonaNamer] Injected {prefs_block.count(chr(10))} lines of '
+                      f'user preferences from prior UI feedback.')
+        except Exception as _exc:  # noqa: BLE001
+            # Memory injection is best-effort; never block the pipeline on it
+            print(f'  [PersonaNamer] (no UI feedback memory loaded: {_exc})')
+
         # Append must-have constraint to prompt if set
         if must_have:
             must_have_str = ', '.join(f'"{t}"' for t in must_have)

agents/state.py

@@ -113,6 +113,12 @@ class PipelineState:
     selected_features: list[str] = field(default_factory=list)
     needs_feature_selection: bool = True
 
+    # Escalation rules (silhouette < target → reselect; N failures → re-engineer)
+    needs_feature_engineering: bool = False
+    consecutive_silhouette_failures: int = 0   # → re-engineer at max_reselect_failures (3)
+    silhouette_fail_for_relax: int = 0         # → relax target at max_relax_failures (5)
+    silhouette_target_override: Optional[float] = None   # dynamic, set by relax logic
+
     # Feedback strings routed to specific agents
     fs_feedback: str = ''
     cluster_feedback: str = ''
@@ -139,6 +145,11 @@ class PipelineState:
     best_silhouette_value: float = -1.0
     best_silhouette_features: list[str] = field(default_factory=list)
 
+    # Composite score of the all-time best iteration (F1 + Silhouette − VIF penalty).
+    # Recomputed every call to update_best so the per-iteration comparison is fair.
+    best_composite_score: float = float('-inf')
+    best_max_vif: float = 0.0
+
     # Dynamic tuning parameters — LLM adjusts these after each failed iteration
     # so agents are NOT locked into hardcoded thresholds.
     tuning_params: dict = field(default_factory=lambda: {
@@ -168,16 +179,74 @@ def update_best_silhouette(self, cr: ClusteringResult, selected_features: list[s
             self.best_silhouette_cluster = cr
             self.best_silhouette_features = list(selected_features)
 
+    @staticmethod
+    def composite_score(silhouette: Optional[float],
+                         cv_f1_macro: Optional[float],
+                         max_vif: Optional[float]) -> float:
+        """Decision metric for the best iteration. Higher is better.
+
+        Weighting (all three deliberately on different scales so each one matters
+        without any single metric dominating):
+          • F1 macro      → ×100  (primary signal: classifier learnability of the labels)
+          • Silhouette    → ×30   (cluster separation quality)
+          • VIF penalty   → −log10(max(1, max_vif)) × 5  (feature multicollinearity)
+
+        A perfect iteration (F1=1.0, Sil=1.0, VIF=1.0) ≈ 100 + 30 − 0 = 130.
+        A terrible iteration (F1=0.2, Sil=0.1, VIF=50) ≈ 20 + 3 − 8.5 ≈ 14.5.
+        Negative silhouette or F1 contribute 0 (not negative) — we don't want
+        a single dead-bad signal to dominate when the others are fine.
+        """
+        import math
+        f1_part  = max(0.0, float(cv_f1_macro or 0.0)) * 100.0
+        sil_part = max(0.0, float(silhouette or 0.0)) * 30.0
+        vif_part = -math.log10(max(1.0, float(max_vif or 1.0))) * 5.0
+        return f1_part + sil_part + vif_part
+
     def update_best(
         self,
         nr: NamingResult,
         cr: ClusteringResult,
         clf: Optional[ClassifierResult] = None,
-    ) -> None:
-        """Keep track of the best (highest avg_confidence + passed gate) result."""
-        if nr.passed:
-            if (self.best_naming_result is None
-                    or nr.avg_confidence > self.best_naming_result.avg_confidence):
-                self.best_naming_result = nr
-                self.best_clustering_result = cr
-                self.best_classifier_result = clf
+        max_vif: Optional[float] = None,
+    ) -> bool:
+        """Keep the iteration with the highest composite score (F1 + Sil − VIF penalty).
+
+        Unlike before, this runs for EVERY iteration that produced a Classifier
+        result — not only iterations whose PersonaNamer cleared the Clarity
+        Gate. That's because the user-facing best-iteration decision combines
+        three signals (F1↑, Silhouette↑, VIF↓), so we must score every iter
+        with a Classifier result, not gate by naming quality alone.
+
+        Returns True if this iteration is the new best (helps the caller log it).
+        """
+        if cr is None or clf is None:
+            return False
+
+        sil = cr.silhouette if cr.silhouette is not None else 0.0
+        f1  = clf.cv_f1_macro if clf.cv_f1_macro is not None else 0.0
+        vif = float(max_vif) if max_vif is not None else self.best_max_vif or 1.0
+        new_score = self.composite_score(sil, f1, vif)
+
+        if new_score > self.best_composite_score:
+            self.best_composite_score = new_score
+            self.best_naming_result = nr
+            self.best_clustering_result = cr
+            self.best_classifier_result = clf
+            self.best_max_vif = vif
+            return True
+        return False
+
+    def current_max_vif(self) -> float:
+        """Largest VIF among the currently-selected features (1.0 if unknown).
+        Sourced from the most recent FeatureSelectionResult; used as the VIF
+        component of the composite score for the current iteration."""
+        if not self.fs_history:
+            return 1.0
+        last = self.fs_history[-1]
+        vt = getattr(last, 'vif_table', None) or {}
+        if not vt:
+            return 1.0
+        try:
+            return max(float(v) for v in vt.values() if v is not None)
+        except (ValueError, TypeError):
+            return 1.0