app.py

import hashlib
import os
import sys
import time
import warnings

import joblib
import pandas as pd
import streamlit as st

warnings.filterwarnings("ignore", category=UserWarning, module="sklearn")

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))

from chatbot.config import Config
from chatbot.preprocessor import Preprocessor
from chatbot.intent_classifier import IntentClassifier
from chatbot.similarity_matcher import SimilarityMatcher

CACHE_FILE = os.path.join("data", "model_cache.joblib")

st.set_page_config(
    page_title="FAQ Microsoft Clarity — QA",
    page_icon="💬",
    layout="wide",
)


def load_faq_data(config):
    data = {}
    for lang_code in config.supported_languages:
        path = os.path.join(config.data_dir, lang_code, "faq.csv")
        if os.path.exists(path):
            df = pd.read_csv(path)
            data[lang_code] = df.to_dict("records")
        else:
            data[lang_code] = []
    return data


def strip_html(text):
    import re
    return re.sub(r"\s+", " ", re.sub(r"<[^>]+>", "", text)).strip()


def process_query(user_input, preprocessor, intent_clf, similarity, config):
    lang = preprocessor.detect_language(user_input)
    processed = preprocessor.preprocess(user_input, lang)

    fallback = "I'm sorry, I couldn't find an answer."
    if lang == "fr":
        fallback = "Désolé, je n'ai pas trouvé de réponse."
    elif lang == "es":
        fallback = "Lo siento, no pude encontrar una respuesta."

    idx, confidence, answer = intent_clf.predict(processed, lang)
    method = "intent"

    if answer is None:
        idx, confidence, answer = similarity.match(processed, lang)
        method = "similarity"

    if answer is None:
        suggestions = similarity.suggest(processed, lang)
        return fallback, 0.0, lang, method, False, suggestions

    return strip_html(answer), confidence, lang, method, True, []


def load_css():
    st.markdown("""
    <style>
    .chat-container {
        max-width: 800px;
        margin: 0 auto;
    }
    .user-msg {
        background: #e8f0fe;
        border-radius: 18px 18px 4px 18px;
        padding: 12px 18px;
        margin: 8px 0 8px auto;
        max-width: 75%;
        color: #1a1a2e;
        font-size: 15px;
        line-height: 1.5;
        border: 1px solid #d0d7de;
    }
    .bot-msg {
        background: #f6f8fa;
        border-radius: 18px 18px 18px 4px;
        padding: 12px 18px;
        margin: 8px auto 8px 0;
        max-width: 85%;
        color: #1a1a2e;
        font-size: 15px;
        line-height: 1.5;
        border: 1px solid #d0d7de;
    }
    .confidence-badge {
        display: inline-block;
        background: #2da44e;
        color: white;
        padding: 2px 10px;
        border-radius: 12px;
        font-size: 12px;
        font-weight: 600;
        margin-top: 6px;
    }
    .confidence-low {
        background: #d1242f;
    }
    .confidence-medium {
        background: #d4920b;
    }
    .method-badge {
        display: inline-block;
        background: #0969da;
        color: white;
        padding: 2px 10px;
        border-radius: 12px;
        font-size: 11px;
        font-weight: 500;
        margin-left: 6px;
    }
    .sidebar-header {
        font-size: 14px;
        font-weight: 600;
        color: #57606a;
        text-transform: uppercase;
        letter-spacing: 0.5px;
        margin-bottom: 8px;
    }
    .stat-card {
        background: #f6f8fa;
        border-radius: 8px;
        padding: 12px;
        margin-bottom: 8px;
        border: 1px solid #d0d7de;
    }
    .stat-number {
        font-size: 22px;
        font-weight: 700;
        color: #1f2328;
    }
    .stat-label {
        font-size: 12px;
        color: #656d76;
    }
    </style>
    """, unsafe_allow_html=True)


def _get_cache_key(config):
    hasher = hashlib.md5()
    for lang_code in config.supported_languages:
        path = os.path.join(config.data_dir, lang_code, "faq.csv")
        if os.path.exists(path):
            st_info = os.stat(path)
            hasher.update(f"{lang_code}:{st_info.st_mtime}:{st_info.st_size}".encode())
    return hasher.hexdigest()


def _load_cached(cache_key):
    if not os.path.exists(CACHE_FILE):
        print("[CACHE] No cache file found")
        return None
    try:
        data = joblib.load(CACHE_FILE)
        if data.get("cache_key") == cache_key:
            size = os.path.getsize(CACHE_FILE)
            print(f"[CACHE] Cache hit — {len(data.get('faq_data',{}))} languages loaded ({size/1024:.0f} KB)")
            return (
                data["faq_data"],
                data["intent_pipelines"],
                data["similarity_vectorizers"],
                data["similarity_faq_data"],
            )
        print("[CACHE] Cache invalid (CSV files changed)")
    except Exception as e:
        print(f"[CACHE] Read error: {e}")
    return None


def _save_cache(cache_key, faq_data, intent_clf, similarity):
    os.makedirs(os.path.dirname(CACHE_FILE) or ".", exist_ok=True)
    data = {
        "cache_key": cache_key,
        "faq_data": faq_data,
        "intent_pipelines": intent_clf.pipelines,
        "similarity_vectorizers": similarity.vectorizers,
        "similarity_faq_data": similarity.faq_data,
    }
    joblib.dump(data, CACHE_FILE)
    size = os.path.getsize(CACHE_FILE)
    print(f"[CACHE] Models saved ({size/1024:.0f} KB)")


def main():
    load_css()
    config = Config()

    st.markdown("""
    <div style="display:flex;align-items:center;gap:14px;margin-bottom:4px;background:#1f2328;padding:16px 20px;border-radius:12px">
        <span style="font-size:2.4rem;line-height:1">💬</span>
        <div>
            <h1 style="margin:0;font-size:1.6rem;font-weight:700;color:#ffffff">FAQ Microsoft Clarity — Question Answering</h1>
            <p style="margin:0;color:#8b949e;font-size:0.85rem">
                Intent matching + Semantic similarity · EN · FR · ES
            </p>
        </div>
    </div>
    """, unsafe_allow_html=True)

    if "messages" not in st.session_state:
        st.session_state.messages = []
    if "models_trained" not in st.session_state:
        st.session_state.models_trained = False

    with st.sidebar:
        st.markdown('<div class="sidebar-header">⚙️ Configuration</div>', unsafe_allow_html=True)

        intent_th = st.slider(
            "Intent Threshold", 0.0, 1.0, config.intent_threshold, 0.05,
            help="If classifier confidence exceeds this threshold, its answer is used"
        )
        sim_th = st.slider(
            "Similarity Threshold", 0.0, 1.0, config.similarity_threshold, 0.05,
            help="If cosine similarity exceeds this threshold, the best FAQ match is returned"
        )
        config.intent_threshold = intent_th
        config.similarity_threshold = sim_th

        st.markdown('<div class="sidebar-header" style="margin-top:20px">📡 Data</div>', unsafe_allow_html=True)

        if st.button("🔄 Reload FAQs", use_container_width=True):
            if os.path.exists(CACHE_FILE):
                os.remove(CACHE_FILE)
                print("[CACHE] Cache deleted (reload requested)")
            st.session_state.models_trained = False
            st.rerun()

        st.markdown('<div class="sidebar-header" style="margin-top:20px">📊 Statistics</div>', unsafe_allow_html=True)

        if "faq_data" in st.session_state:
            for lang, qas in st.session_state.faq_data.items():
                lang_name = config.supported_languages.get(lang, {}).get("name", lang)
                count = len(qas)
                st.markdown(
                    f'<div class="stat-card">'
                    f'<div class="stat-number">{count}</div>'
                    f'<div class="stat-label">FAQs • {lang_name}</div>'
                    f'</div>',
                    unsafe_allow_html=True
                )

        if st.button("🗑️ Clear conversation", use_container_width=True, type="secondary"):
            st.session_state.messages = []
            st.rerun()

    if not st.session_state.models_trained:
        print("[INIT] Starting initialization...")
        loading_placeholder = st.empty()
        progress_bar = st.progress(0)

        cache_key = _get_cache_key(config)
        cached = _load_cached(cache_key)

        if cached:
            faq_data, intent_pipelines, sim_vectorizers, sim_faq_data = cached

            with loading_placeholder.container():
                st.markdown("""
                <div style="text-align:center;padding:3rem 0">
                    <div style="font-size:3rem;margin-bottom:1rem">💾</div>
                    <h3>Restoring models...</h3>
                    <p style="color:#656d76">Loading from disk cache</p>
                </div>
                """, unsafe_allow_html=True)
            progress_bar.progress(30)

            st.session_state.faq_data = faq_data
            preprocessor = Preprocessor(config)

            intent_clf = IntentClassifier(config)
            intent_clf.pipelines = intent_pipelines
            similarity = SimilarityMatcher(config)
            similarity.vectorizers = sim_vectorizers
            similarity.faq_data = sim_faq_data

            progress_bar.progress(80)
            st.session_state.preprocessor = preprocessor
            st.session_state.intent_clf = intent_clf
            st.session_state.similarity = similarity
            st.session_state.models_trained = True
            print("[INIT] Models restored from cache")
        else:
            with loading_placeholder.container():
                st.markdown("""
                <div style="text-align:center;padding:3rem 0">
                    <div style="font-size:3rem;margin-bottom:1rem">⏳</div>
                    <h3>Initializing chatbot...</h3>
                    <p style="color:#656d76">Loading FAQ data</p>
                </div>
                """, unsafe_allow_html=True)
            progress_bar.progress(10)

            print("[DATA] Loading CSV files...")
            faq_data = load_faq_data(config)
            for lang_code, qa_list in faq_data.items():
                print(f"  └─ {lang_code}: {len(qa_list)} FAQs")
            progress_bar.progress(30)

            if not any(faq_data.values()):
                progress_bar.empty()
                loading_placeholder.empty()
                print("[ERROR] No FAQ loaded")
                st.error("No FAQ loaded. Check the CSV files in data/.")
                st.stop()

            st.session_state.faq_data = faq_data

            with loading_placeholder.container():
                st.markdown("""
                <div style="text-align:center;padding:3rem 0">
                    <div style="font-size:3rem;margin-bottom:1rem">🧠</div>
                    <h3>Training models...</h3>
                    <p style="color:#656d76">Preprocessing and vectorizing questions</p>
                </div>
                """, unsafe_allow_html=True)
            progress_bar.progress(40)

            preprocessor = Preprocessor(config)
            intent_clf = IntentClassifier(config)
            similarity = SimilarityMatcher(config)

            lang_list = [lc for lc, qa in faq_data.items() if qa]
            for idx, lang_code in enumerate(lang_list):
                qa_list = faq_data[lang_code]
                questions = [item["question"] for item in qa_list]
                answers = [item["answer"] for item in qa_list]
                t0 = time.time()
                processed_q = [preprocessor.preprocess(q, lang_code) for q in questions]
                intent_clf.train(processed_q, answers, lang_code)
                similarity.fit(processed_q, answers, lang_code, raw_questions=questions)
                elapsed = time.time() - t0
                n_questions = len(questions)
                print(f"[TRAIN] {lang_code}: {n_questions} questions → {n_questions} classes ({elapsed:.1f}s)")
                progress_bar.progress(50 + int((idx + 1) / len(lang_list) * 40))

            progress_bar.progress(95)
            with loading_placeholder.container():
                st.markdown("""
                <div style="text-align:center;padding:3rem 0">
                    <div style="font-size:3rem;margin-bottom:1rem">💾</div>
                    <h3>Saving cache...</h3>
                    <p style="color:#656d76">Writing to disk</p>
                </div>
                """, unsafe_allow_html=True)

            _save_cache(cache_key, faq_data, intent_clf, similarity)
            st.session_state.preprocessor = preprocessor
            st.session_state.intent_clf = intent_clf
            st.session_state.similarity = similarity
            st.session_state.models_trained = True
            print("[INIT] Initialization complete")

        progress_bar.progress(100)
        with loading_placeholder.container():
            st.markdown("""
            <div style="text-align:center;padding:3rem 0">
                <div style="font-size:3rem;margin-bottom:1rem">✅</div>
                <h3 style="color:#1f2328">Ready!</h3>
                <p style="color:#656d76">You can now ask your questions</p>
            </div>
            """, unsafe_allow_html=True)

        time.sleep(0.6)
        loading_placeholder.empty()
        progress_bar.empty()

    preprocessor = st.session_state.preprocessor
    intent_clf = st.session_state.intent_clf
    similarity = st.session_state.similarity

    if not st.session_state.messages:
        with st.chat_message("assistant"):
            st.markdown("""👋 **Welcome to the Microsoft Clarity FAQ!**  
Ask a question in English, French or Spanish, or click a suggestion below to get started.""")

        faq_data = st.session_state.faq_data
        cols = st.columns(4)
        en_qs = [q["question"] for q in faq_data.get("en", [])]
        fr_qs = [q["question"] for q in faq_data.get("fr", [])]
        es_qs = [q["question"] for q in faq_data.get("es", [])]
        suggestions = [
            en_qs[0] if en_qs else "What is Clarity?",
            fr_qs[0] if fr_qs else "Qu'est-ce que Clarity ?",
            es_qs[0] if es_qs else "¿Qué es Clarity?",
            en_qs[1] if len(en_qs) > 1 else en_qs[0] if en_qs else "How does Clarity work?",
        ]
        for i, suggestion in enumerate(suggestions):
            with cols[i]:
                if st.button(suggestion, use_container_width=True):
                    print(f"[SUGGEST] \"{suggestion}\"")
                    answer, confidence, lang, method, found, suggestions = process_query(
                        suggestion, preprocessor, intent_clf, similarity, config
                    )
                    lang_display = config.supported_languages.get(lang, {}).get("name", lang)
                    status = "✓" if found else "✗"
                    print(f"  └─ {status} {method} · confidence {confidence:.0%} · {lang_display}")
                    st.session_state.messages.append({"role": "user", "content": suggestion})
                    st.session_state.messages.append({
                        "role": "assistant", "content": answer,
                        "confidence": confidence, "method": method,
                        "lang_display": lang_display,
                        "suggestions": suggestions,
                    })
                    st.rerun()

    for msg in st.session_state.messages:
        with st.chat_message(msg["role"]):
            st.markdown(msg["content"])
            if "confidence" in msg:
                conf = msg["confidence"]
                if conf >= 0.7:
                    badge_class = ""
                elif conf >= 0.4:
                    badge_class = "confidence-medium"
                else:
                    badge_class = "confidence-low"
                st.markdown(
                    f'<span class="confidence-badge {badge_class}">Confidence {conf:.0%}</span>'
                    f'<span class="method-badge">{msg["method"]}</span>'
                    f'<span style="font-size:11px;color:#656d76;margin-left:8px">{msg["lang_display"]}</span>',
                    unsafe_allow_html=True
                )
            sug = msg.get("suggestions")
            if sug:
                st.markdown("💡 **Did you mean one of these?**")
                sug_cols = st.columns(len(sug))
                for i, (sug_q, _) in enumerate(sug):
                    with sug_cols[i]:
                        key = f"sug_hist_{hash(str(msg))}_{i}"
                        if st.button(f"📌 {sug_q}", use_container_width=True, key=key):
                            a, c, l, m, f, _ = process_query(
                                sug_q, preprocessor, intent_clf, similarity, config
                            )
                            ld = config.supported_languages.get(l, {}).get("name", l)
                            st.session_state.messages.append({"role": "user", "content": sug_q})
                            st.session_state.messages.append({
                                "role": "assistant", "content": a,
                                "confidence": c, "method": m,
                                "lang_display": ld, "suggestions": [],
                            })
                            st.rerun()

    if prompt := st.chat_input("Ask a question..."):
        st.session_state.messages.append({"role": "user", "content": prompt})
        with st.chat_message("user"):
            st.markdown(prompt)

        t0 = time.time()
        answer, confidence, lang, method, found, suggestions = process_query(
            prompt, preprocessor, intent_clf, similarity, config
        )
        elapsed = time.time() - t0
        lang_display = config.supported_languages.get(lang, {}).get("name", lang)
        status = "✓" if found else "✗"
        print(f"[QUERY] \"{prompt[:60]}{'...' if len(prompt)>60 else ''}\"")
        print(f"  └─ {status} {lang_display} · {method} · confidence {confidence:.0%} ({elapsed:.2f}s)")

        with st.chat_message("assistant"):
            st.markdown(answer)
            st.markdown(
                f'<span class="confidence-badge {'confidence-low' if not found else ''}">'
                f'Confidence {confidence:.0%}</span>'
                f'<span class="method-badge">{method}</span>'
                f'<span style="font-size:11px;color:#656d76;margin-left:8px">{lang_display}</span>',
                unsafe_allow_html=True
            )
            if not found and suggestions:
                st.markdown("💡 **Did you mean one of these?**")
                sug_cols = st.columns(len(suggestions))
                for i, (sug_q, _) in enumerate(suggestions):
                    with sug_cols[i]:
                        if st.button(f"📌 {sug_q}", use_container_width=True, key=f"sug_{sug_q[:20]}_{i}_{hash(prompt)}"):
                            answer2, conf2, lang2, method2, found2, _ = process_query(
                                sug_q, preprocessor, intent_clf, similarity, config
                            )
                            lang_display2 = config.supported_languages.get(lang2, {}).get("name", lang2)
                            st.session_state.messages.append({"role": "user", "content": sug_q})
                            st.session_state.messages.append({
                                "role": "assistant", "content": answer2,
                                "confidence": conf2, "method": method2,
                                "lang_display": lang_display2,
                                "suggestions": [],
                            })
                            st.rerun()

        st.session_state.messages.append({
            "role": "assistant",
            "content": answer,
            "confidence": confidence,
            "method": method,
            "lang_display": lang_display,
            "suggestions": suggestions,
        })


if __name__ == "__main__":
    main()