scraper.py

import json
import os
import re
import pandas as pd
import requests
from bs4 import BeautifulSoup


class FAQScraper:
    def __init__(self, config):
        self.config = config

    def extract_from_url(self, url):
        try:
            response = requests.get(url, timeout=15, headers={
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
            })
            response.raise_for_status()
        except requests.RequestException as e:
            print(f"[ERROR] Failed to fetch {url}: {e}")
            return []

        soup = BeautifulSoup(response.text, "html.parser")
        scripts = soup.find_all("script", type="application/ld+json")
        for script in scripts:
            try:
                data = json.loads(script.string)
            except (json.JSONDecodeError, TypeError):
                continue
            items = self._extract_faqpage_items(data)
            if items:
                return items
        return []

    def _extract_faqpage_items(self, data):
        items = []
        if isinstance(data, dict):
            if data.get("@type") == "FAQPage":
                items = data.get("mainEntity", [])
            elif isinstance(data.get("@graph"), list):
                for item in data["@graph"]:
                    if item.get("@type") == "FAQPage":
                        items = item.get("mainEntity", [])
                        break
        elif isinstance(data, list):
            for item in data:
                if item.get("@type") == "FAQPage":
                    items = item.get("mainEntity", [])
                    break

        qa_pairs = []
        for entity in items:
            if entity.get("@type") == "Question":
                question = entity.get("name", "").strip()
                answer = entity.get("acceptedAnswer", {}).get("text", "").strip()
                if question and answer:
                    qa_pairs.append({"question": question, "answer": self._clean_html(answer)})
        return qa_pairs

    def _clean_html(self, text):
        clean = re.sub(r"<[^>]+>", "", text)
        clean = re.sub(r"\s+", " ", clean).strip()
        return clean

    def scrape_all(self, urls_config):
        for lang_code, url in urls_config.items():
            print(f"[SCRAPER] Fetching {lang_code} from {url}")
            qa_pairs = self.extract_from_url(url)
            if qa_pairs:
                df = pd.DataFrame(qa_pairs)
                out_dir = os.path.join(self.config.data_dir, lang_code)
                os.makedirs(out_dir, exist_ok=True)
                path = os.path.join(out_dir, "faq.csv")
                df.to_csv(path, index=False, encoding="utf-8")
                print(f"[SCRAPER] Saved {len(qa_pairs)} pairs to {path}")
            else:
                print(f"[SCRAPER] No FAQPage found at {url}")