scripts/transformer_data_preprocessing.py

"""
Transformer Data Preprocessing Module
======================================
Enhanced preprocessing pipeline for Nepali hate speech classification.

This module provides:
- Script detection (Devanagari/Romanized/English/Mixed)
- Transliteration (Romanized → Devanagari)
- Translation (English → Nepali)
- Emoji semantic mapping with feature extraction
- Text normalization

Usage:
------
from scripts.transformer_data_preprocessing import HateSpeechPreprocessor

# Initialize preprocessor
preprocessor = HateSpeechPreprocessor(
    model_type="xlmr",
    translate_english=True
)

# Preprocess single text
processed_text, emoji_features = preprocessor.preprocess("Your text here")

# Preprocess batch
texts_list = ["text1", "text2", "text3"]
processed_texts, features_list = preprocessor.preprocess_batch(texts_list)
"""

import re
import emoji
import regex
from typing import Any, Literal, Optional, Tuple, Dict, List
from deep_translator import GoogleTranslator
from functools import lru_cache
import logging

# Setup logging
logging.basicConfig(level=logging.WARNING)
logger = logging.getLogger(__name__)

# Try to import transliteration (optional)
try:
    from indic_transliteration import sanscript
    from indic_transliteration.sanscript import transliterate
    TRANSLITERATION_AVAILABLE = True
except ImportError:
    TRANSLITERATION_AVAILABLE = False
    logger.warning("indic_transliteration not available. Transliteration disabled.")


# ============================================================================
# COMPREHENSIVE EMOJI MAPPINGS
# ============================================================================

EMOJI_TO_NEPALI = {
    # Positive emotions
    '😂': 'हाँसो', '🤣': 'ठूलो_हाँसो', '😀': 'खुशी', '😁': 'खुशी', '😃': 'खुशी',
    '😄': 'खुशी', '😅': 'नर्भस_हाँसो', '😆': 'हाँसो', '😊': 'मुस्कान', '☺️': 'मुस्कान',
    '😉': 'आँखा_झिम्काउने', '🙂': 'मुस्कान', '🙃': 'उल्टो_मुस्कान', '😌': 'शान्त',
    '😍': 'माया', '🥰': 'माया', '😘': 'चुम्बन', '😗': 'चुम्बन', '😙': 'चुम्बन', '😚': 'चुम्बन',
    '🤗': 'अँगालो', '🤩': 'चकित', '🥳': 'उत्सव', '🤤': 'लालसा',
    
    # Mockery & Sarcasm
    '😏': 'व्यंग्य', '😜': 'जिब्रो_देखाउने', '😝': 'जिब्रो_देखाउने', '😛': 'जिब्रो',
    '🙄': 'आँखा_घुमाउने', '😤': 'निराश', '😑': 'अभिव्यक्तिहीन', '😐': 'तटस्थ',
    '😬': 'तनाव', '🤨': 'शंकास्पद', '🤫': 'चुपचाप', '🤭': 'हात_छोप्ने', 
    '🤥': 'झूठ', '😶': 'मौन',
    
    # Anger & Hate
    '😠': 'रिस', '😡': 'ठूलो_रिस', '🤬': 'गाली', '😈': 'खराब', '👿': 'खराब',
    '💢': 'क्रोध', '🔪': 'हिंसा', '💣': 'हिंसा', '🗡️': 'तरवार', '⚔️': 'युद्ध',
    '💥': 'विस्फोट', '🔫': 'बन्दुक', '🧨': 'विस्फोटक', '☠️': 'मृत्यु', '💀': 'खोपडी',
    '👹': 'राक्षस', '👺': 'दानव', '🤡': 'जोकर', '🖤': 'कालो_मन',
    '😾': 'रिसाएको', '👊': 'मुक्का', '✊': 'मुक्का',
    
    # Offensive Gestures
    '🖕': 'अपमान', '👎': 'नकारात्मक', '👎🏻': 'नकारात्मक', '👎🏼': 'नकारात्मक',
    '👎🏽': 'नकारात्मक', '👎🏾': 'नकारात्मक', '👎🏿': 'नकारात्मक',
    '🖕🏻': 'अपमान', '🖕🏼': 'अपमान', '🖕🏽': 'अपमान', '🖕🏾': 'अपमान', '🖕🏿': 'अपमान',
    
    # Sadness
    '😭': 'रुवाइ', '😢': 'रुवाइ', '😿': 'रुवाइ', '😔': 'उदास', '😞': 'उदास',
    '😒': 'उदास', '😓': 'चिन्तित', '😟': 'चिन्तित', '😕': 'अलमलिएको',
    '🙁': 'तल्लो_मुख', '☹️': 'दुःखी', '😩': 'थकित', '😫': 'थकित',
    '😖': 'भ्रमित', '😣': 'अडिग', '😥': 'निराश', '🥺': 'बिन्ती',
    
    # Fear & Shock
    '😨': 'डर', '😰': 'चिन्तित_पसिना', '😱': 'चिच्याउने', '😳': 'लजाउने',
    '🤯': 'मन_उडेको', '😵': 'चक्कर', '😲': 'चकित', '😯': 'छक्क',
    
    # Disgust
    '🤢': 'बान्ता', '🤮': 'बान्ता', '🤧': 'हाच्छ्यूँ', '😷': 'बिरामी',
    '🤒': 'ज्वरो', '🤕': 'घाइते', '🥴': 'मात्तिएको', '😪': 'निद्रा',
    
    # Positive Gestures
    '👍': 'सकारात्मक', '👍🏻': 'सकारात्मक', '👍🏼': 'सकारात्मक', 
    '👍🏽': 'सकारात्मक', '👍🏾': 'सकारात्मक', '👍🏿': 'सकारात्मक',
    '👏': 'तालि', '🙌': 'उत्सव', '👌': 'ठीक_छ', '🤝': 'हात_मिलाउनु',
    '🙏': 'नमस्कार', '🤲': 'प्रार्थना', '💪': 'शक्ति', '✌️': 'शान्ति',
    
    # Hearts
    '❤️': 'माया', '🧡': 'माया', '💛': 'माया', '💚': 'माया', '💙': 'माया',
    '💜': 'माया', '🤍': 'सेतो_मन', '🤎': 'खैरो_मन', '❣️': 'माया', 
    '💕': 'माया', '💞': 'माया', '💓': 'माया', '💗': 'माया',
    '💖': 'माया', '💘': 'माया', '💝': 'माया', '💔': 'टुटेको_मन',
    
    # Symbols
    '🔥': 'आगो', '💯': 'पूर्ण', '💨': 'हावा', '💫': 'चमक',
    '⭐': 'तारा', '✨': 'चमक', '🌟': 'चम्किलो_तारा',
    '🚫': 'निषेध', '⛔': 'प्रवेश_निषेध', '❌': 'रद्द', '❎': 'गलत',
    
    # People
    '👫': 'जोडी', '👬': 'पुरुष_जोडी', '👭': 'महिला_जोडी', '👨\u200d👩\u200d👧\u200d👦': 'परिवार',
    '👶': 'बच्चा', '👦': 'केटा', '👧': 'केटी', '👨': 'पुरुष', '👩': 'महिला',
    '👴': 'बूढो', '👵': 'बूढी', '🧒': 'बालक', '👱': 'गोरो', '🧔': 'दाह्री',
    
    # Country
    '🇳🇵': 'नेपाल', '🇮🇳': 'भारत', '🇵🇰': 'पाकिस्तान', '🇧🇩': 'बंगलादेश',
    '🇨🇳': 'चीन', '🇺🇸': 'अमेरिका', '🏴': 'झण्डा',
    
    # Animals
    '🐕': 'कुकुर', '🐖': 'सुँगुर', '🐀': 'मुसा', '🐍': 'सर्प', '🦂': 'बिच्छी',
    '🐒': 'बाँदर', '🐵': 'बाँदर_अनुहार', '🦍': 'गोरिल्ला', '🐗': 'जङ्गली_सुँगुर',
    
    # Other
    '🤔': 'सोच', '🧐': 'अनुसन्धान', '😴': 'सुत्ने', '💩': 'मल',
    '👻': 'भूत', '🤖': 'रोबोट', '👽': 'विदेशी', '🎭': 'मुखौटा',
    
    # === EXPANDED COMMON EMOJIS ===
    
    # Celebrations & Party
    '🎉': 'उत्सव', '🎊': 'पार्टी', '🎈': 'बेलुन', '🎁': 'उपहार',
    '🎂': 'केक', '🍰': 'मिठाई', '🥂': 'चश्मा', '🍾': 'शराब',
    
    # Food & Drink (common in casual/hate contexts)
    '🍕': 'पिज्जा', '🍔': 'बर्गर', '🍗': 'चिकन', '🍖': 'मासु',
    '🍺': 'बियर', '🍻': 'पार्टी', '☕': 'चिया', '🍵': 'चिया',
    '🍜': 'नूडल', '🍛': 'करी', '🍲': 'खाना', '🥘': 'परिकार',
    
    # Sports & Activities
    '⚽': 'फुटबल', '🏏': 'क्रिकेट', '🏀': 'बास्केटबल', '🎮': 'खेल',
    '🏆': 'ट्रफी', '🥇': 'स्वर्ण', '🥈': 'रजत', '🥉': 'कांस्य',
    
    # Weather & Nature
    '☀️': 'घाम', '🌙': 'चन्द्रमा', '🌧️': 'पानी', '⛈️': 'आँधी',
    '❄️': 'हिउँ', '🌈': 'इन्द्रेणी', '⚡': 'बिजुली', '🌪️': 'बतास',
    
    # Technology & Modern
    '📱': 'मोबाइल', '💻': 'कम्प्युटर', '📷': 'क्यामेरा', '🎥': 'भिडियो',
    '🖥️': 'कम्प्युटर', '⌨️': 'किबोर्ड', '🖱️': 'माउस', '📡': 'एन्टेना',
    
    # Time & Clock
    '⏰': 'घडी', '⏳': 'समय', '⌛': 'बालुवा_घडी', '🕐': 'एक_बजे',
    
    # Objects
    '📚': 'किताब', '📖': 'खुल्ला_किताब', '✏️': 'पेन्सिल', '📝': 'लेख',
    '🎤': 'माइक', '🎧': 'हेडफोन', '📢': 'घोषणा', '📣': 'चिल्लाउने',
    
    # Miscellaneous Common
    '✅': 'ठीक', '☑️': 'जाँच', '💯': 'सय', '🆗': 'ठीक',
    '🆕': 'नयाँ', '🆓': 'मुक्त', '🔴': 'रातो', '🟢': 'हरियो',
}

# Emoji categories for feature extraction
HATE_RELATED_EMOJIS = {
    '😠', '😡', '🤬', '😈', '👿', '💢', '👊', '✊',
    '🔪', '💣', '🗡️', '⚔️', '💥', '🔫', '🧨', '☠️', '💀',
    '🖕', '🖕🏻', '🖕🏼', '🖕🏽', '🖕🏾', '🖕🏿',
    '👎', '👎🏻', '👎🏼', '👎🏽', '👎🏾', '👎🏿',
    '👹', '👺', '🤡', '🖤', '💔',
    '🐕', '🐖', '🐀', '🐍', '🦂', '🐒', '🐵', '🦍', '🐗',
    '💩', '😾',
}

MOCKERY_EMOJIS = {
    '😏', '😜', '😝', '😛', '🙄', '😤', '🙃',
    '😑', '😐', '😬', '🤨', '🤫', '🤭', '🤥',
    '🤡', '👻', '🎭',
}

POSITIVE_EMOJIS = {
    '😊', '😀', '😁', '😃', '😄', '☺️', '🙂', '😌', '🥰', '😍',
    '❤️', '🧡', '💛', '💚', '💙', '💜', '🤍', '🤎',
    '💕', '💞', '💓', '💗', '💖', '💘', '💝', '❣️',
    '👍', '👍🏻', '👍🏼', '👍🏽', '👍🏾', '👍🏿',
    '🙏', '👏', '🙌', '👌', '🤝', '✌️',
    '🥳', '🎉', '🎊', '⭐', '✨', '🌟',
}

SADNESS_EMOJIS = {
    '😭', '😢', '😿', '😔', '😞', '😒', '😓', '😟', '😕',
    '🙁', '☹️', '😩', '😫', '😖', '😣', '😥', '🥺',
}

FEAR_EMOJIS = {
    '😨', '😰', '😱', '😳', '🤯', '😵', '😲', '😯',
}

DISGUST_EMOJIS = {
    '🤢', '🤮', '🤧', '😷', '🤒', '🤕', '🥴',
}


# ============================================================================
# NORMALIZATION MAPPINGS
# ============================================================================

DIRGHIKARAN_MAP = {
    "\u200d": "",  # Zero-width joiner
    "\u200c": "",  # Zero-width non-joiner
    "।": ".",      # Devanagari danda
    "॥": ".",      # Double danda
}


# ============================================================================
# TYPE DEFINITIONS
# ============================================================================

ScriptType = Literal["devanagari", "romanized_nepali", "english", "mixed", "other"]


# ============================================================================
# EMOJI FEATURE EXTRACTION
# ============================================================================

def extract_emoji_features(text: str) -> Dict[str, int]:
    """
    Extract comprehensive emoji-based semantic features
    
    Returns 18 features:
    - 6 binary flags (has_X_emoji)
    - 6 count features (X_emoji_count)
    - 6 derived features (total, ratio, mixed_sentiment, unknown tracking)
    """
    emojis_found = [c for c in text if c in emoji.EMOJI_DATA]
    
    hate_count = sum(1 for e in emojis_found if e in HATE_RELATED_EMOJIS)
    mockery_count = sum(1 for e in emojis_found if e in MOCKERY_EMOJIS)
    positive_count = sum(1 for e in emojis_found if e in POSITIVE_EMOJIS)
    sadness_count = sum(1 for e in emojis_found if e in SADNESS_EMOJIS)
    fear_count = sum(1 for e in emojis_found if e in FEAR_EMOJIS)
    disgust_count = sum(1 for e in emojis_found if e in DISGUST_EMOJIS)
    
    # Track unknown emojis (not in our mapping)
    known_emojis = set(EMOJI_TO_NEPALI.keys())
    unknown_emojis = [e for e in emojis_found if e not in known_emojis]
    unknown_count = len(unknown_emojis)
    
    return {
        # Binary flags
        'has_hate_emoji': 1 if hate_count > 0 else 0,
        'has_mockery_emoji': 1 if mockery_count > 0 else 0,
        'has_positive_emoji': 1 if positive_count > 0 else 0,
        'has_sadness_emoji': 1 if sadness_count > 0 else 0,
        'has_fear_emoji': 1 if fear_count > 0 else 0,
        'has_disgust_emoji': 1 if disgust_count > 0 else 0,
        
        # Count features
        'hate_emoji_count': hate_count,
        'mockery_emoji_count': mockery_count,
        'positive_emoji_count': positive_count,
        'sadness_emoji_count': sadness_count,
        'fear_emoji_count': fear_count,
        'disgust_emoji_count': disgust_count,
        'total_emoji_count': len(emojis_found),
        
        # Derived features
        'hate_to_positive_ratio': hate_count / max(positive_count, 1),
        'has_mixed_sentiment': 1 if (hate_count > 0 and positive_count > 0) else 0,
        
        # NEW: Unknown emoji tracking
        'unknown_emoji_count': unknown_count,
        'has_unknown_emoji': 1 if unknown_count > 0 else 0,
        'known_emoji_ratio': (len(emojis_found) - unknown_count) / max(len(emojis_found), 1),
    }


def remove_emojis_for_detection(text: str) -> str:
    """Remove emojis temporarily for script detection"""
    return emoji.replace_emoji(text, replace="")


# ============================================================================
# SCRIPT DETECTION
# ============================================================================

def detect_script_type(text: str) -> Tuple[ScriptType, dict]:
    """
    Detect the dominant script type ignoring emojis
    
    Returns:
        Tuple of (script_type, detection_details)
    """
    if not text or not text.strip():
        return "other", {"confidence": 0.0, "reason": "empty_text"}
    
    # Remove emojis before detection
    text_no_emoji = remove_emojis_for_detection(text)
    
    if not text_no_emoji.strip():
        return "other", {"confidence": 0.5, "reason": "emoji_only"}
    
    letters = regex.findall(r"\p{L}", text_no_emoji)
    letter_count = len(letters)
    
    if letter_count == 0:
        return "other", {"confidence": 0.0, "reason": "no_letters"}
    
    devanagari_chars = regex.findall(r"\p{Devanagari}", text_no_emoji)
    dev_count = len(devanagari_chars)
    dev_ratio = dev_count / letter_count
    
    latin_chars = regex.findall(r"[a-zA-Z]", text_no_emoji)
    latin_count = len(latin_chars)
    latin_ratio = latin_count / letter_count
    
    # Romanized Nepali patterns
    romanized_nepali_patterns = [
        # Common words
        r'\b[xX]u\b', r'\b[xX]um?\b', r'\bhajur\b', r'\bdai\b', r'\bbhai\b', r'\bdidi\b',
        r'\bbahini\b', r'\bsanghai\b', r'\bsunu\b', r'\bhera\b', r'\bsun\b',
        
        # Particles & Postpositions
        r'\bko\b', r'\bki\b', r'\bka\b', r'\bho\b', r'\btyo\b', r'\byo\b', r'\bta\b',
        r'\bma\b', r'\bma?i\b', r'\bla[ie]?\b', r'\bnai?\b', r'\bpani\b', r'\bni\b',
        
        # Verbs
        r'\bhun[ae]\b', r'\bhunchha\b', r'\bhunuhunchha\b', r'\bgar\w+\b', r'\bgarna\b',
        r'\bx[ao]\b', r'\bxa\b', r'\bxan\b', r'\bxaina\b', r'\bxu\b',
        r'\bchain\b', r'\bchaina\b', r'\bthiy[oe]\b', r'\bhola\b', r'\bhos\b',
        r'\bbhan\w*\b', r'\bbol\w*\b', r'\bher\w*\b',
        
        # Common adjectives/states
        r'\bkh[ou]s[hi]?\b', r'\bkhusi\b', r'\bkhushi\b', r'\bramro\b', r'\bnaramro\b',
        r'\bthulo\b', r'\bsano\b', r'\brasilo\b', r'\bmitho\b', r'\btikhi\b',
        r'\bdherei\b', r'\baliali\b', r'\bastai\b', r'\blastai\b',
        
        # Question words
        r'\bkina\b', r'\bkasari\b', r'\bkahile\b', r'\bkaha[n]?\b', r'\bke\b', r'\bko\b',
        
        # Pronouns
        r'\bma\b', r'\btimi\b', r'\btapai\b', r'\buha\b', r'\buni\b', r'\byini\b',
        r'\bmero\b', r'\btimro\b', r'\buhako\b', r'\buniko\b', r'\bhamro\b',
        
        # Common nouns
        r'\bmanxe\b', r'\bmanchhe\b', r'\bmanche\b', r'\bharu\b', r'\bdes[ha]?\b',
        r'\bgha?r\b', r'\bthau\b', r'\bsamay\b', r'\bbela\b',
        
        # Nepali-specific endings (transliterated)
        r'\w+[ae]ko\b', r'\w+[ae]ki\b', r'\w+dai\b', r'\w+lai\b',
        r'\w+ma\b', r'\w+xa\b', r'\w+hun[ae]\b', r'\w+thiyo\b',
    ]
    
    romanized_indicators = sum(1 for pattern in romanized_nepali_patterns 
                               if re.search(pattern, text_no_emoji, re.IGNORECASE))
    
    # Calculate Romanized Nepali score
    romanized_score = 0.0
    if latin_ratio > 0.5 and dev_ratio < 0.3:
        if romanized_indicators > 0:
            romanized_score = min(0.5 + (romanized_indicators * 0.15), 0.95)
        else:
            # Check for typical Romanized Nepali patterns
            romanized_patterns = re.findall(r'\b\w*[aeiou](?:h)?\b', text_no_emoji.lower())
            if any(word.endswith(('xu', 'ro', 'no', 'lo', 'ko', 'ho')) 
                   for word in romanized_patterns):
                romanized_score = 0.4
            else:
                romanized_score = 0.3
    
    # English indicators (EXPANDED)
    english_indicators = [
        # Articles & Determiners
        'the', 'a', 'an', 'this', 'that', 'these', 'those', 'some', 'any', 'all', 'every',
        
        # Pronouns
        'i', 'you', 'he', 'she', 'it', 'we', 'they', 'me', 'him', 'her', 'us', 'them',
        'my', 'your', 'his', 'her', 'its', 'our', 'their', 'mine', 'yours', 'ours', 'theirs',
        'myself', 'yourself', 'himself', 'herself', 'itself', 'ourselves', 'themselves',
        
        # Common verbs (be, have, do)
        'is', 'am', 'are', 'was', 'were', 'be', 'been', 'being',
        'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'done',
        'will', 'would', 'shall', 'should', 'can', 'could', 'may', 'might', 'must',
        
        # Common verbs (action)
        'get', 'got', 'go', 'went', 'gone', 'make', 'made', 'take', 'took', 'taken',
        'come', 'came', 'see', 'saw', 'seen', 'know', 'knew', 'known', 'say', 'said',
        'tell', 'told', 'think', 'thought', 'give', 'gave', 'given', 'find', 'found',
        
        # Question words
        'what', 'which', 'who', 'whom', 'whose', 'when', 'where', 'why', 'how',
        
        # Prepositions
        'in', 'on', 'at', 'to', 'for', 'of', 'with', 'from', 'by', 'about', 'as',
        'into', 'through', 'over', 'under', 'after', 'before', 'between', 'among',
        
        # Conjunctions
        'and', 'or', 'but', 'so', 'yet', 'nor', 'because', 'if', 'when', 'while',
        'although', 'though', 'unless', 'since', 'until', 'where', 'whether',
        
        # Negations
        'not', 'no', 'never', 'none', 'nothing', 'nobody', 'nowhere', 'neither',
        
        # Common adjectives
        'good', 'bad', 'great', 'big', 'small', 'long', 'short', 'high', 'low',
        'old', 'new', 'young', 'early', 'late', 'right', 'wrong', 'true', 'false',
        'hot', 'cold', 'happy', 'sad', 'angry', 'nice', 'beautiful', 'ugly',
        
        # Sentiment words (hate speech relevant)
        'hate', 'love', 'like', 'dislike', 'stupid', 'dumb', 'idiot', 'fool',
        'kill', 'die', 'dead', 'death', 'fuck', 'shit', 'ass', 'damn', 'hell',
        'worst', 'terrible', 'horrible', 'awful', 'disgusting', 'pathetic',
        
        # Common nouns
        'man', 'woman', 'people', 'person', 'thing', 'time', 'day', 'year',
        'way', 'work', 'life', 'world', 'country', 'place', 'home', 'hand',
        
        # Very & Adverbs
        'very', 'really', 'quite', 'too', 'so', 'just', 'only', 'even', 'also',
        'well', 'much', 'more', 'most', 'less', 'least', 'still', 'already',
    ]
    english_words = [w.lower() for w in re.findall(r'\b\w+\b', text_no_emoji)]
    english_count = sum(1 for w in english_words if w in english_indicators)
    english_ratio = english_count / len(english_words) if english_words else 0
    
    # Detection details
    details = {
        "devanagari_count": dev_count,
        "devanagari_ratio": dev_ratio,
        "latin_count": latin_count,
        "latin_ratio": latin_ratio,
        "romanized_indicators": romanized_indicators,
        "english_ratio": english_ratio,
        "letter_count": letter_count
    }
    
    # Decision logic
    if dev_ratio >= 0.8:
        return "devanagari", {**details, "confidence": dev_ratio, "reason": "dominant_devanagari"}
    
    elif dev_ratio >= 0.4:
        return "mixed", {**details, "confidence": 0.7, "reason": "mixed_with_devanagari"}
    
    elif romanized_score > 0.5 and dev_ratio < 0.2:
        return "romanized_nepali", {**details, "confidence": romanized_score, "reason": "romanized_nepali_detected"}
    
    elif english_ratio > 0.2 and romanized_score < 0.4:
        return "english", {**details, "confidence": min(english_ratio + 0.3, 0.9), "reason": "english_detected"}
    
    elif latin_ratio > 0.5 and romanized_score > 0.3:
        return "romanized_nepali", {**details, "confidence": romanized_score, "reason": "likely_romanized_nepali"}
    
    elif latin_ratio > 0.8:
        if english_ratio > 0.1:
            return "english", {**details, "confidence": 0.6, "reason": "likely_english"}
        else:
            return "romanized_nepali", {**details, "confidence": 0.5, "reason": "ambiguous_latin_script"}
    
    else:
        return "other", {**details, "confidence": 0.3, "reason": "insufficient_indicators"}



# ============================================================================
# TEXT PROCESSING FUNCTIONS
# ============================================================================

def clean_text_basic(text: str) -> str:
    """Basic text cleaning"""
    # Remove URLs
    text = re.sub(r"http\S+|www\S+", "", text)
    # Remove mentions
    text = re.sub(r"@\w+", "", text)
    # Remove hashtag symbol but keep text
    text = re.sub(r"#(\w+)", r"\1", text)
    # Remove quotes (single and double, including smart quotes)
    text = text.replace('"', '').replace("'", '').replace('"', '').replace('"', '').replace(''', '').replace(''', '')
    # Normalize whitespace
    text = re.sub(r"\s+", " ", text).strip()
    return text


def normalize_devanagari(text: str) -> str:
    """Normalize Devanagari-specific characters"""
    for k, v in DIRGHIKARAN_MAP.items():
        text = text.replace(k, v)
    return text


def replace_emojis_semantic(text: str, preserve_spacing: bool = True, 
                           preserve_unknown: bool = True) -> str:
    """
    Replace emojis with Nepali text
    
    Args:
        text: Input text with emojis
        preserve_spacing: Add spaces around Nepali replacements
        preserve_unknown: Keep emojis not in EMOJI_TO_NEPALI mapping (default: True)
                         If False, unknown emojis will be removed
    
    Returns:
        Text with emojis replaced (or preserved)
    """
    # Replace known emojis with Nepali translations
    for emoji_char, nepali_text in EMOJI_TO_NEPALI.items():
        if preserve_spacing:
            text = text.replace(emoji_char, f" {nepali_text} ")
        else:
            text = text.replace(emoji_char, nepali_text)
    
    # Handle unknown emojis
    if not preserve_unknown:
        # OLD BEHAVIOR: Remove all remaining emojis
        text = emoji.replace_emoji(text, replace=" ")
    # else: NEW BEHAVIOR: Keep unknown emojis as-is
    # This allows the model to process them directly
    
    return text


def transliterate_romanized_nepali(text: str) -> str:
    """Transliterate Romanized Nepali to Devanagari"""
    if not TRANSLITERATION_AVAILABLE:
        return text
    
    try:
        result = transliterate(text, sanscript.ITRANS, sanscript.DEVANAGARI)
        return result if result else text
    except Exception as e:
        logger.warning(f"Transliteration failed: {e}")
        return text


# ============================================================================
# CACHED TRANSLATOR
# ============================================================================

class CachedNepaliTranslator:
    """Translator with LRU cache for efficiency"""
    
    def __init__(self, cache_size: int = 2000):
        self.translator = GoogleTranslator(source='en', target='ne')
        self.cache_size = cache_size
        self._translate_cached = lru_cache(maxsize=cache_size)(self._translate_single)
    
    def _translate_single(self, text: str) -> str:
        if not text or not text.strip():
            return ""
        try:
            result = self.translator.translate(text.strip())
            return result if result else text
        except Exception as e:
            logger.error(f"Translation failed: {str(e)}")
            return text
    
    def translate(self, text: str, fallback_to_original: bool = True) -> str:
        if not text or not text.strip():
            return ""
        try:
            return self._translate_cached(text.strip())
        except Exception as e:
            if fallback_to_original:
                logger.warning(f"Translation failed, using original: {str(e)}")
                return text
            raise
    
    def get_cache_info(self) -> dict:
        """Get cache statistics"""
        cache_info = self._translate_cached.cache_info()
        return {
            'hits': cache_info.hits,
            'misses': cache_info.misses,
            'size': cache_info.currsize,
            'max_size': cache_info.maxsize,
            'hit_rate': cache_info.hits / (cache_info.hits + cache_info.misses) 
                       if (cache_info.hits + cache_info.misses) > 0 else 0.0
        }


def translate_latin_spans(text: str, translator: CachedNepaliTranslator) -> str:
    """Translate Latin word spans in Devanagari text"""
    def repl(match):
        latin_text = match.group(0)
        translated = translator.translate(latin_text, fallback_to_original=True)
        return f" {translated} "
    
    return re.sub(r"[A-Za-z][A-Za-z\s]{2,}", repl, text)


# ============================================================================
# MAIN PREPROCESSOR CLASS
# ============================================================================

class HateSpeechPreprocessor:
    """
    Main preprocessing pipeline for Nepali hate speech classification
    
    Pipeline:
    1. Extract emoji features (before any processing)
    2. Detect script type (ignoring emojis)
    3. Apply script-specific processing
    4. Replace emojis with Nepali text
    5. Normalize Devanagari
    """

    def __init__(
        self,
        model_type: Literal["xlmr", "mbert", "nepalibert"] = "xlmr",
        translate_english: bool = True,
        cache_size: int = 2000
    ):
        self.model_type = model_type
        self.translate_english = translate_english
        self.translator = CachedNepaliTranslator(cache_size) if translate_english else None

    def preprocess(self, text: str, verbose: bool = False) -> Tuple[str, Dict[str, int]]:
        """
        Preprocess a single text
        
        Args:
            text: Input text
            verbose: Print processing steps
        
        Returns:
            Tuple of (preprocessed_text, emoji_features)
        """
        if not isinstance(text, str) or not text.strip():
            return "", {
                'has_hate_emoji': 0, 'has_mockery_emoji': 0, 'has_positive_emoji': 0,
                'has_sadness_emoji': 0, 'has_fear_emoji': 0, 'has_disgust_emoji': 0,
                'hate_emoji_count': 0, 'mockery_emoji_count': 0, 'positive_emoji_count': 0,
                'sadness_emoji_count': 0, 'fear_emoji_count': 0, 'disgust_emoji_count': 0,
                'total_emoji_count': 0, 'hate_to_positive_ratio': 0.0, 'has_mixed_sentiment': 0
            }

        original_text = text
        
        # Step 1: Extract emoji features
        emoji_features = extract_emoji_features(original_text)
        
        # Step 2: Detect script type
        script_type, details = detect_script_type(text)
        
        if verbose:
            print(f"Script detected: {script_type} (confidence: {details.get('confidence', 0):.2%})")
        
        # Step 3: Basic cleaning
        text = clean_text_basic(text)
        
        # Step 4: Script-specific processing
        if script_type == "devanagari":
            processed = text
            if self.translate_english and self.translator:
                processed = translate_latin_spans(processed, self.translator)
        
        elif script_type == "romanized_nepali":
            processed = transliterate_romanized_nepali(text)
        
        elif script_type == "english":
            if self.translate_english and self.translator:
                processed = self.translator.translate(text, fallback_to_original=True)
            else:
                processed = text
        
        elif script_type == "mixed":
            processed = transliterate_romanized_nepali(text)
            if self.translate_english and self.translator:
                processed = translate_latin_spans(processed, self.translator)
        else:
            processed = text
        
        # Step 5: Replace emojis
        processed = replace_emojis_semantic(processed)
        
        # Step 6: Normalize
        final = normalize_devanagari(processed)
        final = re.sub(r"\s+", " ", final).strip()
        
        if verbose:
            print(f"Original: {original_text}")
            print(f"Processed: {final}")
            print(f"Emoji features: {emoji_features}")
        
        return final, emoji_features
    
    def preprocess_batch(self, texts: List[str], verbose: bool = False, show_progress: bool = False) -> Tuple[List[str], List[Dict[str, int]]]:
        """
        Preprocess multiple texts
        
        Args:
            texts: List of input texts
            verbose: Print processing steps for each text
            show_progress: Show progress bar (requires tqdm)
        
        Returns:
            Tuple of (preprocessed_texts, emoji_features_list)
        """
        if show_progress:
            try:
                from tqdm import tqdm
                results = [self.preprocess(text, verbose=verbose) for text in tqdm(texts, desc="Preprocessing")]
            except ImportError:
                results = [self.preprocess(text, verbose=verbose) for text in texts]
        else:
            results = [self.preprocess(text, verbose=verbose) for text in texts]
        
        texts_processed = [r[0] for r in results]
        features = [r[1] for r in results]
        return texts_processed, features
    
    def get_stats(self) -> dict:
        """Get preprocessor statistics"""
        stats = {
            'model_type': self.model_type,
            'translation_enabled': self.translate_english,
            'transliteration_available': TRANSLITERATION_AVAILABLE,
        }
        if self.translator:
            stats['cache_info'] = self.translator.get_cache_info()
        return stats


# ============================================================================
# CONVENIENCE FUNCTIONS FOR STREAMLIT
# ============================================================================

def preprocess_text(
    text: str,
    model_type: str = "xlmr",
    translate_english: bool = True,
    verbose: bool = False
) -> Tuple[str, Dict[str, int]]:
    """
    Quick preprocessing function for single text (Streamlit-friendly)
    
    Args:
        text: Input text
        model_type: Model type (xlmr, mbert, nepalibert)
        translate_english: Whether to translate English
        verbose: Print processing steps
    
    Returns:
        Tuple of (preprocessed_text, emoji_features)
    """
    preprocessor = HateSpeechPreprocessor(
        model_type=model_type,
        translate_english=translate_english
    )
    return preprocessor.preprocess(text, verbose=verbose)


def get_script_info(text: str) -> Dict[str, any]:
    """
    Get detailed script detection info (useful for Streamlit display)
    
    Returns:
        Dictionary with script type, confidence, and details
    """
    script_type, details = detect_script_type(text)
    return {
        'script_type': script_type,
        'confidence': details.get('confidence', 0),
        'details': details
    }


def get_emoji_info(text: str) -> Dict[str, Any]:
    """Get detailed information about emojis in text"""
    emojis_found = [c for c in text if c in emoji.EMOJI_DATA]
    known_emojis = set(EMOJI_TO_NEPALI.keys())
    unknown_emojis = [e for e in emojis_found if e not in known_emojis]
    known_emojis_found = [e for e in emojis_found if e in known_emojis]
    
    return {
        'emojis_found': emojis_found,
        'total_count': len(emojis_found),
        'known_emojis': known_emojis_found,
        'known_count': len(known_emojis_found),
        'unknown_emojis': unknown_emojis,
        'unknown_count': len(unknown_emojis),
        'coverage': len(known_emojis_found) / len(emojis_found) if emojis_found else 1.0
    }