src/utils/language_utils.py

from __future__ import annotations

from dataclasses import dataclass
from typing import Optional, List

from langdetect import detect, DetectorFactory, LangDetectException
from lingua import Language, LanguageDetectorBuilder

# Optional INLD (Hinglish detector)
try:
    from INLD.INLD import detect as inld_detect
    INLD_AVAILABLE = True
except ImportError:
    INLD_AVAILABLE = False

# Make langdetect deterministic
DetectorFactory.seed = 0

# Languages covered by the TextDetox toxicity models (15) + Hinglish code "hin"
# en, ru, uk, de, es, am, zh, ar, hi, it, fr, he, hin, ja, tt
SUPPORTED_LANGUAGES = {
    "en", "ru", "uk", "de", "es", "am", "zh", "ar",
    "hi", "it", "fr", "he", "hin", "ja", "tt"
}

# Subset we can currently detect reliably with langdetect+lingua
ALLOWED_LANGUAGES = {
    "en", "ru", "uk", "de", "es", "ar",
    "hi", "zh", "it", "fr", "he", "ja"
}

LINGUA_LANG_MAP = {
    Language.ENGLISH: "en",
    Language.RUSSIAN: "ru",
    Language.UKRAINIAN: "uk",
    Language.GERMAN: "de",
    Language.SPANISH: "es",
    Language.ARABIC: "ar",
    Language.HINDI: "hi",
    Language.CHINESE: "zh",
    Language.ITALIAN: "it",
    Language.FRENCH: "fr",
    Language.HEBREW: "he",
    Language.JAPANESE: "ja",
}

lingua_detector = LanguageDetectorBuilder.from_languages(
    *LINGUA_LANG_MAP.keys()
).build()


@dataclass
class LanguageResult:
    lang: str          # final classification (en, ru, hin, other, ...)
    langdetect: str    # raw langdetect output
    lingua: str        # raw lingua output
    detector: str      # "agreement" | "inld" | "other"


def normalize(code: str) -> str:
    if not code:
        return "unknown"
    code = code.lower().strip()
    if code in {"zh-cn", "zh-tw"}:
        return "zh"
    if code == "iw":
        return "he"
    return code


def run_inld(text: str) -> Optional[str]:
    """
    Run INLD and return 'hin' (Hinglish) if detected.
    INLD returns ['hng'], ['hi'], ['en'], etc.
    """
    if not INLD_AVAILABLE:
        return None
    try:
        result = inld_detect(text)
        if result and result[0].lower() == "hng":
            return "hin"
    except Exception:
        pass
    return None


def detect_language(text: str) -> LanguageResult:
    if not text or not text.strip():
        return LanguageResult("other", "unknown", "unknown", "other")

    text = text.strip()

    # -------- langdetect --------
    try:
        ld_code = normalize(detect(text))
    except LangDetectException:
        ld_code = "unknown"

    # -------- lingua --------
    lingua_lang = lingua_detector.detect_language_of(text)
    if lingua_lang is None:
        li_code = "unknown"
    else:
        li_code = LINGUA_LANG_MAP.get(lingua_lang, "unknown")

    # -------- agreement rule --------
    if ld_code == li_code and ld_code in ALLOWED_LANGUAGES:
        return LanguageResult(
            lang=ld_code,
            langdetect=ld_code,
            lingua=li_code,
            detector="agreement",
        )

    # -------- INLD fallback (Hinglish) --------
    inld_lang = run_inld(text)
    if inld_lang:
        return LanguageResult(
            lang=inld_lang,
            langdetect=ld_code,
            lingua=li_code,
            detector="inld",
        )

    # -------- final fallback --------
    # For now, am + tt and low-confidence cases go here
    return LanguageResult(
        lang="other",
        langdetect=ld_code,
        lingua=li_code,
        detector="other",
    )


def detect_language_code(text: str) -> str:
    """
    Convenience helper: just returns the final language code.
    """
    return detect_language(text).lang


def detect_language_batch(texts: List[str]) -> List[LanguageResult]:
    return [detect_language(t) for t in texts]


def detect_language_codes(texts: List[str]) -> List[str]:
    return [detect_language(t).lang for t in texts]


if __name__ == "__main__":
    tests = [
        "Hello, how are you?",
        "Привет, как дела?",
        "नमस्ते, आप कैसे हैं?",
        "Hola, ¿cómo estás?",
        "こんにちは、お元気ですか？",
        "ye ek example hai",
        "Bhai aaj kya kar rahe ho?",
        "lol ok",
        ""
    ]

    for t in tests:
        r = detect_language(t)
        print(
            f"{t!r:35} → {r.lang:9} | "
            f"ld={r.langdetect:5} | li={r.lingua:5} | via={r.detector}"
        )