from __future__ import annotations from dataclasses import dataclass from typing import Optional from langdetect import detect, DetectorFactory, LangDetectException from lingua import Language, LanguageDetectorBuilder try: from INLD.INLD import detect as inld_detect INLD_AVAILABLE = True except ImportError: INLD_AVAILABLE = False DetectorFactory.seed = 0 ALLOWED_LANGUAGES = { "en", "ru", "uk", "de", "es", "ar", "hi", "zh", "it", "fr", "he", "ja", "hin" } LINGUA_LANG_MAP = { Language.ENGLISH: "en", Language.RUSSIAN: "ru", Language.UKRAINIAN: "uk", Language.GERMAN: "de", Language.SPANISH: "es", Language.ARABIC: "ar", Language.HINDI: "hi", Language.CHINESE: "zh", Language.ITALIAN: "it", Language.FRENCH: "fr", Language.HEBREW: "he", Language.JAPANESE: "ja", } lingua_detector = LanguageDetectorBuilder.from_languages(*LINGUA_LANG_MAP.keys()).build() @dataclass class LanguageResult: lang: str langdetect: str lingua: str detector: str def normalize(code: str) -> str: if not code: return "unknown" code = code.lower().strip() if code in {"zh-cn", "zh-tw"}: return "zh" if code == "iw": return "he" return code def run_inld(text: str) -> Optional[str]: if not INLD_AVAILABLE: return None try: result = inld_detect(text) if result and result[0].lower() == "hng": return "hin" except Exception: pass return None def detect_language(text: str) -> LanguageResult: if not text or not text.strip(): return LanguageResult("other", "unknown", "unknown", "other") text = text.strip() try: ld_code = normalize(detect(text)) except LangDetectException: ld_code = "unknown" lingua_lang = lingua_detector.detect_language_of(text) if lingua_lang is None: li_code = "unknown" else: li_code = LINGUA_LANG_MAP.get(lingua_lang, "unknown") if ld_code == li_code and ld_code in ALLOWED_LANGUAGES: return LanguageResult(lang=ld_code, langdetect=ld_code, lingua=li_code, detector="agreement") inld_lang = run_inld(text) if inld_lang: return LanguageResult(lang=inld_lang, langdetect=ld_code, lingua=li_code, detector="inld") return LanguageResult(lang="other", langdetect=ld_code, lingua=li_code, detector="other")