File size: 4,431 Bytes
1818339 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 | from __future__ import annotations
from dataclasses import dataclass
from typing import Optional, List
from langdetect import detect, DetectorFactory, LangDetectException
from lingua import Language, LanguageDetectorBuilder
# Optional INLD (Hinglish detector)
try:
from INLD.INLD import detect as inld_detect
INLD_AVAILABLE = True
except ImportError:
INLD_AVAILABLE = False
# Make langdetect deterministic
DetectorFactory.seed = 0
# Languages covered by the TextDetox toxicity models (15) + Hinglish code "hin"
# en, ru, uk, de, es, am, zh, ar, hi, it, fr, he, hin, ja, tt
SUPPORTED_LANGUAGES = {
"en", "ru", "uk", "de", "es", "am", "zh", "ar",
"hi", "it", "fr", "he", "hin", "ja", "tt"
}
# Subset we can currently detect reliably with langdetect+lingua
ALLOWED_LANGUAGES = {
"en", "ru", "uk", "de", "es", "ar",
"hi", "zh", "it", "fr", "he", "ja"
}
LINGUA_LANG_MAP = {
Language.ENGLISH: "en",
Language.RUSSIAN: "ru",
Language.UKRAINIAN: "uk",
Language.GERMAN: "de",
Language.SPANISH: "es",
Language.ARABIC: "ar",
Language.HINDI: "hi",
Language.CHINESE: "zh",
Language.ITALIAN: "it",
Language.FRENCH: "fr",
Language.HEBREW: "he",
Language.JAPANESE: "ja",
}
lingua_detector = LanguageDetectorBuilder.from_languages(
*LINGUA_LANG_MAP.keys()
).build()
@dataclass
class LanguageResult:
lang: str # final classification (en, ru, hin, other, ...)
langdetect: str # raw langdetect output
lingua: str # raw lingua output
detector: str # "agreement" | "inld" | "other"
def normalize(code: str) -> str:
if not code:
return "unknown"
code = code.lower().strip()
if code in {"zh-cn", "zh-tw"}:
return "zh"
if code == "iw":
return "he"
return code
def run_inld(text: str) -> Optional[str]:
"""
Run INLD and return 'hin' (Hinglish) if detected.
INLD returns ['hng'], ['hi'], ['en'], etc.
"""
if not INLD_AVAILABLE:
return None
try:
result = inld_detect(text)
if result and result[0].lower() == "hng":
return "hin"
except Exception:
pass
return None
def detect_language(text: str) -> LanguageResult:
if not text or not text.strip():
return LanguageResult("other", "unknown", "unknown", "other")
text = text.strip()
# -------- langdetect --------
try:
ld_code = normalize(detect(text))
except LangDetectException:
ld_code = "unknown"
# -------- lingua --------
lingua_lang = lingua_detector.detect_language_of(text)
if lingua_lang is None:
li_code = "unknown"
else:
li_code = LINGUA_LANG_MAP.get(lingua_lang, "unknown")
# -------- agreement rule --------
if ld_code == li_code and ld_code in ALLOWED_LANGUAGES:
return LanguageResult(
lang=ld_code,
langdetect=ld_code,
lingua=li_code,
detector="agreement",
)
# -------- INLD fallback (Hinglish) --------
inld_lang = run_inld(text)
if inld_lang:
return LanguageResult(
lang=inld_lang,
langdetect=ld_code,
lingua=li_code,
detector="inld",
)
# -------- final fallback --------
# For now, am + tt and low-confidence cases go here
return LanguageResult(
lang="other",
langdetect=ld_code,
lingua=li_code,
detector="other",
)
def detect_language_code(text: str) -> str:
"""
Convenience helper: just returns the final language code.
"""
return detect_language(text).lang
def detect_language_batch(texts: List[str]) -> List[LanguageResult]:
return [detect_language(t) for t in texts]
def detect_language_codes(texts: List[str]) -> List[str]:
return [detect_language(t).lang for t in texts]
if __name__ == "__main__":
tests = [
"Hello, how are you?",
"Привет, как дела?",
"नमस्ते, आप कैसे हैं?",
"Hola, ¿cómo estás?",
"こんにちは、お元気ですか？",
"ye ek example hai",
"Bhai aaj kya kar rahe ho?",
"lol ok",
""
]
for t in tests:
r = detect_language(t)
print(
f"{t!r:35} → {r.lang:9} | "
f"ld={r.langdetect:5} | li={r.lingua:5} | via={r.detector}"
)
|