import json import re import unicodedata SUPPORTED_ASR_CODES = { "af", "am", "ar", "as", "az", "ba", "be", "bg", "bn", "bo", "br", "bs", "ca", "cs", "cy", "da", "de", "el", "en", "es", "et", "eu", "fa", "fi", "fo", "fr", "gl", "gu", "ha", "haw", "he", "hi", "hr", "ht", "hu", "hy", "id", "is", "it", "ja", "jw", "ka", "kk", "km", "kn", "ko", "la", "lb", "ln", "lo", "lt", "lv", "mg", "mi", "mk", "ml", "mn", "mr", "ms", "mt", "my", "ne", "nl", "nn", "no", "oc", "pa", "pl", "ps", "pt", "ro", "ru", "sa", "sd", "si", "sk", "sl", "sn", "so", "sq", "sr", "su", "sv", "sw", "ta", "te", "tg", "th", "tk", "tl", "tr", "tt", "uk", "ur", "uz", "vi", "yi", "yo", "zh", } LANGUAGE_TO_ASR_CODE = { "afrikaans": "af", "amharic": "am", "arabic": "ar", "egyptian arabic": "ar", "bengali": "bn", "bangla": "bn", "bulgarian": "bg", "cantonese": "zh", "chinese": "zh", "creole": "en", "english": "en", "filipino": "tl", "french": "fr", "german": "de", "greek": "el", "hausa": "ha", "hindi": "hi", "igbo": "ig", "indonesian": "id", "italian": "it", "japanese": "ja", "korean": "ko", "malay": "ms", "malaysian": "ms", "mandarin": "zh", "persian": "fa", "farsi": "fa", "portuguese": "pt", "russian": "ru", "spanish": "es", "tagalog": "tl", "tamil": "ta", "telugu": "te", "thai": "th", "turkish": "tr", "ukrainian": "uk", "urdu": "ur", "vietnamese": "vi", "yoruba": "yo", } DIALECT_TO_LANGUAGE = { "aave": "English", "african american vernacular english": "English", "american english": "English", "british english": "English", "chinese english": "English", "filipino english": "English", "ghanaian pidgin english": "English", "indian english": "English", "indonesian english": "English", "jamaican patois": "English", "korean english": "English", "malaysian english": "English", "naija pidgin": "English", "nigerian english": "English", "nigerian pidgin english": "English", "pakistani english": "English", "singaporean english": "English", "spanish creole": "English", "standard": "English", "standard english": "English", "egyptian arabic": "Arabic", "cretan greek": "Greek", "kano": "Hausa", "kano hausa": "Hausa", "katsina": "Hausa", "sokoto": "Hausa", "zaria": "Hausa", "chungcheong satoori": "Korean", "gangwon satoori": "Korean", "gyeongsang satoori": "Korean", "jeju dialect": "Korean", "jeju korean": "Korean", "jeju satoori": "Korean", "jeolla satoori": "Korean", "seoul dialect": "Korean", "asusu igbo": "Igbo", "igbo kolo": "Igbo", "izugbe": "Igbo", "onicha": "Igbo", } def _normalize(value): text = str(value or "").strip() decomposed = unicodedata.normalize("NFKD", text) asciiish = "".join(ch for ch in decomposed if not unicodedata.combining(ch)) return re.sub(r"[^a-z0-9]+", " ", asciiish.lower()).strip() _LANGUAGE_LOOKUP = {_normalize(k): v for k, v in LANGUAGE_TO_ASR_CODE.items()} _DIALECT_LOOKUP = {_normalize(k): v for k, v in DIALECT_TO_LANGUAGE.items()} def parse_asr_hint(hint): if hint is None: return "", "" if isinstance(hint, (list, tuple)): language = str(hint[0] if len(hint) > 0 else "").strip() dialect = str(hint[1] if len(hint) > 1 else "").strip() return language, dialect text = str(hint).strip() if not text: return "", "" if text.startswith("{"): try: data = json.loads(text) language = data.get("language") or data.get("source_language") or data.get("sourceLang") or "" dialect = data.get("dialect") or data.get("source_dialect") or data.get("sourceDialect") or "" return str(language).strip(), str(dialect).strip() except Exception: pass if "|||" in text: language, dialect = (text.split("|||", 1) + [""])[:2] return language.strip(), dialect.strip() return "", text def infer_language_from_dialect(dialect): norm = _normalize(dialect) if not norm: return "" if norm in _DIALECT_LOOKUP: return _DIALECT_LOOKUP[norm] if any(token in norm for token in ("hausa", "kano", "sokoto", "zaria", "katsina")): return "Hausa" if any(token in norm for token in ("korean", "satoori", "saturi", "seoul", "jeju", "jeolla", "gyeongsang", "chungcheong", "gangwon")): return "Korean" if any(token in norm for token in ("arabic", "egyptian", "levantine", "gulf")): return "Arabic" if any(token in norm for token in ("igbo", "onicha", "izugbe", "asusu")): return "Igbo" if any(token in norm for token in ("english", "pidgin", "patois", "aave", "creole")): return "English" return "" def resolve_asr_language_code(language_or_hint="", dialect=""): parsed_language, parsed_dialect = parse_asr_hint(language_or_hint) source_language = parsed_language or "" source_dialect = parsed_dialect or str(dialect or "").strip() if not source_language: source_language = infer_language_from_dialect(source_dialect) norm_language = _normalize(source_language) asr_code = None if norm_language in SUPPORTED_ASR_CODES: asr_code = norm_language else: asr_code = _LANGUAGE_LOOKUP.get(norm_language) if asr_code not in SUPPORTED_ASR_CODES: asr_code = None return asr_code, source_language, source_dialect