src/asr_language.py

import json
import re
import unicodedata


SUPPORTED_ASR_CODES = {
    "af", "am", "ar", "as", "az", "ba", "be", "bg", "bn", "bo", "br", "bs",
    "ca", "cs", "cy", "da", "de", "el", "en", "es", "et", "eu", "fa", "fi",
    "fo", "fr", "gl", "gu", "ha", "haw", "he", "hi", "hr", "ht", "hu", "hy",
    "id", "is", "it", "ja", "jw", "ka", "kk", "km", "kn", "ko", "la", "lb",
    "ln", "lo", "lt", "lv", "mg", "mi", "mk", "ml", "mn", "mr", "ms", "mt",
    "my", "ne", "nl", "nn", "no", "oc", "pa", "pl", "ps", "pt", "ro", "ru",
    "sa", "sd", "si", "sk", "sl", "sn", "so", "sq", "sr", "su", "sv", "sw",
    "ta", "te", "tg", "th", "tk", "tl", "tr", "tt", "uk", "ur", "uz", "vi",
    "yi", "yo", "zh",
}

LANGUAGE_TO_ASR_CODE = {
    "afrikaans": "af",
    "amharic": "am",
    "arabic": "ar",
    "egyptian arabic": "ar",
    "bengali": "bn",
    "bangla": "bn",
    "bulgarian": "bg",
    "cantonese": "zh",
    "chinese": "zh",
    "creole": "en",
    "english": "en",
    "filipino": "tl",
    "french": "fr",
    "german": "de",
    "greek": "el",
    "hausa": "ha",
    "hindi": "hi",
    "igbo": "ig",
    "indonesian": "id",
    "italian": "it",
    "japanese": "ja",
    "korean": "ko",
    "malay": "ms",
    "malaysian": "ms",
    "mandarin": "zh",
    "persian": "fa",
    "farsi": "fa",
    "portuguese": "pt",
    "russian": "ru",
    "spanish": "es",
    "tagalog": "tl",
    "tamil": "ta",
    "telugu": "te",
    "thai": "th",
    "turkish": "tr",
    "ukrainian": "uk",
    "urdu": "ur",
    "vietnamese": "vi",
    "yoruba": "yo",
}

DIALECT_TO_LANGUAGE = {
    "aave": "English",
    "african american vernacular english": "English",
    "american english": "English",
    "british english": "English",
    "chinese english": "English",
    "filipino english": "English",
    "ghanaian pidgin english": "English",
    "indian english": "English",
    "indonesian english": "English",
    "jamaican patois": "English",
    "korean english": "English",
    "malaysian english": "English",
    "naija pidgin": "English",
    "nigerian english": "English",
    "nigerian pidgin english": "English",
    "pakistani english": "English",
    "singaporean english": "English",
    "spanish creole": "English",
    "standard": "English",
    "standard english": "English",
    "egyptian arabic": "Arabic",
    "cretan greek": "Greek",
    "kano": "Hausa",
    "kano hausa": "Hausa",
    "katsina": "Hausa",
    "sokoto": "Hausa",
    "zaria": "Hausa",
    "chungcheong satoori": "Korean",
    "gangwon satoori": "Korean",
    "gyeongsang satoori": "Korean",
    "jeju dialect": "Korean",
    "jeju korean": "Korean",
    "jeju satoori": "Korean",
    "jeolla satoori": "Korean",
    "seoul dialect": "Korean",
    "asusu igbo": "Igbo",
    "igbo kolo": "Igbo",
    "izugbe": "Igbo",
    "onicha": "Igbo",
}


def _normalize(value):
    text = str(value or "").strip()
    decomposed = unicodedata.normalize("NFKD", text)
    asciiish = "".join(ch for ch in decomposed if not unicodedata.combining(ch))
    return re.sub(r"[^a-z0-9]+", " ", asciiish.lower()).strip()


_LANGUAGE_LOOKUP = {_normalize(k): v for k, v in LANGUAGE_TO_ASR_CODE.items()}
_DIALECT_LOOKUP = {_normalize(k): v for k, v in DIALECT_TO_LANGUAGE.items()}


def parse_asr_hint(hint):
    if hint is None:
        return "", ""

    if isinstance(hint, (list, tuple)):
        language = str(hint[0] if len(hint) > 0 else "").strip()
        dialect = str(hint[1] if len(hint) > 1 else "").strip()
        return language, dialect

    text = str(hint).strip()
    if not text:
        return "", ""

    if text.startswith("{"):
        try:
            data = json.loads(text)
            language = data.get("language") or data.get("source_language") or data.get("sourceLang") or ""
            dialect = data.get("dialect") or data.get("source_dialect") or data.get("sourceDialect") or ""
            return str(language).strip(), str(dialect).strip()
        except Exception:
            pass

    if "|||" in text:
        language, dialect = (text.split("|||", 1) + [""])[:2]
        return language.strip(), dialect.strip()

    return "", text


def infer_language_from_dialect(dialect):
    norm = _normalize(dialect)
    if not norm:
        return ""
    if norm in _DIALECT_LOOKUP:
        return _DIALECT_LOOKUP[norm]
    if any(token in norm for token in ("hausa", "kano", "sokoto", "zaria", "katsina")):
        return "Hausa"
    if any(token in norm for token in ("korean", "satoori", "saturi", "seoul", "jeju", "jeolla", "gyeongsang", "chungcheong", "gangwon")):
        return "Korean"
    if any(token in norm for token in ("arabic", "egyptian", "levantine", "gulf")):
        return "Arabic"
    if any(token in norm for token in ("igbo", "onicha", "izugbe", "asusu")):
        return "Igbo"
    if any(token in norm for token in ("english", "pidgin", "patois", "aave", "creole")):
        return "English"
    return ""


def resolve_asr_language_code(language_or_hint="", dialect=""):
    parsed_language, parsed_dialect = parse_asr_hint(language_or_hint)
    source_language = parsed_language or ""
    source_dialect = parsed_dialect or str(dialect or "").strip()

    if not source_language:
        source_language = infer_language_from_dialect(source_dialect)

    norm_language = _normalize(source_language)
    asr_code = None
    if norm_language in SUPPORTED_ASR_CODES:
        asr_code = norm_language
    else:
        asr_code = _LANGUAGE_LOOKUP.get(norm_language)

    if asr_code not in SUPPORTED_ASR_CODES:
        asr_code = None

    return asr_code, source_language, source_dialect