Spaces:
Running
Running
File size: 5,599 Bytes
160aacb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 | import json
import re
import unicodedata
SUPPORTED_ASR_CODES = {
"af", "am", "ar", "as", "az", "ba", "be", "bg", "bn", "bo", "br", "bs",
"ca", "cs", "cy", "da", "de", "el", "en", "es", "et", "eu", "fa", "fi",
"fo", "fr", "gl", "gu", "ha", "haw", "he", "hi", "hr", "ht", "hu", "hy",
"id", "is", "it", "ja", "jw", "ka", "kk", "km", "kn", "ko", "la", "lb",
"ln", "lo", "lt", "lv", "mg", "mi", "mk", "ml", "mn", "mr", "ms", "mt",
"my", "ne", "nl", "nn", "no", "oc", "pa", "pl", "ps", "pt", "ro", "ru",
"sa", "sd", "si", "sk", "sl", "sn", "so", "sq", "sr", "su", "sv", "sw",
"ta", "te", "tg", "th", "tk", "tl", "tr", "tt", "uk", "ur", "uz", "vi",
"yi", "yo", "zh",
}
LANGUAGE_TO_ASR_CODE = {
"afrikaans": "af",
"amharic": "am",
"arabic": "ar",
"egyptian arabic": "ar",
"bengali": "bn",
"bangla": "bn",
"bulgarian": "bg",
"cantonese": "zh",
"chinese": "zh",
"creole": "en",
"english": "en",
"filipino": "tl",
"french": "fr",
"german": "de",
"greek": "el",
"hausa": "ha",
"hindi": "hi",
"igbo": "ig",
"indonesian": "id",
"italian": "it",
"japanese": "ja",
"korean": "ko",
"malay": "ms",
"malaysian": "ms",
"mandarin": "zh",
"persian": "fa",
"farsi": "fa",
"portuguese": "pt",
"russian": "ru",
"spanish": "es",
"tagalog": "tl",
"tamil": "ta",
"telugu": "te",
"thai": "th",
"turkish": "tr",
"ukrainian": "uk",
"urdu": "ur",
"vietnamese": "vi",
"yoruba": "yo",
}
DIALECT_TO_LANGUAGE = {
"aave": "English",
"african american vernacular english": "English",
"american english": "English",
"british english": "English",
"chinese english": "English",
"filipino english": "English",
"ghanaian pidgin english": "English",
"indian english": "English",
"indonesian english": "English",
"jamaican patois": "English",
"korean english": "English",
"malaysian english": "English",
"naija pidgin": "English",
"nigerian english": "English",
"nigerian pidgin english": "English",
"pakistani english": "English",
"singaporean english": "English",
"spanish creole": "English",
"standard": "English",
"standard english": "English",
"egyptian arabic": "Arabic",
"cretan greek": "Greek",
"kano": "Hausa",
"kano hausa": "Hausa",
"katsina": "Hausa",
"sokoto": "Hausa",
"zaria": "Hausa",
"chungcheong satoori": "Korean",
"gangwon satoori": "Korean",
"gyeongsang satoori": "Korean",
"jeju dialect": "Korean",
"jeju korean": "Korean",
"jeju satoori": "Korean",
"jeolla satoori": "Korean",
"seoul dialect": "Korean",
"asusu igbo": "Igbo",
"igbo kolo": "Igbo",
"izugbe": "Igbo",
"onicha": "Igbo",
}
def _normalize(value):
text = str(value or "").strip()
decomposed = unicodedata.normalize("NFKD", text)
asciiish = "".join(ch for ch in decomposed if not unicodedata.combining(ch))
return re.sub(r"[^a-z0-9]+", " ", asciiish.lower()).strip()
_LANGUAGE_LOOKUP = {_normalize(k): v for k, v in LANGUAGE_TO_ASR_CODE.items()}
_DIALECT_LOOKUP = {_normalize(k): v for k, v in DIALECT_TO_LANGUAGE.items()}
def parse_asr_hint(hint):
if hint is None:
return "", ""
if isinstance(hint, (list, tuple)):
language = str(hint[0] if len(hint) > 0 else "").strip()
dialect = str(hint[1] if len(hint) > 1 else "").strip()
return language, dialect
text = str(hint).strip()
if not text:
return "", ""
if text.startswith("{"):
try:
data = json.loads(text)
language = data.get("language") or data.get("source_language") or data.get("sourceLang") or ""
dialect = data.get("dialect") or data.get("source_dialect") or data.get("sourceDialect") or ""
return str(language).strip(), str(dialect).strip()
except Exception:
pass
if "|||" in text:
language, dialect = (text.split("|||", 1) + [""])[:2]
return language.strip(), dialect.strip()
return "", text
def infer_language_from_dialect(dialect):
norm = _normalize(dialect)
if not norm:
return ""
if norm in _DIALECT_LOOKUP:
return _DIALECT_LOOKUP[norm]
if any(token in norm for token in ("hausa", "kano", "sokoto", "zaria", "katsina")):
return "Hausa"
if any(token in norm for token in ("korean", "satoori", "saturi", "seoul", "jeju", "jeolla", "gyeongsang", "chungcheong", "gangwon")):
return "Korean"
if any(token in norm for token in ("arabic", "egyptian", "levantine", "gulf")):
return "Arabic"
if any(token in norm for token in ("igbo", "onicha", "izugbe", "asusu")):
return "Igbo"
if any(token in norm for token in ("english", "pidgin", "patois", "aave", "creole")):
return "English"
return ""
def resolve_asr_language_code(language_or_hint="", dialect=""):
parsed_language, parsed_dialect = parse_asr_hint(language_or_hint)
source_language = parsed_language or ""
source_dialect = parsed_dialect or str(dialect or "").strip()
if not source_language:
source_language = infer_language_from_dialect(source_dialect)
norm_language = _normalize(source_language)
asr_code = None
if norm_language in SUPPORTED_ASR_CODES:
asr_code = norm_language
else:
asr_code = _LANGUAGE_LOOKUP.get(norm_language)
if asr_code not in SUPPORTED_ASR_CODES:
asr_code = None
return asr_code, source_language, source_dialect
|