Release prep: Phase 3 LM (checkpoints/phase3/p3_lm_full), versions/voices/settings, auto-tagger, Gradio app, CC-BY-SA-4.0 weights + Apache-2.0 code, model card
85bc603 verified | #!/usr/bin/env python | |
| """Dynamic Accent auto-tagger: word/span-level language ID for untagged mixed Nigerian text. | |
| Labels: eng (English), pcm (Nigerian Pidgin), ibo (Igbo), yor (Yoruba), hau (Hausa). | |
| Output is the tagged text infer.py expects, e.g. | |
| "omo, I'm very excited" -> "<pcm>omo,</pcm> <eng>I'm very excited</eng>" | |
| How it decides (rule-based, no model needed): | |
| 1. letters: Yoruba-only (ẹ ṣ, tone-marked n/m), Igbo-only (ị ụ ṅ), Hausa-only (ɓ ɗ ƙ ƴ), shared ọ; | |
| 2. lexicons of frequent function words / greetings per language (diacritics folded for lookup); | |
| 3. Pidgin markers ("dey", "abi", "omo", "wetin", "abeg", "una", "no go", "e don", "know say", pronoun + "go" + verb ...): | |
| a clause of plain-ASCII words that contains a Pidgin marker becomes Pidgin as a whole; | |
| 4. clause context: ambiguous words ("na", "o", "ka", "ya", "ma", "ni" ...) and unknown words take the language of the | |
| clause they sit in, then of the neighbouring clause; plain ASCII with no evidence falls back to English. | |
| 5. optional: --lid glotlid adds clause-level scores from GlotLID v3 (cis-lmu/glotlid, fastText, Apache-2.0 + notices, | |
| 1.7 GB download, needs `pip install fasttext`), mainly to separate untagged Igbo/Yoruba/Hausa written without | |
| diacritics from English. | |
| Overrides: lang_map {"omo": "pcm"} forces a word; force_lang renders everything in one language. | |
| Always print the tagged text and let the user fix it: the tagger is a convenience, not ground truth. | |
| python tagger.py "Omo, this traffic no be here o. I go call you later." | |
| python tagger.py --lang_map "omo=pcm,sha=pcm" "..." | |
| """ | |
| import argparse, json, re, sys, unicodedata | |
| LANGS = ["eng", "pcm", "ibo", "yor", "hau"] | |
| NG = ("ibo", "yor", "hau") | |
| TAGGED = re.compile(r"<(eng|ibo|hau|yor|pcm)>") | |
| def fold(w): | |
| """lowercase, NFD, drop every combining mark (tones, dot below, dot above), map hooked letters.""" | |
| w = unicodedata.normalize("NFD", w.lower()) | |
| w = "".join(c for c in w if not unicodedata.combining(c)) | |
| return w.translate(str.maketrans({"ɓ": "b", "ɗ": "d", "ƙ": "k", "ƴ": "y", "’": "'", "ʼ": "'", "‘": "'"})) | |
| def _set(s): | |
| return {fold(x) for x in s.split()} | |
| def _exact(s): | |
| return {unicodedata.normalize("NFC", x.lower()) for x in s.split()} | |
| # --- lexicons (frequent, high-precision words; folded). Words shared across languages are listed in each. | |
| _RAW = { | |
| "ibo": """ | |
| bụ nke ndị ihe ụlọ mmiri kedu ndewo nụ ụmụ ụmụnna nna nne anyị unu gị ha ọ ya ka ma | |
| nwere ga ji dị ọma biko daalụ dalu ụtọ obi ike nri ego akwụkwọ ọrụ maka ebe otu ọzọ ugbu | |
| a m n' na mere meela kwa kwesịrị ịga ịbịa bịa gaa nyere aka mmadụ obodo nwanne nwa nwoke nwanyị | |
| ụbọchị echi taa ụnyaahụ abalị ututu ehihie mgbede oge afọ ọnwa izu ukwuu nnukwu ntakịrị | |
| mana ọbụna ọbụla ruo site gbasara ebee onye gịnị olee ole kpọmkwem eziokwu nso anọ abụọ atọ | |
| ise isii asaa asatọ itoolu iri mbụ zuru ezu ahụ ahụla ịhụ hụ chere chọrọ chọ mara amaghị | |
| nọ nọrọ bịara gara kwuru kwuo sị siri esi na-esi na-eme eme mee eji ogologo ọkụ ụzọ ahịa ụka | |
| ọnụ ụtụtụ ka ọ dị mma mma nnọọ kpọọ zụta re ree rie riri ụmụaka ezinụlọ chi chukwu ndụ | |
| """, | |
| "yor": """ | |
| ni ti si sí àti àwọn mo o wa wọn kò ṣé jẹ́ ọjọ́ ẹ káàbọ̀ ilé ìlú ọmọ púpọ̀ dáadáa bẹ́ẹ̀ ni | |
| náà yìí kan mi wa yín rẹ̀ rẹ tirẹ̀ fún láti pé ń máa yóò ó a jẹ ṣe lọ wá rí fẹ́ mọ̀ sọ gbọ́ | |
| ẹ̀kọ́ iṣẹ́ owó ọjà òní ọ̀la àná ìpàdé padà ilé-ìwé ọjọ́ ajé ìsinmi alẹ́ àárọ̀ káàárọ̀ káàsán | |
| káalẹ́ ẹ kú ṣé gan an pàdé ìyá bàbá ẹ̀gbọ́n àbúrò ọ̀rẹ́ ènìyàn èèyàn àgbà ọba ìjọba | |
| nítorí ṣùgbọ́n tàbí bí kí kì ìgbà ibi ibo kílódé báwo kíni ta tani mélòó ọ̀kan méjì mẹ́ta | |
| mẹ́rin márùn-ún ẹ̀ẹ̀kan dúpẹ́ ẹ ṣeun ẹ̀ jọ̀ọ́ jọ̀ọ́ oúnjẹ omi ilẹ̀ ojú ọwọ́ ẹsẹ̀ | |
| """, | |
| "hau": """ | |
| da ba ya ta ka ki su mu na ne ce ina yaya barka kuke don haka za idan mai wannan wancan yana | |
| tana suna muna kuna sannu lafiya kwana aiki gobe yau jiya ruwa sai amma kuma shi ita nan | |
| wani wata abin zan zai zata zamu mutane gida kudi gode sosai tafi fita zuwa laima yamma safe | |
| rana dare mako shekara wata yanzu kai ke ku mun sun ban bai bata sani so iya yi yin | |
| farashin tumatur tashi kasuwa abinci ruwan sha babu akwai kamar saboda domin tare har tun | |
| daga cikin waje gaba baya sama kasa me wane wace ina yaushe nawa daya biyu uku hudu biyar | |
| """, | |
| "pcm": """ | |
| dey abi omo wetin abeg una wahala sabi oga sef sha oya jare ehn ehen shey kpele pikin | |
| chop comot waka belle palava wan wey dem don am siddon gist gbam yawa kampe sharp sharp | |
| nawa wahala kolo japa ginger vex dash ogbonge padi paddy shine eye | |
| """, | |
| "eng": """ | |
| the a an and or but if then so because of to in on at for with from by about as into over | |
| after before between through during without under again further here there when where why how | |
| all any both each few more most other some such no nor not only own same than too very can | |
| will just should now i me my we our you your he him his she her it its they them their what | |
| which who whom this that these those am is are was were be been being have has had having do | |
| does did doing would could might must shall may please thank thanks yes hello hi good morning | |
| evening afternoon night today tomorrow yesterday week year time day people man woman child | |
| children team meeting report school market road weather rain safe everyone everybody | |
| new old big small great right left back home work job money food water house car family | |
| friend friends know think say said tell told make made take took get got give gave go went | |
| come came see saw look want need like love help call send sent finish start stop keep let put | |
| read write talk speak discuss three two one four five six seven eight nine ten first second | |
| last next soon late early ready sure also still really very much many lot thing things way | |
| out up down off well done going coming doing nothing something everything anything model | |
| """, | |
| } | |
| LEX = {l: _set(v) for l, v in _RAW.items()} # folded (diacritics dropped) for lookup | |
| LEX_EXACT = {l: _exact(v) for l, v in _RAW.items()} # as written | |
| _N_LEX = {} | |
| for _l, _v in LEX.items(): | |
| for _w in _v: | |
| _N_LEX[_w] = _N_LEX.get(_w, 0) + 1 # >1 = word shared by several languages (na, o, ka, ya ...) | |
| # words that are Pidgin markers on their own (strong evidence) | |
| PCM_STRONG = _set("dey abi omo wetin abeg una wahala sabi sef oya jare shey pikin chop comot waka palava wan wey dem siddon vex japa ehn ehen") | |
| PCM_BIGRAMS = {("no", "go"), ("no", "be"), ("no", "fit"), ("no", "dey"), ("e", "don"), ("e", "dey"), ("e", "no"), | |
| ("know", "say"), ("tell", "am"), ("well", "well"), ("how", "far"), ("how", "body"), ("na", "so"), | |
| ("make", "we"), ("make", "i"), ("make", "una"), ("make", "dem"), ("i", "no"), ("we", "no"), | |
| ("you", "no"), ("dem", "no"), ("no", "easy"), ("no", "forget"), ("no", "vex"), ("fit", "heavy"), | |
| ("don", "come"), ("don", "go"), ("for", "inside"), ("una", "well"), ("e", "go"), ("na", "you"), | |
| ("na", "me"), ("na", "im"), ("na", "dem"), ("na", "una"), ("give", "you"), ("lie", "give")} | |
| PRON = _set("i we you dem una e im him she he they") | |
| NOT_VERB_AFTER_GO = _set("to home there back out away on in up down off ahead") | |
| PCM_AMBIG_END = _set("o") # clause-final particle "o" in an ASCII clause -> Pidgin evidence | |
| YOR_CH = set("ẹṣ") # after NFC lowercase; also e/s + U+0323 | |
| IBO_CH = set("ịụṅ") | |
| HAU_CH = set("ɓɗƙƴ") | |
| WORD = re.compile(r"[^\W\d_](?:[^\W\d_]|['’ʼ\-\u0300-\u036f])*", re.U) | |
| CLAUSE_END = re.compile(r"[,;:.?!]") | |
| def char_scores(w): | |
| s = dict.fromkeys(LANGS, 0.0) | |
| nfd = unicodedata.normalize("NFD", w.lower()) | |
| base = None | |
| for c in nfd: | |
| if unicodedata.combining(c): | |
| if c == "\u0323": # dot below | |
| if base in "es": | |
| s["yor"] += 3 | |
| elif base in "iu": | |
| s["ibo"] += 3 | |
| elif base == "o": | |
| s["yor"] += 1; s["ibo"] += 1 | |
| elif c == "\u0307" and base == "n": | |
| s["ibo"] += 3 | |
| elif c in "\u0300\u0301\u0304": # tones | |
| if base in "nm": | |
| s["yor"] += 2 | |
| else: | |
| s["yor"] += 0.8; s["ibo"] += 0.3 | |
| continue | |
| base = c | |
| if c in HAU_CH: | |
| s["hau"] += 4 | |
| if not any(s.values()) and re.fullmatch(r"[a-z'\-]+", fold(w)): | |
| s["ascii"] = 1 | |
| return s | |
| def word_scores(w): | |
| s = char_scores(w) | |
| ascii_ = s.pop("ascii", 0) | |
| f = fold(w) | |
| parts = [f] + (f.split("-") if "-" in f else []) | |
| if f.startswith("n'") or re.match(r"^na-[a-z]", f): | |
| s["ibo"] += 3 | |
| ex = unicodedata.normalize("NFC", w.lower()) | |
| for p in parts: | |
| amb = 0.5 if _N_LEX.get(p, 0) > 1 else 1.0 # shared words: let the clause decide | |
| for l in LANGS: | |
| if p in LEX[l]: | |
| w_ = 1.5 if (p == f and ex in LEX_EXACT[l]) or ascii_ and p in LEX_EXACT[l] else 0.7 | |
| s[l] += amb * (w_ if l != "pcm" else 1.0) | |
| if f in PCM_STRONG and ascii_: # Pidgin markers are plain ASCII ("ọmọ" is Yoruba) | |
| s["pcm"] += 4 | |
| # orthography hints (weak) | |
| if ascii_: | |
| if re.search(r"(th|wh|ck|tion|ing$|ould$|ght)", f): | |
| s["eng"] += 1 | |
| if re.match(r"^(nw|kw|gw|mm|nn|nk|ng[a-z])", f) and f not in LEX["eng"]: | |
| s["ibo"] += 0.7 | |
| if re.search(r"(ts|^ky|^'y|aa|ii|uu)", f) and f not in LEX["eng"]: | |
| s["hau"] += 0.5 | |
| return s, bool(ascii_) | |
| def tokenize(text): | |
| """-> list of (start, end, word) for words; everything else is glue.""" | |
| return [(m.start(), m.end(), m.group(0)) for m in WORD.finditer(text)] | |
| def clauses(text, toks): | |
| """clause index for each word: a new clause starts after , ; : . ? !""" | |
| ends = [m.start() for m in CLAUSE_END.finditer(text)] | |
| out, k = [], 0 | |
| for a, b, w in toks: | |
| while k < len(ends) and ends[k] < a: | |
| k += 1 | |
| out.append(k) | |
| return out | |
| def _norm(d): | |
| t = sum(max(0.0, v) for v in d.values()) | |
| return {k: (max(0.0, v) / t if t else 0.0) for k, v in d.items()} | |
| class Tagger: | |
| def __init__(self, lid=None, lang_map=None): | |
| self.lang_map = {fold(k): v for k, v in (lang_map or {}).items()} | |
| self.lid = None | |
| if lid == "glotlid": | |
| import fasttext | |
| from huggingface_hub import hf_hub_download | |
| self.lid = fasttext.load_model(hf_hub_download("cis-lmu/glotlid", "model.bin")) | |
| def _lid_scores(self, text): | |
| labs, ps = self.lid.predict(text.replace("\n", " "), k=20) | |
| m = {"__label__eng_Latn": "eng", "__label__pcm_Latn": "pcm", "__label__ibo_Latn": "ibo", | |
| "__label__yor_Latn": "yor", "__label__hau_Latn": "hau"} | |
| s = dict.fromkeys(LANGS, 0.0) | |
| for l, p in zip(labs, ps): | |
| if l in m: | |
| s[m[l]] += float(p) | |
| return s | |
| def word_langs(self, text): | |
| toks = tokenize(text) | |
| if not toks: | |
| return toks, [] | |
| cl = clauses(text, toks) | |
| n = len(toks) | |
| sc, asc = zip(*[word_scores(w) for _, _, w in toks]) | |
| sc = [dict(s) for s in sc] | |
| f = [fold(w) for _, _, w in toks] | |
| # Pidgin bigrams and "pronoun go verb" | |
| for i in range(n - 1): | |
| if cl[i] != cl[i + 1]: | |
| continue | |
| if (f[i], f[i + 1]) in PCM_BIGRAMS: | |
| sc[i]["pcm"] += 2.5; sc[i + 1]["pcm"] += 2.5 | |
| if i + 2 < n and f[i] in PRON and f[i + 1] == "go" and cl[i + 2] == cl[i] and f[i + 2] not in NOT_VERB_AFTER_GO \ | |
| and f[i + 2] in LEX["eng"] | LEX["pcm"]: | |
| for j in (i, i + 1, i + 2): | |
| sc[j]["pcm"] += 2.5 | |
| # clause-level evidence | |
| C = sorted(set(cl)) | |
| cs = {c: dict.fromkeys(LANGS, 0.0) for c in C} | |
| for i in range(n): | |
| for l in LANGS: | |
| cs[cl[i]][l] += sc[i][l] | |
| # clause-final "o" in an ASCII clause = Pidgin particle (Yoruba "o" clauses carry other Yoruba evidence) | |
| for i in range(n): | |
| last = i == n - 1 or cl[i + 1] != cl[i] | |
| if last and f[i] in PCM_AMBIG_END and asc[i]: | |
| others = [j for j in range(n) if cl[j] == cl[i] and j != i] | |
| if others and all(asc[j] for j in others) and cs[cl[i]]["yor"] < 2: | |
| sc[i]["pcm"] += 3; cs[cl[i]]["pcm"] += 3 | |
| if self.lid is not None: | |
| for c in C: | |
| ws = [toks[i][2] for i in range(n) if cl[i] == c] | |
| for l, v in self._lid_scores(" ".join(ws)).items(): | |
| cs[c][l] += 2.0 * v | |
| out = [] | |
| for i in range(n): | |
| c = cl[i] | |
| ctx = _norm(cs[c]) | |
| nb = dict.fromkeys(LANGS, 0.0) # neighbouring clauses, weaker | |
| for d in (-1, 1): | |
| if c + d in cs: | |
| for l, v in _norm(cs[c + d]).items(): | |
| nb[l] += 0.3 * v | |
| tot = {l: sc[i][l] + 1.5 * ctx[l] + nb[l] for l in LANGS} | |
| # plain-ASCII words in a clause without Nigerian-language evidence: English or Pidgin | |
| if asc[i] and max(sc[i][l] for l in NG) < 1.0: | |
| ng_ctx = max(cs[c][l] for l in NG) | |
| if ng_ctx < 1.5 * max(cs[c]["eng"], cs[c]["pcm"], 0.1): | |
| pc = cs[c]["pcm"] >= 2.5 | |
| out.append("pcm" if pc else ("eng" if sc[i]["pcm"] < 4 else "pcm")) | |
| continue | |
| best = max(tot, key=tot.get) | |
| if tot[best] <= 0: | |
| best = "eng" | |
| out.append(best) | |
| for i in range(n): # user overrides | |
| if f[i] in self.lang_map: | |
| out[i] = self.lang_map[f[i]] | |
| return toks, out | |
| def tag(self, text, force_lang=None): | |
| text = unicodedata.normalize("NFC", text.strip()) | |
| if TAGGED.search(text): | |
| return text # already tagged: leave alone | |
| if force_lang: | |
| return f"<{force_lang}>{text}</{force_lang}>" | |
| toks, langs = self.word_langs(text) | |
| if not toks: | |
| return f"<eng>{text}</eng>" | |
| # spans: consecutive same-language words; glue (spaces/punct) stays with the preceding word | |
| spans, cur_l, a = [], langs[0], 0 | |
| for i in range(1, len(toks)): | |
| if langs[i] != cur_l: | |
| b = toks[i][0] | |
| seg = text[a:b] | |
| spans.append((cur_l, seg)) | |
| cur_l, a = langs[i], b | |
| spans.append((cur_l, text[a:])) | |
| lead = text[:toks[0][0]] | |
| out = [] | |
| for k, (l, s) in enumerate(spans): | |
| s2 = (lead + s) if k == 0 else s | |
| core = s2.rstrip() | |
| out.append(f"<{l}>{core}</{l}>") | |
| return " ".join(out) | |
| def parse_map(s): | |
| m = {} | |
| for kv in (s or "").split(","): | |
| if "=" in kv: | |
| k, v = kv.split("=", 1) | |
| v = v.strip() | |
| if v not in LANGS: | |
| raise ValueError(f"--lang_map: unknown language '{v}' (use {LANGS})") | |
| m[k.strip()] = v | |
| return m | |
| def spans_of(tagged): | |
| return [(m.group(1), m.group(2)) for m in re.finditer(r"<(eng|ibo|hau|yor|pcm)>(.*?)</\1>", tagged, re.S)] | |
| def word_labels(tagged): | |
| """gold/pred word-level labels from a tagged string.""" | |
| out = [] | |
| for l, s in spans_of(tagged): | |
| out += [(fold(w), l) for _, _, w in tokenize(s)] | |
| return out | |
| def strip_tags(tagged): | |
| return re.sub(r"\s+", " ", re.sub(r"</?(eng|ibo|hau|yor|pcm)>", "", tagged)).strip() | |
| def main(): | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("text") | |
| ap.add_argument("--lang_map", default="", help='per-word overrides, e.g. "omo=pcm,sha=pcm"') | |
| ap.add_argument("--force_lang", choices=LANGS) | |
| ap.add_argument("--lid", choices=["none", "glotlid"], default="none") | |
| a = ap.parse_args() | |
| t = Tagger(None if a.lid == "none" else a.lid, parse_map(a.lang_map)) | |
| print(t.tag(a.text, a.force_lang)) | |
| if __name__ == "__main__": | |
| main() | |