import gradio as gr from transformers import AutoTokenizer, AutoModelForSequenceClassification, pipeline from huggingface_hub import hf_hub_download import fasttext # provided by the `fasttext-predict` package (see requirements.txt) — # ships prebuilt wheels for py3.13, unlike fasttext-wheel which fails # to compile from source on newer GCC (missing include upstream) # --------------------------------------------------------------------------- # Model registry — all LID models published under olaverse. # Two different underlying formats: # - "transformers": standard AutoTokenizer + AutoModelForSequenceClassification # - "fasttext": a .bin file loaded directly with the fasttext library # (lid-lite-25 is NOT a transformers model — it's a character-n-gram # linear classifier, hence no tokenizer/config to load via AutoTokenizer) # Lazy-loaded on first use so the Space doesn't need to hold all of these in # memory (or pay startup time for all of them) before anyone's clicked anything. # --------------------------------------------------------------------------- MODELS = { "lid-neural-5 (Nigerian, 4 langs)": { "type": "transformers", "repo": "olaverse/lid-neural-5", "note": "Yoruba, Hausa, Igbo, Nigerian Pidgin", }, "lid-neural-5.1 (Nigerian, 4 langs, sentence-level)": { "type": "transformers", "repo": "olaverse/lid-neural-5.1", "note": "Hausa, Yoruba, Igbo, Nigerian Pidgin — built on mist-encoder-base-ng, tuned for short/sentence-level text (97.6% acc)", }, "lid-lite-25 (fastText, passages)": { "type": "fasttext", "repo": "olaverse/lid-lite-25", "filename": "passages.bin", "note": "25 languages, fastText character n-gram model, tuned for long-form passages", }, "lid-lite-25 (fastText, short queries)": { "type": "fasttext", "repo": "olaverse/lid-lite-25", "filename": "questions.bin", "note": "25 languages, fastText character n-gram model, tuned for short questions/queries", }, "lid-neural-25.1 (XLM-R, passages)": { "type": "transformers", "repo": "olaverse/lid-neural-25.1", "note": "25 languages, tuned for long-form text", }, "lid-neural-25.2 (XLM-R, short queries)": { "type": "transformers", "repo": "olaverse/lid-neural-25.2", "note": "25 languages, tuned for short questions/queries", }, } LANGUAGE_NAMES = { # Nigerian 4-lang models (lid-neural-5, lid-neural-5.1) "yor": "Yoruba", "hau": "Hausa", "ibo": "Igbo", "pcm": "Nigerian Pidgin", # 25-lang models (lid-lite-25, lid-neural-25.1/.2) — actual output labels # are ISO 639-3 three-letter codes, NOT the ISO 639-1 codes in the HF # `language:` YAML tag. Mapping the wrong code format meant unmatched # predictions fell back to showing the raw code instead of a name. "afr": "Afrikaans", "amh": "Amharic", "deu": "German", "eng": "English", "fra": "French", "hin": "Hindi", "ind": "Indonesian", "ita": "Italian", "jpn": "Japanese", "kor": "Korean", "nld": "Dutch", "pol": "Polish", "por": "Portuguese", "rus": "Russian", "sna": "Shona", "som": "Somali", "spa": "Spanish", "swa": "Swahili", "tur": "Turkish", "vie": "Vietnamese", "xho": "Xhosa", "zul": "Zulu", } import gc class FastTextWrapper: """Mimics the shape of a transformers text-classification pipeline output ([[{'label': ..., 'score': ...}, ...]]) so downstream code doesn't need to branch on model type.""" def __init__(self, ft_model): self._model = ft_model def __call__(self, text: str, top_k: int = 5): clean = text.replace("\n", " ").strip() labels, probs = self._model.predict(clean, k=top_k) results = [ {"label": label.replace("__label__", ""), "score": float(prob)} for label, prob in zip(labels, probs) ] return [results] # Only ONE model is kept in memory at a time. Loading a new one evicts # whatever was previously cached. This trades re-download/reload time on # every model switch for a flat, predictable memory footprint — important # on a free/quota-limited CPU Space running several separate checkpoints. _CACHED_LABEL = None _CACHED_PIPELINE = None def _unload_current(): global _CACHED_LABEL, _CACHED_PIPELINE if _CACHED_PIPELINE is not None: del _CACHED_PIPELINE _CACHED_PIPELINE = None _CACHED_LABEL = None gc.collect() def get_pipeline(model_label: str): global _CACHED_LABEL, _CACHED_PIPELINE if _CACHED_LABEL == model_label and _CACHED_PIPELINE is not None: return _CACHED_PIPELINE # Evict whatever's currently loaded before loading the new one. _unload_current() entry = MODELS[model_label] if entry["type"] == "fasttext": local_path = hf_hub_download(repo_id=entry["repo"], filename=entry["filename"]) ft_model = fasttext.load_model(local_path) pipe = FastTextWrapper(ft_model) else: repo = entry["repo"] tok = AutoTokenizer.from_pretrained(repo) model = AutoModelForSequenceClassification.from_pretrained(repo) pipe = pipeline("text-classification", model=model, tokenizer=tok, top_k=5) _CACHED_LABEL = model_label _CACHED_PIPELINE = pipe return pipe def detect_language(text: str, model_label: str, progress=gr.Progress()): if not text or not text.strip(): return "Please enter some text." progress(0.2, desc="Running..." if _CACHED_LABEL == model_label else f"Loading {model_label}...") try: pipe = get_pipeline(model_label) except Exception as e: return f"⚠️ Could not load `{MODELS[model_label]['repo']}`: {e}" progress(0.7, desc="Classifying...") results = pipe(text) top = results[0] if isinstance(results[0], list) else results output = f"## 🇳🇬 Results — `{model_label}`\n\n" output += f"*{MODELS[model_label]['note']}*\n\n" for i, r in enumerate(top): code = r["label"] score = r["score"] name = LANGUAGE_NAMES.get(code, code) bar = "█" * int(score * 20) emoji = "🥇" if i == 0 else "🥈" if i == 1 else "🥉" if i == 2 else " " output += f"{emoji} **{name}** `{code}` — {score*100:.1f}% {bar}\n\n" return output def compare_all(text: str, progress=gr.Progress()): """Runs text through each model in turn. Since only one model is cached at a time, this reloads each checkpoint sequentially (slower than if all four were held in memory), trading speed for a flat memory footprint.""" if not text or not text.strip(): return "Please enter some text." sections = [] labels = list(MODELS.keys()) for i, label in enumerate(labels): progress((i + 1) / len(labels), desc=f"Running {label}...") try: pipe = get_pipeline(label) results = pipe(text) top = (results[0] if isinstance(results[0], list) else results)[:3] lines = [f"### `{label}`"] for r in top: code = r["label"] score = r["score"] name = LANGUAGE_NAMES.get(code, code) lines.append(f"- **{name}** `{code}` — {score*100:.1f}%") sections.append("\n".join(lines)) except Exception as e: sections.append(f"### `{label}`\n⚠️ Could not load: {e}") return "\n\n".join(sections) with gr.Blocks() as demo: gr.Markdown(""" # 🇳🇬 Nigerian & Multilingual Language Identifier All language-identification models by [olaverse](https://huggingface.co/olaverse), in one Space. Pick a single model to test, or run your text through all of them at once to compare. """) with gr.Tab("Single model"): with gr.Row(): with gr.Column(): model_dd = gr.Dropdown( choices=list(MODELS.keys()), value=list(MODELS.keys())[0], label="Model", ) text_input = gr.Textbox( label="Enter text", placeholder="Type or paste text in any supported language...", lines=4, ) gr.Examples( examples=[ ["Bawo ni, se daadaa ni?"], ["Sannu, yaya kake?"], ["Kedu, ọ dị mma?"], ["How you dey, e don do?"], ["Ẹ káàárọ̀, ẹ káàbọ̀"], ], inputs=text_input, ) detect_btn = gr.Button("🔍 Detect Language", variant="primary") with gr.Column(): output = gr.Markdown() detect_btn.click(fn=detect_language, inputs=[text_input, model_dd], outputs=output) text_input.submit(fn=detect_language, inputs=[text_input, model_dd], outputs=output) with gr.Tab("Compare all models"): gr.Markdown( "Runs the same text through all six model/checkpoint combinations and shows top-3 " "predictions from each. Only one model is kept in memory at a time, so this reloads " "each checkpoint in turn — expect it to take a bit longer than the single-model tab." ) compare_input = gr.Textbox( label="Enter text", placeholder="Type or paste text in any supported language...", lines=4, ) compare_btn = gr.Button("🔍 Compare All", variant="primary") compare_output = gr.Markdown() compare_btn.click(fn=compare_all, inputs=compare_input, outputs=compare_output) compare_input.submit(fn=compare_all, inputs=compare_input, outputs=compare_output) gr.Markdown(""" --- **Model notes:** - `lid-neural-5` — Nigerian-focused, 4 languages (Yoruba, Hausa, Igbo, Pidgin) - `lid-neural-5.1` — Nigerian-focused, same 4 languages, sentence-level tuned on `mist-encoder-base-ng` (97.6% acc; most residual error involves Pidgin, which shares vocabulary with the others) - `lid-lite-25` — fastText (character n-gram, CPU-only), 25 languages, two checkpoints (passages / short queries) - `lid-neural-25.1` / `.2` — XLM-R fine-tunes, 25 languages, tuned for passages vs. short queries respectively. Known limitation across both `-25` families: Zulu/Xhosa confusion on short text (see model cards). """) if __name__ == "__main__": demo.launch(theme=gr.themes.Soft())