Spaces:
Paused
Paused
| import gradio as gr | |
| from transformers import AutoTokenizer, AutoModelForSequenceClassification, pipeline | |
| from huggingface_hub import hf_hub_download | |
| import fasttext # provided by the `fasttext-predict` package (see requirements.txt) — | |
| # ships prebuilt wheels for py3.13, unlike fasttext-wheel which fails | |
| # to compile from source on newer GCC (missing <cstdint> include upstream) | |
| # --------------------------------------------------------------------------- | |
| # Model registry — all LID models published under olaverse. | |
| # Two different underlying formats: | |
| # - "transformers": standard AutoTokenizer + AutoModelForSequenceClassification | |
| # - "fasttext": a .bin file loaded directly with the fasttext library | |
| # (lid-lite-25 is NOT a transformers model — it's a character-n-gram | |
| # linear classifier, hence no tokenizer/config to load via AutoTokenizer) | |
| # Lazy-loaded on first use so the Space doesn't need to hold all of these in | |
| # memory (or pay startup time for all of them) before anyone's clicked anything. | |
| # --------------------------------------------------------------------------- | |
| MODELS = { | |
| "lid-neural-5 (Nigerian, 4 langs)": { | |
| "type": "transformers", | |
| "repo": "olaverse/lid-neural-5", | |
| "note": "Yoruba, Hausa, Igbo, Nigerian Pidgin", | |
| }, | |
| "lid-neural-5.1 (Nigerian, 4 langs, sentence-level)": { | |
| "type": "transformers", | |
| "repo": "olaverse/lid-neural-5.1", | |
| "note": "Hausa, Yoruba, Igbo, Nigerian Pidgin — built on mist-encoder-base-ng, tuned for short/sentence-level text (97.6% acc)", | |
| }, | |
| "lid-lite-25 (fastText, passages)": { | |
| "type": "fasttext", | |
| "repo": "olaverse/lid-lite-25", | |
| "filename": "passages.bin", | |
| "note": "25 languages, fastText character n-gram model, tuned for long-form passages", | |
| }, | |
| "lid-lite-25 (fastText, short queries)": { | |
| "type": "fasttext", | |
| "repo": "olaverse/lid-lite-25", | |
| "filename": "questions.bin", | |
| "note": "25 languages, fastText character n-gram model, tuned for short questions/queries", | |
| }, | |
| "lid-neural-25.1 (XLM-R, passages)": { | |
| "type": "transformers", | |
| "repo": "olaverse/lid-neural-25.1", | |
| "note": "25 languages, tuned for long-form text", | |
| }, | |
| "lid-neural-25.2 (XLM-R, short queries)": { | |
| "type": "transformers", | |
| "repo": "olaverse/lid-neural-25.2", | |
| "note": "25 languages, tuned for short questions/queries", | |
| }, | |
| } | |
| LANGUAGE_NAMES = { | |
| # Nigerian 4-lang models (lid-neural-5, lid-neural-5.1) | |
| "yor": "Yoruba", "hau": "Hausa", "ibo": "Igbo", "pcm": "Nigerian Pidgin", | |
| # 25-lang models (lid-lite-25, lid-neural-25.1/.2) — actual output labels | |
| # are ISO 639-3 three-letter codes, NOT the ISO 639-1 codes in the HF | |
| # `language:` YAML tag. Mapping the wrong code format meant unmatched | |
| # predictions fell back to showing the raw code instead of a name. | |
| "afr": "Afrikaans", "amh": "Amharic", "deu": "German", "eng": "English", | |
| "fra": "French", "hin": "Hindi", "ind": "Indonesian", "ita": "Italian", | |
| "jpn": "Japanese", "kor": "Korean", "nld": "Dutch", "pol": "Polish", | |
| "por": "Portuguese", "rus": "Russian", "sna": "Shona", "som": "Somali", | |
| "spa": "Spanish", "swa": "Swahili", "tur": "Turkish", "vie": "Vietnamese", | |
| "xho": "Xhosa", "zul": "Zulu", | |
| } | |
| import gc | |
| class FastTextWrapper: | |
| """Mimics the shape of a transformers text-classification pipeline output | |
| ([[{'label': ..., 'score': ...}, ...]]) so downstream code doesn't need | |
| to branch on model type.""" | |
| def __init__(self, ft_model): | |
| self._model = ft_model | |
| def __call__(self, text: str, top_k: int = 5): | |
| clean = text.replace("\n", " ").strip() | |
| labels, probs = self._model.predict(clean, k=top_k) | |
| results = [ | |
| {"label": label.replace("__label__", ""), "score": float(prob)} | |
| for label, prob in zip(labels, probs) | |
| ] | |
| return [results] | |
| # Only ONE model is kept in memory at a time. Loading a new one evicts | |
| # whatever was previously cached. This trades re-download/reload time on | |
| # every model switch for a flat, predictable memory footprint — important | |
| # on a free/quota-limited CPU Space running several separate checkpoints. | |
| _CACHED_LABEL = None | |
| _CACHED_PIPELINE = None | |
| def _unload_current(): | |
| global _CACHED_LABEL, _CACHED_PIPELINE | |
| if _CACHED_PIPELINE is not None: | |
| del _CACHED_PIPELINE | |
| _CACHED_PIPELINE = None | |
| _CACHED_LABEL = None | |
| gc.collect() | |
| def get_pipeline(model_label: str): | |
| global _CACHED_LABEL, _CACHED_PIPELINE | |
| if _CACHED_LABEL == model_label and _CACHED_PIPELINE is not None: | |
| return _CACHED_PIPELINE | |
| # Evict whatever's currently loaded before loading the new one. | |
| _unload_current() | |
| entry = MODELS[model_label] | |
| if entry["type"] == "fasttext": | |
| local_path = hf_hub_download(repo_id=entry["repo"], filename=entry["filename"]) | |
| ft_model = fasttext.load_model(local_path) | |
| pipe = FastTextWrapper(ft_model) | |
| else: | |
| repo = entry["repo"] | |
| tok = AutoTokenizer.from_pretrained(repo) | |
| model = AutoModelForSequenceClassification.from_pretrained(repo) | |
| pipe = pipeline("text-classification", model=model, tokenizer=tok, top_k=5) | |
| _CACHED_LABEL = model_label | |
| _CACHED_PIPELINE = pipe | |
| return pipe | |
| def detect_language(text: str, model_label: str, progress=gr.Progress()): | |
| if not text or not text.strip(): | |
| return "Please enter some text." | |
| progress(0.2, desc="Running..." if _CACHED_LABEL == model_label else f"Loading {model_label}...") | |
| try: | |
| pipe = get_pipeline(model_label) | |
| except Exception as e: | |
| return f"⚠️ Could not load `{MODELS[model_label]['repo']}`: {e}" | |
| progress(0.7, desc="Classifying...") | |
| results = pipe(text) | |
| top = results[0] if isinstance(results[0], list) else results | |
| output = f"## 🇳🇬 Results — `{model_label}`\n\n" | |
| output += f"*{MODELS[model_label]['note']}*\n\n" | |
| for i, r in enumerate(top): | |
| code = r["label"] | |
| score = r["score"] | |
| name = LANGUAGE_NAMES.get(code, code) | |
| bar = "█" * int(score * 20) | |
| emoji = "🥇" if i == 0 else "🥈" if i == 1 else "🥉" if i == 2 else " " | |
| output += f"{emoji} **{name}** `{code}` — {score*100:.1f}% {bar}\n\n" | |
| return output | |
| def compare_all(text: str, progress=gr.Progress()): | |
| """Runs text through each model in turn. Since only one model is cached | |
| at a time, this reloads each checkpoint sequentially (slower than if all | |
| four were held in memory), trading speed for a flat memory footprint.""" | |
| if not text or not text.strip(): | |
| return "Please enter some text." | |
| sections = [] | |
| labels = list(MODELS.keys()) | |
| for i, label in enumerate(labels): | |
| progress((i + 1) / len(labels), desc=f"Running {label}...") | |
| try: | |
| pipe = get_pipeline(label) | |
| results = pipe(text) | |
| top = (results[0] if isinstance(results[0], list) else results)[:3] | |
| lines = [f"### `{label}`"] | |
| for r in top: | |
| code = r["label"] | |
| score = r["score"] | |
| name = LANGUAGE_NAMES.get(code, code) | |
| lines.append(f"- **{name}** `{code}` — {score*100:.1f}%") | |
| sections.append("\n".join(lines)) | |
| except Exception as e: | |
| sections.append(f"### `{label}`\n⚠️ Could not load: {e}") | |
| return "\n\n".join(sections) | |
| with gr.Blocks() as demo: | |
| gr.Markdown(""" | |
| # 🇳🇬 Nigerian & Multilingual Language Identifier | |
| All language-identification models by [olaverse](https://huggingface.co/olaverse), in one Space. | |
| Pick a single model to test, or run your text through all of them at once to compare. | |
| """) | |
| with gr.Tab("Single model"): | |
| with gr.Row(): | |
| with gr.Column(): | |
| model_dd = gr.Dropdown( | |
| choices=list(MODELS.keys()), | |
| value=list(MODELS.keys())[0], | |
| label="Model", | |
| ) | |
| text_input = gr.Textbox( | |
| label="Enter text", | |
| placeholder="Type or paste text in any supported language...", | |
| lines=4, | |
| ) | |
| gr.Examples( | |
| examples=[ | |
| ["Bawo ni, se daadaa ni?"], | |
| ["Sannu, yaya kake?"], | |
| ["Kedu, ọ dị mma?"], | |
| ["How you dey, e don do?"], | |
| ["Ẹ káàárọ̀, ẹ káàbọ̀"], | |
| ], | |
| inputs=text_input, | |
| ) | |
| detect_btn = gr.Button("🔍 Detect Language", variant="primary") | |
| with gr.Column(): | |
| output = gr.Markdown() | |
| detect_btn.click(fn=detect_language, inputs=[text_input, model_dd], outputs=output) | |
| text_input.submit(fn=detect_language, inputs=[text_input, model_dd], outputs=output) | |
| with gr.Tab("Compare all models"): | |
| gr.Markdown( | |
| "Runs the same text through all six model/checkpoint combinations and shows top-3 " | |
| "predictions from each. Only one model is kept in memory at a time, so this reloads " | |
| "each checkpoint in turn — expect it to take a bit longer than the single-model tab." | |
| ) | |
| compare_input = gr.Textbox( | |
| label="Enter text", | |
| placeholder="Type or paste text in any supported language...", | |
| lines=4, | |
| ) | |
| compare_btn = gr.Button("🔍 Compare All", variant="primary") | |
| compare_output = gr.Markdown() | |
| compare_btn.click(fn=compare_all, inputs=compare_input, outputs=compare_output) | |
| compare_input.submit(fn=compare_all, inputs=compare_input, outputs=compare_output) | |
| gr.Markdown(""" | |
| --- | |
| **Model notes:** | |
| - `lid-neural-5` — Nigerian-focused, 4 languages (Yoruba, Hausa, Igbo, Pidgin) | |
| - `lid-neural-5.1` — Nigerian-focused, same 4 languages, sentence-level tuned on `mist-encoder-base-ng` (97.6% acc; most residual error involves Pidgin, which shares vocabulary with the others) | |
| - `lid-lite-25` — fastText (character n-gram, CPU-only), 25 languages, two checkpoints (passages / short queries) | |
| - `lid-neural-25.1` / `.2` — XLM-R fine-tunes, 25 languages, tuned for passages vs. short queries respectively. | |
| Known limitation across both `-25` families: Zulu/Xhosa confusion on short text (see model cards). | |
| """) | |
| if __name__ == "__main__": | |
| demo.launch(theme=gr.themes.Soft()) |