| """© KAND CA 2026 - Arabic tokenizer comparison Space. |
| |
| A tokenizer is the cheapest thing to get wrong in an Arabic pipeline and the |
| hardest to see. Every model card quotes parameters and context length; almost |
| none tell you that the same Arabic paragraph costs 52% more context on one |
| tokenizer than another, which is a 52% tax on every prompt, every document and |
| every embedding you will ever run through it. |
| |
| This puts the number on screen. Paste Arabic - MSA, dialect, or code-switched - |
| and see how many tokens each tokenizer actually spends on it. |
| |
| The interesting comparison is not big-vocab vs small-vocab. Mistral and Emhotob |
| both have ~32K entries; Emhotob spends all of them on Arabic and Mistral spends |
| almost none, and the gap that opens up is the whole point. |
| """ |
| import html |
| import statistics |
|
|
| import gradio as gr |
| from transformers import AutoTokenizer |
|
|
| TOKENIZERS = [ |
| ("Emhotob 32K (Arabic-only)", "oddadmix/50M-2048-Emhotob", "Arabic-only"), |
| ("Gemma-4", "google/gemma-4-31B-it", "multilingual"), |
| ("Qwen3.6", "Qwen/Qwen3.6-27B", "multilingual"), |
| ("Qwen3.5", "Qwen/Qwen3.5-4B", "multilingual"), |
| ("Mistral-7B v0.3", "mistralai/Mistral-7B-v0.3", "western"), |
| ("GPT-2", "openai-community/gpt2", "western"), |
| ] |
|
|
| EXAMPLES = { |
| "فصحى — MSA news": ( |
| "أعلنت وزارة الاقتصاد أن معدل النمو المتوقع خلال العام المقبل سيبلغ نحو " |
| "أربعة في المئة، مدفوعاً بارتفاع الصادرات غير النفطية وتحسن أداء قطاع " |
| "السياحة، فيما أشار التقرير إلى أن الاستثمارات الأجنبية المباشرة سجلت " |
| "زيادة ملحوظة مقارنة بالفترة نفسها من العام الماضي."), |
| "مصري — Egyptian": ( |
| "يا جماعة أنا رايح السوق دلوقتي عشان أجيب شوية حاجات للبيت، لو محتاجين " |
| "حاجة قولولي بسرعة قبل ما أنزل. الأسعار غليت أوي الفترة دي وبقى لازم " |
| "الواحد يحسب حسابه كويس قبل ما يشتري أي حاجة."), |
| "مغربي — Moroccan Darija": ( |
| "واش نتا فاهم شنو كايقع؟ الأسعار طلعات بزاف هاد الشهر وكلشي كايشتكي. " |
| "غادي نمشي للسوق دابا باش نشوف شنو كاين، ولكن ما كنظنش غادي نلقى شي حاجة " |
| "رخيصة. الله يسهل علينا."), |
| "خليجي — Gulf": ( |
| "شلونك؟ أنا بروح الدوام بدري اليوم لأن عندي اجتماع مهم مع المدير. " |
| "بعدين بمر على المحل عشان أشتري أغراض البيت، وإذا خلصت بدري بجيك."), |
| "Code-switched": ( |
| "الـ deployment اتعمل امبارح على الـ production server بس فيه issue في " |
| "الـ latency، محتاجين نعمل profiling للـ database queries عشان نشوف " |
| "الـ bottleneck فين بالظبط."), |
| "English (control)": ( |
| "The ministry announced that the expected growth rate for the coming year " |
| "will reach approximately four percent, driven by rising non-oil exports " |
| "and improved performance in the tourism sector."), |
| } |
|
|
| _CACHE = {} |
|
|
|
|
| def get(repo): |
| if repo not in _CACHE: |
| _CACHE[repo] = AutoTokenizer.from_pretrained(repo) |
| return _CACHE[repo] |
|
|
|
|
| PALETTE = ["#dbeafe", "#fef3c7", "#dcfce7", "#fae8ff", "#ffe4e6", "#e0f2fe"] |
|
|
|
|
| def render_tokens(tk, ids): |
| """Colour each token so the segmentation is visible, not just counted. |
| |
| Tokens are decoded ONE ID AT A TIME rather than read off |
| `convert_ids_to_tokens`. For a byte-level BPE - which Emhotob, Gemma, Qwen |
| and GPT-2 all are - that method returns the byte-mangled form, so Arabic |
| comes back as 'أعÙĦÙĨت' instead of 'أعلنت'. Decoding per id gives the |
| real characters, which is the whole point of showing the split. |
| """ |
| out = [] |
| for i, tid in enumerate(ids): |
| s = tk.decode([tid]) |
| lead = s.startswith(" ") |
| s = html.escape(s.strip()) or "␣" |
| out.append( |
| f'<span style="background:{PALETTE[i % len(PALETTE)]};' |
| f'padding:3px 5px;margin:2px;border-radius:4px;' |
| f'display:inline-block;color:#111;font-size:15px;' |
| f'border-left:{"3px solid #94a3b8" if lead else "0"}">{s}</span>') |
| return ('<div dir="rtl" style="line-height:2.4;direction:rtl;' |
| 'text-align:right;padding:10px;background:#fafafa;' |
| 'border-radius:8px;border:1px solid #e5e7eb">' |
| + "".join(out) + "</div>") |
|
|
|
|
| def compare(text, show_for): |
| text = (text or "").strip() |
| if not text: |
| raise gr.Error("اكتب أو الصق نصاً عربياً أولاً. / Enter some text first.") |
| n_words = len(text.split()) |
| n_chars = len(text) |
|
|
| rows, counts = [], {} |
| for name, repo, kind in TOKENIZERS: |
| try: |
| tk = get(repo) |
| ids = tk(text, add_special_tokens=False)["input_ids"] |
| counts[name] = len(ids) |
| rows.append([name, kind, f"{tk.vocab_size:,}", len(ids), |
| round(len(ids) / max(n_words, 1), 3), |
| round(n_chars / max(len(ids), 1), 2)]) |
| except Exception as e: |
| rows.append([name, kind, "—", None, None, None]) |
|
|
| base = counts.get("Emhotob 32K (Arabic-only)") |
| for r in rows: |
| r.append(round(r[3] / base, 2) if (base and r[3]) else None) |
|
|
| ok = [r for r in rows if r[3]] |
| ok.sort(key=lambda r: r[3]) |
| best, worst = ok[0], ok[-1] |
| summary = ( |
| f"### {n_words} كلمة · {n_chars:,} حرف\n\n" |
| f"**{best[0]}** is most efficient at **{best[3]:,} tokens** " |
| f"({best[4]} tok/word). **{worst[0]}** needs **{worst[3]:,}** " |
| f"— **{round(worst[3]/best[3], 2)}×** as many for the same text.\n\n" |
| f"On a 128K context window that difference is " |
| f"**{int(128000/best[4]) - int(128000/worst[4]):,} fewer words** of room.") |
|
|
| tk = get(dict((n, r) for n, r, _ in TOKENIZERS)[show_for]) |
| ids = tk(text, add_special_tokens=False)["input_ids"] |
| return rows, summary, render_tokens(tk, ids) |
|
|
|
|
| with gr.Blocks(title="Arabic Tokenizer Comparison") as demo: |
| gr.Markdown( |
| "# 🔤 Arabic Tokenizer Comparison\n" |
| "### كم رمزاً يكلّفك النص العربي؟\n\n" |
| "The same Arabic paragraph can cost **50% more context** on one tokenizer " |
| "than another. That is a tax on every prompt, document and embedding you " |
| "run — and no model card mentions it. Paste Arabic below and see.\n\n" |
| "Note that **Mistral and Emhotob both have ~32K vocabularies**. The gap " |
| "between them is not vocabulary *size*, it is what the vocabulary is " |
| "*spent on*.") |
|
|
| with gr.Row(): |
| with gr.Column(scale=3): |
| text = gr.Textbox(label="النص / Text", lines=8, rtl=True, |
| text_align="right", value=EXAMPLES["فصحى — MSA news"]) |
| gr.Examples(examples=[[v] for v in EXAMPLES.values()], inputs=[text], |
| example_labels=list(EXAMPLES), label="أمثلة / Examples") |
| with gr.Column(scale=2): |
| summary = gr.Markdown() |
| show_for = gr.Dropdown([n for n, _, _ in TOKENIZERS], |
| value="Emhotob 32K (Arabic-only)", |
| label="Show token split for") |
| run = gr.Button("قارِن / Compare", variant="primary") |
|
|
| table = gr.Dataframe( |
| headers=["tokenizer", "kind", "vocab", "tokens", "tok/word", "chars/tok", "×Emhotob"], |
| datatype=["str", "str", "str", "number", "number", "number", "number"], |
| label="fewer tokens = better", wrap=True) |
| viz = gr.HTML(label="token split — a grey edge marks a token that begins with a space") |
|
|
| gr.Markdown( |
| "---\n" |
| "**Emhotob 32K** is the Arabic-only byte-level BPE behind " |
| "[Nawah](https://huggingface.co/oddadmix/Nawah-50M-RAG-Support-2K) and " |
| "[50M-2048-Emhotob](https://huggingface.co/oddadmix/50M-2048-Emhotob). " |
| "Every one of its 32,000 entries is spent on Arabic, which is why it " |
| "beats vocabularies 8× its size on Arabic text — and why it does *worse* " |
| "on English, which it was never meant to handle.\n\n" |
| "Measured on MSA it reaches ~1.39 tokens/word against Gemma-4's ~2.11. " |
| "On dialect its lead narrows — dialectal orthography is where an " |
| "MSA-trained vocabulary is weakest.\n\n" |
| "© KAND CA 2026 — PROJECT NAWAH") |
|
|
| run.click(compare, [text, show_for], [table, summary, viz]) |
| text.submit(compare, [text, show_for], [table, summary, viz]) |
| show_for.change(compare, [text, show_for], [table, summary, viz]) |
|
|
| if __name__ == "__main__": |
| demo.queue(max_size=24).launch(theme=gr.themes.Soft(primary_hue="teal")) |
|
|
