Update app.py

app.py

@@ -1,111 +1,1521 @@
-from gradio import Interface
+"""
+Arabic Tokenizer Arena Pro - Advanced Arabic Tokenization Analysis Platform
+============================================================================
+A comprehensive research and production-grade tool for evaluating Arabic tokenizers
+across multiple dimensions: efficiency, coverage, morphological awareness, and more.
+
+Supports:
+- Arabic-specific tokenizers (Aranizer, AraBERT, CAMeLBERT, MARBERT, etc.)
+- Major LLM tokenizers (Jais, AceGPT, Falcon-Arabic, ALLaM, Qwen, Llama, Mistral, GPT)
+- Comprehensive evaluation metrics based on latest research
+"""
+
 import gradio as gr
-import aranizer
-from aranizer import (
-    aranizer_bpe50k, aranizer_bpe64k, aranizer_bpe86k,
-    aranizer_sp32k, aranizer_sp50k, aranizer_sp64k, aranizer_sp86k
-)
-from transformers import AutoTokenizer, logging
-from huggingface_hub import login
+import json
+import re
+import time
+import unicodedata
+from typing import Dict, List, Tuple, Optional, Any
+from dataclasses import dataclass, field
+from enum import Enum
 import os
 
-# Retrieve your Hugging Face token from the environment variable
+# Hugging Face authentication
 HF_TOKEN = os.getenv('HF_TOKEN')
-
 if HF_TOKEN:
-    HF_TOKEN = HF_TOKEN.strip()  # Remove any leading or trailing whitespace/newlines
+    HF_TOKEN = HF_TOKEN.strip()
+    from huggingface_hub import login
     login(token=HF_TOKEN)
 
-# Load additional tokenizers from transformers
-gpt_13b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-13B")
-gpt_7b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-7B")
-jais_13b_tokenizer = AutoTokenizer.from_pretrained("inception-mbzuai/jais-13b")
-arabert_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv2")
-
-# Try to load the gated tokenizer
-try:
-    meta_llama_tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B")
-except Exception as e:
-    meta_llama_tokenizer = None
-    logging.warning(f"Could not load meta-llama/Meta-Llama-3-8B tokenizer: {e}")
-
-# List of available tokenizers and a dictionary to load them
-tokenizer_options = [
-    "aranizer_bpe50k", "aranizer_bpe64k", "aranizer_bpe86k",
-    "aranizer_sp32k", "aranizer_sp50k", "aranizer_sp64k", "aranizer_sp86k",
-    "FreedomIntelligence/AceGPT-13B",
-    "FreedomIntelligence/AceGPT-7B",
-    "inception-mbzuai/jais-13b",
-    "aubmindlab/bert-base-arabertv2"
+from transformers import AutoTokenizer, logging
+logging.set_verbosity_error()
+
+# ============================================================================
+# DATA CLASSES AND ENUMS
+# ============================================================================
+
+class TokenizerType(Enum):
+    ARABIC_SPECIFIC = "Arabic-Specific"
+    MULTILINGUAL_LLM = "Multilingual LLM"
+    ARABIC_LLM = "Arabic LLM"
+    ENCODER_ONLY = "Encoder-Only (BERT)"
+    DECODER_ONLY = "Decoder-Only (GPT)"
+
+class TokenizerAlgorithm(Enum):
+    BPE = "Byte-Pair Encoding (BPE)"
+    BBPE = "Byte-Level BPE"
+    WORDPIECE = "WordPiece"
+    SENTENCEPIECE = "SentencePiece"
+    UNIGRAM = "Unigram"
+    TIKTOKEN = "Tiktoken"
+
+@dataclass
+class TokenizerInfo:
+    """Metadata about a tokenizer"""
+    name: str
+    model_id: str
+    type: TokenizerType
+    algorithm: TokenizerAlgorithm
+    vocab_size: int
+    description: str
+    organization: str
+    arabic_support: str  # Native, Adapted, Limited
+    dialect_support: List[str] = field(default_factory=list)
+    special_features: List[str] = field(default_factory=list)
+
+@dataclass
+class TokenizationMetrics:
+    """Comprehensive tokenization evaluation metrics"""
+    # Basic counts
+    total_tokens: int
+    total_words: int
+    total_characters: int
+    total_bytes: int
+    
+    # Efficiency metrics
+    fertility: float  # tokens per word (lower is better, 1.0 is ideal)
+    compression_ratio: float  # bytes per token (higher is better)
+    char_per_token: float  # characters per token
+    
+    # Coverage metrics
+    oov_count: int  # out-of-vocabulary tokens (UNK)
+    oov_percentage: float
+    single_token_words: int  # words tokenized as single token
+    single_token_retention_rate: float  # STRR metric
+    
+    # Morphological metrics
+    avg_subwords_per_word: float
+    max_subwords_per_word: int
+    continued_words_ratio: float  # words split into multiple tokens
+    
+    # Arabic-specific metrics
+    arabic_char_count: int
+    arabic_token_count: int
+    arabic_fertility: float
+    diacritic_preservation: bool
+    
+    # Performance metrics
+    tokenization_time_ms: float
+    
+    # Token details
+    tokens: List[str] = field(default_factory=list)
+    token_ids: List[int] = field(default_factory=list)
+    decoded_text: str = ""
+
+# ============================================================================
+# TOKENIZER REGISTRY - Comprehensive list of Arabic tokenizers
+# ============================================================================
+
+TOKENIZER_REGISTRY: Dict[str, TokenizerInfo] = {
+    # ========== ARABIC-SPECIFIC BERT MODELS ==========
+    "aubmindlab/bert-base-arabertv2": TokenizerInfo(
+        name="AraBERT v2",
+        model_id="aubmindlab/bert-base-arabertv2",
+        type=TokenizerType.ENCODER_ONLY,
+        algorithm=TokenizerAlgorithm.WORDPIECE,
+        vocab_size=64000,
+        description="Arabic BERT with Farasa segmentation, optimized for MSA",
+        organization="AUB MIND Lab",
+        arabic_support="Native",
+        dialect_support=["MSA"],
+        special_features=["Farasa preprocessing", "Morphological segmentation"]
+    ),
+    "aubmindlab/bert-large-arabertv2": TokenizerInfo(
+        name="AraBERT v2 Large",
+        model_id="aubmindlab/bert-large-arabertv2",
+        type=TokenizerType.ENCODER_ONLY,
+        algorithm=TokenizerAlgorithm.WORDPIECE,
+        vocab_size=64000,
+        description="Large Arabic BERT with enhanced capacity",
+        organization="AUB MIND Lab",
+        arabic_support="Native",
+        dialect_support=["MSA"],
+        special_features=["Large model", "Farasa preprocessing"]
+    ),
+    "CAMeL-Lab/bert-base-arabic-camelbert-mix": TokenizerInfo(
+        name="CAMeLBERT Mix",
+        model_id="CAMeL-Lab/bert-base-arabic-camelbert-mix",
+        type=TokenizerType.ENCODER_ONLY,
+        algorithm=TokenizerAlgorithm.WORDPIECE,
+        vocab_size=30000,
+        description="Pre-trained on MSA, DA, and Classical Arabic mix",
+        organization="CAMeL Lab NYU Abu Dhabi",
+        arabic_support="Native",
+        dialect_support=["MSA", "DA", "CA"],
+        special_features=["Multi-variant Arabic", "Classical Arabic support"]
+    ),
+    "CAMeL-Lab/bert-base-arabic-camelbert-msa": TokenizerInfo(
+        name="CAMeLBERT MSA",
+        model_id="CAMeL-Lab/bert-base-arabic-camelbert-msa",
+        type=TokenizerType.ENCODER_ONLY,
+        algorithm=TokenizerAlgorithm.WORDPIECE,
+        vocab_size=30000,
+        description="Specialized for Modern Standard Arabic",
+        organization="CAMeL Lab NYU Abu Dhabi",
+        arabic_support="Native",
+        dialect_support=["MSA"],
+        special_features=["MSA optimized"]
+    ),
+    "CAMeL-Lab/bert-base-arabic-camelbert-da": TokenizerInfo(
+        name="CAMeLBERT DA",
+        model_id="CAMeL-Lab/bert-base-arabic-camelbert-da",
+        type=TokenizerType.ENCODER_ONLY,
+        algorithm=TokenizerAlgorithm.WORDPIECE,
+        vocab_size=30000,
+        description="Specialized for Dialectal Arabic",
+        organization="CAMeL Lab NYU Abu Dhabi",
+        arabic_support="Native",
+        dialect_support=["Egyptian", "Gulf", "Levantine", "Maghrebi"],
+        special_features=["Dialect optimized"]
+    ),
+    "CAMeL-Lab/bert-base-arabic-camelbert-ca": TokenizerInfo(
+        name="CAMeLBERT CA",
+        model_id="CAMeL-Lab/bert-base-arabic-camelbert-ca",
+        type=TokenizerType.ENCODER_ONLY,
+        algorithm=TokenizerAlgorithm.WORDPIECE,
+        vocab_size=30000,
+        description="Specialized for Classical Arabic",
+        organization="CAMeL Lab NYU Abu Dhabi",
+        arabic_support="Native",
+        dialect_support=["Classical"],
+        special_features=["Classical Arabic", "Religious texts"]
+    ),
+    "UBC-NLP/MARBERT": TokenizerInfo(
+        name="MARBERT",
+        model_id="UBC-NLP/MARBERT",
+        type=TokenizerType.ENCODER_ONLY,
+        algorithm=TokenizerAlgorithm.WORDPIECE,
+        vocab_size=100000,
+        description="Multi-dialectal Arabic BERT trained on Twitter data",
+        organization="UBC NLP",
+        arabic_support="Native",
+        dialect_support=["MSA", "Egyptian", "Gulf", "Levantine", "Maghrebi"],
+        special_features=["Twitter data", "100K vocabulary", "Multi-dialect"]
+    ),
+    "UBC-NLP/ARBERT": TokenizerInfo(
+        name="ARBERT",
+        model_id="UBC-NLP/ARBERT",
+        type=TokenizerType.ENCODER_ONLY,
+        algorithm=TokenizerAlgorithm.WORDPIECE,
+        vocab_size=100000,
+        description="Arabic BERT focused on MSA with large vocabulary",
+        organization="UBC NLP",
+        arabic_support="Native",
+        dialect_support=["MSA"],
+        special_features=["100K vocabulary", "MSA focused"]
+    ),
+    
+    # ========== ARABIC-SPECIFIC LLMs ==========
+    "inception-mbzuai/jais-13b": TokenizerInfo(
+        name="Jais 13B",
+        model_id="inception-mbzuai/jais-13b",
+        type=TokenizerType.ARABIC_LLM,
+        algorithm=TokenizerAlgorithm.SENTENCEPIECE,
+        vocab_size=84992,
+        description="World's most advanced Arabic LLM, trained from scratch",
+        organization="Inception/MBZUAI",
+        arabic_support="Native",
+        dialect_support=["MSA", "Gulf", "Egyptian", "Levantine"],
+        special_features=["Arabic-first", "Lowest fertility", "UAE-native"]
+    ),
+    "inceptionai/jais-family-30b-8k-chat": TokenizerInfo(
+        name="Jais 30B Chat",
+        model_id="inceptionai/jais-family-30b-8k-chat",
+        type=TokenizerType.ARABIC_LLM,
+        algorithm=TokenizerAlgorithm.SENTENCEPIECE,
+        vocab_size=84992,
+        description="Enhanced 30B version with chat capabilities",
+        organization="Inception AI",
+        arabic_support="Native",
+        dialect_support=["MSA", "Gulf", "Egyptian", "Levantine"],
+        special_features=["30B parameters", "Chat optimized", "8K context"]
+    ),
+    "FreedomIntelligence/AceGPT-13B": TokenizerInfo(
+        name="AceGPT 13B",
+        model_id="FreedomIntelligence/AceGPT-13B",
+        type=TokenizerType.ARABIC_LLM,
+        algorithm=TokenizerAlgorithm.SENTENCEPIECE,
+        vocab_size=32000,
+        description="Arabic-enhanced LLaMA with cultural alignment",
+        organization="Freedom Intelligence",
+        arabic_support="Adapted",
+        dialect_support=["MSA"],
+        special_features=["LLaMA-based", "Cultural alignment", "RLHF"]
+    ),
+    "FreedomIntelligence/AceGPT-7B": TokenizerInfo(
+        name="AceGPT 7B",
+        model_id="FreedomIntelligence/AceGPT-7B",
+        type=TokenizerType.ARABIC_LLM,
+        algorithm=TokenizerAlgorithm.SENTENCEPIECE,
+        vocab_size=32000,
+        description="Smaller Arabic-enhanced LLaMA variant",
+        organization="Freedom Intelligence",
+        arabic_support="Adapted",
+        dialect_support=["MSA"],
+        special_features=["LLaMA-based", "Efficient"]
+    ),
+    
+    # ========== MULTILINGUAL LLMs WITH ARABIC ==========
+    "Qwen/Qwen2.5-7B": TokenizerInfo(
+        name="Qwen 2.5 7B",
+        model_id="Qwen/Qwen2.5-7B",
+        type=TokenizerType.MULTILINGUAL_LLM,
+        algorithm=TokenizerAlgorithm.BPE,
+        vocab_size=151936,
+        description="Alibaba's multilingual LLM with 30+ language support",
+        organization="Alibaba Qwen",
+        arabic_support="Supported",
+        dialect_support=["MSA"],
+        special_features=["152K vocab", "128K context", "30+ languages"]
+    ),
+    "google/gemma-2-9b": TokenizerInfo(
+        name="Gemma 2 9B",
+        model_id="google/gemma-2-9b",
+        type=TokenizerType.MULTILINGUAL_LLM,
+        algorithm=TokenizerAlgorithm.SENTENCEPIECE,
+        vocab_size=256000,
+        description="Google's efficient multilingual model",
+        organization="Google",
+        arabic_support="Supported",
+        dialect_support=["MSA"],
+        special_features=["256K vocab", "Efficient architecture"]
+    ),
+    "mistralai/Mistral-7B-v0.3": TokenizerInfo(
+        name="Mistral 7B v0.3",
+        model_id="mistralai/Mistral-7B-v0.3",
+        type=TokenizerType.MULTILINGUAL_LLM,
+        algorithm=TokenizerAlgorithm.SENTENCEPIECE,
+        vocab_size=32768,
+        description="Efficient multilingual model with sliding window attention",
+        organization="Mistral AI",
+        arabic_support="Limited",
+        dialect_support=["MSA"],
+        special_features=["Sliding window", "Efficient"]
+    ),
+    "mistralai/Mistral-Nemo-Base-2407": TokenizerInfo(
+        name="Mistral Nemo",
+        model_id="mistralai/Mistral-Nemo-Base-2407",
+        type=TokenizerType.MULTILINGUAL_LLM,
+        algorithm=TokenizerAlgorithm.TIKTOKEN,
+        vocab_size=131072,
+        description="Uses Tekken tokenizer, optimized for multilingual",
+        organization="Mistral AI + NVIDIA",
+        arabic_support="Supported",
+        dialect_support=["MSA"],
+        special_features=["Tekken tokenizer", "131K vocab", "Multilingual optimized"]
+    ),
+    "google/mt5-base": TokenizerInfo(
+        name="mT5 Base",
+        model_id="google/mt5-base",
+        type=TokenizerType.MULTILINGUAL_LLM,
+        algorithm=TokenizerAlgorithm.SENTENCEPIECE,
+        vocab_size=250112,
+        description="Multilingual T5 covering 101 languages",
+        organization="Google",
+        arabic_support="Supported",
+        dialect_support=["MSA"],
+        special_features=["250K vocab", "101 languages", "Seq2Seq"]
+    ),
+    "xlm-roberta-base": TokenizerInfo(
+        name="XLM-RoBERTa Base",
+        model_id="xlm-roberta-base",
+        type=TokenizerType.MULTILINGUAL_LLM,
+        algorithm=TokenizerAlgorithm.SENTENCEPIECE,
+        vocab_size=250002,
+        description="Cross-lingual model covering 100 languages",
+        organization="Facebook AI",
+        arabic_support="Supported",
+        dialect_support=["MSA"],
+        special_features=["250K vocab", "100 languages"]
+    ),
+    "bert-base-multilingual-cased": TokenizerInfo(
+        name="mBERT",
+        model_id="bert-base-multilingual-cased",
+        type=TokenizerType.MULTILINGUAL_LLM,
+        algorithm=TokenizerAlgorithm.WORDPIECE,
+        vocab_size=119547,
+        description="Original multilingual BERT, baseline for comparison",
+        organization="Google",
+        arabic_support="Limited",
+        dialect_support=["MSA"],
+        special_features=["Baseline model", "104 languages"]
+    ),
+}
+
+# Try to load gated models
+GATED_MODELS = [
+    ("meta-llama/Meta-Llama-3-8B", TokenizerInfo(
+        name="Llama 3 8B",
+        model_id="meta-llama/Meta-Llama-3-8B",
+        type=TokenizerType.MULTILINGUAL_LLM,
+        algorithm=TokenizerAlgorithm.BPE,
+        vocab_size=128256,
+        description="Meta's latest LLM with improved multilingual",
+        organization="Meta AI",
+        arabic_support="Limited",
+        dialect_support=["MSA"],
+        special_features=["128K vocab", "Improved tokenizer"]
+    )),
+    ("meta-llama/Llama-2-7b-hf", TokenizerInfo(
+        name="Llama 2 7B",
+        model_id="meta-llama/Llama-2-7b-hf",
+        type=TokenizerType.MULTILINGUAL_LLM,
+        algorithm=TokenizerAlgorithm.SENTENCEPIECE,
+        vocab_size=32000,
+        description="Meta's Llama 2 base model",
+        organization="Meta AI",
+        arabic_support="Limited",
+        dialect_support=["MSA"],
+        special_features=["32K vocab", "Foundation model"]
+    )),
+    ("tiiuae/falcon-7b", TokenizerInfo(
+        name="Falcon 7B",
+        model_id="tiiuae/falcon-7b",
+        type=TokenizerType.MULTILINGUAL_LLM,
+        algorithm=TokenizerAlgorithm.BPE,
+        vocab_size=65024,
+        description="TII's powerful open-source LLM",
+        organization="Technology Innovation Institute",
+        arabic_support="Limited",
+        dialect_support=["MSA"],
+        special_features=["65K vocab", "RefinedWeb trained"]
+    )),
 ]
 
-if meta_llama_tokenizer:
-    tokenizer_options.append("meta-llama/Meta-Llama-3-8B")
-
-tokenizers = {
-    "aranizer_bpe50k": aranizer_bpe50k.get_tokenizer,
-    "aranizer_bpe64k": aranizer_bpe64k.get_tokenizer,
-    "aranizer_bpe86k": aranizer_bpe86k.get_tokenizer,
-    "aranizer_sp32k": aranizer_sp32k.get_tokenizer,
-    "aranizer_sp50k": aranizer_sp50k.get_tokenizer,
-    "aranizer_sp64k": aranizer_sp64k.get_tokenizer,
-    "aranizer_sp86k": aranizer_sp86k.get_tokenizer,
-    "FreedomIntelligence/AceGPT-13B": lambda: gpt_13b_tokenizer,
-    "FreedomIntelligence/AceGPT-7B": lambda: gpt_7b_tokenizer,
-    "inception-mbzuai/jais-13b": lambda: jais_13b_tokenizer,
-    "aubmindlab/bert-base-arabertv2": lambda: arabert_tokenizer
-}
-
-if meta_llama_tokenizer:
-    tokenizers["meta-llama/Meta-Llama-3-8B"] = lambda: meta_llama_tokenizer
-
-def compare_tokenizers(tokenizer_index, text):
-    tokenizer_name = tokenizer_options[tokenizer_index]
-    tokenizer = tokenizers[tokenizer_name]()
+# ============================================================================
+# TOKENIZER LOADER AND CACHE
+# ============================================================================
+
+class TokenizerManager:
+    """Manages tokenizer loading and caching"""
+    
+    def __init__(self):
+        self._cache: Dict[str, Any] = {}
+        self._available: Dict[str, TokenizerInfo] = {}
+        self._initialize_available_tokenizers()
+    
+    def _initialize_available_tokenizers(self):
+        """Check which tokenizers are available and can be loaded"""
+        print("Initializing tokenizer registry...")
+        
+        # Add all base tokenizers
+        for model_id, info in TOKENIZER_REGISTRY.items():
+            try:
+                # Quick check if tokenizer can be loaded
+                _ = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
+                self._available[model_id] = info
+                print(f"  ✓ {info.name}")
+            except Exception as e:
+                print(f"  ✗ {info.name}: {str(e)[:50]}")
+        
+        # Try gated models
+        for model_id, info in GATED_MODELS:
+            try:
+                _ = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
+                self._available[model_id] = info
+                print(f"  ✓ {info.name} (gated)")
+            except Exception as e:
+                print(f"  ✗ {info.name} (gated): {str(e)[:50]}")
+        
+        print(f"\nTotal available tokenizers: {len(self._available)}")
+    
+    def get_tokenizer(self, model_id: str):
+        """Get tokenizer from cache or load it"""
+        if model_id not in self._cache:
+            self._cache[model_id] = AutoTokenizer.from_pretrained(
+                model_id, 
+                trust_remote_code=True
+            )
+        return self._cache[model_id]
+    
+    def get_available_tokenizers(self) -> Dict[str, TokenizerInfo]:
+        return self._available
+    
+    def get_tokenizer_choices(self) -> List[str]:
+        """Get list of tokenizer display names for dropdown"""
+        return [f"{info.name} ({info.organization})" for info in self._available.values()]
+    
+    def get_model_id_from_choice(self, choice: str) -> str:
+        """Convert display choice back to model ID"""
+        for model_id, info in self._available.items():
+            if f"{info.name} ({info.organization})" == choice:
+                return model_id
+        return list(self._available.keys())[0]
+
+# Global tokenizer manager
+tokenizer_manager = TokenizerManager()
+
+# ============================================================================
+# ARABIC TEXT UTILITIES
+# ============================================================================
+
+def is_arabic_char(char: str) -> bool:
+    """Check if character is Arabic"""
+    if len(char) != 1:
+        return False
+    code = ord(char)
+    return (
+        (0x0600 <= code <= 0x06FF) or  # Arabic
+        (0x0750 <= code <= 0x077F) or  # Arabic Supplement
+        (0x08A0 <= code <= 0x08FF) or  # Arabic Extended-A
+        (0xFB50 <= code <= 0xFDFF) or  # Arabic Presentation Forms-A
+        (0xFE70 <= code <= 0xFEFF)     # Arabic Presentation Forms-B
+    )
+
+def count_arabic_chars(text: str) -> int:
+    """Count Arabic characters in text"""
+    return sum(1 for c in text if is_arabic_char(c))
+
+def has_diacritics(text: str) -> bool:
+    """Check if text contains Arabic diacritics (tashkeel)"""
+    diacritics = set('ًٌٍَُِّْٰ')
+    return any(c in diacritics for c in text)
+
+def normalize_arabic(text: str) -> str:
+    """Basic Arabic normalization"""
+    # Normalize alef variants
+    text = re.sub('[إأآا]', 'ا', text)
+    # Normalize yeh
+    text = re.sub('ى', 'ي', text)
+    # Normalize teh marbuta
+    text = re.sub('ة', 'ه', text)
+    return text
+
+def get_arabic_words(text: str) -> List[str]:
+    """Extract Arabic words from text"""
+    # Split on whitespace and filter for words containing Arabic
+    words = text.split()
+    return [w for w in words if any(is_arabic_char(c) for c in w)]
+
+# ============================================================================
+# TOKENIZATION ANALYSIS ENGINE
+# ============================================================================
+
+def analyze_tokenization(
+    text: str, 
+    model_id: str,
+    tokenizer_info: TokenizerInfo
+) -> TokenizationMetrics:
+    """Perform comprehensive tokenization analysis"""
+    
+    tokenizer = tokenizer_manager.get_tokenizer(model_id)
+    
+    # Time the tokenization
+    start_time = time.perf_counter()
     tokens = tokenizer.tokenize(text)
-    encoded_output = tokenizer.encode(text, add_special_tokens=True)
-    decoded_text = tokenizer.decode(encoded_output, skip_special_tokens=True)
+    token_ids = tokenizer.encode(text, add_special_tokens=False)
+    tokenization_time = (time.perf_counter() - start_time) * 1000
+    
+    decoded = tokenizer.decode(token_ids, skip_special_tokens=True)
+    
+    # Basic counts
+    words = text.split()
+    total_words = len(words)
+    total_tokens = len(tokens)
+    total_characters = len(text)
+    total_bytes = len(text.encode('utf-8'))
+    
+    # Efficiency metrics
+    fertility = total_tokens / max(total_words, 1)
+    compression_ratio = total_bytes / max(total_tokens, 1)
+    char_per_token = total_characters / max(total_tokens, 1)
+    
+    # OOV analysis
+    unk_token = tokenizer.unk_token if hasattr(tokenizer, 'unk_token') else '[UNK]'
+    oov_count = sum(1 for t in tokens if t == unk_token or '[UNK]' in str(t))
+    oov_percentage = (oov_count / max(total_tokens, 1)) * 100
+    
+    # Single Token Retention Rate (STRR)
+    single_token_words = 0
+    subwords_per_word = []
+    
+    for word in words:
+        word_tokens = tokenizer.tokenize(word)
+        subwords_per_word.append(len(word_tokens))
+        if len(word_tokens) == 1:
+            single_token_words += 1
+    
+    strr = single_token_words / max(total_words, 1)
+    avg_subwords = sum(subwords_per_word) / max(len(subwords_per_word), 1)
+    max_subwords = max(subwords_per_word) if subwords_per_word else 0
+    continued_ratio = (total_words - single_token_words) / max(total_words, 1)
+    
+    # Arabic-specific metrics
+    arabic_char_count = count_arabic_chars(text)
+    arabic_words = get_arabic_words(text)
+    arabic_tokens_count = 0
+    
+    for token in tokens:
+        if any(is_arabic_char(c) for c in str(token)):
+            arabic_tokens_count += 1
+    
+    arabic_fertility = arabic_tokens_count / max(len(arabic_words), 1) if arabic_words else 0
+    diacritic_preserved = has_diacritics(text) == has_diacritics(decoded)
+    
+    return TokenizationMetrics(
+        total_tokens=total_tokens,
+        total_words=total_words,
+        total_characters=total_characters,
+        total_bytes=total_bytes,
+        fertility=fertility,
+        compression_ratio=compression_ratio,
+        char_per_token=char_per_token,
+        oov_count=oov_count,
+        oov_percentage=oov_percentage,
+        single_token_words=single_token_words,
+        single_token_retention_rate=strr,
+        avg_subwords_per_word=avg_subwords,
+        max_subwords_per_word=max_subwords,
+        continued_words_ratio=continued_ratio,
+        arabic_char_count=arabic_char_count,
+        arabic_token_count=arabic_tokens_count,
+        arabic_fertility=arabic_fertility,
+        diacritic_preservation=diacritic_preserved,
+        tokenization_time_ms=tokenization_time,
+        tokens=tokens,
+        token_ids=token_ids,
+        decoded_text=decoded
+    )
 
-    # Ensure the tokens are properly decoded
-    tokens_display = [token.encode('utf-8').decode('utf-8') if isinstance(token, bytes) else token for token in tokens]
+# ============================================================================
+# UI GENERATION FUNCTIONS
+# ============================================================================
 
-    # Prepare the results to be displayed in HTML format
-    tokens_html = "".join([
-        f"<span style='background-color:#eeeeee; color: #333333; padding:4px; margin:2px; border-radius:3px; border:1px solid #cccccc;'>{token}</span>" 
-        for token in tokens_display
+def generate_token_visualization(tokens: List[str], token_ids: List[int]) -> str:
+    """Generate beautiful HTML visualization of tokens"""
+    
+    # Color palette for tokens (alternating for clarity)
+    colors = [
+        ('#1a1a2e', '#eaeaea'),  # Dark blue bg, light text
+        ('#16213e', '#f0f0f0'),
+        ('#0f3460', '#ffffff'),
+        ('#533483', '#f5f5f5'),
+        ('#e94560', '#ffffff'),
+        ('#0f4c75', '#f0f0f0'),
+        ('#3282b8', '#ffffff'),
+        ('#bbe1fa', '#1a1a2e'),
+    ]
+    
+    html_parts = []
+    for i, (token, tid) in enumerate(zip(tokens, token_ids)):
+        bg, fg = colors[i % len(colors)]
+        # Escape HTML entities
+        display_token = token.replace('<', '&lt;').replace('>', '&gt;')
+        
+        # Determine if token is Arabic
+        is_arabic = any(is_arabic_char(c) for c in token)
+        direction = 'rtl' if is_arabic else 'ltr'
+        
+        html_parts.append(f'''
+            <span class="token" style="
+                background: {bg}; 
+                color: {fg};
+                direction: {direction};
+            " title="ID: {tid}">
+                {display_token}
+                <span class="token-id">{tid}</span>
+            </span>
+        ''')
+    
+    return f'''
+    <div class="token-container">
+        {''.join(html_parts)}
+    </div>
+    '''
+
+def generate_metrics_card(metrics: TokenizationMetrics, info: TokenizerInfo) -> str:
+    """Generate metrics visualization card"""
+    
+    # Determine quality indicators
+    fertility_quality = "excellent" if metrics.fertility < 1.5 else "good" if metrics.fertility < 2.5 else "poor"
+    strr_quality = "excellent" if metrics.single_token_retention_rate > 0.5 else "good" if metrics.single_token_retention_rate > 0.3 else "poor"
+    compression_quality = "excellent" if metrics.compression_ratio > 4 else "good" if metrics.compression_ratio > 2.5 else "poor"
+    
+    return f'''
+    <div class="metrics-grid">
+        <div class="metric-card primary">
+            <div class="metric-icon">📊</div>
+            <div class="metric-value">{metrics.total_tokens}</div>
+            <div class="metric-label">Total Tokens</div>
+        </div>
+        
+        <div class="metric-card {fertility_quality}">
+            <div class="metric-icon">🎯</div>
+            <div class="metric-value">{metrics.fertility:.3f}</div>
+            <div class="metric-label">Fertility (tokens/word)</div>
+            <div class="metric-hint">Lower is better (1.0 ideal)</div>
+        </div>
+        
+        <div class="metric-card {compression_quality}">
+            <div class="metric-icon">📦</div>
+            <div class="metric-value">{metrics.compression_ratio:.2f}</div>
+            <div class="metric-label">Compression (bytes/token)</div>
+            <div class="metric-hint">Higher is better</div>
+        </div>
+        
+        <div class="metric-card {strr_quality}">
+            <div class="metric-icon">✨</div>
+            <div class="metric-value">{metrics.single_token_retention_rate:.1%}</div>
+            <div class="metric-label">Single Token Rate (STRR)</div>
+            <div class="metric-hint">Higher is better</div>
+        </div>
+        
+        <div class="metric-card">
+            <div class="metric-icon">📝</div>
+            <div class="metric-value">{metrics.char_per_token:.2f}</div>
+            <div class="metric-label">Characters/Token</div>
+        </div>
+        
+        <div class="metric-card">
+            <div class="metric-icon">⚡</div>
+            <div class="metric-value">{metrics.tokenization_time_ms:.2f}ms</div>
+            <div class="metric-label">Processing Time</div>
+        </div>
+        
+        <div class="metric-card arabic">
+            <div class="metric-icon">🔤</div>
+            <div class="metric-value">{metrics.arabic_fertility:.3f}</div>
+            <div class="metric-label">Arabic Fertility</div>
+            <div class="metric-hint">Arabic-specific efficiency</div>
+        </div>
+        
+        <div class="metric-card">
+            <div class="metric-icon">{"✅" if metrics.oov_percentage == 0 else "⚠️"}</div>
+            <div class="metric-value">{metrics.oov_percentage:.1f}%</div>
+            <div class="metric-label">OOV Rate</div>
+            <div class="metric-hint">Lower is better (0% ideal)</div>
+        </div>
+    </div>
+    '''
+
+def generate_tokenizer_info_card(info: TokenizerInfo) -> str:
+    """Generate tokenizer information card"""
+    
+    dialect_badges = ' '.join([
+        f'<span class="dialect-badge">{d}</span>' 
+        for d in info.dialect_support
     ])
-    encoded_html = "".join([
-        f"<span style='background-color:#e0e0e0; color: #000000; padding:4px; margin:2px; border-radius:3px; border:1px solid #aaaaaa;'>{token}</span>" 
-        for token in encoded_output
+    
+    feature_badges = ' '.join([
+        f'<span class="feature-badge">{f}</span>' 
+        for f in info.special_features
     ])
-    decoded_html = f"<div style='background-color:#f5f5f5; color: #444444; padding:10px; border-radius:3px; border:1px solid #999999;'>{decoded_text}</div>"
-
-    results_html = f"""
-    <div style='font-family: Arial, sans-serif;'>
-        <h3 style='color: #2e7d32;'>Tokenizer: {tokenizer_name}</h3>
-        <p><strong>Tokens:</strong> {tokens_html}</p>
-        <p><strong>Encoded:</strong> {encoded_html}</p>
-        <p><strong>Decoded:</strong> {decoded_html}</p>
+    
+    support_class = info.arabic_support.lower().replace(' ', '-')
+    
+    return f'''
+    <div class="tokenizer-info">
+        <div class="tokenizer-header">
+            <h3>{info.name}</h3>
+            <span class="org-badge">{info.organization}</span>
+        </div>
+        <p class="tokenizer-desc">{info.description}</p>
+        <div class="tokenizer-meta">
+            <div class="meta-row">
+                <span class="meta-label">Type:</span>
+                <span class="meta-value">{info.type.value}</span>
+            </div>
+            <div class="meta-row">
+                <span class="meta-label">Algorithm:</span>
+                <span class="meta-value">{info.algorithm.value}</span>
+            </div>
+            <div class="meta-row">
+                <span class="meta-label">Vocab Size:</span>
+                <span class="meta-value">{info.vocab_size:,}</span>
+            </div>
+            <div class="meta-row">
+                <span class="meta-label">Arabic Support:</span>
+                <span class="support-badge {support_class}">{info.arabic_support}</span>
+            </div>
+        </div>
+        <div class="tokenizer-badges">
+            <div class="badge-group">
+                <span class="badge-label">Dialects:</span>
+                {dialect_badges}
+            </div>
+            <div class="badge-group">
+                <span class="badge-label">Features:</span>
+                {feature_badges}
+            </div>
+        </div>
     </div>
-    """
-    return results_html
+    '''
 
-# Define the Gradio interface components with a dropdown for model selection
-inputs_component = [
-    gr.Dropdown(choices=tokenizer_options, label="Select Tokenizer", type="index"),
-    gr.Textbox(lines=2, placeholder="اكتب النص هنا...", label="Input Text")
-]
+# ============================================================================
+# MAIN ANALYSIS FUNCTION
+# ============================================================================
+
+def analyze_single_tokenizer(tokenizer_choice: str, text: str) -> Tuple[str, str, str, str]:
+    """Analyze text with a single tokenizer"""
+    
+    if not text.strip():
+        return (
+            "<p class='warning'>Please enter some text to analyze.</p>",
+            "",
+            "",
+            ""
+        )
+    
+    model_id = tokenizer_manager.get_model_id_from_choice(tokenizer_choice)
+    info = tokenizer_manager.get_available_tokenizers()[model_id]
+    
+    try:
+        metrics = analyze_tokenization(text, model_id, info)
+        
+        # Generate all outputs
+        info_html = generate_tokenizer_info_card(info)
+        metrics_html = generate_metrics_card(metrics, info)
+        tokens_html = generate_token_visualization(metrics.tokens, metrics.token_ids)
+        
+        # Decoded text output
+        decoded_html = f'''
+        <div class="decoded-section">
+            <h4>Decoded Output</h4>
+            <div class="decoded-text" dir="auto">{metrics.decoded_text}</div>
+            <div class="decoded-meta">
+                <span>Diacritics preserved: {"✅ Yes" if metrics.diacritic_preservation else "❌ No"}</span>
+            </div>
+        </div>
+        '''
+        
+        return info_html, metrics_html, tokens_html, decoded_html
+        
+    except Exception as e:
+        error_html = f'''
+        <div class="error-card">
+            <h4>Error analyzing with {info.name}</h4>
+            <p>{str(e)}</p>
+        </div>
+        '''
+        return error_html, "", "", ""
+
+def compare_tokenizers(tokenizer_choices: List[str], text: str) -> str:
+    """Compare multiple tokenizers side by side"""
+    
+    if not text.strip():
+        return "<p class='warning'>Please enter some text to analyze.</p>"
+    
+    if not tokenizer_choices or len(tokenizer_choices) < 2:
+        return "<p class='warning'>Please select at least 2 tokenizers to compare.</p>"
+    
+    results = []
+    
+    for choice in tokenizer_choices:
+        model_id = tokenizer_manager.get_model_id_from_choice(choice)
+        info = tokenizer_manager.get_available_tokenizers()[model_id]
+        
+        try:
+            metrics = analyze_tokenization(text, model_id, info)
+            results.append((info, metrics))
+        except Exception as e:
+            continue
+    
+    if not results:
+        return "<p class='error'>Failed to analyze with any selected tokenizers.</p>"
+    
+    # Sort by fertility (best first)
+    results.sort(key=lambda x: x[1].fertility)
+    
+    # Generate comparison table
+    table_rows = []
+    for i, (info, metrics) in enumerate(results):
+        rank_class = "rank-1" if i == 0 else "rank-2" if i == 1 else "rank-3" if i == 2 else ""
+        
+        table_rows.append(f'''
+        <tr class="{rank_class}">
+            <td class="rank-cell">{i + 1}</td>
+            <td class="name-cell">
+                <strong>{info.name}</strong>
+                <span class="org-small">{info.organization}</span>
+            </td>
+            <td class="metric-cell">{metrics.total_tokens}</td>
+            <td class="metric-cell highlight">{metrics.fertility:.3f}</td>
+            <td class="metric-cell">{metrics.compression_ratio:.2f}</td>
+            <td class="metric-cell">{metrics.single_token_retention_rate:.1%}</td>
+            <td class="metric-cell">{metrics.arabic_fertility:.3f}</td>
+            <td class="metric-cell">{metrics.oov_percentage:.1f}%</td>
+            <td class="metric-cell">{metrics.tokenization_time_ms:.2f}ms</td>
+        </tr>
+        ''')
+    
+    return f'''
+    <div class="comparison-container">
+        <h3>Tokenizer Comparison Results</h3>
+        <p class="comparison-subtitle">Ranked by fertility (lower is better)</p>
+        <table class="comparison-table">
+            <thead>
+                <tr>
+                    <th>#</th>
+                    <th>Tokenizer</th>
+                    <th>Tokens</th>
+                    <th>Fertility ↓</th>
+                    <th>Compression</th>
+                    <th>STRR</th>
+                    <th>Arabic Fertility</th>
+                    <th>OOV %</th>
+                    <th>Time</th>
+                </tr>
+            </thead>
+            <tbody>
+                {''.join(table_rows)}
+            </tbody>
+        </table>
+        <div class="comparison-legend">
+            <span class="legend-item"><span class="legend-color rank-1"></span> Best</span>
+            <span class="legend-item"><span class="legend-color rank-2"></span> Runner-up</span>
+            <span class="legend-item"><span class="legend-color rank-3"></span> Third</span>
+        </div>
+    </div>
+    '''
+
+# ============================================================================
+# CSS STYLES
+# ============================================================================
+
+CUSTOM_CSS = """
+/* ===== GLOBAL STYLES ===== */
+:root {
+    --primary: #0d47a1;
+    --primary-light: #1976d2;
+    --primary-dark: #002171;
+    --accent: #ff6f00;
+    --accent-light: #ffa040;
+    --success: #2e7d32;
+    --warning: #f57c00;
+    --error: #c62828;
+    --bg-dark: #0a0a0f;
+    --bg-card: #12121a;
+    --bg-elevated: #1a1a24;
+    --text-primary: #f5f5f5;
+    --text-secondary: #b0b0b0;
+    --border: #2a2a3a;
+    --gradient-1: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+    --gradient-2: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);
+    --gradient-arabic: linear-gradient(135deg, #11998e 0%, #38ef7d 100%);
+}
+
+.gradio-container {
+    background: var(--bg-dark) !important;
+    font-family: 'IBM Plex Sans Arabic', 'Segoe UI', system-ui, sans-serif !important;
+}
+
+/* ===== HEADER STYLES ===== */
+.header-section {
+    text-align: center;
+    padding: 2rem;
+    background: var(--gradient-1);
+    border-radius: 16px;
+    margin-bottom: 2rem;
+}
+
+.header-section h1 {
+    font-size: 2.5rem;
+    font-weight: 700;
+    color: white;
+    margin-bottom: 0.5rem;
+    text-shadow: 0 2px 10px rgba(0,0,0,0.3);
+}
+
+.header-section p {
+    color: rgba(255,255,255,0.9);
+    font-size: 1.1rem;
+}
+
+/* ===== TOKEN VISUALIZATION ===== */
+.token-container {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 8px;
+    padding: 1.5rem;
+    background: var(--bg-card);
+    border-radius: 12px;
+    border: 1px solid var(--border);
+    direction: rtl;
+}
+
+.token {
+    display: inline-flex;
+    flex-direction: column;
+    align-items: center;
+    padding: 8px 12px;
+    border-radius: 8px;
+    font-family: 'IBM Plex Mono', monospace;
+    font-size: 0.95rem;
+    transition: transform 0.2s, box-shadow 0.2s;
+    cursor: default;
+}
+
+.token:hover {
+    transform: translateY(-2px);
+    box-shadow: 0 4px 12px rgba(0,0,0,0.3);
+}
+
+.token-id {
+    font-size: 0.7rem;
+    opacity: 0.7;
+    margin-top: 4px;
+}
+
+/* ===== METRICS GRID ===== */
+.metrics-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(180px, 1fr));
+    gap: 1rem;
+    padding: 1rem;
+}
+
+.metric-card {
+    background: var(--bg-card);
+    border: 1px solid var(--border);
+    border-radius: 12px;
+    padding: 1.25rem;
+    text-align: center;
+    transition: transform 0.2s, border-color 0.2s;
+}
+
+.metric-card:hover {
+    transform: translateY(-3px);
+    border-color: var(--primary-light);
+}
+
+.metric-card.excellent {
+    border-color: var(--success);
+    background: linear-gradient(to bottom, rgba(46, 125, 50, 0.1), transparent);
+}
+
+.metric-card.good {
+    border-color: var(--primary-light);
+    background: linear-gradient(to bottom, rgba(25, 118, 210, 0.1), transparent);
+}
+
+.metric-card.poor {
+    border-color: var(--warning);
+    background: linear-gradient(to bottom, rgba(245, 124, 0, 0.1), transparent);
+}
+
+.metric-card.primary {
+    background: var(--gradient-1);
+}
+
+.metric-card.arabic {
+    background: linear-gradient(to bottom, rgba(17, 153, 142, 0.2), transparent);
+    border-color: #11998e;
+}
+
+.metric-icon {
+    font-size: 1.5rem;
+    margin-bottom: 0.5rem;
+}
+
+.metric-value {
+    font-size: 1.75rem;
+    font-weight: 700;
+    color: var(--text-primary);
+    margin-bottom: 0.25rem;
+}
+
+.metric-label {
+    font-size: 0.85rem;
+    color: var(--text-secondary);
+    margin-bottom: 0.25rem;
+}
+
+.metric-hint {
+    font-size: 0.7rem;
+    color: var(--text-secondary);
+    opacity: 0.7;
+}
+
+/* ===== TOKENIZER INFO ===== */
+.tokenizer-info {
+    background: var(--bg-card);
+    border: 1px solid var(--border);
+    border-radius: 12px;
+    padding: 1.5rem;
+}
+
+.tokenizer-header {
+    display: flex;
+    align-items: center;
+    gap: 1rem;
+    margin-bottom: 1rem;
+}
+
+.tokenizer-header h3 {
+    margin: 0;
+    color: var(--text-primary);
+    font-size: 1.5rem;
+}
+
+.org-badge {
+    background: var(--gradient-1);
+    padding: 4px 12px;
+    border-radius: 20px;
+    font-size: 0.8rem;
+    color: white;
+}
+
+.tokenizer-desc {
+    color: var(--text-secondary);
+    margin-bottom: 1rem;
+    line-height: 1.6;
+}
+
+.tokenizer-meta {
+    display: grid;
+    grid-template-columns: repeat(2, 1fr);
+    gap: 0.75rem;
+    margin-bottom: 1rem;
+}
+
+.meta-row {
+    display: flex;
+    gap: 0.5rem;
+}
+
+.meta-label {
+    color: var(--text-secondary);
+    font-size: 0.85rem;
+}
+
+.meta-value {
+    color: var(--text-primary);
+    font-weight: 500;
+}
+
+.support-badge {
+    padding: 2px 8px;
+    border-radius: 4px;
+    font-size: 0.8rem;
+}
+
+.support-badge.native {
+    background: var(--success);
+    color: white;
+}
+
+.support-badge.adapted {
+    background: var(--primary-light);
+    color: white;
+}
+
+.support-badge.supported {
+    background: var(--warning);
+    color: white;
+}
+
+.support-badge.limited {
+    background: var(--error);
+    color: white;
+}
+
+.tokenizer-badges {
+    display: flex;
+    flex-direction: column;
+    gap: 0.75rem;
+}
+
+.badge-group {
+    display: flex;
+    flex-wrap: wrap;
+    align-items: center;
+    gap: 0.5rem;
+}
+
+.badge-label {
+    color: var(--text-secondary);
+    font-size: 0.85rem;
+}
+
+.dialect-badge, .feature-badge {
+    background: var(--bg-elevated);
+    border: 1px solid var(--border);
+    padding: 4px 10px;
+    border-radius: 6px;
+    font-size: 0.75rem;
+    color: var(--text-primary);
+}
+
+/* ===== COMPARISON TABLE ===== */
+.comparison-container {
+    background: var(--bg-card);
+    border-radius: 12px;
+    padding: 1.5rem;
+    border: 1px solid var(--border);
+}
+
+.comparison-container h3 {
+    color: var(--text-primary);
+    margin-bottom: 0.25rem;
+}
+
+.comparison-subtitle {
+    color: var(--text-secondary);
+    font-size: 0.9rem;
+    margin-bottom: 1.5rem;
+}
+
+.comparison-table {
+    width: 100%;
+    border-collapse: collapse;
+    font-size: 0.9rem;
+}
+
+.comparison-table th {
+    background: var(--bg-elevated);
+    color: var(--text-secondary);
+    padding: 12px 8px;
+    text-align: left;
+    font-weight: 500;
+    border-bottom: 2px solid var(--border);
+}
+
+.comparison-table td {
+    padding: 12px 8px;
+    border-bottom: 1px solid var(--border);
+    color: var(--text-primary);
+}
+
+.comparison-table tr.rank-1 {
+    background: linear-gradient(90deg, rgba(46, 125, 50, 0.2), transparent);
+}
+
+.comparison-table tr.rank-2 {
+    background: linear-gradient(90deg, rgba(25, 118, 210, 0.15), transparent);
+}
+
+.comparison-table tr.rank-3 {
+    background: linear-gradient(90deg, rgba(245, 124, 0, 0.1), transparent);
+}
+
+.rank-cell {
+    font-weight: 700;
+    text-align: center;
+}
+
+.name-cell strong {
+    display: block;
+}
+
+.org-small {
+    font-size: 0.75rem;
+    color: var(--text-secondary);
+}
+
+.metric-cell {
+    text-align: center;
+}
+
+.metric-cell.highlight {
+    font-weight: 700;
+    color: var(--accent-light);
+}
+
+.comparison-legend {
+    display: flex;
+    gap: 1.5rem;
+    margin-top: 1rem;
+    padding-top: 1rem;
+    border-top: 1px solid var(--border);
+}
+
+.legend-item {
+    display: flex;
+    align-items: center;
+    gap: 0.5rem;
+    font-size: 0.85rem;
+    color: var(--text-secondary);
+}
+
+.legend-color {
+    width: 16px;
+    height: 16px;
+    border-radius: 4px;
+}
+
+.legend-color.rank-1 { background: var(--success); }
+.legend-color.rank-2 { background: var(--primary-light); }
+.legend-color.rank-3 { background: var(--warning); }
+
+/* ===== DECODED SECTION ===== */
+.decoded-section {
+    background: var(--bg-card);
+    border: 1px solid var(--border);
+    border-radius: 12px;
+    padding: 1.5rem;
+}
+
+.decoded-section h4 {
+    color: var(--text-primary);
+    margin-bottom: 1rem;
+}
+
+.decoded-text {
+    background: var(--bg-elevated);
+    padding: 1rem;
+    border-radius: 8px;
+    font-family: 'IBM Plex Sans Arabic', serif;
+    font-size: 1.1rem;
+    line-height: 1.8;
+    color: var(--text-primary);
+}
+
+.decoded-meta {
+    margin-top: 1rem;
+    font-size: 0.85rem;
+    color: var(--text-secondary);
+}
+
+/* ===== UTILITY CLASSES ===== */
+.warning {
+    background: linear-gradient(to right, rgba(245, 124, 0, 0.1), transparent);
+    border-left: 4px solid var(--warning);
+    padding: 1rem;
+    border-radius: 0 8px 8px 0;
+    color: var(--text-primary);
+}
+
+.error-card {
+    background: linear-gradient(to right, rgba(198, 40, 40, 0.1), transparent);
+    border-left: 4px solid var(--error);
+    padding: 1rem;
+    border-radius: 0 8px 8px 0;
+}
+
+.error-card h4 {
+    color: var(--error);
+    margin-bottom: 0.5rem;
+}
+
+.error-card p {
+    color: var(--text-secondary);
+}
+"""
+
+# ============================================================================
+# SAMPLE TEXTS FOR TESTING
+# ============================================================================
+
+SAMPLE_TEXTS = {
+    "MSA News": "أعلنت وزارة التربية والتعليم عن بدء العام الدراسي الجديد في الأول من سبتمبر، حيث ستعود المدارس لاستقبال الطلاب بعد العطلة الصيفية الطويلة.",
+    "MSA Formal": "إن تطوير تقنيات الذكاء الاصطناعي يمثل نقلة نوعية في مجال معالجة اللغات الطبيعية، وخاصة فيما يتعلق باللغة العربية ذات الخصائص المورفولوجية الغنية.",
+    "Egyptian Dialect": "ازيك يا صاحبي؟ إيه أخبارك؟ عامل إيه النهارده؟ قولي هنروح فين بكره؟",
+    "Gulf Dialect": "شلونك؟ شخبارك؟ وش تسوي الحين؟ ودك تروح وياي للسوق؟",
+    "Levantine Dialect": "كيفك؟ شو أخبارك؟ شو عم تعمل هلق؟ بدك تيجي معي على السوق؟",
+    "Classical Arabic (Quran)": "بِسْمِ اللَّهِ الرَّحْمَٰنِ الرَّحِيمِ ۝ الْحَمْدُ لِلَّهِ رَبِّ الْعَالَمِينَ",
+    "Poetry": "وما من كاتبٍ إلا سيفنى ويُبقي الدهرُ ما كتبت يداهُ",
+    "Technical": "يستخدم نموذج المحولات آلية الانتباه الذاتي لمعالجة تسلسلات النصوص بشكل متوازي.",
+    "Mixed Arabic-English": "The Arabic language العربية is a Semitic language with over 400 million speakers worldwide.",
+    "With Diacritics": "إِنَّ اللَّهَ وَمَلَائِكَتَهُ يُصَلُّونَ عَلَى النَّبِيِّ",
+}
+
+# ============================================================================
+# GRADIO INTERFACE
+# ============================================================================
 
-outputs_component = gr.HTML(label="Results")
+def create_interface():
+    """Create the Gradio interface"""
+    
+    available_tokenizers = tokenizer_manager.get_tokenizer_choices()
+    
+    # Group tokenizers by type for better organization
+    arabic_specific = [t for t in available_tokenizers if any(x in t for x in ['AraBERT', 'CAMeL', 'MARBERT', 'ARBERT'])]
+    arabic_llms = [t for t in available_tokenizers if any(x in t for x in ['Jais', 'AceGPT'])]
+    multilingual = [t for t in available_tokenizers if t not in arabic_specific and t not in arabic_llms]
+    
+    with gr.Blocks(css=CUSTOM_CSS, title="Arabic Tokenizer Arena Pro", theme=gr.themes.Base(
+        primary_hue="blue",
+        secondary_hue="purple",
+        neutral_hue="slate",
+        font=["IBM Plex Sans Arabic", "system-ui", "sans-serif"]
+    )) as demo:
+        
+        # Header
+        gr.HTML("""
+        <div class="header-section">
+            <h1>🏟️ Arabic Tokenizer Arena Pro</h1>
+            <p>Advanced research & production platform for Arabic tokenization analysis</p>
+        </div>
+        """)
+        
+        with gr.Tabs():
+            # ===== TAB 1: Single Tokenizer Analysis =====
+            with gr.TabItem("🔬 Single Analysis", id="single"):
+                with gr.Row():
+                    with gr.Column(scale=1):
+                        tokenizer_dropdown = gr.Dropdown(
+                            choices=available_tokenizers,
+                            value=available_tokenizers[0] if available_tokenizers else None,
+                            label="Select Tokenizer",
+                            info="Choose a tokenizer to analyze"
+                        )
+                        
+                        sample_dropdown = gr.Dropdown(
+                            choices=list(SAMPLE_TEXTS.keys()),
+                            label="Sample Texts",
+                            info="Select a sample or enter custom text"
+                        )
+                        
+                        input_text = gr.Textbox(
+                            lines=4,
+                            placeholder="اكتب النص العربي هنا...\nEnter Arabic text here...",
+                            label="Input Text",
+                            rtl=True
+                        )
+                        
+                        analyze_btn = gr.Button("🔍 Analyze", variant="primary", size="lg")
+                    
+                    with gr.Column(scale=2):
+                        info_output = gr.HTML(label="Tokenizer Information")
+                
+                metrics_output = gr.HTML(label="Evaluation Metrics")
+                tokens_output = gr.HTML(label="Token Visualization")
+                decoded_output = gr.HTML(label="Decoded Output")
+                
+                # Event handlers
+                sample_dropdown.change(
+                    lambda x: SAMPLE_TEXTS.get(x, ""),
+                    inputs=[sample_dropdown],
+                    outputs=[input_text]
+                )
+                
+                analyze_btn.click(
+                    analyze_single_tokenizer,
+                    inputs=[tokenizer_dropdown, input_text],
+                    outputs=[info_output, metrics_output, tokens_output, decoded_output]
+                )
+            
+            # ===== TAB 2: Comparison Mode =====
+            with gr.TabItem("⚖️ Compare Tokenizers", id="compare"):
+                with gr.Row():
+                    with gr.Column(scale=1):
+                        compare_tokenizers_select = gr.CheckboxGroup(
+                            choices=available_tokenizers,
+                            value=available_tokenizers[:5] if len(available_tokenizers) >= 5 else available_tokenizers,
+                            label="Select Tokenizers to Compare",
+                            info="Choose 2 or more tokenizers"
+                        )
+                        
+                        compare_sample = gr.Dropdown(
+                            choices=list(SAMPLE_TEXTS.keys()),
+                            label="Sample Texts"
+                        )
+                        
+                        compare_text = gr.Textbox(
+                            lines=4,
+                            placeholder="اكتب النص العربي هنا...",
+                            label="Input Text",
+                            rtl=True
+                        )
+                        
+                        compare_btn = gr.Button("⚖️ Compare", variant="primary", size="lg")
+                    
+                    with gr.Column(scale=2):
+                        comparison_output = gr.HTML(label="Comparison Results")
+                
+                compare_sample.change(
+                    lambda x: SAMPLE_TEXTS.get(x, ""),
+                    inputs=[compare_sample],
+                    outputs=[compare_text]
+                )
+                
+                compare_btn.click(
+                    compare_tokenizers,
+                    inputs=[compare_tokenizers_select, compare_text],
+                    outputs=[comparison_output]
+                )
+            
+            # ===== TAB 3: Metrics Reference =====
+            with gr.TabItem("📖 Metrics Guide", id="guide"):
+                gr.Markdown("""
+                ## Tokenization Evaluation Metrics Guide
+                
+                ### Efficiency Metrics
+                
+                | Metric | Description | Ideal Value | Why It Matters |
+                |--------|-------------|-------------|----------------|
+                | **Fertility** | Tokens per word | 1.0 | Lower fertility = fewer tokens = faster inference & lower cost |
+                | **Compression Ratio** | Bytes per token | Higher is better | Better compression = more efficient encoding |
+                | **Chars/Token** | Characters per token | Higher is better | More characters per token = better vocabulary utilization |
+                
+                ### Coverage Metrics
+                
+                | Metric | Description | Ideal Value | Why It Matters |
+                |--------|-------------|-------------|----------------|
+                | **OOV Rate** | Out-of-vocabulary percentage | 0% | Lower OOV = better vocabulary coverage |
+                | **STRR** | Single Token Retention Rate | Higher is better | More words preserved as single tokens = better semantic boundaries |
+                | **Continued Words Ratio** | Words split into multiple tokens | Lower is better | Fewer splits = better word boundary preservation |
+                
+                ### Arabic-Specific Metrics
+                
+                | Metric | Description | Why It Matters |
+                |--------|-------------|----------------|
+                | **Arabic Fertility** | Tokens per Arabic word | Arabic-specific efficiency measure |
+                | **Diacritic Preservation** | Whether tashkeel is preserved | Important for religious & educational texts |
+                
+                ### Research Background
+                
+                These metrics are based on recent research including:
+                - *"A Comprehensive Analysis of Various Tokenizers for Arabic LLMs"* (2024)
+                - *"Evaluating Various Tokenizers for Arabic Text Classification"* (Alyafeai et al.)
+                - *"Beyond Fertility: STRR as a Metric for Multilingual Tokenization"* (2025)
+                - *"Arabic Stable LM: Adapting Stable LM to Arabic"* (2024)
+                
+                ### Tokenizer Algorithm Types
+                
+                - **BPE (Byte-Pair Encoding)**: Iteratively merges frequent character pairs
+                - **Byte-Level BPE**: BPE applied to UTF-8 bytes instead of characters
+                - **WordPiece**: Google's variant, used in BERT models
+                - **SentencePiece**: Language-independent, uses unigram model
+                - **Unigram**: Probabilistic subword model
+                - **Tiktoken**: OpenAI's optimized BPE implementation
+                """)
+            
+            # ===== TAB 4: About =====
+            with gr.TabItem("ℹ️ About", id="about"):
+                gr.Markdown(f"""
+                ## Arabic Tokenizer Arena Pro
+                
+                A comprehensive platform for evaluating Arabic tokenizers across multiple dimensions.
+                
+                ### Available Tokenizers: {len(available_tokenizers)}
+                
+                **Arabic-Specific Models:**
+                {chr(10).join(['- ' + t for t in arabic_specific])}
+                
+                **Arabic LLMs:**
+                {chr(10).join(['- ' + t for t in arabic_llms])}
+                
+                **Multilingual LLMs:**
+                {chr(10).join(['- ' + t for t in multilingual])}
+                
+                ### Features
+                
+                ✅ Comprehensive efficiency metrics (fertility, compression, STRR)  
+                ✅ Arabic-specific analysis (dialect support, diacritic preservation)  
+                ✅ Side-by-side tokenizer comparison  
+                ✅ Beautiful token visualization  
+                ✅ Support for MSA, dialectal Arabic, and Classical Arabic  
+                ✅ Research-backed evaluation methodology  
+                
+                ### Use Cases
+                
+                - **Research**: Compare tokenizers for Arabic NLP experiments
+                - **Production**: Select optimal tokenizer for deployment
+                - **Education**: Understand how different algorithms handle Arabic
+                - **Optimization**: Identify cost-efficient tokenizers for API usage
+                
+                ---
+                
+                Built with ❤️ for the Arabic NLP community
+                """)
+        
+        return demo
 
-# Setting up the interface
-iface = Interface(
-    fn=compare_tokenizers, 
-    inputs=inputs_component, 
-    outputs=outputs_component, 
-    title="Arabic Tokenizer Arena",
-    live=True
-)
+# ============================================================================
+# MAIN
+# ============================================================================
 
-# Launching the Gradio app
-iface.launch()
+if __name__ == "__main__":
+    demo = create_interface()
+    demo.launch(share=True)