Upload benchmark_pypi_full.py with huggingface_hub

benchmark_pypi_full.py

@@ -0,0 +1,338 @@
+#!/usr/bin/env python3
+"""
+Tokenizer Parity Benchmark - Compare SARF tokenizers against state-of-the-art.
+
+This script compares SARFTokenizer (from deeplatent-nlp) against GPT-4o, Gemma-3,
+Command-R, Fanar, Qwen3, and other popular tokenizers.
+
+Datasets:
+- Benchmark data (60k samples): https://huggingface.co/datasets/almaghrabima/deeplatent-benchmark-data
+- Eval test data: https://huggingface.co/datasets/almaghrabima/eval-test-data
+
+Usage:
+    pip install -r requirements.txt
+    python benchmark_pypi.py
+
+Requirements: see benchmarks/requirements.txt
+"""
+
+import os
+import re
+import json
+import time
+import random
+
+import pyarrow.parquet as pq
+
+# Import from PyPI package
+from deeplatent import SARFTokenizer, version, RUST_AVAILABLE
+
+print(f"deeplatent-nlp version: {version()}")
+print(f"Rust available: {RUST_AVAILABLE}")
+
+
+# ── Tokenizer wrappers ──────────────────────────────────────────────
+
+class SarfTokenizerWrapper:
+    """SARF tokenizer using PyPI package."""
+
+    def __init__(self, name_or_path: str, display_name: str = "SARFTokenizer"):
+        self._tok = SARFTokenizer.from_pretrained(name_or_path)
+        self._name = display_name
+
+    def encode(self, text: str) -> list:
+        return self._tok.encode(text)
+
+    @property
+    def vocab_size(self) -> int:
+        return self._tok.vocab_size
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+
+class TiktokenTokenizer:
+    def __init__(self, encoding_name: str, display_name: str = None):
+        import tiktoken
+        self._enc = tiktoken.get_encoding(encoding_name)
+        self._name = display_name or encoding_name
+
+    def encode(self, text: str) -> list:
+        return self._enc.encode(text, allowed_special="all")
+
+    @property
+    def vocab_size(self) -> int:
+        return self._enc.n_vocab
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+
+class HFTokenizer:
+    def __init__(self, model_id: str, display_name: str = None):
+        from transformers import AutoTokenizer
+        try:
+            self._tok = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
+        except Exception:
+            self._tok = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True, use_fast=False)
+        self._name = display_name or model_id.split("/")[-1]
+
+    def encode(self, text: str) -> list:
+        return self._tok.encode(text, add_special_tokens=False)
+
+    @property
+    def vocab_size(self) -> int:
+        return len(self._tok)
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+
+# ── Data loading ─────────────────────────────────────────────────────
+
+AR_DETECT = re.compile(r'[\u0600-\u06FF]')
+
+# HuggingFace datasets
+HF_BENCHMARK_DATA = "almaghrabima/deeplatent-benchmark-data"  # 60k samples (30k AR + 30k EN)
+HF_EVAL_DATA = "almaghrabima/eval-test-data"  # Eval test data
+
+
+def load_samples_from_hf(dataset_id: str = HF_BENCHMARK_DATA):
+    """
+    Load Arabic and English samples from HuggingFace dataset.
+
+    Args:
+        dataset_id: HuggingFace dataset ID
+            - "almaghrabima/deeplatent-benchmark-data" (default): 60k samples for benchmarking
+            - "almaghrabima/eval-test-data": Eval test data
+
+    Returns:
+        Tuple of (arabic_samples, english_samples)
+    """
+    from huggingface_hub import hf_hub_download
+
+    cache_dir = os.path.expanduser("~/.cache/deeplatent/benchmark_data")
+    os.makedirs(cache_dir, exist_ok=True)
+
+    # Download parquet files from HF
+    ar_path = hf_hub_download(
+        repo_id=dataset_id,
+        filename="arabic_samples.parquet",
+        repo_type="dataset",
+        cache_dir=cache_dir,
+    )
+    en_path = hf_hub_download(
+        repo_id=dataset_id,
+        filename="english_samples.parquet",
+        repo_type="dataset",
+        cache_dir=cache_dir,
+    )
+
+    # Load samples
+    ar_table = pq.read_table(ar_path)
+    en_table = pq.read_table(en_path)
+
+    ar_samples = ar_table.column("text").to_pylist()
+    en_samples = en_table.column("text").to_pylist()
+
+    print(f"Loaded {len(ar_samples)} Arabic, {len(en_samples)} English samples from {dataset_id}")
+    return ar_samples, en_samples
+
+
+# ── Metrics ─────────────────────────────────────────────────────────
+
+AR_WORD = re.compile(r'[\u0600-\u06FF]+')
+EN_WORD = re.compile(r'[a-zA-Z]+')
+
+
+def compute_metrics(tokenizer, ar_texts: list, en_texts: list) -> dict:
+    """Compute fertility and parity metrics."""
+    ar_total_chars = ar_total_tokens = ar_total_words = ar_total_word_tokens = 0
+
+    for text in ar_texts:
+        tokens = tokenizer.encode(text)
+        ar_total_chars += len(text)
+        ar_total_tokens += len(tokens)
+        words = AR_WORD.findall(text)
+        ar_total_words += len(words)
+        for w in words:
+            ar_total_word_tokens += len(tokenizer.encode(w))
+
+    en_total_chars = en_total_tokens = en_total_words = en_total_word_tokens = 0
+
+    for text in en_texts:
+        tokens = tokenizer.encode(text)
+        en_total_chars += len(text)
+        en_total_tokens += len(tokens)
+        words = EN_WORD.findall(text)
+        en_total_words += len(words)
+        for w in words:
+            en_total_word_tokens += len(tokenizer.encode(w))
+
+    ar_fertility = ar_total_word_tokens / ar_total_words if ar_total_words else 0
+    ar_cpt = ar_total_chars / ar_total_tokens if ar_total_tokens else 0
+    en_fertility = en_total_word_tokens / en_total_words if en_total_words else 0
+    en_cpt = en_total_chars / en_total_tokens if en_total_tokens else 0
+    parity = ar_cpt / en_cpt if en_cpt else 0
+
+    return {
+        "ar_fertility": ar_fertility,
+        "ar_cpt": ar_cpt,
+        "en_fertility": en_fertility,
+        "en_cpt": en_cpt,
+        "parity": parity,
+        "avg_fertility": (ar_fertility + en_fertility) / 2,
+    }
+
+
+# ── Configuration ───────────────────────────────────────────────────
+
+# SARF tokenizers
+SARF_TOKENIZERS = [
+    ("SARFTokenizer", "/root/.cache/deeplatent/SARFTokenizer"),
+]
+
+# Baseline tokenizers
+BASELINE_TOKENIZERS = [
+    ("GPT-4o", "tiktoken", "o200k_base"),
+    ("GPT-4", "tiktoken", "cl100k_base"),
+    ("Gemma-3-4B", "hf", "unsloth/gemma-3-4b-it"),
+    ("Command-R-Arabic", "hf", "CohereLabs/c4ai-command-r7b-arabic-02-2025"),
+    ("Fanar-1-9B", "hf", "QCRI/Fanar-1-9B-Instruct"),
+    ("Qwen3-4B", "hf", "Qwen/Qwen3-4B-Instruct-2507"),
+    ("Hala-9B", "hf", "hammh0a/Hala-9B"),
+    ("Falcon-H1-7B", "hf", "tiiuae/Falcon-H1-7B-Instruct"),
+    ("ALLaM-7B", "hf", "humain-ai/ALLaM-7B-Instruct-preview"),
+    ("Mistral-7B-v0.3", "hf", "mistralai/Mistral-7B-Instruct-v0.3"),
+]
+
+NUM_RUNS = 1
+SAMPLES_PER_RUN = 60000
+
+
+# ── Main ────────────────────────────────────────────────────────────
+
+def main():
+    print("=" * 100)
+    print("TOKENIZER PARITY BENCHMARK")
+    print("Dataset: almaghrabima/deeplatent-benchmark-data")
+    print("=" * 100)
+
+    # Load tokenizers
+    print("\nLoading tokenizers...")
+    tokenizers = []
+
+    for name, hf_repo in SARF_TOKENIZERS:
+        print(f"  {name}...", end=" ", flush=True)
+        try:
+            tok = SarfTokenizerWrapper(hf_repo, name)
+            print(f"OK (vocab={tok.vocab_size:,})")
+            tokenizers.append(tok)
+        except Exception as e:
+            print(f"FAILED: {e}")
+
+    for name, typ, source in BASELINE_TOKENIZERS:
+        print(f"  {name}...", end=" ", flush=True)
+        try:
+            if typ == "tiktoken":
+                tok = TiktokenTokenizer(source, name)
+            else:
+                tok = HFTokenizer(source, name)
+            print(f"OK (vocab={tok.vocab_size:,})")
+            tokenizers.append(tok)
+        except Exception as e:
+            print(f"FAILED: {e}")
+
+    print(f"\nLoaded {len(tokenizers)} tokenizers.")
+
+    # Load all samples from HuggingFace
+    print("\nLoading evaluation data from HuggingFace...")
+    all_ar, all_en = load_samples_from_hf(HF_BENCHMARK_DATA)
+
+    # Run benchmark 5 times
+    all_runs = {tok.name: [] for tok in tokenizers}
+
+    for run in range(NUM_RUNS):
+        print(f"\n{'='*80}")
+        print(f"RUN {run+1}/{NUM_RUNS}")
+        print(f"{'='*80}")
+
+        random.seed(42 + run)
+        ar_sample = random.sample(all_ar, min(SAMPLES_PER_RUN, len(all_ar)))
+        en_sample = random.sample(all_en, min(SAMPLES_PER_RUN, len(all_en)))
+        print(f"Sampled {len(ar_sample)} AR, {len(en_sample)} EN")
+
+        for tok in tokenizers:
+            print(f"  {tok.name}...", end=" ", flush=True)
+            t0 = time.time()
+            m = compute_metrics(tok, ar_sample, en_sample)
+            all_runs[tok.name].append(m)
+            print(f"parity={m['parity']:.4f} ({time.time()-t0:.1f}s)")
+
+    # Compute averages
+    print("\n" + "=" * 100)
+    print("COMPUTING AVERAGES")
+    print("=" * 100)
+
+    results = []
+    for tok in tokenizers:
+        runs = all_runs[tok.name]
+        n = len(runs)
+
+        parity_vals = [r["parity"] for r in runs]
+        parity_avg = sum(parity_vals) / n
+        parity_std = (sum((v - parity_avg)**2 for v in parity_vals) / n) ** 0.5
+
+        avg = {
+            "name": tok.name,
+            "vocab_size": tok.vocab_size,
+            "ar_fertility_avg": sum(r["ar_fertility"] for r in runs) / n,
+            "en_fertility_avg": sum(r["en_fertility"] for r in runs) / n,
+            "avg_fertility_avg": sum(r["avg_fertility"] for r in runs) / n,
+            "ar_cpt_avg": sum(r["ar_cpt"] for r in runs) / n,
+            "en_cpt_avg": sum(r["en_cpt"] for r in runs) / n,
+            "parity_avg": parity_avg,
+            "parity_std": parity_std,
+            "runs": runs,
+        }
+        results.append(avg)
+
+    # Print table (no ranking)
+    print("\n" + "=" * 140)
+    print(f"FINAL RESULTS (averaged over {NUM_RUNS} runs, {SAMPLES_PER_RUN} samples each)")
+    print("=" * 140)
+    header = f"{'Tokenizer':<22} {'Vocab':>10} {'AR Fert':>10} {'EN Fert':>10} {'Avg Fert':>10} {'AR C/T':>10} {'EN C/T':>10} {'Parity':>10} {'±Std':>8}"
+    print(header)
+    print("-" * 140)
+
+    for r in results:
+        is_sarf = "SARF" in r["name"]
+        marker = " ***" if is_sarf else ""
+        print(f"{r['name']:<22} {r['vocab_size']:>10,} {r['ar_fertility_avg']:>10.3f} {r['en_fertility_avg']:>10.3f} {r['avg_fertility_avg']:>10.3f} {r['ar_cpt_avg']:>10.3f} {r['en_cpt_avg']:>10.3f} {r['parity_avg']:>10.4f} {r['parity_std']:>7.4f}{marker}")
+
+    print("=" * 140)
+    print("*** = SARF tokenizers (using PyPI deeplatent-nlp)")
+    print("Parity = AR chars/token ÷ EN chars/token (1.0 = equal treatment)")
+
+    # Save results
+    output = {
+        "package": "deeplatent-nlp",
+        "version": version(),
+        "dataset": HF_BENCHMARK_DATA,
+        "num_runs": NUM_RUNS,
+        "samples_per_run": SAMPLES_PER_RUN,
+        "results": [{k: v for k, v in r.items() if k != "runs"} for r in results],
+        "detailed_runs": {r["name"]: r["runs"] for r in results},
+    }
+
+    output_path = "benchmark_results.json"
+    with open(output_path, "w") as f:
+        json.dump(output, f, indent=2, ensure_ascii=False)
+    print(f"\nResults saved to {output_path}")
+
+
+if __name__ == "__main__":
+    main()