almaghrabima
/

SARFTokenizer

@@ -1,338 +0,0 @@
-#!/usr/bin/env python3
-"""
-Tokenizer Parity Benchmark - Compare SARF tokenizers against state-of-the-art.
-This script compares SARFTokenizer (from deeplatent-nlp) against GPT-4o, Gemma-3,
-Command-R, Fanar, Qwen3, and other popular tokenizers.
-Datasets:
-- Benchmark data (60k samples): https://huggingface.co/datasets/almaghrabima/deeplatent-benchmark-data
-- Eval test data: https://huggingface.co/datasets/almaghrabima/eval-test-data
-Usage:
-    pip install -r requirements.txt
-    python benchmark_pypi.py
-Requirements: see benchmarks/requirements.txt
-"""
-import os
-import re
-import json
-import time
-import random
-import pyarrow.parquet as pq
-# Import from PyPI package
-from deeplatent import SARFTokenizer, version, RUST_AVAILABLE
-print(f"deeplatent-nlp version: {version()}")
-print(f"Rust available: {RUST_AVAILABLE}")
-# ── Tokenizer wrappers ──────────────────────────────────────────────
-class SarfTokenizerWrapper:
-    """SARF tokenizer using PyPI package."""
-    def __init__(self, name_or_path: str, display_name: str = "SARFTokenizer"):
-        self._tok = SARFTokenizer.from_pretrained(name_or_path)
-        self._name = display_name
-    def encode(self, text: str) -> list:
-        return self._tok.encode(text)
-    @property
-    def vocab_size(self) -> int:
-        return self._tok.vocab_size
-    @property
-    def name(self) -> str:
-        return self._name
-class TiktokenTokenizer:
-    def __init__(self, encoding_name: str, display_name: str = None):
-        import tiktoken
-        self._enc = tiktoken.get_encoding(encoding_name)
-        self._name = display_name or encoding_name
-    def encode(self, text: str) -> list:
-        return self._enc.encode(text, allowed_special="all")
-    @property
-    def vocab_size(self) -> int:
-        return self._enc.n_vocab
-    @property
-    def name(self) -> str:
-        return self._name
-class HFTokenizer:
-    def __init__(self, model_id: str, display_name: str = None):
-        from transformers import AutoTokenizer
-        try:
-            self._tok = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
-        except Exception:
-            self._tok = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True, use_fast=False)
-        self._name = display_name or model_id.split("/")[-1]
-    def encode(self, text: str) -> list:
-        return self._tok.encode(text, add_special_tokens=False)
-    @property
-    def vocab_size(self) -> int:
-        return len(self._tok)
-    @property
-    def name(self) -> str:
-        return self._name
-# ── Data loading ─────────────────────────────────────────────────────
-AR_DETECT = re.compile(r'[\u0600-\u06FF]')
-# HuggingFace datasets
-HF_BENCHMARK_DATA = "almaghrabima/deeplatent-benchmark-data"  # 60k samples (30k AR + 30k EN)
-HF_EVAL_DATA = "almaghrabima/eval-test-data"  # Eval test data
-def load_samples_from_hf(dataset_id: str = HF_BENCHMARK_DATA):
-    """
-    Load Arabic and English samples from HuggingFace dataset.
-    Args:
-        dataset_id: HuggingFace dataset ID
-            - "almaghrabima/deeplatent-benchmark-data" (default): 60k samples for benchmarking
-            - "almaghrabima/eval-test-data": Eval test data
-    Returns:
-        Tuple of (arabic_samples, english_samples)
-    """
-    from huggingface_hub import hf_hub_download
-    cache_dir = os.path.expanduser("~/.cache/deeplatent/benchmark_data")
-    os.makedirs(cache_dir, exist_ok=True)
-    # Download parquet files from HF
-    ar_path = hf_hub_download(
-        repo_id=dataset_id,
-        filename="arabic_samples.parquet",
-        repo_type="dataset",
-        cache_dir=cache_dir,
-    )
-    en_path = hf_hub_download(
-        repo_id=dataset_id,
-        filename="english_samples.parquet",
-        repo_type="dataset",
-        cache_dir=cache_dir,
-    )
-    # Load samples
-    ar_table = pq.read_table(ar_path)
-    en_table = pq.read_table(en_path)
-    ar_samples = ar_table.column("text").to_pylist()
-    en_samples = en_table.column("text").to_pylist()
-    print(f"Loaded {len(ar_samples)} Arabic, {len(en_samples)} English samples from {dataset_id}")
-    return ar_samples, en_samples
-# ── Metrics ─────────────────────────────────────────────────────────
-AR_WORD = re.compile(r'[\u0600-\u06FF]+')
-EN_WORD = re.compile(r'[a-zA-Z]+')
-def compute_metrics(tokenizer, ar_texts: list, en_texts: list) -> dict:
-    """Compute fertility and parity metrics."""
-    ar_total_chars = ar_total_tokens = ar_total_words = ar_total_word_tokens = 0
-    for text in ar_texts:
-        tokens = tokenizer.encode(text)
-        ar_total_chars += len(text)
-        ar_total_tokens += len(tokens)
-        words = AR_WORD.findall(text)
-        ar_total_words += len(words)
-        for w in words:
-            ar_total_word_tokens += len(tokenizer.encode(w))
-    en_total_chars = en_total_tokens = en_total_words = en_total_word_tokens = 0
-    for text in en_texts:
-        tokens = tokenizer.encode(text)
-        en_total_chars += len(text)
-        en_total_tokens += len(tokens)
-        words = EN_WORD.findall(text)
-        en_total_words += len(words)
-        for w in words:
-            en_total_word_tokens += len(tokenizer.encode(w))
-    ar_fertility = ar_total_word_tokens / ar_total_words if ar_total_words else 0
-    ar_cpt = ar_total_chars / ar_total_tokens if ar_total_tokens else 0
-    en_fertility = en_total_word_tokens / en_total_words if en_total_words else 0
-    en_cpt = en_total_chars / en_total_tokens if en_total_tokens else 0
-    parity = ar_cpt / en_cpt if en_cpt else 0
-    return {
-        "ar_fertility": ar_fertility,
-        "ar_cpt": ar_cpt,
-        "en_fertility": en_fertility,
-        "en_cpt": en_cpt,
-        "parity": parity,
-        "avg_fertility": (ar_fertility + en_fertility) / 2,
-    }
-# ── Configuration ───────────────────────────────────────────────────
-# SARF tokenizers from HuggingFace
-SARF_TOKENIZERS = [
-    ("SARFTokenizer", "almaghrabima/SARFTokenizer"),
-]
-# Baseline tokenizers
-BASELINE_TOKENIZERS = [
-    ("GPT-4o", "tiktoken", "o200k_base"),
-    ("GPT-4", "tiktoken", "cl100k_base"),
-    ("Gemma-3-4B", "hf", "google/gemma-3-4b-it"),
-    ("Command-R-Arabic", "hf", "CohereLabs/c4ai-command-r7b-arabic-02-2025"),
-    ("Fanar-1-9B", "hf", "QCRI/Fanar-1-9B-Instruct"),
-    ("Qwen3-4B", "hf", "Qwen/Qwen3-4B-Instruct-2507"),
-]
-NUM_RUNS = 5
-SAMPLES_PER_RUN = 5000
-# ── Main ────────────────────────────────────────────────────────────
-def main():
-    print("=" * 100)
-    print("TOKENIZER PARITY BENCHMARK")
-    print("Dataset: almaghrabima/deeplatent-benchmark-data")
-    print("=" * 100)
-    # Load tokenizers
-    print("\nLoading tokenizers...")
-    tokenizers = []
-    for name, hf_repo in SARF_TOKENIZERS:
-        print(f"  {name}...", end=" ", flush=True)
-        try:
-            tok = SarfTokenizerWrapper(hf_repo, name)
-            print(f"OK (vocab={tok.vocab_size:,})")
-            tokenizers.append(tok)
-        except Exception as e:
-            print(f"FAILED: {e}")
-    for name, typ, source in BASELINE_TOKENIZERS:
-        print(f"  {name}...", end=" ", flush=True)
-        try:
-            if typ == "tiktoken":
-                tok = TiktokenTokenizer(source, name)
-            else:
-                tok = HFTokenizer(source, name)
-            print(f"OK (vocab={tok.vocab_size:,})")
-            tokenizers.append(tok)
-        except Exception as e:
-            print(f"FAILED: {e}")
-    print(f"\nLoaded {len(tokenizers)} tokenizers.")
-    # Load all samples from HuggingFace
-    print("\nLoading evaluation data from HuggingFace...")
-    all_ar, all_en = load_samples_from_hf(HF_BENCHMARK_DATA)
-    # Run benchmark 5 times
-    all_runs = {tok.name: [] for tok in tokenizers}
-    for run in range(NUM_RUNS):
-        print(f"\n{'='*80}")
-        print(f"RUN {run+1}/{NUM_RUNS}")
-        print(f"{'='*80}")
-        random.seed(42 + run)
-        ar_sample = random.sample(all_ar, min(SAMPLES_PER_RUN, len(all_ar)))
-        en_sample = random.sample(all_en, min(SAMPLES_PER_RUN, len(all_en)))
-        print(f"Sampled {len(ar_sample)} AR, {len(en_sample)} EN")
-        for tok in tokenizers:
-            print(f"  {tok.name}...", end=" ", flush=True)
-            t0 = time.time()
-            m = compute_metrics(tok, ar_sample, en_sample)
-            all_runs[tok.name].append(m)
-            print(f"parity={m['parity']:.4f} ({time.time()-t0:.1f}s)")
-    # Compute averages
-    print("\n" + "=" * 100)
-    print("COMPUTING AVERAGES")
-    print("=" * 100)
-    results = []
-    for tok in tokenizers:
-        runs = all_runs[tok.name]
-        n = len(runs)
-        parity_vals = [r["parity"] for r in runs]
-        parity_avg = sum(parity_vals) / n
-        parity_std = (sum((v - parity_avg)**2 for v in parity_vals) / n) ** 0.5
-        avg = {
-            "name": tok.name,
-            "vocab_size": tok.vocab_size,
-            "ar_fertility_avg": sum(r["ar_fertility"] for r in runs) / n,
-            "en_fertility_avg": sum(r["en_fertility"] for r in runs) / n,
-            "avg_fertility_avg": sum(r["avg_fertility"] for r in runs) / n,
-            "ar_cpt_avg": sum(r["ar_cpt"] for r in runs) / n,
-            "en_cpt_avg": sum(r["en_cpt"] for r in runs) / n,
-            "parity_avg": parity_avg,
-            "parity_std": parity_std,
-            "runs": runs,
-        }
-        results.append(avg)
-    # Sort by parity (closer to 1.0)
-    results_sorted = sorted(results, key=lambda r: abs(1.0 - r["parity_avg"]))
-    # Print table
-    print("\n" + "=" * 140)
-    print(f"FINAL RESULTS (averaged over {NUM_RUNS} runs, {SAMPLES_PER_RUN} samples each)")
-    print("=" * 140)
-    header = f"{'Rank':<5} {'Tokenizer':<22} {'Vocab':>10} {'AR Fert':>10} {'EN Fert':>10} {'Avg Fert':>10} {'AR C/T':>10} {'EN C/T':>10} {'Parity':>10} {'±Std':>8}"
-    print(header)
-    print("-" * 140)
-    for rank, r in enumerate(results_sorted, 1):
-        is_best = rank == 1
-        is_sarf = "SARF" in r["name"]
-        marker = " 🏆" if is_best else (" ***" if is_sarf else "")
-        print(f"{rank:<5} {r['name']:<22} {r['vocab_size']:>10,} {r['ar_fertility_avg']:>10.3f} {r['en_fertility_avg']:>10.3f} {r['avg_fertility_avg']:>10.3f} {r['ar_cpt_avg']:>10.3f} {r['en_cpt_avg']:>10.3f} {r['parity_avg']:>10.4f} {r['parity_std']:>7.4f}{marker}")
-    print("=" * 140)
-    print("*** = SARF tokenizers (using PyPI deeplatent-nlp) | 🏆 = Best parity (closest to 1.0)")
-    print("Parity = AR chars/token ÷ EN chars/token (1.0 = equal treatment)")
-    # Save results
-    output = {
-        "package": "deeplatent-nlp",
-        "version": version(),
-        "dataset": HF_BENCHMARK_DATA,
-        "num_runs": NUM_RUNS,
-        "samples_per_run": SAMPLES_PER_RUN,
-        "results": [{k: v for k, v in r.items() if k != "runs"} for r in results_sorted],
-        "detailed_runs": {r["name"]: r["runs"] for r in results_sorted},
-    }
-    output_path = "benchmark_results.json"
-    with open(output_path, "w") as f:
-        json.dump(output, f, indent=2, ensure_ascii=False)
-    print(f"\nResults saved to {output_path}")
-if __name__ == "__main__":
-    main()