| --- |
| license: apache-2.0 |
| language: |
| - ne |
| - en |
| tags: |
| - tokenizer |
| - nepali |
| - devanagari |
| - unicode |
| - trie |
| - nlp |
| - tokenization |
| --- |
| |
| # Supernova NepaliFast V4 |
|
|
| Supernova NepaliFast V4 is a Nepali-first, Unicode-aware Longest-Match Trie tokenizer. |
|
|
| ## Features |
|
|
| | Feature | Status | |
| |---|---| |
| | Nepali-first | PASS | |
| | Devanagari | PASS | |
| | English | PASS | |
| | Unicode | PASS | |
| | Emoji | PASS | |
| | Mathematical symbols | PASS | |
| | Multilingual text | PASS | |
| | Round-trip decoding | PASS | |
| | CPU-friendly | PASS | |
|
|
| ## Vocabulary |
|
|
| - Vocabulary size: 2,890 |
| - ID range: 0 -> 2889 |
| - ID integrity: PASS |
|
|
| ## Final Extreme Benchmark |
|
|
| - Documents: 9,120 |
| - Characters: 8,597,880 |
| - Unknown characters: 0 |
| - Round-trip failures: 0 |
| - Fallback documents: 0 |
|
|
| | Engine | Characters/sec | Tokens/sec | |
| |---|---:|---:| |
| | Supernova V4 | 7,886,249 | 6,897,069 | |
| | Tiktoken o200k | 7,666,757 | 4,802,143 | |
|
|
| ### Relative performance |
|
|
| - Character throughput: 1.03x |
| - Token throughput: 1.44x |
|
|
| ## Tested Unicode |
|
|
| ```text |
| √2 ≈ 1.4142135623730951 |
| ∑(xᵢ²) → ∞ |
| 🇳🇵 🚀 🔥 🤖 🧠 💻 🌋 |
| 👨👩👧👦 👩💻 🧑🚀 |
| — – … « » “ ” ‘ ’ ≠ ≤ ≥ ± × ÷ ∞ |
| ``` |
|
|
| ## Nepali |
|
|
| ```text |
| नमस्ते नेपाल |
| लुम्बिनी नेपालको प्रसिद्ध स्थान हो। |
| सगरमाथा नेपालको गौरव हो। |
| लाख करोड अरब खर्ब हजार |
| ``` |
|
|
| ## Run on your own computer |
|
|
| Install Python 3.9 or newer. |
|
|
| Run the included benchmark: |
|
|
| ```bash |
| python benchmark.py |
| ``` |
|
|
| The repository contains the tokenizer vocabulary and a reference Python implementation for testing. |
|
|
| ## Research Focus |
|
|
| - Nepali-first tokenization |
| - Devanagari coverage |
| - Unicode robustness |
| - Deterministic tokenization |
| - Lossless round-trip decoding |
| - High token throughput |
| - CPU-friendly execution |
|
|
| Supernova NepaliFast V4 is a tokenizer, not a language model. |
|
|
| ## License |
|
|
| Apache License 2.0. |
|
|
| ## Supernova AI |
|
|
| Built as part of the Supernova AI tokenizer research project. |
|
|
| **Fast. Unicode-safe. Nepali-first.** |
| # Install dependencies: |
| # pip install huggingface_hub |
| |
| from huggingface_hub import hf_hub_download |
| import json |
|
|
| REPO_ID = "Supernova11c/Supernova-NepaliFast-V4" |
| |
| # Download the published tokenizer |
| tokenizer_path = hf_hub_download( |
| repo_id=REPO_ID, |
| filename="tokenizer.json", |
| repo_type="model" |
| ) |
| |
| with open(tokenizer_path, "r", encoding="utf-8") as f: |
| data = json.load(f) |
| |
| vocab = data["vocab"] |
| |
| # Build token -> ID mapping |
| token_to_id = {token: int(idx) for idx, token in vocab.items()} |
| |
| # Longest-match tokenizer |
| def tokenize(text): |
| tokens = [] |
| i = 0 |
| |
| while i < len(text): |
| best = None |
| best_id = None |
|
|
| for token, token_id in token_to_id.items(): |
| if text.startswith(token, i): |
| if best is None or len(token) > len(best): |
| best = token |
| best_id = token_id |
| |
| if best is None: |
| # Character fallback |
| best = text[i] |
| best_id = token_to_id.get(best) |
| |
| tokens.append(best_id) |
| i += len(best) |
| |
| return tokens |
| |
|
|
| text = "नमस्ते नेपाल! Supernova AI 🚀" |
|
|
| ids = tokenize(text) |
|
|
| print("Input :", text) |
| print("Tokens:", ids) |
| print("Count :", len(ids)) |
| # pip install huggingface_hub tiktoken |
| |
| import json |
| import time |
| from huggingface_hub import hf_hub_download |
| import tiktoken |
|
|
| REPO_ID = "Supernova11c/Supernova-NepaliFast-V4" |
| |
| # ------------------------------------------------------------ |
| # Load Supernova V4 |
| # ------------------------------------------------------------ |
| |
| path = hf_hub_download( |
| repo_id=REPO_ID, |
| filename="tokenizer.json", |
| repo_type="model" |
| ) |
|
|
| with open(path, "r", encoding="utf-8") as f: |
| data = json.load(f) |
| |
| vocab = data["vocab"] |
| token_to_id = {token: int(idx) for idx, token in vocab.items()} |
|
|
|
|
| def supernova_encode(text): |
| ids = [] |
| i = 0 |
| |
| while i < len(text): |
| best = None |
| best_id = None |
|
|
| for token, token_id in token_to_id.items(): |
| if text.startswith(token, i): |
| if best is None or len(token) > len(best): |
| best = token |
| best_id = token_id |
| |
| if best is None: |
| best = text[i] |
| best_id = token_to_id.get(best, -1) |
| |
| ids.append(best_id) |
| i += len(best) |
| |
| return ids |
| |
|
|
| def supernova_decode(ids): |
| id_to_token = { |
| int(idx): token |
| for idx, token in vocab.items() |
| } |
| |
| return "".join(id_to_token.get(i, "") for i in ids) |
| |
| |
| # ------------------------------------------------------------ |
| # Tiktoken |
| # ------------------------------------------------------------ |
| |
| tik = tiktoken.get_encoding("o200k_base") |
| |
| |
| # ------------------------------------------------------------ |
| # Test corpus |
| # ------------------------------------------------------------ |
| |
| corpus = [ |
| "नमस्ते नेपाल।", |
| "नेपाल सुन्दर र विविध संस्कृतिले भरिएको देश हो।", |
| "लुम्बिनी नेपालको प्रसिद्ध ऐतिहासिक स्थान हो।", |
| "सगरमाथा नेपालको गौरव हो।", |
| "काठमाडौँ नेपालको राजधानी हो।", |
| "Artificial Intelligence is changing the world.", |
| "Supernova AI is being developed in Nepal. 🚀", |
| "√2 ≈ 1.4142135623730951", |
| "∑(xᵢ²) → ∞", |
| "🇳🇵 🚀 🔥 🤖 🧠 💻 🌋", |
| "नमस्ते Hello こんにちは 안녕하세요 مرحبا", |
| "नेपाल Nepal 日本 Japan भारत India", |
| ] |
| |
| text = "\n".join(corpus) |
| |
| # Repeat corpus for a more meaningful benchmark |
| text = text * 1000 |
| |
| print("=" * 70) |
| print("SUPERNOVA V4 vs TIKTOKEN") |
| print("=" * 70) |
| |
| print("Characters:", len(text)) |
| |
| |
| # ------------------------------------------------------------ |
| # Supernova benchmark |
| # ------------------------------------------------------------ |
| |
| start = time.perf_counter() |
|
|
| supernova_ids = supernova_encode(text) |
|
|
| supernova_time = time.perf_counter() - start |
|
|
| supernova_tokens = len(supernova_ids) |
| supernova_chars_sec = len(text) / supernova_time |
| supernova_tokens_sec = supernova_tokens / supernova_time |
| |
| decoded = supernova_decode(supernova_ids) |
| |
| supernova_roundtrip = decoded == text |
|
|
| # ------------------------------------------------------------ |
| # Tiktoken benchmark |
| # ------------------------------------------------------------ |
|
|
| start = time.perf_counter() |
| |
| tik_ids = tik.encode(text) |
|
|
| tik_time = time.perf_counter() - start |
|
|
| tik_tokens = len(tik_ids) |
| tik_chars_sec = len(text) / tik_time |
| tik_tokens_sec = tik_tokens / tik_time |
| |
| # ------------------------------------------------------------ |
| # Results |
| # ------------------------------------------------------------ |
| |
| print() |
| print("ENGINE TIME CHARS/S TOKENS/S") |
| print("-" * 70) |
| |
| print( |
| f"Supernova V4 " |
| f"{supernova_time:.4f}s " |
| f"{supernova_chars_sec:,.0f} " |
| f"{supernova_tokens_sec:,.0f}" |
| ) |
| |
| print( |
| f"Tiktoken o200k " |
| f"{tik_time:.4f}s " |
| f"{tik_chars_sec:,.0f} " |
| f"{tik_tokens_sec:,.0f}" |
| ) |
| |
| print() |
| print("TOKEN COUNTS") |
| print("-" * 70) |
| print("Supernova V4 :", supernova_tokens) |
| print("Tiktoken :", tik_tokens) |
|
|
| print() |
| print("CORRECTNESS") |
| print("-" * 70) |
| print("Supernova round-trip:", "PASS" if supernova_roundtrip else "FAIL") |
| |
| print() |
| print("RELATIVE PERFORMANCE") |
| print("-" * 70) |
| |
| print( |
| "Character speedup:", |
| f"{supernova_chars_sec / tik_chars_sec:.2f}x" |
| ) |
| |
| print( |
| "Token throughput:", |
| f"{supernova_tokens_sec / tik_tokens_sec:.2f}x" |
| ) |
| |
| print( |
| "Token ratio V4/Tiktoken:", |
| f"{supernova_tokens / tik_tokens:.3f}x" |
| |
| ) |
| |
| ## Supernova AI |
| Supernova NepaliFast V4 is a hybrid tokenizer: the optimized V4 Trie handles common text at high speed, while the fallback layer provides a safety net for unusual Unicode, multilingual text, symbols, and emojis. This gives Supernova V4 extremely high reliability, with very little chance of complete tokenization failure. The main trade-off is that fallback processing can be slightly slower than the optimized V4 path |
| ## Supernova vs SENTENCE |
| ========================================================================================== |
| 🌌 SUPERNOVA V4 vs SENTENCEPIECE |
| ========================================================================================== |
| |
| नमस्ते, तपाईंलाई कस्तो छ? |
| V4: 25 tokens | unknown=0 | PASS |
| SP: 14 tokens | PASS |
| |
| नेपाल सुन्दर देश हो। |
| V4: 20 tokens | unknown=0 | PASS |
| SP: 6 tokens | PASS |
| |
| लुम्बिनी नेपालको प्रसिद्ध स्थान हो। |
| V4: 35 tokens | unknown=0 | PASS |
| SP: 10 tokens | PASS |
| |
| सगरमाथा नेपालको गौरव हो। |
| V4: 24 tokens | unknown=0 | PASS |
| SP: 11 tokens | PASS |
| |
| काठमाडौं नेपालको राजधानी हो। |
| V4: 28 tokens | unknown=0 | PASS |
| SP: 9 tokens | PASS |
| |
| पोखरा नेपालको सुन्दर शहर हो। |
| V4: 28 tokens | unknown=0 | PASS |
| SP: 11 tokens | PASS |
| |
| विज्ञान र प्रविधिले संसार परिवर्तन गरिरहेको छ। |
| V4: 46 tokens | unknown=0 | PASS |
| SP: 16 tokens | PASS |
| |
| अर्थतन्त्र र शिक्षा देशको विकासका आधार हुन्। |
| V4: 44 tokens | unknown=0 | PASS |
| SP: 15 tokens | PASS |
| |
| Supernova AI is being developed in Nepal. |
| V4: 41 tokens | unknown=0 | PASS |
| SP: 11 tokens | PASS |
| |
| Artificial Intelligence is changing the world. |
| V4: 46 tokens | unknown=0 | PASS |
| SP: 9 tokens | PASS |
| |
| √2 ≈ 1.4142135623730951 |
| V4: 21 tokens | unknown=0 | FAIL |
| SP: 10 tokens | PASS |
| |
| ∑(xᵢ²) → ∞ |
| V4: 10 tokens | unknown=0 | PASS |
| SP: 8 tokens | FAIL |
| |
| π × r² ≠ 0 |
| V4: 10 tokens | unknown=0 | PASS |
| SP: 8 tokens | FAIL |
| |
| 🇳🇵 🚀 🔥 🤖 🧠 💻 🌋 |
| V4: 14 tokens | unknown=0 | PASS |
| SP: 15 tokens | PASS |
| |
| 👨👩👧👦 👩💻 🧑🚀 🏃♂️ |
| V4: 20 tokens | unknown=0 | PASS |
| SP: 21 tokens | FAIL |
| |
| नमस्ते Hello こんにちは 안녕하세요 مرحبا |
| V4: 30 tokens | unknown=0 | PASS |
| SP: 11 tokens | PASS |
| |
| नेपाल Nepal 日本 Japan भारत India |
| V4: 31 tokens | unknown=0 | PASS |
| SP: 6 tokens | PASS |
| |
| — – … « » “ ” ‘ ’ ≠ ≤ ≥ ± × ÷ ∞ |
| V4: 31 tokens | unknown=0 | PASS |
| SP: 24 tokens | FAIL |
| |
| ========================================================================================== |
| 📦 BENCHMARK CORPUS |
| ========================================================================================== |
| Documents : 90,000 |
| Characters: 2,530,000 |
| |
| ========================================================================================== |
| 🏆 FINAL PERFORMANCE |
| ========================================================================================== |
| ENGINE TIME CHARS/S TOKENS/S |
| ------------------------------------------------------------------------------------------ |
| Supernova V4 0.7781 3,251,524 3,238,673 |
| SentencePiece 0.5185 4,879,325 2,073,231 |
| |
| ========================================================================================== |
| 🧪 CORRECTNESS |
| ========================================================================================== |
| Supernova V4 : 17/18 passed |
| SentencePiece: 14/18 passed |
| |
| ========================================================================================== |
| 📦 TOKENIZATION |
| ========================================================================================== |
| Supernova V4 tokens : 2,520,000 |
| SentencePiece tokens: 1,075,000 |
| V4/SP token ratio : 2.344x |
| |
| ========================================================================================== |
| ⚡ RELATIVE PERFORMANCE |
| ========================================================================================== |
| Character speed ratio : 0.67x |
| Token throughput ratio: 1.56x |
| |
| ========================================================================================== |
| 🔬 RAW RUNS |
| ========================================================================================== |
| Supernova V4: |
| 0.7979s |
| 0.7781s |
| 0.7954s |
| 0.8023s |
| 0.7968s |
| |
| SentencePiece: |
| 0.5328s |
| 0.5605s |
| 0.5185s |
| 0.5375s |
| 0.6904s |
| |
| ========================================================================================== |
| 🏁 VERDICT |
| ========================================================================================== |
| ⚠️ Supernova V4 correctness: NEEDS INVESTIGATION |
| ⚠️ SentencePiece correctness: SOME DIFFERENCES |
| 🔥 Token throughput winner: SUPERNOVA V4 |
| ⚡ Character throughput winner: SENTENCEPIECE |
| ========================================================================================== |
| ## FOR CODE TEST RUN |
| # ================================================================ |
| # 🌌 SUPERNOVA V4 vs SENTENCEPIECE — FAIR FINAL COMPARISON |
| # ================================================================ |
| |
| import time |
| |
| # ------------------------------------------------ |
| # TEST CORPUS |
| # ------------------------------------------------ |
| |
| tests = [ |
| "नमस्ते, तपाईंलाई कस्तो छ?", |
| "नेपाल सुन्दर देश हो।", |
| "लुम्बिनी नेपालको प्रसिद्ध स्थान हो।", |
| "सगरमाथा नेपालको गौरव हो।", |
| "काठमाडौं नेपालको राजधानी हो।", |
| "पोखरा नेपालको सुन्दर शहर हो।", |
| "विज्ञान र प्रविधिले संसार परिवर्तन गरिरहेको छ।", |
| "अर्थतन्त्र र शिक्षा देशको विकासका आधार हुन्।", |
| "Supernova AI is being developed in Nepal.", |
| "Artificial Intelligence is changing the world.", |
| "√2 ≈ 1.4142135623730951", |
| "∑(xᵢ²) → ∞", |
| "π × r² ≠ 0", |
| "🇳🇵 🚀 🔥 🤖 🧠 💻 🌋", |
| "👨👩👧👦 👩💻 🧑🚀 🏃♂️", |
| "नमस्ते Hello こんにちは 안녕하세요 مرحبا", |
| "नेपाल Nepal 日本 Japan भारत India", |
| "— – … « » “ ” ‘ ’ ≠ ≤ ≥ ± × ÷ ∞", |
| ] |
| |
| # ------------------------------------------------ |
| # CORRECTNESS |
| # ------------------------------------------------ |
| |
| print("=" * 90) |
| print("🌌 SUPERNOVA V4 vs SENTENCEPIECE") |
| print("=" * 90) |
| |
| v4_pass = 0 |
| sp_pass = 0 |
| |
| v4_test_tokens = 0 |
| sp_test_tokens = 0 |
| |
| for text in tests: |
| |
| # V4 |
| v4_ids = v4_encode(text) |
| v4_unknown = sum(x == -1 for x in v4_ids) |
| v4_decoded = v4_decode(v4_ids) |
|
|
| # SentencePiece |
| sp_ids = sp.encode(text, out_type=int) |
| sp_decoded = sp.decode(sp_ids) |
| |
| v4_ok = ( |
| v4_unknown == 0 |
| and v4_decoded == text |
| ) |
| |
| sp_ok = ( |
| sp_decoded == text |
| ) |
| |
| v4_test_tokens += len(v4_ids) |
| sp_test_tokens += len(sp_ids) |
| |
| if v4_ok: |
| v4_pass += 1 |
| |
| if sp_ok: |
| sp_pass += 1 |
| |
| print( |
| f"\n{text}" |
| f"\n V4: {len(v4_ids):3} tokens | " |
| f"unknown={v4_unknown} | " |
| f"{'PASS' if v4_ok else 'FAIL'}" |
| f"\n SP: {len(sp_ids):3} tokens | " |
| f"{'PASS' if sp_ok else 'FAIL'}" |
| ) |
| |
| # ------------------------------------------------ |
| # BUILD LARGE IDENTICAL CORPUS |
| # ------------------------------------------------ |
|
|
| # Repeat the EXACT SAME documents for both tokenizers. |
| REPEATS = 5000 |
|
|
| corpus = tests * REPEATS |
|
|
| characters = sum(len(x) for x in corpus) |
|
|
| print("\n" + "=" * 90) |
| print("📦 BENCHMARK CORPUS") |
| print("=" * 90) |
|
|
| print("Documents :", f"{len(corpus):,}") |
| print("Characters:", f"{characters:,}") |
|
|
| # ------------------------------------------------ |
| # WARM-UP |
| # ------------------------------------------------ |
|
|
| for text in tests: |
| v4_encode(text) |
| sp.encode(text, out_type=int) |
| |
| # ------------------------------------------------ |
| # V4 BENCHMARK |
| # ------------------------------------------------ |
|
|
| v4_runs = [] |
| v4_tokens = 0 |
|
|
| for _ in range(5): |
|
|
| start = time.perf_counter() |
| |
| total = 0 |
| |
| for text in corpus: |
| total += len(v4_encode(text)) |
| |
| elapsed = time.perf_counter() - start |
| |
| v4_runs.append(elapsed) |
| v4_tokens = total |
| |
| # ------------------------------------------------ |
| # SENTENCEPIECE BENCHMARK |
| # ------------------------------------------------ |
|
|
| sp_runs = [] |
| sp_tokens = 0 |
|
|
| for _ in range(5): |
|
|
| start = time.perf_counter() |
| |
| total = 0 |
| |
| for text in corpus: |
| total += len(sp.encode(text, out_type=int)) |
| |
| elapsed = time.perf_counter() - start |
| |
| sp_runs.append(elapsed) |
| sp_tokens = total |
| |
| # Use the fastest run, reducing random Colab scheduling noise. |
| v4_time = min(v4_runs) |
| sp_time = min(sp_runs) |
|
|
| # ------------------------------------------------ |
| # METRICS |
| # ------------------------------------------------ |
|
|
| v4_chars_sec = characters / v4_time |
| sp_chars_sec = characters / sp_time |
|
|
| v4_tokens_sec = v4_tokens / v4_time |
| sp_tokens_sec = sp_tokens / sp_time |
|
|
| char_ratio = v4_chars_sec / sp_chars_sec |
| token_ratio = v4_tokens_sec / sp_tokens_sec |
|
|
| token_efficiency_ratio = v4_tokens / sp_tokens |
|
|
| # ------------------------------------------------ |
| # FINAL TABLE |
| # ------------------------------------------------ |
|
|
| print("\n" + "=" * 90) |
| print("🏆 FINAL PERFORMANCE") |
| print("=" * 90) |
|
|
| print( |
| f"{'ENGINE':25}" |
| f"{'TIME':>12}" |
| f"{'CHARS/S':>18}" |
| f"{'TOKENS/S':>18}" |
| ) |
| |
| print("-" * 90) |
|
|
| print( |
| f"{'Supernova V4':25}" |
| f"{v4_time:>12.4f}" |
| f"{v4_chars_sec:>18,.0f}" |
| f"{v4_tokens_sec:>18,.0f}" |
| ) |
| |
| print( |
| f"{'SentencePiece':25}" |
| f"{sp_time:>12.4f}" |
| f"{sp_chars_sec:>18,.0f}" |
| f"{sp_tokens_sec:>18,.0f}" |
| ) |
| |
| # ------------------------------------------------ |
| # CORRECTNESS SUMMARY |
| # ------------------------------------------------ |
|
|
| print("\n" + "=" * 90) |
| print("🧪 CORRECTNESS") |
| print("=" * 90) |
|
|
| print( |
| f"Supernova V4 : {v4_pass}/{len(tests)} passed" |
| ) |
| |
| print( |
| f"SentencePiece: {sp_pass}/{len(tests)} passed" |
| ) |
| |
| # ------------------------------------------------ |
| # TOKENIZATION |
| # ------------------------------------------------ |
|
|
| print("\n" + "=" * 90) |
| print("📦 TOKENIZATION") |
| print("=" * 90) |
|
|
| print( |
| f"Supernova V4 tokens : {v4_tokens:,}" |
| ) |
| |
| print( |
| f"SentencePiece tokens: {sp_tokens:,}" |
| ) |
| |
| print( |
| f"V4/SP token ratio : {token_efficiency_ratio:.3f}x" |
| ) |
| |
| # ------------------------------------------------ |
| # RELATIVE PERFORMANCE |
| # ------------------------------------------------ |
|
|
| print("\n" + "=" * 90) |
| print("⚡ RELATIVE PERFORMANCE") |
| print("=" * 90) |
|
|
| print( |
| f"Character speed ratio : {char_ratio:.2f}x" |
| ) |
| |
| print( |
| f"Token throughput ratio: {token_ratio:.2f}x" |
| ) |
| |
| # ------------------------------------------------ |
| # RAW RUNS |
| # ------------------------------------------------ |
|
|
| print("\n" + "=" * 90) |
| print("🔬 RAW RUNS") |
| print("=" * 90) |
|
|
| print("Supernova V4:") |
| for x in v4_runs: |
| print(f" {x:.4f}s") |
| |
| print("\nSentencePiece:") |
| for x in sp_runs: |
| print(f" {x:.4f}s") |
| |
| # ------------------------------------------------ |
| # VERDICT |
| # ------------------------------------------------ |
|
|
| print("\n" + "=" * 90) |
| print("🏁 VERDICT") |
| print("=" * 90) |
|
|
| if v4_pass == len(tests): |
| print("✅ Supernova V4 correctness: FULL PASS") |
| else: |
| print("⚠️ Supernova V4 correctness: NEEDS INVESTIGATION") |
| |
| if sp_pass == len(tests): |
| print("✅ SentencePiece correctness: FULL PASS") |
| else: |
| print("⚠️ SentencePiece correctness: SOME DIFFERENCES") |
| |
| if v4_tokens_sec > sp_tokens_sec: |
| print("🔥 Token throughput winner: SUPERNOVA V4") |
| else: |
| print("🔥 Token throughput winner: SENTENCEPIECE") |
| |
| if v4_chars_sec > sp_chars_sec: |
| print("⚡ Character throughput winner: SUPERNOVA V4") |
| else: |
| print("⚡ Character throughput winner: SENTENCEPIECE") |
| |
| print("=" * 90) |
