benchmark.py

import json
from pathlib import Path

ROOT = Path(__file__).parent

with open(ROOT / "tokenizer.json", "r", encoding="utf-8") as f:
    data = json.load(f)

id_to_token = {
    int(k): v
    for k, v in data["vocab"].items()
}

token_to_id = {
    token: idx
    for idx, token in id_to_token.items()
}

tokens = sorted(
    token_to_id,
    key=len,
    reverse=True
)

def encode(text):
    ids = []
    i = 0

    while i < len(text):
        found = None

        for token in tokens:
            if text.startswith(token, i):
                found = token
                break

        if found is None:
            ids.append(-1)
            i += 1
        else:
            ids.append(token_to_id[found])
            i += len(found)

    return ids

def decode(ids):
    return ''.join(
        id_to_token.get(i, '')
        for i in ids
        if i >= 0
    )

tests = [
    "नमस्ते नेपाल 🇳🇵",
    "लाख करोड अरब खर्ब हजार",
    "नेपाल सुन्दर देश हो।",
    "√2 ≈ 1.4142135623730951",
    "∑(xᵢ²) → ∞",
    "🇳🇵 🚀 🔥 🤖 🧠 💻 🌋",
    "नमस्ते Hello こんにちは 안녕하세요 مرحبا",
    "नेपाल Nepal 日本 Japan भारत India",
    "— – … « » “ ” ‘ ’ ≠ ≤ ≥ ± × ÷ ∞"
]

print("=" * 70)
print("SUPERNOVA NEPALIFAST V4 LOCAL TEST")
print("=" * 70)

for text in tests:
    ids = encode(text)
    unknown = sum(x == -1 for x in ids)
    decoded = decode(ids)

    print("\nText:", text)
    print("Tokens:", len(ids))
    print("Unknown:", unknown)
    print("Roundtrip:", decoded == text)