from pathlib import Path

from tokenizers import Tokenizer


root = Path(__file__).resolve().parent
tokenizer_path = root / "tokenizer.json"
tokenizer = Tokenizer.from_file(str(tokenizer_path))

if tokenizer.get_vocab_size() != 65_536:
    raise RuntimeError(f"Unexpected vocabulary size: {tokenizer.get_vocab_size()}")

samples = [
    "المملكة العربية السعودية",
    "السلام عليكم ورحمة الله وبركاته",
    "Artificial intelligence والذكاء الاصطناعي",
    "ياهلا ومرحبا فيكم",
]

for text in samples:
    encoding = tokenizer.encode(text, add_special_tokens=False)
    decoded = tokenizer.decode(encoding.ids)
    if decoded != text:
        raise RuntimeError(f"Round-trip failure: {text!r} -> {decoded!r}")

print("Tokenizer validation passed")
print(f"Vocabulary size: {tokenizer.get_vocab_size():,}")