File size: 886 Bytes
d1d3fe6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | from pathlib import Path
from tokenizers import Tokenizer
root = Path(__file__).resolve().parent
tokenizer_path = root / "tokenizer.json"
tokenizer = Tokenizer.from_file(str(tokenizer_path))
if tokenizer.get_vocab_size() != 65_536:
raise RuntimeError(f"Unexpected vocabulary size: {tokenizer.get_vocab_size()}")
samples = [
"المملكة العربية السعودية",
"السلام عليكم ورحمة الله وبركاته",
"Artificial intelligence والذكاء الاصطناعي",
"ياهلا ومرحبا فيكم",
]
for text in samples:
encoding = tokenizer.encode(text, add_special_tokens=False)
decoded = tokenizer.decode(encoding.ids)
if decoded != text:
raise RuntimeError(f"Round-trip failure: {text!r} -> {decoded!r}")
print("Tokenizer validation passed")
print(f"Vocabulary size: {tokenizer.get_vocab_size():,}")
|