| from pathlib import Path | |
| from tokenizers import Tokenizer | |
| root = Path(__file__).resolve().parent | |
| tokenizer_path = root / "tokenizer.json" | |
| tokenizer = Tokenizer.from_file(str(tokenizer_path)) | |
| if tokenizer.get_vocab_size() != 65_536: | |
| raise RuntimeError(f"Unexpected vocabulary size: {tokenizer.get_vocab_size()}") | |
| samples = [ | |
| "المملكة العربية السعودية", | |
| "السلام عليكم ورحمة الله وبركاته", | |
| "Artificial intelligence والذكاء الاصطناعي", | |
| "ياهلا ومرحبا فيكم", | |
| ] | |
| for text in samples: | |
| encoding = tokenizer.encode(text, add_special_tokens=False) | |
| decoded = tokenizer.decode(encoding.ids) | |
| if decoded != text: | |
| raise RuntimeError(f"Round-trip failure: {text!r} -> {decoded!r}") | |
| print("Tokenizer validation passed") | |
| print(f"Vocabulary size: {tokenizer.get_vocab_size():,}") | |