| from transformers import PreTrainedTokenizerFast | |
| REPO_ID = "Samai-sa/SamAI-Tokenizer" | |
| tokenizer = PreTrainedTokenizerFast.from_pretrained(REPO_ID) | |
| text = "المملكة العربية السعودية" | |
| encoding = tokenizer(text, add_special_tokens=False) | |
| print("Text:", text) | |
| print("Token IDs:", encoding["input_ids"]) | |
| print("Tokens:", tokenizer.convert_ids_to_tokens(encoding["input_ids"])) | |
| print("Token count:", len(encoding["input_ids"])) | |
| print("Decoded:", tokenizer.decode(encoding["input_ids"])) | |