File size: 507 Bytes
ea80f98 d1d3fe6 ea80f98 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 | from transformers import PreTrainedTokenizerFast
REPO_ID = "Samai-sa/SamAI-Tokenizer"
tokenizer = PreTrainedTokenizerFast.from_pretrained(REPO_ID)
text = "المملكة العربية السعودية"
encoding = tokenizer(text, add_special_tokens=False)
print("Text:", text)
print("Token IDs:", encoding["input_ids"])
print("Tokens:", tokenizer.convert_ids_to_tokens(encoding["input_ids"]))
print("Token count:", len(encoding["input_ids"]))
print("Decoded:", tokenizer.decode(encoding["input_ids"]))
|