from transformers import PreTrainedTokenizerFast


REPO_ID = "Samai-sa/SamAI-Tokenizer"

tokenizer = PreTrainedTokenizerFast.from_pretrained(REPO_ID)

text = "المملكة العربية السعودية"
encoding = tokenizer(text, add_special_tokens=False)

print("Text:", text)
print("Token IDs:", encoding["input_ids"])
print("Tokens:", tokenizer.convert_ids_to_tokens(encoding["input_ids"]))
print("Token count:", len(encoding["input_ids"]))
print("Decoded:", tokenizer.decode(encoding["input_ids"]))