feat: simplified mon tokenizer in hf format, updated tags

README.md

@@ -20,11 +20,26 @@ sentencepiece tokenizer for mon language with 4,000 vocabulary.
 ```python
 from transformers import AutoTokenizer
 
+# Load the tokenizer
 tokenizer = AutoTokenizer.from_pretrained("janakhpon/mon_tokenizer")
 
-text = "ဘာသာမန် ပရူပရာတံဂှ် ကၠောန်ဗဒှ်လဝ်ရ။"
+# Example text
+text = "ပ္ဍဲအခိင်မာံနဲသဵု မဒှ်ဘဝကွးဘာတက္ကသိုလ်ဂှ် ပါလုပ်ချဳဓရာင်ကၠုင် ပ္ဍဲပရေင်ကမၠောန်ယေန်သၞာင် ကေုာံ လိက်ပတ်မန် ဗွဲကတိုင်ကၟဟ်ရ။"
+
+# Tokenize the text
 tokens = tokenizer(text, return_tensors="pt")
-decoded = tokenizer.decode(tokens["input_ids"][0])
+input_ids = tokens["input_ids"][0]
+
+# Print token IDs
+print("Token IDs:", input_ids.tolist())
+
+# Print tokens
+token_list = tokenizer.convert_ids_to_tokens(input_ids)
+print("Tokens:", token_list)
+
+# Decode back to text
+decoded = tokenizer.decode(input_ids, skip_special_tokens=True)
+print("Decoded text:", decoded)
 ```
 
 ## details