Update README.md

from transformers import T5Tokenizer
text = "сайн уу"

mn_tokenizer = T5Tokenizer(vocab_file="mn_tokenizer.model")

tokens = mn_tokenizer.tokenize(text)
encoded = mn_tokenizer.encode(text)
decoded = mn_tokenizer.decode(encoded)

print("Original:", text)
print("Mongolian tokenizer tokens:", tokens)
print("Decoded :", decoded)

README.md

@@ -1,12 +0,0 @@
-from transformers import T5Tokenizer
-text = "сайн уу"
-
-mn_tokenizer = T5Tokenizer(vocab_file="mn_tokenizer.model")
-
-tokens = mn_tokenizer.tokenize(text)
-encoded = mn_tokenizer.encode(text)
-decoded = mn_tokenizer.decode(encoded)
-
-print("Original:", text)
-print("Mongolian tokenizer tokens:", tokens)
-print("Decoded :", decoded)