goryden
/

mn_tokenizer

Model card Files Files and versions

goryden commited on May 13, 2025

Commit

d4c0e45

·

verified ·

1 Parent(s): 8838402

Update README.md

Files changed (1) hide show

README.md +1 -1

README.md CHANGED Viewed

@@ -9,7 +9,7 @@ language:
 from transformers import AutoTokenizer
 text = "сайн уу"
-AutoTokenizer.from_pretrained("goryden/mn_tokenizer")
 tokens = mn_tokenizer.tokenize(text)
 encoded = mn_tokenizer.encode(text)

 from transformers import AutoTokenizer
 text = "сайн уу"
+mn_tokenizer = AutoTokenizer.from_pretrained("goryden/mn_tokenizer")
 tokens = mn_tokenizer.tokenize(text)
 encoded = mn_tokenizer.encode(text)