Update README.md

README.md

@@ -1,24 +1,31 @@
+
+Mistral擴充詞表只包含教育部常用8000字
+
+
 ```python
-from transformers import LlamaTokenizer
+from transformers import AutoTokenizer
 
-tokenizer = LlamaTokenizer.from_pretrained(  
-        'ocisd4/llama_tokenizer_ext_zhtw',  
+tokenizer = AutoTokenizer.from_pretrained(  
+        'ocisd4/mistral_tokenizer_ext',  
         pad_token='<unk>',  
         add_bos_token=True,  
         add_eos_token=False  
 )  
 
-#vocab size: 36128     
+print('vocab size:', tokenizer.vocab_size)   
+#vocab size: 35712     
 
 print(tokenizer.tokenize('今天天氣真好！'))   
 #['▁', '今', '天', '天', '氣', '真', '好', '！']
 
 print(tokenizer.encode('今天天氣真好！'))  
-#[1, 29871, 31482, 30408, 30408, 32045, 30848, 31076, 30584]
+#[1, 28705, 30316, 29354, 29354, 32004, 29974, 29530, 29267]
 
 print(tokenizer.decode(tokenizer.encode('今天天氣真好！')))  
-# <s>今天天氣真好！
+#<s> 今天天氣真好！
 ```
 
 
+Mistral
+