Update README.md

README.md

@@ -3,12 +3,13 @@ from transformers import LlamaTokenizer
 
 tokenizer = LlamaTokenizer.from_pretrained(
         'ocisd4/openllama_tokenizer_ext_zh',
-        pad_token="<pad>",
         add_bos_token=True,
         add_eos_token=False,
         use_auth_token='True',
 )
 
+tokenizer.pad_token_id = tokenizer.vocab_size -1
+
 print('vocab size:',tokenizer.vocab_size)
 #vocab size: 52928
 
@@ -33,4 +34,4 @@ print(tokenizer.decode(tokenizer.encode(text)))
 
 ### updated
 #### 2023-06-02
-  - add special tokens: <|output|>, <|input|>, <|sep|>, <|emb|>, <|rwd|>, <|ctx|>
+  - add special tokens: <|pad|><|output|>, <|input|>, <|sep|>, <|emb|>, <|rwd|>, <|ctx|>