Update README.md
Browse files
README.md
CHANGED
|
@@ -40,3 +40,15 @@ Designed for Banking and Fintech Institutions
|
|
| 40 |
- model.resize_token_embeddings(len(tokenizer.tokenizer)) # или просто len(tokenizer.tokenizer)
|
| 41 |
|
| 42 |
- print("New Embedding size for you chat script:", model.get_input_embeddings().weight.shape[0])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
- model.resize_token_embeddings(len(tokenizer.tokenizer)) # или просто len(tokenizer.tokenizer)
|
| 41 |
|
| 42 |
- print("New Embedding size for you chat script:", model.get_input_embeddings().weight.shape[0])
|
| 43 |
+
|
| 44 |
+
- # Tesr Tokenizer size !
|
| 45 |
+
(venv_ji) root@jirack2:# python -c '
|
| 46 |
+
from transformers import AutoTokenizer
|
| 47 |
+
tok = AutoTokenizer.from_pretrained("./jirack_code_tokenizer_fixed")
|
| 48 |
+
print("Vocab size:", len(tok))
|
| 49 |
+
print("pad_token_id:", tok.pad_token_id)
|
| 50 |
+
print("eos_token_id:", tok.eos_token_id)
|
| 51 |
+
'
|
| 52 |
+
Vocab size: 128259
|
| 53 |
+
pad_token_id: 128001
|
| 54 |
+
eos_token_id: 128001
|