openhands commited on
Commit ·
5b5d351
1
Parent(s): a44d22b
Set pad_token=<|im_end|> (id=11), eos_token_id=11, unk_token_id=0
Browse files- tokenizer_config.json +6 -2
tokenizer_config.json
CHANGED
|
@@ -8024,5 +8024,9 @@
|
|
| 8024 |
],
|
| 8025 |
"model_max_length": 262144,
|
| 8026 |
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 8027 |
-
"unk_token": "<unk>"
|
| 8028 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8024 |
],
|
| 8025 |
"model_max_length": 262144,
|
| 8026 |
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 8027 |
+
"unk_token": "<unk>",
|
| 8028 |
+
"pad_token": "<|im_end|>",
|
| 8029 |
+
"pad_token_id": 11,
|
| 8030 |
+
"eos_token_id": 11,
|
| 8031 |
+
"unk_token_id": 0
|
| 8032 |
+
}
|