Upload tokenizer
Browse files- special_tokens_map.json +2 -1
- tokenizer_config.json +1 -0
special_tokens_map.json
CHANGED
|
@@ -77,5 +77,6 @@
|
|
| 77 |
"normalized": false,
|
| 78 |
"rstrip": false,
|
| 79 |
"single_word": false
|
| 80 |
-
}
|
|
|
|
| 81 |
}
|
|
|
|
| 77 |
"normalized": false,
|
| 78 |
"rstrip": false,
|
| 79 |
"single_word": false
|
| 80 |
+
},
|
| 81 |
+
"pad_token": "<|eot_id|>"
|
| 82 |
}
|
tokenizer_config.json
CHANGED
|
@@ -2141,5 +2141,6 @@
|
|
| 2141 |
"attention_mask"
|
| 2142 |
],
|
| 2143 |
"model_max_length": 131072,
|
|
|
|
| 2144 |
"tokenizer_class": "PreTrainedTokenizerFast"
|
| 2145 |
}
|
|
|
|
| 2141 |
"attention_mask"
|
| 2142 |
],
|
| 2143 |
"model_max_length": 131072,
|
| 2144 |
+
"pad_token": "<|eot_id|>",
|
| 2145 |
"tokenizer_class": "PreTrainedTokenizerFast"
|
| 2146 |
}
|