Upload tokenizer
Browse files- special_tokens_map.json +7 -1
special_tokens_map.json
CHANGED
|
@@ -285,7 +285,13 @@
|
|
| 285 |
"rstrip": false,
|
| 286 |
"single_word": false
|
| 287 |
},
|
| 288 |
-
"sep_token":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 289 |
"unk_token": {
|
| 290 |
"content": "<|UNKNOWN|>",
|
| 291 |
"lstrip": false,
|
|
|
|
| 285 |
"rstrip": false,
|
| 286 |
"single_word": false
|
| 287 |
},
|
| 288 |
+
"sep_token": {
|
| 289 |
+
"content": "<EOL>",
|
| 290 |
+
"lstrip": false,
|
| 291 |
+
"normalized": false,
|
| 292 |
+
"rstrip": false,
|
| 293 |
+
"single_word": false
|
| 294 |
+
},
|
| 295 |
"unk_token": {
|
| 296 |
"content": "<|UNKNOWN|>",
|
| 297 |
"lstrip": false,
|