Upload tokenizer
Browse files- special_tokens_map.json +1 -7
special_tokens_map.json
CHANGED
|
@@ -285,13 +285,7 @@
|
|
| 285 |
"rstrip": false,
|
| 286 |
"single_word": false
|
| 287 |
},
|
| 288 |
-
"sep_token":
|
| 289 |
-
"content": "<EOL>",
|
| 290 |
-
"lstrip": false,
|
| 291 |
-
"normalized": false,
|
| 292 |
-
"rstrip": false,
|
| 293 |
-
"single_word": false
|
| 294 |
-
},
|
| 295 |
"unk_token": {
|
| 296 |
"content": "<|UNKNOWN|>",
|
| 297 |
"lstrip": false,
|
|
|
|
| 285 |
"rstrip": false,
|
| 286 |
"single_word": false
|
| 287 |
},
|
| 288 |
+
"sep_token": "<EOL>",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 289 |
"unk_token": {
|
| 290 |
"content": "<|UNKNOWN|>",
|
| 291 |
"lstrip": false,
|