Upload tokenizer_config.json
Browse files- tokenizer_config.json +1 -13
tokenizer_config.json
CHANGED
|
@@ -1,13 +1 @@
|
|
| 1 |
-
{"eos_token":
|
| 2 |
-
{"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"},
|
| 3 |
-
"bos_token":
|
| 4 |
-
{"content": "<bos>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"},
|
| 5 |
-
"pad_token":
|
| 6 |
-
{"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"},
|
| 7 |
-
"mask_token":
|
| 8 |
-
{"content": "<mask>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"},
|
| 9 |
-
"cls_token":
|
| 10 |
-
{"content": "<cls>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"},
|
| 11 |
-
"sep_token":
|
| 12 |
-
{"content": "<sep>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}
|
| 13 |
-
}
|
|
|
|
| 1 |
+
{"pad_token": {"content": "[PAD]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "bos_token": {"content": "[BOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "[EOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "mask_token": {"content": "[MASK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "cls_token": {"content": "[CLS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "sep_token": {"content": "[SEP]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "model_max_length": 2048, "tokenizer_class": "PerceiverTokenizer"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|