tokenizer_config: drop transformers-5-only extra_special_tokens list (breaks 4.x load)
Browse files- tokenizer_config.json +2 -9
tokenizer_config.json
CHANGED
|
@@ -5,15 +5,8 @@
|
|
| 5 |
"cls_token": "[CLS]",
|
| 6 |
"do_lower_case": false,
|
| 7 |
"eos_token": "[SEP]",
|
| 8 |
-
"extra_special_tokens": [
|
| 9 |
-
"[PAD]",
|
| 10 |
-
"[CLS]",
|
| 11 |
-
"[SEP]"
|
| 12 |
-
],
|
| 13 |
-
"is_local": false,
|
| 14 |
-
"local_files_only": false,
|
| 15 |
"mask_token": "[MASK]",
|
| 16 |
-
"model_max_length":
|
| 17 |
"pad_token": "[PAD]",
|
| 18 |
"sep_token": "[SEP]",
|
| 19 |
"split_by_punct": false,
|
|
@@ -21,4 +14,4 @@
|
|
| 21 |
"unk_id": 3,
|
| 22 |
"unk_token": "[UNK]",
|
| 23 |
"vocab_type": "spm"
|
| 24 |
-
}
|
|
|
|
| 5 |
"cls_token": "[CLS]",
|
| 6 |
"do_lower_case": false,
|
| 7 |
"eos_token": "[SEP]",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
"mask_token": "[MASK]",
|
| 9 |
+
"model_max_length": 224,
|
| 10 |
"pad_token": "[PAD]",
|
| 11 |
"sep_token": "[SEP]",
|
| 12 |
"split_by_punct": false,
|
|
|
|
| 14 |
"unk_id": 3,
|
| 15 |
"unk_token": "[UNK]",
|
| 16 |
"vocab_type": "spm"
|
| 17 |
+
}
|