Upload tokenizer_config.json with huggingface_hub
Browse files- tokenizer_config.json +7 -2
tokenizer_config.json
CHANGED
|
@@ -69,5 +69,10 @@
|
|
| 69 |
"pad_token": "[PAD]",
|
| 70 |
"sep_token": "[SEP]",
|
| 71 |
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 72 |
-
"unk_token": "[UNK]"
|
| 73 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
"pad_token": "[PAD]",
|
| 70 |
"sep_token": "[SEP]",
|
| 71 |
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 72 |
+
"unk_token": "[UNK]",
|
| 73 |
+
"model_type": "modernbert",
|
| 74 |
+
"model_input_names": [
|
| 75 |
+
"input_ids",
|
| 76 |
+
"attention_mask"
|
| 77 |
+
]
|
| 78 |
+
}
|