try to fix added_tokens error
Browse files- tokenizer_config.json +2 -1
tokenizer_config.json
CHANGED
|
@@ -7,5 +7,6 @@
|
|
| 7 |
"bos_token": "/s",
|
| 8 |
"eos_token": "s",
|
| 9 |
"unk_token": "unk",
|
| 10 |
-
"pad_token": "pad"
|
|
|
|
| 11 |
}
|
|
|
|
| 7 |
"bos_token": "/s",
|
| 8 |
"eos_token": "s",
|
| 9 |
"unk_token": "unk",
|
| 10 |
+
"pad_token": "pad",
|
| 11 |
+
"added_tokens": []
|
| 12 |
}
|