Upload tokenizer.json
#2
by vrdn23 - opened
- tokenizer.json +9 -2
tokenizer.json
CHANGED
|
@@ -54,9 +54,16 @@
|
|
| 54 |
"special": true
|
| 55 |
}
|
| 56 |
],
|
| 57 |
-
"normalizer":
|
|
|
|
|
|
|
| 58 |
"pre_tokenizer": {
|
| 59 |
-
"type": "
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
},
|
| 61 |
"post_processor": {
|
| 62 |
"type": "RobertaProcessing",
|
|
|
|
| 54 |
"special": true
|
| 55 |
}
|
| 56 |
],
|
| 57 |
+
"normalizer": {
|
| 58 |
+
"type": "Lowercase"
|
| 59 |
+
},
|
| 60 |
"pre_tokenizer": {
|
| 61 |
+
"type": "Split",
|
| 62 |
+
"pattern": {
|
| 63 |
+
"String": ""
|
| 64 |
+
},
|
| 65 |
+
"behavior": "Removed",
|
| 66 |
+
"invert": false
|
| 67 |
},
|
| 68 |
"post_processor": {
|
| 69 |
"type": "RobertaProcessing",
|