End of training
Browse files- tokenizer.json +1 -1
tokenizer.json
CHANGED
|
@@ -46,7 +46,7 @@
|
|
| 46 |
{
|
| 47 |
"type": "Split",
|
| 48 |
"pattern": {
|
| 49 |
-
"Regex": "
|
| 50 |
},
|
| 51 |
"behavior": "Removed",
|
| 52 |
"invert": true
|
|
|
|
| 46 |
{
|
| 47 |
"type": "Split",
|
| 48 |
"pattern": {
|
| 49 |
+
"Regex": "'s|'t|'re|'ve|'m|'ll|'d|[\\p{L}]+|[\\p{N}]|[^\\s\\p{L}\\p{N}]+"
|
| 50 |
},
|
| 51 |
"behavior": "Removed",
|
| 52 |
"invert": true
|