Commit ·
9eaffad
1
Parent(s): 47f62c8
Upload tokenizer
Browse files- added_tokens.json +1 -0
- tokenizer.json +9 -0
added_tokens.json
CHANGED
|
@@ -18,6 +18,7 @@
|
|
| 18 |
"6": 519,
|
| 19 |
"7": 520,
|
| 20 |
"8": 521,
|
|
|
|
| 21 |
":": 505,
|
| 22 |
";": 506,
|
| 23 |
"?": 503
|
|
|
|
| 18 |
"6": 519,
|
| 19 |
"7": 520,
|
| 20 |
"8": 521,
|
| 21 |
+
"9": 522,
|
| 22 |
":": 505,
|
| 23 |
";": 506,
|
| 24 |
"?": 503
|
tokenizer.json
CHANGED
|
@@ -245,6 +245,15 @@
|
|
| 245 |
"rstrip": false,
|
| 246 |
"normalized": true,
|
| 247 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 248 |
}
|
| 249 |
],
|
| 250 |
"normalizer": {
|
|
|
|
| 245 |
"rstrip": false,
|
| 246 |
"normalized": true,
|
| 247 |
"special": false
|
| 248 |
+
},
|
| 249 |
+
{
|
| 250 |
+
"id": 522,
|
| 251 |
+
"content": "9",
|
| 252 |
+
"single_word": false,
|
| 253 |
+
"lstrip": false,
|
| 254 |
+
"rstrip": false,
|
| 255 |
+
"normalized": true,
|
| 256 |
+
"special": false
|
| 257 |
}
|
| 258 |
],
|
| 259 |
"normalizer": {
|