Upload processor
Browse files- added_tokens.json +4 -3
- tokenizer.json +13 -4
added_tokens.json
CHANGED
|
@@ -10,13 +10,14 @@
|
|
| 10 |
"<invoice/>": 57530,
|
| 11 |
"<letter/>": 57531,
|
| 12 |
"<memo/>": 57532,
|
| 13 |
-
"<news/>":
|
| 14 |
"<news_article/>": 57533,
|
| 15 |
-
"<note/>":
|
| 16 |
"<passport/>": 57544,
|
| 17 |
"<presentation/>": 57534,
|
| 18 |
"<questionnaire/>": 57535,
|
| 19 |
-
"<
|
|
|
|
| 20 |
"<resume/>": 57536,
|
| 21 |
"<s_class>": 57537,
|
| 22 |
"<s_iitcdip>": 57538,
|
|
|
|
| 10 |
"<invoice/>": 57530,
|
| 11 |
"<letter/>": 57531,
|
| 12 |
"<memo/>": 57532,
|
| 13 |
+
"<news/>": 57547,
|
| 14 |
"<news_article/>": 57533,
|
| 15 |
+
"<note/>": 57548,
|
| 16 |
"<passport/>": 57544,
|
| 17 |
"<presentation/>": 57534,
|
| 18 |
"<questionnaire/>": 57535,
|
| 19 |
+
"<receipt/>": 57546,
|
| 20 |
+
"<report/>": 57549,
|
| 21 |
"<resume/>": 57536,
|
| 22 |
"<s_class>": 57537,
|
| 23 |
"<s_iitcdip>": 57538,
|
tokenizer.json
CHANGED
|
@@ -149,8 +149,8 @@
|
|
| 149 |
"single_word": false,
|
| 150 |
"lstrip": false,
|
| 151 |
"rstrip": false,
|
| 152 |
-
"normalized":
|
| 153 |
-
"special":
|
| 154 |
},
|
| 155 |
{
|
| 156 |
"id": 57532,
|
|
@@ -280,7 +280,7 @@
|
|
| 280 |
},
|
| 281 |
{
|
| 282 |
"id": 57546,
|
| 283 |
-
"content": "<
|
| 284 |
"single_word": false,
|
| 285 |
"lstrip": false,
|
| 286 |
"rstrip": false,
|
|
@@ -289,7 +289,7 @@
|
|
| 289 |
},
|
| 290 |
{
|
| 291 |
"id": 57547,
|
| 292 |
-
"content": "<
|
| 293 |
"single_word": false,
|
| 294 |
"lstrip": false,
|
| 295 |
"rstrip": false,
|
|
@@ -298,6 +298,15 @@
|
|
| 298 |
},
|
| 299 |
{
|
| 300 |
"id": 57548,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 301 |
"content": "<report/>",
|
| 302 |
"single_word": false,
|
| 303 |
"lstrip": false,
|
|
|
|
| 149 |
"single_word": false,
|
| 150 |
"lstrip": false,
|
| 151 |
"rstrip": false,
|
| 152 |
+
"normalized": false,
|
| 153 |
+
"special": true
|
| 154 |
},
|
| 155 |
{
|
| 156 |
"id": 57532,
|
|
|
|
| 280 |
},
|
| 281 |
{
|
| 282 |
"id": 57546,
|
| 283 |
+
"content": "<receipt/>",
|
| 284 |
"single_word": false,
|
| 285 |
"lstrip": false,
|
| 286 |
"rstrip": false,
|
|
|
|
| 289 |
},
|
| 290 |
{
|
| 291 |
"id": 57547,
|
| 292 |
+
"content": "<news/>",
|
| 293 |
"single_word": false,
|
| 294 |
"lstrip": false,
|
| 295 |
"rstrip": false,
|
|
|
|
| 298 |
},
|
| 299 |
{
|
| 300 |
"id": 57548,
|
| 301 |
+
"content": "<note/>",
|
| 302 |
+
"single_word": false,
|
| 303 |
+
"lstrip": false,
|
| 304 |
+
"rstrip": false,
|
| 305 |
+
"normalized": true,
|
| 306 |
+
"special": false
|
| 307 |
+
},
|
| 308 |
+
{
|
| 309 |
+
"id": 57549,
|
| 310 |
"content": "<report/>",
|
| 311 |
"single_word": false,
|
| 312 |
"lstrip": false,
|