Upload processor

added_tokens.json

@@ -1,90 +1,26 @@
 {
-  "</s_BIC>": 50344,
-  "</s_IBAN>": 50380,
-  "</s_bank_account_number>": 50353,
-  "</s_const_symbol>": 50369,
-  "</s_currency>": 50360,
-  "</s_customer_name>": 50346,
-  "</s_customer_register_id>": 50382,
-  "</s_customer_tax_id>": 50359,
-  "</s_due_date>": 50378,
-  "</s_invoice_number>": 50365,
-  "</s_issue_date>": 50356,
-  "</s_payment>": 50377,
-  "</s_supplier_name>": 50366,
-  "</s_supplier_register_id>": 50345,
-  "</s_supplier_tax_id>": 50364,
-  "</s_taxable_supply_date>": 50373,
-  "</s_total_price>": 50349,
-  "</s_total_vat>": 50370,
-  "</s_variable_symbol>": 50371,
-  "<parsing>": 50361,
-  "<reserved_10>": 50392,
-  "<reserved_11>": 50393,
-  "<reserved_12>": 50394,
-  "<reserved_13>": 50395,
-  "<reserved_14>": 50396,
-  "<reserved_15>": 50397,
-  "<reserved_16>": 50398,
-  "<reserved_17>": 50399,
-  "<reserved_18>": 50400,
-  "<reserved_19>": 50401,
-  "<reserved_1>": 50383,
-  "<reserved_20>": 50402,
-  "<reserved_21>": 50403,
-  "<reserved_22>": 50404,
-  "<reserved_23>": 50405,
-  "<reserved_24>": 50406,
-  "<reserved_25>": 50407,
-  "<reserved_26>": 50408,
-  "<reserved_27>": 50409,
-  "<reserved_28>": 50410,
-  "<reserved_29>": 50411,
-  "<reserved_2>": 50384,
-  "<reserved_30>": 50412,
-  "<reserved_31>": 50413,
-  "<reserved_32>": 50414,
-  "<reserved_33>": 50415,
-  "<reserved_34>": 50416,
-  "<reserved_35>": 50417,
-  "<reserved_36>": 50418,
-  "<reserved_37>": 50419,
-  "<reserved_38>": 50420,
-  "<reserved_39>": 50421,
-  "<reserved_3>": 50385,
-  "<reserved_40>": 50422,
-  "<reserved_41>": 50423,
-  "<reserved_42>": 50424,
-  "<reserved_43>": 50425,
-  "<reserved_44>": 50426,
-  "<reserved_45>": 50427,
-  "<reserved_46>": 50428,
-  "<reserved_47>": 50429,
-  "<reserved_48>": 50430,
-  "<reserved_49>": 50431,
-  "<reserved_4>": 50386,
-  "<reserved_5>": 50387,
-  "<reserved_6>": 50388,
-  "<reserved_7>": 50389,
-  "<reserved_8>": 50390,
-  "<reserved_9>": 50391,
-  "<s_BIC>": 50372,
-  "<s_IBAN>": 50363,
-  "<s_bank_account_number>": 50367,
-  "<s_const_symbol>": 50368,
-  "<s_currency>": 50362,
-  "<s_customer_name>": 50352,
-  "<s_customer_register_id>": 50375,
-  "<s_customer_tax_id>": 50355,
-  "<s_due_date>": 50381,
-  "<s_invoice_number>": 50354,
-  "<s_issue_date>": 50358,
-  "<s_payment>": 50379,
-  "<s_supplier_name>": 50357,
-  "<s_supplier_register_id>": 50351,
-  "<s_supplier_tax_id>": 50348,
-  "<s_taxable_supply_date>": 50374,
-  "<s_total_price>": 50347,
-  "<s_total_vat>": 50376,
-  "<s_variable_symbol>": 50350
+  "<parsing>": 50344,
+  "<reserved_10>": 50354,
+  "<reserved_11>": 50355,
+  "<reserved_12>": 50356,
+  "<reserved_13>": 50357,
+  "<reserved_14>": 50358,
+  "<reserved_15>": 50359,
+  "<reserved_16>": 50360,
+  "<reserved_17>": 50361,
+  "<reserved_18>": 50362,
+  "<reserved_19>": 50363,
+  "<reserved_1>": 50345,
+  "<reserved_20>": 50364,
+  "<reserved_21>": 50365,
+  "<reserved_22>": 50366,
+  "<reserved_23>": 50367,
+  "<reserved_2>": 50346,
+  "<reserved_3>": 50347,
+  "<reserved_4>": 50348,
+  "<reserved_5>": 50349,
+  "<reserved_6>": 50350,
+  "<reserved_7>": 50351,
+  "<reserved_8>": 50352,
+  "<reserved_9>": 50353
 }

special_tokens_map.json

@@ -1,277 +1,11 @@
 {
   "additional_special_tokens": [
-    {
-      "content": "</s_BIC>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</s_supplier_register_id>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</s_customer_name>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<s_total_price>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<s_supplier_tax_id>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</s_total_price>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<s_variable_symbol>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<s_supplier_register_id>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<s_customer_name>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</s_bank_account_number>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<s_invoice_number>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<s_customer_tax_id>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</s_issue_date>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<s_supplier_name>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<s_issue_date>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</s_customer_tax_id>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</s_currency>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
     {
       "content": "<parsing>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false
-    },
-    {
-      "content": "<s_currency>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<s_IBAN>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</s_supplier_tax_id>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</s_invoice_number>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</s_supplier_name>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<s_bank_account_number>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<s_const_symbol>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</s_const_symbol>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</s_total_vat>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</s_variable_symbol>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<s_BIC>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</s_taxable_supply_date>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<s_taxable_supply_date>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<s_customer_register_id>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<s_total_vat>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</s_payment>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</s_due_date>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<s_payment>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</s_IBAN>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<s_due_date>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</s_customer_register_id>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
     }
   ],
   "eos_token": {

tokenizer.json

@@ -2,13 +2,13 @@
   "version": "1.0",
   "truncation": {
     "direction": "Right",
-    "max_length": 128,
+    "max_length": 512,
     "strategy": "LongestFirst",
     "stride": 0
   },
   "padding": {
     "strategy": {
-      "Fixed": 128
+      "Fixed": 512
     },
     "direction": "Right",
     "pad_to_multiple_of": null,
@@ -946,159 +946,6 @@
     },
     {
       "id": 50344,
-      "content": "</s_BIC>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50345,
-      "content": "</s_supplier_register_id>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50346,
-      "content": "</s_customer_name>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50347,
-      "content": "<s_total_price>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50348,
-      "content": "<s_supplier_tax_id>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50349,
-      "content": "</s_total_price>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50350,
-      "content": "<s_variable_symbol>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50351,
-      "content": "<s_supplier_register_id>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50352,
-      "content": "<s_customer_name>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50353,
-      "content": "</s_bank_account_number>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50354,
-      "content": "<s_invoice_number>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50355,
-      "content": "<s_customer_tax_id>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50356,
-      "content": "</s_issue_date>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50357,
-      "content": "<s_supplier_name>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50358,
-      "content": "<s_issue_date>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50359,
-      "content": "</s_customer_tax_id>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50360,
-      "content": "</s_currency>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50361,
       "content": "<parsing>",
       "single_word": false,
       "lstrip": false,
@@ -1107,196 +954,7 @@
       "special": true
     },
     {
-      "id": 50362,
-      "content": "<s_currency>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50363,
-      "content": "<s_IBAN>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50364,
-      "content": "</s_supplier_tax_id>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50365,
-      "content": "</s_invoice_number>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50366,
-      "content": "</s_supplier_name>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50367,
-      "content": "<s_bank_account_number>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50368,
-      "content": "<s_const_symbol>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50369,
-      "content": "</s_const_symbol>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50370,
-      "content": "</s_total_vat>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50371,
-      "content": "</s_variable_symbol>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50372,
-      "content": "<s_BIC>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50373,
-      "content": "</s_taxable_supply_date>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50374,
-      "content": "<s_taxable_supply_date>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50375,
-      "content": "<s_customer_register_id>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50376,
-      "content": "<s_total_vat>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50377,
-      "content": "</s_payment>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50378,
-      "content": "</s_due_date>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50379,
-      "content": "<s_payment>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50380,
-      "content": "</s_IBAN>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50381,
-      "content": "<s_due_date>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50382,
-      "content": "</s_customer_register_id>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50383,
+      "id": 50345,
       "content": "<reserved_1>",
       "single_word": false,
       "lstrip": false,
@@ -1305,7 +963,7 @@
       "special": false
     },
     {
-      "id": 50384,
+      "id": 50346,
       "content": "<reserved_2>",
       "single_word": false,
       "lstrip": false,
@@ -1314,7 +972,7 @@
       "special": false
     },
     {
-      "id": 50385,
+      "id": 50347,
       "content": "<reserved_3>",
       "single_word": false,
       "lstrip": false,
@@ -1323,7 +981,7 @@
       "special": false
     },
     {
-      "id": 50386,
+      "id": 50348,
       "content": "<reserved_4>",
       "single_word": false,
       "lstrip": false,
@@ -1332,7 +990,7 @@
       "special": false
     },
     {
-      "id": 50387,
+      "id": 50349,
       "content": "<reserved_5>",
       "single_word": false,
       "lstrip": false,
@@ -1341,7 +999,7 @@
       "special": false
     },
     {
-      "id": 50388,
+      "id": 50350,
       "content": "<reserved_6>",
       "single_word": false,
       "lstrip": false,
@@ -1350,7 +1008,7 @@
       "special": false
     },
     {
-      "id": 50389,
+      "id": 50351,
       "content": "<reserved_7>",
       "single_word": false,
       "lstrip": false,
@@ -1359,7 +1017,7 @@
       "special": false
     },
     {
-      "id": 50390,
+      "id": 50352,
       "content": "<reserved_8>",
       "single_word": false,
       "lstrip": false,
@@ -1368,7 +1026,7 @@
       "special": false
     },
     {
-      "id": 50391,
+      "id": 50353,
       "content": "<reserved_9>",
       "single_word": false,
       "lstrip": false,
@@ -1377,7 +1035,7 @@
       "special": false
     },
     {
-      "id": 50392,
+      "id": 50354,
       "content": "<reserved_10>",
       "single_word": false,
       "lstrip": false,
@@ -1386,7 +1044,7 @@
       "special": false
     },
     {
-      "id": 50393,
+      "id": 50355,
       "content": "<reserved_11>",
       "single_word": false,
       "lstrip": false,
@@ -1395,7 +1053,7 @@
       "special": false
     },
     {
-      "id": 50394,
+      "id": 50356,
       "content": "<reserved_12>",
       "single_word": false,
       "lstrip": false,
@@ -1404,7 +1062,7 @@
       "special": false
     },
     {
-      "id": 50395,
+      "id": 50357,
       "content": "<reserved_13>",
       "single_word": false,
       "lstrip": false,
@@ -1413,7 +1071,7 @@
       "special": false
     },
     {
-      "id": 50396,
+      "id": 50358,
       "content": "<reserved_14>",
       "single_word": false,
       "lstrip": false,
@@ -1422,7 +1080,7 @@
       "special": false
     },
     {
-      "id": 50397,
+      "id": 50359,
       "content": "<reserved_15>",
       "single_word": false,
       "lstrip": false,
@@ -1431,7 +1089,7 @@
       "special": false
     },
     {
-      "id": 50398,
+      "id": 50360,
       "content": "<reserved_16>",
       "single_word": false,
       "lstrip": false,
@@ -1440,7 +1098,7 @@
       "special": false
     },
     {
-      "id": 50399,
+      "id": 50361,
       "content": "<reserved_17>",
       "single_word": false,
       "lstrip": false,
@@ -1449,7 +1107,7 @@
       "special": false
     },
     {
-      "id": 50400,
+      "id": 50362,
       "content": "<reserved_18>",
       "single_word": false,
       "lstrip": false,
@@ -1458,7 +1116,7 @@
       "special": false
     },
     {
-      "id": 50401,
+      "id": 50363,
       "content": "<reserved_19>",
       "single_word": false,
       "lstrip": false,
@@ -1467,7 +1125,7 @@
       "special": false
     },
     {
-      "id": 50402,
+      "id": 50364,
       "content": "<reserved_20>",
       "single_word": false,
       "lstrip": false,
@@ -1476,7 +1134,7 @@
       "special": false
     },
     {
-      "id": 50403,
+      "id": 50365,
       "content": "<reserved_21>",
       "single_word": false,
       "lstrip": false,
@@ -1485,7 +1143,7 @@
       "special": false
     },
     {
-      "id": 50404,
+      "id": 50366,
       "content": "<reserved_22>",
       "single_word": false,
       "lstrip": false,
@@ -1494,247 +1152,13 @@
       "special": false
     },
     {
-      "id": 50405,
+      "id": 50367,
       "content": "<reserved_23>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": true,
       "special": false
-    },
-    {
-      "id": 50406,
-      "content": "<reserved_24>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50407,
-      "content": "<reserved_25>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50408,
-      "content": "<reserved_26>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50409,
-      "content": "<reserved_27>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50410,
-      "content": "<reserved_28>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50411,
-      "content": "<reserved_29>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50412,
-      "content": "<reserved_30>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50413,
-      "content": "<reserved_31>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50414,
-      "content": "<reserved_32>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50415,
-      "content": "<reserved_33>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50416,
-      "content": "<reserved_34>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50417,
-      "content": "<reserved_35>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50418,
-      "content": "<reserved_36>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50419,
-      "content": "<reserved_37>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50420,
-      "content": "<reserved_38>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50421,
-      "content": "<reserved_39>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50422,
-      "content": "<reserved_40>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50423,
-      "content": "<reserved_41>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50424,
-      "content": "<reserved_42>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50425,
-      "content": "<reserved_43>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50426,
-      "content": "<reserved_44>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50427,
-      "content": "<reserved_45>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50428,
-      "content": "<reserved_46>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50429,
-      "content": "<reserved_47>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50430,
-      "content": "<reserved_48>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 50431,
-      "content": "<reserved_49>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
     }
   ],
   "normalizer": {

tokenizer_config.json

@@ -826,142 +826,6 @@
       "special": true
     },
     "50344": {
-      "content": "</s_BIC>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50345": {
-      "content": "</s_supplier_register_id>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50346": {
-      "content": "</s_customer_name>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50347": {
-      "content": "<s_total_price>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50348": {
-      "content": "<s_supplier_tax_id>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50349": {
-      "content": "</s_total_price>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50350": {
-      "content": "<s_variable_symbol>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50351": {
-      "content": "<s_supplier_register_id>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50352": {
-      "content": "<s_customer_name>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50353": {
-      "content": "</s_bank_account_number>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50354": {
-      "content": "<s_invoice_number>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50355": {
-      "content": "<s_customer_tax_id>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50356": {
-      "content": "</s_issue_date>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50357": {
-      "content": "<s_supplier_name>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50358": {
-      "content": "<s_issue_date>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50359": {
-      "content": "</s_customer_tax_id>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50360": {
-      "content": "</s_currency>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50361": {
       "content": "<parsing>",
       "lstrip": false,
       "normalized": false,
@@ -969,175 +833,7 @@
       "single_word": false,
       "special": true
     },
-    "50362": {
-      "content": "<s_currency>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50363": {
-      "content": "<s_IBAN>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50364": {
-      "content": "</s_supplier_tax_id>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50365": {
-      "content": "</s_invoice_number>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50366": {
-      "content": "</s_supplier_name>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50367": {
-      "content": "<s_bank_account_number>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50368": {
-      "content": "<s_const_symbol>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50369": {
-      "content": "</s_const_symbol>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50370": {
-      "content": "</s_total_vat>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50371": {
-      "content": "</s_variable_symbol>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50372": {
-      "content": "<s_BIC>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50373": {
-      "content": "</s_taxable_supply_date>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50374": {
-      "content": "<s_taxable_supply_date>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50375": {
-      "content": "<s_customer_register_id>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50376": {
-      "content": "<s_total_vat>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50377": {
-      "content": "</s_payment>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50378": {
-      "content": "</s_due_date>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50379": {
-      "content": "<s_payment>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50380": {
-      "content": "</s_IBAN>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50381": {
-      "content": "<s_due_date>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50382": {
-      "content": "</s_customer_register_id>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50383": {
+    "50345": {
       "content": "<reserved_1>",
       "lstrip": false,
       "normalized": true,
@@ -1145,7 +841,7 @@
       "single_word": false,
       "special": false
     },
-    "50384": {
+    "50346": {
       "content": "<reserved_2>",
       "lstrip": false,
       "normalized": true,
@@ -1153,7 +849,7 @@
       "single_word": false,
       "special": false
     },
-    "50385": {
+    "50347": {
       "content": "<reserved_3>",
       "lstrip": false,
       "normalized": true,
@@ -1161,7 +857,7 @@
       "single_word": false,
       "special": false
     },
-    "50386": {
+    "50348": {
       "content": "<reserved_4>",
       "lstrip": false,
       "normalized": true,
@@ -1169,7 +865,7 @@
       "single_word": false,
       "special": false
     },
-    "50387": {
+    "50349": {
       "content": "<reserved_5>",
       "lstrip": false,
       "normalized": true,
@@ -1177,7 +873,7 @@
       "single_word": false,
       "special": false
     },
-    "50388": {
+    "50350": {
       "content": "<reserved_6>",
       "lstrip": false,
       "normalized": true,
@@ -1185,7 +881,7 @@
       "single_word": false,
       "special": false
     },
-    "50389": {
+    "50351": {
       "content": "<reserved_7>",
       "lstrip": false,
       "normalized": true,
@@ -1193,7 +889,7 @@
       "single_word": false,
       "special": false
     },
-    "50390": {
+    "50352": {
       "content": "<reserved_8>",
       "lstrip": false,
       "normalized": true,
@@ -1201,7 +897,7 @@
       "single_word": false,
       "special": false
     },
-    "50391": {
+    "50353": {
       "content": "<reserved_9>",
       "lstrip": false,
       "normalized": true,
@@ -1209,7 +905,7 @@
       "single_word": false,
       "special": false
     },
-    "50392": {
+    "50354": {
       "content": "<reserved_10>",
       "lstrip": false,
       "normalized": true,
@@ -1217,7 +913,7 @@
       "single_word": false,
       "special": false
     },
-    "50393": {
+    "50355": {
       "content": "<reserved_11>",
       "lstrip": false,
       "normalized": true,
@@ -1225,7 +921,7 @@
       "single_word": false,
       "special": false
     },
-    "50394": {
+    "50356": {
       "content": "<reserved_12>",
       "lstrip": false,
       "normalized": true,
@@ -1233,7 +929,7 @@
       "single_word": false,
       "special": false
     },
-    "50395": {
+    "50357": {
       "content": "<reserved_13>",
       "lstrip": false,
       "normalized": true,
@@ -1241,7 +937,7 @@
       "single_word": false,
       "special": false
     },
-    "50396": {
+    "50358": {
       "content": "<reserved_14>",
       "lstrip": false,
       "normalized": true,
@@ -1249,7 +945,7 @@
       "single_word": false,
       "special": false
     },
-    "50397": {
+    "50359": {
       "content": "<reserved_15>",
       "lstrip": false,
       "normalized": true,
@@ -1257,7 +953,7 @@
       "single_word": false,
       "special": false
     },
-    "50398": {
+    "50360": {
       "content": "<reserved_16>",
       "lstrip": false,
       "normalized": true,
@@ -1265,7 +961,7 @@
       "single_word": false,
       "special": false
     },
-    "50399": {
+    "50361": {
       "content": "<reserved_17>",
       "lstrip": false,
       "normalized": true,
@@ -1273,7 +969,7 @@
       "single_word": false,
       "special": false
     },
-    "50400": {
+    "50362": {
       "content": "<reserved_18>",
       "lstrip": false,
       "normalized": true,
@@ -1281,7 +977,7 @@
       "single_word": false,
       "special": false
     },
-    "50401": {
+    "50363": {
       "content": "<reserved_19>",
       "lstrip": false,
       "normalized": true,
@@ -1289,7 +985,7 @@
       "single_word": false,
       "special": false
     },
-    "50402": {
+    "50364": {
       "content": "<reserved_20>",
       "lstrip": false,
       "normalized": true,
@@ -1297,7 +993,7 @@
       "single_word": false,
       "special": false
     },
-    "50403": {
+    "50365": {
       "content": "<reserved_21>",
       "lstrip": false,
       "normalized": true,
@@ -1305,7 +1001,7 @@
       "single_word": false,
       "special": false
     },
-    "50404": {
+    "50366": {
       "content": "<reserved_22>",
       "lstrip": false,
       "normalized": true,
@@ -1313,263 +1009,17 @@
       "single_word": false,
       "special": false
     },
-    "50405": {
+    "50367": {
       "content": "<reserved_23>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": false
-    },
-    "50406": {
-      "content": "<reserved_24>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50407": {
-      "content": "<reserved_25>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50408": {
-      "content": "<reserved_26>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50409": {
-      "content": "<reserved_27>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50410": {
-      "content": "<reserved_28>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50411": {
-      "content": "<reserved_29>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50412": {
-      "content": "<reserved_30>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50413": {
-      "content": "<reserved_31>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50414": {
-      "content": "<reserved_32>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50415": {
-      "content": "<reserved_33>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50416": {
-      "content": "<reserved_34>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50417": {
-      "content": "<reserved_35>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50418": {
-      "content": "<reserved_36>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50419": {
-      "content": "<reserved_37>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50420": {
-      "content": "<reserved_38>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50421": {
-      "content": "<reserved_39>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50422": {
-      "content": "<reserved_40>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50423": {
-      "content": "<reserved_41>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50424": {
-      "content": "<reserved_42>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50425": {
-      "content": "<reserved_43>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50426": {
-      "content": "<reserved_44>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50427": {
-      "content": "<reserved_45>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50428": {
-      "content": "<reserved_46>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50429": {
-      "content": "<reserved_47>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50430": {
-      "content": "<reserved_48>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50431": {
-      "content": "<reserved_49>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
     }
   },
   "additional_special_tokens": [
-    "</s_BIC>",
-    "</s_supplier_register_id>",
-    "</s_customer_name>",
-    "<s_total_price>",
-    "<s_supplier_tax_id>",
-    "</s_total_price>",
-    "<s_variable_symbol>",
-    "<s_supplier_register_id>",
-    "<s_customer_name>",
-    "</s_bank_account_number>",
-    "<s_invoice_number>",
-    "<s_customer_tax_id>",
-    "</s_issue_date>",
-    "<s_supplier_name>",
-    "<s_issue_date>",
-    "</s_customer_tax_id>",
-    "</s_currency>",
-    "<parsing>",
-    "<s_currency>",
-    "<s_IBAN>",
-    "</s_supplier_tax_id>",
-    "</s_invoice_number>",
-    "</s_supplier_name>",
-    "<s_bank_account_number>",
-    "<s_const_symbol>",
-    "</s_const_symbol>",
-    "</s_total_vat>",
-    "</s_variable_symbol>",
-    "<s_BIC>",
-    "</s_taxable_supply_date>",
-    "<s_taxable_supply_date>",
-    "<s_customer_register_id>",
-    "<s_total_vat>",
-    "</s_payment>",
-    "</s_due_date>",
-    "<s_payment>",
-    "</s_IBAN>",
-    "<s_due_date>",
-    "</s_customer_register_id>"
+    "<parsing>"
   ],
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",