Upload processor

added_tokens.json

@@ -1,11 +1,11 @@
 {
-  "</line>": 57525,
-  "</s_HW>": 57531,
+  "</line>": 57528,
+  "</s_HW>": 57530,
   "</s_P>": 57529,
   "<MathOCR>": 57526,
-  "<line>": 57530,
-  "<s_HW>": 57527,
-  "<s_P>": 57528,
+  "<line>": 57525,
+  "<s_HW>": 57531,
+  "<s_P>": 57527,
   "<s_iitcdip>": 57523,
   "<s_synthdog>": 57524,
   "<sep/>": 57522

tokenizer_config.json

@@ -65,7 +65,7 @@
       "special": true
     },
     "57525": {
-      "content": "</line>",
+      "content": "<line>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -81,7 +81,7 @@
       "special": false
     },
     "57527": {
-      "content": "<s_HW>",
+      "content": "<s_P>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -89,7 +89,7 @@
       "special": false
     },
     "57528": {
-      "content": "<s_P>",
+      "content": "</line>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -105,7 +105,7 @@
       "special": false
     },
     "57530": {
-      "content": "<line>",
+      "content": "</s_HW>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -113,7 +113,7 @@
       "special": false
     },
     "57531": {
-      "content": "</s_HW>",
+      "content": "<s_HW>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -130,11 +130,18 @@
   "cls_token": "<s>",
   "eos_token": "</s>",
   "mask_token": "<mask>",
+  "max_length": 512,
   "model_max_length": 1000000000000000019884624838656,
+  "pad_to_multiple_of": null,
   "pad_token": "<pad>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
   "processor_class": "DonutProcessor",
   "sep_token": "</s>",
   "sp_model_kwargs": {},
+  "stride": 0,
   "tokenizer_class": "XLMRobertaTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "<unk>"
 }