Upload processor

added_tokens.json

@@ -1,31 +1,50 @@
 {
-  "</s_Cooking>": 57548,
-  "</s_Fish>": 57530,
-  "</s_Gear>": 57544,
-  "</s_Latin>": 57532,
-  "</s_Packing>": 57542,
-  "</s_Peche>": 57528,
-  "</s_Provenance>": 57550,
-  "</s_Size>": 57546,
-  "</s_Szone>": 57540,
-  "</s_SzoneCode>": 57536,
-  "</s_Zone>": 57538,
-  "</s_ZoneCode>": 57534,
-  "</s_ner>": 57526,
-  "<s_Cooking>": 57547,
-  "<s_Fish>": 57529,
-  "<s_Gear>": 57543,
-  "<s_Latin>": 57531,
-  "<s_Packing>": 57541,
-  "<s_Peche>": 57527,
-  "<s_Provenance>": 57549,
-  "<s_Size>": 57545,
-  "<s_Szone>": 57539,
-  "<s_SzoneCode>": 57535,
-  "<s_Zone>": 57537,
-  "<s_ZoneCode>": 57533,
-  "<s_iitcdip>": 57523,
-  "<s_ner>": 57525,
-  "<s_synthdog>": 57524,
-  "<sep/>": 57522
+  "</s_Cooking>": 57567,
+  "</s_Fish>": 57549,
+  "</s_Gear>": 57563,
+  "</s_Latin>": 57551,
+  "</s_Packing>": 57561,
+  "</s_Peche>": 57547,
+  "</s_Provenance>": 57569,
+  "</s_Size>": 57565,
+  "</s_Szone>": 57559,
+  "</s_SzoneCode>": 57555,
+  "</s_Zone>": 57557,
+  "</s_ZoneCode>": 57553,
+  "</s_answer>": 57523,
+  "</s_ner>": 57545,
+  "</s_question>": 57524,
+  "<Fish/>": 57533,
+  "<Gear/>": 57539,
+  "<Latin/>": 57538,
+  "<O/>": 57532,
+  "<Packing/>": 57536,
+  "<Peche/>": 57534,
+  "<SZone/>": 57542,
+  "<Size/>": 57537,
+  "<SzoneCode/>": 57543,
+  "<Zone/>": 57540,
+  "<ZoneCode/>": 57541,
+  "<no/>": 57525,
+  "<s_Cooking>": 57566,
+  "<s_Fish>": 57548,
+  "<s_Gear>": 57562,
+  "<s_Latin>": 57550,
+  "<s_Packing>": 57560,
+  "<s_Peche>": 57546,
+  "<s_Provenance>": 57568,
+  "<s_Size>": 57564,
+  "<s_Szone>": 57558,
+  "<s_SzoneCode>": 57554,
+  "<s_Zone>": 57556,
+  "<s_ZoneCode>": 57552,
+  "<s_answer>": 57526,
+  "<s_docvqa>": 57527,
+  "<s_iitcdip>": 57528,
+  "<s_ner>": 57544,
+  "<s_question>": 57529,
+  "<s_synthdog>": 57530,
+  "<sep/>": 57522,
+  "<yes/>": 57531,
+  "Cooking/>": 57535
 }

special_tokens_map.json

@@ -1,7 +1,14 @@
 {
   "additional_special_tokens": [
+    "</s_answer>",
+    "</s_question>",
+    "<no/>",
+    "<s_answer>",
+    "<s_docvqa>",
     "<s_iitcdip>",
-    "<s_synthdog>"
+    "<s_question>",
+    "<s_synthdog>",
+    "<yes/>"
   ],
   "bos_token": "<s>",
   "cls_token": "<s>",

tokenizer.json

@@ -2,13 +2,13 @@
   "version": "1.0",
   "truncation": {
     "direction": "Right",
-    "max_length": 256,
+    "max_length": 128,
     "strategy": "LongestFirst",
     "stride": 0
   },
   "padding": {
     "strategy": {
-      "Fixed": 256
+      "Fixed": 128
     },
     "direction": "Right",
     "pad_to_multiple_of": null,
@@ -73,7 +73,7 @@
     },
     {
       "id": 57523,
-      "content": "<s_iitcdip>",
+      "content": "</s_answer>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -82,7 +82,7 @@
     },
     {
       "id": 57524,
-      "content": "<s_synthdog>",
+      "content": "</s_question>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -91,6 +91,177 @@
     },
     {
       "id": 57525,
+      "content": "<no/>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 57526,
+      "content": "<s_answer>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 57527,
+      "content": "<s_docvqa>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 57528,
+      "content": "<s_iitcdip>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 57529,
+      "content": "<s_question>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 57530,
+      "content": "<s_synthdog>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 57531,
+      "content": "<yes/>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 57532,
+      "content": "<O/>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 57533,
+      "content": "<Fish/>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 57534,
+      "content": "<Peche/>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 57535,
+      "content": "Cooking/>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 57536,
+      "content": "<Packing/>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 57537,
+      "content": "<Size/>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 57538,
+      "content": "<Latin/>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 57539,
+      "content": "<Gear/>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 57540,
+      "content": "<Zone/>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 57541,
+      "content": "<ZoneCode/>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 57542,
+      "content": "<SZone/>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 57543,
+      "content": "<SzoneCode/>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 57544,
       "content": "<s_ner>",
       "single_word": false,
       "lstrip": false,
@@ -99,7 +270,7 @@
       "special": false
     },
     {
-      "id": 57526,
+      "id": 57545,
       "content": "</s_ner>",
       "single_word": false,
       "lstrip": false,
@@ -108,7 +279,7 @@
       "special": false
     },
     {
-      "id": 57527,
+      "id": 57546,
       "content": "<s_Peche>",
       "single_word": false,
       "lstrip": false,
@@ -117,7 +288,7 @@
       "special": false
     },
     {
-      "id": 57528,
+      "id": 57547,
       "content": "</s_Peche>",
       "single_word": false,
       "lstrip": false,
@@ -126,7 +297,7 @@
       "special": false
     },
     {
-      "id": 57529,
+      "id": 57548,
       "content": "<s_Fish>",
       "single_word": false,
       "lstrip": false,
@@ -135,7 +306,7 @@
       "special": false
     },
     {
-      "id": 57530,
+      "id": 57549,
       "content": "</s_Fish>",
       "single_word": false,
       "lstrip": false,
@@ -144,7 +315,7 @@
       "special": false
     },
     {
-      "id": 57531,
+      "id": 57550,
       "content": "<s_Latin>",
       "single_word": false,
       "lstrip": false,
@@ -153,7 +324,7 @@
       "special": false
     },
     {
-      "id": 57532,
+      "id": 57551,
       "content": "</s_Latin>",
       "single_word": false,
       "lstrip": false,
@@ -162,7 +333,7 @@
       "special": false
     },
     {
-      "id": 57533,
+      "id": 57552,
       "content": "<s_ZoneCode>",
       "single_word": false,
       "lstrip": false,
@@ -171,7 +342,7 @@
       "special": false
     },
     {
-      "id": 57534,
+      "id": 57553,
       "content": "</s_ZoneCode>",
       "single_word": false,
       "lstrip": false,
@@ -180,7 +351,7 @@
       "special": false
     },
     {
-      "id": 57535,
+      "id": 57554,
       "content": "<s_SzoneCode>",
       "single_word": false,
       "lstrip": false,
@@ -189,7 +360,7 @@
       "special": false
     },
     {
-      "id": 57536,
+      "id": 57555,
       "content": "</s_SzoneCode>",
       "single_word": false,
       "lstrip": false,
@@ -198,7 +369,7 @@
       "special": false
     },
     {
-      "id": 57537,
+      "id": 57556,
       "content": "<s_Zone>",
       "single_word": false,
       "lstrip": false,
@@ -207,7 +378,7 @@
       "special": false
     },
     {
-      "id": 57538,
+      "id": 57557,
       "content": "</s_Zone>",
       "single_word": false,
       "lstrip": false,
@@ -216,7 +387,7 @@
       "special": false
     },
     {
-      "id": 57539,
+      "id": 57558,
       "content": "<s_Szone>",
       "single_word": false,
       "lstrip": false,
@@ -225,7 +396,7 @@
       "special": false
     },
     {
-      "id": 57540,
+      "id": 57559,
       "content": "</s_Szone>",
       "single_word": false,
       "lstrip": false,
@@ -234,7 +405,7 @@
       "special": false
     },
     {
-      "id": 57541,
+      "id": 57560,
       "content": "<s_Packing>",
       "single_word": false,
       "lstrip": false,
@@ -243,7 +414,7 @@
       "special": false
     },
     {
-      "id": 57542,
+      "id": 57561,
       "content": "</s_Packing>",
       "single_word": false,
       "lstrip": false,
@@ -252,7 +423,7 @@
       "special": false
     },
     {
-      "id": 57543,
+      "id": 57562,
       "content": "<s_Gear>",
       "single_word": false,
       "lstrip": false,
@@ -261,7 +432,7 @@
       "special": false
     },
     {
-      "id": 57544,
+      "id": 57563,
       "content": "</s_Gear>",
       "single_word": false,
       "lstrip": false,
@@ -270,7 +441,7 @@
       "special": false
     },
     {
-      "id": 57545,
+      "id": 57564,
       "content": "<s_Size>",
       "single_word": false,
       "lstrip": false,
@@ -279,7 +450,7 @@
       "special": false
     },
     {
-      "id": 57546,
+      "id": 57565,
       "content": "</s_Size>",
       "single_word": false,
       "lstrip": false,
@@ -288,7 +459,7 @@
       "special": false
     },
     {
-      "id": 57547,
+      "id": 57566,
       "content": "<s_Cooking>",
       "single_word": false,
       "lstrip": false,
@@ -297,7 +468,7 @@
       "special": false
     },
     {
-      "id": 57548,
+      "id": 57567,
       "content": "</s_Cooking>",
       "single_word": false,
       "lstrip": false,
@@ -306,7 +477,7 @@
       "special": false
     },
     {
-      "id": 57549,
+      "id": 57568,
       "content": "<s_Provenance>",
       "single_word": false,
       "lstrip": false,
@@ -315,7 +486,7 @@
       "special": false
     },
     {
-      "id": 57550,
+      "id": 57569,
       "content": "</s_Provenance>",
       "single_word": false,
       "lstrip": false,

tokenizer_config.json

@@ -11,7 +11,7 @@
     "single_word": false
   },
   "model_max_length": 1000000000000000019884624838656,
-  "name_or_path": "naver-clova-ix/donut-base",
+  "name_or_path": "naver-clova-ix/donut-base-finetuned-docvqa",
   "pad_token": "<pad>",
   "processor_class": "DonutProcessor",
   "sep_token": "</s>",