Training done

added_tokens.json

@@ -1,58 +1,46 @@
 {
-  "</s_bar_code>": 57551,
-  "</s_box_no>": 57546,
-  "</s_family>": 57567,
-  "</s_item>": 57559,
-  "</s_item_barcode>": 57561,
-  "</s_item_batch>": 57548,
-  "</s_item_code>": 57532,
-  "</s_item_decsription>": 57571,
-  "</s_item_description>": 57528,
-  "</s_item_quantity>": 57530,
-  "</s_item_quantity_billed>": 57540,
-  "</s_item_quantity_ordered>": 57553,
-  "</s_item_quantity_received>": 57557,
-  "</s_item_quantity_rejected>": 57573,
-  "</s_item_quantity_scanned>": 57555,
-  "</s_item_quantity_shipped>": 57538,
-  "</s_item_quantity_short>": 57569,
-  "</s_item_total_price>": 57544,
-  "</s_item_total_quantity>": 57577,
-  "</s_item_unit>": 57563,
-  "</s_item_unit_price>": 57542,
-  "</s_item_unit_quantity>": 57575,
-  "</s_item_uom>": 57534,
-  "</s_item_weight>": 57536,
+  "</s_box_no>": 57536,
+  "</s_family>": 57538,
+  "</s_item_barcode>": 57540,
+  "</s_item_batch>": 57542,
+  "</s_item_code>": 57528,
+  "</s_item_description>": 57530,
+  "</s_item_quantity>": 57532,
+  "</s_item_quantity_billed>": 57544,
+  "</s_item_quantity_ordered>": 57546,
+  "</s_item_quantity_received>": 57548,
+  "</s_item_quantity_rejected>": 57550,
+  "</s_item_quantity_scanned>": 57552,
+  "</s_item_quantity_shipped>": 57554,
+  "</s_item_quantity_short>": 57556,
+  "</s_item_total_price>": 57558,
+  "</s_item_total_quantity>": 57560,
+  "</s_item_unit_price>": 57562,
+  "</s_item_unit_quantity>": 57534,
+  "</s_item_weight>": 57564,
   "</s_items>": 57526,
-  "</s_supplier_reference>": 57565,
-  "<s_bar_code>": 57550,
-  "<s_box_no>": 57545,
-  "<s_cord-v2>": 57549,
-  "<s_family>": 57566,
+  "<s_box_no>": 57535,
+  "<s_cord-v2>": 57565,
+  "<s_family>": 57537,
   "<s_iitcdip>": 57523,
-  "<s_item>": 57558,
-  "<s_item_barcode>": 57560,
-  "<s_item_batch>": 57547,
-  "<s_item_code>": 57531,
-  "<s_item_decsription>": 57570,
-  "<s_item_description>": 57527,
-  "<s_item_quantity>": 57529,
-  "<s_item_quantity_billed>": 57539,
-  "<s_item_quantity_ordered>": 57552,
-  "<s_item_quantity_received>": 57556,
-  "<s_item_quantity_rejected>": 57572,
-  "<s_item_quantity_scanned>": 57554,
-  "<s_item_quantity_shipped>": 57537,
-  "<s_item_quantity_short>": 57568,
-  "<s_item_total_price>": 57543,
-  "<s_item_total_quantity>": 57576,
-  "<s_item_unit>": 57562,
-  "<s_item_unit_price>": 57541,
-  "<s_item_unit_quantity>": 57574,
-  "<s_item_uom>": 57533,
-  "<s_item_weight>": 57535,
+  "<s_item_barcode>": 57539,
+  "<s_item_batch>": 57541,
+  "<s_item_code>": 57527,
+  "<s_item_description>": 57529,
+  "<s_item_quantity>": 57531,
+  "<s_item_quantity_billed>": 57543,
+  "<s_item_quantity_ordered>": 57545,
+  "<s_item_quantity_received>": 57547,
+  "<s_item_quantity_rejected>": 57549,
+  "<s_item_quantity_scanned>": 57551,
+  "<s_item_quantity_shipped>": 57553,
+  "<s_item_quantity_short>": 57555,
+  "<s_item_total_price>": 57557,
+  "<s_item_total_quantity>": 57559,
+  "<s_item_unit_price>": 57561,
+  "<s_item_unit_quantity>": 57533,
+  "<s_item_weight>": 57563,
   "<s_items>": 57525,
-  "<s_supplier_reference>": 57564,
   "<s_synthdog>": 57524,
   "<sep/>": 57522
 }

tokenizer_config.json

@@ -81,7 +81,7 @@
       "special": false
     },
     "57527": {
-      "content": "<s_item_description>",
+      "content": "<s_item_code>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -89,7 +89,7 @@
       "special": false
     },
     "57528": {
-      "content": "</s_item_description>",
+      "content": "</s_item_code>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -97,7 +97,7 @@
       "special": false
     },
     "57529": {
-      "content": "<s_item_quantity>",
+      "content": "<s_item_description>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -105,7 +105,7 @@
       "special": false
     },
     "57530": {
-      "content": "</s_item_quantity>",
+      "content": "</s_item_description>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -113,7 +113,7 @@
       "special": false
     },
     "57531": {
-      "content": "<s_item_code>",
+      "content": "<s_item_quantity>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -121,7 +121,7 @@
       "special": false
     },
     "57532": {
-      "content": "</s_item_code>",
+      "content": "</s_item_quantity>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -129,7 +129,7 @@
       "special": false
     },
     "57533": {
-      "content": "<s_item_uom>",
+      "content": "<s_item_unit_quantity>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -137,7 +137,7 @@
       "special": false
     },
     "57534": {
-      "content": "</s_item_uom>",
+      "content": "</s_item_unit_quantity>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -145,7 +145,7 @@
       "special": false
     },
     "57535": {
-      "content": "<s_item_weight>",
+      "content": "<s_box_no>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -153,7 +153,7 @@
       "special": false
     },
     "57536": {
-      "content": "</s_item_weight>",
+      "content": "</s_box_no>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -161,7 +161,7 @@
       "special": false
     },
     "57537": {
-      "content": "<s_item_quantity_shipped>",
+      "content": "<s_family>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -169,7 +169,7 @@
       "special": false
     },
     "57538": {
-      "content": "</s_item_quantity_shipped>",
+      "content": "</s_family>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -177,7 +177,7 @@
       "special": false
     },
     "57539": {
-      "content": "<s_item_quantity_billed>",
+      "content": "<s_item_barcode>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -185,7 +185,7 @@
       "special": false
     },
     "57540": {
-      "content": "</s_item_quantity_billed>",
+      "content": "</s_item_barcode>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -193,7 +193,7 @@
       "special": false
     },
     "57541": {
-      "content": "<s_item_unit_price>",
+      "content": "<s_item_batch>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -201,7 +201,7 @@
       "special": false
     },
     "57542": {
-      "content": "</s_item_unit_price>",
+      "content": "</s_item_batch>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -209,7 +209,7 @@
       "special": false
     },
     "57543": {
-      "content": "<s_item_total_price>",
+      "content": "<s_item_quantity_billed>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -217,7 +217,7 @@
       "special": false
     },
     "57544": {
-      "content": "</s_item_total_price>",
+      "content": "</s_item_quantity_billed>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -225,7 +225,7 @@
       "special": false
     },
     "57545": {
-      "content": "<s_box_no>",
+      "content": "<s_item_quantity_ordered>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -233,7 +233,7 @@
       "special": false
     },
     "57546": {
-      "content": "</s_box_no>",
+      "content": "</s_item_quantity_ordered>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -241,7 +241,7 @@
       "special": false
     },
     "57547": {
-      "content": "<s_item_batch>",
+      "content": "<s_item_quantity_received>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -249,7 +249,7 @@
       "special": false
     },
     "57548": {
-      "content": "</s_item_batch>",
+      "content": "</s_item_quantity_received>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -257,7 +257,7 @@
       "special": false
     },
     "57549": {
-      "content": "<s_cord-v2>",
+      "content": "<s_item_quantity_rejected>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -265,7 +265,7 @@
       "special": false
     },
     "57550": {
-      "content": "<s_bar_code>",
+      "content": "</s_item_quantity_rejected>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -273,7 +273,7 @@
       "special": false
     },
     "57551": {
-      "content": "</s_bar_code>",
+      "content": "<s_item_quantity_scanned>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -281,7 +281,7 @@
       "special": false
     },
     "57552": {
-      "content": "<s_item_quantity_ordered>",
+      "content": "</s_item_quantity_scanned>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -289,7 +289,7 @@
       "special": false
     },
     "57553": {
-      "content": "</s_item_quantity_ordered>",
+      "content": "<s_item_quantity_shipped>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -297,7 +297,7 @@
       "special": false
     },
     "57554": {
-      "content": "<s_item_quantity_scanned>",
+      "content": "</s_item_quantity_shipped>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -305,7 +305,7 @@
       "special": false
     },
     "57555": {
-      "content": "</s_item_quantity_scanned>",
+      "content": "<s_item_quantity_short>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -313,7 +313,7 @@
       "special": false
     },
     "57556": {
-      "content": "<s_item_quantity_received>",
+      "content": "</s_item_quantity_short>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -321,7 +321,7 @@
       "special": false
     },
     "57557": {
-      "content": "</s_item_quantity_received>",
+      "content": "<s_item_total_price>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -329,7 +329,7 @@
       "special": false
     },
     "57558": {
-      "content": "<s_item>",
+      "content": "</s_item_total_price>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -337,7 +337,7 @@
       "special": false
     },
     "57559": {
-      "content": "</s_item>",
+      "content": "<s_item_total_quantity>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -345,7 +345,7 @@
       "special": false
     },
     "57560": {
-      "content": "<s_item_barcode>",
+      "content": "</s_item_total_quantity>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -353,7 +353,7 @@
       "special": false
     },
     "57561": {
-      "content": "</s_item_barcode>",
+      "content": "<s_item_unit_price>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -361,7 +361,7 @@
       "special": false
     },
     "57562": {
-      "content": "<s_item_unit>",
+      "content": "</s_item_unit_price>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -369,7 +369,7 @@
       "special": false
     },
     "57563": {
-      "content": "</s_item_unit>",
+      "content": "<s_item_weight>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -377,7 +377,7 @@
       "special": false
     },
     "57564": {
-      "content": "<s_supplier_reference>",
+      "content": "</s_item_weight>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -385,103 +385,7 @@
       "special": false
     },
     "57565": {
-      "content": "</s_supplier_reference>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "57566": {
-      "content": "<s_family>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "57567": {
-      "content": "</s_family>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "57568": {
-      "content": "<s_item_quantity_short>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "57569": {
-      "content": "</s_item_quantity_short>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "57570": {
-      "content": "<s_item_decsription>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "57571": {
-      "content": "</s_item_decsription>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "57572": {
-      "content": "<s_item_quantity_rejected>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "57573": {
-      "content": "</s_item_quantity_rejected>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "57574": {
-      "content": "<s_item_unit_quantity>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "57575": {
-      "content": "</s_item_unit_quantity>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "57576": {
-      "content": "<s_item_total_quantity>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "57577": {
-      "content": "</s_item_total_quantity>",
+      "content": "<s_cord-v2>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -498,18 +402,11 @@
   "cls_token": "<s>",
   "eos_token": "</s>",
   "mask_token": "<mask>",
-  "max_length": 768,
   "model_max_length": 1000000000000000019884624838656,
-  "pad_to_multiple_of": null,
   "pad_token": "<pad>",
-  "pad_token_type_id": 0,
-  "padding_side": "right",
   "processor_class": "DonutProcessor",
   "sep_token": "</s>",
   "sp_model_kwargs": {},
-  "stride": 0,
   "tokenizer_class": "XLMRobertaTokenizer",
-  "truncation_side": "right",
-  "truncation_strategy": "longest_first",
   "unk_token": "<unk>"
 }