Upload folder using huggingface_hub

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e54773dc999718fb34aff7804edab1bf587e55c904953979d912cb90c6ab603
+oid sha256:da593240b956f14fa50de909659d1b8b0d33c8573888fe397600a852171fc8e2
 size 706875632

special_tokens_map.json

@@ -1,15 +1,30 @@
 {
   "additional_special_tokens": [
-    "<|source_id|>",
+    "<|tool_call_start|>",
+    "<|tool_call_end|>",
+    "<|tool_list_start|>",
+    "<|tool_list_end|>",
     "<|source_analysis_start|>",
     "<|source_analysis_end|>",
     "<|source_start|>",
     "<|source_end|>",
+    "<|im_start|>",
+    "<|im_end|>",
     "<|answer_start|>",
     "<|answer_end|>",
-    "<|query_start|>",
-    "<|query_end|>",
+    "<|text_start|>",
+    "<|text_end|>",
+    "<|translation_start|>",
+    "<|translation_end|>",
+    "<|back_translation_start|>",
+    "<|back_translation_end|>",
+    "<|ocr_correction_start|>",
+    "<|ocr_correction_end|>",
+    "<|json_scheme_start|>",
+    "<|json_scheme_end|>",
     "<|source_interpretation_start|>",
-    "<|source_interpretation_end|>"
+    "<|source_interpretation_end|>",
+    "<|query_start|>",
+    "<|query_end|>"
   ]
 }

tokenizer.json

@@ -39,9 +39,99 @@
       "normalized": false,
       "special": true
     },
+    {
+      "id": 65510,
+      "content": "<|tool_call_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65511,
+      "content": "<|tool_call_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65512,
+      "content": "<|tool_list_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65513,
+      "content": "<|tool_list_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65514,
+      "content": "<|source_analysis_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65515,
+      "content": "<|source_analysis_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65516,
+      "content": "<|source_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65517,
+      "content": "<|source_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65518,
+      "content": "<|im_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65519,
+      "content": "<|im_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
     {
       "id": 65520,
-      "content": "<|source_id|>",
+      "content": "<|answer_start|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -50,7 +140,7 @@
     },
     {
       "id": 65521,
-      "content": "<|source_analysis_start|>",
+      "content": "<|answer_end|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -59,7 +149,7 @@
     },
     {
       "id": 65522,
-      "content": "<|source_analysis_end|>",
+      "content": "<|text_start|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -68,7 +158,7 @@
     },
     {
       "id": 65523,
-      "content": "<|source_start|>",
+      "content": "<|text_end|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -77,7 +167,7 @@
     },
     {
       "id": 65524,
-      "content": "<|source_end|>",
+      "content": "<|translation_start|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -86,7 +176,7 @@
     },
     {
       "id": 65525,
-      "content": "<|answer_start|>",
+      "content": "<|translation_end|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -95,7 +185,7 @@
     },
     {
       "id": 65526,
-      "content": "<|answer_end|>",
+      "content": "<|back_translation_start|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -104,7 +194,7 @@
     },
     {
       "id": 65527,
-      "content": "<|query_start|>",
+      "content": "<|back_translation_end|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -113,7 +203,7 @@
     },
     {
       "id": 65528,
-      "content": "<|query_end|>",
+      "content": "<|ocr_correction_start|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -122,7 +212,7 @@
     },
     {
       "id": 65529,
-      "content": "<|source_interpretation_start|>",
+      "content": "<|ocr_correction_end|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -131,12 +221,57 @@
     },
     {
       "id": 65530,
+      "content": "<|json_scheme_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65531,
+      "content": "<|json_scheme_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65532,
+      "content": "<|source_interpretation_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65533,
       "content": "<|source_interpretation_end|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 65534,
+      "content": "<|query_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65535,
+      "content": "<|query_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
@@ -173,7 +308,7 @@
         "single": [
           {
             "SpecialToken": {
-              "id": "<|end_of_text|>",
+              "id": "<|begin_of_text|>",
               "type_id": 0
             }
           },
@@ -187,7 +322,7 @@
         "pair": [
           {
             "SpecialToken": {
-              "id": "<|end_of_text|>",
+              "id": "<|begin_of_text|>",
               "type_id": 0
             }
           },
@@ -199,7 +334,7 @@
           },
           {
             "SpecialToken": {
-              "id": "<|end_of_text|>",
+              "id": "<|begin_of_text|>",
               "type_id": 0
             }
           },
@@ -65776,17 +65911,7 @@
       "reste": 65506,
       "ĠMonat": 65507,
       "å·¥ä½ľ": 65508,
-      "Ġdémocratique": 65509,
-      "Ġsejam": 65510,
-      "ÑĥбеÑĢ": 65511,
-      "Ġaccessoires": 65512,
-      "Ġ709": 65513,
-      "Ġménages": 65514,
-      "Ġprodukty": 65515,
-      "Ġsuum": 65516,
-      "Ġinfluenc": 65517,
-      "Ġcatching": 65518,
-      "ĠÏĮÏĦαν": 65519
+      "Ġdémocratique": 65509
     },
     "merges": [
       "Ġ Ġ",
@@ -131051,17 +131176,7 @@
       "res te",
       "ĠMon at",
       "å·¥ ä½ľ",
-      "Ġdémocra tique",
-      "Ġse jam",
-      "Ñĥб еÑĢ",
-      "Ġaccess oires",
-      "Ġ7 09",
-      "Ġmén ages",
-      "Ġproduk ty",
-      "Ġsu um",
-      "Ġinflu enc",
-      "Ġcatch ing",
-      "ĠÏĮ ÏĦαν"
+      "Ġdémocra tique"
     ]
   }
 }

tokenizer_config.json

@@ -32,8 +32,88 @@
       "single_word": false,
       "special": true
     },
+    "65510": {
+      "content": "<|tool_call_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65511": {
+      "content": "<|tool_call_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65512": {
+      "content": "<|tool_list_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65513": {
+      "content": "<|tool_list_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65514": {
+      "content": "<|source_analysis_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65515": {
+      "content": "<|source_analysis_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65516": {
+      "content": "<|source_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65517": {
+      "content": "<|source_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65518": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65519": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
     "65520": {
-      "content": "<|source_id|>",
+      "content": "<|answer_start|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -41,7 +121,7 @@
       "special": true
     },
     "65521": {
-      "content": "<|source_analysis_start|>",
+      "content": "<|answer_end|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -49,7 +129,7 @@
       "special": true
     },
     "65522": {
-      "content": "<|source_analysis_end|>",
+      "content": "<|text_start|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -57,7 +137,7 @@
       "special": true
     },
     "65523": {
-      "content": "<|source_start|>",
+      "content": "<|text_end|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -65,7 +145,7 @@
       "special": true
     },
     "65524": {
-      "content": "<|source_end|>",
+      "content": "<|translation_start|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -73,7 +153,7 @@
       "special": true
     },
     "65525": {
-      "content": "<|answer_start|>",
+      "content": "<|translation_end|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -81,7 +161,7 @@
       "special": true
     },
     "65526": {
-      "content": "<|answer_end|>",
+      "content": "<|back_translation_start|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -89,7 +169,7 @@
       "special": true
     },
     "65527": {
-      "content": "<|query_start|>",
+      "content": "<|back_translation_end|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -97,7 +177,7 @@
       "special": true
     },
     "65528": {
-      "content": "<|query_end|>",
+      "content": "<|ocr_correction_start|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -105,7 +185,7 @@
       "special": true
     },
     "65529": {
-      "content": "<|source_interpretation_start|>",
+      "content": "<|ocr_correction_end|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -113,26 +193,81 @@
       "special": true
     },
     "65530": {
+      "content": "<|json_scheme_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65531": {
+      "content": "<|json_scheme_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65532": {
+      "content": "<|source_interpretation_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65533": {
       "content": "<|source_interpretation_end|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "65534": {
+      "content": "<|query_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65535": {
+      "content": "<|query_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
   "additional_special_tokens": [
-    "<|source_id|>",
+    "<|tool_call_start|>",
+    "<|tool_call_end|>",
+    "<|tool_list_start|>",
+    "<|tool_list_end|>",
     "<|source_analysis_start|>",
     "<|source_analysis_end|>",
     "<|source_start|>",
     "<|source_end|>",
+    "<|im_start|>",
+    "<|im_end|>",
     "<|answer_start|>",
     "<|answer_end|>",
-    "<|query_start|>",
-    "<|query_end|>",
+    "<|text_start|>",
+    "<|text_end|>",
+    "<|translation_start|>",
+    "<|translation_end|>",
+    "<|back_translation_start|>",
+    "<|back_translation_end|>",
+    "<|ocr_correction_start|>",
+    "<|ocr_correction_end|>",
+    "<|json_scheme_start|>",
+    "<|json_scheme_end|>",
     "<|source_interpretation_start|>",
-    "<|source_interpretation_end|>"
+    "<|source_interpretation_end|>",
+    "<|query_start|>",
+    "<|query_end|>"
   ],
   "clean_up_tokenization_spaces": true,
   "model_max_length": 1000000000000000019884624838656,