Upload tokenizer

tokenizer_config.json

@@ -3,7 +3,7 @@
   "add_eos_token": false,
   "add_prefix_space": null,
   "added_tokens_decoder": {
-    "38959": {
+    "39861": {
       "content": "õ",
       "lstrip": false,
       "normalized": true,
@@ -11,7 +11,7 @@
       "single_word": false,
       "special": false
     },
-    "38960": {
+    "39862": {
       "content": "÷",
       "lstrip": false,
       "normalized": true,
@@ -19,7 +19,7 @@
       "single_word": false,
       "special": false
     },
-    "38961": {
+    "39863": {
       "content": "Á",
       "lstrip": false,
       "normalized": true,
@@ -27,7 +27,7 @@
       "single_word": false,
       "special": false
     },
-    "38962": {
+    "39864": {
       "content": "ý",
       "lstrip": false,
       "normalized": true,
@@ -35,7 +35,7 @@
       "single_word": false,
       "special": false
     },
-    "38963": {
+    "39865": {
       "content": "À",
       "lstrip": false,
       "normalized": true,
@@ -43,7 +43,7 @@
       "single_word": false,
       "special": false
     },
-    "38964": {
+    "39866": {
       "content": "ÿ",
       "lstrip": false,
       "normalized": true,
@@ -51,7 +51,7 @@
       "single_word": false,
       "special": false
     },
-    "38965": {
+    "39867": {
       "content": "ø",
       "lstrip": false,
       "normalized": true,
@@ -59,7 +59,7 @@
       "single_word": false,
       "special": false
     },
-    "38966": {
+    "39868": {
       "content": "ú",
       "lstrip": false,
       "normalized": true,
@@ -67,7 +67,7 @@
       "single_word": false,
       "special": false
     },
-    "38967": {
+    "39869": {
       "content": "þ",
       "lstrip": false,
       "normalized": true,
@@ -75,7 +75,7 @@
       "single_word": false,
       "special": false
     },
-    "38968": {
+    "39870": {
       "content": "ü",
       "lstrip": false,
       "normalized": true,
@@ -83,7 +83,7 @@
       "single_word": false,
       "special": false
     },
-    "38969": {
+    "39871": {
       "content": "ù",
       "lstrip": false,
       "normalized": true,
@@ -91,7 +91,7 @@
       "single_word": false,
       "special": false
     },
-    "38970": {
+    "39872": {
       "content": "ö",
       "lstrip": false,
       "normalized": true,
@@ -99,7 +99,7 @@
       "single_word": false,
       "special": false
     },
-    "38971": {
+    "39873": {
       "content": "û",
       "lstrip": false,
       "normalized": true,
@@ -107,7 +107,7 @@
       "single_word": false,
       "special": false
     },
-    "38972": {
+    "39874": {
       "content": "<｜begin▁of▁sentence｜>",
       "lstrip": false,
       "normalized": true,
@@ -115,7 +115,7 @@
       "single_word": false,
       "special": true
     },
-    "38973": {
+    "39875": {
       "content": "<｜end▁of▁sentence｜>",
       "lstrip": false,
       "normalized": true,
@@ -123,7 +123,7 @@
       "single_word": false,
       "special": true
     },
-    "38974": {
+    "39876": {
       "content": "<｜fim▁hole｜>",
       "lstrip": false,
       "normalized": true,
@@ -131,7 +131,7 @@
       "single_word": false,
       "special": false
     },
-    "38975": {
+    "39877": {
       "content": "<｜fim▁begin｜>",
       "lstrip": false,
       "normalized": true,
@@ -139,7 +139,7 @@
       "single_word": false,
       "special": false
     },
-    "38976": {
+    "39878": {
       "content": "<｜fim▁end｜>",
       "lstrip": false,
       "normalized": true,
@@ -147,7 +147,7 @@
       "single_word": false,
       "special": false
     },
-    "38977": {
+    "39879": {
       "content": "<pad>",
       "lstrip": false,
       "normalized": true,
@@ -155,7 +155,7 @@
       "single_word": false,
       "special": false
     },
-    "38978": {
+    "39880": {
       "content": "<|User|>",
       "lstrip": false,
       "normalized": true,
@@ -163,7 +163,7 @@
       "single_word": false,
       "special": false
     },
-    "38979": {
+    "39881": {
       "content": "<|Assistant|>",
       "lstrip": false,
       "normalized": true,
@@ -171,7 +171,7 @@
       "single_word": false,
       "special": false
     },
-    "38980": {
+    "39882": {
       "content": "<|EOT|>",
       "lstrip": false,
       "normalized": true,