Upload tokenizer

tokenizer.json

@@ -2073,6 +2073,33 @@
       "normalized": false,
       "special": true
     },
+    {
+      "id": 230,
+      "content": "<|file_sep|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 231,
+      "content": "<|repo_name|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 232,
+      "content": "<|repo_tree|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
     {
       "id": 233,
       "content": ">>UNUSED_221<<",
@@ -4691,33 +4718,6 @@
       "rstrip": false,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 32768,
-      "content": "<|file_sep|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32769,
-      "content": "<|repo_name|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32770,
-      "content": "<|repo_tree|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
     }
   ],
   "normalizer": null,
@@ -4999,9 +4999,9 @@
       "<|im_start|>": 227,
       "<|im_end|>": 228,
       "<|system|>": 229,
-      ">>UNUSED_218<<": 230,
-      ">>UNUSED_219<<": 231,
-      ">>UNUSED_220<<": 232,
+      "<|file_sep|>": 230,
+      "<|repo_name|>": 231,
+      "<|repo_tree|>": 232,
       ">>UNUSED_221<<": 233,
       ">>UNUSED_222<<": 234,
       ">>UNUSED_223<<": 235,

tokenizer_config.json

@@ -1840,6 +1840,30 @@
       "single_word": false,
       "special": true
     },
+    "230": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "231": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "232": {
+      "content": "<|repo_tree|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
     "233": {
       "content": ">>UNUSED_221<<",
       "lstrip": false,
@@ -4167,30 +4191,6 @@
       "rstrip": false,
       "single_word": false,
       "special": true
-    },
-    "32768": {
-      "content": "<|file_sep|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32769": {
-      "content": "<|repo_name|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32770": {
-      "content": "<|repo_tree|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
     }
   },
   "additional_special_tokens": [