removed unused special tokens

fix_tokens.py

@@ -79,16 +79,16 @@ DESIRED_MAPPING = [
       SpecialToken(id=100263, content="|||IP_ADDRESS|||"),
       SpecialToken(id=100264, content="<|im_start|>", special=True),
       SpecialToken(id=100265, content="<|im_end|>", special=True),
-      SpecialToken(id=100266, content="<|extra_id_1|>"),
-      SpecialToken(id=100267, content="<|extra_id_2|>"),
-      SpecialToken(id=100268, content="<think>"),
-      SpecialToken(id=100269, content="</think>"),
-      SpecialToken(id=100270, content="<functions>"),
-      SpecialToken(id=100271, content="</functions>"),
-      SpecialToken(id=100272, content="<function_calls>"),
-      SpecialToken(id=100273, content="</function_calls>"),
-      SpecialToken(id=100274, content="<answer>"),
-      SpecialToken(id=100275, content="</answer>"),
+      SpecialToken(id=100266, content="<functions>"),
+      SpecialToken(id=100267, content="</functions>"),
+      SpecialToken(id=100268, content="<function_calls>"),
+      SpecialToken(id=100269, content="</function_calls>"),
+      SpecialToken(id=100270, content="<|extra_id_1|>"),
+      SpecialToken(id=100271, content="<|extra_id_2|>"),
+      SpecialToken(id=100272, content="<|extra_id_3|>"),
+      SpecialToken(id=100273, content="<|extra_id_4|>"),
+      SpecialToken(id=100274, content="<|extra_id_5|>"),
+      SpecialToken(id=100275, content="<|extra_id_6|>"),
       SpecialToken(id=100276, content="<|endofprompt|>", special=True),
       SpecialToken(
         id=100277,

tokenizer.json

@@ -95,7 +95,7 @@
     },
     {
       "id": 100266,
-      "content": "<|extra_id_1|>",
+      "content": "<functions>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -104,7 +104,7 @@
     },
     {
       "id": 100267,
-      "content": "<|extra_id_2|>",
+      "content": "</functions>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -113,7 +113,7 @@
     },
     {
       "id": 100268,
-      "content": "<think>",
+      "content": "<function_calls>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -122,7 +122,7 @@
     },
     {
       "id": 100269,
-      "content": "</think>",
+      "content": "</function_calls>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -131,7 +131,7 @@
     },
     {
       "id": 100270,
-      "content": "<functions>",
+      "content": "<|extra_id_1|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -140,7 +140,7 @@
     },
     {
       "id": 100271,
-      "content": "</functions>",
+      "content": "<|extra_id_2|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -149,7 +149,7 @@
     },
     {
       "id": 100272,
-      "content": "<function_calls>",
+      "content": "<|extra_id_3|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -158,7 +158,7 @@
     },
     {
       "id": 100273,
-      "content": "</function_calls>",
+      "content": "<|extra_id_4|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -167,7 +167,7 @@
     },
     {
       "id": 100274,
-      "content": "<answer>",
+      "content": "<|extra_id_5|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -176,7 +176,7 @@
     },
     {
       "id": 100275,
-      "content": "</answer>",
+      "content": "<|extra_id_6|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -100495,6 +100495,10 @@
       ".WaitFor": 100253,
       "Ġdaycare": 100254,
       "ĠConveyor": 100255,
+      "<functions>": 100266,
+      "</functions>": 100267,
+      "<function_calls>": 100268,
+      "</function_calls>": 100269,
       "<|extra_id_0|>": 100256,
       "<|endoftext|>": 100257,
       "<|fim_prefix|>": 100258,
@@ -100505,16 +100509,12 @@
       "|||IP_ADDRESS|||": 100263,
       "<|im_start|>": 100264,
       "<|im_end|>": 100265,
-      "<|extra_id_1|>": 100266,
-      "<|extra_id_2|>": 100267,
-      "<think>": 100268,
-      "</think>": 100269,
-      "<functions>": 100270,
-      "</functions>": 100271,
-      "<function_calls>": 100272,
-      "</function_calls>": 100273,
-      "<answer>": 100274,
-      "</answer>": 100275,
+      "<|extra_id_1|>": 100270,
+      "<|extra_id_2|>": 100271,
+      "<|extra_id_3|>": 100272,
+      "<|extra_id_4|>": 100273,
+      "<|extra_id_5|>": 100274,
+      "<|extra_id_6|>": 100275,
       "<|endofprompt|>": 100276,
       "<|pad|>": 100277
     },

tokenizer_config.json

@@ -82,7 +82,7 @@
       "special": true
     },
     "100266": {
-      "content": "<|extra_id_1|>",
+      "content": "<functions>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -90,7 +90,7 @@
       "special": false
     },
     "100267": {
-      "content": "<|extra_id_2|>",
+      "content": "</functions>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -98,7 +98,7 @@
       "special": false
     },
     "100268": {
-      "content": "<think>",
+      "content": "<function_calls>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -106,7 +106,7 @@
       "special": false
     },
     "100269": {
-      "content": "</think>",
+      "content": "</function_calls>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -114,7 +114,7 @@
       "special": false
     },
     "100270": {
-      "content": "<functions>",
+      "content": "<|extra_id_1|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -122,7 +122,7 @@
       "special": false
     },
     "100271": {
-      "content": "</functions>",
+      "content": "<|extra_id_2|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -130,7 +130,7 @@
       "special": false
     },
     "100272": {
-      "content": "<function_calls>",
+      "content": "<|extra_id_3|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -138,7 +138,7 @@
       "special": false
     },
     "100273": {
-      "content": "</function_calls>",
+      "content": "<|extra_id_4|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -146,7 +146,7 @@
       "special": false
     },
     "100274": {
-      "content": "<answer>",
+      "content": "<|extra_id_5|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -154,7 +154,7 @@
       "special": false
     },
     "100275": {
-      "content": "</answer>",
+      "content": "<|extra_id_6|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,

vocab.json

@@ -100255,6 +100255,10 @@
   ".WaitFor": 100253,
   "Ġdaycare": 100254,
   "ĠConveyor": 100255,
+  "<functions>": 100266,
+  "</functions>": 100267,
+  "<function_calls>": 100268,
+  "</function_calls>": 100269,
   "<|extra_id_0|>": 100256,
   "<|endoftext|>": 100257,
   "<|fim_prefix|>": 100258,
@@ -100265,16 +100269,12 @@
   "|||IP_ADDRESS|||": 100263,
   "<|im_start|>": 100264,
   "<|im_end|>": 100265,
-  "<|extra_id_1|>": 100266,
-  "<|extra_id_2|>": 100267,
-  "<think>": 100268,
-  "</think>": 100269,
-  "<functions>": 100270,
-  "</functions>": 100271,
-  "<function_calls>": 100272,
-  "</function_calls>": 100273,
-  "<answer>": 100274,
-  "</answer>": 100275,
+  "<|extra_id_1|>": 100270,
+  "<|extra_id_2|>": 100271,
+  "<|extra_id_3|>": 100272,
+  "<|extra_id_4|>": 100273,
+  "<|extra_id_5|>": 100274,
+  "<|extra_id_6|>": 100275,
   "<|endofprompt|>": 100276,
   "<|pad|>": 100277
 }