Revert "Upload tokenizer"

This reverts commit 73c49a12e3243978affacfcaf5427654288b8583.

added_tokens.json

@@ -1,17 +1,13 @@
 {
-  "\"arguments\":": 32014,
-  "\"function\":": 32012,
-  "\"name\":": 32013,
   "<|assistant|>": 32001,
   "<|endoftext|>": 32000,
   "<|end|>": 32007,
-  "<|function_metadata|>": 32008,
   "<|placeholder1|>": 32002,
   "<|placeholder2|>": 32003,
   "<|placeholder3|>": 32004,
-  "<|placeholder4|>": 32005,
-  "<|system|>": 32006,
-  "<|tool_calls|>": 32011,
+  "<|tool_calls|>": 32005,
+  "<|function_metadata|>": 32008,
   "<|tool_results|>": 32009,
+  "<|system|>": 32006,
   "<|user|>": 32010
 }

tokenizer.json

@@ -128,42 +128,6 @@
       "rstrip": true,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 32011,
-      "content": "<|tool_calls|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32012,
-      "content": "\"function\":",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 32013,
-      "content": "\"name\":",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
-    },
-    {
-      "id": 32014,
-      "content": "\"arguments\":",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true,
-      "special": false
     }
   ],
   "normalizer": {
@@ -93496,4 +93460,4 @@
       "▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
     ]
   }
-}
+}

tokenizer_config.json

@@ -67,7 +67,7 @@
       "special": true
     },
     "32005": {
-      "content": "<|placeholder4|>",
+      "content": "<|tool_calls|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": true,
@@ -113,38 +113,6 @@
       "rstrip": true,
       "single_word": false,
       "special": true
-    },
-    "32011": {
-      "content": "<|tool_calls|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32012": {
-      "content": "\"function\":",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "32013": {
-      "content": "\"name\":",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "32014": {
-      "content": "\"arguments\":",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
     }
   },
   "bos_token": "<s>",
@@ -160,12 +128,13 @@
   ],
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
-  "func_bos_token": "<|tool_calls|>",
+  "legacy": false,
   "model_max_length": 4096,
   "pad_token": "<|endoftext|>",
   "padding_side": "left",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
-  "use_default_system_prompt": false
+  "use_default_system_prompt": false,
+  "func_bos_token": "<|tool_calls|>"
 }