Initial tokenizer upload

special_tokens_map.json

@@ -1,33 +1,7 @@
 {
   "additional_special_tokens": [
-    {
-      "content": "[KNOWLEDGE]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "[/KNOWLEDGE]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "[REASONING]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "[/REASONING]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    }
+    "[KNOWLEDGE]",
+    "[REASONING]"
   ],
   "bos_token": {
     "content": "<|begin_of_text|>",

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:929f1ab58b5deea4054d7d06c0100b1d3b4df857a534adf926ca54c1cfaf7666
-size 17210942
+oid sha256:7c3e39f7ef233dc92fc002889a496eb1c1d3282824fd32e594f3e52bda70c0fc
+size 17210576

tokenizer_config.json

@@ -2063,29 +2063,11 @@
       "rstrip": false,
       "single_word": false,
       "special": true
-    },
-    "128258": {
-      "content": "[/KNOWLEDGE]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128259": {
-      "content": "[/REASONING]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
     }
   },
   "additional_special_tokens": [
     "[KNOWLEDGE]",
-    "[/KNOWLEDGE]",
-    "[REASONING]",
-    "[/REASONING]"
+    "[REASONING]"
   ],
   "bos_token": "<|begin_of_text|>",
   "clean_up_tokenization_spaces": true,
@@ -2102,7 +2084,7 @@
   "pad_token_type_id": 0,
   "padding_side": "right",
   "stride": 0,
-  "tokenizer_class": "PreTrainedTokenizer",
+  "tokenizer_class": "PreTrainedTokenizerFast",
   "truncation_side": "right",
   "truncation_strategy": "longest_first"
 }