working tokenizer

config.json

@@ -24,7 +24,7 @@
   "sliding_window": null,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.45.1",
+  "transformers_version": "4.46.1",
   "use_cache": true,
   "vocab_size": 25
 }

generation_config.json

@@ -3,5 +3,5 @@
   "bos_token_id": 1,
   "eos_token_id": 2,
   "pad_token_id": 3,
-  "transformers_version": "4.45.1"
+  "transformers_version": "4.46.1"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:926d327bcb02cde99d75d29fb7e3d2656623e4ba0ed69fe910f759dbf1692e8a
+oid sha256:619fe89397d93fd0a9d08c5ac59c8a0f3235948849bad255c9f4a570f15860b7
 size 142920

special_tokens_map.json

@@ -1,9 +1,9 @@
 {
-  "bos_token": "1",
-  "cls_token": "1",
-  "eos_token": "2",
-  "mask_token": "4",
-  "pad_token": "3",
-  "sep_token": "2",
-  "unk_token": "0"
+  "bos_token": "CLS",
+  "cls_token": "CLS",
+  "eos_token": "SEP",
+  "mask_token": "MASK",
+  "pad_token": "PAD",
+  "sep_token": "SEP",
+  "unk_token": "UNK"
 }

tokenizer.json

@@ -5,93 +5,48 @@
   "added_tokens": [
     {
       "id": 0,
-      "content": "[UNK]",
+      "content": "UNK",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
-      "special": true
+      "normalized": true,
+      "special": false
     },
     {
       "id": 1,
-      "content": "[CLS]",
+      "content": "CLS",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
-      "special": true
+      "normalized": true,
+      "special": false
     },
     {
       "id": 2,
-      "content": "[SEP]",
+      "content": "SEP",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
-      "special": true
+      "normalized": true,
+      "special": false
     },
     {
       "id": 3,
-      "content": "[PAD]",
+      "content": "PAD",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
-      "special": true
+      "normalized": true,
+      "special": false
     },
     {
       "id": 4,
-      "content": "[MASK]",
+      "content": "MASK",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 5,
-      "content": "1",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 6,
-      "content": "2",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 7,
-      "content": "3",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 8,
-      "content": "4",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 14,
-      "content": "0",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
+      "normalized": true,
+      "special": false
     }
   ],
   "normalizer": {
@@ -125,7 +80,7 @@
     "single": [
       {
         "SpecialToken": {
-          "id": "[CLS]",
+          "id": "CLS",
           "type_id": 0
         }
       },
@@ -137,7 +92,7 @@
       },
       {
         "SpecialToken": {
-          "id": "[SEP]",
+          "id": "SEP",
           "type_id": 0
         }
       }
@@ -145,7 +100,7 @@
     "pair": [
       {
         "SpecialToken": {
-          "id": "[CLS]",
+          "id": "CLS",
           "type_id": 0
         }
       },
@@ -157,7 +112,7 @@
       },
       {
         "SpecialToken": {
-          "id": "[SEP]",
+          "id": "SEP",
           "type_id": 0
         }
       },
@@ -169,28 +124,28 @@
       },
       {
         "SpecialToken": {
-          "id": "[SEP]",
+          "id": "SEP",
           "type_id": 1
         }
       }
     ],
     "special_tokens": {
-      "[CLS]": {
-        "id": "[CLS]",
+      "CLS": {
+        "id": "CLS",
         "ids": [
           1
         ],
         "tokens": [
-          "[CLS]"
+          "CLS"
         ]
       },
-      "[SEP]": {
-        "id": "[SEP]",
+      "SEP": {
+        "id": "SEP",
         "ids": [
           2
         ],
         "tokens": [
-          "[SEP]"
+          "SEP"
         ]
       }
     }
@@ -199,23 +154,32 @@
   "model": {
     "type": "WordLevel",
     "vocab": {
-      "[UNK]": 0,
-      "[CLS]": 1,
-      "[SEP]": 2,
-      "[PAD]": 3,
-      "[MASK]": 4,
-      "1": 5,
-      "2": 6,
-      "3": 7,
-      "4": 8,
-      "5": 9,
-      "6": 10,
-      "7": 11,
-      "8": 12,
-      "9": 13,
-      "0": 14,
-      "token": 15
+      "UNK": 0,
+      "CLS": 1,
+      "SEP": 2,
+      "PAD": 3,
+      "MASK": 4,
+      "boken": 5,
+      "coken": 6,
+      "doken": 7,
+      "foken": 8,
+      "goken": 9,
+      "hoken": 10,
+      "joken": 11,
+      "koken": 12,
+      "loken": 13,
+      "moken": 14,
+      "noken": 15,
+      "poken": 16,
+      "qoken": 17,
+      "roken": 18,
+      "soken": 19,
+      "token": 20,
+      "voken": 21,
+      "woken": 22,
+      "xoken": 23,
+      "yoken": 24
     },
-    "unk_token": "[UNK]"
+    "unk_token": "UNK"
   }
 }

tokenizer_config.json

@@ -1,97 +1,57 @@
 {
   "added_tokens_decoder": {
     "0": {
-      "content": "[UNK]",
+      "content": "UNK",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
     },
     "1": {
-      "content": "[CLS]",
+      "content": "CLS",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
     },
     "2": {
-      "content": "[SEP]",
+      "content": "SEP",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
     },
     "3": {
-      "content": "[PAD]",
+      "content": "PAD",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
     },
     "4": {
-      "content": "[MASK]",
+      "content": "MASK",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
-      "special": true
-    },
-    "5": {
-      "content": "1",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "6": {
-      "content": "2",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "7": {
-      "content": "3",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "8": {
-      "content": "4",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "14": {
-      "content": "0",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
+      "special": false
     }
   },
-  "bos_token": "1",
+  "bos_token": "CLS",
   "chat_template": "{% for message in messages %}{{ message['content'] }}{% endfor %}",
   "clean_up_tokenization_spaces": false,
-  "cls_token": "1",
-  "eos_token": "2",
-  "mask_token": "4",
+  "cls_token": "CLS",
+  "eos_token": "SEP",
+  "mask_token": "MASK",
   "max_new_tokens": 1048576,
   "model_max_length": 1048576,
-  "pad_token": "3",
+  "pad_token": "PAD",
   "padding_side": "right",
-  "sep_token": "2",
+  "sep_token": "SEP",
   "tokenizer_class": "PreTrainedTokenizerFast",
-  "unk_token": "0"
+  "unk_token": "UNK"
 }