vocab size 25

config.json

@@ -26,5 +26,5 @@
   "torch_dtype": "bfloat16",
   "transformers_version": "4.46.1",
   "use_cache": true,
-  "vocab_size": 16
+  "vocab_size": 25
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cee59d9498b2a8bcd70242446dfa44404e52ee9d3e75413b42793acd0b53f68c
-size 140616
+oid sha256:8fb0d6686f6933f72ed6dc1838b9a6084d735e50e9f8554615ddf5488a7c6624
+size 142920

tokenizer.json

@@ -9,8 +9,8 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
-      "special": true
+      "normalized": true,
+      "special": false
     },
     {
       "id": 1,
@@ -18,8 +18,8 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
-      "special": true
+      "normalized": true,
+      "special": false
     },
     {
       "id": 2,
@@ -27,8 +27,8 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
-      "special": true
+      "normalized": true,
+      "special": false
     },
     {
       "id": 3,
@@ -36,8 +36,8 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
-      "special": true
+      "normalized": true,
+      "special": false
     },
     {
       "id": 4,
@@ -45,8 +45,8 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
-      "special": true
+      "normalized": true,
+      "special": false
     }
   ],
   "normalizer": {
@@ -169,7 +169,16 @@
       "koken": 12,
       "loken": 13,
       "moken": 14,
-      "token": 15
+      "noken": 15,
+      "poken": 16,
+      "qoken": 17,
+      "roken": 18,
+      "soken": 19,
+      "token": 20,
+      "voken": 21,
+      "woken": 22,
+      "xoken": 23,
+      "yoken": 24
     },
     "unk_token": "[UNK]"
   }

tokenizer_config.json

@@ -3,42 +3,42 @@
     "0": {
       "content": "[UNK]",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
     },
     "1": {
       "content": "[CLS]",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
     },
     "2": {
       "content": "[SEP]",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
     },
     "3": {
       "content": "[PAD]",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
     },
     "4": {
       "content": "[MASK]",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
     }
   },
   "bos_token": "[CLS]",