새롭게 업데이트된 토크나이저

added_tokens.json

@@ -1,10 +1,14 @@
 {
   "$~bos$": 50257,
-  "$~dev$": 50262,
+  "$~dev$": 50260,
   "$~eos$": 50256,
-  "$~me$": 50261,
+  "$~func-continue$": 50263,
+  "$~func-time$": 50262,
+  "$~info$": 50264,
+  "$~me$": 50265,
   "$~pad$": 50258,
-  "$~somebody$": 50263,
-  "$~the_flow_of_thought$": 50260,
+  "$~somebody$": 50266,
+  "$~tfot$": 50261,
+  "$~time$": 50267,
   "$~unk$": 50259
 }

special_tokens_map.json

@@ -1,4 +1,62 @@
 {
+  "additional_special_tokens": [
+    {
+      "content": "$~dev$",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "$~tfot$",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "$~func-time$",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "$~func-continue$",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "$~info$",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "$~me$",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "$~somebody$",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "$~time$",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
   "bos_token": {
     "content": "$~bos$",
     "lstrip": false,

tokenizer.json

@@ -41,8 +41,8 @@
     },
     {
       "id": 50260,
-      "content": "$~the_flow_of_thought$",
-      "single_word": true,
+      "content": "$~dev$",
+      "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
@@ -50,8 +50,8 @@
     },
     {
       "id": 50261,
-      "content": "$~me$",
-      "single_word": true,
+      "content": "$~tfot$",
+      "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
@@ -59,8 +59,8 @@
     },
     {
       "id": 50262,
-      "content": "$~dev$",
-      "single_word": true,
+      "content": "$~func-time$",
+      "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
@@ -68,8 +68,44 @@
     },
     {
       "id": 50263,
+      "content": "$~func-continue$",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 50264,
+      "content": "$~info$",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 50265,
+      "content": "$~me$",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 50266,
       "content": "$~somebody$",
-      "single_word": true,
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 50267,
+      "content": "$~time$",
+      "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,

tokenizer_config.json

@@ -35,38 +35,80 @@
       "special": true
     },
     "50260": {
-      "content": "$~the_flow_of_thought$",
+      "content": "$~dev$",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
-      "single_word": true,
+      "single_word": false,
       "special": true
     },
     "50261": {
-      "content": "$~me$",
+      "content": "$~tfot$",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
-      "single_word": true,
+      "single_word": false,
       "special": true
     },
     "50262": {
-      "content": "$~dev$",
+      "content": "$~func-time$",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
-      "single_word": true,
+      "single_word": false,
       "special": true
     },
     "50263": {
+      "content": "$~func-continue$",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50264": {
+      "content": "$~info$",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50265": {
+      "content": "$~me$",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50266": {
       "content": "$~somebody$",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
-      "single_word": true,
+      "single_word": false,
+      "special": true
+    },
+    "50267": {
+      "content": "$~time$",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
       "special": true
     }
   },
+  "additional_special_tokens": [
+    "$~dev$",
+    "$~tfot$",
+    "$~func-time$",
+    "$~func-continue$",
+    "$~info$",
+    "$~me$",
+    "$~somebody$",
+    "$~time$"
+  ],
   "bos_token": "$~bos$",
   "clean_up_tokenization_spaces": true,
   "eos_token": "$~eos$",