사용자 정의 특수 토큰 제거 : 해당 토큰을 학습하는 것은 매우 비효율적.

added_tokens.json

@@ -1,14 +1,6 @@
 {
-  "$~bos$": 50257,
-  "$~dev$": 50260,
-  "$~eos$": 50256,
-  "$~func-continue$": 50263,
-  "$~func-time$": 50262,
-  "$~info$": 50264,
-  "$~me$": 50265,
-  "$~pad$": 50258,
-  "$~somebody$": 50266,
-  "$~tfot$": 50261,
-  "$~time$": 50267,
-  "$~unk$": 50259
+  "$~^bos$": 50257,
+  "$~^eos$": 50256,
+  "$~^pad$": 50258,
+  "$~^unk$": 50259
 }

special_tokens_map.json

@@ -1,85 +1,27 @@
 {
-  "additional_special_tokens": [
-    {
-      "content": "$~dev$",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "$~tfot$",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "$~func-time$",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "$~func-continue$",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "$~info$",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "$~me$",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "$~somebody$",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "$~time$",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    }
-  ],
   "bos_token": {
-    "content": "$~bos$",
+    "content": "$~^bos$",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "eos_token": {
-    "content": "$~eos$",
+    "content": "$~^eos$",
     "lstrip": false,
     "normalized": true,
     "rstrip": false,
     "single_word": false
   },
   "pad_token": {
-    "content": "$~pad$",
+    "content": "$~^pad$",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "unk_token": {
-    "content": "$~unk$",
+    "content": "$~^unk$",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -5,7 +5,7 @@
   "added_tokens": [
     {
       "id": 50256,
-      "content": "$~eos$",
+      "content": "$~^eos$",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -14,7 +14,7 @@
     },
     {
       "id": 50257,
-      "content": "$~bos$",
+      "content": "$~^bos$",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -23,7 +23,7 @@
     },
     {
       "id": 50258,
-      "content": "$~pad$",
+      "content": "$~^pad$",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -32,79 +32,7 @@
     },
     {
       "id": 50259,
-      "content": "$~unk$",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50260,
-      "content": "$~dev$",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50261,
-      "content": "$~tfot$",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50262,
-      "content": "$~func-time$",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50263,
-      "content": "$~func-continue$",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50264,
-      "content": "$~info$",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50265,
-      "content": "$~me$",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50266,
-      "content": "$~somebody$",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 50267,
-      "content": "$~time$",
+      "content": "$~^unk$",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,

tokenizer_config.json

@@ -3,7 +3,7 @@
   "add_prefix_space": false,
   "added_tokens_decoder": {
     "50256": {
-      "content": "$~eos$",
+      "content": "$~^eos$",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -11,7 +11,7 @@
       "special": true
     },
     "50257": {
-      "content": "$~bos$",
+      "content": "$~^bos$",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -19,7 +19,7 @@
       "special": true
     },
     "50258": {
-      "content": "$~pad$",
+      "content": "$~^pad$",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -27,71 +27,7 @@
       "special": true
     },
     "50259": {
-      "content": "$~unk$",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50260": {
-      "content": "$~dev$",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50261": {
-      "content": "$~tfot$",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50262": {
-      "content": "$~func-time$",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50263": {
-      "content": "$~func-continue$",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50264": {
-      "content": "$~info$",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50265": {
-      "content": "$~me$",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50266": {
-      "content": "$~somebody$",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "50267": {
-      "content": "$~time$",
+      "content": "$~^unk$",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -99,22 +35,12 @@
       "special": true
     }
   },
-  "additional_special_tokens": [
-    "$~dev$",
-    "$~tfot$",
-    "$~func-time$",
-    "$~func-continue$",
-    "$~info$",
-    "$~me$",
-    "$~somebody$",
-    "$~time$"
-  ],
-  "bos_token": "$~bos$",
+  "bos_token": "$~^bos$",
   "clean_up_tokenization_spaces": true,
-  "eos_token": "$~eos$",
+  "eos_token": "$~^eos$",
   "errors": "replace",
   "model_max_length": 8192,
-  "pad_token": "$~pad$",
+  "pad_token": "$~^pad$",
   "tokenizer_class": "GPT2Tokenizer",
-  "unk_token": "$~unk$"
+  "unk_token": "$~^unk$"
 }