Upload tokenizer

special_tokens_map.json

@@ -1,7 +1,37 @@
 {
-  "cls_token": "[CLS]",
-  "mask_token": "[MASK]",
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "unk_token": "[UNK]"
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer.json

@@ -150,22 +150,22 @@
       "[CLS]": 2,
       "[SEP]": 3,
       "[MASK]": 4,
-      "[unused0]": 5,
-      "[unused1]": 6,
-      "[unused2]": 7,
-      "[unused3]": 8,
-      "[unused4]": 9,
-      "[unused5]": 10,
-      "[unused6]": 11,
-      "[unused7]": 12,
-      "[unused8]": 13,
-      "[unused9]": 14,
-      "[unused10]": 15,
-      "[unused11]": 16,
-      "[unused12]": 17,
-      "[unused13]": 18,
-      "[unused14]": 19,
-      "[unused15]": 20,
+      "재택": 5,
+      "무드등": 6,
+      "다용도실": 7,
+      "위층": 8,
+      "전등": 9,
+      "파랗게": 10,
+      "적색": 11,
+      "노랗게": 12,
+      "도록": 13,
+      "낮출": 14,
+      "해주실": 15,
+      "확인해볼": 16,
+      "알려줄": 17,
+      "다락방": 18,
+      "작은방": 19,
+      "큰방": 20,
       "[unused16]": 21,
       "[unused17]": 22,
       "[unused18]": 23,

vocab.txt

@@ -3,22 +3,22 @@
 [CLS]
 [SEP]
 [MASK]
-[unused0]
-[unused1]
-[unused2]
-[unused3]
-[unused4]
-[unused5]
-[unused6]
-[unused7]
-[unused8]
-[unused9]
-[unused10]
-[unused11]
-[unused12]
-[unused13]
-[unused14]
-[unused15]
+재택
+무드등
+다용도실
+위층
+전등
+파랗게
+적색
+노랗게
+도록
+낮출
+해주실
+확인해볼
+알려줄
+다락방
+작은방
+큰방
 [unused16]
 [unused17]
 [unused18]