add tokens
Browse files- added_tokens.json +7 -5
- tokenizer.json +18 -0
added_tokens.json
CHANGED
|
@@ -1,9 +1,11 @@
|
|
| 1 |
{
|
| 2 |
-
"์ฌ๋ฏบ๋์": 50140,
|
| 3 |
-
"์ฐธ์ง์๊ธฌ": 50141,
|
| 4 |
-
"์ด๋ผ๋๋ฅ": 50136,
|
| 5 |
-
"๊ป๋ฑ": 50135,
|
| 6 |
"๊ฝ๊ณ ": 50139,
|
| 7 |
"๋ฎซ์ผ๋ฉด": 50138,
|
| 8 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
}
|
|
|
|
| 1 |
{
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
"๊ฝ๊ณ ": 50139,
|
| 3 |
"๋ฎซ์ผ๋ฉด": 50138,
|
| 4 |
+
"๊ฐ ํ์์": 50142,
|
| 5 |
+
"๋๊ตฌ๋ ๊ฒ": 50137,
|
| 6 |
+
"์ด๋ผ๋๋ฅ": 50136,
|
| 7 |
+
"์ผ๋ง์ ๋แแ๋๋ฐ": 50143,
|
| 8 |
+
"๊ป๋ฑ": 50135,
|
| 9 |
+
"์ฐธ์ง์๊ธฌ": 50141,
|
| 10 |
+
"์ฌ๋ฏบ๋์": 50140
|
| 11 |
}
|
tokenizer.json
CHANGED
|
@@ -110,6 +110,24 @@
|
|
| 110 |
"lstrip": false,
|
| 111 |
"rstrip": false,
|
| 112 |
"normalized": true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 113 |
}
|
| 114 |
],
|
| 115 |
"normalizer": {
|
|
|
|
| 110 |
"lstrip": false,
|
| 111 |
"rstrip": false,
|
| 112 |
"normalized": true
|
| 113 |
+
},
|
| 114 |
+
{
|
| 115 |
+
"id": 50142,
|
| 116 |
+
"special": false,
|
| 117 |
+
"content": "๊ฐ ํ์์",
|
| 118 |
+
"single_word": false,
|
| 119 |
+
"lstrip": false,
|
| 120 |
+
"rstrip": false,
|
| 121 |
+
"normalized": true
|
| 122 |
+
},
|
| 123 |
+
{
|
| 124 |
+
"id": 50143,
|
| 125 |
+
"special": false,
|
| 126 |
+
"content": "์ผ๋ง์ ๋แแ๋๋ฐ",
|
| 127 |
+
"single_word": false,
|
| 128 |
+
"lstrip": false,
|
| 129 |
+
"rstrip": false,
|
| 130 |
+
"normalized": true
|
| 131 |
}
|
| 132 |
],
|
| 133 |
"normalizer": {
|