Upload 2 files
Browse files- tokenizer.json +477 -450
- tokenizer.model +3 -0
tokenizer.json
CHANGED
|
@@ -77,14 +77,29 @@
|
|
| 77 |
"type_id": 0
|
| 78 |
}
|
| 79 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
{
|
| 81 |
"Sequence": {
|
| 82 |
"id": "B",
|
| 83 |
-
"type_id":
|
| 84 |
}
|
| 85 |
}
|
| 86 |
],
|
| 87 |
"special_tokens": {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
"<s>": {
|
| 89 |
"id": "<s>",
|
| 90 |
"ids": [
|
|
@@ -37166,455 +37181,467 @@
|
|
| 37166 |
"倉": 37034,
|
| 37167 |
"켄": 37035,
|
| 37168 |
"釈": 37036,
|
| 37169 |
-
"
|
| 37170 |
-
"
|
| 37171 |
-
"
|
| 37172 |
-
"
|
| 37173 |
-
"
|
| 37174 |
-
"
|
| 37175 |
-
"
|
| 37176 |
-
"
|
| 37177 |
-
"
|
| 37178 |
-
"
|
| 37179 |
-
"
|
| 37180 |
-
"
|
| 37181 |
-
"
|
| 37182 |
-
"
|
| 37183 |
-
"
|
| 37184 |
-
"
|
| 37185 |
-
"
|
| 37186 |
-
"
|
| 37187 |
-
"
|
| 37188 |
-
"
|
| 37189 |
-
"
|
| 37190 |
-
"
|
| 37191 |
-
"
|
| 37192 |
-
"
|
| 37193 |
-
"
|
| 37194 |
-
"
|
| 37195 |
-
"
|
| 37196 |
-
"
|
| 37197 |
-
"
|
| 37198 |
-
"
|
| 37199 |
-
"
|
| 37200 |
-
"
|
| 37201 |
-
"
|
| 37202 |
-
"
|
| 37203 |
-
"
|
| 37204 |
-
"
|
| 37205 |
-
"
|
| 37206 |
-
"
|
| 37207 |
-
"
|
| 37208 |
-
"
|
| 37209 |
-
"
|
| 37210 |
-
"
|
| 37211 |
-
"
|
| 37212 |
-
"
|
| 37213 |
-
"
|
| 37214 |
-
"
|
| 37215 |
-
"
|
| 37216 |
-
"
|
| 37217 |
-
"
|
| 37218 |
-
"
|
| 37219 |
-
"
|
| 37220 |
-
"
|
| 37221 |
-
"
|
| 37222 |
-
"
|
| 37223 |
-
"
|
| 37224 |
-
"
|
| 37225 |
-
"
|
| 37226 |
-
"
|
| 37227 |
-
"
|
| 37228 |
-
"
|
| 37229 |
-
"
|
| 37230 |
-
"
|
| 37231 |
-
"
|
| 37232 |
-
"
|
| 37233 |
-
"
|
| 37234 |
-
"
|
| 37235 |
-
"
|
| 37236 |
-
"
|
| 37237 |
-
"
|
| 37238 |
-
"
|
| 37239 |
-
"
|
| 37240 |
-
"
|
| 37241 |
-
"
|
| 37242 |
-
"
|
| 37243 |
-
"
|
| 37244 |
-
"
|
| 37245 |
-
"
|
| 37246 |
-
"
|
| 37247 |
-
"
|
| 37248 |
-
"
|
| 37249 |
-
"
|
| 37250 |
-
"
|
| 37251 |
-
"
|
| 37252 |
-
"
|
| 37253 |
-
"
|
| 37254 |
-
"
|
| 37255 |
-
"
|
| 37256 |
-
"
|
| 37257 |
-
"
|
| 37258 |
-
"
|
| 37259 |
-
"
|
| 37260 |
-
"
|
| 37261 |
-
"
|
| 37262 |
-
"
|
| 37263 |
-
"
|
| 37264 |
-
"
|
| 37265 |
-
"
|
| 37266 |
-
"
|
| 37267 |
-
"
|
| 37268 |
-
"
|
| 37269 |
-
"
|
| 37270 |
-
"
|
| 37271 |
-
"
|
| 37272 |
-
"
|
| 37273 |
-
"
|
| 37274 |
-
"
|
| 37275 |
-
"
|
| 37276 |
-
"
|
| 37277 |
-
"
|
| 37278 |
-
"
|
| 37279 |
-
"
|
| 37280 |
-
"
|
| 37281 |
-
"
|
| 37282 |
-
"
|
| 37283 |
-
"
|
| 37284 |
-
"
|
| 37285 |
-
"
|
| 37286 |
-
"
|
| 37287 |
-
"
|
| 37288 |
-
"
|
| 37289 |
-
"
|
| 37290 |
-
"
|
| 37291 |
-
"
|
| 37292 |
-
"
|
| 37293 |
-
"
|
| 37294 |
-
"
|
| 37295 |
-
"
|
| 37296 |
-
"
|
| 37297 |
-
"
|
| 37298 |
-
"
|
| 37299 |
-
"
|
| 37300 |
-
"
|
| 37301 |
-
"
|
| 37302 |
-
"
|
| 37303 |
-
"
|
| 37304 |
-
"
|
| 37305 |
-
"
|
| 37306 |
-
"
|
| 37307 |
-
"
|
| 37308 |
-
"
|
| 37309 |
-
"
|
| 37310 |
-
"
|
| 37311 |
-
"
|
| 37312 |
-
"
|
| 37313 |
-
"
|
| 37314 |
-
"
|
| 37315 |
-
"
|
| 37316 |
-
"
|
| 37317 |
-
"
|
| 37318 |
-
"
|
| 37319 |
-
"
|
| 37320 |
-
"
|
| 37321 |
-
"
|
| 37322 |
-
"
|
| 37323 |
-
"
|
| 37324 |
-
"
|
| 37325 |
-
"
|
| 37326 |
-
"
|
| 37327 |
-
"
|
| 37328 |
-
"
|
| 37329 |
-
"
|
| 37330 |
-
"
|
| 37331 |
-
"
|
| 37332 |
-
"
|
| 37333 |
-
"
|
| 37334 |
-
"
|
| 37335 |
-
"
|
| 37336 |
-
"
|
| 37337 |
-
"
|
| 37338 |
-
"
|
| 37339 |
-
"
|
| 37340 |
-
"
|
| 37341 |
-
"
|
| 37342 |
-
"
|
| 37343 |
-
"
|
| 37344 |
-
"
|
| 37345 |
-
"
|
| 37346 |
-
"
|
| 37347 |
-
"
|
| 37348 |
-
"
|
| 37349 |
-
"
|
| 37350 |
-
"
|
| 37351 |
-
"
|
| 37352 |
-
"
|
| 37353 |
-
"
|
| 37354 |
-
"
|
| 37355 |
-
"
|
| 37356 |
-
"
|
| 37357 |
-
"
|
| 37358 |
-
"
|
| 37359 |
-
"
|
| 37360 |
-
"
|
| 37361 |
-
"
|
| 37362 |
-
"
|
| 37363 |
-
"
|
| 37364 |
-
"
|
| 37365 |
-
"
|
| 37366 |
-
"
|
| 37367 |
-
"
|
| 37368 |
-
"
|
| 37369 |
-
"
|
| 37370 |
-
"
|
| 37371 |
-
"
|
| 37372 |
-
"
|
| 37373 |
-
"
|
| 37374 |
-
"
|
| 37375 |
-
"
|
| 37376 |
-
"
|
| 37377 |
-
"
|
| 37378 |
-
"
|
| 37379 |
-
"
|
| 37380 |
-
"
|
| 37381 |
-
"
|
| 37382 |
-
"
|
| 37383 |
-
"
|
| 37384 |
-
"
|
| 37385 |
-
"
|
| 37386 |
-
"
|
| 37387 |
-
"
|
| 37388 |
-
"
|
| 37389 |
-
"
|
| 37390 |
-
"
|
| 37391 |
-
"
|
| 37392 |
-
"
|
| 37393 |
-
"
|
| 37394 |
-
"
|
| 37395 |
-
"
|
| 37396 |
-
"
|
| 37397 |
-
"
|
| 37398 |
-
"
|
| 37399 |
-
"
|
| 37400 |
-
"
|
| 37401 |
-
"
|
| 37402 |
-
"
|
| 37403 |
-
"
|
| 37404 |
-
"
|
| 37405 |
-
"
|
| 37406 |
-
"
|
| 37407 |
-
"
|
| 37408 |
-
"
|
| 37409 |
-
"
|
| 37410 |
-
"
|
| 37411 |
-
"
|
| 37412 |
-
"
|
| 37413 |
-
"
|
| 37414 |
-
"
|
| 37415 |
-
"
|
| 37416 |
-
"
|
| 37417 |
-
"
|
| 37418 |
-
"
|
| 37419 |
-
"
|
| 37420 |
-
"
|
| 37421 |
-
"
|
| 37422 |
-
"
|
| 37423 |
-
"
|
| 37424 |
-
"
|
| 37425 |
-
"
|
| 37426 |
-
"
|
| 37427 |
-
"
|
| 37428 |
-
"
|
| 37429 |
-
"
|
| 37430 |
-
"
|
| 37431 |
-
"
|
| 37432 |
-
"
|
| 37433 |
-
"
|
| 37434 |
-
"
|
| 37435 |
-
"
|
| 37436 |
-
"
|
| 37437 |
-
"
|
| 37438 |
-
"
|
| 37439 |
-
"
|
| 37440 |
-
"
|
| 37441 |
-
"
|
| 37442 |
-
"
|
| 37443 |
-
"
|
| 37444 |
-
"
|
| 37445 |
-
"
|
| 37446 |
-
"
|
| 37447 |
-
"
|
| 37448 |
-
"
|
| 37449 |
-
"
|
| 37450 |
-
"
|
| 37451 |
-
"
|
| 37452 |
-
"
|
| 37453 |
-
"
|
| 37454 |
-
"
|
| 37455 |
-
"
|
| 37456 |
-
"
|
| 37457 |
-
"
|
| 37458 |
-
"
|
| 37459 |
-
"
|
| 37460 |
-
"
|
| 37461 |
-
"
|
| 37462 |
-
"
|
| 37463 |
-
"
|
| 37464 |
-
"
|
| 37465 |
-
"
|
| 37466 |
-
"
|
| 37467 |
-
"
|
| 37468 |
-
"
|
| 37469 |
-
"
|
| 37470 |
-
"
|
| 37471 |
-
"
|
| 37472 |
-
"
|
| 37473 |
-
"
|
| 37474 |
-
"
|
| 37475 |
-
"
|
| 37476 |
-
"
|
| 37477 |
-
"
|
| 37478 |
-
"
|
| 37479 |
-
"
|
| 37480 |
-
"
|
| 37481 |
-
"
|
| 37482 |
-
"
|
| 37483 |
-
"
|
| 37484 |
-
"
|
| 37485 |
-
"
|
| 37486 |
-
"
|
| 37487 |
-
"
|
| 37488 |
-
"
|
| 37489 |
-
"
|
| 37490 |
-
"
|
| 37491 |
-
"
|
| 37492 |
-
"
|
| 37493 |
-
"
|
| 37494 |
-
"
|
| 37495 |
-
"
|
| 37496 |
-
"
|
| 37497 |
-
"
|
| 37498 |
-
"
|
| 37499 |
-
"
|
| 37500 |
-
"
|
| 37501 |
-
"
|
| 37502 |
-
"
|
| 37503 |
-
"
|
| 37504 |
-
"
|
| 37505 |
-
"
|
| 37506 |
-
"
|
| 37507 |
-
"
|
| 37508 |
-
"
|
| 37509 |
-
"
|
| 37510 |
-
"
|
| 37511 |
-
"
|
| 37512 |
-
"
|
| 37513 |
-
"
|
| 37514 |
-
"
|
| 37515 |
-
"
|
| 37516 |
-
"
|
| 37517 |
-
"
|
| 37518 |
-
"
|
| 37519 |
-
"
|
| 37520 |
-
"
|
| 37521 |
-
"
|
| 37522 |
-
"
|
| 37523 |
-
"
|
| 37524 |
-
"
|
| 37525 |
-
"
|
| 37526 |
-
"
|
| 37527 |
-
"
|
| 37528 |
-
"
|
| 37529 |
-
"
|
| 37530 |
-
"
|
| 37531 |
-
"
|
| 37532 |
-
"
|
| 37533 |
-
"
|
| 37534 |
-
"
|
| 37535 |
-
"
|
| 37536 |
-
"
|
| 37537 |
-
"
|
| 37538 |
-
"
|
| 37539 |
-
"
|
| 37540 |
-
"
|
| 37541 |
-
"
|
| 37542 |
-
"
|
| 37543 |
-
"
|
| 37544 |
-
"
|
| 37545 |
-
"
|
| 37546 |
-
"
|
| 37547 |
-
"
|
| 37548 |
-
"
|
| 37549 |
-
"
|
| 37550 |
-
"
|
| 37551 |
-
"
|
| 37552 |
-
"
|
| 37553 |
-
"
|
| 37554 |
-
"
|
| 37555 |
-
"
|
| 37556 |
-
"
|
| 37557 |
-
"
|
| 37558 |
-
"
|
| 37559 |
-
"
|
| 37560 |
-
"
|
| 37561 |
-
"
|
| 37562 |
-
"
|
| 37563 |
-
"
|
| 37564 |
-
"
|
| 37565 |
-
"
|
| 37566 |
-
"
|
| 37567 |
-
"
|
| 37568 |
-
"
|
| 37569 |
-
"
|
| 37570 |
-
"
|
| 37571 |
-
"
|
| 37572 |
-
"
|
| 37573 |
-
"
|
| 37574 |
-
"
|
| 37575 |
-
"
|
| 37576 |
-
"
|
| 37577 |
-
"
|
| 37578 |
-
"
|
| 37579 |
-
"
|
| 37580 |
-
"
|
| 37581 |
-
"
|
| 37582 |
-
"
|
| 37583 |
-
"
|
| 37584 |
-
"
|
| 37585 |
-
"
|
| 37586 |
-
"
|
| 37587 |
-
"
|
| 37588 |
-
"
|
| 37589 |
-
"
|
| 37590 |
-
"
|
| 37591 |
-
"
|
| 37592 |
-
"
|
| 37593 |
-
"
|
| 37594 |
-
"
|
| 37595 |
-
"
|
| 37596 |
-
"
|
| 37597 |
-
"
|
| 37598 |
-
"
|
| 37599 |
-
"
|
| 37600 |
-
"
|
| 37601 |
-
"
|
| 37602 |
-
"
|
| 37603 |
-
"
|
| 37604 |
-
"
|
| 37605 |
-
"
|
| 37606 |
-
"
|
| 37607 |
-
"
|
| 37608 |
-
"
|
| 37609 |
-
"
|
| 37610 |
-
"
|
| 37611 |
-
"
|
| 37612 |
-
"
|
| 37613 |
-
"
|
| 37614 |
-
"
|
| 37615 |
-
"
|
| 37616 |
-
"
|
| 37617 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37618 |
},
|
| 37619 |
"merges": [
|
| 37620 |
"▁ t",
|
|
|
|
| 77 |
"type_id": 0
|
| 78 |
}
|
| 79 |
},
|
| 80 |
+
{
|
| 81 |
+
"SpecialToken": {
|
| 82 |
+
"id": "<s>",
|
| 83 |
+
"type_id": 1
|
| 84 |
+
}
|
| 85 |
+
},
|
| 86 |
{
|
| 87 |
"Sequence": {
|
| 88 |
"id": "B",
|
| 89 |
+
"type_id": 1
|
| 90 |
}
|
| 91 |
}
|
| 92 |
],
|
| 93 |
"special_tokens": {
|
| 94 |
+
"</s>": {
|
| 95 |
+
"id": "</s>",
|
| 96 |
+
"ids": [
|
| 97 |
+
2
|
| 98 |
+
],
|
| 99 |
+
"tokens": [
|
| 100 |
+
"</s>"
|
| 101 |
+
]
|
| 102 |
+
},
|
| 103 |
"<s>": {
|
| 104 |
"id": "<s>",
|
| 105 |
"ids": [
|
|
|
|
| 37181 |
"倉": 37034,
|
| 37182 |
"켄": 37035,
|
| 37183 |
"釈": 37036,
|
| 37184 |
+
"ᠠ": 37037,
|
| 37185 |
+
"괄": 37038,
|
| 37186 |
+
"맷": 37039,
|
| 37187 |
+
"枠": 37040,
|
| 37188 |
+
"삶": 37041,
|
| 37189 |
+
"誕": 37042,
|
| 37190 |
+
"띠": 37043,
|
| 37191 |
+
"덤": 37044,
|
| 37192 |
+
"ぜ": 37045,
|
| 37193 |
+
"軸": 37046,
|
| 37194 |
+
"냉": 37047,
|
| 37195 |
+
"緑": 37048,
|
| 37196 |
+
"뱅": 37049,
|
| 37197 |
+
"납": 37050,
|
| 37198 |
+
"엽": 37051,
|
| 37199 |
+
"썼": 37052,
|
| 37200 |
+
"끄": 37053,
|
| 37201 |
+
"웰": 37054,
|
| 37202 |
+
"맛": 37055,
|
| 37203 |
+
"緯": 37056,
|
| 37204 |
+
"災": 37057,
|
| 37205 |
+
"흰": 37058,
|
| 37206 |
+
"븐": 37059,
|
| 37207 |
+
"氷": 37060,
|
| 37208 |
+
"겪": 37061,
|
| 37209 |
+
"診": 37062,
|
| 37210 |
+
"냥": 37063,
|
| 37211 |
+
"隣": 37064,
|
| 37212 |
+
"퀸": 37065,
|
| 37213 |
+
"쿼": 37066,
|
| 37214 |
+
"뤼": 37067,
|
| 37215 |
+
"쿨": 37068,
|
| 37216 |
+
"뷰": 37069,
|
| 37217 |
+
"뮌": 37070,
|
| 37218 |
+
"앵": 37071,
|
| 37219 |
+
"샘": 37072,
|
| 37220 |
+
"잃": 37073,
|
| 37221 |
+
"큼": 37074,
|
| 37222 |
+
"힐": 37075,
|
| 37223 |
+
"돼": 37076,
|
| 37224 |
+
"첼": 37077,
|
| 37225 |
+
"벡": 37078,
|
| 37226 |
+
"붉": 37079,
|
| 37227 |
+
"傾": 37080,
|
| 37228 |
+
"낱": 37081,
|
| 37229 |
+
"풋": 37082,
|
| 37230 |
+
"엠": 37083,
|
| 37231 |
+
"鎖": 37084,
|
| 37232 |
+
"첨": 37085,
|
| 37233 |
+
"歯": 37086,
|
| 37234 |
+
"낳": 37087,
|
| 37235 |
+
"択": 37088,
|
| 37236 |
+
"聴": 37089,
|
| 37237 |
+
"넬": 37090,
|
| 37238 |
+
"딕": 37091,
|
| 37239 |
+
"찍": 37092,
|
| 37240 |
+
"흘": 37093,
|
| 37241 |
+
"룡": 37094,
|
| 37242 |
+
"낼": 37095,
|
| 37243 |
+
"慶": 37096,
|
| 37244 |
+
"弁": 37097,
|
| 37245 |
+
"縁": 37098,
|
| 37246 |
+
"듯": 37099,
|
| 37247 |
+
"℃": 37100,
|
| 37248 |
+
"賀": 37101,
|
| 37249 |
+
"팽": 37102,
|
| 37250 |
+
"늬": 37103,
|
| 37251 |
+
"襲": 37104,
|
| 37252 |
+
"좁": 37105,
|
| 37253 |
+
"팩": 37106,
|
| 37254 |
+
"掛": 37107,
|
| 37255 |
+
"룸": 37108,
|
| 37256 |
+
"齢": 37109,
|
| 37257 |
+
"姉": 37110,
|
| 37258 |
+
"繰": 37111,
|
| 37259 |
+
"傘": 37112,
|
| 37260 |
+
"翌": 37113,
|
| 37261 |
+
"펙": 37114,
|
| 37262 |
+
"릉": 37115,
|
| 37263 |
+
"鏡": 37116,
|
| 37264 |
+
"衝": 37117,
|
| 37265 |
+
"밑": 37118,
|
| 37266 |
+
"뿌": 37119,
|
| 37267 |
+
"숨": 37120,
|
| 37268 |
+
"겔": 37121,
|
| 37269 |
+
"렘": 37122,
|
| 37270 |
+
"綱": 37123,
|
| 37271 |
+
"偉": 37124,
|
| 37272 |
+
"ヶ": 37125,
|
| 37273 |
+
"飾": 37126,
|
| 37274 |
+
"貢": 37127,
|
| 37275 |
+
"씬": 37128,
|
| 37276 |
+
"摂": 37129,
|
| 37277 |
+
"臓": 37130,
|
| 37278 |
+
"竜": 37131,
|
| 37279 |
+
"핫": 37132,
|
| 37280 |
+
"톰": 37133,
|
| 37281 |
+
"갤": 37134,
|
| 37282 |
+
"댄": 37135,
|
| 37283 |
+
"腸": 37136,
|
| 37284 |
+
"顕": 37137,
|
| 37285 |
+
"답": 37138,
|
| 37286 |
+
"탠": 37139,
|
| 37287 |
+
"誇": 37140,
|
| 37288 |
+
"縄": 37141,
|
| 37289 |
+
"퀘": 37142,
|
| 37290 |
+
"덩": 37143,
|
| 37291 |
+
"儀": 37144,
|
| 37292 |
+
"엇": 37145,
|
| 37293 |
+
"ぽ": 37146,
|
| 37294 |
+
"셋": 37147,
|
| 37295 |
+
"鎌": 37148,
|
| 37296 |
+
"펀": 37149,
|
| 37297 |
+
"덮": 37150,
|
| 37298 |
+
"첩": 37151,
|
| 37299 |
+
"損": 37152,
|
| 37300 |
+
"셸": 37153,
|
| 37301 |
+
"눌": 37154,
|
| 37302 |
+
"겉": 37155,
|
| 37303 |
+
"햄": 37156,
|
| 37304 |
+
"쌓": 37157,
|
| 37305 |
+
"納": 37158,
|
| 37306 |
+
"顧": 37159,
|
| 37307 |
+
"묶": 37160,
|
| 37308 |
+
"렐": 37161,
|
| 37309 |
+
"왼": 37162,
|
| 37310 |
+
"듣": 37163,
|
| 37311 |
+
"깔": 37164,
|
| 37312 |
+
"캔": 37165,
|
| 37313 |
+
"꼭": 37166,
|
| 37314 |
+
"닛": 37167,
|
| 37315 |
+
"貫": 37168,
|
| 37316 |
+
"옷": 37169,
|
| 37317 |
+
"/": 37170,
|
| 37318 |
+
"훨": 37171,
|
| 37319 |
+
"슴": 37172,
|
| 37320 |
+
"뿔": 37173,
|
| 37321 |
+
"슐": 37174,
|
| 37322 |
+
"審": 37175,
|
| 37323 |
+
"벗": 37176,
|
| 37324 |
+
"깃": 37177,
|
| 37325 |
+
"銘": 37178,
|
| 37326 |
+
"펑": 37179,
|
| 37327 |
+
"꿈": 37180,
|
| 37328 |
+
"縦": 37181,
|
| 37329 |
+
"뒷": 37182,
|
| 37330 |
+
"쓸": 37183,
|
| 37331 |
+
"鋼": 37184,
|
| 37332 |
+
"넥": 37185,
|
| 37333 |
+
"룰": 37186,
|
| 37334 |
+
"貿": 37187,
|
| 37335 |
+
"偵": 37188,
|
| 37336 |
+
"곰": 37189,
|
| 37337 |
+
"펄": 37190,
|
| 37338 |
+
"젊": 37191,
|
| 37339 |
+
"覇": 37192,
|
| 37340 |
+
"좀": 37193,
|
| 37341 |
+
"섞": 37194,
|
| 37342 |
+
"뤄": 37195,
|
| 37343 |
+
"봄": 37196,
|
| 37344 |
+
"킷": 37197,
|
| 37345 |
+
"칩": 37198,
|
| 37346 |
+
"ᠨ": 37199,
|
| 37347 |
+
"밥": 37200,
|
| 37348 |
+
"떻": 37201,
|
| 37349 |
+
"奪": 37202,
|
| 37350 |
+
"겠": 37203,
|
| 37351 |
+
"紛": 37204,
|
| 37352 |
+
"툴": 37205,
|
| 37353 |
+
"빼": 37206,
|
| 37354 |
+
"針": 37207,
|
| 37355 |
+
"춤": 37208,
|
| 37356 |
+
"繊": 37209,
|
| 37357 |
+
"랩": 37210,
|
| 37358 |
+
"賛": 37211,
|
| 37359 |
+
"힙": 37212,
|
| 37360 |
+
"첸": 37213,
|
| 37361 |
+
"遅": 37214,
|
| 37362 |
+
"塚": 37215,
|
| 37363 |
+
"벼": 37216,
|
| 37364 |
+
"툰": 37217,
|
| 37365 |
+
"렴": 37218,
|
| 37366 |
+
"뇨": 37219,
|
| 37367 |
+
"끈": 37220,
|
| 37368 |
+
"譲": 37221,
|
| 37369 |
+
"앗": 37222,
|
| 37370 |
+
"콥": 37223,
|
| 37371 |
+
"렛": 37224,
|
| 37372 |
+
"끔": 37225,
|
| 37373 |
+
"빗": 37226,
|
| 37374 |
+
"뱀": 37227,
|
| 37375 |
+
"랴": 37228,
|
| 37376 |
+
"浄": 37229,
|
| 37377 |
+
"딜": 37230,
|
| 37378 |
+
"裏": 37231,
|
| 37379 |
+
"뽑": 37232,
|
| 37380 |
+
"鎮": 37233,
|
| 37381 |
+
"ぱ": 37234,
|
| 37382 |
+
"遷": 37235,
|
| 37383 |
+
"윙": 37236,
|
| 37384 |
+
"딴": 37237,
|
| 37385 |
+
"싼": 37238,
|
| 37386 |
+
"깥": 37239,
|
| 37387 |
+
"繋": 37240,
|
| 37388 |
+
"殻": 37241,
|
| 37389 |
+
"깝": 37242,
|
| 37390 |
+
"앱": 37243,
|
| 37391 |
+
"恵": 37244,
|
| 37392 |
+
"섹": 37245,
|
| 37393 |
+
"訓": 37246,
|
| 37394 |
+
"汎": 37247,
|
| 37395 |
+
"륜": 37248,
|
| 37396 |
+
"닷": 37249,
|
| 37397 |
+
"봇": 37250,
|
| 37398 |
+
"紋": 37251,
|
| 37399 |
+
"噴": 37252,
|
| 37400 |
+
"敎": 37253,
|
| 37401 |
+
"訪": 37254,
|
| 37402 |
+
"棄": 37255,
|
| 37403 |
+
"숲": 37256,
|
| 37404 |
+
"붓": 37257,
|
| 37405 |
+
"幣": 37258,
|
| 37406 |
+
"삽": 37259,
|
| 37407 |
+
"슘": 37260,
|
| 37408 |
+
"汚": 37261,
|
| 37409 |
+
"獣": 37262,
|
| 37410 |
+
"넨": 37263,
|
| 37411 |
+
"銅": 37264,
|
| 37412 |
+
"ぬ": 37265,
|
| 37413 |
+
"輝": 37266,
|
| 37414 |
+
"뮬": 37267,
|
| 37415 |
+
"糸": 37268,
|
| 37416 |
+
"혐": 37269,
|
| 37417 |
+
"찌": 37270,
|
| 37418 |
+
"덱": 37271,
|
| 37419 |
+
"頻": 37272,
|
| 37420 |
+
"戯": 37273,
|
| 37421 |
+
"轄": 37274,
|
| 37422 |
+
"払": 37275,
|
| 37423 |
+
"ᡳ": 37276,
|
| 37424 |
+
"薩": 37277,
|
| 37425 |
+
"셉": 37278,
|
| 37426 |
+
"%": 37279,
|
| 37427 |
+
"稲": 37280,
|
| 37428 |
+
"넌": 37281,
|
| 37429 |
+
"ᠣ": 37282,
|
| 37430 |
+
"끊": 37283,
|
| 37431 |
+
"陣": 37284,
|
| 37432 |
+
"謀": 37285,
|
| 37433 |
+
"諱": 37286,
|
| 37434 |
+
"煙": 37287,
|
| 37435 |
+
"꺼": 37288,
|
| 37436 |
+
"쾌": 37289,
|
| 37437 |
+
"郷": 37290,
|
| 37438 |
+
"돔": 37291,
|
| 37439 |
+
"캅": 37292,
|
| 37440 |
+
"빵": 37293,
|
| 37441 |
+
"섰": 37294,
|
| 37442 |
+
"묵": 37295,
|
| 37443 |
+
"홋": 37296,
|
| 37444 |
+
"톡": 37297,
|
| 37445 |
+
"겹": 37298,
|
| 37446 |
+
"뚜": 37299,
|
| 37447 |
+
"띄": 37300,
|
| 37448 |
+
"婦": 37301,
|
| 37449 |
+
"緩": 37302,
|
| 37450 |
+
"拝": 37303,
|
| 37451 |
+
"갱": 37304,
|
| 37452 |
+
"郵": 37305,
|
| 37453 |
+
"託": 37306,
|
| 37454 |
+
"벳": 37307,
|
| 37455 |
+
"흉": 37308,
|
| 37456 |
+
"훌": 37309,
|
| 37457 |
+
"닭": 37310,
|
| 37458 |
+
"蘭": 37311,
|
| 37459 |
+
"뻗": 37312,
|
| 37460 |
+
"晩": 37313,
|
| 37461 |
+
"譜": 37314,
|
| 37462 |
+
"꺾": 37315,
|
| 37463 |
+
"꼴": 37316,
|
| 37464 |
+
"剛": 37317,
|
| 37465 |
+
"닿": 37318,
|
| 37466 |
+
"둑": 37319,
|
| 37467 |
+
"渉": 37320,
|
| 37468 |
+
"姫": 37321,
|
| 37469 |
+
"捜": 37322,
|
| 37470 |
+
"蘇": 37323,
|
| 37471 |
+
"짝": 37324,
|
| 37472 |
+
"奨": 37325,
|
| 37473 |
+
"舎": 37326,
|
| 37474 |
+
"贈": 37327,
|
| 37475 |
+
"탱": 37328,
|
| 37476 |
+
"햇": 37329,
|
| 37477 |
+
"潟": 37330,
|
| 37478 |
+
"軟": 37331,
|
| 37479 |
+
"鍵": 37332,
|
| 37480 |
+
"럿": 37333,
|
| 37481 |
+
"陥": 37334,
|
| 37482 |
+
"핸": 37335,
|
| 37483 |
+
"꾼": 37336,
|
| 37484 |
+
"엣": 37337,
|
| 37485 |
+
"冊": 37338,
|
| 37486 |
+
"껍": 37339,
|
| 37487 |
+
"벵": 37340,
|
| 37488 |
+
"ᠭ": 37341,
|
| 37489 |
+
"횡": 37342,
|
| 37490 |
+
"콧": 37343,
|
| 37491 |
+
"멍": 37344,
|
| 37492 |
+
"굳": 37345,
|
| 37493 |
+
"輩": 37346,
|
| 37494 |
+
"討": 37347,
|
| 37495 |
+
"긍": 37348,
|
| 37496 |
+
"隻": 37349,
|
| 37497 |
+
"膚": 37350,
|
| 37498 |
+
"콕": 37351,
|
| 37499 |
+
"킥": 37352,
|
| 37500 |
+
"헐": 37353,
|
| 37501 |
+
"띤": 37354,
|
| 37502 |
+
"彫": 37355,
|
| 37503 |
+
"곽": 37356,
|
| 37504 |
+
"젖": 37357,
|
| 37505 |
+
"룽": 37358,
|
| 37506 |
+
"듭": 37359,
|
| 37507 |
+
"옵": 37360,
|
| 37508 |
+
"揚": 37361,
|
| 37509 |
+
"앉": 37362,
|
| 37510 |
+
"싶": 37363,
|
| 37511 |
+
"놈": 37364,
|
| 37512 |
+
"貧": 37365,
|
| 37513 |
+
"꿀": 37366,
|
| 37514 |
+
"늑": 37367,
|
| 37515 |
+
"콰": 37368,
|
| 37516 |
+
"塁": 37369,
|
| 37517 |
+
"帥": 37370,
|
| 37518 |
+
"眞": 37371,
|
| 37519 |
+
"딱": 37372,
|
| 37520 |
+
"옌": 37373,
|
| 37521 |
+
"償": 37374,
|
| 37522 |
+
"啓": 37375,
|
| 37523 |
+
"뀐": 37376,
|
| 37524 |
+
"밸": 37377,
|
| 37525 |
+
"춰": 37378,
|
| 37526 |
+
"굽": 37379,
|
| 37527 |
+
"亀": 37380,
|
| 37528 |
+
"貞": 37381,
|
| 37529 |
+
"澤": 37382,
|
| 37530 |
+
"菓": 37383,
|
| 37531 |
+
"샹": 37384,
|
| 37532 |
+
"寛": 37385,
|
| 37533 |
+
"穀": 37386,
|
| 37534 |
+
"덧": 37387,
|
| 37535 |
+
"훗": 37388,
|
| 37536 |
+
"냄": 37389,
|
| 37537 |
+
"캡": 37390,
|
| 37538 |
+
"隷": 37391,
|
| 37539 |
+
"똑": 37392,
|
| 37540 |
+
"쌀": 37393,
|
| 37541 |
+
"粋": 37394,
|
| 37542 |
+
"獄": 37395,
|
| 37543 |
+
"킴": 37396,
|
| 37544 |
+
"맘": 37397,
|
| 37545 |
+
"飼": 37398,
|
| 37546 |
+
"舗": 37399,
|
| 37547 |
+
"債": 37400,
|
| 37548 |
+
"닫": 37401,
|
| 37549 |
+
"鈴": 37402,
|
| 37550 |
+
"埼": 37403,
|
| 37551 |
+
"팟": 37404,
|
| 37552 |
+
"썬": 37405,
|
| 37553 |
+
"剰": 37406,
|
| 37554 |
+
"畿": 37407,
|
| 37555 |
+
"궐": 37408,
|
| 37556 |
+
"홉": 37409,
|
| 37557 |
+
"옆": 37410,
|
| 37558 |
+
"鑑": 37411,
|
| 37559 |
+
"髪": 37412,
|
| 37560 |
+
"닮": 37413,
|
| 37561 |
+
"흙": 37414,
|
| 37562 |
+
"끓": 37415,
|
| 37563 |
+
"賢": 37416,
|
| 37564 |
+
"엉": 37417,
|
| 37565 |
+
"鶴": 37418,
|
| 37566 |
+
"ᡠ": 37419,
|
| 37567 |
+
"撲": 37420,
|
| 37568 |
+
"낌": 37421,
|
| 37569 |
+
"늦": 37422,
|
| 37570 |
+
"옴": 37423,
|
| 37571 |
+
"덟": 37424,
|
| 37572 |
+
"얄": 37425,
|
| 37573 |
+
"訂": 37426,
|
| 37574 |
+
"綴": 37427,
|
| 37575 |
+
"挿": 37428,
|
| 37576 |
+
"髄": 37429,
|
| 37577 |
+
"묻": 37430,
|
| 37578 |
+
"렷": 37431,
|
| 37579 |
+
"듐": 37432,
|
| 37580 |
+
"듈": 37433,
|
| 37581 |
+
"鉛": 37434,
|
| 37582 |
+
"勧": 37435,
|
| 37583 |
+
"溝": 37436,
|
| 37584 |
+
"샬": 37437,
|
| 37585 |
+
"閲": 37438,
|
| 37586 |
+
"鐵": 37439,
|
| 37587 |
+
"녔": 37440,
|
| 37588 |
+
"傑": 37441,
|
| 37589 |
+
"酔": 37442,
|
| 37590 |
+
"腫": 37443,
|
| 37591 |
+
"팡": 37444,
|
| 37592 |
+
"튬": 37445,
|
| 37593 |
+
"얇": 37446,
|
| 37594 |
+
"퐁": 37447,
|
| 37595 |
+
"윗": 37448,
|
| 37596 |
+
"륭": 37449,
|
| 37597 |
+
"擬": 37450,
|
| 37598 |
+
"툼": 37451,
|
| 37599 |
+
"ㆍ": 37452,
|
| 37600 |
+
"턱": 37453,
|
| 37601 |
+
"罰": 37454,
|
| 37602 |
+
"犠": 37455,
|
| 37603 |
+
"黨": 37456,
|
| 37604 |
+
"ᠡ": 37457,
|
| 37605 |
+
"왈": 37458,
|
| 37606 |
+
"왓": 37459,
|
| 37607 |
+
"誠": 37460,
|
| 37608 |
+
"巣": 37461,
|
| 37609 |
+
"솜": 37462,
|
| 37610 |
+
"쿤": 37463,
|
| 37611 |
+
"퓌": 37464,
|
| 37612 |
+
"쏘": 37465,
|
| 37613 |
+
"ᠰ": 37466,
|
| 37614 |
+
"駐": 37467,
|
| 37615 |
+
"ᠢ": 37468,
|
| 37616 |
+
"쫓": 37469,
|
| 37617 |
+
"돕": 37470,
|
| 37618 |
+
"ᠯ": 37471,
|
| 37619 |
+
"綬": 37472,
|
| 37620 |
+
"ᠤ": 37473,
|
| 37621 |
+
"줌": 37474,
|
| 37622 |
+
"쉐": 37475,
|
| 37623 |
+
"壌": 37476,
|
| 37624 |
+
"賊": 37477,
|
| 37625 |
+
"諡": 37478,
|
| 37626 |
+
"偽": 37479,
|
| 37627 |
+
"팜": 37480,
|
| 37628 |
+
"맵": 37481,
|
| 37629 |
+
"瘍": 37482,
|
| 37630 |
+
"潤": 37483,
|
| 37631 |
+
"斎": 37484,
|
| 37632 |
+
"呉": 37485,
|
| 37633 |
+
"飽": 37486,
|
| 37634 |
+
"漁": 37487,
|
| 37635 |
+
"겁": 37488,
|
| 37636 |
+
"貯": 37489,
|
| 37637 |
+
"蓋": 37490,
|
| 37638 |
+
"핍": 37491,
|
| 37639 |
+
"짱": 37492,
|
| 37640 |
+
"藍": 37493,
|
| 37641 |
+
"倣": 37494,
|
| 37642 |
+
"墜": 37495,
|
| 37643 |
+
"틈": 37496,
|
| 37644 |
+
"詰": 37497
|
| 37645 |
},
|
| 37646 |
"merges": [
|
| 37647 |
"▁ t",
|
tokenizer.model
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5307c4650ccc9d9b5cdd1245a7f88a234eabe3d8e342cc5b8b577a5e30e927e9
|
| 3 |
+
size 565699
|