| { |
| "version": "1.0", |
| "truncation": null, |
| "padding": null, |
| "added_tokens": [ |
| { |
| "id": 0, |
| "content": "<pad>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 1, |
| "content": "<unk>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 2, |
| "content": "<s>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 3, |
| "content": "</s>", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 504, |
| "content": "__ace__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 505, |
| "content": "__ace_Latn__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 506, |
| "content": "__acm__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 507, |
| "content": "__acq__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 508, |
| "content": "__aeb__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 509, |
| "content": "__afr__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 510, |
| "content": "__ajp__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 511, |
| "content": "__aka__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 512, |
| "content": "__amh__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 513, |
| "content": "__apc__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 514, |
| "content": "__arb__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 515, |
| "content": "__ars__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 516, |
| "content": "__ary__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 517, |
| "content": "__arz__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 518, |
| "content": "__asm__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 519, |
| "content": "__ast__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 520, |
| "content": "__awa__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 521, |
| "content": "__ayr__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 522, |
| "content": "__azb__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 523, |
| "content": "__azj__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 524, |
| "content": "__bak__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 525, |
| "content": "__bam__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 526, |
| "content": "__ban__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 527, |
| "content": "__bel__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 528, |
| "content": "__bem__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 529, |
| "content": "__ben__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 530, |
| "content": "__bho__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 531, |
| "content": "__bjn__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 532, |
| "content": "__bjn_Latn__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 533, |
| "content": "__bod__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 534, |
| "content": "__bos__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 535, |
| "content": "__bug__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 536, |
| "content": "__bul__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 537, |
| "content": "__cat__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 538, |
| "content": "__ceb__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 539, |
| "content": "__ces__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 540, |
| "content": "__cjk__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 541, |
| "content": "__ckb__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 542, |
| "content": "__crh__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 543, |
| "content": "__cym__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 544, |
| "content": "__dan__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 545, |
| "content": "__deu__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 546, |
| "content": "__dik__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 547, |
| "content": "__dyu__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 548, |
| "content": "__dzo__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 549, |
| "content": "__ell__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 550, |
| "content": "__eng__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 551, |
| "content": "__epo__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 552, |
| "content": "__est__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 553, |
| "content": "__eus__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 554, |
| "content": "__ewe__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 555, |
| "content": "__fao__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 556, |
| "content": "__pes__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 557, |
| "content": "__fij__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 558, |
| "content": "__fin__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 559, |
| "content": "__fon__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 560, |
| "content": "__fra__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 561, |
| "content": "__fur__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 562, |
| "content": "__fuv__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 563, |
| "content": "__gla__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 564, |
| "content": "__gle__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 565, |
| "content": "__glg__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 566, |
| "content": "__grn__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 567, |
| "content": "__guj__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 568, |
| "content": "__hat__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 569, |
| "content": "__hau__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 570, |
| "content": "__heb__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 571, |
| "content": "__hin__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 572, |
| "content": "__hne__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 573, |
| "content": "__hrv__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 574, |
| "content": "__hun__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 575, |
| "content": "__hye__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 576, |
| "content": "__ibo__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 577, |
| "content": "__ilo__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 578, |
| "content": "__ind__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 579, |
| "content": "__isl__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 580, |
| "content": "__ita__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 581, |
| "content": "__jav__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 582, |
| "content": "__jpn__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 583, |
| "content": "__kab__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 584, |
| "content": "__kac__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 585, |
| "content": "__kam__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 586, |
| "content": "__kan__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 587, |
| "content": "__kas__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 588, |
| "content": "__kas_Deva__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 589, |
| "content": "__kat__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 590, |
| "content": "__knc__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 591, |
| "content": "__knc_Latn__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 592, |
| "content": "__kaz__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 593, |
| "content": "__kbp__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 594, |
| "content": "__kea__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 595, |
| "content": "__khm__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 596, |
| "content": "__kik__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 597, |
| "content": "__kin__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 598, |
| "content": "__kir__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 599, |
| "content": "__kmb__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 600, |
| "content": "__kon__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 601, |
| "content": "__kor__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 602, |
| "content": "__kmr__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 603, |
| "content": "__lao__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 604, |
| "content": "__lvs__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 605, |
| "content": "__lij__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 606, |
| "content": "__lim__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 607, |
| "content": "__lin__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 608, |
| "content": "__lit__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 609, |
| "content": "__lmo__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 610, |
| "content": "__ltg__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 611, |
| "content": "__ltz__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 612, |
| "content": "__lua__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 613, |
| "content": "__lug__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 614, |
| "content": "__luo__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 615, |
| "content": "__lus__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 616, |
| "content": "__mag__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 617, |
| "content": "__mai__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 618, |
| "content": "__mal__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 619, |
| "content": "__mar__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 620, |
| "content": "__min__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 621, |
| "content": "__mkd__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 622, |
| "content": "__plt__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 623, |
| "content": "__mlt__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 624, |
| "content": "__mni__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 625, |
| "content": "__khk__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 626, |
| "content": "__mos__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 627, |
| "content": "__mri__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 628, |
| "content": "__zsm__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 629, |
| "content": "__mya__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 630, |
| "content": "__nld__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 631, |
| "content": "__nno__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 632, |
| "content": "__nob__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 633, |
| "content": "__npi__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 634, |
| "content": "__nso__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 635, |
| "content": "__nus__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 636, |
| "content": "__nya__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 637, |
| "content": "__oci__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 638, |
| "content": "__gaz__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 639, |
| "content": "__ory__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 640, |
| "content": "__pag__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 641, |
| "content": "__pan__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 642, |
| "content": "__pap__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 643, |
| "content": "__pol__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 644, |
| "content": "__por__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 645, |
| "content": "__prs__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 646, |
| "content": "__pbt__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 647, |
| "content": "__quy__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 648, |
| "content": "__ron__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 649, |
| "content": "__run__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 650, |
| "content": "__rus__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 651, |
| "content": "__sag__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 652, |
| "content": "__san__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 653, |
| "content": "__sat__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 654, |
| "content": "__scn__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 655, |
| "content": "__shn__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 656, |
| "content": "__sin__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 657, |
| "content": "__slk__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 658, |
| "content": "__slv__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 659, |
| "content": "__smo__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 660, |
| "content": "__sna__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 661, |
| "content": "__snd__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 662, |
| "content": "__som__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 663, |
| "content": "__sot__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 664, |
| "content": "__spa__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 665, |
| "content": "__als__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 666, |
| "content": "__srd__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 667, |
| "content": "__srp__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 668, |
| "content": "__ssw__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 669, |
| "content": "__sun__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 670, |
| "content": "__swe__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 671, |
| "content": "__swh__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 672, |
| "content": "__szl__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 673, |
| "content": "__tam__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 674, |
| "content": "__tat__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 675, |
| "content": "__tel__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 676, |
| "content": "__tgk__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 677, |
| "content": "__tgl__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 678, |
| "content": "__tha__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 679, |
| "content": "__tir__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 680, |
| "content": "__taq__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 681, |
| "content": "__taq_Tfng__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 682, |
| "content": "__tpi__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 683, |
| "content": "__tsn__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 684, |
| "content": "__tso__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 685, |
| "content": "__tuk__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 686, |
| "content": "__tum__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 687, |
| "content": "__tur__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 688, |
| "content": "__twi__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 689, |
| "content": "__tzm__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 690, |
| "content": "__uig__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 691, |
| "content": "__ukr__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 692, |
| "content": "__umb__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 693, |
| "content": "__urd__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 694, |
| "content": "__uzn__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 695, |
| "content": "__vec__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 696, |
| "content": "__vie__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 697, |
| "content": "__war__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 698, |
| "content": "__wol__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 699, |
| "content": "__xho__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 700, |
| "content": "__ydd__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 701, |
| "content": "__yor__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 702, |
| "content": "__yue__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 703, |
| "content": "__cmn__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 704, |
| "content": "__cmn_Hant__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 705, |
| "content": "__zul__", |
| "single_word": false, |
| "lstrip": true, |
| "rstrip": true, |
| "normalized": false, |
| "special": true |
| } |
| ], |
| "normalizer": { |
| "type": "Sequence", |
| "normalizers": [ |
| { |
| "type": "Replace", |
| "pattern": { |
| "Regex": "[\\n\\r\\t]" |
| }, |
| "content": " " |
| }, |
| { |
| "type": "NFKC" |
| }, |
| { |
| "type": "Strip", |
| "strip_left": false, |
| "strip_right": true |
| }, |
| { |
| "type": "Replace", |
| "pattern": { |
| "Regex": " +▁" |
| }, |
| "content": "▁" |
| }, |
| { |
| "type": "Replace", |
| "pattern": { |
| "Regex": "^▁+$" |
| }, |
| "content": "" |
| }, |
| { |
| "type": "Replace", |
| "pattern": { |
| "Regex": " {2,}" |
| }, |
| "content": "▁" |
| } |
| ] |
| }, |
| "pre_tokenizer": { |
| "type": "Metaspace", |
| "replacement": "▁", |
| "prepend_scheme": "first", |
| "split": true |
| }, |
| "post_processor": { |
| "type": "TemplateProcessing", |
| "single": [ |
| { |
| "SpecialToken": { |
| "id": "</s>", |
| "type_id": 0 |
| } |
| }, |
| { |
| "SpecialToken": { |
| "id": "__fra__", |
| "type_id": 0 |
| } |
| }, |
| { |
| "Sequence": { |
| "id": "A", |
| "type_id": 0 |
| } |
| }, |
| { |
| "SpecialToken": { |
| "id": "</s>", |
| "type_id": 0 |
| } |
| } |
| ], |
| "pair": [ |
| { |
| "SpecialToken": { |
| "id": "</s>", |
| "type_id": 0 |
| } |
| }, |
| { |
| "SpecialToken": { |
| "id": "__fra__", |
| "type_id": 0 |
| } |
| }, |
| { |
| "Sequence": { |
| "id": "A", |
| "type_id": 0 |
| } |
| }, |
| { |
| "Sequence": { |
| "id": "B", |
| "type_id": 0 |
| } |
| }, |
| { |
| "SpecialToken": { |
| "id": "</s>", |
| "type_id": 0 |
| } |
| } |
| ], |
| "special_tokens": { |
| "</s>": { |
| "id": "</s>", |
| "ids": [ |
| 3 |
| ], |
| "tokens": [ |
| "</s>" |
| ] |
| }, |
| "__fra__": { |
| "id": "__fra__", |
| "ids": [ |
| 560 |
| ], |
| "tokens": [ |
| "__fra__" |
| ] |
| } |
| } |
| }, |
| "decoder": { |
| "type": "Metaspace", |
| "replacement": "▁", |
| "prepend_scheme": "first", |
| "split": true |
| }, |
| "model": { |
| "type": "BPE", |
| "dropout": null, |
| "unk_token": "<unk>", |
| "continuing_subword_prefix": null, |
| "end_of_word_suffix": null, |
| "fuse_unk": true, |
| "byte_fallback": false, |
| "ignore_merges": false, |
| "vocab": { |
| "<pad>": 0, |
| "<unk>": 1, |
| "<s>": 2, |
| "</s>": 3, |
| "an": 4, |
| "▁n": 5, |
| "▁m": 6, |
| "▁t": 7, |
| "▁k": 8, |
| "▁a": 9, |
| "▁s": 10, |
| "en": 11, |
| "in": 12, |
| "▁d": 13, |
| "er": 14, |
| "la": 15, |
| "▁b": 16, |
| "▁p": 17, |
| "on": 18, |
| "ar": 19, |
| "at": 20, |
| "is": 21, |
| "le": 22, |
| "▁e": 23, |
| "un": 24, |
| "li": 25, |
| "▁o": 26, |
| "▁v": 27, |
| "▁h": 28, |
| "▁c": 29, |
| "▁i": 30, |
| "ak": 31, |
| "as": 32, |
| "es": 33, |
| "am": 34, |
| "▁y": 35, |
| "it": 36, |
| "or": 37, |
| "▁g": 38, |
| "et": 39, |
| "▁w": 40, |
| "al": 41, |
| "▁f": 42, |
| "AT": 43, |
| "IN": 44, |
| "▁<": 45, |
| "ED": 46, |
| "ATA": 47, |
| "MIN": 48, |
| "DATA": 49, |
| "MINED": 50, |
| "▁r": 51, |
| "ir": 52, |
| "ang": 53, |
| "ti": 54, |
| "▁l": 55, |
| "ad": 56, |
| "os": 57, |
| "om": 58, |
| "ur": 59, |
| "el": 60, |
| "▁j": 61, |
| "▁na": 62, |
| "us": 63, |
| "em": 64, |
| "ch": 65, |
| "ik": 66, |
| "um": 67, |
| "ga": 68, |
| "▁\"": 69, |
| "▁A": 70, |
| "ta": 71, |
| "▁M": 72, |
| "il": 73, |
| "▁ا": 74, |
| "im": 75, |
| "ed": 76, |
| "ek": 77, |
| "▁u": 78, |
| "▁de": 79, |
| "wa": 80, |
| "▁.": 81, |
| "▁la": 82, |
| "▁S": 83, |
| "▁N": 84, |
| "id": 85, |
| "te": 86, |
| "ing": 87, |
| "ay": 88, |
| "▁-": 89, |
| "▁ku": 90, |
| "ok": 91, |
| "ra": 92, |
| "ol": 93, |
| "▁1": 94, |
| "▁ب": 95, |
| "▁le": 96, |
| "ia": 97, |
| "▁K": 98, |
| "▁ka": 99, |
| "▁z": 100, |
| "uk": 101, |
| "ha": 102, |
| "ba": 103, |
| "▁(": 104, |
| "ut": 105, |
| "на": 106, |
| "da": 107, |
| "▁in": 108, |
| "▁T": 109, |
| "he": 110, |
| "▁an": 111, |
| "▁ma": 112, |
| "▁م": 113, |
| "▁B": 114, |
| "ya": 115, |
| "ie": 116, |
| "▁I": 117, |
| "ul": 118, |
| "re": 119, |
| "▁с": 120, |
| "▁P": 121, |
| "▁D": 122, |
| "▁क": 123, |
| "ot": 124, |
| "ana": 125, |
| "▁E": 126, |
| "▁sa": 127, |
| "▁б": 128, |
| "od": 129, |
| "▁se": 130, |
| "▁pr": 131, |
| "та": 132, |
| "▁ta": 133, |
| "▁ک": 134, |
| "▁q": 135, |
| "ان": 136, |
| "▁к": 137, |
| "ent": 138, |
| "ov": 139, |
| "▁2": 140, |
| "▁و": 141, |
| "▁п": 142, |
| "hi": 143, |
| "да": 144, |
| "di": 145, |
| "▁,": 146, |
| "ag": 147, |
| "st": 148, |
| "ci": 149, |
| "▁di": 150, |
| "ap": 151, |
| "▁C": 152, |
| "▁en": 153, |
| "ра": 154, |
| "▁L": 155, |
| "▁J": 156, |
| "de": 157, |
| "▁ni": 158, |
| "▁un": 159, |
| "▁ya": 160, |
| "▁ba": 161, |
| "ka": 162, |
| "ng": 163, |
| "▁د": 164, |
| "ен": 165, |
| "▁H": 166, |
| "▁ne": 167, |
| "ла": 168, |
| "na": 169, |
| "▁da": 170, |
| "si": 171, |
| "▁te": 172, |
| "ho": 173, |
| "ri": 174, |
| "▁ng": 175, |
| "ong": 176, |
| "▁li": 177, |
| "▁ت": 178, |
| "▁м": 179, |
| "eng": 180, |
| "bi": 181, |
| "og": 182, |
| "we": 183, |
| "ma": 184, |
| "aka": 185, |
| "tu": 186, |
| "ye": 187, |
| "ara": 188, |
| "..": 189, |
| "and": 190, |
| "▁в": 191, |
| "{{": 192, |
| "▁O": 193, |
| "▁स": 194, |
| "ار": 195, |
| "▁qu": 196, |
| "ja": 197, |
| "▁Y": 198, |
| "ig": 199, |
| "▁wa": 200, |
| "se": 201, |
| "▁to": 202, |
| "ku": 203, |
| "za": 204, |
| "то": 205, |
| "▁ti": 206, |
| "zi": 207, |
| "to": 208, |
| "sa": 209, |
| "eg": 210, |
| "▁{{": 211, |
| "▁G": 212, |
| "▁и": 213, |
| "\".": 214, |
| "те": 215, |
| "▁mu": 216, |
| "▁о": 217, |
| "▁а": 218, |
| "▁si": 219, |
| "op": 220, |
| "▁ह": 221, |
| "be": 222, |
| "ung": 223, |
| "не": 224, |
| "ou": 225, |
| "va": 226, |
| "▁ه": 227, |
| "▁ch": 228, |
| "▁д": 229, |
| "▁at": 230, |
| "ina": 231, |
| "ва": 232, |
| "▁th": 233, |
| "ни": 234, |
| "▁al": 235, |
| "▁س": 236, |
| "▁ل": 237, |
| "▁'": 238, |
| "▁me": 239, |
| "ni": 240, |
| "▁ال": 241, |
| "▁ن": 242, |
| "▁з": 243, |
| "ati": 244, |
| "▁ha": 245, |
| "▁W": 246, |
| "ud": 247, |
| "▁is": 248, |
| "ke": 249, |
| "▁R": 250, |
| "▁म": 251, |
| "yo": 252, |
| "ua": 253, |
| "▁no": 254, |
| "▁на": 255, |
| "je": 256, |
| "ти": 257, |
| "ata": 258, |
| "▁am": 259, |
| "▁V": 260, |
| "▁U": 261, |
| "▁ko": 262, |
| "▁प": 263, |
| "▁ke": 264, |
| "ро": 265, |
| "ре": 266, |
| "ob": 267, |
| "ter": 268, |
| "но": 269, |
| "ani": 270, |
| "ce": 271, |
| "▁re": 272, |
| "pa": 273, |
| "ne": 274, |
| "ere": 275, |
| "bu": 276, |
| "▁F": 277, |
| "pe": 278, |
| "ve": 279, |
| "▁be": 280, |
| "▁on": 281, |
| "ال": 282, |
| "▁ar": 283, |
| "...": 284, |
| "▁ak": 285, |
| "ка": 286, |
| "▁va": 287, |
| "ли": 288, |
| "ari": 289, |
| "ist": 290, |
| "lar": 291, |
| "ama": 292, |
| "ри": 293, |
| "ini": 294, |
| "▁nga": 295, |
| "▁tr": 296, |
| "▁ئ": 297, |
| "bo": 298, |
| "्य": 299, |
| "hu": 300, |
| "▁ر": 301, |
| "ant": 302, |
| "▁je": 303, |
| "ona": 304, |
| "ле": 305, |
| "era": 306, |
| "ki": 307, |
| "▁pa": 308, |
| "ས་": 309, |
| "▁20": 310, |
| "▁per": 311, |
| "ro": 312, |
| "ны": 313, |
| "ca": 314, |
| "▁3": 315, |
| "▁mo": 316, |
| "▁mi": 317, |
| "▁ny": 318, |
| "ika": 319, |
| "ji": 320, |
| "ada": 321, |
| "га": 322, |
| "end": 323, |
| "▁so": 324, |
| "▁अ": 325, |
| "ст": 326, |
| "▁su": 327, |
| "▁پ": 328, |
| "▁א": 329, |
| "▁bi": 330, |
| "ма": 331, |
| "ون": 332, |
| "isa": 333, |
| "▁tu": 334, |
| "▁es": 335, |
| "▁el": 336, |
| "▁ki": 337, |
| "me": 338, |
| "▁đ": 339, |
| "▁que": 340, |
| "wo": 341, |
| "▁er": 342, |
| "го": 343, |
| "▁st": 344, |
| "ita": 345, |
| "lo": 346, |
| "wi": 347, |
| "ug": 348, |
| "▁the": 349, |
| "▁ج": 350, |
| "▁for": 351, |
| "▁त": 352, |
| "ko": 353, |
| "est": 354, |
| "▁у": 355, |
| "vi": 356, |
| "▁ब": 357, |
| "ای": 358, |
| "▁do": 359, |
| "ene": 360, |
| "▁ش": 361, |
| "▁ga": 362, |
| "▁con": 363, |
| "▁ж": 364, |
| "▁ف": 365, |
| "▁по": 366, |
| "▁ye": 367, |
| "▁ي": 368, |
| "ny": 369, |
| "ane": 370, |
| "ི་": 371, |
| "ze": 372, |
| "▁fa": 373, |
| "▁ق": 374, |
| "्र": 375, |
| "ло": 376, |
| "ira": 377, |
| "vo": 378, |
| "ست": 379, |
| "ко": 380, |
| "▁ज": 381, |
| "▁vi": 382, |
| "ند": 383, |
| "up": 384, |
| "▁?": 385, |
| "▁आ": 386, |
| "ང་": 387, |
| "▁as": 388, |
| "uma": 389, |
| "▁po": 390, |
| "ер": 391, |
| "ine": 392, |
| "▁за": 393, |
| "▁أ": 394, |
| "ور": 395, |
| "▁خ": 396, |
| "▁x": 397, |
| "▁т": 398, |
| "▁kan": 399, |
| "ena": 400, |
| "nd": 401, |
| "▁да": 402, |
| "▁bo": 403, |
| "ic": 404, |
| "▁go": 405, |
| "ver": 406, |
| "eh": 407, |
| "▁آ": 408, |
| "▁ra": 409, |
| "▁et": 410, |
| "▁og": 411, |
| "▁é": 412, |
| "▁par": 413, |
| "▁ho": 414, |
| "asi": 415, |
| "asa": 416, |
| "▁ה": 417, |
| "▁ع": 418, |
| "th": 419, |
| "ार": 420, |
| "ib": 421, |
| "▁न": 422, |
| "▁ja": 423, |
| "qu": 424, |
| "▁그": 425, |
| "isi": 426, |
| "so": 427, |
| "any": 428, |
| "ova": 429, |
| "eni": 430, |
| "▁ol": 431, |
| "ele": 432, |
| "во": 433, |
| "ind": 434, |
| "▁hi": 435, |
| "▁4": 436, |
| "▁lo": 437, |
| "▁ба": 438, |
| "▁men": 439, |
| "▁ve": 440, |
| "▁19": 441, |
| "oni": 442, |
| "gi": 443, |
| "ام": 444, |
| "lam": 445, |
| "pr": 446, |
| "ры": 447, |
| "▁za": 448, |
| "po": 449, |
| "▁com": 450, |
| "▁ك": 451, |
| "▁of": 452, |
| "▁та": 453, |
| "▁द": 454, |
| "▁ag": 455, |
| "▁im": 456, |
| "ու": 457, |
| "iri": 458, |
| "olo": 459, |
| "00": 460, |
| "▁ক": 461, |
| "▁du": 462, |
| "are": 463, |
| "no": 464, |
| "ین": 465, |
| "ို": 466, |
| "▁не": 467, |
| "ts": 468, |
| "han": 469, |
| "iti": 470, |
| "ات": 471, |
| "eri": 472, |
| "лы": 473, |
| "lu": 474, |
| "▁व": 475, |
| "ab": 476, |
| "▁ang": 477, |
| "ore": 478, |
| "ge": 479, |
| "tr": 480, |
| "▁5": 481, |
| "pi": 482, |
| "▁के": 483, |
| "▁dan": 484, |
| "்க": 485, |
| "yi": 486, |
| "ди": 487, |
| "ju": 488, |
| "де": 489, |
| "▁nd": 490, |
| "▁τ": 491, |
| "▁man": 492, |
| "ula": 493, |
| "▁ш": 494, |
| "kan": 495, |
| "iz": 496, |
| "اد": 497, |
| "fa": 498, |
| "una": 499, |
| "adi": 500, |
| "▁х": 501, |
| "ate": 502, |
| "니다": 503 |
| }, |
| "merges": [ |
| [ |
| "MIN", |
| "ED" |
| ], |
| [ |
| "▁n", |
| "ga" |
| ], |
| [ |
| "▁p", |
| "er" |
| ], |
| [ |
| "▁t", |
| "he" |
| ], |
| [ |
| "▁f", |
| "or" |
| ], |
| [ |
| "▁c", |
| "on" |
| ], |
| [ |
| "▁k", |
| "an" |
| ], |
| [ |
| "▁p", |
| "ar" |
| ], |
| [ |
| "▁m", |
| "en" |
| ], |
| [ |
| "▁c", |
| "om" |
| ], |
| [ |
| "▁a", |
| "ng" |
| ], |
| [ |
| "▁d", |
| "an" |
| ], |
| [ |
| "▁m", |
| "an" |
| ] |
| ] |
| } |
| } |
