Upload mod_tokenizers_zero_padded/flexitok--mod-tokenizers-zero-padded-rtl_3digit_super_mapping.json with huggingface_hub

mod_tokenizers_zero_padded/flexitok--mod-tokenizers-zero-padded-rtl_3digit_super_mapping.json

@@ -0,0 +1 @@
+{"0": 110927, "1": 110930, "2": 110925, "3": 110926, "4": 110929, "5": 110928, "6": 0, "7": 1, "8": 10926, "9": 22037, "10": 33148, "11": 44259, "12": 55370, "13": 66481, "14": 77592, "15": 88703, "16": 99814, "17": 10927, "18": 13149, "19": 14260, "20": 15371, "21": 16482, "22": 17593, "23": 18704, "24": 19815, "25": 20926, "26": 22038, "27": 25371, "28": 26482, "29": 27593, "30": 28704, "31": 29815, "32": 30926, "33": 32037, "34": 33149, "35": 37593, "36": 38704, "37": 39815, "38": 40926, "39": 42037, "40": 43148, "41": 44260, "42": 49815, "43": 50926, "44": 52037, "45": 53148, "46": 54259, "47": 55371, "48": 62037, "49": 63148, "50": 64259, "51": 65370, "52": 66482, "53": 74259, "54": 75370, "55": 76481, "56": 77593, "57": 86481, "58": 87592, "59": 88704, "60": 98703, "61": 99815, "62": 12038, "63": 24260, "64": 36482, "65": 48704, "66": 60926, "67": 73148, "68": 85370, "69": 97592, "70": 109814, "71": 23149, "72": 47593, "73": 72037, "74": 96481, "75": 107592, "76": 108703, "77": 34260, "78": 35371, "79": 45371, "80": 46482, "81": 83148, "82": 84259, "83": 94259, "84": 95370, "85": 105370, "86": 106481, "87": 5453, "88": 6542, "89": 7642, "90": 8741, "91": 9837, "92": 12039, "93": 12261, "94": 12372, "95": 12483, "96": 12594, "97": 12705, "98": 12816, "99": 12927, "100": 13038, "101": 23150, "102": 23372, "103": 23483, "104": 23594, "105": 23705, "106": 23816, "107": 23927, "108": 24038, "109": 24149, "110": 24261, "111": 24594, "112": 24705, "113": 24816, "114": 24927, "115": 25038, "116": 25149, "117": 25260, "118": 23261, "119": 34261, "120": 34483, "121": 34594, "122": 34705, "123": 34816, "124": 34927, "125": 35038, "126": 35149, "127": 35260, "128": 35372, "129": 35705, "130": 35816, "131": 35927, "132": 36038, "133": 36149, "134": 36260, "135": 36371, "136": 36483, "137": 36927, "138": 37038, "139": 37149, "140": 37260, "141": 37371, "142": 37482, "143": 34372, "144": 35594, "145": 35483, "146": 45372, "147": 45594, "148": 45705, "149": 45816, "150": 45927, "151": 46038, "152": 46149, "153": 46260, "154": 46371, "155": 46483, "156": 46816, "157": 46927, "158": 47038, "159": 47149, "160": 47260, "161": 47371, "162": 47482, "163": 47594, "164": 48038, "165": 48149, "166": 48260, "167": 48371, "168": 48482, "169": 48593, "170": 48705, "171": 49260, "172": 49371, "173": 49482, "174": 49593, "175": 49704, "176": 45483, "177": 46705, "178": 47927, "179": 46594, "180": 56483, "181": 56705, "182": 56816, "183": 56927, "184": 57038, "185": 57149, "186": 57260, "187": 57371, "188": 57482, "189": 57594, "190": 57927, "191": 58038, "192": 58149, "193": 58260, "194": 58371, "195": 58482, "196": 58593, "197": 58705, "198": 59149, "199": 59260, "200": 59371, "201": 59482, "202": 59593, "203": 59704, "204": 59816, "205": 60371, "206": 60482, "207": 60593, "208": 60704, "209": 60815, "210": 60927, "211": 61593, "212": 61704, "213": 61815, "214": 61926, "215": 56594, "216": 57816, "217": 59038, "218": 60260, "219": 57705, "220": 60149, "221": 58816, "222": 58927, "223": 59927, "224": 60038, "225": 67594, "226": 67816, "227": 67927, "228": 68038, "229": 68149, "230": 68260, "231": 68371, "232": 68482, "233": 68593, "234": 68705, "235": 69038, "236": 69149, "237": 69260, "238": 69371, "239": 69482, "240": 69593, "241": 69704, "242": 69816, "243": 70260, "244": 70371, "245": 70482, "246": 70593, "247": 70704, "248": 70815, "249": 70927, "250": 71482, "251": 71593, "252": 71704, "253": 71815, "254": 71926, "255": 72038, "256": 72704, "257": 72815, "258": 72926, "259": 73037, "260": 73149, "261": 73926, "262": 74037, "263": 74148, "264": 67705, "265": 68927, "266": 70149, "267": 71371, "268": 72593, "269": 68816, "270": 71260, "271": 69927, "272": 70038, "273": 71038, "274": 71149, "275": 78705, "276": 78927, "277": 79038, "278": 79149, "279": 79260, "280": 79371, "281": 79482, "282": 79593, "283": 79704, "284": 79816, "285": 80149, "286": 80260, "287": 80371, "288": 80482, "289": 80593, "290": 80704, "291": 80815, "292": 80927, "293": 81371, "294": 81482, "295": 81593, "296": 81704, "297": 81815, "298": 81926, "299": 82038, "300": 82593, "301": 82704, "302": 82815, "303": 82926, "304": 83037, "305": 83149, "306": 83815, "307": 83926, "308": 84037, "309": 84148, "310": 84260, "311": 85037, "312": 85148, "313": 85259, "314": 85371, "315": 86259, "316": 86370, "317": 78816, "318": 80038, "319": 81260, "320": 82482, "321": 83704, "322": 84926, "323": 79927, "324": 82371, "325": 84815, "326": 81038, "327": 81149, "328": 82149, "329": 82260, "330": 89816, "331": 90038, "332": 90149, "333": 90260, "334": 90371, "335": 90482, "336": 90593, "337": 90704, "338": 90815, "339": 90927, "340": 91260, "341": 91371, "342": 91482, "343": 91593, "344": 91704, "345": 91815, "346": 91926, "347": 92038, "348": 92482, "349": 92593, "350": 92704, "351": 92815, "352": 92926, "353": 93037, "354": 93149, "355": 93704, "356": 93815, "357": 93926, "358": 94037, "359": 94148, "360": 94260, "361": 94926, "362": 95037, "363": 95148, "364": 95259, "365": 95371, "366": 96148, "367": 96259, "368": 96370, "369": 96482, "370": 97370, "371": 97481, "372": 97593, "373": 98592, "374": 89927, "375": 91149, "376": 92371, "377": 93593, "378": 94815, "379": 96037, "380": 97259, "381": 91038, "382": 93482, "383": 95926, "384": 92149, "385": 92260, "386": 93260, "387": 93371, "388": 100927, "389": 101149, "390": 101260, "391": 101371, "392": 101482, "393": 101593, "394": 101704, "395": 101815, "396": 101926, "397": 102038, "398": 102371, "399": 102482, "400": 102593, "401": 102704, "402": 102815, "403": 102926, "404": 103037, "405": 103149, "406": 103593, "407": 103704, "408": 103815, "409": 103926, "410": 104037, "411": 104148, "412": 104260, "413": 104815, "414": 104926, "415": 105037, "416": 105148, "417": 105259, "418": 105371, "419": 106037, "420": 106148, "421": 106259, "422": 106370, "423": 106482, "424": 107259, "425": 107370, "426": 107481, "427": 107593, "428": 108481, "429": 108592, "430": 108704, "431": 109703, "432": 109815, "433": 101038, "434": 102260, "435": 103482, "436": 104704, "437": 105926, "438": 107148, "439": 108370, "440": 109592, "441": 102149, "442": 104593, "443": 107037, "444": 109481, "445": 103260, "446": 103371, "447": 104371, "448": 104482, "449": 10928, "450": 11039, "451": 11150, "452": 11261, "453": 11372, "454": 11483, "455": 11594, "456": 11705, "457": 11816, "458": 11927, "459": 13150, "460": 13261, "461": 13372, "462": 13483, "463": 13594, "464": 13705, "465": 13816, "466": 13927, "467": 14038, "468": 14149, "469": 14261, "470": 14372, "471": 14483, "472": 14594, "473": 14705, "474": 14816, "475": 14927, "476": 15038, "477": 15149, "478": 15260, "479": 15372, "480": 15483, "481": 15594, "482": 15705, "483": 15816, "484": 15927, "485": 16038, "486": 16149, "487": 16260, "488": 16371, "489": 16483, "490": 16594, "491": 16705, "492": 16816, "493": 16927, "494": 17038, "495": 17149, "496": 17260, "497": 17371, "498": 17482, "499": 17594, "500": 17705, "501": 17816, "502": 17927, "503": 18038, "504": 18149, "505": 18260, "506": 18371, "507": 18482, "508": 18593, "509": 18705, "510": 18816, "511": 18927, "512": 19038, "513": 19149, "514": 19260, "515": 19371, "516": 19482, "517": 19593, "518": 19704, "519": 19816, "520": 19927, "521": 20038, "522": 20149, "523": 20260, "524": 20371, "525": 20482, "526": 20593, "527": 20704, "528": 20815, "529": 20927, "530": 21038, "531": 21149, "532": 21260, "533": 21371, "534": 21482, "535": 21593, "536": 21704, "537": 21815, "538": 21926, "539": 22039, "540": 22150, "541": 22261, "542": 22372, "543": 22483, "544": 22594, "545": 22705, "546": 22816, "547": 22927, "548": 23038, "549": 25372, "550": 25483, "551": 25594, "552": 25705, "553": 25816, "554": 25927, "555": 26038, "556": 26149, "557": 26260, "558": 26371, "559": 26483, "560": 26594, "561": 26705, "562": 26816, "563": 26927, "564": 27038, "565": 27149, "566": 27260, "567": 27371, "568": 27482, "569": 27594, "570": 27705, "571": 27816, "572": 27927, "573": 28038, "574": 28149, "575": 28260, "576": 28371, "577": 28482, "578": 28593, "579": 28705, "580": 28816, "581": 28927, "582": 29038, "583": 29149, "584": 29260, "585": 29371, "586": 29482, "587": 29593, "588": 29704, "589": 29816, "590": 29927, "591": 30038, "592": 30149, "593": 30260, "594": 30371, "595": 30482, "596": 30593, "597": 30704, "598": 30815, "599": 30927, "600": 31038, "601": 31149, "602": 31260, "603": 31371, "604": 31482, "605": 31593, "606": 31704, "607": 31815, "608": 31926, "609": 32038, "610": 32149, "611": 32260, "612": 32371, "613": 32482, "614": 32593, "615": 32704, "616": 32815, "617": 32926, "618": 33037, "619": 33150, "620": 33261, "621": 33372, "622": 33483, "623": 33594, "624": 33705, "625": 33816, "626": 33927, "627": 34038, "628": 34149, "629": 37594, "630": 37705, "631": 37816, "632": 37927, "633": 38038, "634": 38149, "635": 38260, "636": 38371, "637": 38482, "638": 38593, "639": 38705, "640": 38816, "641": 38927, "642": 39038, "643": 39149, "644": 39260, "645": 39371, "646": 39482, "647": 39593, "648": 39704, "649": 39816, "650": 39927, "651": 40038, "652": 40149, "653": 40260, "654": 40371, "655": 40482, "656": 40593, "657": 40704, "658": 40815, "659": 40927, "660": 41038, "661": 41149, "662": 41260, "663": 41371, "664": 41482, "665": 41593, "666": 41704, "667": 41815, "668": 41926, "669": 42038, "670": 42149, "671": 42260, "672": 42371, "673": 42482, "674": 42593, "675": 42704, "676": 42815, "677": 42926, "678": 43037, "679": 43149, "680": 43260, "681": 43371, "682": 43482, "683": 43593, "684": 43704, "685": 43815, "686": 43926, "687": 44037, "688": 44148, "689": 44261, "690": 44372, "691": 44483, "692": 44594, "693": 44705, "694": 44816, "695": 44927, "696": 45038, "697": 45149, "698": 45260, "699": 49816, "700": 49927, "701": 50038, "702": 50149, "703": 50260, "704": 50371, "705": 50482, "706": 50593, "707": 50704, "708": 50815, "709": 50927, "710": 51038, "711": 51149, "712": 51260, "713": 51371, "714": 51482, "715": 51593, "716": 51704, "717": 51815, "718": 51926, "719": 52038, "720": 52149, "721": 52260, "722": 52371, "723": 52482, "724": 52593, "725": 52704, "726": 52815, "727": 52926, "728": 53037, "729": 53149, "730": 53260, "731": 53371, "732": 53482, "733": 53593, "734": 53704, "735": 53815, "736": 53926, "737": 54037, "738": 54148, "739": 54260, "740": 54371, "741": 54482, "742": 54593, "743": 54704, "744": 54815, "745": 54926, "746": 55037, "747": 55148, "748": 55259, "749": 55372, "750": 55483, "751": 55594, "752": 55705, "753": 55816, "754": 55927, "755": 56038, "756": 56149, "757": 56260, "758": 56371, "759": 62038, "760": 62149, "761": 62260, "762": 62371, "763": 62482, "764": 62593, "765": 62704, "766": 62815, "767": 62926, "768": 63037, "769": 63149, "770": 63260, "771": 63371, "772": 63482, "773": 63593, "774": 63704, "775": 63815, "776": 63926, "777": 64037, "778": 64148, "779": 64260, "780": 64371, "781": 64482, "782": 64593, "783": 64704, "784": 64815, "785": 64926, "786": 65037, "787": 65148, "788": 65259, "789": 65371, "790": 65482, "791": 65593, "792": 65704, "793": 65815, "794": 65926, "795": 66037, "796": 66148, "797": 66259, "798": 66370, "799": 66483, "800": 66594, "801": 66705, "802": 66816, "803": 66927, "804": 67038, "805": 67149, "806": 67260, "807": 67371, "808": 67482, "809": 74260, "810": 74371, "811": 74482, "812": 74593, "813": 74704, "814": 74815, "815": 74926, "816": 75037, "817": 75148, "818": 75259, "819": 75371, "820": 75482, "821": 75593, "822": 75704, "823": 75815, "824": 75926, "825": 76037, "826": 76148, "827": 76259, "828": 76370, "829": 76482, "830": 76593, "831": 76704, "832": 76815, "833": 76926, "834": 77037, "835": 77148, "836": 77259, "837": 77370, "838": 77481, "839": 77594, "840": 77705, "841": 77816, "842": 77927, "843": 78038, "844": 78149, "845": 78260, "846": 78371, "847": 78482, "848": 78593, "849": 86482, "850": 86593, "851": 86704, "852": 86815, "853": 86926, "854": 87037, "855": 87148, "856": 87259, "857": 87370, "858": 87481, "859": 87593, "860": 87704, "861": 87815, "862": 87926, "863": 88037, "864": 88148, "865": 88259, "866": 88370, "867": 88481, "868": 88592, "869": 88705, "870": 88816, "871": 88927, "872": 89038, "873": 89149, "874": 89260, "875": 89371, "876": 89482, "877": 89593, "878": 89704, "879": 98704, "880": 98815, "881": 98926, "882": 99037, "883": 99148, "884": 99259, "885": 99370, "886": 99481, "887": 99592, "888": 99703, "889": 99816, "890": 99927, "891": 100038, "892": 100149, "893": 100260, "894": 100371, "895": 100482, "896": 100593, "897": 100704, "898": 100815, "899": 12150, "900": 24372, "901": 24483, "902": 36594, "903": 36705, "904": 36816, "905": 48816, "906": 48927, "907": 49038, "908": 49149, "909": 61038, "910": 61149, "911": 61260, "912": 61371, "913": 61482, "914": 73260, "915": 73371, "916": 73482, "917": 73593, "918": 73704, "919": 73815, "920": 85482, "921": 85593, "922": 85704, "923": 85815, "924": 85926, "925": 86037, "926": 86148, "927": 97704, "928": 97815, "929": 97926, "930": 98037, "931": 98148, "932": 98259, "933": 98370, "934": 98481, "935": 109926, "936": 110037, "937": 110148, "938": 110259, "939": 110370, "940": 110481, "941": 110592, "942": 110703, "943": 110814, "944": 47705, "945": 47816, "946": 72149, "947": 72260, "948": 72371, "949": 72482, "950": 96593, "951": 96704, "952": 96815, "953": 96926, "954": 97037, "955": 97148, "956": 107704, "957": 107815, "958": 107926, "959": 108037, "960": 108148, "961": 108259, "962": 108815, "963": 108926, "964": 109037, "965": 109148, "966": 109259, "967": 109370, "968": 83260, "969": 83371, "970": 83482, "971": 83593, "972": 84371, "973": 84482, "974": 84593, "975": 84704, "976": 94371, "977": 94482, "978": 94593, "979": 94704, "980": 95482, "981": 95593, "982": 95704, "983": 95815, "984": 105482, "985": 105593, "986": 105704, "987": 105815, "988": 106593, "989": 106704, "990": 106815, "991": 106926, "992": 1094, "993": 1306, "994": 1417, "995": 1528, "996": 1639, "997": 1750, "998": 1861, "999": 1972, "1000": 2083, "1001": 2195, "1002": 2499, "1003": 2610, "1004": 2721, "1005": 2832, "1006": 2943, "1007": 3054, "1008": 3165, "1009": 3277, "1010": 3694, "1011": 3805, "1012": 3916, "1013": 4027, "1014": 4138, "1015": 4249, "1016": 4361, "1017": 4898, "1018": 5009, "1019": 5120, "1020": 5231, "1021": 5342, "1022": 5454, "1023": 6098, "1024": 6209, "1025": 6320, "1026": 6431, "1027": 6543, "1028": 7309, "1029": 7420, "1030": 7531, "1031": 7643, "1032": 8519, "1033": 8630, "1034": 8742, "1035": 9726, "1036": 9838, "1037": 1205, "1038": 2408, "1039": 3583, "1040": 4787, "1041": 5992, "1042": 7204, "1043": 8410, "1044": 9615, "1045": 10815, "1046": 2306, "1047": 4676, "1048": 7096, "1049": 9509, "1050": 10605, "1051": 10711, "1052": 3388, "1053": 3490, "1054": 4472, "1055": 4583, "1056": 8196, "1057": 8304, "1058": 9295, "1059": 9403, "1060": 10391, "1061": 10499, "1062": 56482, "1063": 57593, "1064": 58704, "1065": 59815, "1066": 67593, "1067": 68704, "1068": 69815, "1069": 70926, "1070": 78704, "1071": 79815, "1072": 80926, "1073": 82037, "1074": 89815, "1075": 90926, "1076": 92037, "1077": 93148, "1078": 100926, "1079": 102037, "1080": 103148, "1081": 104259, "1082": 5565, "1083": 5676, "1084": 5787, "1085": 5881, "1086": 6654, "1087": 6765, "1088": 6876, "1089": 6987, "1090": 7754, "1091": 7865, "1092": 7976, "1093": 8087, "1094": 8853, "1095": 8964, "1096": 9075, "1097": 9186, "1098": 9949, "1099": 10060, "1100": 10171, "1101": 10282}