Upload mod_tokenizers/flexitok--mod-tokenizers-ltr_3digit_super_mapping.json with huggingface_hub

mod_tokenizers/flexitok--mod-tokenizers-ltr_3digit_super_mapping.json

@@ -0,0 +1 @@
+{"0": 100003, "1": 100006, "2": 100001, "3": 100002, "4": 100005, "5": 100004, "6": 0, "7": 1, "8": 22, "9": 11133, "10": 22244, "11": 33355, "12": 44466, "13": 55577, "14": 66688, "15": 77799, "16": 88908, "17": 23, "18": 2245, "19": 3356, "20": 4467, "21": 5578, "22": 6689, "23": 7800, "24": 8911, "25": 10022, "26": 11134, "27": 14467, "28": 15578, "29": 16689, "30": 17800, "31": 18911, "32": 20022, "33": 21133, "34": 22245, "35": 26689, "36": 27800, "37": 28911, "38": 30022, "39": 31133, "40": 32244, "41": 33356, "42": 38911, "43": 40022, "44": 41133, "45": 42244, "46": 43355, "47": 44467, "48": 51133, "49": 52244, "50": 53355, "51": 54466, "52": 55578, "53": 63355, "54": 64466, "55": 65577, "56": 66689, "57": 75577, "58": 76688, "59": 77800, "60": 87797, "61": 88909, "62": 1134, "63": 13356, "64": 25578, "65": 37800, "66": 50022, "67": 62244, "68": 74466, "69": 86686, "70": 98890, "71": 12245, "72": 36689, "73": 61133, "74": 85575, "75": 96668, "76": 97779, "77": 23356, "78": 24467, "79": 34467, "80": 35578, "81": 72244, "82": 73355, "83": 83353, "84": 84464, "85": 94446, "86": 95557, "87": 1135, "88": 1357, "89": 1468, "90": 1579, "91": 1690, "92": 1801, "93": 1912, "94": 2023, "95": 2134, "96": 12246, "97": 12468, "98": 12579, "99": 12690, "100": 12801, "101": 12912, "102": 13023, "103": 13134, "104": 13245, "105": 13357, "106": 13690, "107": 13801, "108": 13912, "109": 14023, "110": 14134, "111": 14245, "112": 14356, "113": 12357, "114": 23357, "115": 23579, "116": 23690, "117": 23801, "118": 23912, "119": 24023, "120": 24134, "121": 24245, "122": 24356, "123": 24468, "124": 24801, "125": 24912, "126": 25023, "127": 25134, "128": 25245, "129": 25356, "130": 25467, "131": 25579, "132": 26023, "133": 26134, "134": 26245, "135": 26356, "136": 26467, "137": 26578, "138": 23468, "139": 24690, "140": 24579, "141": 34468, "142": 34690, "143": 34801, "144": 34912, "145": 35023, "146": 35134, "147": 35245, "148": 35356, "149": 35467, "150": 35579, "151": 35912, "152": 36023, "153": 36134, "154": 36245, "155": 36356, "156": 36467, "157": 36578, "158": 36690, "159": 37134, "160": 37245, "161": 37356, "162": 37467, "163": 37578, "164": 37689, "165": 37801, "166": 38356, "167": 38467, "168": 38578, "169": 38689, "170": 38800, "171": 34579, "172": 35801, "173": 37023, "174": 35690, "175": 45578, "176": 46689, "177": 47800, "178": 48911, "179": 45579, "180": 45801, "181": 45912, "182": 46023, "183": 46134, "184": 46245, "185": 46356, "186": 46467, "187": 46578, "188": 46690, "189": 47023, "190": 47134, "191": 47245, "192": 47356, "193": 47467, "194": 47578, "195": 47689, "196": 47801, "197": 48245, "198": 48356, "199": 48467, "200": 48578, "201": 48689, "202": 48800, "203": 48912, "204": 49467, "205": 49578, "206": 49689, "207": 49800, "208": 49911, "209": 50023, "210": 50689, "211": 50800, "212": 50911, "213": 51022, "214": 45690, "215": 46912, "216": 48134, "217": 49356, "218": 46801, "219": 49245, "220": 47912, "221": 48023, "222": 49023, "223": 49134, "224": 56689, "225": 57800, "226": 58911, "227": 60022, "228": 56690, "229": 56912, "230": 57023, "231": 57134, "232": 57245, "233": 57356, "234": 57467, "235": 57578, "236": 57689, "237": 57801, "238": 58134, "239": 58245, "240": 58356, "241": 58467, "242": 58578, "243": 58689, "244": 58800, "245": 58912, "246": 59356, "247": 59467, "248": 59578, "249": 59689, "250": 59800, "251": 59911, "252": 60023, "253": 60578, "254": 60689, "255": 60800, "256": 60911, "257": 61022, "258": 61134, "259": 61800, "260": 61911, "261": 62022, "262": 62133, "263": 62245, "264": 63022, "265": 63133, "266": 63244, "267": 56801, "268": 58023, "269": 59245, "270": 60467, "271": 61689, "272": 57912, "273": 60356, "274": 59023, "275": 59134, "276": 60134, "277": 60245, "278": 67800, "279": 68911, "280": 70022, "281": 71133, "282": 67801, "283": 68023, "284": 68134, "285": 68245, "286": 68356, "287": 68467, "288": 68578, "289": 68689, "290": 68800, "291": 68912, "292": 69245, "293": 69356, "294": 69467, "295": 69578, "296": 69689, "297": 69800, "298": 69911, "299": 70023, "300": 70467, "301": 70578, "302": 70689, "303": 70800, "304": 70911, "305": 71022, "306": 71134, "307": 71689, "308": 71800, "309": 71911, "310": 72022, "311": 72133, "312": 72245, "313": 72911, "314": 73022, "315": 73133, "316": 73244, "317": 73356, "318": 74133, "319": 74244, "320": 74355, "321": 74467, "322": 75355, "323": 75466, "324": 67912, "325": 69134, "326": 70356, "327": 71578, "328": 72800, "329": 74022, "330": 69023, "331": 71467, "332": 73911, "333": 70134, "334": 70245, "335": 71245, "336": 71356, "337": 78911, "338": 80022, "339": 81133, "340": 82242, "341": 78912, "342": 79134, "343": 79245, "344": 79356, "345": 79467, "346": 79578, "347": 79689, "348": 79800, "349": 79911, "350": 80023, "351": 80356, "352": 80467, "353": 80578, "354": 80689, "355": 80800, "356": 80911, "357": 81022, "358": 81134, "359": 81576, "360": 81687, "361": 81798, "362": 81909, "363": 82020, "364": 82131, "365": 82243, "366": 82798, "367": 82909, "368": 83020, "369": 83131, "370": 83242, "371": 83354, "372": 84020, "373": 84131, "374": 84242, "375": 84353, "376": 84465, "377": 85242, "378": 85353, "379": 85464, "380": 85576, "381": 86464, "382": 86575, "383": 86687, "384": 87686, "385": 79023, "386": 80245, "387": 81465, "388": 82687, "389": 83909, "390": 85131, "391": 86353, "392": 80134, "393": 82576, "394": 85020, "395": 81245, "396": 81354, "397": 82354, "398": 82465, "399": 90020, "400": 91131, "401": 92233, "402": 93335, "403": 90021, "404": 90243, "405": 90354, "406": 90465, "407": 90576, "408": 90687, "409": 90798, "410": 90909, "411": 91020, "412": 91132, "413": 91456, "414": 91567, "415": 91678, "416": 91789, "417": 91900, "418": 92011, "419": 92122, "420": 92234, "421": 92669, "422": 92780, "423": 92891, "424": 93002, "425": 93113, "426": 93224, "427": 93336, "428": 93891, "429": 94002, "430": 94113, "431": 94224, "432": 94335, "433": 94447, "434": 95113, "435": 95224, "436": 95335, "437": 95446, "438": 95558, "439": 96335, "440": 96446, "441": 96557, "442": 96669, "443": 97557, "444": 97668, "445": 97780, "446": 98779, "447": 98891, "448": 90132, "449": 91345, "450": 92558, "451": 93780, "452": 95002, "453": 96224, "454": 97446, "455": 98668, "456": 91243, "457": 93669, "458": 96113, "459": 98557, "460": 92345, "461": 92447, "462": 93447, "463": 93558, "464": 24, "465": 135, "466": 246, "467": 357, "468": 468, "469": 579, "470": 690, "471": 801, "472": 912, "473": 1023, "474": 2246, "475": 2357, "476": 2468, "477": 2579, "478": 2690, "479": 2801, "480": 2912, "481": 3023, "482": 3134, "483": 3245, "484": 3357, "485": 3468, "486": 3579, "487": 3690, "488": 3801, "489": 3912, "490": 4023, "491": 4134, "492": 4245, "493": 4356, "494": 4468, "495": 4579, "496": 4690, "497": 4801, "498": 4912, "499": 5023, "500": 5134, "501": 5245, "502": 5356, "503": 5467, "504": 5579, "505": 5690, "506": 5801, "507": 5912, "508": 6023, "509": 6134, "510": 6245, "511": 6356, "512": 6467, "513": 6578, "514": 6690, "515": 6801, "516": 6912, "517": 7023, "518": 7134, "519": 7245, "520": 7356, "521": 7467, "522": 7578, "523": 7689, "524": 7801, "525": 7912, "526": 8023, "527": 8134, "528": 8245, "529": 8356, "530": 8467, "531": 8578, "532": 8689, "533": 8800, "534": 8912, "535": 9023, "536": 9134, "537": 9245, "538": 9356, "539": 9467, "540": 9578, "541": 9689, "542": 9800, "543": 9911, "544": 10023, "545": 10134, "546": 10245, "547": 10356, "548": 10467, "549": 10578, "550": 10689, "551": 10800, "552": 10911, "553": 11022, "554": 11135, "555": 11246, "556": 11357, "557": 11468, "558": 11579, "559": 11690, "560": 11801, "561": 11912, "562": 12023, "563": 12134, "564": 14468, "565": 14579, "566": 14690, "567": 14801, "568": 14912, "569": 15023, "570": 15134, "571": 15245, "572": 15356, "573": 15467, "574": 15579, "575": 15690, "576": 15801, "577": 15912, "578": 16023, "579": 16134, "580": 16245, "581": 16356, "582": 16467, "583": 16578, "584": 16690, "585": 16801, "586": 16912, "587": 17023, "588": 17134, "589": 17245, "590": 17356, "591": 17467, "592": 17578, "593": 17689, "594": 17801, "595": 17912, "596": 18023, "597": 18134, "598": 18245, "599": 18356, "600": 18467, "601": 18578, "602": 18689, "603": 18800, "604": 18912, "605": 19023, "606": 19134, "607": 19245, "608": 19356, "609": 19467, "610": 19578, "611": 19689, "612": 19800, "613": 19911, "614": 20023, "615": 20134, "616": 20245, "617": 20356, "618": 20467, "619": 20578, "620": 20689, "621": 20800, "622": 20911, "623": 21022, "624": 21134, "625": 21245, "626": 21356, "627": 21467, "628": 21578, "629": 21689, "630": 21800, "631": 21911, "632": 22022, "633": 22133, "634": 22246, "635": 22357, "636": 22468, "637": 22579, "638": 22690, "639": 22801, "640": 22912, "641": 23023, "642": 23134, "643": 23245, "644": 26690, "645": 26801, "646": 26912, "647": 27023, "648": 27134, "649": 27245, "650": 27356, "651": 27467, "652": 27578, "653": 27689, "654": 27801, "655": 27912, "656": 28023, "657": 28134, "658": 28245, "659": 28356, "660": 28467, "661": 28578, "662": 28689, "663": 28800, "664": 28912, "665": 29023, "666": 29134, "667": 29245, "668": 29356, "669": 29467, "670": 29578, "671": 29689, "672": 29800, "673": 29911, "674": 30023, "675": 30134, "676": 30245, "677": 30356, "678": 30467, "679": 30578, "680": 30689, "681": 30800, "682": 30911, "683": 31022, "684": 31134, "685": 31245, "686": 31356, "687": 31467, "688": 31578, "689": 31689, "690": 31800, "691": 31911, "692": 32022, "693": 32133, "694": 32245, "695": 32356, "696": 32467, "697": 32578, "698": 32689, "699": 32800, "700": 32911, "701": 33022, "702": 33133, "703": 33244, "704": 33357, "705": 33468, "706": 33579, "707": 33690, "708": 33801, "709": 33912, "710": 34023, "711": 34134, "712": 34245, "713": 34356, "714": 38912, "715": 39023, "716": 39134, "717": 39245, "718": 39356, "719": 39467, "720": 39578, "721": 39689, "722": 39800, "723": 39911, "724": 40023, "725": 40134, "726": 40245, "727": 40356, "728": 40467, "729": 40578, "730": 40689, "731": 40800, "732": 40911, "733": 41022, "734": 41134, "735": 41245, "736": 41356, "737": 41467, "738": 41578, "739": 41689, "740": 41800, "741": 41911, "742": 42022, "743": 42133, "744": 42245, "745": 42356, "746": 42467, "747": 42578, "748": 42689, "749": 42800, "750": 42911, "751": 43022, "752": 43133, "753": 43244, "754": 43356, "755": 43467, "756": 43578, "757": 43689, "758": 43800, "759": 43911, "760": 44022, "761": 44133, "762": 44244, "763": 44355, "764": 44468, "765": 44579, "766": 44690, "767": 44801, "768": 44912, "769": 45023, "770": 45134, "771": 45245, "772": 45356, "773": 45467, "774": 51134, "775": 51245, "776": 51356, "777": 51467, "778": 51578, "779": 51689, "780": 51800, "781": 51911, "782": 52022, "783": 52133, "784": 52245, "785": 52356, "786": 52467, "787": 52578, "788": 52689, "789": 52800, "790": 52911, "791": 53022, "792": 53133, "793": 53244, "794": 53356, "795": 53467, "796": 53578, "797": 53689, "798": 53800, "799": 53911, "800": 54022, "801": 54133, "802": 54244, "803": 54355, "804": 54467, "805": 54578, "806": 54689, "807": 54800, "808": 54911, "809": 55022, "810": 55133, "811": 55244, "812": 55355, "813": 55466, "814": 55579, "815": 55690, "816": 55801, "817": 55912, "818": 56023, "819": 56134, "820": 56245, "821": 56356, "822": 56467, "823": 56578, "824": 63356, "825": 63467, "826": 63578, "827": 63689, "828": 63800, "829": 63911, "830": 64022, "831": 64133, "832": 64244, "833": 64355, "834": 64467, "835": 64578, "836": 64689, "837": 64800, "838": 64911, "839": 65022, "840": 65133, "841": 65244, "842": 65355, "843": 65466, "844": 65578, "845": 65689, "846": 65800, "847": 65911, "848": 66022, "849": 66133, "850": 66244, "851": 66355, "852": 66466, "853": 66577, "854": 66690, "855": 66801, "856": 66912, "857": 67023, "858": 67134, "859": 67245, "860": 67356, "861": 67467, "862": 67578, "863": 67689, "864": 75578, "865": 75689, "866": 75800, "867": 75911, "868": 76022, "869": 76133, "870": 76244, "871": 76355, "872": 76466, "873": 76577, "874": 76689, "875": 76800, "876": 76911, "877": 77022, "878": 77133, "879": 77244, "880": 77355, "881": 77466, "882": 77577, "883": 77688, "884": 77801, "885": 77912, "886": 78023, "887": 78134, "888": 78245, "889": 78356, "890": 78467, "891": 78578, "892": 78689, "893": 78800, "894": 87798, "895": 87909, "896": 88020, "897": 88131, "898": 88242, "899": 88353, "900": 88464, "901": 88575, "902": 88686, "903": 88797, "904": 88910, "905": 89021, "906": 89132, "907": 89243, "908": 89354, "909": 89465, "910": 89576, "911": 89687, "912": 89798, "913": 89909, "914": 1246, "915": 13468, "916": 13579, "917": 25690, "918": 25801, "919": 25912, "920": 37912, "921": 38023, "922": 38134, "923": 38245, "924": 50134, "925": 50245, "926": 50356, "927": 50467, "928": 50578, "929": 62356, "930": 62467, "931": 62578, "932": 62689, "933": 62800, "934": 62911, "935": 74578, "936": 74689, "937": 74800, "938": 74911, "939": 75022, "940": 75133, "941": 75244, "942": 86798, "943": 86909, "944": 87020, "945": 87131, "946": 87242, "947": 87353, "948": 87464, "949": 87575, "950": 99002, "951": 99113, "952": 99224, "953": 99335, "954": 99446, "955": 99557, "956": 99668, "957": 99779, "958": 99890, "959": 36801, "960": 36912, "961": 61245, "962": 61356, "963": 61467, "964": 61578, "965": 85687, "966": 85798, "967": 85909, "968": 86020, "969": 86131, "970": 86242, "971": 96780, "972": 96891, "973": 97002, "974": 97113, "975": 97224, "976": 97335, "977": 97891, "978": 98002, "979": 98113, "980": 98224, "981": 98335, "982": 98446, "983": 72356, "984": 72467, "985": 72578, "986": 72689, "987": 73467, "988": 73578, "989": 73689, "990": 73800, "991": 83465, "992": 83576, "993": 83687, "994": 83798, "995": 84576, "996": 84687, "997": 84798, "998": 84909, "999": 94558, "1000": 94669, "1001": 94780, "1002": 94891, "1003": 95669, "1004": 95780, "1005": 95891, "1006": 96002}