Upload tokenizer

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,540 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|sound_0000|>": 151668,
+  "<|sound_0001|>": 151669,
+  "<|sound_0002|>": 151670,
+  "<|sound_0003|>": 151671,
+  "<|sound_0004|>": 151672,
+  "<|sound_0005|>": 151673,
+  "<|sound_0006|>": 151674,
+  "<|sound_0007|>": 151675,
+  "<|sound_0008|>": 151676,
+  "<|sound_0009|>": 151677,
+  "<|sound_0010|>": 151678,
+  "<|sound_0011|>": 151679,
+  "<|sound_0012|>": 151680,
+  "<|sound_0013|>": 151681,
+  "<|sound_0014|>": 151682,
+  "<|sound_0015|>": 151683,
+  "<|sound_0016|>": 151684,
+  "<|sound_0017|>": 151685,
+  "<|sound_0018|>": 151686,
+  "<|sound_0019|>": 151687,
+  "<|sound_0020|>": 151688,
+  "<|sound_0021|>": 151689,
+  "<|sound_0022|>": 151690,
+  "<|sound_0023|>": 151691,
+  "<|sound_0024|>": 151692,
+  "<|sound_0025|>": 151693,
+  "<|sound_0026|>": 151694,
+  "<|sound_0027|>": 151695,
+  "<|sound_0028|>": 151696,
+  "<|sound_0029|>": 151697,
+  "<|sound_0030|>": 151698,
+  "<|sound_0031|>": 151699,
+  "<|sound_0032|>": 151700,
+  "<|sound_0033|>": 151701,
+  "<|sound_0034|>": 151702,
+  "<|sound_0035|>": 151703,
+  "<|sound_0036|>": 151704,
+  "<|sound_0037|>": 151705,
+  "<|sound_0038|>": 151706,
+  "<|sound_0039|>": 151707,
+  "<|sound_0040|>": 151708,
+  "<|sound_0041|>": 151709,
+  "<|sound_0042|>": 151710,
+  "<|sound_0043|>": 151711,
+  "<|sound_0044|>": 151712,
+  "<|sound_0045|>": 151713,
+  "<|sound_0046|>": 151714,
+  "<|sound_0047|>": 151715,
+  "<|sound_0048|>": 151716,
+  "<|sound_0049|>": 151717,
+  "<|sound_0050|>": 151718,
+  "<|sound_0051|>": 151719,
+  "<|sound_0052|>": 151720,
+  "<|sound_0053|>": 151721,
+  "<|sound_0054|>": 151722,
+  "<|sound_0055|>": 151723,
+  "<|sound_0056|>": 151724,
+  "<|sound_0057|>": 151725,
+  "<|sound_0058|>": 151726,
+  "<|sound_0059|>": 151727,
+  "<|sound_0060|>": 151728,
+  "<|sound_0061|>": 151729,
+  "<|sound_0062|>": 151730,
+  "<|sound_0063|>": 151731,
+  "<|sound_0064|>": 151732,
+  "<|sound_0065|>": 151733,
+  "<|sound_0066|>": 151734,
+  "<|sound_0067|>": 151735,
+  "<|sound_0068|>": 151736,
+  "<|sound_0069|>": 151737,
+  "<|sound_0070|>": 151738,
+  "<|sound_0071|>": 151739,
+  "<|sound_0072|>": 151740,
+  "<|sound_0073|>": 151741,
+  "<|sound_0074|>": 151742,
+  "<|sound_0075|>": 151743,
+  "<|sound_0076|>": 151744,
+  "<|sound_0077|>": 151745,
+  "<|sound_0078|>": 151746,
+  "<|sound_0079|>": 151747,
+  "<|sound_0080|>": 151748,
+  "<|sound_0081|>": 151749,
+  "<|sound_0082|>": 151750,
+  "<|sound_0083|>": 151751,
+  "<|sound_0084|>": 151752,
+  "<|sound_0085|>": 151753,
+  "<|sound_0086|>": 151754,
+  "<|sound_0087|>": 151755,
+  "<|sound_0088|>": 151756,
+  "<|sound_0089|>": 151757,
+  "<|sound_0090|>": 151758,
+  "<|sound_0091|>": 151759,
+  "<|sound_0092|>": 151760,
+  "<|sound_0093|>": 151761,
+  "<|sound_0094|>": 151762,
+  "<|sound_0095|>": 151763,
+  "<|sound_0096|>": 151764,
+  "<|sound_0097|>": 151765,
+  "<|sound_0098|>": 151766,
+  "<|sound_0099|>": 151767,
+  "<|sound_0100|>": 151768,
+  "<|sound_0101|>": 151769,
+  "<|sound_0102|>": 151770,
+  "<|sound_0103|>": 151771,
+  "<|sound_0104|>": 151772,
+  "<|sound_0105|>": 151773,
+  "<|sound_0106|>": 151774,
+  "<|sound_0107|>": 151775,
+  "<|sound_0108|>": 151776,
+  "<|sound_0109|>": 151777,
+  "<|sound_0110|>": 151778,
+  "<|sound_0111|>": 151779,
+  "<|sound_0112|>": 151780,
+  "<|sound_0113|>": 151781,
+  "<|sound_0114|>": 151782,
+  "<|sound_0115|>": 151783,
+  "<|sound_0116|>": 151784,
+  "<|sound_0117|>": 151785,
+  "<|sound_0118|>": 151786,
+  "<|sound_0119|>": 151787,
+  "<|sound_0120|>": 151788,
+  "<|sound_0121|>": 151789,
+  "<|sound_0122|>": 151790,
+  "<|sound_0123|>": 151791,
+  "<|sound_0124|>": 151792,
+  "<|sound_0125|>": 151793,
+  "<|sound_0126|>": 151794,
+  "<|sound_0127|>": 151795,
+  "<|sound_0128|>": 151796,
+  "<|sound_0129|>": 151797,
+  "<|sound_0130|>": 151798,
+  "<|sound_0131|>": 151799,
+  "<|sound_0132|>": 151800,
+  "<|sound_0133|>": 151801,
+  "<|sound_0134|>": 151802,
+  "<|sound_0135|>": 151803,
+  "<|sound_0136|>": 151804,
+  "<|sound_0137|>": 151805,
+  "<|sound_0138|>": 151806,
+  "<|sound_0139|>": 151807,
+  "<|sound_0140|>": 151808,
+  "<|sound_0141|>": 151809,
+  "<|sound_0142|>": 151810,
+  "<|sound_0143|>": 151811,
+  "<|sound_0144|>": 151812,
+  "<|sound_0145|>": 151813,
+  "<|sound_0146|>": 151814,
+  "<|sound_0147|>": 151815,
+  "<|sound_0148|>": 151816,
+  "<|sound_0149|>": 151817,
+  "<|sound_0150|>": 151818,
+  "<|sound_0151|>": 151819,
+  "<|sound_0152|>": 151820,
+  "<|sound_0153|>": 151821,
+  "<|sound_0154|>": 151822,
+  "<|sound_0155|>": 151823,
+  "<|sound_0156|>": 151824,
+  "<|sound_0157|>": 151825,
+  "<|sound_0158|>": 151826,
+  "<|sound_0159|>": 151827,
+  "<|sound_0160|>": 151828,
+  "<|sound_0161|>": 151829,
+  "<|sound_0162|>": 151830,
+  "<|sound_0163|>": 151831,
+  "<|sound_0164|>": 151832,
+  "<|sound_0165|>": 151833,
+  "<|sound_0166|>": 151834,
+  "<|sound_0167|>": 151835,
+  "<|sound_0168|>": 151836,
+  "<|sound_0169|>": 151837,
+  "<|sound_0170|>": 151838,
+  "<|sound_0171|>": 151839,
+  "<|sound_0172|>": 151840,
+  "<|sound_0173|>": 151841,
+  "<|sound_0174|>": 151842,
+  "<|sound_0175|>": 151843,
+  "<|sound_0176|>": 151844,
+  "<|sound_0177|>": 151845,
+  "<|sound_0178|>": 151846,
+  "<|sound_0179|>": 151847,
+  "<|sound_0180|>": 151848,
+  "<|sound_0181|>": 151849,
+  "<|sound_0182|>": 151850,
+  "<|sound_0183|>": 151851,
+  "<|sound_0184|>": 151852,
+  "<|sound_0185|>": 151853,
+  "<|sound_0186|>": 151854,
+  "<|sound_0187|>": 151855,
+  "<|sound_0188|>": 151856,
+  "<|sound_0189|>": 151857,
+  "<|sound_0190|>": 151858,
+  "<|sound_0191|>": 151859,
+  "<|sound_0192|>": 151860,
+  "<|sound_0193|>": 151861,
+  "<|sound_0194|>": 151862,
+  "<|sound_0195|>": 151863,
+  "<|sound_0196|>": 151864,
+  "<|sound_0197|>": 151865,
+  "<|sound_0198|>": 151866,
+  "<|sound_0199|>": 151867,
+  "<|sound_0200|>": 151868,
+  "<|sound_0201|>": 151869,
+  "<|sound_0202|>": 151870,
+  "<|sound_0203|>": 151871,
+  "<|sound_0204|>": 151872,
+  "<|sound_0205|>": 151873,
+  "<|sound_0206|>": 151874,
+  "<|sound_0207|>": 151875,
+  "<|sound_0208|>": 151876,
+  "<|sound_0209|>": 151877,
+  "<|sound_0210|>": 151878,
+  "<|sound_0211|>": 151879,
+  "<|sound_0212|>": 151880,
+  "<|sound_0213|>": 151881,
+  "<|sound_0214|>": 151882,
+  "<|sound_0215|>": 151883,
+  "<|sound_0216|>": 151884,
+  "<|sound_0217|>": 151885,
+  "<|sound_0218|>": 151886,
+  "<|sound_0219|>": 151887,
+  "<|sound_0220|>": 151888,
+  "<|sound_0221|>": 151889,
+  "<|sound_0222|>": 151890,
+  "<|sound_0223|>": 151891,
+  "<|sound_0224|>": 151892,
+  "<|sound_0225|>": 151893,
+  "<|sound_0226|>": 151894,
+  "<|sound_0227|>": 151895,
+  "<|sound_0228|>": 151896,
+  "<|sound_0229|>": 151897,
+  "<|sound_0230|>": 151898,
+  "<|sound_0231|>": 151899,
+  "<|sound_0232|>": 151900,
+  "<|sound_0233|>": 151901,
+  "<|sound_0234|>": 151902,
+  "<|sound_0235|>": 151903,
+  "<|sound_0236|>": 151904,
+  "<|sound_0237|>": 151905,
+  "<|sound_0238|>": 151906,
+  "<|sound_0239|>": 151907,
+  "<|sound_0240|>": 151908,
+  "<|sound_0241|>": 151909,
+  "<|sound_0242|>": 151910,
+  "<|sound_0243|>": 151911,
+  "<|sound_0244|>": 151912,
+  "<|sound_0245|>": 151913,
+  "<|sound_0246|>": 151914,
+  "<|sound_0247|>": 151915,
+  "<|sound_0248|>": 151916,
+  "<|sound_0249|>": 151917,
+  "<|sound_0250|>": 151918,
+  "<|sound_0251|>": 151919,
+  "<|sound_0252|>": 151920,
+  "<|sound_0253|>": 151921,
+  "<|sound_0254|>": 151922,
+  "<|sound_0255|>": 151923,
+  "<|sound_0256|>": 151924,
+  "<|sound_0257|>": 151925,
+  "<|sound_0258|>": 151926,
+  "<|sound_0259|>": 151927,
+  "<|sound_0260|>": 151928,
+  "<|sound_0261|>": 151929,
+  "<|sound_0262|>": 151930,
+  "<|sound_0263|>": 151931,
+  "<|sound_0264|>": 151932,
+  "<|sound_0265|>": 151933,
+  "<|sound_0266|>": 151934,
+  "<|sound_0267|>": 151935,
+  "<|sound_0268|>": 151936,
+  "<|sound_0269|>": 151937,
+  "<|sound_0270|>": 151938,
+  "<|sound_0271|>": 151939,
+  "<|sound_0272|>": 151940,
+  "<|sound_0273|>": 151941,
+  "<|sound_0274|>": 151942,
+  "<|sound_0275|>": 151943,
+  "<|sound_0276|>": 151944,
+  "<|sound_0277|>": 151945,
+  "<|sound_0278|>": 151946,
+  "<|sound_0279|>": 151947,
+  "<|sound_0280|>": 151948,
+  "<|sound_0281|>": 151949,
+  "<|sound_0282|>": 151950,
+  "<|sound_0283|>": 151951,
+  "<|sound_0284|>": 151952,
+  "<|sound_0285|>": 151953,
+  "<|sound_0286|>": 151954,
+  "<|sound_0287|>": 151955,
+  "<|sound_0288|>": 151956,
+  "<|sound_0289|>": 151957,
+  "<|sound_0290|>": 151958,
+  "<|sound_0291|>": 151959,
+  "<|sound_0292|>": 151960,
+  "<|sound_0293|>": 151961,
+  "<|sound_0294|>": 151962,
+  "<|sound_0295|>": 151963,
+  "<|sound_0296|>": 151964,
+  "<|sound_0297|>": 151965,
+  "<|sound_0298|>": 151966,
+  "<|sound_0299|>": 151967,
+  "<|sound_0300|>": 151968,
+  "<|sound_0301|>": 151969,
+  "<|sound_0302|>": 151970,
+  "<|sound_0303|>": 151971,
+  "<|sound_0304|>": 151972,
+  "<|sound_0305|>": 151973,
+  "<|sound_0306|>": 151974,
+  "<|sound_0307|>": 151975,
+  "<|sound_0308|>": 151976,
+  "<|sound_0309|>": 151977,
+  "<|sound_0310|>": 151978,
+  "<|sound_0311|>": 151979,
+  "<|sound_0312|>": 151980,
+  "<|sound_0313|>": 151981,
+  "<|sound_0314|>": 151982,
+  "<|sound_0315|>": 151983,
+  "<|sound_0316|>": 151984,
+  "<|sound_0317|>": 151985,
+  "<|sound_0318|>": 151986,
+  "<|sound_0319|>": 151987,
+  "<|sound_0320|>": 151988,
+  "<|sound_0321|>": 151989,
+  "<|sound_0322|>": 151990,
+  "<|sound_0323|>": 151991,
+  "<|sound_0324|>": 151992,
+  "<|sound_0325|>": 151993,
+  "<|sound_0326|>": 151994,
+  "<|sound_0327|>": 151995,
+  "<|sound_0328|>": 151996,
+  "<|sound_0329|>": 151997,
+  "<|sound_0330|>": 151998,
+  "<|sound_0331|>": 151999,
+  "<|sound_0332|>": 152000,
+  "<|sound_0333|>": 152001,
+  "<|sound_0334|>": 152002,
+  "<|sound_0335|>": 152003,
+  "<|sound_0336|>": 152004,
+  "<|sound_0337|>": 152005,
+  "<|sound_0338|>": 152006,
+  "<|sound_0339|>": 152007,
+  "<|sound_0340|>": 152008,
+  "<|sound_0341|>": 152009,
+  "<|sound_0342|>": 152010,
+  "<|sound_0343|>": 152011,
+  "<|sound_0344|>": 152012,
+  "<|sound_0345|>": 152013,
+  "<|sound_0346|>": 152014,
+  "<|sound_0347|>": 152015,
+  "<|sound_0348|>": 152016,
+  "<|sound_0349|>": 152017,
+  "<|sound_0350|>": 152018,
+  "<|sound_0351|>": 152019,
+  "<|sound_0352|>": 152020,
+  "<|sound_0353|>": 152021,
+  "<|sound_0354|>": 152022,
+  "<|sound_0355|>": 152023,
+  "<|sound_0356|>": 152024,
+  "<|sound_0357|>": 152025,
+  "<|sound_0358|>": 152026,
+  "<|sound_0359|>": 152027,
+  "<|sound_0360|>": 152028,
+  "<|sound_0361|>": 152029,
+  "<|sound_0362|>": 152030,
+  "<|sound_0363|>": 152031,
+  "<|sound_0364|>": 152032,
+  "<|sound_0365|>": 152033,
+  "<|sound_0366|>": 152034,
+  "<|sound_0367|>": 152035,
+  "<|sound_0368|>": 152036,
+  "<|sound_0369|>": 152037,
+  "<|sound_0370|>": 152038,
+  "<|sound_0371|>": 152039,
+  "<|sound_0372|>": 152040,
+  "<|sound_0373|>": 152041,
+  "<|sound_0374|>": 152042,
+  "<|sound_0375|>": 152043,
+  "<|sound_0376|>": 152044,
+  "<|sound_0377|>": 152045,
+  "<|sound_0378|>": 152046,
+  "<|sound_0379|>": 152047,
+  "<|sound_0380|>": 152048,
+  "<|sound_0381|>": 152049,
+  "<|sound_0382|>": 152050,
+  "<|sound_0383|>": 152051,
+  "<|sound_0384|>": 152052,
+  "<|sound_0385|>": 152053,
+  "<|sound_0386|>": 152054,
+  "<|sound_0387|>": 152055,
+  "<|sound_0388|>": 152056,
+  "<|sound_0389|>": 152057,
+  "<|sound_0390|>": 152058,
+  "<|sound_0391|>": 152059,
+  "<|sound_0392|>": 152060,
+  "<|sound_0393|>": 152061,
+  "<|sound_0394|>": 152062,
+  "<|sound_0395|>": 152063,
+  "<|sound_0396|>": 152064,
+  "<|sound_0397|>": 152065,
+  "<|sound_0398|>": 152066,
+  "<|sound_0399|>": 152067,
+  "<|sound_0400|>": 152068,
+  "<|sound_0401|>": 152069,
+  "<|sound_0402|>": 152070,
+  "<|sound_0403|>": 152071,
+  "<|sound_0404|>": 152072,
+  "<|sound_0405|>": 152073,
+  "<|sound_0406|>": 152074,
+  "<|sound_0407|>": 152075,
+  "<|sound_0408|>": 152076,
+  "<|sound_0409|>": 152077,
+  "<|sound_0410|>": 152078,
+  "<|sound_0411|>": 152079,
+  "<|sound_0412|>": 152080,
+  "<|sound_0413|>": 152081,
+  "<|sound_0414|>": 152082,
+  "<|sound_0415|>": 152083,
+  "<|sound_0416|>": 152084,
+  "<|sound_0417|>": 152085,
+  "<|sound_0418|>": 152086,
+  "<|sound_0419|>": 152087,
+  "<|sound_0420|>": 152088,
+  "<|sound_0421|>": 152089,
+  "<|sound_0422|>": 152090,
+  "<|sound_0423|>": 152091,
+  "<|sound_0424|>": 152092,
+  "<|sound_0425|>": 152093,
+  "<|sound_0426|>": 152094,
+  "<|sound_0427|>": 152095,
+  "<|sound_0428|>": 152096,
+  "<|sound_0429|>": 152097,
+  "<|sound_0430|>": 152098,
+  "<|sound_0431|>": 152099,
+  "<|sound_0432|>": 152100,
+  "<|sound_0433|>": 152101,
+  "<|sound_0434|>": 152102,
+  "<|sound_0435|>": 152103,
+  "<|sound_0436|>": 152104,
+  "<|sound_0437|>": 152105,
+  "<|sound_0438|>": 152106,
+  "<|sound_0439|>": 152107,
+  "<|sound_0440|>": 152108,
+  "<|sound_0441|>": 152109,
+  "<|sound_0442|>": 152110,
+  "<|sound_0443|>": 152111,
+  "<|sound_0444|>": 152112,
+  "<|sound_0445|>": 152113,
+  "<|sound_0446|>": 152114,
+  "<|sound_0447|>": 152115,
+  "<|sound_0448|>": 152116,
+  "<|sound_0449|>": 152117,
+  "<|sound_0450|>": 152118,
+  "<|sound_0451|>": 152119,
+  "<|sound_0452|>": 152120,
+  "<|sound_0453|>": 152121,
+  "<|sound_0454|>": 152122,
+  "<|sound_0455|>": 152123,
+  "<|sound_0456|>": 152124,
+  "<|sound_0457|>": 152125,
+  "<|sound_0458|>": 152126,
+  "<|sound_0459|>": 152127,
+  "<|sound_0460|>": 152128,
+  "<|sound_0461|>": 152129,
+  "<|sound_0462|>": 152130,
+  "<|sound_0463|>": 152131,
+  "<|sound_0464|>": 152132,
+  "<|sound_0465|>": 152133,
+  "<|sound_0466|>": 152134,
+  "<|sound_0467|>": 152135,
+  "<|sound_0468|>": 152136,
+  "<|sound_0469|>": 152137,
+  "<|sound_0470|>": 152138,
+  "<|sound_0471|>": 152139,
+  "<|sound_0472|>": 152140,
+  "<|sound_0473|>": 152141,
+  "<|sound_0474|>": 152142,
+  "<|sound_0475|>": 152143,
+  "<|sound_0476|>": 152144,
+  "<|sound_0477|>": 152145,
+  "<|sound_0478|>": 152146,
+  "<|sound_0479|>": 152147,
+  "<|sound_0480|>": 152148,
+  "<|sound_0481|>": 152149,
+  "<|sound_0482|>": 152150,
+  "<|sound_0483|>": 152151,
+  "<|sound_0484|>": 152152,
+  "<|sound_0485|>": 152153,
+  "<|sound_0486|>": 152154,
+  "<|sound_0487|>": 152155,
+  "<|sound_0488|>": 152156,
+  "<|sound_0489|>": 152157,
+  "<|sound_0490|>": 152158,
+  "<|sound_0491|>": 152159,
+  "<|sound_0492|>": 152160,
+  "<|sound_0493|>": 152161,
+  "<|sound_0494|>": 152162,
+  "<|sound_0495|>": 152163,
+  "<|sound_0496|>": 152164,
+  "<|sound_0497|>": 152165,
+  "<|sound_0498|>": 152166,
+  "<|sound_0499|>": 152167,
+  "<|sound_0500|>": 152168,
+  "<|sound_0501|>": 152169,
+  "<|sound_0502|>": 152170,
+  "<|sound_0503|>": 152171,
+  "<|sound_0504|>": 152172,
+  "<|sound_0505|>": 152173,
+  "<|sound_0506|>": 152174,
+  "<|sound_0507|>": 152175,
+  "<|sound_0508|>": 152176,
+  "<|sound_0509|>": 152177,
+  "<|sound_0510|>": 152178,
+  "<|sound_0511|>": 152179,
+  "<|sound_0512|>": 152180,
+  "<|sound_end|>": 151666,
+  "<|sound_start|>": 151665,
+  "<|text_to_sementic|>": 151667,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

special_tokens_map.json

@@ -0,0 +1,39 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|sound_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|sound_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|text_to_sementic|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f649ce2d043c0d297c9234ff917770b4d816a97cf66bead316b30efbb0bbcef
+size 11520458