GleemanAI / added_tokens.json
Khewa153's picture
Adding tokenizer files
e1f636c verified
{
"<extra_id_0>": 147,
"<extra_id_10>": 137,
"<extra_id_11>": 136,
"<extra_id_12>": 135,
"<extra_id_13>": 134,
"<extra_id_14>": 133,
"<extra_id_15>": 132,
"<extra_id_16>": 131,
"<extra_id_17>": 130,
"<extra_id_18>": 129,
"<extra_id_19>": 128,
"<extra_id_1>": 146,
"<extra_id_20>": 127,
"<extra_id_21>": 126,
"<extra_id_22>": 125,
"<extra_id_23>": 124,
"<extra_id_24>": 123,
"<extra_id_25>": 122,
"<extra_id_26>": 121,
"<extra_id_27>": 120,
"<extra_id_28>": 119,
"<extra_id_29>": 118,
"<extra_id_2>": 145,
"<extra_id_30>": 117,
"<extra_id_31>": 116,
"<extra_id_32>": 115,
"<extra_id_33>": 114,
"<extra_id_34>": 113,
"<extra_id_35>": 112,
"<extra_id_36>": 111,
"<extra_id_37>": 110,
"<extra_id_38>": 109,
"<extra_id_39>": 108,
"<extra_id_3>": 144,
"<extra_id_40>": 107,
"<extra_id_41>": 106,
"<extra_id_42>": 105,
"<extra_id_43>": 104,
"<extra_id_44>": 103,
"<extra_id_45>": 102,
"<extra_id_46>": 101,
"<extra_id_47>": 100,
"<extra_id_48>": 99,
"<extra_id_49>": 98,
"<extra_id_4>": 143,
"<extra_id_50>": 97,
"<extra_id_51>": 96,
"<extra_id_52>": 95,
"<extra_id_53>": 94,
"<extra_id_54>": 93,
"<extra_id_55>": 92,
"<extra_id_56>": 91,
"<extra_id_57>": 90,
"<extra_id_58>": 89,
"<extra_id_59>": 88,
"<extra_id_5>": 142,
"<extra_id_60>": 87,
"<extra_id_61>": 86,
"<extra_id_62>": 85,
"<extra_id_63>": 84,
"<extra_id_64>": 83,
"<extra_id_65>": 82,
"<extra_id_66>": 81,
"<extra_id_67>": 80,
"<extra_id_68>": 79,
"<extra_id_69>": 78,
"<extra_id_6>": 141,
"<extra_id_70>": 77,
"<extra_id_71>": 76,
"<extra_id_72>": 75,
"<extra_id_73>": 74,
"<extra_id_74>": 73,
"<extra_id_75>": 72,
"<extra_id_76>": 71,
"<extra_id_77>": 70,
"<extra_id_78>": 69,
"<extra_id_79>": 68,
"<extra_id_7>": 140,
"<extra_id_80>": 67,
"<extra_id_81>": 66,
"<extra_id_82>": 65,
"<extra_id_83>": 64,
"<extra_id_84>": 63,
"<extra_id_85>": 62,
"<extra_id_86>": 61,
"<extra_id_87>": 60,
"<extra_id_88>": 59,
"<extra_id_89>": 58,
"<extra_id_8>": 139,
"<extra_id_90>": 57,
"<extra_id_91>": 56,
"<extra_id_92>": 55,
"<extra_id_93>": 54,
"<extra_id_94>": 53,
"<extra_id_95>": 52,
"<extra_id_96>": 51,
"<extra_id_97>": 50,
"<extra_id_98>": 49,
"<extra_id_99>": 48,
"<extra_id_9>": 138,
"<pad>": 148
}