Adding tokenizer files
e1f636c
verified
|
|
{ |
|
|
"<extra_id_0>": 147, |
|
|
"<extra_id_10>": 137, |
|
|
"<extra_id_11>": 136, |
|
|
"<extra_id_12>": 135, |
|
|
"<extra_id_13>": 134, |
|
|
"<extra_id_14>": 133, |
|
|
"<extra_id_15>": 132, |
|
|
"<extra_id_16>": 131, |
|
|
"<extra_id_17>": 130, |
|
|
"<extra_id_18>": 129, |
|
|
"<extra_id_19>": 128, |
|
|
"<extra_id_1>": 146, |
|
|
"<extra_id_20>": 127, |
|
|
"<extra_id_21>": 126, |
|
|
"<extra_id_22>": 125, |
|
|
"<extra_id_23>": 124, |
|
|
"<extra_id_24>": 123, |
|
|
"<extra_id_25>": 122, |
|
|
"<extra_id_26>": 121, |
|
|
"<extra_id_27>": 120, |
|
|
"<extra_id_28>": 119, |
|
|
"<extra_id_29>": 118, |
|
|
"<extra_id_2>": 145, |
|
|
"<extra_id_30>": 117, |
|
|
"<extra_id_31>": 116, |
|
|
"<extra_id_32>": 115, |
|
|
"<extra_id_33>": 114, |
|
|
"<extra_id_34>": 113, |
|
|
"<extra_id_35>": 112, |
|
|
"<extra_id_36>": 111, |
|
|
"<extra_id_37>": 110, |
|
|
"<extra_id_38>": 109, |
|
|
"<extra_id_39>": 108, |
|
|
"<extra_id_3>": 144, |
|
|
"<extra_id_40>": 107, |
|
|
"<extra_id_41>": 106, |
|
|
"<extra_id_42>": 105, |
|
|
"<extra_id_43>": 104, |
|
|
"<extra_id_44>": 103, |
|
|
"<extra_id_45>": 102, |
|
|
"<extra_id_46>": 101, |
|
|
"<extra_id_47>": 100, |
|
|
"<extra_id_48>": 99, |
|
|
"<extra_id_49>": 98, |
|
|
"<extra_id_4>": 143, |
|
|
"<extra_id_50>": 97, |
|
|
"<extra_id_51>": 96, |
|
|
"<extra_id_52>": 95, |
|
|
"<extra_id_53>": 94, |
|
|
"<extra_id_54>": 93, |
|
|
"<extra_id_55>": 92, |
|
|
"<extra_id_56>": 91, |
|
|
"<extra_id_57>": 90, |
|
|
"<extra_id_58>": 89, |
|
|
"<extra_id_59>": 88, |
|
|
"<extra_id_5>": 142, |
|
|
"<extra_id_60>": 87, |
|
|
"<extra_id_61>": 86, |
|
|
"<extra_id_62>": 85, |
|
|
"<extra_id_63>": 84, |
|
|
"<extra_id_64>": 83, |
|
|
"<extra_id_65>": 82, |
|
|
"<extra_id_66>": 81, |
|
|
"<extra_id_67>": 80, |
|
|
"<extra_id_68>": 79, |
|
|
"<extra_id_69>": 78, |
|
|
"<extra_id_6>": 141, |
|
|
"<extra_id_70>": 77, |
|
|
"<extra_id_71>": 76, |
|
|
"<extra_id_72>": 75, |
|
|
"<extra_id_73>": 74, |
|
|
"<extra_id_74>": 73, |
|
|
"<extra_id_75>": 72, |
|
|
"<extra_id_76>": 71, |
|
|
"<extra_id_77>": 70, |
|
|
"<extra_id_78>": 69, |
|
|
"<extra_id_79>": 68, |
|
|
"<extra_id_7>": 140, |
|
|
"<extra_id_80>": 67, |
|
|
"<extra_id_81>": 66, |
|
|
"<extra_id_82>": 65, |
|
|
"<extra_id_83>": 64, |
|
|
"<extra_id_84>": 63, |
|
|
"<extra_id_85>": 62, |
|
|
"<extra_id_86>": 61, |
|
|
"<extra_id_87>": 60, |
|
|
"<extra_id_88>": 59, |
|
|
"<extra_id_89>": 58, |
|
|
"<extra_id_8>": 139, |
|
|
"<extra_id_90>": 57, |
|
|
"<extra_id_91>": 56, |
|
|
"<extra_id_92>": 55, |
|
|
"<extra_id_93>": 54, |
|
|
"<extra_id_94>": 53, |
|
|
"<extra_id_95>": 52, |
|
|
"<extra_id_96>": 51, |
|
|
"<extra_id_97>": 50, |
|
|
"<extra_id_98>": 49, |
|
|
"<extra_id_99>": 48, |
|
|
"<extra_id_9>": 138, |
|
|
"<pad>": 148 |
|
|
} |
|
|
|