dalat5 / src /tokeniser /added_tokens.json
crossroderick's picture
Pre-v5 update for the tokeniser (training date pushed to the 25th)
794cf97
{
"<extra_id_0>": 40099,
"<extra_id_10>": 40089,
"<extra_id_11>": 40088,
"<extra_id_12>": 40087,
"<extra_id_13>": 40086,
"<extra_id_14>": 40085,
"<extra_id_15>": 40084,
"<extra_id_16>": 40083,
"<extra_id_17>": 40082,
"<extra_id_18>": 40081,
"<extra_id_19>": 40080,
"<extra_id_1>": 40098,
"<extra_id_20>": 40079,
"<extra_id_21>": 40078,
"<extra_id_22>": 40077,
"<extra_id_23>": 40076,
"<extra_id_24>": 40075,
"<extra_id_25>": 40074,
"<extra_id_26>": 40073,
"<extra_id_27>": 40072,
"<extra_id_28>": 40071,
"<extra_id_29>": 40070,
"<extra_id_2>": 40097,
"<extra_id_30>": 40069,
"<extra_id_31>": 40068,
"<extra_id_32>": 40067,
"<extra_id_33>": 40066,
"<extra_id_34>": 40065,
"<extra_id_35>": 40064,
"<extra_id_36>": 40063,
"<extra_id_37>": 40062,
"<extra_id_38>": 40061,
"<extra_id_39>": 40060,
"<extra_id_3>": 40096,
"<extra_id_40>": 40059,
"<extra_id_41>": 40058,
"<extra_id_42>": 40057,
"<extra_id_43>": 40056,
"<extra_id_44>": 40055,
"<extra_id_45>": 40054,
"<extra_id_46>": 40053,
"<extra_id_47>": 40052,
"<extra_id_48>": 40051,
"<extra_id_49>": 40050,
"<extra_id_4>": 40095,
"<extra_id_50>": 40049,
"<extra_id_51>": 40048,
"<extra_id_52>": 40047,
"<extra_id_53>": 40046,
"<extra_id_54>": 40045,
"<extra_id_55>": 40044,
"<extra_id_56>": 40043,
"<extra_id_57>": 40042,
"<extra_id_58>": 40041,
"<extra_id_59>": 40040,
"<extra_id_5>": 40094,
"<extra_id_60>": 40039,
"<extra_id_61>": 40038,
"<extra_id_62>": 40037,
"<extra_id_63>": 40036,
"<extra_id_64>": 40035,
"<extra_id_65>": 40034,
"<extra_id_66>": 40033,
"<extra_id_67>": 40032,
"<extra_id_68>": 40031,
"<extra_id_69>": 40030,
"<extra_id_6>": 40093,
"<extra_id_70>": 40029,
"<extra_id_71>": 40028,
"<extra_id_72>": 40027,
"<extra_id_73>": 40026,
"<extra_id_74>": 40025,
"<extra_id_75>": 40024,
"<extra_id_76>": 40023,
"<extra_id_77>": 40022,
"<extra_id_78>": 40021,
"<extra_id_79>": 40020,
"<extra_id_7>": 40092,
"<extra_id_80>": 40019,
"<extra_id_81>": 40018,
"<extra_id_82>": 40017,
"<extra_id_83>": 40016,
"<extra_id_84>": 40015,
"<extra_id_85>": 40014,
"<extra_id_86>": 40013,
"<extra_id_87>": 40012,
"<extra_id_88>": 40011,
"<extra_id_89>": 40010,
"<extra_id_8>": 40091,
"<extra_id_90>": 40009,
"<extra_id_91>": 40008,
"<extra_id_92>": 40007,
"<extra_id_93>": 40006,
"<extra_id_94>": 40005,
"<extra_id_95>": 40004,
"<extra_id_96>": 40003,
"<extra_id_97>": 40002,
"<extra_id_98>": 40001,
"<extra_id_99>": 40000,
"<extra_id_9>": 40090
}