Romance-cleaned-1 / added_tokens.json
MarkGG's picture
Upload tokenizer
c4f1f49
{
"EXT.": 52000,
"INT.": 52001,
"[EX0]": 52004,
"[EX10]": 52014,
"[EX11]": 52015,
"[EX12]": 52016,
"[EX13]": 52017,
"[EX14]": 52018,
"[EX15]": 52019,
"[EX16]": 52020,
"[EX17]": 52021,
"[EX18]": 52022,
"[EX19]": 52023,
"[EX1]": 52005,
"[EX20]": 52024,
"[EX21]": 52025,
"[EX22]": 52026,
"[EX23]": 52027,
"[EX24]": 52028,
"[EX25]": 52029,
"[EX26]": 52030,
"[EX27]": 52031,
"[EX28]": 52032,
"[EX29]": 52033,
"[EX2]": 52006,
"[EX30]": 52034,
"[EX31]": 52035,
"[EX32]": 52036,
"[EX33]": 52037,
"[EX34]": 52038,
"[EX35]": 52039,
"[EX36]": 52040,
"[EX37]": 52041,
"[EX38]": 52042,
"[EX39]": 52043,
"[EX3]": 52007,
"[EX40]": 52044,
"[EX41]": 52045,
"[EX42]": 52046,
"[EX43]": 52047,
"[EX44]": 52048,
"[EX45]": 52049,
"[EX46]": 52050,
"[EX47]": 52051,
"[EX48]": 52052,
"[EX49]": 52053,
"[EX4]": 52008,
"[EX50]": 52054,
"[EX51]": 52055,
"[EX52]": 52056,
"[EX53]": 52057,
"[EX54]": 52058,
"[EX55]": 52059,
"[EX56]": 52060,
"[EX57]": 52061,
"[EX58]": 52062,
"[EX59]": 52063,
"[EX5]": 52009,
"[EX60]": 52064,
"[EX61]": 52065,
"[EX62]": 52066,
"[EX63]": 52067,
"[EX64]": 52068,
"[EX65]": 52069,
"[EX66]": 52070,
"[EX67]": 52071,
"[EX68]": 52072,
"[EX69]": 52073,
"[EX6]": 52010,
"[EX70]": 52074,
"[EX71]": 52075,
"[EX72]": 52076,
"[EX73]": 52077,
"[EX74]": 52078,
"[EX75]": 52079,
"[EX76]": 52080,
"[EX77]": 52081,
"[EX78]": 52082,
"[EX79]": 52083,
"[EX7]": 52011,
"[EX80]": 52084,
"[EX81]": 52085,
"[EX82]": 52086,
"[EX83]": 52087,
"[EX84]": 52088,
"[EX85]": 52089,
"[EX86]": 52090,
"[EX87]": 52091,
"[EX88]": 52092,
"[EX89]": 52093,
"[EX8]": 52012,
"[EX90]": 52094,
"[EX91]": 52095,
"[EX92]": 52096,
"[EX93]": 52097,
"[EX94]": 52098,
"[EX95]": 52099,
"[EX96]": 52100,
"[EX97]": 52101,
"[EX98]": 52102,
"[EX99]": 52103,
"[EX9]": 52013,
"[FRL]": 52002,
"[MRL]": 52003
}