AkaraAlpha / tokenizers.json
Darayut's picture
Release AkaraAlpha
d728d26 verified
Raw History Blame Contribute Delete
1.75 kB
{
"latin": {
"word_index": {
"<unk>": 1,
"a": 2,
"o": 3,
"h": 4,
"e": 5,
"n": 6,
"r": 7,
"k": 8,
"t": 9,
"u": 10,
"s": 11,
"m": 12,
"b": 13,
"i": 14,
"c": 15,
"p": 16,
"l": 17,
"g": 18,
"y": 19,
"v": 20,
"d": 21,
"j": 22,
"z": 23,
"f": 24,
"w": 25,
"x": 26,
"q": 27
},
"max_len": 25
},
"khmer": {
"word_index": {
"<unk>": 1,
"แŸ’": 2,
"แžถ": 3,
"แžš": 4,
"แž€": 5,
"แžŸ": 6,
"แž”": 7,
"แž": 8,
"แž“": 9,
"แž˜": 10,
"แž„": 11,
"แžท": 12,
"แž›": 13,
"แžข": 14,
"แž…": 15,
"แžป": 16,
"แžœ": 17,
"แž‘": 18,
"แž™": 19,
"แŸ‹": 20,
"แž–": 21,
"แŸ†": 22,
"แž ": 23,
"แŸ„": 24,
"แž‡": 25,
"แŸ": 26,
"แž‚": 27,
"แžธ": 28,
"แžŽ": 29,
"แž‰": 30,
"แžผ": 31,
"แž—": 32,
"แŸ‚": 33,
"แž’": 34,
"แžŠ": 35,
"แž": 36,
"แžพ": 37,
"แŸ‰": 38,
"แŸ‡": 39,
"แžน": 40,
"แžฝ": 41,
"แžก": 42,
"แž": 43,
"แž•": 44,
"แŸ€": 45,
"แŸ": 46,
"แŸˆ": 47,
"แž†": 48,
"แŸ": 49,
"แŸƒ": 50,
"แŸŠ": 51,
"แž‹": 52,
"แžบ": 53,
"แŸ—": 54,
"แžŒ": 55,
"แžง": 56,
"แŸ…": 57,
"แžƒ": 58,
"แžˆ": 59,
"แŸŒ": 60,
"แžฟ": 61,
"แžฅ": 62,
"แžฑ": 63,
"แžซ": 64,
"แžฏ": 65,
"แž": 66,
"แŸŽ": 67,
"แŸ": 68,
"แžช": 69,
"แžฉ": 70,
"แžฒ": 71,
"แžฆ": 72,
"แžฎ": 73,
"แžฐ": 74,
"แžณ": 75,
"แŸ”": 76,
"แžญ": 77,
"แžฌ": 78,
"\t": 79,
"\n": 80
},
"max_len": 24
},
"padding_id": 0,
"start_token": "\t",
"end_token": "\n",
"oov_token": "<unk>"
}