CogNet-1B / tokenizer_v3.json
thefinalboss's picture
Upload tokenizer_v3.json with huggingface_hub
88b19bd verified
Raw
History Blame Contribute Delete
2.04 kB
{
"vocab_size": 136,
"char_to_id": {
"0": 0,
"1": 1,
"2": 2,
"3": 3,
"32": 4,
"33": 5,
"34": 6,
"35": 7,
"36": 8,
"37": 9,
"38": 10,
"39": 11,
"40": 12,
"41": 13,
"42": 14,
"43": 15,
"44": 16,
"45": 17,
"46": 18,
"47": 19,
"48": 20,
"49": 21,
"50": 22,
"51": 23,
"52": 24,
"53": 25,
"54": 26,
"55": 27,
"56": 28,
"57": 29,
"58": 30,
"59": 31,
"60": 32,
"61": 33,
"62": 34,
"63": 35,
"64": 36,
"65": 37,
"66": 38,
"67": 39,
"68": 40,
"69": 41,
"70": 42,
"71": 43,
"72": 44,
"73": 45,
"74": 46,
"75": 47,
"76": 48,
"77": 49,
"78": 50,
"79": 51,
"80": 52,
"81": 53,
"82": 54,
"83": 55,
"84": 56,
"85": 57,
"86": 58,
"87": 59,
"88": 60,
"89": 61,
"90": 62,
"91": 63,
"92": 64,
"93": 65,
"94": 66,
"95": 67,
"96": 68,
"97": 69,
"98": 70,
"99": 71,
"100": 72,
"101": 73,
"102": 74,
"103": 75,
"104": 76,
"105": 77,
"106": 78,
"107": 79,
"108": 80,
"109": 81,
"110": 82,
"111": 83,
"112": 84,
"113": 85,
"114": 86,
"115": 87,
"116": 88,
"117": 89,
"118": 90,
"119": 91,
"120": 92,
"121": 93,
"122": 94,
"123": 95,
"124": 96,
"125": 97,
"126": 98,
"192": 99,
"193": 100,
"194": 101,
"195": 102,
"196": 103,
"197": 104,
"199": 105,
"200": 106,
"201": 107,
"202": 108,
"203": 109,
"204": 110,
"205": 111,
"206": 112,
"207": 113,
"210": 114,
"211": 115,
"212": 116,
"213": 117,
"214": 118,
"217": 119,
"218": 120,
"219": 121,
"220": 122,
"224": 123,
"225": 124,
"226": 125,
"227": 126,
"228": 127,
"229": 128,
"231": 129,
"232": 130,
"233": 131,
"234": 132,
"235": 133,
"236": 134,
"237": 135
}
}