tokenizer_config.json

{
    "name": "CharacterTokenizer",
    "char_ords": [
        3090,
        355,
        1139,
        777,
        1060,
        183,
        32,
        1049,
        1122,
        1106,
        8056,
        947,
        1026,
        36,
        191,
        8051,
        1058,
        1103,
        10004,
        776,
        8736,
        8747,
        90,
        8592,
        1116,
        353,
        9829,
        324,
        1097,
        1066,
        1513,
        1032,
        367,
        1514,
        23565,
        1084,
        1054,
        896,
        185,
        345,
        8222,
        251,
        1040,
        1085,
        1112,
        1062,
        44,
        700,
        8704,
        33310,
        1117,
        1038,
        12515,
        40,
        1104,
        1031,
        64,
        1079,
        943,
        240,
        72,
        20869,
        39,
        1073,
        352,
        196,
        299,
        8059,
        9794,
        8203,
        1025,
        8249,
        63,
        80,
        1141,
        31639,
        729,
        966,
        1169,
        1050,
        101,
        246,
        1046,
        1089,
        1076,
        12476,
        112,
        1053,
        8166,
        87,
        27475,
        318,
        47,
        968,
        61608,
        49,
        8016,
        12384,
        1512,
        8183,
        12364,
        9472,
        12523,
        937,
        21152,
        61558,
        184,
        1081,
        37,
        19977,
        10024,
        1115,
        513,
        365,
        20238,
        7940,
        1056,
        231,
        83,
        768,
        932,
        972,
        224,
        9,
        9656,
        52,
        331,
        8541,
        1199,
        113,
        8746,
        26494,
        33,
        1503,
        1217,
        43,
        38646,
        941,
        944,
        8539,
        926,
        12459,
        226,
        1063,
        1078,
        172,
        34,
        970,
        199,
        8211,
        128165,
        9786,
        1047,
        731,
        64257,
        5,
        204,
        130,
        55,
        34892,
        107,
        1493,
        189,
        167,
        12377,
        12408,
        181,
        252,
        22303,
        920,
        235,
        8540,
        8218,
        1211,
        12365,
        95,
        259,
        973,
        1111,
        8212,
        903,
        202,
        961,
        58,
        56,
        250,
        12298,
        65,
        933,
        2,
        1107,
        379,
        1241,
        8057,
        12473,
        62,
        347,
        75,
        1114,
        1082,
        178,
        151,
        915,
        53,
        9679,
        82,
        1092,
        928,
        8115,
        33021,
        929,
        194,
        269,
        232,
        150,
        7985,
        65281,
        77,
        10052,
        116,
        12418,
        958,
        942,
        205,
        73,
        1099,
        1105,
        65039,
        201,
        195,
        618,
        111,
        61482,
        974,
        977,
        42,
        1071,
        1101,
        274,
        344,
        727,
        127,
        8243,
        245,
        79,
        938,
        12540,
        967,
        1102,
        957,
        124,
        12383,
        8722,
        12358,
        1119,
        268,
        1055,
        8730,
        27498,
        8194,
        108,
        242,
        8453,
        1088,
        254,
        12517,
        122,
        8221,
        351,
        234,
        192,
        945,
        24231,
        8594,
        88,
        10240,
        948,
        8060,
        8017,
        1067,
        8216,
        228,
        328,
        61449,
        98,
        279,
        161,
        99,
        8226,
        1497,
        12463,
        8150,
        12417,
        27468,
        9825,
        295,
        119,
        8531,
        369,
        221,
        1091,
        110,
        173,
        8195,
        8596,
        1029,
        8197,
        65279,
        8049,
        211,
        106,
        23383,
        23376,
        916,
        160,
        239,
        28783,
        19979,
        114,
        220,
        103,
        8164,
        8729,
        1187,
        9824,
        337,
        1086,
        8542,
        8225,
        9524,
        188,
        248,
        8593,
        8240,
        118,
        123,
        1096,
        923,
        21475,
        8741,
        9633,
        257,
        12434,
        180,
        1036,
        8364,
        61620,
        955,
        197,
        190,
        8,
        186,
        244,
        8470,
        382,
        10,
        281,
        1440,
        8743,
        380,
        179,
        8242,
        230,
        934,
        363,
        981,
        833,
        200,
        12512,
        8001,
        96,
        218,
        8482,
        1041,
        41,
        1095,
        7990,
        9658,
        1068,
        54,
        25126,
        84,
        925,
        12414,
        70,
        8776,
        8532,
        253,
        960,
        8055,
        917,
        236,
        28779,
        950,
        57,
        38,
        81,
        61448,
        182,
        935,
        1140,
        8058,
        67,
        1027,
        215,
        8322,
        85,
        1138,
        1028,
        271,
        166,
        1069,
        946,
        187,
        7936,
        177,
        969,
        8224,
        171,
        8190,
        922,
        12525,
        105,
        1144,
        28792,
        126,
        338,
        16,
        65533,
        7969,
        1256,
        965,
        20307,
        12395,
        1061,
        8052,
        12424,
        225,
        25731,
        12,
        8486,
        59,
        1030,
        61,
        464,
        12471,
        287,
        1042,
        1502,
        8800,
        35,
        1175,
        1257,
        733,
        7937,
        918,
        255,
        9632,
        900,
        8054,
        76,
        1087,
        174,
        1524,
        50,
        60,
        8230,
        48,
        120,
        964,
        29226,
        8038,
        121,
        8021,
        1110,
        1100,
        1044,
        1113,
        971,
        949,
        963,
        241,
        263,
        931,
        1094,
        17114,
        237,
        8220,
        1045,
        176,
        9650,
        1090,
        378,
        1080,
        1033,
        243,
        65293,
        36814,
        1168,
        69,
        198,
        914,
        9830,
        163,
        1120,
        12288,
        953,
        902,
        1059,
        8048,
        212,
        14,
        1492,
        321,
        951,
        904,
        1072,
        93,
        227,
        71,
        1098,
        952,
        1093,
        104,
        1488,
        7957,
        1008,
        74,
        125,
        216,
        12426,
        45,
        1075,
        7952,
        9792,
        1083,
        38754,
        277,
        959,
        1500,
        46,
        97,
        12427,
        956,
        59393,
        720,
        305,
        8202,
        1052,
        7883,
        8213,
        1074,
        8125,
        1118,
        1043,
        128519,
        357,
        1051,
        8595,
        233,
        12360,
        962,
        665,
        34241,
        927,
        12481,
        1489,
        89,
        109,
        210,
        1057,
        275,
        92,
        12507,
        12356,
        65288,
        8239,
        247,
        710,
        78,
        1064,
        261,
        921,
        164,
        8165,
        25239,
        1123,
        924,
        8206,
        769,
        1039,
        322,
        115,
        9574,
        912,
        66,
        1108,
        63719,
        32946,
        229,
        8134,
        117,
        8287,
        94,
        1077,
        940,
        51,
        214,
        1070,
        38651,
        381,
        913,
        1065,
        8020,
        249,
        91,
        9827,
        346,
        8179,
        1034,
        86,
        339,
        350,
        8209,
        919,
        100,
        712,
        8706,
        1035,
        223,
        273,
        1048,
        954,
        8196,
        12398,
        8182,
        65289,
        283,
        1109,
        238,
        21322,
        8805,
        4,
        8543,
        68,
        8210,
        8217,
        8250,
        102,
        169,
        42012
    ],
    "model_max_length": 2048
}