SLLM-0.6 / tokenizer.json
Veirded's picture
Upload 4 files
dac9a15 verified
Raw History Blame Contribute Delete
15.4 kB
{"version": "sllm-bpe-v017", "special_tokens": ["<pad>", "<end_of_text>", "<|user|>", "<|ai|>"], "merges": [[36, 120, 260], [36, 101, 261], [109, 114, 262], [36, 119, 263], [108, 105, 264], [262, 107, 265], [115, 121, 266], [118, 105, 267], [36, 123, 268], [36, 106, 269], [260, 264, 270], [36, 113, 271], [101, 120, 272], [114, 104, 273], [36, 103, 274], [36, 102, 275], [109, 119, 276], [260, 115, 277], [115, 118, 278], [105, 118, 279], [36, 77, 280], [109, 120, 281], [36, 112, 282], [36, 115, 283], [36, 116, 284], [36, 108, 285], [119, 120, 286], [36, 104, 287], [36, 107, 288], [112, 105, 289], [36, 276, 290], [36, 125, 291], [111, 105, 292], [101, 118, 293], [101, 125, 294], [101, 112, 295], [291, 266, 296], [113, 105, 297], [115, 115, 298], [36, 114, 299], [122, 105, 300], [260, 108, 301], [261, 273, 302], [112, 125, 303], [101, 114, 304], [36, 88, 305], [263, 115, 306], [36, 105, 307], [105, 105, 308], [283, 106, 309], [109, 103, 310], [107, 108, 311], [115, 123, 312], [115, 114, 313], [36, 262, 314], [36, 267, 315], [103, 108, 316], [275, 105, 317], [261, 113, 318], [269, 278, 319], [271, 125, 320], [119, 105, 321], [108, 272, 322], [109, 311, 323], [263, 120, 324], [36, 121, 325], [105, 114, 326], [105, 120, 327], [105, 119, 328], [283, 114, 329], [112, 112, 330], [285, 101, 331], [36, 281, 332], [101, 103, 333], [36, 118, 334], [266, 120, 335], [115, 120, 336], [323, 120, 337], [274, 115, 338], [305, 322, 339], [105, 104, 340], [122, 279, 341], [101, 119, 342], [282, 109, 343], [121, 120, 344], [109, 104, 345], [101, 104, 346], [261, 267, 347], [111, 265, 348], [120, 279, 349], [295, 303, 350], [263, 121, 351], [121, 286, 352], [36, 110, 353], [266, 273, 354], [260, 118, 355], [101, 292, 356], [112, 104, 357], [118, 115, 358], [114, 265, 359], [343, 292, 360], [296, 118, 361], [268, 281, 362], [362, 108, 363], [114, 120, 364], [298, 104, 365], [301, 272, 366], [109, 118, 367], [36, 335, 368], [325, 116, 369], [36, 264, 370], [105, 103, 371], [262, 104, 372], [306, 297, 373], [261, 114, 374], [299, 105, 375], [261, 120, 376], [263, 116, 377], [109, 297, 378], [288, 115, 379], [353, 352, 380], [121, 112, 381], [120, 265, 382], [287, 115, 383], [121, 114, 384], [118, 101, 385], [261, 112, 386], [269, 308, 387], [331, 300, 388], [282, 115, 389], [101, 286, 390], [108, 115, 391], [118, 109, 392], [123, 294, 393], [298, 111, 394], [36, 91, 395], [280, 120, 396], [387, 112, 397], [108, 265, 398], [305, 264, 399], [36, 81, 400], [271, 115, 401], [109, 313, 402], [36, 87, 403], [341, 125, 404], [264, 114, 405], [284, 112, 406], [104, 294, 407], [121, 118, 408], [106, 381, 409], [267, 272, 410], [113, 116, 411], [293, 104, 412], [116, 116, 413], [333, 111, 414], [267, 119, 415], [268, 294, 416], [263, 354, 417], [285, 115, 418], [304, 120, 419], [287, 294, 420], [326, 120, 421], [278, 359, 422], [288, 327, 423], [117, 121, 424], [272, 316, 425], [271, 121, 426], [109, 112, 427], [107, 105, 428], [268, 105, 429], [271, 356, 430], [271, 422, 431], [36, 111, 432], [271, 105, 433], [266, 311, 434], [293, 120, 435], [105, 116, 436], [268, 278, 437], [109, 330, 438], [261, 102, 439], [310, 105, 440], [106, 349, 441], [301, 276, 442], [417, 119, 443], [261, 441, 444], [268, 405, 445], [260, 378, 446], [281, 105, 447], [105, 273, 448], [393, 119, 449], [281, 120, 450], [379, 265, 451], [101, 262, 452], [345, 105, 453], [36, 278, 454], [274, 108, 455], [386, 449, 456], [317, 286, 457], [274, 304, 458], [106, 106, 459], [276, 120, 460], [389, 300, 461], [263, 108, 462], [268, 308, 463], [261, 330, 464], [266, 118, 465], [400, 125, 466], [450, 289, 467], [282, 467, 468], [36, 76, 469], [315, 350, 470], [272, 105, 471], [109, 300, 472], [277, 407, 473], [274, 112, 474], [288, 365, 475], [273, 279, 476], [115, 267, 477], [307, 404, 478], [36, 69, 479], [313, 105, 480], [269, 384, 481], [36, 424, 482], [36, 122, 483], [36, 72, 484], [288, 410, 485], [101, 121, 486], [307, 124, 487], [375, 123, 488], [125, 265, 489], [120, 398, 490], [351, 316, 491], [358, 113, 492], [299, 336, 493], [109, 289, 494], [293, 113, 495], [101, 348, 496], [102, 289, 497], [298, 114, 498], [103, 105, 499], [333, 105, 500], [282, 105, 501], [101, 122, 502], [269, 492, 503], [116, 105, 504], [121, 267, 505], [264, 118, 506], [36, 93, 507], [307, 114, 508], [283, 357, 509], [334, 337, 510], [269, 262, 511], [439, 335, 512], [287, 105, 513], [426, 316, 514], [101, 300, 515], [268, 295, 516], [268, 108, 517], [109, 113, 518], [109, 105, 519], [371, 120, 520], [268, 342, 521], [299, 440, 522], [109, 328, 523], [308, 104, 524], [122, 265, 525], [274, 272, 526], [121, 119, 527], [261, 118, 528], [292, 273, 529], [463, 529, 530], [103, 111, 531], [263, 103, 532], [115, 125, 533], [109, 106, 534], [261, 103, 535], [279, 119, 536], [109, 107, 537], [312, 114, 538], [342, 119, 539], [284, 105, 540], [313, 107, 541], [268, 115, 542], [120, 115, 543], [307, 300, 544], [301, 118, 545], [115, 112, 546], [272, 402, 547], [299, 524, 548], [299, 337, 549], [315, 112, 550], [101, 116, 551], [309, 106, 552], [299, 115, 553], [507, 266, 554], [298, 113, 555], [282, 394, 556], [114, 498, 557], [444, 557, 558], [338, 113, 559], [270, 125, 560], [275, 344, 561], [295, 112, 562], [267, 104, 563], [344, 105, 564], [121, 113, 565], [310, 111, 566], [101, 273, 567], [119, 453, 568], [275, 118, 569], [425, 265, 570], [266, 119, 571], [101, 113, 572], [36, 75, 573], [395, 105, 574], [36, 292, 575], [110, 533, 576], [275, 414, 577], [115, 321, 578], [109, 116, 579], [284, 358, 580], [268, 438, 581], [292, 120, 582], [517, 494, 583], [101, 297, 584], [545, 434, 585], [299, 312, 586], [277, 115, 587], [114, 337, 588], [101, 124, 589], [101, 111, 590], [284, 115, 591], [36, 71, 592], [270, 113, 593], [415, 108, 594], [269, 392, 595], [261, 119, 596], [411, 289, 597], [120, 108, 598], [268, 372, 599], [430, 119, 600], [274, 289, 601], [315, 346, 602], [36, 73, 603], [120, 105, 604], [297, 364, 605], [275, 125, 606], [418, 297, 607], [275, 267, 608], [595, 448, 609], [301, 262, 610], [374, 125, 611], [272, 279, 612], [542, 476, 613], [278, 120, 614], [262, 105, 615], [550, 589, 616], [260, 105, 617], [264, 104, 618], [275, 112, 619], [109, 126, 620], [101, 126, 621], [285, 312, 622], [36, 84, 623], [289, 436, 624], [502, 278, 625], [298, 348, 626], [275, 394, 627], [118, 125, 628], [287, 538, 629], [329, 105, 630], [268, 495, 631], [106, 477, 632], [437, 111, 633], [317, 632, 634], [397, 265, 635], [101, 107, 636], [269, 112, 637], [263, 308, 638], [269, 372, 639], [292, 104, 640], [324, 435, 641], [101, 428, 642], [277, 588, 643], [613, 409, 644], [266, 357, 645], [269, 594, 646], [575, 436, 647], [314, 543, 648], [406, 304, 649], [119, 119, 650], [109, 327, 651], [36, 74, 652], [301, 265, 653], [260, 356, 654], [484, 115, 655], [482, 651, 656], [282, 541, 657], [101, 497, 658], [511, 350, 659], [355, 489, 660], [334, 555, 661], [383, 107, 662], [269, 625, 663], [406, 294, 664], [126, 125, 665], [268, 391, 666], [36, 83, 667], [432, 114, 668], [263, 114, 669], [486, 321, 670], [317, 103, 671], [283, 341, 672], [106, 262, 673], [346, 105, 674], [123, 114, 675], [516, 111, 676], [115, 297, 677], [268, 419, 678], [266, 321, 679], [351, 114, 680], [663, 447, 681], [317, 120, 682], [317, 104, 683], [102, 105, 684], [301, 578, 685], [112, 106, 686], [269, 105, 687], [325, 114, 688], [121, 350, 689], [115, 116, 690], [391, 321, 691], [113, 278, 692], [671, 670, 693], [447, 303, 694], [544, 359, 695], [328, 119, 696], [673, 694, 697], [459, 308, 698], [413, 265, 699], [288, 584, 700], [270, 267, 701], [118, 312, 702], [120, 378, 703], [305, 691, 704], [377, 448, 705], [307, 293, 706], [263, 113, 707], [338, 698, 708], [263, 624, 709], [102, 278, 710], [418, 504, 711], [334, 115, 712], [268, 570, 713], [284, 108, 714], [325, 119, 715], [269, 354, 716], [36, 82, 717], [284, 121, 718], [373, 490, 719], [36, 80, 720], [284, 267, 721], [540, 500, 722], [331, 119, 723], [263, 123, 724], [282, 460, 725], [284, 344, 726], [308, 114, 727], [119, 546, 728], [274, 264, 729], [312, 119, 730], [301, 304, 731], [109, 571, 732], [274, 101, 733], [264, 267, 734], [368, 568, 735], [261, 104, 736], [112, 116, 737], [263, 281, 738], [728, 564, 739], [338, 665, 740], [528, 354, 741], [275, 367, 742], [323, 710, 743], [36, 404, 744], [268, 322, 745], [116, 279, 746], [118, 276, 747], [408, 114, 748], [275, 281, 749], [513, 697, 750], [106, 120, 751], [285, 679, 752], [370, 346, 753], [298, 112, 754], [275, 115, 755], [338, 597, 756], [104, 119, 757], [375, 743, 758], [315, 286, 759], [621, 265, 760], [399, 125, 761], [103, 120, 762], [756, 604, 763], [397, 119, 764], [281, 103, 765], [333, 120, 766], [401, 267, 767], [114, 125, 768], [275, 121, 769], [288, 112, 770], [355, 125, 771], [267, 105, 772], [112, 120, 773], [370, 737, 774], [272, 506, 775], [338, 112, 776], [395, 322, 777], [36, 534, 778], [293, 111, 779], [282, 337, 780], [307, 273, 781], [539, 310, 782], [271, 262, 783], [355, 579, 784], [282, 486, 785], [439, 739, 786], [666, 289, 787], [432, 372, 788], [107, 327, 789], [508, 576, 790], [703, 119, 791], [318, 760, 792], [559, 106, 793], [763, 303, 794], [401, 122, 795], [284, 279, 796], [610, 111, 797], [535, 120, 798], [274, 313, 799], [304, 107, 800], [282, 626, 801], [263, 391, 802], [106, 105, 803], [377, 421, 804], [263, 112, 805], [608, 590, 806], [112, 265, 807], [281, 265, 808], [285, 465, 809], [682, 349, 810], [331, 413, 811], [114, 279, 812], [434, 120, 813], [263, 101, 814], [722, 409, 815], [324, 294, 816], [326, 265, 817], [106, 520, 818], [284, 435, 819], [115, 113, 820], [334, 452, 821], [101, 427, 822], [275, 727, 823], [279, 125, 824], [281, 125, 825], [603, 114, 826], [653, 119, 827], [321, 104, 828], [287, 262, 829], [36, 465, 830], [423, 382, 831], [262, 120, 832], [274, 266, 833], [104, 125, 834], [601, 304, 835], [119, 310, 836], [271, 496, 837], [415, 119, 838], [36, 70, 839], [119, 116, 840], [384, 316, 841], [121, 531, 842], [826, 576, 843], [108, 365, 844], [314, 286, 845], [274, 391, 846], [287, 371, 847], [109, 357, 848], [275, 537, 849], [463, 111, 850], [124, 120, 851], [321, 686, 852], [331, 104, 853], [315, 103, 854], [765, 405, 855], [268, 612, 856], [109, 295, 857], [616, 265, 858], [432, 855, 859], [287, 109, 860], [474, 782, 861], [795, 519, 862], [667, 108, 863], [399, 267, 864], [276, 108, 865], [105, 293, 866], [105, 286, 867], [287, 328, 868], [264, 119, 869], [270, 367, 870], [101, 267, 871], [285, 412, 872], [283, 675, 873], [115, 278, 874], [351, 267, 875], [271, 304, 876], [331, 273, 877], [114, 107, 878], [796, 818, 879], [351, 118, 880], [401, 286, 881], [429, 775, 882], [261, 107, 883], [269, 115, 884], [269, 118, 885], [389, 103, 886], [403, 677, 887], [260, 123, 888], [599, 312, 889], [478, 490, 890], [272, 265, 891], [263, 264, 892], [289, 119, 893], [338, 297, 894], [282, 390, 895], [461, 303, 896], [370, 118, 897], [287, 392, 898], [288, 336, 899], [324, 278, 900], [105, 267, 901], [123, 412, 902], [274, 118, 903], [293, 125, 904], [120, 125, 905], [289, 104, 906], [518, 471, 907], [260, 748, 908], [426, 836, 909], [36, 78, 910], [113, 102, 911], [121, 116, 912], [263, 297, 913], [274, 754, 914], [285, 276, 915], [102, 102, 916], [268, 101, 917], [261, 393, 918], [288, 118, 919], [305, 108, 920], [324, 267, 921], [363, 335, 922], [109, 267, 923], [602, 265, 924], [886, 295, 925], [263, 265, 926], [263, 266, 927], [773, 907, 928], [268, 425, 929], [793, 614, 930], [668, 312, 931], [271, 293, 932], [913, 330, 933], [121, 295, 934], [324, 118, 935], [284, 118, 936], [325, 928, 937], [275, 385, 938], [481, 768, 939], [43, 119, 940], [284, 310, 941], [482, 566, 942], [738, 382, 943], [493, 398, 944], [327, 119, 945], [544, 114, 946], [429, 267, 947], [573, 365, 948], [513, 112, 949], [860, 459, 950], [306, 878, 951], [109, 303, 952], [261, 367, 953], [287, 276, 954], [690, 289, 955], [116, 112, 956], [121, 105, 957], [263, 111, 958], [829, 812, 959], [325, 476, 960], [706, 303, 961], [609, 119, 962], [324, 293, 963], [462, 312, 964], [285, 121, 965], [484, 345, 966], [36, 109, 967], [534, 120, 968], [272, 119, 969], [36, 86, 970], [263, 105, 971], [43, 120, 972], [126, 126, 973], [263, 371, 974], [910, 352, 975], [377, 371, 976], [433, 692, 977], [263, 109, 978], [310, 295, 979], [315, 295, 980], [262, 111, 981], [275, 527, 982], [610, 348, 983], [109, 497, 984], [308, 116, 985], [336, 115, 986], [109, 279, 987], [950, 901, 988], [115, 273, 989], [887, 791, 990], [285, 281, 991], [315, 106, 992], [307, 342, 993], [261, 413, 994], [287, 345, 995], [358, 650, 996], [263, 272, 997], [107, 125, 998], [284, 779, 999], [377, 336, 1000], [556, 119, 1001], [375, 851, 1002], [326, 499, 1003], [367, 286, 1004], [383, 265, 1005], [601, 293, 1006], [260, 410, 1007], [785, 311, 1008], [798, 689, 1009], [287, 118, 1010], [308, 120, 1011], [331, 525, 1012], [462, 645, 1013], [617, 101, 1014], [714, 986, 1015], [269, 365, 1016], [268, 262, 1017], [274, 295, 1018], [344, 328, 1019], [687, 123, 1020], [725, 817, 1021], [487, 103, 1022], [487, 120, 1023], [288, 385, 1024], [288, 412, 1025], [783, 1019, 1026], [102, 118, 1027], [267, 572, 1028], [284, 109, 1029], [116, 747, 1030], [469, 515, 1031], [326, 119, 1032], [880, 1030, 1033], [649, 359, 1034], [758, 844, 1035], [602, 125, 1036], [540, 955, 1037], [272, 408, 1038], [288, 472, 1039], [406, 419, 1040], [832, 642, 1041], [288, 968, 1042], [277, 120, 1043], [669, 414, 1044], [707, 562, 1045], [282, 327, 1046], [437, 357, 1047], [627, 119, 1048], [105, 346, 1049], [789, 506, 1050], [113, 115, 1051], [569, 265, 1052], [1018, 113, 1053], [110, 520, 1054], [282, 841, 1055], [113, 674, 1056], [262, 125, 1057], [355, 109, 1058], [288, 121, 1059], [408, 265, 1060], [718, 973, 1061], [343, 525, 1062], [401, 300, 1063], [1023, 385, 1064], [307, 364, 1065], [383, 278, 1066], [101, 367, 1067], [114, 336, 1068], [375, 341, 1069], [102, 115, 1070], [692, 702, 1071], [501, 515, 1072], [107, 304, 1073], [277, 1071, 1074], [811, 125, 1075], [927, 116, 1076], [126, 105, 1077], [101, 102, 1078], [269, 1004, 1079], [284, 390, 1080], [483, 1041, 1081], [345, 104, 1082], [478, 480, 1083], [619, 304, 1084], [288, 267, 1085], [274, 394, 1086], [268, 734, 1087], [104, 279, 1088], [420, 119, 1089], [458, 1068, 1090], [776, 278, 1091], [271, 276, 1092], [274, 293, 1093], [528, 120, 1094], [1073, 620, 1095], [304, 428, 1096], [401, 605, 1097], [413, 340, 1098], [272, 505, 1099], [113, 279, 1100], [287, 385, 1101], [287, 1060, 1102], [598, 407, 1103], [742, 1103, 1104], [314, 568, 1105], [274, 408, 1106], [483, 276, 1107], [276, 106, 1108], [580, 1054, 1109], [997, 1108, 1110], [334, 384, 1111], [361, 852, 1112], [809, 119, 1113], [926, 289, 1114], [424, 105, 1115], [277, 1050, 1116], [715, 689, 1117], [274, 425, 1118], [333, 348, 1119], [712, 346, 1120], [429, 330, 1121], [304, 119, 1122], [967, 411, 1123], [1029, 371, 1124], [36, 453, 1125], [282, 471, 1126], [104, 874, 1127], [121, 107, 1128], [331, 367, 1129], [113, 119, 1130], [291, 866, 1131], [978, 597, 1132], [313, 120, 1133], [364, 427, 1134], [724, 1011, 1135], [883, 452, 1136], [1084, 582, 1137], [502, 125, 1138], [501, 390, 1139], [324, 438, 1140], [325, 1134, 1141], [976, 857, 1142], [268, 118, 1143], [329, 499, 1144], [469, 105, 1145], [103, 618, 1146], [271, 674, 1147], [1110, 489, 1148], [263, 327, 1149], [454, 1095, 1150], [755, 124, 1151]], "vocab_size": 1152}