tokenizer.json

{
    "version": "1.0",
    "model": {
        "type": "BPE",
        "vocab": {
            "[PAD]": 0,
            "<s>": 1,
            "</s>": 2,
            "[INST]": 3,
            "[/INST]": 4,
            "\t": 5,
            "\n": 6,
            " ": 7,
            "!": 8,
            "\"": 9,
            "#": 10,
            "$": 11,
            "%": 12,
            "&": 13,
            "'": 14,
            "(": 15,
            ")": 16,
            "*": 17,
            "+": 18,
            ",": 19,
            "-": 20,
            ".": 21,
            "/": 22,
            "0": 23,
            "1": 24,
            "2": 25,
            "3": 26,
            "4": 27,
            "5": 28,
            "6": 29,
            "7": 30,
            "8": 31,
            "9": 32,
            ":": 33,
            ";": 34,
            "<": 35,
            "=": 36,
            ">": 37,
            "?": 38,
            "@": 39,
            "A": 40,
            "B": 41,
            "C": 42,
            "D": 43,
            "E": 44,
            "F": 45,
            "G": 46,
            "H": 47,
            "I": 48,
            "J": 49,
            "K": 50,
            "L": 51,
            "M": 52,
            "N": 53,
            "O": 54,
            "P": 55,
            "Q": 56,
            "R": 57,
            "S": 58,
            "T": 59,
            "U": 60,
            "V": 61,
            "W": 62,
            "X": 63,
            "Y": 64,
            "Z": 65,
            "[": 66,
            "]": 67,
            "^": 68,
            "_": 69,
            "`": 70,
            "a": 71,
            "b": 72,
            "c": 73,
            "d": 74,
            "e": 75,
            "f": 76,
            "g": 77,
            "h": 78,
            "i": 79,
            "j": 80,
            "k": 81,
            "l": 82,
            "m": 83,
            "n": 84,
            "o": 85,
            "p": 86,
            "q": 87,
            "r": 88,
            "s": 89,
            "t": 90,
            "u": 91,
            "v": 92,
            "w": 93,
            "x": 94,
            "y": 95,
            "z": 96,
            "{": 97,
            "|": 98,
            "}": 99,
            "~": 100
        },
        "merges": []
    },
    "pre_tokenizer": {
        "type": "ByteLevel"
    },
    "decoder": {
        "type": "ByteLevel"
    },
    "normalizer": {
        "type": "Lowercase"
    },
    "post_processor": {
        "type": "TemplateProcessing",
        "single": [
            {
                "type": "SpecialToken",
                "id": "<s>",
                "ids": 1,
                "tokens": [
                    "<s>"
                ]
            },
            {
                "type": "Sequence",
                "text": " "
            },
            {
                "type": "Feature",
                "feature": "tokens"
            },
            {
                "type": "SpecialToken",
                "id": "</s>",
                "ids": 2,
                "tokens": [
                    "</s>"
                ]
            }
        ],
        "pair": [],
        "special_tokens": {
            "<s>": {
                "id": "<s>",
                "ids": 1,
                "tokens": [
                    "<s>"
                ]
            },
            "</s>": {
                "id": "</s>",
                "ids": 2,
                "tokens": [
                    "</s>"
                ]
            }
        }
    },
    "added_tokens": [
        {
            "id": 3,
            "content": "[INST]",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": false
        },
        {
            "id": 4,
            "content": "[/INST]",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": false
        }
    ]
}