Mistral / tokenizer.json
Delower's picture
Upload tokenizer
97987c8 verified
Raw History Blame Contribute Delete
3.3 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "[BOS]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "[EOS]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true,
"use_regex": true
},
"post_processor": null,
"decoder": null,
"model": {
"type": "BPE",
"dropout": null,
"unk_token": "[PAD]",
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"[PAD]": 0,
"[BOS]": 1,
"[EOS]": 2,
"A": 3,
"B": 4,
"C": 5,
"D": 6,
"E": 7,
"F": 8,
"G": 9,
"H": 10,
"I": 11,
"K": 12,
"L": 13,
"M": 14,
"N": 15,
"O": 16,
"P": 17,
"Q": 18,
"R": 19,
"S": 20,
"T": 21,
"U": 22,
"V": 23,
"W": 24,
"X": 25,
"Y": 26,
"[": 27,
"]": 28,
"Ġ": 29,
"OS": 30,
"][": 31,
"AC": 32,
"AD": 33,
"BOS": 34,
"DE": 35,
"EOS": 36,
"FG": 37,
"HI": 38,
"KL": 39,
"MN": 40,
"PQ": 41,
"PAD": 42,
"RS": 43,
"TU": 44,
"VW": 45,
"XY": 46,
"Ġ[": 47,
"ACDE": 48,
"FGHI": 49,
"KLMN": 50,
"PQRS": 51,
"TUVW": 52,
"ACDEFGHI": 53,
"KLMNPQRS": 54,
"TUVWXY": 55,
"ACDEFGHIKLMNPQRS": 56,
"ACDEFGHIKLMNPQRSTUVWXY": 57
},
"merges": [
[
"O",
"S"
],
[
"]",
"["
],
[
"A",
"C"
],
[
"A",
"D"
],
[
"B",
"OS"
],
[
"D",
"E"
],
[
"E",
"OS"
],
[
"F",
"G"
],
[
"H",
"I"
],
[
"K",
"L"
],
[
"M",
"N"
],
[
"P",
"Q"
],
[
"P",
"AD"
],
[
"R",
"S"
],
[
"T",
"U"
],
[
"V",
"W"
],
[
"X",
"Y"
],
[
"Ġ",
"["
],
[
"AC",
"DE"
],
[
"FG",
"HI"
],
[
"KL",
"MN"
],
[
"PQ",
"RS"
],
[
"TU",
"VW"
],
[
"ACDE",
"FGHI"
],
[
"KLMN",
"PQRS"
],
[
"TUVW",
"XY"
],
[
"ACDEFGHI",
"KLMNPQRS"
],
[
"ACDEFGHIKLMNPQRS",
"TUVWXY"
]
]
}
}