Hugging Face's logo

XXVICK
/
Myst_Ai-25M-Coder

Text Generation
PyTorch
code
English
python
from-scratch
Model card Files
xet
 Community
Myst_Ai-25M-Coder / code_bpe.json
XXVICK's picture
XXVICK
Upload code_bpe.json with huggingface_hub
ef7d0ec verified
Raw History Blame Contribute Delete
9.93 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<|unk|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<|pad|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"post_processor": null,
"decoder": null,
"model": {
"type": "BPE",
"dropout": null,
"unk_token": "<|unk|>",
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"<|unk|>": 0,
"<|pad|>": 1,
"<|endoftext|>": 2,
"(": 3,
")": 4,
"+": 5,
",": 6,
"-": 7,
".": 8,
"/": 9,
"0": 10,
"1": 11,
"2": 12,
":": 13,
"<": 14,
"=": 15,
">": 16,
"N": 17,
"[": 18,
"]": 19,
"_": 20,
"a": 21,
"b": 22,
"c": 23,
"d": 24,
"e": 25,
"f": 26,
"g": 27,
"h": 28,
"i": 29,
"k": 30,
"l": 31,
"m": 32,
"n": 33,
"o": 34,
"p": 35,
"q": 36,
"r": 37,
"s": 38,
"t": 39,
"u": 40,
"v": 41,
"w": 42,
"x": 43,
"y": 44,
"|": 45,
"Ċ": 46,
"č": 47,
"Ġ": 48,
"ĠĠ": 49,
"čĊ": 50,
"ar": 51,
"ĠĠĠ": 52,
"Ġ=": 53,
"arr": 54,
"id": 55,
"le": 56,
"mid": 57,
"Ġi": 58,
"et": 59,
"Ġ<": 60,
"Ġarr": 61,
"ex": 62,
"gh": 63,
"igh": 64,
"or": 65,
"Ġx": 66,
"Ġmid": 67,
"ĠĠĠĠ": 68,
"ext": 69,
"ĠĠĠĠĠĠĠ": 70,
"el": 71,
"ft": 72,
"Ġr": 73,
"Ġif": 74,
"de": 75,
"high": 76,
"iv": 77,
"lo": 78,
"ot": 79,
"piv": 80,
"rn": 81,
"urn": 82,
"Ġ+": 83,
"Ġ1": 84,
"Ġhigh": 85,
"Ġpiv": 86,
"eturn": 87,
"Ġreturn": 88,
"low": 89,
"Ġpivot": 90,
"):": 91,
"al": 92,
"ck": 93,
"do": 94,
"en": 95,
"for": 96,
"get": 97,
"ick": 98,
"next": 99,
"qu": 100,
"sor": 101,
"sel": 102,
"tar": 103,
"val": 104,
"|>": 105,
"Ġ-": 106,
"Ġ[": 107,
"ĠčĊ": 108,
"Ġle": 109,
"Ġde": 110,
"Ġlow": 111,
"Ġfor": 112,
"Ġqu": 113,
"Ġtar": 114,
"Ġin": 115,
"Ġ<|": 116,
"ftext": 117,
"doftext": 118,
"endoftext": 119,
"icksor": 120,
"self": 121,
"Ġdef": 122,
"Ġquicksor": 123,
"Ġtarget": 124,
"Ġquicksort": 125,
"//": 126,
"No": 127,
"__": 128,
"dle": 129,
"in": 130,
"se": 131,
"Ġ2": 132,
"Ġel": 133,
"Ġnext": 134,
"Ġval": 135,
"Ġself": 136,
"Ġ//": 137,
"Ġ==": 138,
"Ġ<=": 139,
"ight": 140,
"Ġmiddle": 141,
"Ġlen": 142,
"as": 143,
"bin": 144,
"ch": 145,
"cl": 146,
"hi": 147,
"if": 148,
"it": 149,
"ne": 150,
"right": 151,
"whi": 152,
"Ġ(": 153,
"Ġ0": 154,
"Ġ>": 155,
"ĠNo": 156,
"Ġ__": 157,
"Ġbin": 158,
"Ġcl": 159,
"Ġwhi": 160,
"ary": 161,
"arch": 162,
"len": 163,
"left": 164,
"Ġright": 165,
"Ġleft": 166,
"None": 167,
"__(": 168,
"init": 169,
"search": 170,
"Ġelse": 171,
"Ġelif": 172,
"ass": 173,
"ĠNode": 174,
"Ġbinary": 175,
"Ġclass": 176,
"Ġwhile": 177
},
"merges": [
[
"Ġ",
"Ġ"
],
[
"č",
"Ċ"
],
[
"a",
"r"
],
[
"ĠĠ",
"Ġ"
],
[
"Ġ",
"="
],
[
"ar",
"r"
],
[
"i",
"d"
],
[
"l",
"e"
],
[
"m",
"id"
],
[
"Ġ",
"i"
],
[
"e",
"t"
],
[
"Ġ",
"<"
],
[
"Ġ",
"arr"
],
[
"e",
"x"
],
[
"g",
"h"
],
[
"i",
"gh"
],
[
"o",
"r"
],
[
"Ġ",
"x"
],
[
"Ġ",
"mid"
],
[
"ĠĠ",
"ĠĠ"
],
[
"ex",
"t"
],
[
"ĠĠĠĠ",
"ĠĠĠ"
],
[
"e",
"l"
],
[
"f",
"t"
],
[
"Ġ",
"r"
],
[
"Ġi",
"f"
],
[
"d",
"e"
],
[
"h",
"igh"
],
[
"i",
"v"
],
[
"l",
"o"
],
[
"o",
"t"
],
[
"p",
"iv"
],
[
"r",
"n"
],
[
"u",
"rn"
],
[
"Ġ",
"+"
],
[
"Ġ",
"1"
],
[
"Ġ",
"high"
],
[
"Ġ",
"piv"
],
[
"et",
"urn"
],
[
"Ġr",
"eturn"
],
[
"lo",
"w"
],
[
"Ġpiv",
"ot"
],
[
")",
":"
],
[
"a",
"l"
],
[
"c",
"k"
],
[
"d",
"o"
],
[
"e",
"n"
],
[
"f",
"or"
],
[
"g",
"et"
],
[
"i",
"ck"
],
[
"n",
"ext"
],
[
"q",
"u"
],
[
"s",
"or"
],
[
"s",
"el"
],
[
"t",
"ar"
],
[
"v",
"al"
],
[
"|",
">"
],
[
"Ġ",
"-"
],
[
"Ġ",
"["
],
[
"Ġ",
"čĊ"
],
[
"Ġ",
"le"
],
[
"Ġ",
"de"
],
[
"Ġ",
"low"
],
[
"Ġ",
"for"
],
[
"Ġ",
"qu"
],
[
"Ġ",
"tar"
],
[
"Ġi",
"n"
],
[
"Ġ<",
"|"
],
[
"ft",
"ext"
],
[
"do",
"ftext"
],
[
"en",
"doftext"
],
[
"ick",
"sor"
],
[
"sel",
"f"
],
[
"Ġde",
"f"
],
[
"Ġqu",
"icksor"
],
[
"Ġtar",
"get"
],
[
"Ġquicksor",
"t"
],
[
"/",
"/"
],
[
"N",
"o"
],
[
"_",
"_"
],
[
"d",
"le"
],
[
"i",
"n"
],
[
"s",
"e"
],
[
"Ġ",
"2"
],
[
"Ġ",
"el"
],
[
"Ġ",
"next"
],
[
"Ġ",
"val"
],
[
"Ġ",
"self"
],
[
"Ġ",
"//"
],
[
"Ġ=",
"="
],
[
"Ġ<",
"="
],
[
"igh",
"t"
],
[
"Ġmid",
"dle"
],
[
"Ġle",
"n"
],
[
"a",
"s"
],
[
"b",
"in"
],
[
"c",
"h"
],
[
"c",
"l"
],
[
"h",
"i"
],
[
"i",
"f"
],
[
"i",
"t"
],
[
"n",
"e"
],
[
"r",
"ight"
],
[
"w",
"hi"
],
[
"Ġ",
"("
],
[
"Ġ",
"0"
],
[
"Ġ",
">"
],
[
"Ġ",
"No"
],
[
"Ġ",
"__"
],
[
"Ġ",
"bin"
],
[
"Ġ",
"cl"
],
[
"Ġ",
"whi"
],
[
"ar",
"y"
],
[
"ar",
"ch"
],
[
"le",
"n"
],
[
"le",
"ft"
],
[
"Ġr",
"ight"
],
[
"Ġle",
"ft"
],
[
"No",
"ne"
],
[
"__",
"("
],
[
"in",
"it"
],
[
"se",
"arch"
],
[
"Ġel",
"se"
],
[
"Ġel",
"if"
],
[
"as",
"s"
],
[
"ĠNo",
"de"
],
[
"Ġbin",
"ary"
],
[
"Ġcl",
"ass"
],
[
"Ġwhi",
"le"
]
]
}
}