DuckyModel_V1 / tokenizer.json
AxisCommunity's picture
Upload 4 files
142808c verified
raw
history blame
22.1 kB
{
"version": "1.0",
"truncation": {
"direction": "Right",
"max_length": 128,
"strategy": "LongestFirst",
"stride": 0
},
"padding": {
"strategy": {
"Fixed": 128
},
"direction": "Right",
"pad_to_multiple_of": null,
"pad_id": 3,
"pad_type_id": 0,
"pad_token": "<pad>"
},
"added_tokens": [
{
"id": 0,
"content": "<unk>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<bos>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<eos>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "<pad>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
}
],
"pair": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
}
],
"special_tokens": {}
},
"decoder": null,
"model": {
"type": "BPE",
"dropout": null,
"unk_token": "<unk>",
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"<unk>": 0,
"<bos>": 1,
"<eos>": 2,
"<pad>": 3,
"'": 4,
"(": 5,
")": 6,
",": 7,
"-": 8,
".": 9,
":": 10,
"?": 11,
"H": 12,
"I": 13,
"J": 14,
"M": 15,
"O": 16,
"P": 17,
"R": 18,
"S": 19,
"T": 20,
"W": 21,
"a": 22,
"b": 23,
"c": 24,
"d": 25,
"e": 26,
"f": 27,
"h": 28,
"i": 29,
"k": 30,
"l": 31,
"m": 32,
"n": 33,
"o": 34,
"p": 35,
"r": 36,
"s": 37,
"t": 38,
"u": 39,
"v": 40,
"w": 41,
"x": 42,
"y": 43,
"£": 44,
"§": 45,
"¯": 46,
"°": 47,
"±": 48,
"²": 49,
"³": 50,
"´": 51,
"µ": 52,
"¶": 53,
"·": 54,
"¸": 55,
"¹": 56,
"º": 57,
"»": 58,
"¼": 59,
"½": 60,
"¾": 61,
"¿": 62,
"Ð": 63,
"Ñ": 64,
"Ò": 65,
"Ġ": 66,
"Ģ": 67,
"ģ": 68,
"Ĥ": 69,
"ĥ": 70,
"ħ": 71,
"Ī": 72,
"ĭ": 73,
"Į": 74,
"IJ": 75,
"ij": 76,
"Ĵ": 77,
"ĵ": 78,
"ĸ": 79,
"ļ": 80,
"Ľ": 81,
"ľ": 82,
"а": 83,
"ĠÐ": 84,
"аÐ": 85,
"Ñĭ": 86,
"о": 87,
"Ġa": 88,
"ÑĢ": 89,
"ÑĤ": 90,
"Ñĸ": 91,
"on": 92,
"л": 93,
"н": 94,
"аÑ": 95,
"е": 96,
"Ġc": 97,
"ct": 98,
"in": 99,
"Ġs": 100,
"Ġe": 101,
"le": 102,
"at": 103,
"..": 104,
"д": 105,
"ÑĪ": 106,
"ĠÒ": 107,
"Ñĸн": 108,
"ÑĪÑĸн": 109,
"ол": 110,
"Ġd": 111,
"Ġt": 112,
"or": 113,
"pt": 114,
"re": 115,
"se": 116,
"is": 117,
"ÑĤÐ": 118,
"an": 119,
"ÑĢаÐ": 120,
"fu": 121,
"Ġfu": 122,
"Ġfun": 123,
"и": 124,
"ÑĢи": 125,
"ect": 126,
"nect": 127,
"onnect": 128,
"Ġl": 129,
"ан": 130,
"ÑĭÑ": 131,
"¶Ð°Ñ": 132,
"ĠÑĸ": 133,
"ģаÑ": 134,
"ĠжаÑ": 135,
"ал": 136,
"де": 137,
"ĠÒĽ": 138,
"ÑĪÑĸнде": 139,
"ĠÑĸÑĪÑĸнде": 140,
"ģаÑĥ": 141,
"ĠжаÑģаÑĥ": 142,
"з": 143,
"Ġв": 144,
"th": 145,
"ou": 146,
"ĠH": 147,
"Ġin": 148,
"Ġex": 149,
"mp": 150,
"mple": 151,
"ion": 152,
"Ġcre": 153,
"ction": 154,
"ate": 155,
"Ġfunction": 156,
"Ġcreate": 157,
"Ġ..": 158,
"Ġ...": 159,
"ab": 160,
"ase": 161,
"Ġconnect": 162,
"atab": 163,
"Ġdatab": 164,
"Ġto": 165,
"Ġdatabase": 166,
"()": 167,
"od": 168,
"Ġm": 169,
"ce": 170,
"ry": 171,
"Ġtry": 172,
"cept": 173,
"Ġp": 174,
"Ġsor": 175,
"ist": 176,
"Ġlist": 177,
"Ġsort": 178,
"el": 179,
"Ġb": 180,
"lo": 181,
"dle": 182,
"han": 183,
"rr": 184,
"Ġhan": 185,
"Ġerr": 186,
"ors": 187,
"Ġhandle": 188,
"Ġerrors": 189,
"³Ð¾": 190,
"¼Ñĭ": 191,
"го": 192,
"IJл": 193,
"ĠÐIJл": 194,
"ÑĤмÑĭ": 195,
"ÑĢиÑĤмÑĭ": 196,
"гоÑĢиÑĤмÑĭ": 197,
"ĠÐIJлгоÑĢиÑĤмÑĭ": 198,
"Ja": 199,
"Sc": 200,
"aSc": 201,
"ipt": 202,
"ript": 203,
"vaSc": 204,
"ĠJa": 205,
"vaScript": 206,
"ĠJavaScript": 207,
"Py": 208,
"ĠPy": 209,
"thon": 210,
"ĠPython": 211,
"We": 212,
"±Ð¾": 213,
"·ÑĢаÐ": 214,
"ºÐ°": 215,
"ĠWe": 216,
"ÑĤка": 217,
"ÑĢазÑĢаÐ": 218,
"±Ð¾ÑĤка": 219,
"ĠWeb": 220,
"ÑĢазÑĢабоÑĤка": 221,
"´Ð°Ð½": 222,
"·Ñĭ": 223,
"ijаÐ": 224,
"Ġдан": 225,
"ĠÐijаÐ": 226,
"нÑĭÑ": 227,
"ĠданнÑĭÑ": 228,
"ĠÐijазÑĭ": 229,
"ĠданнÑĭÑħ": 230,
"£Ñĭ": 231,
"¯ÑĪÑĸн": 232,
"±Ð¾Ð»": 233,
"´Ñĭ": 234,
"Òĵ": 235,
"Ò£Ñĭ": 236,
"ģал": 237,
"ľÑĭÑ": 238,
"Ġбол": 239,
"ĠÐľÑĭÑ": 240,
"ай": 241,
"адÑĭ": 242,
"ÑĭÒ£Ñĭ": 243,
"дан": 244,
"ĠÒ¯ÑĪÑĸн": 245,
"олдан": 246,
"алай": 247,
"ĠÒĽÐ¾Ð»Ð´Ð°Ð½": 248,
"ĠÒĽÐ°Ð»Ð°Ð¹": 249,
"ĠжаÑģаÑĥÒĵ": 250,
"ĠболадÑĭ": 251,
"ĠÐľÑĭÑģал": 252,
"ÑĭÒ£Ñĭз": 253,
"ĠÒĽÐ¾Ð»Ð´Ð°Ð½ÑĭÒ£Ñĭз": 254,
"ĠжаÑģаÑĥÒĵа": 255,
"§ÑĤ": 256,
"±Ñĭ": 257,
"¸Ñ": 258,
"¹ÑĤ": 259,
"¼Ðµ": 260,
"¿Ð¾Ð»": 261,
"¿ÑĢи": 262,
"бÑĭ": 263,
"йÑĤ": 264,
"ме": 265,
"пол": 266,
"Ñĥ": 267,
"ÑĮ": 268,
"ģпол": 269,
"Ĵо": 270,
"ļаÐ": 271,
"ĠЧÑĤ": 272,
"ĠиÑ": 273,
"ĠпÑĢи": 274,
"ĠÐĴо": 275,
"ĠÐļаÐ": 276,
"обÑĭ": 277,
"зÑĥ": 278,
"йÑĤе": 279,
"меÑĢ": 280,
"ÑĮзÑĥ": 281,
"ģполÑĮзÑĥ": 282,
"ĠЧÑĤобÑĭ": 283,
"ĠиÑģполÑĮзÑĥ": 284,
"ĠпÑĢимеÑĢ": 285,
"ĠÐĴоÑĤ": 286,
"ĠÐļак": 287,
"ĠиÑģполÑĮзÑĥйÑĤе": 288,
"To": 289,
"ample": 290,
"ere": 291,
"hou": 292,
"ld": 293,
"ow": 294,
"use": 295,
"you": 296,
"ĠI": 297,
"Ġis": 298,
"ĠTo": 299,
"Ġuse": 300,
"Ġyou": 301,
"Ġan": 302,
"Ġcan": 303,
"Ġshou": 304,
"ĠHere": 305,
"ĠHow": 306,
"Ġexample": 307,
"Ġshould": 308,
"as": 309,
"ef": 310,
"Ġdef": 311,
"Ġfunc": 312,
"():": 313,
"Ġpas": 314,
"Ġpass": 315,
"Ġexcept": 316,
"OR": 317,
"ĠOR": 318,
"odel": 319,
"Ġmodel": 320,
"ĠORM": 321,
"Ġmodels": 322,
"imple": 323,
"oo": 324,
"Ġsimple": 325,
"Ġloo": 326,
"Ġloop": 327,
"connect": 328,
"Ġdb": 329,
"eth": 330,
"il": 331,
"uil": 332,
"ods": 333,
"Ġmeth": 334,
"Ġbuil": 335,
"Ġmethods": 336,
"Ġbuilt": 337,
"ck": 338,
"ex": 339,
"Ġblo": 340,
"cks": 341,
"except": 342,
"Ġblocks": 343,
"')": 344,
"('": 345,
"Hel": 346,
"rin": 347,
"Ġprin": 348,
"Hello": 349,
"Ġprint": 350
},
"merges": [
[
"Ð",
"°"
],
[
"Ġ",
"Ð"
],
[
"а",
"Ð"
],
[
"Ñ",
"ĭ"
],
[
"Ð",
"¾"
],
[
"Ġ",
"a"
],
[
"Ñ",
"Ģ"
],
[
"Ñ",
"Ĥ"
],
[
"Ñ",
"ĸ"
],
[
"o",
"n"
],
[
"Ð",
"»"
],
[
"Ð",
"½"
],
[
"а",
"Ñ"
],
[
"Ð",
"µ"
],
[
"Ġ",
"c"
],
[
"c",
"t"
],
[
"i",
"n"
],
[
"Ġ",
"s"
],
[
"Ġ",
"e"
],
[
"l",
"e"
],
[
"a",
"t"
],
[
".",
"."
],
[
"Ð",
"´"
],
[
"Ñ",
"Ī"
],
[
"Ġ",
"Ò"
],
[
"Ñĸ",
"н"
],
[
"ÑĪ",
"Ñĸн"
],
[
"о",
"л"
],
[
"Ġ",
"d"
],
[
"Ġ",
"t"
],
[
"o",
"r"
],
[
"p",
"t"
],
[
"r",
"e"
],
[
"s",
"e"
],
[
"i",
"s"
],
[
"ÑĤ",
"Ð"
],
[
"a",
"n"
],
[
"ÑĢ",
"аÐ"
],
[
"f",
"u"
],
[
"Ġ",
"fu"
],
[
"Ġfu",
"n"
],
[
"Ð",
"¸"
],
[
"ÑĢ",
"и"
],
[
"e",
"ct"
],
[
"n",
"ect"
],
[
"on",
"nect"
],
[
"Ġ",
"l"
],
[
"аÐ",
"½"
],
[
"Ñĭ",
"Ñ"
],
[
"¶",
"аÑ"
],
[
"Ġ",
"Ñĸ"
],
[
"ģ",
"аÑ"
],
[
"ĠÐ",
"¶Ð°Ñ"
],
[
"аÐ",
"»"
],
[
"д",
"е"
],
[
"ĠÒ",
"Ľ"
],
[
"ÑĪÑĸн",
"де"
],
[
"ĠÑĸ",
"ÑĪÑĸнде"
],
[
"ģаÑ",
"ĥ"
],
[
"ĠжаÑ",
"ģаÑĥ"
],
[
"Ð",
"·"
],
[
"ĠÐ",
"²"
],
[
"t",
"h"
],
[
"o",
"u"
],
[
"Ġ",
"H"
],
[
"Ġ",
"in"
],
[
"Ġe",
"x"
],
[
"m",
"p"
],
[
"mp",
"le"
],
[
"i",
"on"
],
[
"Ġc",
"re"
],
[
"ct",
"ion"
],
[
"at",
"e"
],
[
"Ġfun",
"ction"
],
[
"Ġcre",
"ate"
],
[
"Ġ",
".."
],
[
"Ġ..",
"."
],
[
"a",
"b"
],
[
"a",
"se"
],
[
"Ġc",
"onnect"
],
[
"at",
"ab"
],
[
"Ġd",
"atab"
],
[
"Ġt",
"o"
],
[
"Ġdatab",
"ase"
],
[
"(",
")"
],
[
"o",
"d"
],
[
"Ġ",
"m"
],
[
"c",
"e"
],
[
"r",
"y"
],
[
"Ġt",
"ry"
],
[
"ce",
"pt"
],
[
"Ġ",
"p"
],
[
"Ġs",
"or"
],
[
"is",
"t"
],
[
"Ġl",
"ist"
],
[
"Ġsor",
"t"
],
[
"e",
"l"
],
[
"Ġ",
"b"
],
[
"l",
"o"
],
[
"d",
"le"
],
[
"h",
"an"
],
[
"r",
"r"
],
[
"Ġ",
"han"
],
[
"Ġe",
"rr"
],
[
"or",
"s"
],
[
"Ġhan",
"dle"
],
[
"Ġerr",
"ors"
],
[
"³",
"о"
],
[
"¼",
"Ñĭ"
],
[
"Ð",
"³Ð¾"
],
[
"IJ",
"л"
],
[
"ĠÐ",
"IJл"
],
[
"ÑĤÐ",
"¼Ñĭ"
],
[
"ÑĢи",
"ÑĤмÑĭ"
],
[
"го",
"ÑĢиÑĤмÑĭ"
],
[
"ĠÐIJл",
"гоÑĢиÑĤмÑĭ"
],
[
"J",
"a"
],
[
"S",
"c"
],
[
"a",
"Sc"
],
[
"i",
"pt"
],
[
"r",
"ipt"
],
[
"v",
"aSc"
],
[
"Ġ",
"Ja"
],
[
"vaSc",
"ript"
],
[
"ĠJa",
"vaScript"
],
[
"P",
"y"
],
[
"Ġ",
"Py"
],
[
"th",
"on"
],
[
"ĠPy",
"thon"
],
[
"W",
"e"
],
[
"±",
"о"
],
[
"·",
"ÑĢаÐ"
],
[
"º",
"а"
],
[
"Ġ",
"We"
],
[
"ÑĤÐ",
"ºÐ°"
],
[
"ÑĢаÐ",
"·ÑĢаÐ"
],
[
"±Ð¾",
"ÑĤка"
],
[
"ĠWe",
"b"
],
[
"ÑĢазÑĢаÐ",
"±Ð¾ÑĤка"
],
[
"´",
"ан"
],
[
"·",
"Ñĭ"
],
[
"ij",
"аÐ"
],
[
"ĠÐ",
"´Ð°Ð½"
],
[
"ĠÐ",
"ijаÐ"
],
[
"н",
"ÑĭÑ"
],
[
"Ġдан",
"нÑĭÑ"
],
[
"ĠÐijаÐ",
"·Ñĭ"
],
[
"ĠданнÑĭÑ",
"ħ"
],
[
"£",
"Ñĭ"
],
[
"¯",
"ÑĪÑĸн"
],
[
"±",
"ол"
],
[
"´",
"Ñĭ"
],
[
"Ò",
"ĵ"
],
[
"Ò",
"£Ñĭ"
],
[
"ģ",
"ал"
],
[
"ľ",
"ÑĭÑ"
],
[
"ĠÐ",
"±Ð¾Ð»"
],
[
"ĠÐ",
"ľÑĭÑ"
],
[
"аÐ",
"¹"
],
[
"аÐ",
"´Ñĭ"
],
[
"Ñĭ",
"Ò£Ñĭ"
],
[
"д",
"ан"
],
[
"ĠÒ",
"¯ÑĪÑĸн"
],
[
"ол",
"дан"
],
[
"ал",
"ай"
],
[
"ĠÒĽ",
"олдан"
],
[
"ĠÒĽ",
"алай"
],
[
"ĠжаÑģаÑĥ",
"Òĵ"
],
[
"Ġбол",
"адÑĭ"
],
[
"ĠÐľÑĭÑ",
"ģал"
],
[
"ÑĭÒ£Ñĭ",
"з"
],
[
"ĠÒĽÐ¾Ð»Ð´Ð°Ð½",
"ÑĭÒ£Ñĭз"
],
[
"ĠжаÑģаÑĥÒĵ",
"а"
],
[
"§",
"ÑĤ"
],
[
"±",
"Ñĭ"
],
[
"¸",
"Ñ"
],
[
"¹",
"ÑĤ"
],
[
"¼",
"е"
],
[
"¿",
"ол"
],
[
"¿",
"ÑĢи"
],
[
"Ð",
"±Ñĭ"
],
[
"Ð",
"¹ÑĤ"
],
[
"Ð",
"¼Ðµ"
],
[
"Ð",
"¿Ð¾Ð»"
],
[
"Ñ",
"ĥ"
],
[
"Ñ",
"Į"
],
[
"ģ",
"пол"
],
[
"Ĵ",
"о"
],
[
"ļ",
"аÐ"
],
[
"ĠÐ",
"§ÑĤ"
],
[
"ĠÐ",
"¸Ñ"
],
[
"ĠÐ",
"¿ÑĢи"
],
[
"ĠÐ",
"Ĵо"
],
[
"ĠÐ",
"ļаÐ"
],
[
"о",
"бÑĭ"
],
[
"з",
"Ñĥ"
],
[
"йÑĤ",
"е"
],
[
"ме",
"ÑĢ"
],
[
"ÑĮ",
"зÑĥ"
],
[
"ģпол",
"ÑĮзÑĥ"
],
[
"ĠЧÑĤ",
"обÑĭ"
],
[
"ĠиÑ",
"ģполÑĮзÑĥ"
],
[
"ĠпÑĢи",
"меÑĢ"
],
[
"ĠÐĴо",
"ÑĤ"
],
[
"ĠÐļаÐ",
"º"
],
[
"ĠиÑģполÑĮзÑĥ",
"йÑĤе"
],
[
"T",
"o"
],
[
"a",
"mple"
],
[
"e",
"re"
],
[
"h",
"ou"
],
[
"l",
"d"
],
[
"o",
"w"
],
[
"u",
"se"
],
[
"y",
"ou"
],
[
"Ġ",
"I"
],
[
"Ġ",
"is"
],
[
"Ġ",
"To"
],
[
"Ġ",
"use"
],
[
"Ġ",
"you"
],
[
"Ġa",
"n"
],
[
"Ġc",
"an"
],
[
"Ġs",
"hou"
],
[
"ĠH",
"ere"
],
[
"ĠH",
"ow"
],
[
"Ġex",
"ample"
],
[
"Ġshou",
"ld"
],
[
"a",
"s"
],
[
"e",
"f"
],
[
"Ġd",
"ef"
],
[
"Ġfun",
"c"
],
[
"()",
":"
],
[
"Ġp",
"as"
],
[
"Ġpas",
"s"
],
[
"Ġex",
"cept"
],
[
"O",
"R"
],
[
"Ġ",
"OR"
],
[
"od",
"el"
],
[
"Ġm",
"odel"
],
[
"ĠOR",
"M"
],
[
"Ġmodel",
"s"
],
[
"i",
"mple"
],
[
"o",
"o"
],
[
"Ġs",
"imple"
],
[
"Ġl",
"oo"
],
[
"Ġloo",
"p"
],
[
"c",
"onnect"
],
[
"Ġd",
"b"
],
[
"e",
"th"
],
[
"i",
"l"
],
[
"u",
"il"
],
[
"od",
"s"
],
[
"Ġm",
"eth"
],
[
"Ġb",
"uil"
],
[
"Ġmeth",
"ods"
],
[
"Ġbuil",
"t"
],
[
"c",
"k"
],
[
"e",
"x"
],
[
"Ġb",
"lo"
],
[
"ck",
"s"
],
[
"ex",
"cept"
],
[
"Ġblo",
"cks"
],
[
"'",
")"
],
[
"(",
"'"
],
[
"H",
"el"
],
[
"r",
"in"
],
[
"Ġp",
"rin"
],
[
"Hel",
"lo"
],
[
"Ġprin",
"t"
]
]
}
}