Upload tokenizer

Browse files

Files changed (4) hide show

special_tokens_map.json +7 -0
tokenizer.json +316 -306
tokenizer_config.json +10 -1
vocab.json +1 -1

special_tokens_map.json CHANGED Viewed

@@ -19,6 +19,13 @@
     "rstrip": false,
     "single_word": false
   },
   "unk_token": {
     "content": "UNK",
     "lstrip": false,

     "rstrip": false,
     "single_word": false
   },
+  "pad_token": {
+    "content": "PAD",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "unk_token": {
     "content": "UNK",
     "lstrip": false,

tokenizer.json CHANGED Viewed

@@ -59,7 +59,7 @@
     },
     {
       "id": 6,
-      "content": "EoT",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -68,6 +68,15 @@
     },
     {
       "id": 7,
       "content": "BoT",
       "single_word": false,
       "lstrip": false,
@@ -111,311 +120,312 @@
       "BoS": 3,
       "EoS": 4,
       "UNK": 5,
-      "EoT": 6,
-      "BoT": 7,
-      "!": 8,
-      "\"": 9,
-      "#": 10,
-      "$": 11,
-      "%": 12,
-      "&": 13,
-      "'": 14,
-      "(": 15,
-      ")": 16,
-      "*": 17,
-      "+": 18,
-      ",": 19,
-      "-": 20,
-      ".": 21,
-      "/": 22,
-      "0": 23,
-      "1": 24,
-      "2": 25,
-      "3": 26,
-      "4": 27,
-      "5": 28,
-      "6": 29,
-      "7": 30,
-      "8": 31,
-      "9": 32,
-      ":": 33,
-      ";": 34,
-      "<": 35,
-      "=": 36,
-      ">": 37,
-      "?": 38,
-      "@": 39,
-      "A": 40,
-      "B": 41,
-      "C": 42,
-      "D": 43,
-      "E": 44,
-      "F": 45,
-      "G": 46,
-      "H": 47,
-      "I": 48,
-      "J": 49,
-      "K": 50,
-      "L": 51,
-      "M": 52,
-      "N": 53,
-      "O": 54,
-      "P": 55,
-      "Q": 56,
-      "R": 57,
-      "S": 58,
-      "T": 59,
-      "U": 60,
-      "V": 61,
-      "W": 62,
-      "X": 63,
-      "Y": 64,
-      "Z": 65,
-      "[": 66,
-      "\\": 67,
-      "]": 68,
-      "^": 69,
-      "_": 70,
-      "`": 71,
-      "a": 72,
-      "b": 73,
-      "c": 74,
-      "d": 75,
-      "e": 76,
-      "f": 77,
-      "g": 78,
-      "h": 79,
-      "i": 80,
-      "j": 81,
-      "k": 82,
-      "l": 83,
-      "m": 84,
-      "n": 85,
-      "o": 86,
-      "p": 87,
-      "q": 88,
-      "r": 89,
-      "s": 90,
-      "t": 91,
-      "u": 92,
-      "v": 93,
-      "w": 94,
-      "x": 95,
-      "y": 96,
-      "z": 97,
-      "{": 98,
-      "|": 99,
-      "}": 100,
-      "~": 101,
-      "¡": 102,
-      "¢": 103,
-      "£": 104,
-      "¤": 105,
-      "¥": 106,
-      "¦": 107,
-      "§": 108,
-      "¨": 109,
-      "©": 110,
-      "ª": 111,
-      "«": 112,
-      "¬": 113,
-      "®": 114,
-      "¯": 115,
-      "°": 116,
-      "±": 117,
-      "²": 118,
-      "³": 119,
-      "´": 120,
-      "µ": 121,
-      "¶": 122,
-      "·": 123,
-      "¸": 124,
-      "¹": 125,
-      "º": 126,
-      "»": 127,
-      "¼": 128,
-      "½": 129,
-      "¾": 130,
-      "¿": 131,
-      "À": 132,
-      "Á": 133,
-      "Â": 134,
-      "Ã": 135,
-      "Ä": 136,
-      "Å": 137,
-      "Æ": 138,
-      "Ç": 139,
-      "È": 140,
-      "É": 141,
-      "Ê": 142,
-      "Ë": 143,
-      "Ì": 144,
-      "Í": 145,
-      "Î": 146,
-      "Ï": 147,
-      "Ð": 148,
-      "Ñ": 149,
-      "Ò": 150,
-      "Ó": 151,
-      "Ô": 152,
-      "Õ": 153,
-      "Ö": 154,
-      "×": 155,
-      "Ø": 156,
-      "Ù": 157,
-      "Ú": 158,
-      "Û": 159,
-      "Ü": 160,
-      "Ý": 161,
-      "Þ": 162,
-      "ß": 163,
-      "à": 164,
-      "á": 165,
-      "â": 166,
-      "ã": 167,
-      "ä": 168,
-      "å": 169,
-      "æ": 170,
-      "ç": 171,
-      "è": 172,
-      "é": 173,
-      "ê": 174,
-      "ë": 175,
-      "ì": 176,
-      "í": 177,
-      "î": 178,
-      "ï": 179,
-      "ð": 180,
-      "ñ": 181,
-      "ò": 182,
-      "ó": 183,
-      "ô": 184,
-      "õ": 185,
-      "ö": 186,
-      "÷": 187,
-      "ø": 188,
-      "ù": 189,
-      "ú": 190,
-      "û": 191,
-      "ü": 192,
-      "ý": 193,
-      "þ": 194,
-      "ÿ": 195,
-      "Ā": 196,
-      "ā": 197,
-      "Ă": 198,
-      "ă": 199,
-      "Ą": 200,
-      "ą": 201,
-      "Ć": 202,
-      "ć": 203,
-      "Ĉ": 204,
-      "ĉ": 205,
-      "Ċ": 206,
-      "ċ": 207,
-      "Č": 208,
-      "č": 209,
-      "Ď": 210,
-      "ď": 211,
-      "Đ": 212,
-      "đ": 213,
-      "Ē": 214,
-      "ē": 215,
-      "Ĕ": 216,
-      "ĕ": 217,
-      "Ė": 218,
-      "ė": 219,
-      "Ę": 220,
-      "ę": 221,
-      "Ě": 222,
-      "ě": 223,
-      "Ĝ": 224,
-      "ĝ": 225,
-      "Ğ": 226,
-      "ğ": 227,
-      "Ġ": 228,
-      "ġ": 229,
-      "Ģ": 230,
-      "ģ": 231,
-      "Ĥ": 232,
-      "ĥ": 233,
-      "Ħ": 234,
-      "ħ": 235,
-      "Ĩ": 236,
-      "ĩ": 237,
-      "Ī": 238,
-      "ī": 239,
-      "Ĭ": 240,
-      "ĭ": 241,
-      "Į": 242,
-      "į": 243,
-      "İ": 244,
-      "ı": 245,
-      "Ĳ": 246,
-      "ĳ": 247,
-      "Ĵ": 248,
-      "ĵ": 249,
-      "Ķ": 250,
-      "ķ": 251,
-      "ĸ": 252,
-      "Ĺ": 253,
-      "ĺ": 254,
-      "Ļ": 255,
-      "ļ": 256,
-      "Ľ": 257,
-      "ľ": 258,
-      "Ŀ": 259,
-      "ŀ": 260,
-      "Ł": 261,
-      "ł": 262,
-      "Ń": 263,
-      "Ġn": 264,
-      "Ġ|": 265,
-      "ah": 266,
-      "Ġah": 267,
-      "Ġaha": 268,
-      "Ġ1": 269,
-      "Ġ4": 270,
-      "Ġ3": 271,
-      "Ġ2": 272,
-      "ai": 273,
-      "wai": 274,
-      "Ġwai": 275,
-      "Ġwait": 276,
-      "10": 277,
-      "Ġ[": 278,
-      "Ġ]": 279,
-      "Ġl": 280,
-      "11": 281,
-      "12": 282,
-      "Ġ6": 283,
-      "Ġ7": 284,
-      "Bo": 285,
-      "Eo": 286,
-      "ĠEo": 287,
-      "Ġ5": 288,
-      "Ġ8": 289,
-      "13": 290,
-      "Ġ9": 291,
-      "Ġ10": 292,
-      "ĠBo": 293,
-      "ĠEoS": 294,
-      "ĠEoT": 295,
-      "ĠBoT": 296,
-      "14": 297,
-      "Ġ11": 298,
-      "Ġ12": 299,
-      "15": 300,
-      "Ġ13": 301,
-      "Ġ14": 302,
-      "16": 303,
-      "Ġ15": 304,
-      "Ġ16": 305,
-      "17": 306,
-      "Ġ17": 307,
-      "Ġ18": 308,
-      "Ġ19": 309,
-      "Ġ20": 310
     },
     "merges": [
       [

     },
     {
       "id": 6,
+      "content": "PAD",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
     },
     {
       "id": 7,
+      "content": "EoT",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 8,
       "content": "BoT",
       "single_word": false,
       "lstrip": false,
       "BoS": 3,
       "EoS": 4,
       "UNK": 5,
+      "PAD": 6,
+      "EoT": 7,
+      "BoT": 8,
+      "!": 9,
+      "\"": 10,
+      "#": 11,
+      "$": 12,
+      "%": 13,
+      "&": 14,
+      "'": 15,
+      "(": 16,
+      ")": 17,
+      "*": 18,
+      "+": 19,
+      ",": 20,
+      "-": 21,
+      ".": 22,
+      "/": 23,
+      "0": 24,
+      "1": 25,
+      "2": 26,
+      "3": 27,
+      "4": 28,
+      "5": 29,
+      "6": 30,
+      "7": 31,
+      "8": 32,
+      "9": 33,
+      ":": 34,
+      ";": 35,
+      "<": 36,
+      "=": 37,
+      ">": 38,
+      "?": 39,
+      "@": 40,
+      "A": 41,
+      "B": 42,
+      "C": 43,
+      "D": 44,
+      "E": 45,
+      "F": 46,
+      "G": 47,
+      "H": 48,
+      "I": 49,
+      "J": 50,
+      "K": 51,
+      "L": 52,
+      "M": 53,
+      "N": 54,
+      "O": 55,
+      "P": 56,
+      "Q": 57,
+      "R": 58,
+      "S": 59,
+      "T": 60,
+      "U": 61,
+      "V": 62,
+      "W": 63,
+      "X": 64,
+      "Y": 65,
+      "Z": 66,
+      "[": 67,
+      "\\": 68,
+      "]": 69,
+      "^": 70,
+      "_": 71,
+      "`": 72,
+      "a": 73,
+      "b": 74,
+      "c": 75,
+      "d": 76,
+      "e": 77,
+      "f": 78,
+      "g": 79,
+      "h": 80,
+      "i": 81,
+      "j": 82,
+      "k": 83,
+      "l": 84,
+      "m": 85,
+      "n": 86,
+      "o": 87,
+      "p": 88,
+      "q": 89,
+      "r": 90,
+      "s": 91,
+      "t": 92,
+      "u": 93,
+      "v": 94,
+      "w": 95,
+      "x": 96,
+      "y": 97,
+      "z": 98,
+      "{": 99,
+      "|": 100,
+      "}": 101,
+      "~": 102,
+      "¡": 103,
+      "¢": 104,
+      "£": 105,
+      "¤": 106,
+      "¥": 107,
+      "¦": 108,
+      "§": 109,
+      "¨": 110,
+      "©": 111,
+      "ª": 112,
+      "«": 113,
+      "¬": 114,
+      "®": 115,
+      "¯": 116,
+      "°": 117,
+      "±": 118,
+      "²": 119,
+      "³": 120,
+      "´": 121,
+      "µ": 122,
+      "¶": 123,
+      "·": 124,
+      "¸": 125,
+      "¹": 126,
+      "º": 127,
+      "»": 128,
+      "¼": 129,
+      "½": 130,
+      "¾": 131,
+      "¿": 132,
+      "À": 133,
+      "Á": 134,
+      "Â": 135,
+      "Ã": 136,
+      "Ä": 137,
+      "Å": 138,
+      "Æ": 139,
+      "Ç": 140,
+      "È": 141,
+      "É": 142,
+      "Ê": 143,
+      "Ë": 144,
+      "Ì": 145,
+      "Í": 146,
+      "Î": 147,
+      "Ï": 148,
+      "Ð": 149,
+      "Ñ": 150,
+      "Ò": 151,
+      "Ó": 152,
+      "Ô": 153,
+      "Õ": 154,
+      "Ö": 155,
+      "×": 156,
+      "Ø": 157,
+      "Ù": 158,
+      "Ú": 159,
+      "Û": 160,
+      "Ü": 161,
+      "Ý": 162,
+      "Þ": 163,
+      "ß": 164,
+      "à": 165,
+      "á": 166,
+      "â": 167,
+      "ã": 168,
+      "ä": 169,
+      "å": 170,
+      "æ": 171,
+      "ç": 172,
+      "è": 173,
+      "é": 174,
+      "ê": 175,
+      "ë": 176,
+      "ì": 177,
+      "í": 178,
+      "î": 179,
+      "ï": 180,
+      "ð": 181,
+      "ñ": 182,
+      "ò": 183,
+      "ó": 184,
+      "ô": 185,
+      "õ": 186,
+      "ö": 187,
+      "÷": 188,
+      "ø": 189,
+      "ù": 190,
+      "ú": 191,
+      "û": 192,
+      "ü": 193,
+      "ý": 194,
+      "þ": 195,
+      "ÿ": 196,
+      "Ā": 197,
+      "ā": 198,
+      "Ă": 199,
+      "ă": 200,
+      "Ą": 201,
+      "ą": 202,
+      "Ć": 203,
+      "ć": 204,
+      "Ĉ": 205,
+      "ĉ": 206,
+      "Ċ": 207,
+      "ċ": 208,
+      "Č": 209,
+      "č": 210,
+      "Ď": 211,
+      "ď": 212,
+      "Đ": 213,
+      "đ": 214,
+      "Ē": 215,
+      "ē": 216,
+      "Ĕ": 217,
+      "ĕ": 218,
+      "Ė": 219,
+      "ė": 220,
+      "Ę": 221,
+      "ę": 222,
+      "Ě": 223,
+      "ě": 224,
+      "Ĝ": 225,
+      "ĝ": 226,
+      "Ğ": 227,
+      "ğ": 228,
+      "Ġ": 229,
+      "ġ": 230,
+      "Ģ": 231,
+      "ģ": 232,
+      "Ĥ": 233,
+      "ĥ": 234,
+      "Ħ": 235,
+      "ħ": 236,
+      "Ĩ": 237,
+      "ĩ": 238,
+      "Ī": 239,
+      "ī": 240,
+      "Ĭ": 241,
+      "ĭ": 242,
+      "Į": 243,
+      "į": 244,
+      "İ": 245,
+      "ı": 246,
+      "Ĳ": 247,
+      "ĳ": 248,
+      "Ĵ": 249,
+      "ĵ": 250,
+      "Ķ": 251,
+      "ķ": 252,
+      "ĸ": 253,
+      "Ĺ": 254,
+      "ĺ": 255,
+      "Ļ": 256,
+      "ļ": 257,
+      "Ľ": 258,
+      "ľ": 259,
+      "Ŀ": 260,
+      "ŀ": 261,
+      "Ł": 262,
+      "ł": 263,
+      "Ń": 264,
+      "Ġn": 265,
+      "Ġ|": 266,
+      "ah": 267,
+      "Ġah": 268,
+      "Ġaha": 269,
+      "Ġ1": 270,
+      "Ġ4": 271,
+      "Ġ3": 272,
+      "Ġ2": 273,
+      "ai": 274,
+      "wai": 275,
+      "Ġwai": 276,
+      "Ġwait": 277,
+      "10": 278,
+      "Ġ[": 279,
+      "Ġ]": 280,
+      "Ġl": 281,
+      "11": 282,
+      "12": 283,
+      "Ġ6": 284,
+      "Ġ7": 285,
+      "Bo": 286,
+      "Eo": 287,
+      "ĠEo": 288,
+      "Ġ5": 289,
+      "Ġ8": 290,
+      "13": 291,
+      "Ġ9": 292,
+      "Ġ10": 293,
+      "ĠBo": 294,
+      "ĠEoS": 295,
+      "ĠEoT": 296,
+      "ĠBoT": 297,
+      "14": 298,
+      "Ġ11": 299,
+      "Ġ12": 300,
+      "15": 301,
+      "Ġ13": 302,
+      "Ġ14": 303,
+      "16": 304,
+      "Ġ15": 305,
+      "Ġ16": 306,
+      "17": 307,
+      "Ġ17": 308,
+      "Ġ18": 309,
+      "Ġ19": 310,
+      "Ġ20": 311
     },
     "merges": [
       [

tokenizer_config.json CHANGED Viewed

@@ -50,7 +50,7 @@
       "special": true
     },
     "6": {
-      "content": "EoT",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -58,6 +58,14 @@
       "special": true
     },
     "7": {
       "content": "BoT",
       "lstrip": false,
       "normalized": false,
@@ -77,6 +85,7 @@
   "eos_token": "EoS",
   "extra_special_tokens": {},
   "model_max_length": 1024,
   "tokenizer_class": "GPT2Tokenizer",
   "unk_token": "UNK"
 }

       "special": true
     },
     "6": {
+      "content": "PAD",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "special": true
     },
     "7": {
+      "content": "EoT",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
       "content": "BoT",
       "lstrip": false,
       "normalized": false,
   "eos_token": "EoS",
   "extra_special_tokens": {},
   "model_max_length": 1024,
+  "pad_token": "PAD",
   "tokenizer_class": "GPT2Tokenizer",
   "unk_token": "UNK"
 }

vocab.json CHANGED Viewed

@@ -1 +1 @@

- {"aha":0,"wait":1,"<|endoftext|>":2,"BoS":3,"EoS":4,"UNK":5,"~~EoT~~":6,"~~BoT~~":7,"!":8,"\"":9,"#":10,"$":11,"%":12,"&":13,"'":14,"(":15,")":16,"*":17,"+":18,",":19,"-":20,".":21,"/":22,"0":23,"1":24,"2":25,"3":26,"4":27,"5":28,"6":29,"7":30,"8":31,"9":32,":":33,";":34,"<":35,"=":36,">":37,"?":38,"@":39,"A":40,"B":41,"C":42,"D":43,"E":44,"F":45,"G":46,"H":47,"I":48,"J":49,"K":50,"L":51,"M":52,"N":53,"O":54,"P":55,"Q":56,"R":57,"S":58,"T":59,"U":60,"V":61,"W":62,"X":63,"Y":64,"Z":65,"[":66,"\\":67,"]":68,"^":69,"_":70,"`":71,"a":72,"b":73,"c":74,"d":75,"e":76,"f":77,"g":78,"h":79,"i":80,"j":81,"k":82,"l":83,"m":84,"n":85,"o":86,"p":87,"q":88,"r":89,"s":90,"t":91,"u":92,"v":93,"w":94,"x":95,"y":96,"z":97,"{":98,"|":99,"}":~~100~~,"~":~~101~~,"¡":~~102~~,"¢":~~103~~,"£":~~104~~,"¤":~~105~~,"¥":~~106~~,"¦":~~107~~,"§":~~108~~,"¨":~~109~~,"©":~~110~~,"ª":~~111~~,"«":~~112~~,"¬":~~113~~,"®":~~114~~,"¯":~~115~~,"°":~~116~~,"±":~~117~~,"²":~~118~~,"³":~~119~~,"´":~~120~~,"µ":~~121~~,"¶":~~122~~,"·":~~123~~,"¸":~~124~~,"¹":~~125~~,"º":~~126~~,"»":~~127~~,"¼":~~128~~,"½":~~129~~,"¾":~~130~~,"¿":~~131~~,"À":~~132~~,"Á":~~133~~,"Â":~~134~~,"Ã":~~135~~,"Ä":~~136~~,"Å":~~137~~,"Æ":~~138~~,"Ç":~~139~~,"È":~~140~~,"É":~~141~~,"Ê":~~142~~,"Ë":~~143~~,"Ì":~~144~~,"Í":~~145~~,"Î":~~146~~,"Ï":~~147~~,"Ð":~~148~~,"Ñ":~~149~~,"Ò":~~150~~,"Ó":~~151~~,"Ô":~~152~~,"Õ":~~153~~,"Ö":~~154~~,"×":~~155~~,"Ø":~~156~~,"Ù":~~157~~,"Ú":~~158~~,"Û":~~159~~,"Ü":~~160~~,"Ý":~~161~~,"Þ":~~162~~,"ß":~~163~~,"à":~~164~~,"á":~~165~~,"â":~~166~~,"ã":~~167~~,"ä":~~168~~,"å":~~169~~,"æ":~~170~~,"ç":~~171~~,"è":~~172~~,"é":~~173~~,"ê":~~174~~,"ë":~~175~~,"ì":~~176~~,"í":~~177~~,"î":~~178~~,"ï":~~179~~,"ð":~~180~~,"ñ":~~181~~,"ò":~~182~~,"ó":~~183~~,"ô":~~184~~,"õ":~~185~~,"ö":~~186~~,"÷":~~187~~,"ø":~~188~~,"ù":~~189~~,"ú":~~190~~,"û":~~191~~,"ü":~~192~~,"ý":~~193~~,"þ":~~194~~,"ÿ":~~195~~,"Ā":~~196~~,"ā":~~197~~,"Ă":~~198~~,"ă":~~199~~,"Ą":~~200~~,"ą":~~201~~,"Ć":~~202~~,"ć":~~203~~,"Ĉ":~~204~~,"ĉ":~~205~~,"Ċ":~~206~~,"ċ":~~207~~,"Č":~~208~~,"č":~~209~~,"Ď":~~210~~,"ď":~~211~~,"Đ":~~212~~,"đ":~~213~~,"Ē":~~214~~,"ē":~~215~~,"Ĕ":~~216~~,"ĕ":~~217~~,"Ė":~~218~~,"ė":~~219~~,"Ę":~~220~~,"ę":~~221~~,"Ě":~~222~~,"ě":~~223~~,"Ĝ":~~224~~,"ĝ":~~225~~,"Ğ":~~226~~,"ğ":~~227~~,"Ġ":~~228~~,"ġ":~~229~~,"Ģ":~~230~~,"ģ":~~231~~,"Ĥ":~~232~~,"ĥ":~~233~~,"Ħ":~~234~~,"ħ":~~235~~,"Ĩ":~~236~~,"ĩ":~~237~~,"Ī":~~238~~,"ī":~~239~~,"Ĭ":~~240~~,"ĭ":~~241~~,"Į":~~242~~,"į":~~243~~,"İ":~~244~~,"ı":~~245~~,"Ĳ":~~246~~,"ĳ":~~247~~,"Ĵ":~~248~~,"ĵ":~~249~~,"Ķ":~~250~~,"ķ":~~251~~,"ĸ":~~252~~,"Ĺ":~~253~~,"ĺ":~~254~~,"Ļ":~~255~~,"ļ":~~256~~,"Ľ":~~257~~,"ľ":~~258~~,"Ŀ":~~259~~,"ŀ":~~260~~,"Ł":~~261~~,"ł":~~262~~,"Ń":~~263~~,"Ġn":~~264~~,"Ġ|":~~265~~,"ah":~~266~~,"Ġah":~~267~~,"Ġaha":~~268~~,"Ġ1":~~269~~,"Ġ4":~~270~~,"Ġ3":~~271~~,"Ġ2":~~272~~,"ai":~~273~~,"wai":~~274~~,"Ġwai":~~275~~,"Ġwait":~~276~~,"10":~~277~~,"Ġ[":~~278~~,"Ġ]":~~279~~,"Ġl":~~280~~,"11":~~281~~,"12":~~282~~,"Ġ6":~~283~~,"Ġ7":~~284~~,"Bo":~~285~~,"Eo":~~286~~,"ĠEo":~~287~~,"Ġ5":~~288~~,"Ġ8":~~289~~,"13":~~290~~,"Ġ9":~~291~~,"Ġ10":~~292~~,"ĠBo":~~293~~,"ĠEoS":~~294~~,"ĠEoT":~~295~~,"ĠBoT":~~296~~,"14":~~297~~,"Ġ11":~~298~~,"Ġ12":~~299~~,"15":~~300~~,"Ġ13":~~301~~,"Ġ14":~~302~~,"16":~~303~~,"Ġ15":~~304~~,"Ġ16":~~305~~,"17":~~306~~,"Ġ17":~~307~~,"Ġ18":~~308~~,"Ġ19":~~309~~,"Ġ20":~~310~~}

+ {"aha":0,"wait":1,"<|endoftext|>":2,"BoS":3,"EoS":4,"UNK":5,"PAD":6,"EoT":7,"BoT":8,"!":9,"\"":10,"#":11,"$":12,"%":13,"&":14,"'":15,"(":16,")":17,"*":18,"+":19,",":20,"-":21,".":22,"/":23,"0":24,"1":25,"2":26,"3":27,"4":28,"5":29,"6":30,"7":31,"8":32,"9":33,":":34,";":35,"<":36,"=":37,">":38,"?":39,"@":40,"A":41,"B":42,"C":43,"D":44,"E":45,"F":46,"G":47,"H":48,"I":49,"J":50,"K":51,"L":52,"M":53,"N":54,"O":55,"P":56,"Q":57,"R":58,"S":59,"T":60,"U":61,"V":62,"W":63,"X":64,"Y":65,"Z":66,"[":67,"\\":68,"]":69,"^":70,"_":71,"`":72,"a":73,"b":74,"c":75,"d":76,"e":77,"f":78,"g":79,"h":80,"i":81,"j":82,"k":83,"l":84,"m":85,"n":86,"o":87,"p":88,"q":89,"r":90,"s":91,"t":92,"u":93,"v":94,"w":95,"x":96,"y":97,"z":98,"{":99,"|":100,"}":101,"~":102,"¡":103,"¢":104,"£":105,"¤":106,"¥":107,"¦":108,"§":109,"¨":110,"©":111,"ª":112,"«":113,"¬":114,"®":115,"¯":116,"°":117,"±":118,"²":119,"³":120,"´":121,"µ":122,"¶":123,"·":124,"¸":125,"¹":126,"º":127,"»":128,"¼":129,"½":130,"¾":131,"¿":132,"À":133,"Á":134,"Â":135,"Ã":136,"Ä":137,"Å":138,"Æ":139,"Ç":140,"È":141,"É":142,"Ê":143,"Ë":144,"Ì":145,"Í":146,"Î":147,"Ï":148,"Ð":149,"Ñ":150,"Ò":151,"Ó":152,"Ô":153,"Õ":154,"Ö":155,"×":156,"Ø":157,"Ù":158,"Ú":159,"Û":160,"Ü":161,"Ý":162,"Þ":163,"ß":164,"à":165,"á":166,"â":167,"ã":168,"ä":169,"å":170,"æ":171,"ç":172,"è":173,"é":174,"ê":175,"ë":176,"ì":177,"í":178,"î":179,"ï":180,"ð":181,"ñ":182,"ò":183,"ó":184,"ô":185,"õ":186,"ö":187,"÷":188,"ø":189,"ù":190,"ú":191,"û":192,"ü":193,"ý":194,"þ":195,"ÿ":196,"Ā":197,"ā":198,"Ă":199,"ă":200,"Ą":201,"ą":202,"Ć":203,"ć":204,"Ĉ":205,"ĉ":206,"Ċ":207,"ċ":208,"Č":209,"č":210,"Ď":211,"ď":212,"Đ":213,"đ":214,"Ē":215,"ē":216,"Ĕ":217,"ĕ":218,"Ė":219,"ė":220,"Ę":221,"ę":222,"Ě":223,"ě":224,"Ĝ":225,"ĝ":226,"Ğ":227,"ğ":228,"Ġ":229,"ġ":230,"Ģ":231,"ģ":232,"Ĥ":233,"ĥ":234,"Ħ":235,"ħ":236,"Ĩ":237,"ĩ":238,"Ī":239,"ī":240,"Ĭ":241,"ĭ":242,"Į":243,"į":244,"İ":245,"ı":246,"Ĳ":247,"ĳ":248,"Ĵ":249,"ĵ":250,"Ķ":251,"ķ":252,"ĸ":253,"Ĺ":254,"ĺ":255,"Ļ":256,"ļ":257,"Ľ":258,"ľ":259,"Ŀ":260,"ŀ":261,"Ł":262,"ł":263,"Ń":264,"Ġn":265,"Ġ|":266,"ah":267,"Ġah":268,"Ġaha":269,"Ġ1":270,"Ġ4":271,"Ġ3":272,"Ġ2":273,"ai":274,"wai":275,"Ġwai":276,"Ġwait":277,"10":278,"Ġ[":279,"Ġ]":280,"Ġl":281,"11":282,"12":283,"Ġ6":284,"Ġ7":285,"Bo":286,"Eo":287,"ĠEo":288,"Ġ5":289,"Ġ8":290,"13":291,"Ġ9":292,"Ġ10":293,"ĠBo":294,"ĠEoS":295,"ĠEoT":296,"ĠBoT":297,"14":298,"Ġ11":299,"Ġ12":300,"15":301,"Ġ13":302,"Ġ14":303,"16":304,"Ġ15":305,"Ġ16":306,"17":307,"Ġ17":308,"Ġ18":309,"Ġ19":310,"Ġ20":311}