Upload tokenizer

tokenizer.json

@@ -229,895 +229,894 @@
       "x": 53,
       "y": 54,
       "z": 55,
-      "Ġ": 56,
-      "Ġt": 57,
-      "Ġth": 58,
+      "Ċ": 56,
+      "Ġ": 57,
+      "Ġt": 58,
       "Ġa": 59,
-      "in": 60,
-      "Ġw": 61,
-      "Ġthe": 62,
-      "re": 63,
-      "un": 64,
-      "Ġs": 65,
-      "Ġo": 66,
-      "Ġi": 67,
-      "at": 68,
-      "unk": 69,
-      "er": 70,
-      "nd": 71,
-      "Ġc": 72,
+      "he": 60,
+      "in": 61,
+      "re": 62,
+      "Ġs": 63,
+      "Ġthe": 64,
+      "Ġo": 65,
+      "Ġw": 66,
+      "er": 67,
+      "nd": 68,
+      "Ġc": 69,
+      "at": 70,
+      "Ġb": 71,
+      "it": 72,
       "ou": 73,
-      "Ġb": 74,
-      "Ġm": 75,
-      "Ġh": 76,
+      "Ġf": 74,
+      "on": 75,
+      "Ġm": 76,
       "en": 77,
-      "ing": 78,
-      "Ġ<": 79,
-      "Ġto": 80,
-      "on": 81,
-      "Ġp": 82,
-      "Ġf": 83,
-      "Ġand": 84,
-      "Ġd": 85,
-      "es": 86,
-      "an": 87,
-      "or": 88,
+      "Ġp": 78,
+      "is": 79,
+      "ing": 80,
+      "Ġto": 81,
+      "es": 82,
+      "Ġth": 83,
+      "Ġh": 84,
+      "or": 85,
+      "an": 86,
+      "Ġd": 87,
+      "ed": 88,
       "Ġof": 89,
-      "al": 90,
+      "Ġand": 90,
       "Ġin": 91,
-      "Ġ'": 92,
-      "Ġl": 93,
-      "ed": 94,
-      "is": 95,
-      "it": 96,
-      "ar": 97,
-      "Ġy": 98,
-      "Ġthat": 99,
-      "Ġg": 100,
-      "om": 101,
-      "Ġwe": 102,
-      "Ġn": 103,
-      "as": 104,
-      "le": 105,
-      "ve": 106,
-      "ic": 107,
-      "Ġe": 108,
-      "Ġwh": 109,
-      "Ġyou": 110,
-      "th": 111,
-      "Ġit": 112,
-      "ll": 113,
-      "Ġbe": 114,
-      "us": 115,
-      "ly": 116,
+      "al": 92,
+      "ar": 93,
+      "Ġl": 94,
+      "as": 95,
+      "Ġn": 96,
+      "om": 97,
+      "ic": 98,
+      "Ġg": 99,
+      "Ġe": 100,
+      "le": 101,
+      "ve": 102,
+      "Ġbe": 103,
+      "ll": 104,
+      "ion": 105,
+      "ly": 106,
+      "ent": 107,
+      "ot": 108,
+      "ut": 109,
+      "us": 110,
+      "Ġre": 111,
+      "Ġi": 112,
+      "Ġon": 113,
+      "Ġthat": 114,
+      "Ġis": 115,
+      "Ġy": 116,
       "ow": 117,
-      "Ġis": 118,
-      "ut": 119,
-      "ion": 120,
-      "Ġon": 121,
-      "ent": 122,
-      "Ġre": 123,
-      "Ġha": 124,
-      "ct": 125,
-      "ro": 126,
-      "ld": 127,
-      "ver": 128,
-      "and": 129,
-      "ot": 130,
-      "Ġthis": 131,
-      "et": 132,
-      "se": 133,
-      "ay": 134,
-      "ig": 135,
-      "st": 136,
-      "Ġst": 137,
-      "id": 138,
-      "ur": 139,
-      "ch": 140,
-      "Ġso": 141,
-      "ke": 142,
-      "Ġfor": 143,
-      "ce": 144,
-      "ere": 145,
-      "Ġwas": 146,
-      "ir": 147,
-      "im": 148,
-      "Ġthey": 149,
-      "Ġan": 150,
-      "Ġdo": 151,
-      "ra": 152,
-      "Ġli": 153,
-      "ith": 154,
-      "Ġare": 155,
-      "op": 156,
-      "Ġse": 157,
-      "Ġgo": 158,
-      "ally": 159,
+      "et": 118,
+      "id": 119,
+      "Ġit": 120,
+      "ac": 121,
+      "ay": 122,
+      "Ġha": 123,
+      "ld": 124,
+      "the": 125,
+      "Ġhe": 126,
+      "Ġwe": 127,
+      "ro": 128,
+      "ver": 129,
+      "Ġst": 130,
+      "ig": 131,
+      "Ġyou": 132,
+      "st": 133,
+      "Ġfor": 134,
+      "Ġwh": 135,
+      "ir": 136,
+      "se": 137,
+      "ur": 138,
+      "im": 139,
+      "ct": 140,
+      "am": 141,
+      "ith": 142,
+      "and": 143,
+      "ke": 144,
+      "ad": 145,
+      "un": 146,
+      "Ġan": 147,
+      "'s": 148,
+      "Ġare": 149,
+      "Ġwith": 150,
+      "Ġas": 151,
+      "Ġr": 152,
+      "Ġwas": 153,
+      "ch": 154,
+      "Ġse": 155,
+      "th": 156,
+      "oo": 157,
+      "ce": 158,
+      "if": 159,
       "Ġmo": 160,
-      "Ġhave": 161,
-      "am": 162,
-      "ome": 163,
-      "Ġab": 164,
-      "Ġwith": 165,
-      "Ġk": 166,
-      "ht": 167,
-      "Ġcan": 168,
+      "ht": 161,
+      "op": 162,
+      "ill": 163,
+      "Ġhave": 164,
+      "ation": 165,
+      "ers": 166,
+      "Ġne": 167,
+      "il": 168,
       "ol": 169,
-      "und": 170,
-      "Ġas": 171,
-      "Ġhe": 172,
-      "Ġwhat": 173,
-      "out": 174,
-      "ation": 175,
-      "Ġne": 176,
-      "Ġwor": 177,
-      "ge": 178,
-      "Ġsh": 179,
-      "Ġj": 180,
-      "ad": 181,
-      "ould": 182,
-      "Ġme": 183,
-      "Ġch": 184,
+      "Ġnot": 170,
+      "Ġde": 171,
+      "Ġcom": 172,
+      "Ġfr": 173,
+      "ter": 174,
+      "red": 175,
+      "ul": 176,
+      "Ġdo": 177,
+      "Ġch": 178,
+      "ge": 179,
+      "ive": 180,
+      "ould": 181,
+      "ight": 182,
+      "Ġone": 183,
+      "Ġj": 184,
       "Ġat": 185,
-      "very": 186,
-      "Ġabout": 187,
-      "ill": 188,
-      "ant": 189,
-      "ers": 190,
-      "Ġpro": 191,
-      "Ġone": 192,
-      "ul": 193,
-      "Ġbut": 194,
-      "il": 195,
-      "ust": 196,
-      "Ġr": 197,
-      "ight": 198,
-      "Ġnot": 199,
-      "Ġall": 200,
-      "Ġfr": 201,
-      "Ġex": 202,
-      "Ġthere": 203,
-      "Ġmy": 204,
-      "if": 205,
-      "Ġus": 206,
-      "oo": 207,
-      "Ġcom": 208,
-      "ter": 209,
-      "Ġde": 210,
-      "Ġpe": 211,
-      "Ġor": 212,
-      "ive": 213,
-      "art": 214,
-      "Ġsome": 215,
-      "ther": 216,
-      "up": 217,
-      "Ġcon": 218,
-      "ople": 219,
-      "ear": 220,
-      "ies": 221,
-      "um": 222,
-      "now": 223,
-      "Ġpeople": 224,
-      "ess": 225,
-      "ck": 226,
-      "Ġlike": 227,
-      "ate": 228,
-      "ity": 229,
-      "ure": 230,
-      "Ġma": 231,
-      "Ġour": 232,
-      "so": 233,
-      "est": 234,
-      "ous": 235,
-      "ist": 236,
-      "ble": 237,
-      "Ġfrom": 238,
-      "Ġbec": 239,
-      "pe": 240,
-      "Ġal": 241,
-      "Ġv": 242,
-      "use": 243,
-      "all": 244,
-      "qu": 245,
-      "Ġte": 246,
-      "ind": 247,
-      "Ġout": 248,
-      "Ġif": 249,
-      "ause": 250,
-      "pp": 251,
-      "Ġint": 252,
-      "Ġjust": 253,
-      "ink": 254,
-      "red": 255,
-      "Ġtw": 256,
-      "Ġthese": 257,
-      "Ġco": 258,
-      "Ġtr": 259,
-      "ook": 260,
-      "Ġthing": 261,
-      "Ġthem": 262,
-      "Ġsee": 263,
-      "ie": 264,
-      "Ġknow": 265,
-      "oug": 266,
-      "ort": 267,
-      "od": 268,
-      "Ġup": 269,
-      "one": 270,
-      "os": 271,
-      "Ġwho": 272,
-      "Ġwhen": 273,
-      "Ġthink": 274,
-      "ci": 275,
-      "Ġpl": 276,
-      "ine": 277,
-      "Ġget": 278,
-      "our": 279,
-      "the": 280,
-      "Ġvery": 281,
-      "Ġby": 282,
-      "Ġbecause": 283,
-      "iv": 284,
-      "Ġhad": 285,
-      "Ġhow": 286,
-      "Ġyear": 287,
-      "el": 288,
-      "ich": 289,
-      "Ġmore": 290,
-      "Ġtim": 291,
-      "thing": 292,
-      "Ġgoing": 293,
-      "ect": 294,
-      "Ġim": 295,
-      "Ġact": 296,
-      "Ġyour": 297,
-      "ain": 298,
-      "Ġwere": 299,
-      "Ġwould": 300,
-      "em": 301,
-      "Ġnow": 302,
-      "Ġtheir": 303,
-      "Ġen": 304,
-      "og": 305,
-      "ment": 306,
-      "Ġsa": 307,
-      "ast": 308,
-      "Ġta": 309,
-      "ame": 310,
-      "Ġlook": 311,
-      "we": 312,
-      "act": 313,
-      "Ġwork": 314,
-      "Ġcomp": 315,
-      "Ġwant": 316,
-      "iz": 317,
-      "Ġreally": 318,
-      "Ġun": 319,
-      "Ġworld": 320,
-      "Ġle": 321,
-      "Ġway": 322,
-      "ack": 323,
-      "ually": 324,
-      "ven": 325,
-      "ose": 326,
-      "Ġtwo": 327,
-      "ard": 328,
-      "Ġwhich": 329,
-      "reat": 330,
-      "Ġtime": 331,
-      "ab": 332,
-      "Ġno": 333,
-      "Ġevery": 334,
-      "ty": 335,
-      "Ġhere": 336,
-      "are": 337,
-      "wh": 338,
-      "ap": 339,
-      "ag": 340,
-      "ans": 341,
-      "ough": 342,
-      "Ġman": 343,
-      "res": 344,
-      "Ġthen": 345,
-      "Ġhas": 346,
-      "Ġcould": 347,
-      "Ġper": 348,
+      "Ġor": 186,
+      "Ġcon": 187,
+      "Ġthey": 188,
+      "Ġsa": 189,
+      "Ġk": 190,
+      "ant": 191,
+      "ess": 192,
+      "Ġpro": 193,
+      "ous": 194,
+      "pe": 195,
+      "Ġcan": 196,
+      "Ġex": 197,
+      "Ġab": 198,
+      "Ġthis": 199,
+      "ain": 200,
+      "Ġu": 201,
+      "Ġsh": 202,
+      "Ġwor": 203,
+      "est": 204,
+      "ate": 205,
+      "Ġtr": 206,
+      "Ġv": 207,
+      "out": 208,
+      "ist": 209,
+      "ab": 210,
+      "ust": 211,
+      "ome": 212,
+      "Ġsu": 213,
+      "Ġme": 214,
+      "Ġtw": 215,
+      "ally": 216,
+      "Ġso": 217,
+      "Ġal": 218,
+      "ies": 219,
+      "pp": 220,
+      "her": 221,
+      "Ġgo": 222,
+      "Ġbut": 223,
+      "Ġli": 224,
+      "Ġwhe": 225,
+      "Ġfrom": 226,
+      "ity": 227,
+      "Ġsaid": 228,
+      "qu": 229,
+      "um": 230,
+      "our": 231,
+      "art": 232,
+      "'t": 233,
+      "Ġus": 234,
+      "res": 235,
+      "very": 236,
+      "ie": 237,
+      "all": 238,
+      "Ġall": 239,
+      "one": 240,
+      "Ġpl": 241,
+      "ine": 242,
+      "os": 243,
+      "ear": 244,
+      "ort": 245,
+      "Ġpe": 246,
+      "we": 247,
+      "oug": 248,
+      "nt": 249,
+      "ment": 250,
+      "Ġabout": 251,
+      "ap": 252,
+      "Ġby": 253,
+      "Ġout": 254,
+      "king": 255,
+      "ople": 256,
+      "Ġmore": 257,
+      "Ġtheir": 258,
+      "iv": 259,
+      "Ġwill": 260,
+      "el": 261,
+      "ast": 262,
+      "Ġwho": 263,
+      "Ġma": 264,
+      "Ġte": 265,
+      "Ġhas": 266,
+      "Ġint": 267,
+      "Ġpeople": 268,
+      "ard": 269,
+      "Ġup": 270,
+      "Ġmy": 271,
+      "Ġle": 272,
+      "Ġwhat": 273,
+      "Ġthere": 274,
+      "ack": 275,
+      "ound": 276,
+      "ind": 277,
+      "fe": 278,
+      "und": 279,
+      "ty": 280,
+      "Ġtim": 281,
+      "Ġun": 282,
+      "ther": 283,
+      "mp": 284,
+      "Ġar": 285,
+      "Ġthem": 286,
+      "Ġen": 287,
+      "Ġlike": 288,
+      "Ġtwo": 289,
+      "but": 290,
+      "ci": 291,
+      "ven": 292,
+      "ure": 293,
+      "ood": 294,
+      "now": 295,
+      "ays": 296,
+      "ong": 297,
+      "ag": 298,
+      "Ġyear": 299,
+      "ich": 300,
+      "Ġhad": 301,
+      "so": 302,
+      "Ġwould": 303,
+      "Ġget": 304,
+      "ud": 305,
+      "Ġwere": 306,
+      "ough": 307,
+      "Ġfe": 308,
+      "Ġyour": 309,
+      "act": 310,
+      "Ġhis": 311,
+      "Ġgr": 312,
+      "ousand": 313,
+      "Ġthousand": 314,
+      "Ġpr": 315,
+      "Ġsome": 316,
+      "Ġbec": 317,
+      "are": 318,
+      "Ġman": 319,
+      "undred": 320,
+      "Ġour": 321,
+      "Ġhundred": 322,
+      "ame": 323,
+      "Ġjust": 324,
+      "em": 325,
+      "Ġcl": 326,
+      "ide": 327,
+      "ree": 328,
+      "Ġcomp": 329,
+      "Ġwhen": 330,
+      "ks": 331,
+      "Ġshe": 332,
+      "Ġco": 333,
+      "use": 334,
+      "Ġtime": 335,
+      "Ġbr": 336,
+      "ans": 337,
+      "age": 338,
+      "Ġthan": 339,
+      "ib": 340,
+      "Ġsp": 341,
+      "ace": 342,
+      "Ġper": 343,
+      "Ġno": 344,
+      "Ġif": 345,
+      "Ġwhich": 346,
+      "reat": 347,
+      "Ġother": 348,
       "ions": 349,
-      "but": 350,
-      "Ġqu": 351,
-      "ide": 352,
-      "ud": 353,
-      "Ġfe": 354,
-      "Ġwill": 355,
-      "Ġdon": 356,
-      "Ġinto": 357,
-      "ace": 358,
-      "ong": 359,
-      "ild": 360,
-      "Ġdid": 361,
-      "Ġyears": 362,
-      "Ġag": 363,
-      "Ġactually": 364,
-      "ence": 365,
-      "Ġwhere": 366,
-      "Ġother": 367,
-      "Ġam": 368,
-      "Ġsp": 369,
-      "Ġthan": 370,
-      "Ġthings": 371,
-      "itt": 372,
-      "Ġpo": 373,
-      "age": 374,
-      "Ġexp": 375,
-      "alk": 376,
-      "ore": 377,
-      "nt": 378,
-      "ree": 379,
-      "ical": 380,
-      "Ġnew": 381,
-      "Ġbeen": 382,
-      "Ġsay": 383,
-      "Ġcl": 384,
-      "ach": 385,
-      "iff": 386,
-      "de": 387,
-      "ousand": 388,
-      "uch": 389,
-      "ff": 390,
-      "Ġany": 391,
-      "unt": 392,
-      "Ġthousand": 393,
-      "Ġmake": 394,
-      "Ġneed": 395,
-      "Ġdiff": 396,
-      "Ġpart": 397,
-      "Ġro": 398,
-      "Ġshe": 399,
-      "Ġar": 400,
-      "te": 401,
-      "Ġstart": 402,
-      "undred": 403,
-      "ri": 404,
-      "Ġdis": 405,
-      "for": 406,
-      "Ġhundred": 407,
-      "Ġhapp": 408,
-      "fe": 409,
-      "Ġright": 410,
-      "irst": 411,
-      "Ġover": 412,
-      "Ġimp": 413,
-      "Ġthose": 414,
-      "ath": 415,
-      "Ġbu": 416,
-      "ry": 417,
-      "Ġchan": 418,
-      "Ġgr": 419,
-      "Ġfirst": 420,
-      "Ġlo": 421,
-      "be": 422,
-      "Ġsomething": 423,
-      "Ġspe": 424,
-      "own": 425,
-      "round": 426,
-      "Ġback": 427,
-      "ite": 428,
-      "ory": 429,
-      "ice": 430,
-      "olog": 431,
-      "ia": 432,
-      "Ġfo": 433,
-      "Ġcall": 434,
-      "ign": 435,
-      "pt": 436,
-      "ater": 437,
-      "ult": 438,
-      "co": 439,
-      "Ġmost": 440,
-      "ass": 441,
-      "Ġalso": 442,
-      "Ġdiffere": 443,
-      "ens": 444,
-      "Ġac": 445,
+      "Ġad": 350,
+      "ore": 351,
+      "Ġbeen": 352,
+      "Ġlo": 353,
+      "Ġta": 354,
+      "Ġknow": 355,
+      "Ġam": 356,
+      "Ġqu": 357,
+      "ice": 358,
+      "hing": 359,
+      "Ġnew": 360,
+      "ip": 361,
+      "Ġpo": 362,
+      "ause": 363,
+      "ink": 364,
+      "per": 365,
+      "ry": 366,
+      "Ġhow": 367,
+      "for": 368,
+      "ild": 369,
+      "Ġher": 370,
+      "te": 371,
+      "able": 372,
+      "Ġany": 373,
+      "Ġdis": 374,
+      "Ġag": 375,
+      "og": 376,
+      "ose": 377,
+      "ings": 378,
+      "Ġwant": 379,
+      "iz": 380,
+      "Ġinto": 381,
+      "Ġro": 382,
+      "ite": 383,
+      "ult": 384,
+      "Ġaf": 385,
+      "Ġac": 386,
+      "Ġpart": 387,
+      "ical": 388,
+      "od": 389,
+      "ence": 390,
+      "Ġres": 391,
+      "Ġevery": 392,
+      "Ġcould": 393,
+      "Ġover": 394,
+      "ook": 395,
+      "Ġvery": 396,
+      "ount": 397,
+      "Ġimp": 398,
+      "ix": 399,
+      "Ġcont": 400,
+      "ren": 401,
+      "ass": 402,
+      "Ġdr": 403,
+      "pl": 404,
+      "Ġbecause": 405,
+      "Ġwork": 406,
+      "own": 407,
+      "ated": 408,
+      "ck": 409,
+      "ue": 410,
+      "Ġoff": 411,
+      "Ġthink": 412,
+      "Ġspe": 413,
+      "Ġsc": 414,
+      "ance": 415,
+      "Ġneed": 416,
+      "Ġyears": 417,
+      "ach": 418,
+      "Ġway": 419,
+      "int": 420,
+      "pt": 421,
+      "Ġnow": 422,
+      "vel": 423,
+      "Ġgoing": 424,
+      "nder": 425,
+      "Ġbl": 426,
+      "Ġsee": 427,
+      "'re": 428,
+      "ect": 429,
+      "ress": 430,
+      "Ġonly": 431,
+      "enty": 432,
+      "Ġworld": 433,
+      "Ġmake": 434,
+      "ang": 435,
+      "Ġback": 436,
+      "ep": 437,
+      "Ġdon": 438,
+      "uch": 439,
+      "ations": 440,
+      "Ġthree": 441,
+      "ire": 442,
+      "ia": 443,
+      "Ġdid": 444,
+      "Ġalso": 445,
       "sel": 446,
-      "Ġsaid": 447,
-      "ip": 448,
-      "ac": 449,
-      "Ġlot": 450,
-      "Ġaf": 451,
-      "Ġgot": 452,
-      "that": 453,
-      "ug": 454,
-      "Ġph": 455,
-      "Ġkind": 456,
-      "ue": 457,
-      "Ġmuch": 458,
-      "ittle": 459,
-      "ves": 460,
-      "ber": 461,
-      "int": 462,
-      "under": 463,
-      "Ġlife": 464,
-      "way": 465,
-      "Ġlittle": 466,
-      "Ġeven": 467,
-      "Ġwell": 468,
-      "Ġdes": 469,
-      "lo": 470,
-      "Ġday": 471,
-      "Ġsy": 472,
-      "Ġbet": 473,
-      "king": 474,
-      "Ġtalk": 475,
-      "Ġonly": 476,
-      "ple": 477,
-      "Ġthr": 478,
-      "ving": 479,
-      "ish": 480,
-      "Ġres": 481,
-      "cent": 482,
-      "Ġher": 483,
-      "able": 484,
-      "vel": 485,
-      "Ġpr": 486,
-      "Ġunder": 487,
-      "ub": 488,
-      "ations": 489,
-      "Ġad": 490,
-      "ated": 491,
-      "Ġhis": 492,
-      "Ġmany": 493,
-      "ms": 494,
-      "Ġcont": 495,
-      "Ġtra": 496,
-      "Ġbr": 497,
-      "ph": 498,
-      "ody": 499
+      "Ġcr": 447,
+      "Ġdif": 448,
+      "itt": 449,
+      "Ġstr": 450,
+      "omet": 451,
+      "Ġthese": 452,
+      "ile": 453,
+      "Ġbet": 454,
+      "Ġmost": 455,
+      "Ġact": 456,
+      "Ġfir": 457,
+      "ens": 458,
+      "other": 459,
+      "wh": 460,
+      "ign": 461,
+      "ents": 462,
+      "ves": 463,
+      "they": 464,
+      "Ġpre": 465,
+      "ater": 466,
+      "ving": 467,
+      "Ġwhere": 468,
+      "Ġfl": 469,
+      "Ġthr": 470,
+      "ife": 471,
+      "Ġfive": 472,
+      "ub": 473,
+      "Ġinc": 474,
+      "Ġmuch": 475,
+      "ick": 476,
+      "Ġgood": 477,
+      "ors": 478,
+      "ial": 479,
+      "Ġnine": 480,
+      "ber": 481,
+      "Ġeven": 482,
+      "cess": 483,
+      "this": 484,
+      "du": 485,
+      "Ġmar": 486,
+      "cent": 487,
+      "ug": 488,
+      "Ġhim": 489,
+      "Ġfirst": 490,
+      "wn": 491,
+      "ory": 492,
+      "Ġthen": 493,
+      "Ġsix": 494,
+      "Ġcomm": 495,
+      "ade": 496,
+      "Ġits": 497,
+      "ak": 498,
+      "Ġsay": 499
     },
     "merges": [
       "Ġ t",
-      "Ġt h",
       "Ġ a",
+      "h e",
       "i n",
-      "Ġ w",
-      "Ġth e",
       "r e",
-      "u n",
       "Ġ s",
+      "Ġt he",
       "Ġ o",
-      "Ġ i",
-      "a t",
-      "un k",
+      "Ġ w",
       "e r",
       "n d",
       "Ġ c",
-      "o u",
+      "a t",
       "Ġ b",
+      "i t",
+      "o u",
+      "Ġ f",
+      "o n",
       "Ġ m",
-      "Ġ h",
       "e n",
+      "Ġ p",
+      "i s",
       "in g",
-      "Ġ <",
       "Ġt o",
-      "o n",
-      "Ġ p",
-      "Ġ f",
-      "Ġa nd",
-      "Ġ d",
       "e s",
-      "a n",
+      "Ġt h",
+      "Ġ h",
       "o r",
+      "a n",
+      "Ġ d",
+      "e d",
       "Ġo f",
-      "a l",
+      "Ġa nd",
       "Ġ in",
-      "Ġ '",
-      "Ġ l",
-      "e d",
-      "i s",
-      "i t",
+      "a l",
       "a r",
-      "Ġ y",
-      "Ġth at",
-      "Ġ g",
-      "o m",
-      "Ġw e",
-      "Ġ n",
+      "Ġ l",
       "a s",
-      "l e",
-      "v e",
+      "Ġ n",
+      "o m",
       "i c",
+      "Ġ g",
       "Ġ e",
-      "Ġw h",
-      "Ġy ou",
-      "t h",
-      "Ġi t",
-      "l l",
+      "l e",
+      "v e",
       "Ġb e",
-      "u s",
-      "l y",
-      "o w",
-      "Ġi s",
-      "u t",
+      "l l",
       "i on",
-      "Ġo n",
+      "l y",
       "en t",
+      "o t",
+      "u t",
+      "u s",
       "Ġ re",
+      "Ġ i",
+      "Ġo n",
+      "Ġth at",
+      "Ġ is",
+      "Ġ y",
+      "o w",
+      "e t",
+      "i d",
+      "Ġ it",
+      "a c",
+      "a y",
       "Ġh a",
-      "c t",
-      "r o",
       "l d",
+      "t he",
+      "Ġ he",
+      "Ġw e",
+      "r o",
       "v er",
-      "a nd",
-      "o t",
-      "Ġth is",
-      "e t",
-      "s e",
-      "a y",
+      "Ġs t",
       "i g",
+      "Ġy ou",
       "s t",
-      "Ġs t",
-      "i d",
-      "u r",
-      "c h",
-      "Ġs o",
-      "k e",
       "Ġf or",
-      "c e",
-      "e re",
-      "Ġw as",
+      "Ġw h",
       "i r",
+      "s e",
+      "u r",
       "i m",
-      "Ġthe y",
-      "Ġa n",
-      "Ġd o",
-      "r a",
-      "Ġl i",
+      "c t",
+      "a m",
       "it h",
+      "a nd",
+      "k e",
+      "a d",
+      "u n",
+      "Ġa n",
+      "' s",
       "Ġa re",
-      "o p",
+      "Ġw ith",
+      "Ġa s",
+      "Ġ r",
+      "Ġw as",
+      "c h",
       "Ġs e",
-      "Ġg o",
-      "al ly",
+      "t h",
+      "o o",
+      "c e",
+      "i f",
       "Ġm o",
-      "Ġha ve",
-      "a m",
-      "om e",
-      "Ġa b",
-      "Ġw ith",
-      "Ġ k",
       "h t",
-      "Ġc an",
-      "o l",
-      "un d",
-      "Ġa s",
-      "Ġh e",
-      "Ġwh at",
-      "ou t",
-      "at ion",
-      "Ġn e",
-      "Ġw or",
-      "g e",
-      "Ġs h",
-      "Ġ j",
-      "a d",
-      "ou ld",
-      "Ġm e",
-      "Ġc h",
-      "Ġa t",
-      "ver y",
-      "Ġab out",
+      "o p",
       "i ll",
-      "an t",
+      "Ġha ve",
+      "at ion",
       "er s",
-      "Ġp ro",
-      "Ġon e",
-      "u l",
-      "Ġb ut",
+      "Ġn e",
       "i l",
-      "us t",
-      "Ġ r",
-      "ig ht",
+      "o l",
       "Ġn ot",
-      "Ġa ll",
-      "Ġf r",
-      "Ġe x",
-      "Ġthe re",
-      "Ġm y",
-      "i f",
-      "Ġ us",
-      "o o",
+      "Ġd e",
       "Ġc om",
+      "Ġf r",
       "t er",
-      "Ġd e",
-      "Ġp e",
-      "Ġo r",
+      "re d",
+      "u l",
+      "Ġd o",
+      "Ġc h",
+      "g e",
       "i ve",
-      "ar t",
-      "Ġs ome",
-      "th er",
-      "u p",
+      "ou ld",
+      "ig ht",
+      "Ġon e",
+      "Ġ j",
+      "Ġa t",
+      "Ġo r",
       "Ġc on",
-      "op le",
-      "e ar",
-      "i es",
-      "u m",
-      "n ow",
-      "Ġpe ople",
+      "Ġthe y",
+      "Ġs a",
+      "Ġ k",
+      "an t",
       "es s",
-      "c k",
-      "Ġli ke",
-      "at e",
-      "it y",
-      "u re",
-      "Ġm a",
-      "Ġo ur",
-      "s o",
-      "es t",
+      "Ġp ro",
       "ou s",
-      "is t",
-      "b le",
-      "Ġfr om",
-      "Ġbe c",
       "p e",
-      "Ġa l",
+      "Ġc an",
+      "Ġe x",
+      "Ġa b",
+      "Ġth is",
+      "a in",
+      "Ġ u",
+      "Ġs h",
+      "Ġw or",
+      "es t",
+      "at e",
+      "Ġt r",
       "Ġ v",
-      "us e",
-      "al l",
-      "q u",
-      "Ġt e",
-      "in d",
-      "Ġo ut",
-      "Ġi f",
-      "a use",
-      "p p",
-      "Ġin t",
-      "Ġj ust",
-      "in k",
-      "re d",
+      "ou t",
+      "is t",
+      "a b",
+      "us t",
+      "om e",
+      "Ġs u",
+      "Ġm e",
       "Ġt w",
-      "Ġthe se",
-      "Ġc o",
-      "Ġt r",
-      "oo k",
-      "Ġth ing",
-      "Ġthe m",
-      "Ġse e",
+      "al ly",
+      "Ġs o",
+      "Ġa l",
+      "i es",
+      "p p",
+      "he r",
+      "Ġg o",
+      "Ġb ut",
+      "Ġl i",
+      "Ġw he",
+      "Ġfr om",
+      "it y",
+      "Ġsa id",
+      "q u",
+      "u m",
+      "ou r",
+      "ar t",
+      "' t",
+      "Ġ us",
+      "re s",
+      "ver y",
       "i e",
-      "Ġk now",
-      "ou g",
-      "or t",
-      "o d",
-      "Ġ up",
+      "al l",
+      "Ġa ll",
       "on e",
-      "o s",
-      "Ġwh o",
-      "Ġwh en",
-      "Ġth ink",
-      "c i",
       "Ġp l",
       "in e",
-      "Ġg et",
-      "ou r",
-      "th e",
-      "Ġ very",
+      "o s",
+      "e ar",
+      "or t",
+      "Ġp e",
+      "w e",
+      "ou g",
+      "n t",
+      "m ent",
+      "Ġab out",
+      "a p",
       "Ġb y",
-      "Ġbec ause",
-      "i v",
-      "Ġha d",
-      "Ġh ow",
-      "Ġy ear",
-      "e l",
-      "ic h",
+      "Ġo ut",
+      "k ing",
+      "op le",
       "Ġmo re",
-      "Ġt im",
-      "th ing",
-      "Ġgo ing",
-      "e ct",
-      "Ġi m",
-      "Ġa ct",
-      "Ġyou r",
-      "a in",
-      "Ġwe re",
-      "Ġw ould",
-      "e m",
-      "Ġn ow",
       "Ġthe ir",
-      "Ġ en",
-      "o g",
-      "m ent",
-      "Ġs a",
+      "i v",
+      "Ġw ill",
+      "e l",
       "as t",
-      "Ġt a",
-      "am e",
-      "Ġl ook",
-      "w e",
-      "a ct",
-      "Ġwor k",
-      "Ġcom p",
-      "Ġw ant",
-      "i z",
-      "Ġre ally",
-      "Ġ un",
-      "Ġwor ld",
-      "Ġl e",
-      "Ġw ay",
-      "a ck",
-      "u ally",
-      "v en",
-      "o se",
-      "Ġtw o",
+      "Ġwh o",
+      "Ġm a",
+      "Ġt e",
+      "Ġh as",
+      "Ġin t",
+      "Ġpe ople",
       "ar d",
-      "Ġwh ich",
-      "re at",
-      "Ġtim e",
-      "a b",
-      "Ġn o",
-      "Ġe very",
+      "Ġu p",
+      "Ġm y",
+      "Ġl e",
+      "Ġwh at",
+      "Ġthe re",
+      "ac k",
+      "ou nd",
+      "in d",
+      "f e",
+      "u nd",
       "t y",
-      "Ġh ere",
-      "a re",
-      "w h",
-      "a p",
+      "Ġt im",
+      "Ġ un",
+      "the r",
+      "m p",
+      "Ġa r",
+      "Ġthe m",
+      "Ġ en",
+      "Ġli ke",
+      "Ġtw o",
+      "b ut",
+      "c i",
+      "v en",
+      "u re",
+      "oo d",
+      "n ow",
+      "ay s",
+      "on g",
       "a g",
-      "an s",
+      "Ġy ear",
+      "ic h",
+      "Ġha d",
+      "s o",
+      "Ġw ould",
+      "Ġg et",
+      "u d",
+      "Ġwe re",
       "oug h",
+      "Ġf e",
+      "Ġyou r",
+      "ac t",
+      "Ġh is",
+      "Ġg r",
+      "ous and",
+      "Ġth ousand",
+      "Ġp r",
+      "Ġs ome",
+      "Ġbe c",
+      "a re",
       "Ġm an",
-      "re s",
-      "Ġthe n",
-      "Ġh as",
-      "Ġc ould",
-      "Ġp er",
-      "ion s",
-      "b ut",
-      "Ġ qu",
+      "und red",
+      "Ġo ur",
+      "Ġh undred",
+      "am e",
+      "Ġj ust",
+      "e m",
+      "Ġc l",
       "id e",
-      "u d",
-      "Ġf e",
-      "Ġw ill",
-      "Ġd on",
-      "Ġint o",
-      "a ce",
-      "on g",
-      "i ld",
-      "Ġd id",
-      "Ġyear s",
-      "Ġa g",
-      "Ġact ually",
-      "en ce",
-      "Ġwh ere",
-      "Ġo ther",
-      "Ġa m",
-      "Ġs p",
-      "Ġth an",
-      "Ġthing s",
-      "it t",
-      "Ġp o",
+      "re e",
+      "Ġcom p",
+      "Ġwhe n",
+      "k s",
+      "Ġs he",
+      "Ġc o",
+      "us e",
+      "Ġtim e",
+      "Ġb r",
+      "an s",
       "a ge",
-      "Ġex p",
-      "al k",
+      "Ġth an",
+      "i b",
+      "Ġs p",
+      "ac e",
+      "Ġp er",
+      "Ġn o",
+      "Ġi f",
+      "Ġwh ich",
+      "re at",
+      "Ġo ther",
+      "ion s",
+      "Ġa d",
       "o re",
-      "n t",
-      "re e",
-      "ic al",
-      "Ġne w",
       "Ġbe en",
-      "Ġs ay",
-      "Ġc l",
-      "a ch",
-      "if f",
-      "d e",
-      "ous and",
-      "u ch",
-      "f f",
-      "Ġan y",
-      "un t",
-      "Ġth ousand",
-      "Ġma ke",
-      "Ġne ed",
-      "Ġd iff",
-      "Ġp art",
-      "Ġ ro",
-      "Ġsh e",
-      "Ġa r",
+      "Ġl o",
+      "Ġt a",
+      "Ġk now",
+      "Ġa m",
+      "Ġ qu",
+      "ic e",
+      "h ing",
+      "Ġne w",
+      "i p",
+      "Ġp o",
+      "a use",
+      "in k",
+      "p er",
+      "r y",
+      "Ġh ow",
+      "f or",
+      "i ld",
+      "Ġhe r",
       "t e",
-      "Ġst art",
-      "und red",
-      "r i",
+      "ab le",
+      "Ġan y",
       "Ġd is",
-      "f or",
-      "Ġh undred",
-      "Ġha pp",
-      "f e",
-      "Ġr ight",
-      "ir st",
+      "Ġa g",
+      "o g",
+      "o se",
+      "ing s",
+      "Ġw ant",
+      "i z",
+      "Ġint o",
+      "Ġ ro",
+      "it e",
+      "ul t",
+      "Ġa f",
+      "Ġa c",
+      "Ġp art",
+      "ic al",
+      "o d",
+      "en ce",
+      "Ġre s",
+      "Ġe very",
+      "Ġc ould",
       "Ġo ver",
-      "Ġim p",
-      "Ġth ose",
-      "at h",
-      "Ġb u",
-      "r y",
-      "Ġch an",
-      "Ġg r",
-      "Ġf irst",
-      "Ġl o",
-      "b e",
-      "Ġsome thing",
-      "Ġs pe",
+      "oo k",
+      "Ġ very",
+      "ou nt",
+      "Ġi mp",
+      "i x",
+      "Ġcon t",
+      "re n",
+      "as s",
+      "Ġd r",
+      "p l",
+      "Ġbec ause",
+      "Ġwor k",
       "ow n",
-      "ro und",
+      "at ed",
+      "c k",
+      "u e",
+      "Ġof f",
+      "Ġth ink",
+      "Ġs pe",
+      "Ġs c",
+      "an ce",
+      "Ġne ed",
+      "Ġyear s",
+      "ac h",
+      "Ġw ay",
+      "in t",
+      "p t",
+      "Ġn ow",
+      "ve l",
+      "Ġgo ing",
+      "nd er",
+      "Ġb l",
+      "Ġse e",
+      "' re",
+      "e ct",
+      "res s",
+      "Ġon ly",
+      "ent y",
+      "Ġwor ld",
+      "Ġma ke",
+      "an g",
       "Ġb ack",
-      "it e",
-      "or y",
-      "ic e",
-      "ol og",
+      "e p",
+      "Ġd on",
+      "u ch",
+      "ation s",
+      "Ġth ree",
+      "i re",
       "i a",
-      "Ġf o",
-      "Ġc all",
-      "ig n",
-      "p t",
-      "at er",
-      "ul t",
-      "c o",
-      "Ġmo st",
-      "as s",
+      "Ġd id",
       "Ġal so",
-      "Ġdiff ere",
-      "en s",
-      "Ġa c",
       "se l",
-      "Ġsa id",
-      "i p",
-      "a c",
-      "Ġl ot",
-      "Ġa f",
-      "Ġg ot",
-      "th at",
-      "u g",
-      "Ġp h",
-      "Ġk ind",
-      "u e",
-      "Ġm uch",
-      "itt le",
+      "Ġc r",
+      "Ġd if",
+      "it t",
+      "Ġst r",
+      "om et",
+      "Ġthe se",
+      "i le",
+      "Ġbe t",
+      "Ġmo st",
+      "Ġa ct",
+      "Ġf ir",
+      "en s",
+      "ot her",
+      "w h",
+      "ig n",
+      "ent s",
       "v es",
+      "the y",
+      "Ġp re",
+      "at er",
+      "v ing",
+      "Ġwhe re",
+      "Ġf l",
+      "Ġth r",
+      "if e",
+      "Ġf ive",
+      "u b",
+      "Ġin c",
+      "Ġm uch",
+      "ic k",
+      "Ġg ood",
+      "or s",
+      "i al",
+      "Ġn ine",
       "b er",
-      "in t",
-      "und er",
-      "Ġli fe",
-      "w ay",
-      "Ġl ittle",
       "Ġe ven",
-      "Ġwe ll",
-      "Ġd es",
-      "l o",
-      "Ġd ay",
-      "Ġs y",
-      "Ġbe t",
-      "k ing",
-      "Ġt alk",
-      "Ġon ly",
-      "p le",
-      "Ġth r",
-      "v ing",
-      "is h",
-      "Ġre s",
+      "c ess",
+      "th is",
+      "d u",
+      "Ġm ar",
       "c ent",
-      "Ġh er",
-      "a ble",
-      "ve l",
-      "Ġp r",
-      "Ġ under",
-      "u b",
-      "ation s",
-      "Ġa d",
-      "at ed",
-      "Ġh is",
-      "Ġman y",
-      "m s",
-      "Ġcon t",
-      "Ġt ra",
-      "Ġb r",
-      "p h",
-      "od y"
+      "u g",
+      "Ġh im",
+      "Ġfir st",
+      "w n",
+      "or y",
+      "Ġthe n",
+      "Ġs ix",
+      "Ġcom m",
+      "ad e",
+      "Ġit s",
+      "a k",
+      "Ġs ay"
     ]
   }
 }

tokenizer_config.json

@@ -1,5 +1,6 @@
 {
   "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
   "eos_token": "</s>",
   "mask_token": "<mask>",
   "model_max_length": 1000000000000000019884624838656,