Upload tokenizer

tokenizer.json

@@ -289,181 +289,223 @@
       "ł": 254,
       "Ń": 255,
       "th": 256,
-      "the": 257,
-      "Ġthe": 258,
-      "Ġi": 259,
-      "Ġa": 260,
-      "ĠÎ": 261,
-      "en": 262,
-      "re": 263,
-      "Ġo": 264,
-      "si": 265,
-      "Ġis": 266,
-      "al": 267,
-      "ri": 268,
-      "at": 269,
-      "es": 270,
-      "le": 271,
-      "on": 272,
-      "ν": 273,
-      "Ïģ": 274,
-      "Ġf": 275,
-      "ĠÏ": 276,
-      "Ġof": 277,
-      "nd": 278,
-      "Ġ2": 279,
-      "ÏģÎ": 280,
-      "an": 281,
-      "he": 282,
-      "Ġb": 283,
-      "Ġc": 284,
-      "Ġe": 285,
-      "Ġs": 286,
-      "Ġt": 287,
-      "Eu": 288,
-      "ion": 289,
-      "la": 290,
-      "mu": 291,
-      "om": 292,
-      "or": 293,
-      "ore": 294,
-      "se": 295,
-      "ten": 296,
-      "ε": 297,
-      "ο": 298,
-      "ĠT": 299,
-      "Ġsi": 300,
-      "ĠEu": 301,
-      "Ġand": 302,
-      "Ġfu": 303,
-      "ĠÏĦ": 304,
-      "mula": 305,
-      "ormula": 306,
-      "Ġsid": 307,
-      "'s": 308,
-      "ag": 309,
-      "et": 310,
-      "hy": 311,
-      "po": 312,
-      "qu": 313,
-      "use": 314,
-      "¹Ï": 315,
-      "ί": 316,
-      "α": 317,
-      "η": 318,
-      "ÏĤ": 319,
-      "Ïħ": 320,
-      "Ġ+": 321,
-      "Ġ-": 322,
-      "Ġ=": 323,
-      "Ġl": 324,
-      "Ġth": 325,
-      "Ġre": 326,
-      "İν": 327,
-      "ther": 328,
-      "Ġin": 329,
-      "Ġγ": 330,
-      "eng": 331,
-      "ent": 332,
-      "Ġother": 333,
-      "rig": 334,
-      "ler": 335,
-      "Ġformula": 336,
-      "ĠThe": 337,
-      "ĠEuler": 338,
-      "Ġsides": 339,
-      "It": 340,
-      "Py": 341,
-      "am": 342,
-      "are": 343,
-      "ct": 344,
-      "gle": 345,
-      "hi": 346,
-      "ht": 347,
-      "in": 348,
-      "li": 349,
-      "lat": 350,
-      "nct": 351,
-      "ple": 352,
-      "ry": 353,
-      "um": 354,
-      "wo": 355,
-      "whe": 356,
-      "³Ï": 357,
-      "γÏ": 358,
-      "ÏĢ": 359,
-      "ÏĦ": 360,
-      "Ġg": 361,
-      "Ġn": 362,
-      "Ġhy": 363,
-      "Ġrig": 364,
-      "ĠIt": 365,
-      "ĠPy": 366,
-      "ĥη": 367,
-      "īν": 368,
-      "thag": 369,
-      "Ġan": 370,
-      "Ġα": 371,
-      "Ġβ": 372,
-      "Ġμ": 373,
-      "ndam": 374,
-      "ÏģιÏ": 375,
-      "Ġcom": 376,
-      "Ġex": 377,
-      "Ġsqu": 378,
-      "Ġtwo": 379,
-      "ions": 380,
-      "omet": 381,
-      "orem": 382,
-      "orean": 383,
-      "tenuse": 384,
-      "οÏħ": 385,
-      "Ġfunct": 386,
-      "Ġfundam": 387,
-      "potenuse": 388,
-      "Ġleng": 389,
-      "Ġthat": 390,
-      "Ġrelat": 391,
-      "ental": 392,
-      "plex": 393,
-      "where": 394,
-      "Ġhypotenuse": 395,
-      "Ġright": 396,
-      "ĠPythag": 397,
-      "Ġcomplex": 398,
-      "Ġsquare": 399,
-      "Ġfundamental": 400,
-      "Ġlength": 401,
-      "ĠPythagorean": 402
+      "ν": 257,
+      "ÏĦ": 258,
+      "the": 259,
+      "Ïģ": 260,
+      "ι": 261,
+      "Ġthe": 262,
+      "ε": 263,
+      "Ġi": 264,
+      "Ïģι": 265,
+      "Ġa": 266,
+      "ÏĦÏģι": 267,
+      "en": 268,
+      "re": 269,
+      "ĠÎ": 270,
+      "ÏĦÏģιÎ": 271,
+      "Ġo": 272,
+      "νÎ": 273,
+      "Ïī": 274,
+      "si": 275,
+      "Ïİ": 276,
+      "Ġis": 277,
+      "al": 278,
+      "ri": 279,
+      "Ïĥ": 280,
+      "Ïİν": 281,
+      "at": 282,
+      "es": 283,
+      "le": 284,
+      "on": 285,
+      "³Ïī": 286,
+      "¼Îµ": 287,
+      "¿Î": 288,
+      "Ġf": 289,
+      "ĠÏĦÏģιÎ": 290,
+      "Ġof": 291,
+      "³ÏīνÎ": 292,
+      "nd": 293,
+      "ºÏİν": 294,
+      "η": 295,
+      "Ġ2": 296,
+      "ÏĦÏģικÏİν": 297,
+      "¼ÎµÏĦÏģικÏİν": 298,
+      "¿Î¼ÎµÏĦÏģικÏİν": 299,
+      "ĠÏĦÏģιγÏīνÎ": 300,
+      "ĠÏĦÏģιγÏīνομεÏĦÏģικÏİν": 301,
+      "an": 302,
+      "he": 303,
+      "Ïħ": 304,
+      "Ġb": 305,
+      "Ġc": 306,
+      "Ġe": 307,
+      "Ġs": 308,
+      "Ġt": 309,
+      "Eu": 310,
+      "ion": 311,
+      "la": 312,
+      "mu": 313,
+      "om": 314,
+      "or": 315,
+      "ore": 316,
+      "se": 317,
+      "ten": 318,
+      "α": 319,
+      "ĠT": 320,
+      "Ġsi": 321,
+      "ĠEu": 322,
+      "Ġand": 323,
+      "Ġfu": 324,
+      "mula": 325,
+      "ormula": 326,
+      "Ġsid": 327,
+      "'s": 328,
+      "ag": 329,
+      "et": 330,
+      "hy": 331,
+      "po": 332,
+      "qu": 333,
+      "use": 334,
+      "ÏĢ": 335,
+      "ÏĤ": 336,
+      "Ġ+": 337,
+      "Ġ-": 338,
+      "Ġ=": 339,
+      "Ġl": 340,
+      "Ġth": 341,
+      "ĠÏĦ": 342,
+      "Ġre": 343,
+      "ĠÏĥ": 344,
+      "ther": 345,
+      "εÎ": 346,
+      "Ġin": 347,
+      "eng": 348,
+      "ent": 349,
+      "Ġγ": 350,
+      "Ġother": 351,
+      "rig": 352,
+      "Ïĥη": 353,
+      "ler": 354,
+      "Ġformula": 355,
+      "ĠThe": 356,
+      "ĠEuler": 357,
+      "Ġsides": 358,
+      "It": 359,
+      "Py": 360,
+      "am": 361,
+      "are": 362,
+      "ct": 363,
+      "gle": 364,
+      "hi": 365,
+      "ht": 366,
+      "in": 367,
+      "li": 368,
+      "lat": 369,
+      "nct": 370,
+      "ple": 371,
+      "ry": 372,
+      "um": 373,
+      "wo": 374,
+      "whe": 375,
+      "ºÎ": 376,
+      "¿Ïģ": 377,
+      "¿Ïħ": 378,
+      "Ġg": 379,
+      "Ġn": 380,
+      "Ġhy": 381,
+      "Ġrig": 382,
+      "ĠIt": 383,
+      "ĠPy": 384,
+      "thag": 385,
+      "ÏĦÎ": 386,
+      "Ġan": 387,
+      "Ġα": 388,
+      "Ġβ": 389,
+      "να": 390,
+      "Ïīν": 391,
+      "ndam": 392,
+      "Ġcom": 393,
+      "Ġex": 394,
+      "Ġsqu": 395,
+      "Ġtwo": 396,
+      "ions": 397,
+      "omet": 398,
+      "orem": 399,
+      "orean": 400,
+      "tenuse": 401,
+      "Ġfunct": 402,
+      "Ġfundam": 403,
+      "potenuse": 404,
+      "ÏĢÎ": 405,
+      "Ġleng": 406,
+      "Ġthat": 407,
+      "ĠÏĦη": 408,
+      "Ġrelat": 409,
+      "ĠÏĥÏħ": 410,
+      "εί": 411,
+      "ental": 412,
+      "plex": 413,
+      "where": 414,
+      "Ġhypotenuse": 415,
+      "Ġright": 416,
+      "ĠPythag": 417,
+      "Ġcomplex": 418,
+      "Ġsquare": 419,
+      "Ġfundamental": 420,
+      "Ġlength": 421,
+      "ĠPythagorean": 422
     },
     "merges": [
       "t h",
+      "Î ½",
+      "Ï Ħ",
       "th e",
+      "Ï ģ",
+      "Î ¹",
       "Ġ the",
+      "Î µ",
       "Ġ i",
+      "Ïģ ι",
       "Ġ a",
-      "Ġ Î",
+      "ÏĦ Ïģι",
       "e n",
       "r e",
+      "Ġ Î",
+      "ÏĦÏģι Î",
       "Ġ o",
+      "ν Î",
+      "Ï ī",
       "s i",
+      "Ï İ",
       "Ġi s",
       "a l",
       "r i",
+      "Ï ĥ",
+      "Ïİ Î½",
       "a t",
       "e s",
       "l e",
       "o n",
-      "Î ½",
-      "Ï ģ",
+      "³ Ïī",
+      "¼ ε",
+      "¿ Î",
       "Ġ f",
-      "Ġ Ï",
+      "Ġ ÏĦÏģιÎ",
       "Ġo f",
+      "³Ïī νÎ",
       "n d",
+      "º Ïİν",
+      "Î ·",
       "Ġ 2",
-      "Ïģ Î",
+      "ÏĦÏģιΠºÏİν",
+      "¼Îµ ÏĦÏģικÏİν",
+      "¿Î ¼ÎµÏĦÏģικÏİν",
+      "ĠÏĦÏģιΠ³ÏīνÎ",
+      "ĠÏĦÏģιγÏīνΠ¿Î¼ÎµÏĦÏģικÏİν",
       "a n",
       "h e",
+      "Ï ħ",
       "Ġ b",
       "Ġ c",
       "Ġ e",
@@ -478,14 +520,12 @@
       "o re",
       "s e",
       "t en",
-      "Î µ",
-      "Î ¿",
+      "Î ±",
       "Ġ T",
       "Ġ si",
       "Ġ Eu",
       "Ġa nd",
       "Ġf u",
-      "ĠÏ Ħ",
       "mu la",
       "or mula",
       "Ġsi d",
@@ -496,26 +536,25 @@
       "p o",
       "q u",
       "u se",
-      "¹ Ï",
-      "Î ¯",
-      "Î ±",
-      "Î ·",
+      "Ï Ģ",
       "Ï Ĥ",
-      "Ï ħ",
       "Ġ +",
       "Ġ -",
       "Ġ =",
       "Ġ l",
       "Ġ th",
+      "Ġ ÏĦ",
       "Ġ re",
-      "İ Î½",
+      "Ġ Ïĥ",
       "the r",
+      "ε Î",
       "Ġi n",
-      "ĠÎ ³",
       "en g",
       "en t",
+      "ĠÎ ³",
       "Ġo ther",
       "ri g",
+      "Ïĥ η",
       "le r",
       "Ġf ormula",
       "ĠT he",
@@ -538,25 +577,23 @@
       "u m",
       "w o",
       "w he",
-      "³ Ï",
-      "Î ³Ï",
-      "Ï Ģ",
-      "Ï Ħ",
+      "º Î",
+      "¿ Ïģ",
+      "¿ Ïħ",
       "Ġ g",
       "Ġ n",
       "Ġ hy",
       "Ġ rig",
       "Ġ It",
       "Ġ Py",
-      "ĥ η",
-      "ī ν",
       "th ag",
+      "ÏĦ Î",
       "Ġa n",
       "ĠÎ ±",
       "ĠÎ ²",
-      "ĠÎ ¼",
+      "νΠ±",
+      "Ïī ν",
       "nd am",
-      "ÏģÎ ¹Ï",
       "Ġc om",
       "Ġe x",
       "Ġs qu",
@@ -566,13 +603,16 @@
       "ore m",
       "ore an",
       "ten use",
-      "ο Ïħ",
       "Ġfu nct",
       "Ġfu ndam",
       "po tenuse",
+      "ÏĢ Î",
       "Ġl eng",
       "Ġth at",
+      "ĠÏĦ η",
       "Ġre lat",
+      "ĠÏĥ Ïħ",
+      "εΠ¯",
       "ent al",
       "ple x",
       "whe re",