Upload tokenizer

Browse files

Files changed (3) hide show

added_tokens.json +6 -543
tokenizer.json +0 -0
tokenizer_config.json +0 -0

added_tokens.json CHANGED Viewed

@@ -1,545 +1,8 @@
 {
-  " ": 32642,
-  "$$": 32100,
-  ";;": 32638,
-  "\\AA": 32101,
-  "\\C": 32102,
-  "\\Delta": 32103,
-  "\\Deltatoo": 32104,
-  "\\Documents": 32105,
-  "\\Downarrow": 32106,
-  "\\Gamma": 32107,
-  "\\GammaRACT": 32108,
-  "\\Gammaacts": 32109,
-  "\\Gammadoes": 32110,
-  "\\Im": 32111,
-  "\\Lambda": 32112,
-  "\\LambdaMPLE": 32113,
-  "\\LambdaPTER": 32114,
-  "\\LambdaRK": 32115,
-  "\\LambdaYNAUD": 32116,
-  "\\Lambdamodule": 32117,
-  "\\Lambdamodules": 32118,
-  "\\Lambdappendix": 32119,
-  "\\Lambdas": 32120,
-  "\\Lambdaut": 32121,
-  "\\Leftarrow": 32122,
-  "\\Leftrightarrow": 32124,
-  "\\Longleftarrow": 32125,
-  "\\Longleftrightarrow": 32126,
-  "\\Longrightarrow": 32127,
-  "\\M": 32128,
-  "\\Math": 32129,
-  "\\Minv": 32130,
-  "\\Omega": 32131,
-  "\\Perp": 32132,
-  "\\PerpI": 32133,
-  "\\Phi": 32134,
-  "\\Pi": 32135,
-  "\\Pisentence": 32136,
-  "\\Pisentences": 32137,
-  "\\Psi": 32138,
-  "\\Psiitself": 32139,
-  "\\Psipre": 32140,
-  "\\Psir": 32141,
-  "\\Re": 32142,
-  "\\Rightarrow": 32143,
-  "\\S": 32145,
-  "\\SA": 32146,
-  "\\SD": 32147,
-  "\\SE": 32148,
-  "\\SF": 32149,
-  "\\SIII": 32150,
-  "\\SIV": 32151,
-  "\\SIX": 32152,
-  "\\SVII": 32153,
-  "\\Sigma": 32154,
-  "\\Subset": 32155,
-  "\\Theta": 32156,
-  "\\Uparrow": 32158,
-  "\\Upsilon": 32159,
-  "\\Users": 32160,
-  "\\Varangle": 32161,
-  "\\Vdash": 32162,
-  "\\Xi": 32163,
-  "\\aleph": 32164,
-  "\\alldifferent": 32165,
-  "\\alpha": 32166,
-  "\\alphapointwise": 32167,
-  "\\alphareversed": 32168,
-  "\\alphasurjective": 32169,
-  "\\amalg": 32170,
-  "\\angle": 32171,
-  "\\approx": 32172,
-  "\\arccos": 32173,
-  "\\arcsin": 32174,
-  "\\arctan": 32175,
-  "\\arg": 32176,
-  "\\ast": 32177,
-  "\\asymp": 32178,
-  "\\atop": 32179,
-  "\\backsim": 32180,
-  "\\backslash": 32181,
-  "\\bar": 32182,
-  "\\because": 32183,
-  "\\begin": 32184,
-  "\\beta": 32185,
-  "\\betaa": 32186,
-  "\\betaencharacter": 32187,
-  "\\betaencharakter": 32188,
-  "\\betaencharaktere": 32189,
-  "\\betaenoharaktere": 32190,
-  "\\betaerwesentliche": 32191,
-  "\\betaeverywhere": 32192,
-  "\\betao": 32193,
-  "\\beth": 32194,
-  "\\bigcap": 32195,
-  "\\bigcirc": 32196,
-  "\\bigcup": 32197,
-  "\\bigodot": 32198,
-  "\\bigoplus": 32199,
-  "\\bigotimes": 32200,
-  "\\bigsqcup": 32201,
-  "\\bigvee": 32202,
-  "\\bigwedge": 32203,
-  "\\binom": 32204,
-  "\\bmod": 32205,
-  "\\bmodL": 32206,
-  "\\boldsymbol": 32207,
-  "\\bowtie": 32208,
-  "\\boxminus": 32209,
-  "\\boxplus": 32210,
-  "\\boxtimes": 32211,
-  "\\breve": 32212,
-  "\\bullet": 32213,
-  "\\bumpeq": 32214,
-  "\\cap": 32215,
-  "\\capV": 32216,
-  "\\cdot": 32217,
-  "\\cdotFUNCTIONS": 32218,
-  "\\cdotideal": 32219,
-  "\\cdotr": 32220,
-  "\\cdots": 32221,
-  "\\cdotss": 32222,
-  "\\cdotvalues": 32223,
-  "\\check": 32224,
-  "\\checkmark": 32225,
-  "\\chi": 32226,
-  "\\circ": 32227,
-  "\\circdet": 32228,
-  "\\circf": 32229,
-  "\\circlearrowleft": 32230,
-  "\\circlearrowright": 32231,
-  "\\circledR": 32232,
-  "\\circledast": 32233,
-  "\\cjkend": 32234,
-  "\\cjkstart": 32235,
-  "\\cline": 32236,
-  "\\complement": 32237,
-  "\\cong": 32238,
-  "\\coprod": 32239,
-  "\\coprodAut": 32240,
-  "\\coprodU": 32241,
-  "\\cos": 32242,
-  "\\cosh": 32243,
-  "\\cot": 32244,
-  "\\cox": 32245,
-  "\\csc": 32246,
-  "\\cup": 32247,
-  "\\curlyvee": 32248,
-  "\\curlywedge": 32249,
-  "\\curvearrowleft": 32250,
-  "\\curvearrowright": 32251,
-  "\\dagger": 32252,
-  "\\dashv": 32253,
-  "\\ddagger": 32254,
-  "\\dddo": 32255,
-  "\\dddot": 32256,
-  "\\ddo": 32257,
-  "\\ddot": 32258,
-  "\\ddots": 32259,
-  "\\delta": 32260,
-  "\\deltafunctor": 32261,
-  "\\diamond": 32262,
-  "\\digamma": 32263,
-  "\\directory": 32264,
-  "\\div": 32265,
-  "\\dot": 32266,
-  "\\doteq": 32267,
-  "\\doteqdot": 32268,
-  "\\dots": 32269,
-  "\\downarrow": 32270,
-  "\\downharpoonright": 32271,
-  "\\ell": 32272,
-  "\\ellholds": 32273,
-  "\\ellrepresentation": 32274,
-  "\\emptyset": 32275,
-  "\\end": 32276,
-  "\\epsilon": 32277,
-  "\\epsilonKer": 32278,
-  "\\equiv": 32279,
-  "\\eta": 32280,
-  "\\ex": 32281,
-  "\\exists": 32282,
-  "\\existsc": 32283,
-  "\\existsclosed": 32284,
-  "\\exp": 32285,
-  "\\fallingdotseq": 32286,
-  "\\forall": 32287,
-  "\\forallsomething": 32288,
-  "\\frac": 32289,
-  "\\frown": 32290,
-  "\\gamma": 32291,
-  "\\gammavalues": 32292,
-  "\\gcd": 32293,
-  "\\geq": 32294,
-  "\\geqq": 32295,
-  "\\geqslant": 32296,
-  "\\gg": 32297,
-  "\\grave": 32298,
-  "\\gtrdot": 32299,
-  "\\gtreqless": 32300,
-  "\\gtrless": 32301,
-  "\\gtrsim": 32302,
-  "\\hat": 32303,
-  "\\hbar": 32304,
-  "\\hdashline": 32305,
-  "\\hline": 32306,
-  "\\hlinee": 32307,
-  "\\hookleftarrow": 32308,
-  "\\hookrightarrow": 32309,
-  "\\hyunj": 32310,
-  "\\iiint": 32311,
-  "\\iint": 32312,
-  "\\imath": 32313,
-  "\\in": 32314,
-  "\\inG": 32315,
-  "\\inMag": 32316,
-  "\\inX": 32317,
-  "\\inf": 32318,
-  "\\infty": 32319,
-  "\\inftyalso": 32320,
-  "\\inftyo": 32321,
-  "\\inftyramifies": 32322,
-  "\\inm": 32323,
-  "\\inn": 32324,
-  "\\int": 32325,
-  "\\intormula": 32326,
-  "\\iota": 32327,
-  "\\jmath": 32328,
-  "\\kappa": 32329,
-  "\\kappai": 32330,
-  "\\kappath": 32331,
-  "\\lambda": 32332,
-  "\\lambdaa": 32333,
-  "\\lambdafinite": 32334,
-  "\\lambdasemiring": 32335,
-  "\\langle": 32336,
-  "\\langlee": 32337,
-  "\\langleu": 32338,
-  "\\langley": 32339,
-  "\\latex": 32340,
-  "\\lceil": 32341,
-  "\\lceilk": 32342,
-  "\\lceiln": 32343,
-  "\\lcm": 32344,
-  "\\ldot": 32345,
-  "\\ldots": 32346,
-  "\\ldotsPROOF": 32347,
-  "\\ldotsetc": 32348,
-  "\\leadsto": 32349,
-  "\\left": 32350,
-  "\\leftarrow": 32123,
-  "\\leftleftarrows": 32351,
-  "\\leftrightarrow": 32352,
-  "\\leftrightarrows": 32353,
-  "\\leftrightharpoons": 32354,
-  "\\leq": 32355,
-  "\\leqq": 32356,
-  "\\leqslant": 32357,
-  "\\lessdot": 32358,
-  "\\lesseqgtr": 32359,
-  "\\lessgtr": 32360,
-  "\\lesssim": 32361,
-  "\\lfloor": 32362,
-  "\\lg": 32363,
-  "\\lim": 32364,
-  "\\liminf": 32365,
-  "\\limsup": 32366,
-  "\\ll": 32367,
-  "\\llbracket": 32368,
-  "\\llcorner": 32369,
-  "\\lll": 32370,
-  "\\ln": 32371,
-  "\\log": 32372,
-  "\\longdiv": 32373,
-  "\\longleftarrow": 32374,
-  "\\longleftrightarrow": 32375,
-  "\\longleftrightarrowEfi": 32376,
-  "\\longmapsto": 32377,
-  "\\longmapstomult": 32378,
-  "\\longrightarrow": 32379,
-  "\\lrcorner": 32380,
-  "\\ltimes": 32381,
-  "\\lvert": 32382,
-  "\\mapsto": 32383,
-  "\\mapstoW": 32384,
-  "\\mathbb": 32385,
-  "\\mathbbR": 32386,
-  "\\mathbbZrank": 32387,
-  "\\mathbbZtakes": 32388,
-  "\\mathbbl": 32389,
-  "\\mathbf": 32390,
-  "\\mathcal": 32391,
-  "\\mathcalM": 32392,
-  "\\mathcalom": 32393,
-  "\\mathfrak": 32394,
-  "\\mathrm": 32395,
-  "\\mathscr": 32396,
-  "\\mathscrB": 32397,
-  "\\mathscrS": 32398,
-  "\\mathscrf": 32399,
-  "\\matshcr": 32400,
-  "\\max": 32401,
-  "\\measuredangle": 32402,
-  "\\mho": 32403,
-  "\\mid": 32404,
-  "\\min": 32405,
-  "\\models": 32406,
-  "\\mp": 32407,
-  "\\mu": 32408,
-  "\\mudominant": 32409,
-  "\\muexists": 32410,
-  "\\mufinite": 32411,
-  "\\multicolumn": 32412,
-  "\\multimap": 32413,
-  "\\multirow": 32414,
-  "\\n": 32415,
-  "\\nLeftarrow": 32416,
-  "\\nRightarrow": 32417,
-  "\\nVdash": 32418,
-  "\\nabla": 32419,
-  "\\natural": 32420,
-  "\\nearrow": 32421,
-  "\\neg": 32422,
-  "\\neq": 32423,
-  "\\neqId": 32424,
-  "\\neqid": 32425,
-  "\\nexists": 32426,
-  "\\ngtr": 32427,
-  "\\ni": 32428,
-  "\\nleftarrow": 32429,
-  "\\nless": 32430,
-  "\\nmid": 32431,
-  "\\nmidDisc": 32432,
-  "\\not": 32433,
-  "\\notin": 32434,
-  "\\nprec": 32435,
-  "\\npreceq": 32436,
-  "\\nrightarrow": 32437,
-  "\\nsim": 32438,
-  "\\nsubseteq": 32439,
-  "\\nsucc": 32440,
-  "\\nsucceq": 32441,
-  "\\nsupseteq": 32442,
-  "\\nu": 32443,
-  "\\nvdash": 32444,
-  "\\nwarrow": 32445,
-  "\\odot": 32446,
-  "\\oepratorname": 32447,
-  "\\oiiint": 32448,
-  "\\oiint": 32449,
-  "\\oint": 32450,
-  "\\omega": 32451,
-  "\\omegaest": 32452,
-  "\\ominus": 32453,
-  "\\operatorname": 32454,
-  "\\operatornamemap": 32455,
-  "\\oplus": 32456,
-  "\\oslash": 32457,
-  "\\otimes": 32458,
-  "\\otimese": 32459,
-  "\\overbrace": 32460,
-  "\\overleftarrow": 32461,
-  "\\overleftrightarrow": 32462,
-  "\\overlin": 32463,
-  "\\overline": 32464,
-  "\\overparen": 32465,
-  "\\overrightarrow": 32466,
-  "\\p": 32467,
-  "\\parallel": 32468,
-  "\\partial": 32469,
-  "\\partialM": 32470,
-  "\\partiala": 32471,
-  "\\partialb": 32472,
-  "\\perp": 32473,
-  "\\phi": 32474,
-  "\\phicommute": 32475,
-  "\\pi": 32476,
-  "\\pifinite": 32477,
-  "\\pitchfork": 32478,
-  "\\pitwist": 32479,
-  "\\pm": 32480,
-  "\\prec": 32481,
-  "\\preccurlyeq": 32482,
-  "\\preceq": 32483,
-  "\\precsim": 32484,
-  "\\prime": 32485,
-  "\\prod": 32486,
-  "\\propto": 32487,
-  "\\psi": 32488,
-  "\\psidoes": 32489,
-  "\\qquad": 32490,
-  "\\qua": 32491,
-  "\\quad": 32492,
-  "\\quada": 32493,
-  "\\rangle": 32494,
-  "\\rceil": 32495,
-  "\\rfloor": 32496,
-  "\\rho": 32497,
-  "\\rhofollows": 32498,
-  "\\rhooperator": 32499,
-  "\\right": 32500,
-  "\\rightarrow": 32144,
-  "\\rightarrowA": 32501,
-  "\\rightarrowC": 32502,
-  "\\rightarrowFinord": 32503,
-  "\\rightarrowSet": 32504,
-  "\\rightarrowSets": 32505,
-  "\\rightarrowf": 32506,
-  "\\rightharpoonup": 32507,
-  "\\rightleftarrows": 32508,
-  "\\rightleftharpoons": 32509,
-  "\\rightrightarrows": 32510,
-  "\\rightsquigarrow": 32511,
-  "\\risingdotseq": 32512,
-  "\\rrbracket": 32513,
-  "\\rtimes": 32514,
-  "\\rvert": 32515,
-  "\\searrow": 32516,
-  "\\sec": 32517,
-  "\\section": 32518,
-  "\\setminus": 32519,
-  "\\sharp": 32520,
-  "\\sigma": 32521,
-  "\\sim": 32522,
-  "\\simeq": 32523,
-  "\\simeqid": 32524,
-  "\\simr": 32525,
-  "\\sin": 32526,
-  "\\sinh": 32527,
-  "\\smile": 32528,
-  "\\sqcap": 32529,
-  "\\sqcup": 32530,
-  "\\sqrt": 32531,
-  "\\sqsubset": 32532,
-  "\\sqsubseteq": 32533,
-  "\\sqsupset": 32534,
-  "\\sqsupseteq": 32535,
-  "\\square": 32536,
-  "\\stackrel": 32537,
-  "\\star": 32538,
-  "\\starEXERCISE": 32539,
-  "\\subsection": 32540,
-  "\\subset": 32541,
-  "\\subsetA": 32542,
-  "\\subsetM": 32543,
-  "\\subseteq": 32544,
-  "\\subseteqW": 32545,
-  "\\subsetneq": 32546,
-  "\\substack": 32547,
-  "\\subsubsection": 32548,
-  "\\succ": 32549,
-  "\\succcurlyeq": 32550,
-  "\\succeq": 32551,
-  "\\succsim": 32552,
-  "\\sum": 32553,
-  "\\sup": 32554,
-  "\\supset": 32555,
-  "\\supseteq": 32556,
-  "\\supseteqq": 32557,
-  "\\supsetneq": 32558,
-  "\\supsetneqq": 32559,
-  "\\swarrow": 32560,
-  "\\tag": 32561,
-  "\\tan": 32562,
-  "\\tanh": 32563,
-  "\\tau": 32564,
-  "\\taualent": 32565,
-  "\\tauariety": 32566,
-  "\\tauphism": 32567,
-  "\\tauphistn": 32568,
-  "\\taupure": 32569,
-  "\\text": 32570,
-  "\\textbf": 32571,
-  "\\textrm": 32572,
-  "\\textrmK": 32573,
-  "\\therefore": 32574,
-  "\\theta": 32575,
-  "\\thetaalone": 32576,
-  "\\thetaexplicit": 32577,
-  "\\thetaworks": 32578,
-  "\\tilde": 32579,
-  "\\times": 32580,
-  "\\timesm": 32581,
-  "\\to": 32157,
-  "\\top": 32582,
-  "\\triangle": 32583,
-  "\\triangleleft": 32584,
-  "\\triangleq": 32585,
-  "\\triangleright": 32586,
-  "\\ulcorner": 32587,
-  "\\underbrace": 32588,
-  "\\underline": 32589,
-  "\\underset": 32590,
-  "\\unlhd": 32591,
-  "\\unrhd": 32592,
-  "\\uparrow": 32593,
-  "\\upharpoonleft": 32594,
-  "\\upharpoonright": 32595,
-  "\\uplus": 32596,
-  "\\urcorner": 32597,
-  "\\vDash": 32598,
-  "\\varangle": 32599,
-  "\\varepsilon": 32600,
-  "\\varepsilona": 32601,
-  "\\varepsilonarrow": 32602,
-  "\\varepsilonarrows": 32603,
-  "\\varinjlim": 32604,
-  "\\varkappa": 32605,
-  "\\varliminf": 32606,
-  "\\varlimsup": 32607,
-  "\\varnothing": 32608,
-  "\\varphi": 32609,
-  "\\varphiare": 32610,
-  "\\varphiitself": 32611,
-  "\\varphiw": 32612,
-  "\\varpi": 32613,
-  "\\varprojlim": 32614,
-  "\\varrho": 32615,
-  "\\varsigma": 32616,
-  "\\varsubsetneqq": 32617,
-  "\\vartheta": 32618,
-  "\\varthetaa": 32619,
-  "\\vdash": 32620,
-  "\\vdots": 32621,
-  "\\vec": 32622,
-  "\\vecara": 32623,
-  "\\vee": 32624,
-  "\\vigvec": 32625,
-  "\\wedge": 32626,
-  "\\wedgeT": 32627,
-  "\\wedgeid": 32628,
-  "\\widehat": 32629,
-  "\\widehatat": 32630,
-  "\\widetilde": 32631,
-  "\\wp": 32632,
-  "\\xi": 32633,
-  "\\xlongequal": 32634,
-  "\\xrightarrow": 32635,
-  "\\your": 32636,
-  "\\zeta": 32637,
-  "^": 32641,
-  "{": 32639,
-  "}": 32640
 }

 {
+  " ": 32105,
+  ";;": 32100,
+  "\\": 32103,
+  "^": 32104,
+  "{": 32101,
+  "}": 32102
 }

tokenizer.json CHANGED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json CHANGED Viewed

The diff for this file is too large to render. See raw diff