Im2LatexTransformer / tokenizer.json
pedrolcs63's picture
Upload folder using huggingface_hub
b0377a8 verified
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<PAD>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<SOS>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<EOS>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "<UNK>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "Split",
"pattern": {
"String": " "
},
"behavior": "Removed",
"invert": false
},
"post_processor": null,
"decoder": {
"type": "WordPiece",
"prefix": "##",
"cleanup": true
},
"model": {
"type": "WordLevel",
"vocab": {
"<PAD>": 0,
"<SOS>": 1,
"<EOS>": 2,
"<UNK>": 3,
"\\widetilde": 4,
"\\gamma": 5,
"_": 6,
"{": 7,
"\\mathrm": 8,
"h": 9,
"o": 10,
"p": 11,
"f": 12,
"}": 13,
"\\simeq": 14,
"\\sum": 15,
"n": 16,
">": 17,
"0": 18,
"G": 19,
"\\frac": 20,
"(": 21,
"-": 22,
"a": 23,
")": 24,
"^": 25,
"2": 26,
"1": 27,
"\\cal": 28,
"L": 29,
"g": 30,
"i": 31,
"j": 32,
"=": 33,
",": 34,
"\\": 35,
"H": 36,
"k": 37,
"S": 38,
"s": 39,
"t": 40,
"\\pi": 41,
"\\sqrt": 42,
"N": 43,
"5": 44,
"3": 45,
"\\left(": 46,
"+": 47,
"\\bar": 48,
"\\right)": 49,
"\\hat": 50,
"\\sp": 51,
"\\dagger": 52,
"\\,": 53,
".": 54,
"*": 55,
"d": 56,
"\\kappa": 57,
"\\phi": 58,
"J": 59,
"B": 60,
"\\prime": 61,
"A": 62,
"r": 63,
"\\lambda": 64,
"\\eta": 65,
"\\partial": 66,
"\\mu": 67,
"F": 68,
"\\nu": 69,
"e": 70,
"x": 71,
"V": 72,
"\\tilde": 73,
"m": 74,
"6": 75,
"\\left\\{": 76,
"b": 77,
"c": 78,
"\\right\\}": 79,
"\\delta": 80,
"~": 81,
"\\quad": 82,
"\\!": 83,
":": 84,
"\\rho": 85,
"q": 86,
"P": 87,
"\\alpha": 88,
"\\theta": 89,
"\\Phi": 90,
"w": 91,
"\\bigtriangleup": 92,
"D": 93,
"z": 94,
"\\left[": 95,
"u": 96,
"\\right]": 97,
"l": 98,
"8": 99,
"T": 100,
"\\psi": 101,
"\\operatorname": 102,
"\\textstyle": 103,
"\\int": 104,
"y": 105,
"E": 106,
"/": 107,
"v": 108,
"\\langle": 109,
"\\rangle": 110,
"\\times": 111,
"4": 112,
"\\xi": 113,
"R": 114,
"C": 115,
"\\sim": 116,
"Q": 117,
"\\cdots": 118,
"\\beta": 119,
"\\begin{array}": 120,
"\\omega": 121,
"&": 122,
"\\\\": 123,
"\\end{array}": 124,
"\\Omega": 125,
"\\;": 126,
"\\qquad": 127,
"\\chi": 128,
"\\{": 129,
"[": 130,
"]": 131,
"\\}": 132,
"K": 133,
"\\dot": 134,
"X": 135,
"\\pm": 136,
";": 137,
"W": 138,
"\\Pi": 139,
"\\ell": 140,
"\\varepsilon": 141,
"\\overline": 142,
"\\:": 143,
"|": 144,
"\\infty": 145,
"\\vspace": 146,
"\\right.": 147,
"\\equiv": 148,
"\\scriptstyle": 149,
"\\forall": 150,
"\\in": 151,
"\\varphi": 152,
"\\Sigma": 153,
"M": 154,
"\\tau": 155,
"\\Gamma": 156,
"\\epsilon": 157,
"\\zeta": 158,
"\\hspace": 159,
"\\sigma": 160,
"!": 161,
"\\bf": 162,
"\\Lambda": 163,
"9": 164,
"Z": 165,
"<": 166,
"\\vec": 167,
"I": 168,
"\\mathop": 169,
"\\prod": 170,
"\\left.": 171,
"\\nonumber": 172,
"\\Delta": 173,
"\\varpi": 174,
"\\small": 175,
"\\left|": 176,
"\\right|": 177,
"U": 178,
"O": 179,
"\\ddot": 180,
"\\nabla": 181,
"\\tiny": 182,
"\\wedge": 183,
"\\boldmath": 184,
"\\mathcal": 185,
"\\Psi": 186,
"\\hbar": 187,
"\\rightarrow": 188,
"\\it": 189,
"\\stackrel": 190,
"\\leftrightarrow": 191,
"\\approx": 192,
"7": 193,
"Y": 194,
"\\Big": 195,
"\\star": 196,
"\\Bigg": 197,
"\\vert": 198,
"\\underline": 199,
"\\cdot": 200,
"\\widehat": 201,
"\\smallskip": 202,
"\\slash": 203,
"\\left\\langle": 204,
"\\right\\rangle": 205,
"\\operatorname*": 206,
"\\ldots": 207,
"\\d": 208,
"\\longleftrightarrow": 209,
"\\triangle": 210,
"\\vartheta": 211,
"\\big": 212,
"\\displaystyle": 213,
"\\to": 214,
"'": 215,
"\\dag": 216,
"\\mapsto": 217,
"\\Xi": 218,
"\\mid": 219,
"\\c": 220,
"\\mathbf": 221,
"\\dots": 222,
"\\otimes": 223,
"\\footnotesize": 224,
"\\thinspace": 225,
"\\oint": 226,
"\\neq": 227,
"\\varrho": 228,
"\\breve": 229,
"\\ast": 230,
"\\binom": 231,
"\\bigl": 232,
"\\bigr": 233,
"\\geq": 234,
"\\biggl": 235,
"\\phantom": 236,
"\\scriptsize": 237,
"\\leq": 238,
"\\mp": 239,
"\\left\\vert": 240,
"\\longrightarrow": 241,
"\\tt": 242,
"\\Bigl": 243,
"\\Bigr": 244,
"\\Theta": 245,
"\\le": 246,
"\\propto": 247,
"\\textrm": 248,
"--": 249,
"\\imath": 250,
"\\jmath": 251,
"\\oplus": 252,
"\\subset": 253,
"\\circ": 254,
"\\bigg": 255,
"\\overleftarrow": 256,
"\\overrightarrow": 257,
"\\Rightarrow": 258,
"\\perp": 259,
"\\parallel": 260,
"\\Re": 261,
"\\Upsilon": 262,
"\\wp": 263,
"\\b": 264,
"\\biggr": 265,
"\\ge": 266,
"\\bot": 267,
"\\L": 268,
"\\rightharpoonup": 269,
"\\cong": 270,
"\\hline": 271,
"\\Longrightarrow": 272,
"\\lbrack": 273,
"\\rbrack": 274,
"\\rightleftharpoons": 275,
"\\Im": 276,
"\\varsigma": 277,
"\\ll": 278,
"\\left<": 279,
"\\right>": 280,
"\\mathbin": 281,
"\\scriptscriptstyle": 282,
"\\mathit": 283,
"\\supset": 284,
"\\raisebox": 285,
"ule": 286,
"0.4": 287,
"pt": 288,
"20": 289,
"\\label": 290,
"\\cap": 291,
"\\backslash": 292,
"\\acute": 293,
"\\vphantom": 294,
"\\Leftrightarrow": 295,
"\\textup": 296,
"\\underbrace": 297,
"`": 298,
"\\not": 299,
"\\vee": 300,
"\\sf": 301,
"\\lceil": 302,
"\\rfloor": 303,
"\\left\\lfloor": 304,
"\\right\\rfloor": 305,
"\\check": 306,
"\\i": 307,
"\\mit": 308,
"\\colon": 309,
"\\bigoplus": 310,
"\\bullet": 311,
"\\ne": 312,
"\\|": 313,
"\\Biggl": 314,
"\\Biggr": 315,
"\\gg": 316,
"\\l": 317,
"\\right\\vert": 318,
"\\sb": 319,
"\\Longleftrightarrow": 320,
"\\left\\lbrace": 321,
"\\right\\rbrace": 322,
"\\subseteq": 323,
"\\do": 324,
"\\em": 325,
"\\atop": 326,
"\\mathsf": 327,
"\\large": 328,
"\\pmod": 329,
"\"": 330,
"\\vdots": 331,
"\\flat": 332,
"\\lbrace": 333,
"\\cup": 334,
"\\raise": 335,
"\\left\\|": 336,
"\\right\\|": 337,
"\\ni": 338,
"\\o": 339,
"\\land": 340,
"\\kern": 341,
"\\llap": 342,
"\\relax": 343,
"\\iota": 344,
"\\rbrace": 345,
"\\/": 346,
"\\protect": 347,
"\\bigotimes": 348,
"\\odot": 349,
"\\textbf": 350,
"\\textit": 351,
"\\&": 352,
"\\vdash": 353,
"\\sharp": 354,
"\\enspace": 355,
"\\bmod": 356,
"\\mathaccent": 357,
"\\grave": 358,
"\\downarrow": 359,
"\\setlength": 360,
"\\unitlength": 361,
"\\thicklines": 362,
"\\put": 363,
"\\circle": 364,
"\\emptyset": 365,
"\\sl": 366,
"\\fbox": 367,
"\\'": 368,
"\\uparrow": 369,
"\\lefteqn": 370,
"\\O": 371,
"\\doteq": 372,
"\\upsilon": 373,
"\\j": 374,
"\\ddots": 375,
"\\ref": 376,
"\\S": 377,
"\\#": 378,
"\\Huge": 379,
"\\pounds": 380,
"\\setminus": 381,
"\\makebox": 382,
"\\longmapsto": 383,
"\\cdotp": 384,
"\\-": 385,
"\\leftarrow": 386,
"0.9": 387,
"cm": 388,
"\\Vert": 389,
"0.14": 390,
"mm": 391,
"8.5": 392,
"\\succeq": 393,
"\\diamond": 394,
"\\space": 395,
"\\aleph": 396,
"\\buildrel": 397,
"\\hfill": 398,
"[object": 399,
"Object]": 400,
"\\_": 401,
"\\Large": 402,
"\\P": 403,
"\\bigtriangledown": 404,
"\\left/": 405,
"\\bigcup": 406,
"\\textsf": 407,
"\\diamondsuit": 408,
"\\lfloor": 409,
"\\overbrace": 410,
"\\hphantom": 411,
"\\^": 412,
"\\bigsqcup": 413,
"\\left\\lbrack": 414,
"\\right\\rbrack": 415,
"\\mkern": 416,
"\\brack": 417,
"\\textnormal": 418,
"\\noalign": 419,
"\\vskip": 420,
"\\line": 421,
"\\framebox": 422,
"\\texttt": 423,
"\\bigcap": 424,
"\\protecte": 425,
"?": 426,
"\\normalsize": 427,
"1.7": 428,
"\\protectu": 429,
"\\asymp": 430,
"\\lower": 431,
"\\hrule": 432,
"\\vrule": 433,
"\\supseteq": 434,
"\\medskip": 435,
"---": 436,
"\\hookrightarrow": 437,
"\\bigwedge": 438,
"\\exists": 439,
"\\natural": 440,
"\\bigvee": 441,
"\\smash": 442,
"\\biggm": 443,
"\\oslash": 444,
"\\surd": 445,
"\\longleftarrow": 446,
"\\fill": 447,
"\\strut": 448,
"\\setcounter": 449,
"\\renewcommand": 450,
"\\arraystretch": 451,
"\\smile": 452,
"\\lq": 453,
"\\ae": 454,
"\\verb": 455,
"\\unboldmath": 456,
"\\nolinebreak": 457,
"\\sc": 458,
"\\prec": 459,
"\\LARGE": 460,
"\\newcommand": 461,
"\\m": 462,
"\\multicolumn": 463,
"#": 464,
"\\arraycolsep": 465,
"\\protectZ": 466,
"2.1": 467,
"\\ominus": 468,
"\\Biggm": 469,
"\\rightarrowfill": 470,
"\\cite": 471,
"\\vcenter": 472,
"\\Bigm": 473,
"\\ooalign": 474,
"\\hfil": 475,
"\\crcr": 476,
"\\parbox": 477,
"\\nearrow": 478,
"\\top": 479,
"10": 480,
"\\multiput": 481,
"\\SS": 482,
"\\bigm": 483,
"\\rlap": 484,
"\\sqcup": 485,
"\\Longleftarrow": 486,
"\\skew": 487,
"\\ddagger": 488,
"\\notin": 489,
"\\protectE": 490,
"\\fboxsep": 491,
"\\triangleright": 492,
"\\succ": 493,
"\\ss": 494,
"\\*": 495,
"0.5": 496,
"\\vline": 497,
"\\mathord": 498,
"\\mathversion": 499,
"\\protectm": 500,
"\\ddag": 501,
"\\null": 502,
"\\triangleleft": 503,
"\\ensuremath": 504,
"\\root": 505,
"\\of": 506,
"\\special": 507,
"\\enskip": 508,
"\\vector": 509,
"\\lgroup": 510,
"\\rgroup": 511,
"\\arrowvert": 512,
"\\overwithdelims": 513,
"\\AA": 514,
"\\mathrel": 515,
"\\sqcap": 516,
"\\left]": 517,
"\\right[": 518,
"\\amalg": 519,
"\\left\\lceil": 520,
"\\right\\rceil": 521,
"\\leavevmode": 522,
"\\mathstrut": 523,
"\\symbol": 524,
"0.1": 525,
"3.1": 526,
"3.2": 527,
"\\footnotemark": 528,
"\\mathopen": 529,
"\\mathclose": 530,
"\\nulldelimiterspace": 531,
"\\brace": 532,
"\\Downarrow": 533,
"\\searrow": 534,
"\\smallint": 535,
"\\itshape": 536,
"\\mskip": 537,
"in": 538,
"0.23": 539,
"\\atopwithdelims": 540,
"\\right/": 541,
"\\vss": 542,
"\\hss": 543
},
"unk_token": "<UNK>"
}
}