Transformers
last-hope-tokenizer / tokenizer.json
NoeMastro's picture
Upload tokenizer
1f9626c verified
Raw
History Blame Contribute Delete
21.8 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "[UNK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "[CLS]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "[SEP]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 4,
"content": "[MASK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": {
"type": "Lowercase"
},
"pre_tokenizer": {
"type": "Whitespace"
},
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
}
],
"pair": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
}
],
"special_tokens": {}
},
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {
"[UNK]": 0,
"[CLS]": 1,
"[SEP]": 2,
"[PAD]": 3,
"[MASK]": 4,
".": 5,
":": 6,
"-": 7,
"a": 8,
"about": 9,
"above": 10,
"acceleration": 11,
"across": 12,
"act": 13,
"action": 14,
"after": 15,
"again": 16,
"age": 17,
"agree": 18,
"air": 19,
"alarm": 20,
"alien": 21,
"all": 22,
"alone": 23,
"amaze": 24,
"ammonia": 25,
"an": 26,
"analyst": 27,
"analyze": 28,
"and": 29,
"angle": 30,
"animal": 31,
"anom-a": 32,
"anom-b": 33,
"anom-c": 34,
"anomaly": 35,
"answer": 36,
"ant": 37,
"antarctica": 38,
"antenna": 39,
"apollo": 40,
"appear": 41,
"arc": 42,
"area": 43,
"argon": 44,
"argue": 45,
"arm": 46,
"around": 47,
"arrive": 48,
"art": 49,
"as": 50,
"ask": 51,
"at": 52,
"atmosphere": 53,
"atom": 54,
"attack": 55,
"automatically": 56,
"avoid": 57,
"away": 58,
"baby": 59,
"back": 60,
"backwards": 61,
"bacteria": 62,
"bad": 63,
"bag": 64,
"ball": 65,
"bandage": 66,
"barrier": 67,
"base": 68,
"battery": 69,
"bay": 70,
"be": 71,
"beat": 72,
"beautiful": 73,
"because": 74,
"become": 75,
"bed": 76,
"bedroom": 77,
"beetle": 78,
"before": 79,
"behind": 80,
"believe": 81,
"belt": 82,
"better": 83,
"between": 84,
"big": 85,
"biology": 86,
"bird": 87,
"birthday": 88,
"bite": 89,
"blind": 90,
"block": 91,
"blow": 92,
"blue": 93,
"body": 94,
"boil": 95,
"bolt": 96,
"bone": 97,
"book": 98,
"boost": 99,
"borrow": 100,
"both": 101,
"bottle": 102,
"box": 103,
"brain": 104,
"brave": 105,
"break": 106,
"breathe": 107,
"breed": 108,
"bring": 109,
"broken": 110,
"brown": 111,
"bug": 112,
"build": 113,
"bump": 114,
"burn": 115,
"bus": 116,
"busy": 117,
"but": 118,
"button": 119,
"by": 120,
"cabin": 121,
"cable": 122,
"cake": 123,
"calculate": 124,
"calibration": 125,
"call": 126,
"calm": 127,
"camera": 128,
"can": 129,
"candle": 130,
"capacitor": 131,
"capacity": 132,
"car": 133,
"carbon": 134,
"care": 135,
"cargo": 136,
"cat": 137,
"cause": 138,
"ceiling": 139,
"celebrate": 140,
"cell": 141,
"centrifuge": 142,
"cetocite": 143,
"chain": 144,
"chair": 145,
"change": 146,
"charge": 147,
"chase": 148,
"check": 149,
"cheer": 150,
"chemical": 151,
"chemistry": 152,
"chicken": 153,
"child": 154,
"chloe": 155,
"circuit": 156,
"city": 157,
"class": 158,
"clean": 159,
"clever": 160,
"click": 161,
"climb": 162,
"clock": 163,
"close": 164,
"cloud": 165,
"coat": 166,
"cold": 167,
"collapse": 168,
"collector": 169,
"collision": 170,
"color": 171,
"come": 172,
"comfortable": 173,
"compass": 174,
"complete": 175,
"computer": 176,
"conflict": 177,
"constant": 178,
"consume": 179,
"contest": 180,
"continue": 181,
"control": 182,
"cook": 183,
"cool": 184,
"copy": 185,
"correct": 186,
"could": 187,
"count": 188,
"cover": 189,
"crash": 190,
"crazy": 191,
"create": 192,
"creature": 193,
"crew": 194,
"cross": 195,
"crush": 196,
"cry": 197,
"cryoplast": 198,
"culture": 199,
"cup": 200,
"cure": 201,
"curious": 202,
"cut": 203,
"damage": 204,
"dance": 205,
"danger": 206,
"dangerous": 207,
"dark": 208,
"data": 209,
"day": 210,
"dead": 211,
"death": 212,
"debate": 213,
"decay": 214,
"deck": 215,
"deep": 216,
"delay": 217,
"density": 218,
"design": 219,
"desk": 220,
"destroy": 221,
"detail": 222,
"dial": 223,
"die": 224,
"diet": 225,
"difference": 226,
"different": 227,
"dig": 228,
"digital": 229,
"dim": 230,
"direction": 231,
"dirt": 232,
"dirty": 233,
"disagree": 234,
"disease": 235,
"distance": 236,
"dizzy": 237,
"do": 238,
"doctor": 239,
"dog": 240,
"door": 241,
"double": 242,
"down": 243,
"drag": 244,
"draw": 245,
"dream": 246,
"drive": 247,
"dry": 248,
"duck": 249,
"dust": 250,
"each": 251,
"early": 252,
"earth": 253,
"easy": 254,
"eat": 255,
"ecosystem": 256,
"edge": 257,
"egg": 258,
"eject": 259,
"electric": 260,
"electronic": 261,
"element": 262,
"elephant": 263,
"empty": 264,
"end": 265,
"enemy": 266,
"energy": 267,
"engine": 268,
"engineer": 269,
"english": 270,
"enjoy": 271,
"enough": 272,
"ensure": 273,
"enter": 274,
"environment": 275,
"equipment": 276,
"error": 277,
"escape": 278,
"ethical": 279,
"evacuate": 280,
"every": 281,
"everything": 282,
"evolution": 283,
"exercise": 284,
"exist": 285,
"exit": 286,
"explain": 287,
"explore": 288,
"explosion": 289,
"explosive": 290,
"eye": 291,
"face": 292,
"fade": 293,
"fail": 294,
"failure": 295,
"fake": 296,
"fall": 297,
"false": 298,
"family": 299,
"far": 300,
"farm": 301,
"fascinating": 302,
"fast": 303,
"fat": 304,
"fault": 305,
"favorite": 306,
"fear": 307,
"feature": 308,
"feel": 309,
"female": 310,
"few": 311,
"field": 312,
"fight": 313,
"fill": 314,
"filter": 315,
"find": 316,
"finger": 317,
"finish": 318,
"fire": 319,
"first": 320,
"fish": 321,
"fist": 322,
"fix": 323,
"float": 324,
"floor": 325,
"flow": 326,
"fly": 327,
"fog": 328,
"follow": 329,
"food": 330,
"foot": 331,
"for": 332,
"force": 333,
"forest": 334,
"forget": 335,
"form": 336,
"forward": 337,
"fragile": 338,
"free": 339,
"freeze": 340,
"frequency": 341,
"fresh": 342,
"friend": 343,
"from": 344,
"front": 345,
"fuel": 346,
"full": 347,
"fun": 348,
"function": 349,
"funny": 350,
"fusion": 351,
"future": 352,
"game": 353,
"games": 354,
"garden": 355,
"gas": 356,
"generate": 357,
"gift": 358,
"girl": 359,
"give": 360,
"glass": 361,
"global": 362,
"glove": 363,
"glow": 364,
"go": 365,
"goal": 366,
"gold": 367,
"good": 368,
"gravity": 369,
"great": 370,
"green": 371,
"greet": 372,
"group": 373,
"grow": 374,
"growl": 375,
"guide": 376,
"guitar": 377,
"habitat": 378,
"hand": 379,
"happen": 380,
"happy": 381,
"hard": 382,
"hat": 383,
"hate": 384,
"have": 385,
"hayes": 386,
"head": 387,
"heal": 388,
"health": 389,
"hear": 390,
"heart": 391,
"heat": 392,
"heavy": 393,
"helium": 394,
"hello": 395,
"helmet": 396,
"help": 397,
"here": 398,
"hibernate": 399,
"hide": 400,
"high": 401,
"history": 402,
"hit": 403,
"hold": 404,
"hole": 405,
"home": 406,
"honor": 407,
"hope": 408,
"horn": 409,
"horse": 410,
"hot": 411,
"house": 412,
"how": 413,
"hugo": 414,
"hull": 415,
"human": 416,
"hurry": 417,
"ice": 418,
"idea": 419,
"ideal": 420,
"if": 421,
"ignore": 422,
"ill": 423,
"image": 424,
"imagine": 425,
"in": 426,
"increase": 427,
"inert": 428,
"injury": 429,
"ink": 430,
"insect": 431,
"inside": 432,
"inspect": 433,
"intense": 434,
"internet": 435,
"into": 436,
"intuition": 437,
"it": 438,
"join": 439,
"joint": 440,
"joke": 441,
"journey": 442,
"joy": 443,
"jump": 444,
"keep": 445,
"key": 446,
"kill": 447,
"kitchen": 448,
"knife": 449,
"know": 450,
"knowledge": 451,
"kryon": 452,
"kryonian": 453,
"laboratory": 454,
"lack": 455,
"land": 456,
"language": 457,
"laser": 458,
"last": 459,
"late": 460,
"later": 461,
"laugh": 462,
"launch": 463,
"laundry": 464,
"law": 465,
"layer": 466,
"lead": 467,
"leader": 468,
"leaf": 469,
"leak": 470,
"learn": 471,
"learning": 472,
"leave": 473,
"left": 474,
"lens": 475,
"less": 476,
"lesson": 477,
"level": 478,
"lever": 479,
"lie": 480,
"life": 481,
"light": 482,
"lightning": 483,
"like": 484,
"limit": 485,
"line": 486,
"lion": 487,
"liquid": 488,
"list": 489,
"listen": 490,
"live": 491,
"load": 492,
"lock": 493,
"logic": 494,
"long": 495,
"look": 496,
"lose": 497,
"lost": 498,
"loud": 499,
"love": 500,
"low": 501,
"lower": 502,
"loyal": 503,
"luck": 504,
"lucky": 505,
"lung": 506,
"machine": 507,
"magnetic": 508,
"maintain": 509,
"maintenance": 510,
"make": 511,
"manual": 512,
"many": 513,
"map": 514,
"marina": 515,
"mark": 516,
"mass": 517,
"mate": 518,
"material": 519,
"math": 520,
"mathieu": 521,
"matter": 522,
"may": 523,
"meal": 524,
"mean": 525,
"measure": 526,
"measured": 527,
"mechanic": 528,
"medicine": 529,
"meet": 530,
"melt": 531,
"membrane": 532,
"memory": 533,
"mess": 534,
"message": 535,
"met": 536,
"metal": 537,
"meteor": 538,
"methane": 539,
"method": 540,
"microbe": 541,
"microphone": 542,
"microscope": 543,
"migrate": 544,
"minute": 545,
"mirror": 546,
"miss": 547,
"mistake": 548,
"mix": 549,
"mode": 550,
"model": 551,
"money": 552,
"monitor": 553,
"monkey": 554,
"moon": 555,
"moral": 556,
"more": 557,
"mosquito": 558,
"motor": 559,
"mount": 560,
"mountain": 561,
"mouth": 562,
"move": 563,
"movie": 564,
"much": 565,
"muscle": 566,
"music": 567,
"must": 568,
"mutate": 569,
"mutates": 570,
"mutation": 571,
"naked": 572,
"natural": 573,
"nature": 574,
"nausea": 575,
"near": 576,
"neck": 577,
"need": 578,
"nerve": 579,
"new": 580,
"news": 581,
"next": 582,
"night": 583,
"nitrogen": 584,
"no": 585,
"noe": 586,
"noise": 587,
"none": 588,
"not": 589,
"nothing": 590,
"now": 591,
"nuclear": 592,
"nucleus": 593,
"number": 594,
"nutrient": 595,
"object": 596,
"observe": 597,
"odd": 598,
"of": 599,
"off": 600,
"okay": 601,
"old": 602,
"on": 603,
"one": 604,
"only": 605,
"open": 606,
"or": 607,
"orbit": 608,
"order": 609,
"organ": 610,
"other": 611,
"out": 612,
"outcome": 613,
"outside": 614,
"over": 615,
"oxygen": 616,
"page": 617,
"pain": 618,
"pan": 619,
"paper": 620,
"part": 621,
"partner": 622,
"past": 623,
"path": 624,
"pattern": 625,
"peace": 626,
"people": 627,
"perfect": 628,
"perfection": 629,
"person": 630,
"petri": 631,
"philippe": 632,
"photo": 633,
"physics": 634,
"pick": 635,
"picture": 636,
"piece": 637,
"pill": 638,
"pilot": 639,
"pipe": 640,
"place": 641,
"plan": 642,
"planet": 643,
"plant": 644,
"play": 645,
"plot": 646,
"pole": 647,
"polish": 648,
"population": 649,
"power": 650,
"praise": 651,
"pray": 652,
"predator": 653,
"prepare": 654,
"pressure": 655,
"prime": 656,
"probe": 657,
"problem": 658,
"process": 659,
"product": 660,
"professor": 661,
"program": 662,
"progress": 663,
"project": 664,
"protect": 665,
"protein": 666,
"protocol": 667,
"proud": 668,
"prove": 669,
"provide": 670,
"pull": 671,
"pure": 672,
"push": 673,
"put": 674,
"puzzle": 675,
"quality": 676,
"question": 677,
"quiet": 678,
"rabbit": 679,
"radiation": 680,
"rain": 681,
"range": 682,
"ray": 683,
"reach": 684,
"reaction": 685,
"reactor": 686,
"read": 687,
"ready": 688,
"real": 689,
"reason": 690,
"recover": 691,
"recovery": 692,
"recruit": 693,
"red": 694,
"reduce": 695,
"refuse": 696,
"release": 697,
"remember": 698,
"repair": 699,
"report": 700,
"reproduce": 701,
"reproduction": 702,
"rescue": 703,
"research": 704,
"researcher": 705,
"resistant": 706,
"resolution": 707,
"resource": 708,
"result": 709,
"return": 710,
"reuse": 711,
"reward": 712,
"ride": 713,
"right": 714,
"rise": 715,
"risk": 716,
"road": 717,
"robot": 718,
"rock": 719,
"rocket": 720,
"roll": 721,
"ronan": 722,
"room": 723,
"rope": 724,
"rot": 725,
"rule": 726,
"run": 727,
"rust": 728,
"sad": 729,
"safe": 730,
"safety": 731,
"same": 732,
"sample": 733,
"sand": 734,
"save": 735,
"say": 736,
"scan": 737,
"scare": 738,
"school": 739,
"schoolteacher": 740,
"science": 741,
"scientist": 742,
"scientists": 743,
"screen": 744,
"sea": 745,
"seal": 746,
"search": 747,
"season": 748,
"seat": 749,
"secret": 750,
"see": 751,
"seek": 752,
"select": 753,
"self": 754,
"send": 755,
"sense": 756,
"sensor": 757,
"session": 758,
"shadow": 759,
"shake": 760,
"shame": 761,
"shape": 762,
"share": 763,
"shark": 764,
"sharp": 765,
"shelf": 766,
"shell": 767,
"shelter": 768,
"shield": 769,
"ship": 770,
"shock": 771,
"shoe": 772,
"short": 773,
"shout": 774,
"show": 775,
"sick": 776,
"side": 777,
"sight": 778,
"sign": 779,
"signal": 780,
"silence": 781,
"silent": 782,
"silk": 783,
"similar": 784,
"simple": 785,
"simulate": 786,
"simulation": 787,
"since": 788,
"sing": 789,
"single": 790,
"sink": 791,
"sit": 792,
"situation": 793,
"size": 794,
"skin": 795,
"sky": 796,
"sleep": 797,
"slow": 798,
"small": 799,
"smart": 800,
"smile": 801,
"smoke": 802,
"snow": 803,
"so": 804,
"sock": 805,
"soil": 806,
"solar": 807,
"solution": 808,
"solve": 809,
"something": 810,
"song": 811,
"soon": 812,
"sound": 813,
"space": 814,
"special": 815,
"species": 816,
"speed": 817,
"spike": 818,
"spin": 819,
"spoon": 820,
"sport": 821,
"spread": 822,
"stable": 823,
"stand": 824,
"star": 825,
"start": 826,
"stay": 827,
"steer": 828,
"stellavore": 829,
"sterling": 830,
"stick": 831,
"still": 832,
"stony": 833,
"stop": 834,
"storage": 835,
"stores": 836,
"storm": 837,
"story": 838,
"strange": 839,
"strict": 840,
"strong": 841,
"structure": 842,
"student": 843,
"study": 844,
"stupid": 845,
"subject": 846,
"success": 847,
"sudden": 848,
"suddenly": 849,
"sum": 850,
"sun": 851,
"sunrise": 852,
"support": 853,
"sure": 854,
"surface": 855,
"surprise": 856,
"survival": 857,
"survive": 858,
"swim": 859,
"switch": 860,
"symbol": 861,
"symbols": 862,
"system": 863,
"table": 864,
"take": 865,
"talk": 866,
"tank": 867,
"tap": 868,
"task": 869,
"taste": 870,
"tea": 871,
"teach": 872,
"teacher": 873,
"team": 874,
"teamwork": 875,
"tear": 876,
"teeth": 877,
"telephone": 878,
"telescope": 879,
"tell": 880,
"temperature": 881,
"territory": 882,
"test": 883,
"text": 884,
"than": 885,
"thank": 886,
"that": 887,
"the": 888,
"theory": 889,
"there": 890,
"they": 891,
"thing": 892,
"think": 893,
"this": 894,
"thorn": 895,
"through": 896,
"thrust": 897,
"thumb": 898,
"thumbs": 899,
"thunder": 900,
"tied": 901,
"tiger": 902,
"tight": 903,
"time": 904,
"tiny": 905,
"to": 906,
"together": 907,
"token": 908,
"too": 909,
"tool": 910,
"topic": 911,
"torch": 912,
"touch": 913,
"tough": 914,
"towel": 915,
"toxic": 916,
"toxin": 917,
"toy": 918,
"trade": 919,
"translate": 920,
"translation": 921,
"translator": 922,
"transparent": 923,
"travel": 924,
"tree": 925,
"triangle": 926,
"trip": 927,
"true": 928,
"trumpet": 929,
"trust": 930,
"truth": 931,
"try": 932,
"tunnel": 933,
"turn": 934,
"under": 935,
"understand": 936,
"universe": 937,
"unload": 938,
"until": 939,
"up": 940,
"upgrade": 941,
"urine": 942,
"use": 943,
"vacuum": 944,
"valve": 945,
"vega": 946,
"vegetable": 947,
"vehicle": 948,
"vent": 949,
"vibration": 950,
"video": 951,
"view": 952,
"visible": 953,
"vital": 954,
"vitamin": 955,
"volume": 956,
"vote": 957,
"wait": 958,
"wake": 959,
"walk": 960,
"wall": 961,
"want": 962,
"warm": 963,
"watch": 964,
"water": 965,
"wave": 966,
"way": 967,
"we": 968,
"wear": 969,
"weather": 970,
"weigh": 971,
"wet": 972,
"what": 973,
"when": 974,
"where": 975,
"while": 976,
"white": 977,
"who": 978,
"why": 979,
"will": 980,
"win": 981,
"wind": 982,
"window": 983,
"wire": 984,
"with": 985,
"without": 986,
"wonder": 987,
"wood": 988,
"word": 989,
"work": 990,
"workshop": 991,
"world": 992,
"worth": 993,
"write": 994,
"wrong": 995,
"year": 996,
"yes": 997,
"yesterday": 998,
"young": 999,
"zero": 1000
},
"unk_token": "[UNK]"
}
}