Upload tokenizer.json with huggingface_hub

tokenizer.json

@@ -22,7 +22,7 @@
       "special": true
     },
     {
-      "id": 2,
+      "id": 29,
       "content": "<bos>",
       "single_word": false,
       "lstrip": false,
@@ -31,7 +31,7 @@
       "special": true
     },
     {
-      "id": 3,
+      "id": 30,
       "content": "<eos>",
       "single_word": false,
       "lstrip": false,
@@ -95,1004 +95,102 @@
     "vocab": {
       "<unk>": 0,
       "<pad>": 1,
-      "<bos>": 2,
-      "<eos>": 3,
       ",": 4,
-      ":": 5,
+      "the": 5,
       ".": 6,
-      "the": 7,
+      "of": 7,
       "and": 8,
       "to": 9,
-      "i": 10,
-      "of": 11,
-      ";": 12,
-      "you": 13,
-      "my": 14,
-      "a": 15,
-      "that": 16,
-      "?": 17,
-      "in": 18,
-      "!": 19,
-      "is": 20,
-      "not": 21,
-      "for": 22,
-      "with": 23,
-      "me": 24,
-      "it": 25,
-      "be": 26,
-      "your": 27,
-      "his": 28,
-      "but": 29,
-      "this": 30,
-      "he": 31,
-      "have": 32,
-      "as": 33,
-      "thou": 34,
-      "him": 35,
-      "so": 36,
-      "what": 37,
-      "thy": 38,
-      "will": 39,
-      "no": 40,
-      "by": 41,
-      "all": 42,
-      "king": 43,
-      "we": 44,
-      "shall": 45,
-      "her": 46,
-      "if": 47,
-      "our": 48,
-      "are": 49,
-      "do": 50,
-      "thee": 51,
-      "now": 52,
-      "lord": 53,
-      "on": 54,
-      "good": 55,
-      "come": 56,
-      "from": 57,
-      "sir": 58,
-      "or": 59,
-      "which": 60,
-      "more": 61,
-      "then": 62,
-      "at": 63,
-      "o": 64,
-      "would": 65,
-      "was": 66,
-      "they": 67,
-      "how": 68,
-      "here": 69,
-      "well": 70,
-      "she": 71,
-      "than": 72,
-      "their": 73,
-      "them": 74,
-      "duke": 75,
-      "am": 76,
-      "hath": 77,
-      "say": 78,
-      "let": 79,
-      "when": 80,
-      "i'll": 81,
-      "one": 82,
-      "go": 83,
-      "were": 84,
-      "love": 85,
-      "may": 86,
-      "us": 87,
-      "make": 88,
-      "upon": 89,
-      "yet": 90,
-      "richard": 91,
-      "like": 92,
-      "there": 93,
-      "must": 94,
-      "should": 95,
-      "an": 96,
-      "first": 97,
-      "queen": 98,
-      "why": 99,
-      "had": 100,
-      "know": 101,
-      "man": 102,
-      "did": 103,
-      "'tis": 104,
-      "where": 105,
-      "see": 106,
-      "some": 107,
-      "too": 108,
-      "death": 109,
-      "give": 110,
-      "who": 111,
-      "speak": 112,
-      "take": 113,
-      "these": 114,
-      "edward": 115,
-      "york": 116,
-      "mine": 117,
-      "such": 118,
-      "up": 119,
-      "henry": 120,
-      "out": 121,
-      "romeo": 122,
-      "can": 123,
-      "father": 124,
-      "tell": 125,
-      "time": 126,
-      "gloucester": 127,
-      "most": 128,
-      "lady": 129,
-      "son": 130,
-      "nor": 131,
-      "vincentio": 132,
-      "hear": 133,
-      "life": 134,
-      "made": 135,
-      "'": 136,
-      "god": 137,
-      "art": 138,
-      "much": 139,
-      "think": 140,
-      "warwick": 141,
-      "heart": 142,
-      "never": 143,
-      "doth": 144,
-      "brother": 145,
-      "before": 146,
-      "both": 147,
-      "cannot": 148,
-      "any": 149,
-      "petruchio": 150,
-      "true": 151,
-      "being": 152,
-      "thus": 153,
-      "ay": 154,
-      "away": 155,
-      "blood": 156,
-      "been": 157,
-      "name": 158,
-      "fair": 159,
-      "coriolanus": 160,
-      "look": 161,
-      "men": 162,
-      "noble": 163,
-      "again": 164,
-      "menenius": 165,
-      "very": 166,
-      "day": 167,
-      "hand": 168,
-      "own": 169,
-      "pray": 170,
-      "done": 171,
-      "juliet": 172,
-      "second": 173,
-      "sweet": 174,
-      "myself": 175,
-      "leave": 176,
-      "great": 177,
-      "therefore": 178,
-      "against": 179,
-      "though": 180,
-      "poor": 181,
-      "honour": 182,
-      "down": 183,
-      "hast": 184,
-      "prince": 185,
-      "angelo": 186,
-      "way": 187,
-      "fear": 188,
-      "old": 189,
-      "nay": 190,
-      "heaven": 191,
-      "call": 192,
-      "till": 193,
-      "eyes": 194,
-      "clarence": 195,
-      "live": 196,
-      "world": 197,
-      "nurse": 198,
-      "stay": 199,
-      "comes": 200,
-      "grace": 201,
-      "many": 202,
-      "even": 203,
-      "stand": 204,
-      "die": 205,
-      "ever": 206,
-      "iii": 207,
-      "nothing": 208,
-      "wife": 209,
-      "bear": 210,
-      "dead": 211,
-      "whose": 212,
-      "night": 213,
-      "--": 214,
-      "other": 215,
-      "bolingbroke": 216,
-      "isabella": 217,
-      "friends": 218,
-      "friar": 219,
-      "leontes": 220,
-      "head": 221,
-      "buckingham": 222,
-      "peace": 223,
-      "unto": 224,
-      "those": 225,
-      "better": 226,
-      "gone": 227,
-      "word": 228,
-      "mother": 229,
-      "off": 230,
-      "tranio": 231,
-      "hence": 232,
-      "house": 233,
-      "two": 234,
-      "could": 235,
-      "lucio": 236,
-      "news": 237,
-      "since": 238,
-      "still": 239,
-      "marcius": 240,
-      "master": 241,
-      "gentleman": 242,
-      "little": 243,
-      "sicinius": 244,
-      "daughter": 245,
-      "lords": 246,
-      "thing": 247,
-      "soul": 248,
-      "put": 249,
-      "once": 250,
-      "whom": 251,
-      "himself": 252,
-      "camillo": 253,
-      "face": 254,
-      "margaret": 255,
-      "o'": 256,
-      "set": 257,
-      "capulet": 258,
-      "else": 259,
-      "ere": 260,
-      "iv": 261,
-      "long": 262,
-      "thine": 263,
-      "elizabeth": 264,
-      "words": 265,
-      "katharina": 266,
-      "none": 267,
-      "might": 268,
-      "rest": 269,
-      "best": 270,
-      "crown": 271,
-      "power": 272,
-      "young": 273,
-      "madam": 274,
-      "tongue": 275,
-      "citizen": 276,
-      "dear": 277,
-      "part": 278,
-      "bring": 279,
-      "gentle": 280,
-      "vi": 281,
-      "farewell": 282,
-      "ii": 283,
-      "keep": 284,
-      "hope": 285,
-      "right": 286,
-      "every": 287,
-      "find": 288,
-      "lucentio": 289,
-      "pardon": 290,
-      "said": 291,
-      "bid": 292,
-      "came": 293,
-      "forth": 294,
-      "hastings": 295,
-      "hortensio": 296,
-      "earth": 297,
-      "hands": 298,
-      "home": 299,
-      "into": 300,
-      "back": 301,
-      "brutus": 302,
-      "cause": 303,
-      "welcome": 304,
-      "about": 305,
-      "baptista": 306,
-      "boy": 307,
-      "dost": 308,
-      "hold": 309,
-      "tears": 310,
-      "please": 311,
-      "rome": 312,
-      "i'": 313,
-      "provost": 314,
-      "that's": 315,
-      "help": 316,
-      "people": 317,
-      "full": 318,
-      "heard": 319,
-      "makes": 320,
-      "thought": 321,
-      "after": 322,
-      "cousin": 323,
-      "indeed": 324,
-      "show": 325,
-      "escalus": 326,
-      "grumio": 327,
-      "husband": 328,
-      "let's": 329,
-      "mind": 330,
-      "place": 331,
-      "cominius": 332,
-      "marry": 333,
-      "pompey": 334,
-      "wilt": 335,
-      "friend": 336,
-      "has": 337,
-      "mistress": 338,
-      "what's": 339,
-      "mean": 340,
-      "shame": 341,
-      "state": 342,
-      "gremio": 343,
-      "within": 344,
-      "clifford": 345,
-      "servant": 346,
-      "while": 347,
-      "he's": 348,
-      "hither": 349,
-      "aufidius": 350,
-      "gracious": 351,
-      "last": 352,
-      "only": 353,
-      "answer": 354,
-      "duchess": 355,
-      "northumberland": 356,
-      "rather": 357,
-      "royal": 358,
-      "eye": 359,
-      "far": 360,
-      "lie": 361,
-      "mercutio": 362,
-      "prove": 363,
-      "thousand": 364,
-      "another": 365,
-      "paulina": 366,
-      "third": 367,
-      "claudio": 368,
-      "meet": 369,
-      "murderer": 370,
-      "child": 371,
-      "shalt": 372,
-      "comfort": 373,
-      "joy": 374,
-      "lay": 375,
-      "lies": 376,
-      "sorrow": 377,
-      "use": 378,
-      "we'll": 379,
-      "benvolio": 380,
-      "beseech": 381,
-      "grief": 382,
-      "polixenes": 383,
-      "autolycus": 384,
-      "less": 385,
-      "sun": 386,
-      "war": 387,
-      "end": 388,
-      "father's": 389,
-      "fellow": 390,
-      "fortune": 391,
-      "happy": 392,
-      "hour": 393,
-      "enough": 394,
-      "holy": 395,
-      "matter": 396,
-      "montague": 397,
-      "thank": 398,
-      "arms": 399,
-      "business": 400,
-      "grave": 401,
-      "hate": 402,
-      "things": 403,
-      "years": 404,
-      "bed": 405,
-      "clown": 406,
-      "lives": 407,
-      "prospero": 408,
-      "truth": 409,
-      "body": 410,
-      "faith": 411,
-      "fall": 412,
-      "light": 413,
-      "pity": 414,
-      "three": 415,
-      "without": 416,
-      "get": 417,
-      "kind": 418,
-      "nature": 419,
-      "save": 420,
-      "to-morrow": 421,
-      "kate": 422,
-      "send": 423,
-      "swear": 424,
-      "there's": 425,
-      "turn": 426,
-      "ah": 427,
-      "believe": 428,
-      "bianca": 429,
-      "false": 430,
-      "fight": 431,
-      "gods": 432,
-      "saw": 433,
-      "tybalt": 434,
-      "follow": 435,
-      "law": 436,
-      "does": 437,
-      "looks": 438,
-      "means": 439,
-      "present": 440,
-      "villain": 441,
-      "yours": 442,
-      "yourself": 443,
-      "anne": 444,
-      "aumerle": 445,
-      "bloody": 446,
-      "foul": 447,
-      "left": 448,
-      "ne'er": 449,
-      "sleep": 450,
-      "talk": 451,
-      "volumnia": 452,
-      "city": 453,
-      "laurence": 454,
-      "uncle": 455,
-      "land": 456,
-      "mercy": 457,
-      "need": 458,
-      "servingman": 459,
-      "told": 460,
-      "woman": 461,
-      "worthy": 462,
-      "messenger": 463,
-      "proud": 464,
-      "sea": 465,
-      "shepherd": 466,
-      "sit": 467,
-      "wish": 468,
-      "because": 469,
-      "breath": 470,
-      "gaunt": 471,
-      "high": 472,
-      "justice": 473,
-      "play": 474,
-      "sword": 475,
-      "together": 476,
-      "biondello": 477,
-      "doubt": 478,
-      "either": 479,
-      "fire": 480,
-      "horse": 481,
-      "john": 482,
-      "majesty": 483,
-      "sister": 484,
-      "through": 485,
-      "thyself": 486,
-      "'twas": 487,
-      "break": 488,
-      "children": 489,
-      "seen": 490,
-      "sound": 491,
-      "sovereign": 492,
-      "under": 493,
-      "woe": 494,
-      "catesby": 495,
-      "days": 496,
-      "heavy": 497,
-      "here's": 498,
-      "maid": 499,
-      "care": 500,
-      "gentlemen": 501,
-      "age": 502,
-      "canst": 503,
-      "content": 504,
-      "ears": 505,
-      "edward's": 506,
-      "florizel": 507,
-      "fly": 508,
-      "haste": 509,
-      "kill": 510,
-      "norfolk": 511,
-      "paris": 512,
-      "purpose": 513,
-      "ready": 514,
-      "says": 515,
-      "signior": 516,
-      "times": 517,
-      "traitor": 518,
-      "wrong": 519,
-      "alone": 520,
-      "each": 521,
-      "grey": 522,
-      "hermione": 523,
-      "late": 524,
-      "liege": 525,
-      "lost": 526,
-      "new": 527,
-      "oath": 528,
-      "reason": 529,
-      "strange": 530,
-      "warrant": 531,
-      "weep": 532,
-      "alas": 533,
-      "brought": 534,
-      "cry": 535,
-      "kiss": 536,
-      "mark": 537,
-      "rivers": 538,
-      "slain": 539,
-      "didst": 540,
-      "france": 541,
-      "free": 542,
-      "ground": 543,
-      "having": 544,
-      "knows": 545,
-      "lest": 546,
-      "loss": 547,
-      "married": 548,
-      "seem": 549,
-      "sent": 550,
-      "spirit": 551,
-      "stands": 552,
-      "straight": 553,
-      "strike": 554,
-      "thoughts": 555,
-      "born": 556,
-      "cold": 557,
-      "found": 558,
-      "near": 559,
-      "person": 560,
-      "remember": 561,
-      "report": 562,
-      "return": 563,
-      "sebastian": 564,
-      "service": 565,
-      "thanks": 566,
-      "ye": 567,
-      "charge": 568,
-      "common": 569,
-      "desire": 570,
-      "hours": 571,
-      "lose": 572,
-      "richmond": 573,
-      "senator": 574,
-      "sight": 575,
-      "sin": 576,
-      "'twere": 577,
-      "air": 578,
-      "fie": 579,
-      "fool": 580,
-      "foot": 581,
-      "general": 582,
-      "god's": 583,
-      "itself": 584,
-      "soon": 585,
-      "sure": 586,
-      "tender": 587,
-      "widow": 588,
-      "yield": 589,
-      "arm": 590,
-      "deep": 591,
-      "duty": 592,
-      "enemy": 593,
-      "fault": 594,
-      "prison": 595,
-      "prithee": 596,
-      "women": 597,
-      "yea": 598,
-      "you'll": 599,
-      "command": 600,
-      "deed": 601,
-      "honest": 602,
-      "known": 603,
-      "mad": 604,
-      "past": 605,
-      "patience": 606,
-      "saint": 607,
-      "she's": 608,
-      "trust": 609,
-      "twenty": 610,
-      "unless": 611,
-      "antonio": 612,
-      "bound": 613,
-      "citizens": 614,
-      "cut": 615,
-      "dare": 616,
-      "fearful": 617,
-      "heavens": 618,
-      "heir": 619,
-      "ill": 620,
-      "lewis": 621,
-      "o'er": 622,
-      "oxford": 623,
-      "something": 624,
-      "ten": 625,
-      "yes": 626,
-      "beat": 627,
-      "beauty": 628,
-      "company": 629,
-      "draw": 630,
-      "ear": 631,
-      "gonzalo": 632,
-      "half": 633,
-      "hearts": 634,
-      "open": 635,
-      "revenge": 636,
-      "serve": 637,
-      "voices": 638,
-      "wear": 639,
-      "withal": 640,
-      "brave": 641,
-      "hark": 642,
-      "is't": 643,
-      "marriage": 644,
-      "neither": 645,
-      "point": 646,
-      "read": 647,
-      "side": 648,
-      "sirrah": 649,
-      "tale": 650,
-      "virtue": 651,
-      "worse": 652,
-      "ask": 653,
-      "become": 654,
-      "change": 655,
-      "counsel": 656,
-      "country": 657,
-      "field": 658,
-      "hell": 659,
-      "king's": 660,
-      "knew": 661,
-      "lancaster": 662,
-      "loving": 663,
-      "methinks": 664,
-      "next": 665,
-      "pleasure": 666,
-      "sake": 667,
-      "seek": 668,
-      "work": 669,
-      "youth": 670,
-      "behold": 671,
-      "brother's": 672,
-      "coming": 673,
-      "course": 674,
-      "court": 675,
-      "enemies": 676,
-      "fast": 677,
-      "further": 678,
-      "gates": 679,
-      "given": 680,
-      "met": 681,
-      "miranda": 682,
-      "pale": 683,
-      "tower": 684,
-      "confess": 685,
-      "elbow": 686,
-      "gave": 687,
-      "ha": 688,
-      "ho": 689,
-      "merry": 690,
-      "rich": 691,
-      "run": 692,
-      "souls": 693,
-      "strength": 694,
-      "themselves": 695,
-      "to-day": 696,
-      "took": 697,
-      "along": 698,
-      "banish'd": 699,
-      "between": 700,
-      "curse": 701,
-      "just": 702,
-      "lips": 703,
-      "pass": 704,
-      "peter": 705,
-      "sad": 706,
-      "same": 707,
-      "sworn": 708,
-      "title": 709,
-      "already": 710,
-      "bad": 711,
-      "bold": 712,
-      "bosom": 713,
-      "dream": 714,
-      "england": 715,
-      "george": 716,
-      "goes": 717,
-      "issue": 718,
-      "loved": 719,
-      "office": 720,
-      "perdita": 721,
-      "pluck": 722,
-      "post": 723,
-      "sons": 724,
-      "spoke": 725,
-      "thomas": 726,
-      "whilst": 727,
-      "wind": 728,
-      "fetch": 729,
-      "hadst": 730,
-      "jest": 731,
-      "learn": 732,
-      "loves": 733,
-      "officer": 734,
-      "over": 735,
-      "soldiers": 736,
-      "stanley": 737,
-      "subject": 738,
-      "tribunes": 739,
-      "dangerous": 740,
-      "entreat": 741,
-      "fit": 742,
-      "foe": 743,
-      "forget": 744,
-      "grant": 745,
-      "green": 746,
-      "often": 747,
-      "presence": 748,
-      "sly": 749,
-      "soldier": 750,
-      "speed": 751,
-      "suit": 752,
-      "thence": 753,
-      "to-night": 754,
-      "voice": 755,
-      "wounds": 756,
-      "attend": 757,
-      "black": 758,
-      "gold": 759,
-      "mariana": 760,
-      "mighty": 761,
-      "order": 762,
-      "quarrel": 763,
-      "rage": 764,
-      "request": 765,
-      "seems": 766,
-      "soft": 767,
-      "strong": 768,
-      "throne": 769,
-      "wars": 770,
-      "wert": 771,
-      "wit": 772,
-      "worth": 773,
-      "appear": 774,
-      "battle": 775,
-      "begin": 776,
-      "brief": 777,
-      "buy": 778,
-      "call'd": 779,
-      "case": 780,
-      "consent": 781,
-      "deeds": 782,
-      "deny": 783,
-      "highness": 784,
-      "kings": 785,
-      "lartius": 786,
-      "london": 787,
-      "look'd": 788,
-      "march": 789,
-      "morning": 790,
-      "princely": 791,
-      "somerset": 792,
-      "throw": 793,
-      "want": 794,
-      "wherein": 795,
-      "win": 796,
-      "above": 797,
-      "almost": 798,
-      "ancient": 799,
-      "breast": 800,
-      "consul": 801,
-      "devil": 802,
-      "earl": 803,
-      "faults": 804,
-      "feel": 805,
-      "got": 806,
-      "harm": 807,
-      "heads": 808,
-      "hot": 809,
-      "kill'd": 810,
-      "man's": 811,
-      "move": 812,
-      "prayers": 813,
-      "presently": 814,
-      "pretty": 815,
-      "ratcliff": 816,
-      "speech": 817,
-      "thither": 818,
-      "valiant": 819,
-      "watch": 820,
-      "water": 821,
-      "whence": 822,
-      "wise": 823,
-      "wonder": 824,
-      "3": 825,
-      "alack": 826,
-      "e'er": 827,
-      "five": 828,
-      "forward": 829,
-      "hang": 830,
-      "knock": 831,
-      "least": 832,
-      "mayor": 833,
-      "measure": 834,
-      "mowbray": 835,
-      "murder": 836,
-      "poison": 837,
-      "roman": 838,
-      "slave": 839,
-      "to't": 840,
-      "towards": 841,
-      "whether": 842,
-      "'em": 843,
-      "'t": 844,
-      "anon": 845,
-      "ariel": 846,
-      "awhile": 847,
-      "close": 848,
-      "curtis": 849,
-      "flesh": 850,
-      "hearing": 851,
-      "hereford": 852,
-      "keeper": 853,
-      "living": 854,
-      "moon": 855,
-      "mortal": 856,
-      "music": 857,
-      "promise": 858,
-      "touch": 859,
-      "virgilia": 860,
-      "virtuous": 861,
-      "witness": 862,
-      "wouldst": 863,
-      "antigonus": 864,
-      "base": 865,
-      "besides": 866,
-      "blows": 867,
-      "breathe": 868,
-      "chance": 869,
-      "deserved": 870,
-      "do't": 871,
-      "goodly": 872,
-      "greater": 873,
-      "grow": 874,
-      "guilty": 875,
-      "lawful": 876,
-      "ourselves": 877,
-      "patient": 878,
-      "pay": 879,
-      "quickly": 880,
-      "toward": 881,
-      "wast": 882,
-      "weeping": 883,
-      "wherefore": 884,
-      "worst": 885,
-      "angry": 886,
-      "beg": 887,
-      "derby": 888,
-      "england's": 889,
-      "exeter": 890,
-      "feast": 891,
-      "fled": 892,
-      "golden": 893,
-      "henry's": 894,
-      "laid": 895,
-      "letters": 896,
-      "longer": 897,
-      "mouth": 898,
-      "needs": 899,
-      "note": 900,
-      "offence": 901,
-      "on't": 902,
-      "pedant": 903,
-      "shows": 904,
-      "stir": 905,
-      "truly": 906,
-      "whither": 907,
-      "worship": 908,
-      "action": 909,
-      "amen": 910,
-      "aside": 911,
-      "banishment": 912,
-      "behind": 913,
-      "bohemia": 914,
-      "book": 915,
-      "choose": 916,
-      "clouds": 917,
-      "defend": 918,
-      "doing": 919,
-      "enter": 920,
-      "hard": 921,
-      "he'll": 922,
-      "held": 923,
-      "in't": 924,
-      "isabel": 925,
-      "knee": 926,
-      "ladies": 927,
-      "liberty": 928,
-      "padua": 929,
-      "plain": 930,
-      "rough": 931,
-      "sense": 932,
-      "shed": 933,
-      "short": 934,
-      "slew": 935,
-      "small": 936,
-      "speaks": 937,
-      "subjects": 938,
-      "sudden": 939,
-      "ta'en": 940,
-      "tear": 941,
-      "town": 942,
-      "until": 943,
-      "won": 944,
-      "year": 945,
-      "act": 946,
-      "bawd": 947,
-      "bears": 948,
-      "blow": 949,
-      "bones": 950,
-      "bride": 951,
-      "broke": 952,
-      "certain": 953,
-      "conscience": 954,
-      "deliver": 955,
-      "dry": 956,
-      "forgot": 957,
-      "hide": 958,
-      "hundred": 959,
-      "kingdom": 960,
-      "lead": 961,
-      "leisure": 962,
-      "letter": 963,
-      "lived": 964,
-      "masters": 965,
-      "neck": 966,
-      "noise": 967,
-      "oft": 968,
-      "piece": 969,
-      "plantagenet": 970,
-      "prisoner": 971,
-      "stood": 972,
-      "victory": 973,
-      "visit": 974,
-      "'twixt": 975,
-      "adieu": 976,
-      "alive": 977,
-      "barnardine": 978,
-      "bitter": 979,
-      "blessed": 980,
-      "bushy": 981,
-      "chamber": 982,
-      "commend": 983,
-      "country's": 984,
-      "danger": 985,
-      "drink": 986,
-      "few": 987,
-      "fine": 988,
-      "forbid": 989,
-      "four": 990,
-      "glad": 991,
-      "glory": 992,
-      "herself": 993,
-      "kept": 994,
-      "knees": 995,
-      "knowledge": 996,
-      "lands": 997,
-      "morrow": 998,
-      "pride": 999
+      "a": 10,
+      "in": 11,
+      "\"": 12,
+      "that": 13,
+      "was": 14,
+      "is": 15,
+      "for": 16,
+      "on": 17,
+      "with": 18,
+      "as": 19,
+      "it": 20,
+      "he": 21,
+      "his": 22,
+      "at": 23,
+      "by": 24,
+      "i": 25,
+      ";": 26,
+      "be": 27,
+      "from": 28,
+      "<bos>": 29,
+      "<eos>": 30,
+      "not": 31,
+      ")": 32,
+      "but": 33,
+      "(": 34,
+      "this": 35,
+      "had": 36,
+      "which": 37,
+      "have": 38,
+      "are": 39,
+      ":": 40,
+      "they": 41,
+      "an": 42,
+      "or": 43,
+      "were": 44,
+      "her": 45,
+      "said": 46,
+      "their": 47,
+      "has": 48,
+      "we": 49,
+      "all": 50,
+      "who": 51,
+      "one": 52,
+      "she": 53,
+      "you": 54,
+      "“": 55,
+      "”": 56,
+      "been": 57,
+      "will": 58,
+      "more": 59,
+      "him": 60,
+      "when": 61,
+      "so": 62,
+      "its": 63,
+      "would": 64,
+      "there": 65,
+      "no": 66,
+      "after": 67,
+      "my": 68,
+      "if": 69,
+      "out": 70,
+      "also": 71,
+      "about": 72,
+      "them": 73,
+      "up": 74,
+      "into": 75,
+      "two": 76,
+      "what": 77,
+      "new": 78,
+      "first": 79,
+      "some": 80,
+      "can": 81,
+      "time": 82,
+      "than": 83,
+      "other": 84,
+      "our": 85,
+      "me": 86,
+      "only": 87,
+      "over": 88,
+      "could": 89,
+      "may": 90,
+      "?": 91,
+      "any": 92,
+      "/": 93,
+      "then": 94,
+      "people": 95,
+      "like": 96,
+      "now": 97,
+      "such": 98,
+      "do": 99
     },
     "unk_token": "<unk>"
   }