IlyasMoutawwakil's picture
Upload folder using huggingface_hub (part 2)
d4aa5aa verified
Raw
History Blame Contribute Delete
21.8 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "[UNK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "[CLS]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "[SEP]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 4,
"content": "[MASK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": {
"type": "BertNormalizer",
"clean_text": true,
"handle_chinese_chars": true,
"strip_accents": null,
"lowercase": true
},
"pre_tokenizer": {
"type": "BertPreTokenizer"
},
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"SpecialToken": {
"id": "[CLS]",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[SEP]",
"type_id": 0
}
}
],
"pair": [
{
"SpecialToken": {
"id": "[CLS]",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "[SEP]",
"type_id": 0
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
},
{
"SpecialToken": {
"id": "[SEP]",
"type_id": 1
}
}
],
"special_tokens": {
"[CLS]": {
"id": "[CLS]",
"ids": [
2
],
"tokens": [
"[CLS]"
]
},
"[SEP]": {
"id": "[SEP]",
"ids": [
3
],
"tokens": [
"[SEP]"
]
}
}
},
"decoder": {
"type": "WordPiece",
"prefix": "##",
"cleanup": true
},
"model": {
"type": "WordPiece",
"unk_token": "[UNK]",
"continuing_subword_prefix": "##",
"max_input_chars_per_word": 100,
"vocab": {
"[PAD]": 0,
"[UNK]": 1,
"[CLS]": 2,
"[SEP]": 3,
"[MASK]": 4,
"!": 5,
"\"": 6,
"#": 7,
"$": 8,
"%": 9,
"&": 10,
"'": 11,
"(": 12,
")": 13,
"*": 14,
"+": 15,
",": 16,
"-": 17,
".": 18,
"/": 19,
"0": 20,
"1": 21,
"2": 22,
"3": 23,
"4": 24,
"5": 25,
"6": 26,
"7": 27,
"8": 28,
"9": 29,
":": 30,
";": 31,
"<": 32,
"=": 33,
">": 34,
"?": 35,
"@": 36,
"[": 37,
"]": 38,
"^": 39,
"`": 40,
"a": 41,
"b": 42,
"c": 43,
"d": 44,
"e": 45,
"f": 46,
"g": 47,
"h": 48,
"i": 49,
"j": 50,
"k": 51,
"l": 52,
"m": 53,
"n": 54,
"o": 55,
"p": 56,
"q": 57,
"r": 58,
"s": 59,
"t": 60,
"u": 61,
"v": 62,
"w": 63,
"x": 64,
"y": 65,
"z": 66,
"~": 67,
"£": 68,
"°": 69,
"²": 70,
"´": 71,
"µ": 72,
"º": 73,
"×": 74,
"ß": 75,
"æ": 76,
"đ": 77,
"ł": 78,
"ə": 79,
"ɡ": 80,
"ɪ": 81,
"ˈ": 82,
"α": 83,
"β": 84,
"γ": 85,
"δ": 86,
"ε": 87,
"η": 88,
"θ": 89,
"ι": 90,
"κ": 91,
"λ": 92,
"μ": 93,
"ν": 94,
"ξ": 95,
"ο": 96,
"ρ": 97,
"ς": 98,
"σ": 99,
"τ": 100,
"υ": 101,
"χ": 102,
"а": 103,
"в": 104,
"з": 105,
"и": 106,
"к": 107,
"н": 108,
"р": 109,
"с": 110,
"ы": 111,
"є": 112,
"ה": 113,
"מ": 114,
"צ": 115,
"ש": 116,
"ا": 117,
"ت": 118,
"ث": 119,
"د": 120,
"س": 121,
"م": 122,
"ي": 123,
"ก": 124,
"ค": 125,
"ง": 126,
"ช": 127,
"ซ": 128,
"ญ": 129,
"ฐ": 130,
"ณ": 131,
"ด": 132,
"ต": 133,
"บ": 134,
"ป": 135,
"พ": 136,
"ภ": 137,
"ม": 138,
"ย": 139,
"ร": 140,
"ล": 141,
"ว": 142,
"ศ": 143,
"ษ": 144,
"ส": 145,
"ห": 146,
"อ": 147,
"า": 148,
"เ": 149,
"แ": 150,
"ไ": 151,
"ᵻ": 152,
"–": 153,
"—": 154,
"‘": 155,
"’": 156,
"“": 157,
"”": 158,
"†": 159,
"‡": 160,
"…": 161,
"⁄": 162,
"₤": 163,
"₹": 164,
"→": 165,
"−": 166,
"い": 167,
"う": 168,
"か": 169,
"き": 170,
"く": 171,
"し": 172,
"た": 173,
"つ": 174,
"な": 175,
"の": 176,
"め": 177,
"も": 178,
"ら": 179,
"ァ": 180,
"ア": 181,
"ウ": 182,
"キ": 183,
"ュ": 184,
"リ": 185,
"ル": 186,
"乙": 187,
"光": 188,
"其": 189,
"具": 190,
"华": 191,
"厂": 192,
"名": 193,
"君": 194,
"命": 195,
"咲": 196,
"場": 197,
"女": 198,
"建": 199,
"戦": 200,
"日": 201,
"旭": 202,
"灯": 203,
"玩": 204,
"琪": 205,
"瘡": 206,
"肖": 207,
"花": 208,
"誓": 209,
"誰": 210,
"许": 211,
"赤": 212,
"運": 213,
"銃": 214,
"願": 215,
"3": 216,
"##p": 217,
"##i": 218,
"##s": 219,
"##o": 220,
"##d": 221,
"##e": 222,
"##u": 223,
"##l": 224,
"##t": 225,
"##f": 226,
"##a": 227,
"##c": 228,
"##h": 229,
"##r": 230,
"##g": 231,
"##k": 232,
"##n": 233,
"##x": 234,
"##v": 235,
"##w": 236,
"##y": 237,
"##z": 238,
"##m": 239,
"##b": 240,
"##q": 241,
"##4": 242,
"##2": 243,
"##3": 244,
"##5": 245,
"##0": 246,
"##1": 247,
"##8": 248,
"##7": 249,
"##ウ": 250,
"##ァ": 251,
"##ル": 252,
"##キ": 253,
"##ュ": 254,
"##リ": 255,
"##ア": 256,
"##ł": 257,
"##9": 258,
"##j": 259,
"##6": 260,
"##の": 261,
"##ɡ": 262,
"##æ": 263,
"##ᵻ": 264,
"##ə": 265,
"##ภ": 266,
"##า": 267,
"##ด": 268,
"##อ": 269,
"##ก": 270,
"##ว": 271,
"##ร": 272,
"##ษ": 273,
"##ฐ": 274,
"##τ": 275,
"##ι": 276,
"##μ": 277,
"##γ": 278,
"##ρ": 279,
"##ο": 280,
"##ς": 281,
"##υ": 282,
"##є": 283,
"##λ": 284,
"##พ": 285,
"##し": 286,
"##も": 287,
"##3": 288,
"##な": 289,
"##き": 290,
"##ซ": 291,
"##ß": 292,
"##а": 293,
"##в": 294,
"##к": 295,
"##з": 296,
"##α": 297,
"##р": 298,
"##с": 299,
"##н": 300,
"##ы": 301,
"##и": 302,
"##ら": 303,
"##เ": 304,
"##ป": 305,
"##ย": 306,
"##ล": 307,
"##ง": 308,
"##ไ": 309,
"##ช": 310,
"##ث": 311,
"##م": 312,
"##د": 313,
"##β": 314,
"##đ": 315,
"##ي": 316,
"##س": 317,
"##ت": 318,
"##ν": 319,
"##χ": 320,
"##แ": 321,
"##ש": 322,
"##מ": 323,
"##צ": 324,
"##ה": 325,
"##ค": 326,
"##ต": 327,
"##ε": 328,
"##ξ": 329,
"##θ": 330,
"##η": 331,
"##た": 332,
"##め": 333,
"##つ": 334,
"##か": 335,
"##º": 336,
"##ณ": 337,
"##ม": 338,
"##ɪ": 339,
"th": 340,
"the": 341,
"##er": 342,
"##in": 343,
"##ed": 344,
"##on": 345,
"##nd": 346,
"##at": 347,
"##or": 348,
"##es": 349,
"in": 350,
"##en": 351,
"##ar": 352,
"of": 353,
"##it": 354,
"##al": 355,
"##an": 356,
"##is": 357,
"##ing": 358,
"##as": 359,
"and": 360,
"##ic": 361,
"to": 362,
"##ou": 363,
"##ion": 364,
"##re": 365,
"##om": 366,
"##el": 367,
"##il": 368,
"##ol": 369,
"##ec": 370,
"##ent": 371,
"##st": 372,
"##th": 373,
"##ur": 374,
"on": 375,
"##am": 376,
"##ad": 377,
"st": 378,
"##et": 379,
"was": 380,
"##ac": 381,
"##ro": 382,
"for": 383,
"##ir": 384,
"wh": 385,
"as": 386,
"##ly": 387,
"re": 388,
"##id": 389,
"##ith": 390,
"##iv": 391,
"##le": 392,
"##ation": 393,
"al": 394,
"##ig": 395,
"##ers": 396,
"is": 397,
"##im": 398,
"with": 399,
"##ch": 400,
"##ay": 401,
"that": 402,
"be": 403,
"##ow": 404,
"##ts": 405,
"##em": 406,
"##ter": 407,
"an": 408,
"sh": 409,
"ch": 410,
"##ul": 411,
"##ut": 412,
"##ot": 413,
"##um": 414,
"he": 415,
"##us": 416,
"con": 417,
"fr": 418,
"by": 419,
"at": 420,
"##os": 421,
"his": 422,
"##est": 423,
"##ra": 424,
"19": 425,
"##ver": 426,
"##ce": 427,
"##and": 428,
"##av": 429,
"##od": 430,
"##ag": 431,
"##ere": 432,
"com": 433,
"se": 434,
"de": 435,
"##ies": 436,
"it": 437,
"20": 438,
"##ain": 439,
"##ct": 440,
"pro": 441,
"su": 442,
"##ist": 443,
"##ess": 444,
"ar": 445,
"##op": 446,
"from": 447,
"or": 448,
"##oc": 449,
"##ew": 450,
"##pp": 451,
"##if": 452,
"##ity": 453,
"were": 454,
"##igh": 455,
"##ich": 456,
"##ish": 457,
"ex": 458,
"pr": 459,
"##un": 460,
"##all": 461,
"pl": 462,
"##ate": 463,
"le": 464,
"##ated": 465,
"##ord": 466,
"##se": 467,
"##ect": 468,
"##ia": 469,
"un": 470,
"##ort": 471,
"##ab": 472,
"are": 473,
"##ear": 474,
"##ill": 475,
"##ak": 476,
"##gh": 477,
"##ber": 478,
"##pt": 479,
"##ther": 480,
"sp": 481,
"tr": 482,
"##ud": 483,
"##ard": 484,
"##ian": 485,
"##art": 486,
"##ld": 487,
"##ial": 488,
"##ong": 489,
"which": 490,
"##ame": 491,
"##ip": 492,
"had": 493,
"##og": 494,
"##ment": 495,
"rec": 496,
"##ive": 497,
"##ough": 498,
"##ound": 499,
"##rit": 500,
"##ov": 501,
"##ore": 502,
"not": 503,
"##ost": 504,
"##our": 505,
"##ant": 506,
"##ell": 507,
"##ap": 508,
"##ue": 509,
"cl": 510,
"en": 511,
"##ib": 512,
"af": 513,
"##ast": 514,
"her": 515,
"##ge": 516,
"##ave": 517,
"##ary": 518,
"ac": 519,
"tw": 520,
"all": 521,
"##ack": 522,
"wor": 523,
"##ust": 524,
"##ical": 525,
"200": 526,
"this": 527,
"she": 528,
"##ine": 529,
"##ight": 530,
"am": 531,
"##pl": 532,
"##so": 533,
"inc": 534,
"also": 535,
"##ure": 536,
"rel": 537,
"##out": 538,
"##ru": 539,
"their": 540,
"ad": 541,
"##ach": 542,
"##oll": 543,
"has": 544,
"comm": 545,
"##ie": 546,
"##qu": 547,
"##ous": 548,
"bo": 549,
"ab": 550,
"fir": 551,
"sc": 552,
"per": 553,
"##ath": 554,
"##ome": 555,
"##ide": 556,
"one": 557,
"##ub": 558,
"after": 559,
"cont": 560,
"##ign": 561,
"new": 562,
"te": 563,
"its": 564,
"res": 565,
"##oun": 566,
"tim": 567,
"201": 568,
"##ition": 569,
"fe": 570,
"first": 571,
"two": 572,
"fin": 573,
"##ater": 574,
"int": 575,
"man": 576,
"##ass": 577,
"##ke": 578,
"##ations": 579,
"pol": 580,
"##wn": 581,
"but": 582,
"##ans": 583,
"go": 584,
"##ual": 585,
"car": 586,
"mar": 587,
"##act": 588,
"they": 589,
"des": 590,
"ind": 591,
"##00": 592,
"im": 593,
"##ond": 594,
"play": 595,
"18": 596,
"##ould": 597,
"off": 598,
"##ory": 599,
"ag": 600,
"have": 601,
"##iz": 602,
"year": 603,
"who": 604,
"##per": 605,
"##orm": 606,
"##uring": 607,
"##own": 608,
"##nder": 609,
"##ire": 610,
"##du": 611,
"bec": 612,
"##ied": 613,
"##ph": 614,
"me": 615,
"co": 616,
"##ff": 617,
"##ance": 618,
"been": 619,
"##ok": 620,
"##ely": 621,
"rep": 622,
"##age": 623,
"dis": 624,
"##ree": 625,
"app": 626,
"##ason": 627,
"##ence": 628,
"us": 629,
"part": 630,
"##lud": 631,
"bl": 632,
"air": 633,
"##ult": 634,
"gr": 635,
"ser": 636,
"##ents": 637,
"##ions": 638,
"##old": 639,
"god": 640,
"over": 641,
"pre": 642,
"sec": 643,
"##ry": 644,
"##ember": 645,
"##ric": 646,
"##ited": 647,
"other": 648,
"ev": 649,
"fl": 650,
"dec": 651,
"up": 652,
"##ite": 653,
"##ile": 654,
"during": 655,
"ne": 656,
"##ities": 657,
"act": 658,
"can": 659,
"time": 660,
"fil": 661,
"under": 662,
"ear": 663,
"mus": 664,
"comp": 665,
"##vi": 666,
"includ": 667,
"##ced": 668,
"##ang": 669,
"##ade": 670,
"##ks": 671,
"##olog": 672,
"##ose": 673,
"form": 674,
"##ase": 675,
"film": 676,
"record": 677,
"##end": 678,
"##one": 679,
"inter": 680,
"season": 681,
"war": 682,
"##ased": 683,
"##erm": 684,
"more": 685,
"##ace": 686,
"alb": 687,
"##ates": 688,
"them": 689,
"##ens": 690,
"ass": 691,
"reg": 692,
"cons": 693,
"we": 694,
"##ubl": 695,
"##ild": 696,
"##cc": 697,
"most": 698,
"##fer": 699,
"##ors": 700,
"##ice": 701,
"rem": 702,
"att": 703,
"album": 704,
"##land": 705,
"##ft": 706,
"char": 707,
"##oin": 708,
"##ward": 709,
"##ted": 710,
"##ail": 711,
"may": 712,
"##ally": 713,
"##ole": 714,
"##able": 715,
"gen": 716,
"no": 717,
"sch": 718,
"cent": 719,
"spec": 720,
"acc": 721,
"him": 722,
"##rough": 723,
"high": 724,
"##gan": 725,
"there": 726,
"ant": 727,
"min": 728,
"##gr": 729,
"mon": 730,
"##ings": 731,
"##les": 732,
"into": 733,
"199": 734,
"##reat": 735,
"##urn": 736,
"bu": 737,
"##ces": 738,
"when": 739,
"end": 740,
"nor": 741,
"out": 742,
"kn": 743,
"some": 744,
"##hip": 745,
"chr": 746,
"city": 747,
"##rib": 748,
"work": 749,
"##ix": 750,
"##form": 751,
"second": 752,
"##ved": 753,
"##ock": 754,
"##ction": 755,
"brit": 756,
"publ": 757,
"sou": 758,
"##ered": 759,
"##ular": 760,
"##orn": 761,
"arch": 762,
"##ve": 763,
"##ames": 764,
"##ordan": 765,
"four": 766,
"jordan": 767,
"hist": 768,
"pe": 769,
"##ational": 770,
"ro": 771,
"##ood": 772,
"##aw": 773,
"team": 774,
"sub": 775,
"em": 776,
"##een": 777,
"three": 778,
"only": 779,
"par": 780,
"##res": 781,
"win": 782,
"bet": 783,
"later": 784,
"##ons": 785,
"num": 786,
"produ": 787,
"again": 788,
"ret": 789,
"##vel": 790,
"##ae": 791,
"##eld": 792,
"rele": 793,
"ph": 794,
"sy": 795,
"##oss": 796,
"imp": 797,
"game": 798,
"li": 799,
"##way": 800,
"through": 801,
"whe": 802,
"inst": 803,
"dep": 804,
"##ished": 805,
"would": 806,
"##aus": 807,
"##ict": 808,
"##erson": 809,
"long": 810,
"while": 811,
"##iss": 812,
"##ount": 813,
"sur": 814,
"##als": 815,
"ma": 816,
"these": 817,
"being": 818,
"about": 819,
"##ri": 820,
"than": 821,
"##ull": 822,
"##vest": 823,
"##ins": 824,
"##ork": 825,
"##ick": 826,
"br": 827,
"col": 828,
"arm": 829,
"how": 830,
"lar": 831,
"##ony": 832,
"many": 833,
"do": 834,
"##tle": 835,
"##eler": 836,
"pres": 837,
"writ": 838,
"such": 839,
"sa": 840,
"sm": 841,
"##te": 842,
"##ake": 843,
"##urch": 844,
"bro": 845,
"##ily": 846,
"so": 847,
"au": 848,
"##tern": 849,
"gal": 850,
"town": 851,
"wheeler": 852,
"##ific": 853,
"##ious": 854,
"##ween": 855,
"between": 856,
"where": 857,
"sever": 858,
"##ism": 859,
"gener": 860,
"##aeolog": 861,
"000": 862,
"bel": 863,
"def": 864,
"north": 865,
"made": 866,
"art": 867,
"##ollow": 868,
"##air": 869,
"pop": 870,
"music": 871,
"archaeolog": 872,
"follow": 873,
"jo": 874,
"song": 875,
"##other": 876,
"##arn": 877,
"el": 878,
"##red": 879,
"##ating": 880,
"ship": 881,
"supp": 882,
"##amp": 883,
"south": 884,
"loc": 885,
"sign": 886,
"exp": 887,
"##ove": 888,
"##though": 889,
"although": 890,
"world": 891,
"##fore": 892,
"##ower": 893,
"christ": 894,
"british": 895,
"##veston": 896,
"galveston": 897,
"day": 898,
"both": 899,
"desc": 900,
"dire": 901,
"germ": 902,
"##iet": 903,
"##row": 904,
"church": 905,
"command": 906,
"met": 907,
"##ten": 908,
"##hn": 909,
"##ised": 910,
"stud": 911,
"contr": 912,
"cre": 913,
"oper": 914,
"lead": 915,
"call": 916,
"div": 917,
"ob": 918,
"number": 919,
"then": 920,
"##oup": 921,
"##ool": 922,
"##ning": 923,
"##eric": 924,
"years": 925,
"gods": 926,
"sing": 927,
"##up": 928,
"##ics": 929,
"care": 930,
"gu": 931,
"nov": 932,
"york": 933,
"##ever": 934,
"state": 935,
"##ife": 936,
"trans": 937,
"used": 938,
"early": 939,
"##ived": 940,
"several": 941,
"##gy": 942,
"##ern": 943,
"##io": 944,
"##ute": 945,
"add": 946,
"main": 947,
"well": 948,
"##ale": 949,
"##sp": 950,
"class": 951,
"##view": 952,
"direct": 953,
"crit": 954,
"my": 955,
"var": 956,
"##ron": 957,
"before": 958,
"prom": 959,
"##ause": 960,
"bar": 961,
"dist": 962,
"dif": 963,
"qu": 964,
"##ral": 965,
"##ste": 966,
"##ected": 967,
"united": 968,
"back": 969,
"orig": 970,
"perform": 971,
"offic": 972,
"tra": 973,
"americ": 974,
"##fl": 975,
"##ures": 976,
"##elf": 977,
"school": 978,
"peop": 979,
"bat": 980,
"sl": 981,
"##ivers": 982,
"against": 983,
"##ress": 984,
"##ists": 985,
"est": 986,
"##kel": 987,
"##raft": 988,
"book": 989,
"disc": 990,
"port": 991,
"##az": 992,
"##ros": 993,
"eng": 994,
"became": 995,
"count": 996,
"found": 997,
"##ty": 998,
"##ery": 999,
"states": 1000,
"##ized": 1001,
"like": 1002,
"origin": 1003,
"hum": 1004,
"##ield": 1005,
"##its": 1006,
"##emb": 1007,
"194": 1008,
"198": 1009,
"antim": 1010,
"people": 1011,
"award": 1012,
"ge": 1013,
"lo": 1014,
"lit": 1015,
"resp": 1016,
"remain": 1017,
"known": 1018,
"##stein": 1019,
"public": 1020,
"fur": 1021,
"however": 1022,
"said": 1023
}
}
}