bs-tokenizer / tokenizer.json
vpalod's picture
Upload folder using huggingface_hub
ca4c739 verified
{
"version": "1.0",
"truncation": null,
"padding": {
"strategy": "BatchLongest",
"direction": "Right",
"pad_to_multiple_of": null,
"pad_id": 940,
"pad_type_id": 0,
"pad_token": "<|pad|>"
},
"added_tokens": [
{
"id": 940,
"content": "<|pad|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 941,
"content": "<|unk|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 942,
"content": "<|bos|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 943,
"content": "<|eos|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "Whitespace"
},
"post_processor": null,
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {
"c0": 0,
"c1": 1,
"c2": 2,
"c3": 3,
"c4": 4,
"c5": 5,
"c6": 6,
"c7": 7,
"c8": 8,
"c9": 9,
"c10": 10,
"c11": 11,
"c12": 12,
"c13": 13,
"c14": 14,
"c15": 15,
"c16": 16,
"c17": 17,
"c18": 18,
"c19": 19,
"c20": 20,
"c21": 21,
"c22": 22,
"c23": 23,
"c24": 24,
"c25": 25,
"c26": 26,
"c27": 27,
"c28": 28,
"c29": 29,
"c30": 30,
"c31": 31,
"c32": 32,
"c33": 33,
"c34": 34,
"c35": 35,
"c36": 36,
"c37": 37,
"c38": 38,
"c39": 39,
"c40": 40,
"c41": 41,
"c42": 42,
"c43": 43,
"c44": 44,
"c45": 45,
"c46": 46,
"c47": 47,
"c48": 48,
"c49": 49,
"c50": 50,
"c51": 51,
"c52": 52,
"c53": 53,
"c54": 54,
"c55": 55,
"c56": 56,
"c57": 57,
"c58": 58,
"c59": 59,
"c60": 60,
"c61": 61,
"c62": 62,
"c63": 63,
"c64": 64,
"c65": 65,
"c66": 66,
"c67": 67,
"c68": 68,
"c69": 69,
"c70": 70,
"c71": 71,
"c72": 72,
"c73": 73,
"c74": 74,
"c75": 75,
"c76": 76,
"c77": 77,
"c78": 78,
"c79": 79,
"c80": 80,
"c81": 81,
"c82": 82,
"c83": 83,
"c84": 84,
"c85": 85,
"c86": 86,
"c87": 87,
"c88": 88,
"c89": 89,
"c90": 90,
"c91": 91,
"c92": 92,
"c93": 93,
"c94": 94,
"c95": 95,
"c96": 96,
"c97": 97,
"c98": 98,
"c99": 99,
"c100": 100,
"c101": 101,
"c102": 102,
"c103": 103,
"c104": 104,
"c105": 105,
"c106": 106,
"c107": 107,
"c108": 108,
"c109": 109,
"c110": 110,
"c111": 111,
"c112": 112,
"c113": 113,
"c114": 114,
"c115": 115,
"c116": 116,
"c117": 117,
"c118": 118,
"c119": 119,
"c120": 120,
"c121": 121,
"c122": 122,
"c123": 123,
"c124": 124,
"c125": 125,
"c126": 126,
"c127": 127,
"c128": 128,
"c129": 129,
"c130": 130,
"c131": 131,
"c132": 132,
"c133": 133,
"c134": 134,
"c135": 135,
"c136": 136,
"c137": 137,
"c138": 138,
"c139": 139,
"c140": 140,
"c141": 141,
"c142": 142,
"c143": 143,
"c144": 144,
"c145": 145,
"c146": 146,
"c147": 147,
"c148": 148,
"c149": 149,
"c150": 150,
"c151": 151,
"c152": 152,
"c153": 153,
"c154": 154,
"c155": 155,
"c156": 156,
"c157": 157,
"c158": 158,
"c159": 159,
"c160": 160,
"c161": 161,
"c162": 162,
"c163": 163,
"c164": 164,
"c165": 165,
"c166": 166,
"c167": 167,
"c168": 168,
"c169": 169,
"c170": 170,
"c171": 171,
"c172": 172,
"c173": 173,
"c174": 174,
"c175": 175,
"c176": 176,
"c177": 177,
"c178": 178,
"c179": 179,
"c180": 180,
"c181": 181,
"c182": 182,
"c183": 183,
"c184": 184,
"c185": 185,
"c186": 186,
"c187": 187,
"c188": 188,
"c189": 189,
"c190": 190,
"c191": 191,
"c192": 192,
"c193": 193,
"c194": 194,
"c195": 195,
"c196": 196,
"c197": 197,
"c198": 198,
"c199": 199,
"c200": 200,
"c201": 201,
"c202": 202,
"c203": 203,
"c204": 204,
"c205": 205,
"c206": 206,
"c207": 207,
"c208": 208,
"c209": 209,
"c210": 210,
"c211": 211,
"c212": 212,
"c213": 213,
"c214": 214,
"c215": 215,
"c216": 216,
"c217": 217,
"c218": 218,
"c219": 219,
"c220": 220,
"c221": 221,
"c222": 222,
"c223": 223,
"c224": 224,
"c225": 225,
"c226": 226,
"c227": 227,
"c228": 228,
"c229": 229,
"c230": 230,
"c231": 231,
"c232": 232,
"c233": 233,
"c234": 234,
"c235": 235,
"c236": 236,
"c237": 237,
"c238": 238,
"c239": 239,
"c240": 240,
"c241": 241,
"c242": 242,
"c243": 243,
"c244": 244,
"c245": 245,
"c246": 246,
"c247": 247,
"c248": 248,
"c249": 249,
"c250": 250,
"c251": 251,
"c252": 252,
"c253": 253,
"c254": 254,
"c255": 255,
"c256": 256,
"c257": 257,
"c258": 258,
"c259": 259,
"c260": 260,
"c261": 261,
"c262": 262,
"c263": 263,
"c264": 264,
"c265": 265,
"c266": 266,
"c267": 267,
"c268": 268,
"c269": 269,
"c270": 270,
"c271": 271,
"c272": 272,
"c273": 273,
"c274": 274,
"c275": 275,
"c276": 276,
"c277": 277,
"c278": 278,
"c279": 279,
"c280": 280,
"c281": 281,
"c282": 282,
"c283": 283,
"c284": 284,
"c285": 285,
"c286": 286,
"c287": 287,
"c288": 288,
"c289": 289,
"c290": 290,
"c291": 291,
"c292": 292,
"c293": 293,
"c294": 294,
"c295": 295,
"c296": 296,
"c297": 297,
"c298": 298,
"c299": 299,
"c300": 300,
"c301": 301,
"c302": 302,
"c303": 303,
"c304": 304,
"c305": 305,
"c306": 306,
"c307": 307,
"c308": 308,
"c309": 309,
"c310": 310,
"c311": 311,
"c312": 312,
"c313": 313,
"c314": 314,
"c315": 315,
"c316": 316,
"c317": 317,
"c318": 318,
"c319": 319,
"c320": 320,
"c321": 321,
"c322": 322,
"c323": 323,
"c324": 324,
"c325": 325,
"c326": 326,
"c327": 327,
"c328": 328,
"c329": 329,
"c330": 330,
"c331": 331,
"c332": 332,
"c333": 333,
"c334": 334,
"c335": 335,
"c336": 336,
"c337": 337,
"c338": 338,
"c339": 339,
"c340": 340,
"c341": 341,
"c342": 342,
"c343": 343,
"c344": 344,
"c345": 345,
"c346": 346,
"c347": 347,
"c348": 348,
"c349": 349,
"c350": 350,
"c351": 351,
"c352": 352,
"c353": 353,
"c354": 354,
"c355": 355,
"c356": 356,
"c357": 357,
"c358": 358,
"c359": 359,
"c360": 360,
"c361": 361,
"c362": 362,
"c363": 363,
"c364": 364,
"c365": 365,
"c366": 366,
"c367": 367,
"c368": 368,
"c369": 369,
"c370": 370,
"c371": 371,
"c372": 372,
"c373": 373,
"c374": 374,
"c375": 375,
"c376": 376,
"c377": 377,
"c378": 378,
"c379": 379,
"c380": 380,
"c381": 381,
"c382": 382,
"c383": 383,
"c384": 384,
"c385": 385,
"c386": 386,
"c387": 387,
"c388": 388,
"c389": 389,
"c390": 390,
"c391": 391,
"c392": 392,
"c393": 393,
"c394": 394,
"c395": 395,
"c396": 396,
"c397": 397,
"c398": 398,
"c399": 399,
"c400": 400,
"c401": 401,
"c402": 402,
"c403": 403,
"c404": 404,
"c405": 405,
"c406": 406,
"c407": 407,
"c408": 408,
"c409": 409,
"c410": 410,
"c411": 411,
"c412": 412,
"c413": 413,
"c414": 414,
"c415": 415,
"c416": 416,
"c417": 417,
"c418": 418,
"c419": 419,
"c420": 420,
"c421": 421,
"c422": 422,
"c423": 423,
"c424": 424,
"c425": 425,
"c426": 426,
"c427": 427,
"c428": 428,
"c429": 429,
"c430": 430,
"c431": 431,
"c432": 432,
"c433": 433,
"c434": 434,
"c435": 435,
"c436": 436,
"c437": 437,
"c438": 438,
"c439": 439,
"c440": 440,
"c441": 441,
"c442": 442,
"c443": 443,
"c444": 444,
"c445": 445,
"c446": 446,
"c447": 447,
"c448": 448,
"c449": 449,
"c450": 450,
"c451": 451,
"c452": 452,
"c453": 453,
"c454": 454,
"c455": 455,
"c456": 456,
"c457": 457,
"c458": 458,
"c459": 459,
"c460": 460,
"c461": 461,
"c462": 462,
"c463": 463,
"c464": 464,
"c465": 465,
"c466": 466,
"c467": 467,
"c468": 468,
"c469": 469,
"c470": 470,
"c471": 471,
"c472": 472,
"c473": 473,
"c474": 474,
"c475": 475,
"c476": 476,
"c477": 477,
"c478": 478,
"c479": 479,
"c480": 480,
"c481": 481,
"c482": 482,
"c483": 483,
"c484": 484,
"c485": 485,
"c486": 486,
"c487": 487,
"c488": 488,
"c489": 489,
"c490": 490,
"c491": 491,
"c492": 492,
"c493": 493,
"c494": 494,
"c495": 495,
"c496": 496,
"c497": 497,
"c498": 498,
"c499": 499,
"c500": 500,
"c501": 501,
"c502": 502,
"c503": 503,
"c504": 504,
"c505": 505,
"c506": 506,
"c507": 507,
"c508": 508,
"c509": 509,
"c510": 510,
"c511": 511,
"c512": 512,
"c513": 513,
"c514": 514,
"c515": 515,
"c516": 516,
"c517": 517,
"c518": 518,
"c519": 519,
"c520": 520,
"c521": 521,
"c522": 522,
"c523": 523,
"c524": 524,
"c525": 525,
"c526": 526,
"c527": 527,
"c528": 528,
"c529": 529,
"c530": 530,
"c531": 531,
"c532": 532,
"c533": 533,
"c534": 534,
"c535": 535,
"c536": 536,
"c537": 537,
"c538": 538,
"c539": 539,
"c540": 540,
"c541": 541,
"c542": 542,
"c543": 543,
"c544": 544,
"c545": 545,
"c546": 546,
"c547": 547,
"c548": 548,
"c549": 549,
"c550": 550,
"c551": 551,
"c552": 552,
"c553": 553,
"c554": 554,
"c555": 555,
"c556": 556,
"c557": 557,
"c558": 558,
"c559": 559,
"c560": 560,
"c561": 561,
"c562": 562,
"c563": 563,
"c564": 564,
"c565": 565,
"c566": 566,
"c567": 567,
"c568": 568,
"c569": 569,
"c570": 570,
"c571": 571,
"c572": 572,
"c573": 573,
"c574": 574,
"c575": 575,
"c576": 576,
"c577": 577,
"c578": 578,
"c579": 579,
"c580": 580,
"c581": 581,
"c582": 582,
"c583": 583,
"c584": 584,
"c585": 585,
"c586": 586,
"c587": 587,
"c588": 588,
"c589": 589,
"c590": 590,
"c591": 591,
"c592": 592,
"c593": 593,
"c594": 594,
"c595": 595,
"c596": 596,
"c597": 597,
"c598": 598,
"c599": 599,
"c600": 600,
"c601": 601,
"c602": 602,
"c603": 603,
"c604": 604,
"c605": 605,
"c606": 606,
"c607": 607,
"c608": 608,
"c609": 609,
"c610": 610,
"c611": 611,
"c612": 612,
"c613": 613,
"c614": 614,
"c615": 615,
"c616": 616,
"c617": 617,
"c618": 618,
"c619": 619,
"c620": 620,
"c621": 621,
"c622": 622,
"c623": 623,
"c624": 624,
"c625": 625,
"c626": 626,
"c627": 627,
"c628": 628,
"c629": 629,
"c630": 630,
"c631": 631,
"c632": 632,
"c633": 633,
"c634": 634,
"c635": 635,
"c636": 636,
"c637": 637,
"c638": 638,
"c639": 639,
"c640": 640,
"c641": 641,
"c642": 642,
"c643": 643,
"c644": 644,
"c645": 645,
"c646": 646,
"c647": 647,
"c648": 648,
"c649": 649,
"c650": 650,
"c651": 651,
"c652": 652,
"c653": 653,
"c654": 654,
"c655": 655,
"c656": 656,
"c657": 657,
"c658": 658,
"c659": 659,
"c660": 660,
"c661": 661,
"c662": 662,
"c663": 663,
"c664": 664,
"c665": 665,
"c666": 666,
"c667": 667,
"c668": 668,
"c669": 669,
"c670": 670,
"c671": 671,
"c672": 672,
"c673": 673,
"c674": 674,
"c675": 675,
"c676": 676,
"c677": 677,
"c678": 678,
"c679": 679,
"c680": 680,
"c681": 681,
"c682": 682,
"c683": 683,
"c684": 684,
"c685": 685,
"c686": 686,
"c687": 687,
"c688": 688,
"c689": 689,
"c690": 690,
"c691": 691,
"c692": 692,
"c693": 693,
"c694": 694,
"c695": 695,
"c696": 696,
"c697": 697,
"c698": 698,
"c699": 699,
"c700": 700,
"c701": 701,
"c702": 702,
"c703": 703,
"c704": 704,
"c705": 705,
"c706": 706,
"c707": 707,
"c708": 708,
"c709": 709,
"c710": 710,
"c711": 711,
"c712": 712,
"c713": 713,
"c714": 714,
"c715": 715,
"c716": 716,
"c717": 717,
"c718": 718,
"c719": 719,
"c720": 720,
"c721": 721,
"c722": 722,
"c723": 723,
"c724": 724,
"c725": 725,
"c726": 726,
"c727": 727,
"c728": 728,
"c729": 729,
"c730": 730,
"c731": 731,
"c732": 732,
"c733": 733,
"c734": 734,
"c735": 735,
"c736": 736,
"c737": 737,
"c738": 738,
"c739": 739,
"c740": 740,
"c741": 741,
"c742": 742,
"c743": 743,
"c744": 744,
"c745": 745,
"c746": 746,
"c747": 747,
"c748": 748,
"c749": 749,
"c750": 750,
"c751": 751,
"c752": 752,
"c753": 753,
"c754": 754,
"c755": 755,
"c756": 756,
"c757": 757,
"c758": 758,
"c759": 759,
"c760": 760,
"c761": 761,
"c762": 762,
"c763": 763,
"c764": 764,
"c765": 765,
"c766": 766,
"c767": 767,
"c768": 768,
"c769": 769,
"c770": 770,
"c771": 771,
"c772": 772,
"c773": 773,
"c774": 774,
"c775": 775,
"c776": 776,
"c777": 777,
"c778": 778,
"c779": 779,
"c780": 780,
"c781": 781,
"c782": 782,
"c783": 783,
"c784": 784,
"c785": 785,
"c786": 786,
"c787": 787,
"c788": 788,
"c789": 789,
"c790": 790,
"c791": 791,
"c792": 792,
"c793": 793,
"c794": 794,
"c795": 795,
"c796": 796,
"c797": 797,
"c798": 798,
"c799": 799,
"c800": 800,
"c801": 801,
"c802": 802,
"c803": 803,
"c804": 804,
"c805": 805,
"c806": 806,
"c807": 807,
"c808": 808,
"c809": 809,
"c810": 810,
"c811": 811,
"c812": 812,
"c813": 813,
"c814": 814,
"c815": 815,
"c816": 816,
"c817": 817,
"c818": 818,
"c819": 819,
"c820": 820,
"c821": 821,
"c822": 822,
"c823": 823,
"c824": 824,
"c825": 825,
"c826": 826,
"c827": 827,
"c828": 828,
"c829": 829,
"c830": 830,
"c831": 831,
"c832": 832,
"c833": 833,
"c834": 834,
"c835": 835,
"c836": 836,
"c837": 837,
"c838": 838,
"c839": 839,
"c840": 840,
"c841": 841,
"c842": 842,
"c843": 843,
"c844": 844,
"c845": 845,
"c846": 846,
"c847": 847,
"c848": 848,
"c849": 849,
"c850": 850,
"c851": 851,
"c852": 852,
"c853": 853,
"c854": 854,
"c855": 855,
"c856": 856,
"c857": 857,
"c858": 858,
"c859": 859,
"c860": 860,
"c861": 861,
"c862": 862,
"c863": 863,
"c864": 864,
"c865": 865,
"c866": 866,
"c867": 867,
"c868": 868,
"c869": 869,
"c870": 870,
"c871": 871,
"c872": 872,
"c873": 873,
"c874": 874,
"c875": 875,
"c876": 876,
"c877": 877,
"c878": 878,
"c879": 879,
"c880": 880,
"c881": 881,
"c882": 882,
"c883": 883,
"c884": 884,
"c885": 885,
"c886": 886,
"c887": 887,
"c888": 888,
"c889": 889,
"c890": 890,
"c891": 891,
"c892": 892,
"c893": 893,
"c894": 894,
"c895": 895,
"c896": 896,
"c897": 897,
"c898": 898,
"c899": 899,
"0": 900,
"1": 901,
"2": 902,
"3": 903,
"4": 904,
"5": 905,
"6": 906,
"7": 907,
"8": 908,
"9": 909,
"10": 910,
"11": 911,
"12": 912,
"13": 913,
"14": 914,
"15": 915,
"16": 916,
"17": 917,
"18": 918,
"19": 919,
"20": 920,
"21": 921,
"22": 922,
"23": 923,
"24": 924,
"25": 925,
"26": 926,
"27": 927,
"28": 928,
"29": 929,
"start": 930,
"goal": 931,
"wall": 932,
"create": 933,
"close": 934,
"plan": 935,
"query": 936,
"reasoning": 937,
"solution": 938,
"end": 939,
"<|pad|>": 940,
"<|unk|>": 941,
"<|bos|>": 942,
"<|eos|>": 943
},
"unk_token": "<|unk|>"
}
}