{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[+=\\-]|[^\\S\\r\\n]*[\\n\\r]+|[^\\S\\r\\n]+" }, "behavior": "Isolated", "invert": false }, { "type": "Split", "pattern": { "Regex": "\\p{N}{1,3}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "post_processor": null, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "": 0, "": 1, "": 2, "!": 3, "\"": 4, "#": 5, "$": 6, "%": 7, "&": 8, "'": 9, "(": 10, ")": 11, "*": 12, "+": 13, ",": 14, "-": 15, ".": 16, "/": 17, "0": 18, "1": 19, "2": 20, "3": 21, "4": 22, "5": 23, "6": 24, "7": 25, "8": 26, "9": 27, ":": 28, ";": 29, "<": 30, "=": 31, ">": 32, "?": 33, "@": 34, "A": 35, "B": 36, "C": 37, "D": 38, "E": 39, "F": 40, "G": 41, "H": 42, "I": 43, "J": 44, "K": 45, "L": 46, "M": 47, "N": 48, "O": 49, "P": 50, "Q": 51, "R": 52, "S": 53, "T": 54, "U": 55, "V": 56, "W": 57, "X": 58, "Y": 59, "Z": 60, "[": 61, "\\": 62, "]": 63, "^": 64, "_": 65, "`": 66, "a": 67, "b": 68, "c": 69, "d": 70, "e": 71, "f": 72, "g": 73, "h": 74, "i": 75, "j": 76, "k": 77, "l": 78, "m": 79, "n": 80, "o": 81, "p": 82, "q": 83, "r": 84, "s": 85, "t": 86, "u": 87, "v": 88, "w": 89, "x": 90, "y": 91, "z": 92, "{": 93, "|": 94, "}": 95, "~": 96, "¡": 97, "¢": 98, "£": 99, "¤": 100, "¥": 101, "¦": 102, "§": 103, "¨": 104, "©": 105, "ª": 106, "«": 107, "¬": 108, "®": 109, "¯": 110, "°": 111, "±": 112, "²": 113, "³": 114, "´": 115, "µ": 116, "¶": 117, "·": 118, "¸": 119, "¹": 120, "º": 121, "»": 122, "¼": 123, "½": 124, "¾": 125, "¿": 126, "À": 127, "Á": 128, "Â": 129, "Ã": 130, "Ä": 131, "Å": 132, "Æ": 133, "Ç": 134, "È": 135, "É": 136, "Ê": 137, "Ë": 138, "Ì": 139, "Í": 140, "Î": 141, "Ï": 142, "Ð": 143, "Ñ": 144, "Ò": 145, "Ó": 146, "Ô": 147, "Õ": 148, "Ö": 149, "×": 150, "Ø": 151, "Ù": 152, "Ú": 153, "Û": 154, "Ü": 155, "Ý": 156, "Þ": 157, "ß": 158, "à": 159, "á": 160, "â": 161, "ã": 162, "ä": 163, "å": 164, "æ": 165, "ç": 166, "è": 167, "é": 168, "ê": 169, "ë": 170, "ì": 171, "í": 172, "î": 173, "ï": 174, "ð": 175, "ñ": 176, "ò": 177, "ó": 178, "ô": 179, "õ": 180, "ö": 181, "÷": 182, "ø": 183, "ù": 184, "ú": 185, "û": 186, "ü": 187, "ý": 188, "þ": 189, "ÿ": 190, "Ā": 191, "ā": 192, "Ă": 193, "ă": 194, "Ą": 195, "ą": 196, "Ć": 197, "ć": 198, "Ĉ": 199, "ĉ": 200, "Ċ": 201, "ċ": 202, "Č": 203, "č": 204, "Ď": 205, "ď": 206, "Đ": 207, "đ": 208, "Ē": 209, "ē": 210, "Ĕ": 211, "ĕ": 212, "Ė": 213, "ė": 214, "Ę": 215, "ę": 216, "Ě": 217, "ě": 218, "Ĝ": 219, "ĝ": 220, "Ğ": 221, "ğ": 222, "Ġ": 223, "ġ": 224, "Ģ": 225, "ģ": 226, "Ĥ": 227, "ĥ": 228, "Ħ": 229, "ħ": 230, "Ĩ": 231, "ĩ": 232, "Ī": 233, "ī": 234, "Ĭ": 235, "ĭ": 236, "Į": 237, "į": 238, "İ": 239, "ı": 240, "IJ": 241, "ij": 242, "Ĵ": 243, "ĵ": 244, "Ķ": 245, "ķ": 246, "ĸ": 247, "Ĺ": 248, "ĺ": 249, "Ļ": 250, "ļ": 251, "Ľ": 252, "ľ": 253, "Ŀ": 254, "ŀ": 255, "Ł": 256, "ł": 257, "Ń": 258, "à¦": 259, "ত": 260, "শ": 261, "া": 262, "à§": 263, "শত": 264, "াà¦": 265, "ি": 266, "à§į": 267, "à§įà¦": 268, "à¦ļ": 269, "ıà¦": 270, "à¦ıà¦": 271, "à¦ıà¦ķ": 272, "ন": 273, "¯à¦": 274, "যà¦": 275, "য়": 276, "ার": 277, "à¦Ł": 278, "াত": 279, "িশ": 280, "à¦ĩ": 281, "প": 282, "ব": 283, "স": 284, "à¦Ĩ": 285, "র": 286, "য়শত": 287, "à¦Ľ": 288, "সাত": 289, "à¦Ĩà¦Ł": 290, "¹à¦¾à¦": 291, "ľà¦¾à¦°": 292, "হাà¦": 293, "à¦¹à¦¾à¦ľà¦¾à¦°": 294, "à§ģ": 295, "ান": 296, "ģà¦ļ": 297, "তি": 298, "দ": 299, "à§įব": 300, "াশ": 301, "বà§įব": 302, "াà¦ģà¦ļ": 303, "à¦ļার": 304, "পাà¦ģà¦ļ": 305, "à§ģà¦ĩ": 306, "তিন": 307, "দà§ģà¦ĩ": 308, "বà§įবà¦ĩ": 309, "°à¦¿à¦¶": 310, "²à§įà¦": 311, "²à¦¿à¦¶": 312, "ষ": 313, "তà§įà¦": 314, "তর": 315, "à§įতর": 316, "à¦ıà¦ķশত": 317, "নয়শত": 318, "পà¦": 319, "à¦Ľà¦¯à¦¼à¦¶à¦¤": 320, "সাতশত": 321, "à¦Ĩà¦Łà¦¶à¦¤": 322, "à¦ļারশত": 323, "পাà¦ģà¦ļশত": 324, "তিনশত": 325, "দà§ģà¦ĩশত": 326, "²à§įলিশ": 327, "তà§įরিশ": 328, "ানবà§įবà¦ĩ": 329, "īন": 330, "Łà¦¿": 331, "à¦īন": 332, "à§ĩ": 333, "à§įন": 334, "à§įà¦Łà¦¿": 335, "à¦Łà§įà¦Łà¦¿": 336, "াতà§įতর": 337, "াশি": 338, "লà§įলিশ": 339, "à¦ļলà§įলিশ": 340, "ানà§įন": 341, "à¦·à¦Łà§įà¦Łà¦¿": 342, "তà§ĩ": 343, "à§ĭ": 344, "à§Į": 345, "িয়": 346, "à¦ļà§ģ": 347, "à¦ļà§Į": 348, "বাà¦": 349, "রানবà§įবà¦ĩ": 350, "পà¦ģà¦ļ": 351, "ŀà§įà¦": 352, "ালà§įলিশ": 353, "ারà§ĭ": 354, "রাশি": 355, "à¦Ľà¦¿à¦¯à¦¼": 356, "পà¦ģ": 357, "পà¦ŀà§įà¦": 358, "পà¦ŀà§įà¦ļ": 359 }, "merges": [ [ "à", "¦" ], [ "à¦", "¤" ], [ "à¦", "¶" ], [ "à¦", "¾" ], [ "à", "§" ], [ "শ", "ত" ], [ "া", "à¦" ], [ "à¦", "¿" ], [ "à§", "į" ], [ "à§į", "à¦" ], [ "à¦", "ļ" ], [ "ı", "à¦" ], [ "à¦", "ıà¦" ], [ "à¦ıà¦", "ķ" ], [ "à¦", "¨" ], [ "¯", "à¦" ], [ "à¦", "¯à¦" ], [ "যà¦", "¼" ], [ "াà¦", "°" ], [ "à¦", "Ł" ], [ "া", "ত" ], [ "ি", "শ" ], [ "à¦", "ĩ" ], [ "à¦", "ª" ], [ "à¦", "¬" ], [ "à¦", "¸" ], [ "à¦", "Ĩ" ], [ "à¦", "°" ], [ "য়", "শত" ], [ "à¦", "Ľ" ], [ "স", "াত" ], [ "à¦Ĩ", "à¦Ł" ], [ "¹", "াà¦" ], [ "ľ", "ার" ], [ "à¦", "¹à¦¾à¦" ], [ "হাà¦", "ľà¦¾à¦°" ], [ "à§", "ģ" ], [ "াà¦", "¨" ], [ "ģ", "à¦ļ" ], [ "ত", "ি" ], [ "à¦", "¦" ], [ "à§įà¦", "¬" ], [ "া", "শ" ], [ "ব", "à§įব" ], [ "াà¦", "ģà¦ļ" ], [ "à¦ļ", "ার" ], [ "প", "াà¦ģà¦ļ" ], [ "à§ģ", "à¦ĩ" ], [ "তি", "ন" ], [ "দ", "à§ģà¦ĩ" ], [ "বà§įব", "à¦ĩ" ], [ "°", "িশ" ], [ "²", "à§įà¦" ], [ "²", "িশ" ], [ "à¦", "·" ], [ "ত", "à§įà¦" ], [ "ত", "র" ], [ "à§į", "তর" ], [ "à¦ıà¦ķ", "শত" ], [ "ন", "য়শত" ], [ "প", "à¦" ], [ "à¦Ľ", "য়শত" ], [ "সাত", "শত" ], [ "à¦Ĩà¦Ł", "শত" ], [ "à¦ļার", "শত" ], [ "পাà¦ģà¦ļ", "শত" ], [ "তিন", "শত" ], [ "দà§ģà¦ĩ", "শত" ], [ "²à§įà¦", "²à¦¿à¦¶" ], [ "তà§įà¦", "°à¦¿à¦¶" ], [ "ান", "বà§įবà¦ĩ" ], [ "ī", "ন" ], [ "Ł", "ি" ], [ "à¦", "īন" ], [ "à§", "ĩ" ], [ "à§įà¦", "¨" ], [ "à§įà¦", "Łà¦¿" ], [ "à¦Ł", "à§įà¦Łà¦¿" ], [ "াত", "à§įতর" ], [ "াশ", "ি" ], [ "à¦", "²à§įলিশ" ], [ "à¦ļ", "লà§įলিশ" ], [ "ান", "à§įন" ], [ "ষ", "à¦Łà§įà¦Łà¦¿" ], [ "ত", "à§ĩ" ], [ "à§", "ĭ" ], [ "à§", "Į" ], [ "ি", "য়" ], [ "à¦ļ", "à§ģ" ], [ "à¦ļ", "à§Į" ], [ "ব", "াà¦" ], [ "র", "ানবà§įবà¦ĩ" ], [ "পà¦", "ģà¦ļ" ], [ "ŀ", "à§įà¦" ], [ "াà¦", "²à§įলিশ" ], [ "ার", "à§ĭ" ], [ "র", "াশি" ], [ "à¦Ľ", "িয়" ], [ "পà¦", "ģ" ], [ "পà¦", "ŀà§įà¦" ], [ "পà¦ŀà§įà¦", "ļ" ] ] } }