{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "[PAD]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "[UNK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "[BOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "[EOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "Whitespace" }, "post_processor": { "type": "TemplateProcessing", "single": [ { "Sequence": { "id": "A", "type_id": 0 } } ], "pair": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "Sequence": { "id": "B", "type_id": 1 } } ], "special_tokens": {} }, "decoder": null, "model": { "type": "WordLevel", "vocab": { "[PAD]": 0, "[UNK]": 1, "[BOS]": 2, "[EOS]": 3, "ĠBRACKET_0": 4, "ĠBRACKET_1": 5, "ĠBRACKET_2": 6, "ĠBRACKET_3": 7, "ĠBRACKET_4": 8, "ĠBRACKET_5": 9, "ĠBRACKET_6": 10, "ĠBRACKET_7": 11, "ĠBRACKET_8": 12, "ĠBRACKET_9": 13, "ĠBRACKET_10": 14, "ĠBRACKET_11": 15, "ĠBRACKET_12": 16, "ĠBRACKET_13": 17, "ĠBRACKET_14": 18, "ĠBRACKET_15": 19, "ĠBRACKET_16": 20, "ĠBRACKET_17": 21, "ĠBRACKET_18": 22, "ĠBRACKET_19": 23, "ĠBRACKET_20": 24, "ĠBRACKET_21": 25, "ĠBRACKET_22": 26, "ĠBRACKET_23": 27, "ĠBRACKET_24": 28, "ĠBRACKET_25": 29, "ĠBRACKET_26": 30, "ĠBRACKET_27": 31, "ĠBRACKET_28": 32, "ĠBRACKET_29": 33, "ĠBRACKET_30": 34, "ĠBRACKET_31": 35, "ĠBRACKET_32": 36, "ĠBRACKET_33": 37, "ĠBRACKET_34": 38, "ĠBRACKET_35": 39, "ĠBRACKET_36": 40, "ĠBRACKET_37": 41, "ĠBRACKET_38": 42, "ĠBRACKET_39": 43, "ĠBRACKET_40": 44, "ĠBRACKET_41": 45, "ĠBRACKET_42": 46, "ĠBRACKET_43": 47, "ĠBRACKET_44": 48, "ĠBRACKET_45": 49, "ĠBRACKET_46": 50, "ĠBRACKET_47": 51, "ĠBRACKET_48": 52, "ĠBRACKET_49": 53, "ĠBRACKET_50": 54, "ĠBRACKET_51": 55, "ĠBRACKET_52": 56, "ĠBRACKET_53": 57, "ĠBRACKET_54": 58, "ĠBRACKET_55": 59, "ĠBRACKET_56": 60, "ĠBRACKET_57": 61, "ĠBRACKET_58": 62, "ĠBRACKET_59": 63, "ĠBRACKET_60": 64, "ĠBRACKET_61": 65, "ĠBRACKET_62": 66, "ĠBRACKET_63": 67, "ĠBRACKET_64": 68, "ĠBRACKET_65": 69, "ĠBRACKET_66": 70, "ĠBRACKET_67": 71, "ĠBRACKET_68": 72, "ĠBRACKET_69": 73, "ĠBRACKET_70": 74, "ĠBRACKET_71": 75, "ĠBRACKET_72": 76, "ĠBRACKET_73": 77, "ĠBRACKET_74": 78, "ĠBRACKET_75": 79, "ĠBRACKET_76": 80, "ĠBRACKET_77": 81, "ĠBRACKET_78": 82, "ĠBRACKET_79": 83, "ĠBRACKET_80": 84, "ĠBRACKET_81": 85, "ĠBRACKET_82": 86, "ĠBRACKET_83": 87, "ĠBRACKET_84": 88, "ĠBRACKET_85": 89, "ĠBRACKET_86": 90, "ĠBRACKET_87": 91, "ĠBRACKET_88": 92, "ĠBRACKET_89": 93, "ĠBRACKET_90": 94, "ĠBRACKET_91": 95, "ĠBRACKET_92": 96, "ĠBRACKET_93": 97, "ĠBRACKET_94": 98, "ĠBRACKET_95": 99, "ĠBRACKET_96": 100, "ĠBRACKET_97": 101, "ĠBRACKET_98": 102, "ĠBRACKET_99": 103, "ĠBRACKET_100": 104, "ĠBRACKET_101": 105, "ĠBRACKET_102": 106, "ĠBRACKET_103": 107, "ĠBRACKET_104": 108, "ĠBRACKET_105": 109, "ĠBRACKET_106": 110, "ĠBRACKET_107": 111, "ĠBRACKET_108": 112, "ĠBRACKET_109": 113, "ĠBRACKET_110": 114, "ĠBRACKET_111": 115, "ĠBRACKET_112": 116, "ĠBRACKET_113": 117, "ĠBRACKET_114": 118, "ĠBRACKET_115": 119, "ĠBRACKET_116": 120, "ĠBRACKET_117": 121, "ĠBRACKET_118": 122, "ĠBRACKET_119": 123, "ĠBRACKET_120": 124, "ĠBRACKET_121": 125, "ĠBRACKET_122": 126, "ĠBRACKET_123": 127, "ĠBRACKET_124": 128, "ĠBRACKET_125": 129, "ĠBRACKET_126": 130, "ĠBRACKET_127": 131 }, "unk_token": "[UNK]" } }