{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "[PAD]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "[BOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "[EOS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "[UNK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "WhitespaceSplit" }, "post_processor": { "type": "TemplateProcessing", "single": [ { "Sequence": { "id": "A", "type_id": 0 } } ], "pair": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "Sequence": { "id": "B", "type_id": 1 } } ], "special_tokens": {} }, "decoder": null, "model": { "type": "WordLevel", "vocab": { "[PAD]": 0, "[BOS]": 1, "[EOS]": 2, "[UNK]": 3, "[ROW_00]": 4, "[ROW_01]": 5, "[ROW_02]": 6, "[ROW_03]": 7, "[ROW_04]": 8, "[ROW_05]": 9, "[ROW_06]": 10, "[ROW_07]": 11, "[ROW_08]": 12, "[ROW_09]": 13, "[ROW_10]": 14, "[ROW_11]": 15, "[ROW_12]": 16, "[ROW_13]": 17, "[ROW_14]": 18, "[ROW_15]": 19, "[ROW_16]": 20, "[ROW_17]": 21, "[ROW_18]": 22, "[ROW_19]": 23, "[ROW_20]": 24, "[ROW_21]": 25, "[ROW_22]": 26, "[ROW_23]": 27, "[ROW_24]": 28, "[ROW_25]": 29, "[ROW_26]": 30, "[ROW_27]": 31, "[ROW_28]": 32, "[ROW_29]": 33, "[ROW_30]": 34, "[ROW_31]": 35, "[ROW_32]": 36, "[ROW_33]": 37, "[ROW_34]": 38, "[ROW_35]": 39, "[ROW_36]": 40, "[ROW_37]": 41, "[ROW_38]": 42, "[ROW_39]": 43, "[ROW_40]": 44, "[ROW_41]": 45, "[ROW_42]": 46, "[ROW_43]": 47, "[ROW_44]": 48, "[ROW_45]": 49, "[ROW_46]": 50, "[ROW_47]": 51, "[ROW_48]": 52, "[ROW_49]": 53, "[ROW_50]": 54, "[ROW_51]": 55, "[ROW_52]": 56, "[ROW_53]": 57, "[ROW_54]": 58, "[ROW_55]": 59, "[ROW_56]": 60, "[ROW_57]": 61, "[ROW_58]": 62, "[ROW_59]": 63, "[ROW_60]": 64, "[ROW_61]": 65, "[ROW_62]": 66, "[ROW_63]": 67, ";": 68, "<": 69, "=": 70, ">": 71, "?": 72, "@": 73, "A": 74, "B": 75, "C": 76, "D": 77, "E": 78, "F": 79, "G": 80, "H": 81, "I": 82, "J": 83, "K": 84, "L": 85, "M": 86, "N": 87, "O": 88, "P": 89, "Q": 90, "R": 91, "S": 92, "T": 93, "U": 94, "V": 95, "W": 96, "X": 97, "Y": 98, "Z": 99, "[": 100, "\\": 101, "]": 102, "^": 103, "_": 104, "`": 105, "a": 106, "b": 107, "c": 108, "d": 109, "e": 110, "f": 111, "g": 112, "h": 113, "i": 114, "j": 115, "k": 116, "l": 117, "m": 118, "n": 119, "o": 120, "p": 121, "q": 122, "r": 123, "s": 124, "t": 125, "u": 126, "v": 127, "w": 128, "x": 129, "y": 130, "z": 131, "~": 132 }, "unk_token": "[UNK]" } }