| {"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":16,"content":"1βξ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":17,"content":"βΆ0ξ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":18,"content":"β’1βξ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":19,"content":"βΆ#ξ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":20,"content":"0βξ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":21,"content":"1βξ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":22,"content":"0ββ£ξ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false}],"normalizer":null,"pre_tokenizer":null,"post_processor":null,"decoder":{"type":"Sequence","decoders":[{"type":"Replace","pattern":{"Regex":"[ξ-ξΏ]"},"content":""},{"type":"Fuse"}]},"model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":null,"end_of_word_suffix":null,"fuse_unk":false,"byte_fallback":false,"ignore_merges":false,"vocab":{"#":0,"0":1,"1":2,"β’":3,"β£":4,"βΆ":5,"β·":6,"β":7,"β":8,"0βΆ":16,"1βΆ":17,"β’βΆ":18,"β#":19,"β0":20,"β1":21,"ββ£":22},"merges":[["0","βΆ"],["1","βΆ"],["β’","βΆ"],["β","#"],["β","0"],["β","1"],["β","β£"]]}} |