Hugging Face's logo

fMindstormer
/
tokcpu

Transformers
tokenizer
brainfuck
turing-machine
esoteric
Model card Files
xet
 Community
tokcpu / increment /tokenizer.json
fMindstormer's picture
fMindstormer
Tokenizer CPU
e833aa9 verified
Raw History Blame Contribute Delete
2.75 kB
{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":27,"content":"0ⓐ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":28,"content":"1Ⓒ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":29,"content":"1ⓐ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":30,"content":"β’·0","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":31,"content":"β’·_","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":32,"content":"1Ⓒ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":33,"content":"βŠ’β’·_","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":34,"content":"⊒1Ⓒ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":35,"content":"0ⓐ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":36,"content":"1ⓐ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":37,"content":"β’·_","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":38,"content":"β’·_βŠ£ξ€","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":39,"content":"β“’1","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":40,"content":"β’·0","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":41,"content":"β“’1","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":42,"content":"β“’1βŠ£ξ€","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false}],"normalizer":null,"pre_tokenizer":null,"post_processor":null,"decoder":{"type":"Sequence","decoders":[{"type":"Replace","pattern":{"Regex":"[ξ€€-ξ€Ώ]"},"content":""},{"type":"Fuse"}]},"model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":null,"end_of_word_suffix":null,"fuse_unk":false,"byte_fallback":false,"ignore_merges":false,"vocab":{"0":0,"1":1,"_":2,"⊒":3,"⊣":4,"β’Ά":5,"β’·":6,"β’Έ":7,"ⓐ":8,"β“‘":9,"β“’":10,"0β’Ά":27,"0β’·":28,"1β’Ά":29,"1β’·":30,"_β’Ά":31,"_β’·":32,"βŠ’β’Ά":33,"βŠ’β’·":34,"ⓐ0":35,"ⓐ1":36,"ⓐ_":37,"β“βŠ£":38,"β“‘0":39,"β“‘1":40,"β“‘_":41,"β“‘βŠ£":42},"merges":[["0","β’Ά"],["0","β’·"],["1","β’Ά"],["1","β’·"],["_","β’Ά"],["_","β’·"],["⊒","β’Ά"],["⊒","β’·"],["ⓐ","0"],["ⓐ","1"],["ⓐ","_"],["ⓐ","⊣"],["β“‘","0"],["β“‘","1"],["β“‘","_"],["β“‘","⊣"]]}}