Tarul commited on
Commit
7dd7dcb
·
verified ·
1 Parent(s): 7a36299

Upload weights/tokenizer.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. weights/tokenizer.json +9 -0
weights/tokenizer.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "pad_id": 0,
3
+ "bos_id": 1,
4
+ "sep_id": 2,
5
+ "caps_len": 32,
6
+ "type": "char+byte-fallback-buckets",
7
+ "charset": "abcdefghijklmnopqrstuvwxyz ,:-:'",
8
+ "note": "ids: 0 pad,1 bos,2 sep,3-6 punct,7-32 a-z,33-62 byte buckets"
9
+ }