enankobh1 commited on
Commit
5d04690
·
1 Parent(s): cb7f1fa

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +27 -28
vocab.json CHANGED
@@ -1,33 +1,32 @@
1
  {
2
- "'": 11,
3
- "/": 9,
4
- "[PAD]": 30,
5
- "[UNK]": 29,
6
- "a": 14,
7
- "b": 7,
8
- "c": 8,
9
- "d": 6,
10
- "e": 5,
11
- "f": 15,
12
  "g": 23,
13
- "h": 20,
14
- "i": 12,
15
- "j": 0,
16
- "k": 1,
17
  "l": 18,
18
- "m": 17,
19
- "n": 16,
20
- "o": 13,
21
  "p": 2,
22
- "q": 22,
23
- "r": 24,
24
- "s": 4,
25
- "t": 21,
26
- "u": 27,
27
- "v": 19,
28
- "w": 28,
29
- "x": 3,
30
- "y": 10,
31
- "z": 26,
32
- "|": 25
33
  }
 
1
  {
2
+ "'": 19,
3
+ "[PAD]": 29,
4
+ "[UNK]": 28,
5
+ "a": 27,
6
+ "b": 12,
7
+ "c": 25,
8
+ "d": 1,
9
+ "e": 26,
10
+ "f": 14,
 
11
  "g": 23,
12
+ "h": 13,
13
+ "i": 24,
14
+ "j": 20,
15
+ "k": 9,
16
  "l": 18,
17
+ "m": 22,
18
+ "n": 0,
19
+ "o": 5,
20
  "p": 2,
21
+ "q": 17,
22
+ "r": 6,
23
+ "s": 21,
24
+ "t": 16,
25
+ "u": 8,
26
+ "v": 7,
27
+ "w": 4,
28
+ "x": 11,
29
+ "y": 15,
30
+ "z": 3,
31
+ "|": 10
32
  }