Hugging Face's logo

KTibow
/
bytetokenizer

Model card Files
xet
Community
KTibow commited on
Commit
8a22647
·
verified ·
1 Parent(s): 5f48de0

Update tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +96 -95
tokenizer.json CHANGED
@@ -27,101 +27,102 @@
27
  "byte_fallback": false,
28
  "vocab": {
29
  "[UNK]": 0,
30
- "0": 1,
31
- "1": 2,
32
- "2": 3,
33
- "3": 4,
34
- "4": 5,
35
- "5": 6,
36
- "6": 7,
37
- "7": 8,
38
- "8": 9,
39
- "9": 10,
40
- "a": 11,
41
- "b": 12,
42
- "c": 13,
43
- "d": 14,
44
- "e": 15,
45
- "f": 16,
46
- "g": 17,
47
- "h": 18,
48
- "i": 19,
49
- "j": 20,
50
- "k": 21,
51
- "l": 22,
52
- "m": 23,
53
- "n": 24,
54
- "o": 25,
55
- "p": 26,
56
- "q": 27,
57
- "r": 28,
58
- "s": 29,
59
- "t": 30,
60
- "u": 31,
61
- "v": 32,
62
- "w": 33,
63
- "x": 34,
64
- "y": 35,
65
- "z": 36,
66
- "A": 37,
67
- "B": 38,
68
- "C": 39,
69
- "D": 40,
70
- "E": 41,
71
- "F": 42,
72
- "G": 43,
73
- "H": 44,
74
- "I": 45,
75
- "J": 46,
76
- "K": 47,
77
- "L": 48,
78
- "M": 49,
79
- "N": 50,
80
- "O": 51,
81
- "P": 52,
82
- "Q": 53,
83
- "R": 54,
84
- "S": 55,
85
- "T": 56,
86
- "U": 57,
87
- "V": 58,
88
- "W": 59,
89
- "X": 60,
90
- "Y": 61,
91
- "Z": 62,
92
- "!": 63,
93
- "\"": 64,
94
- "#": 65,
95
- "$": 66,
96
- "%": 67,
97
- "&": 68,
98
- "'": 69,
99
- "(": 70,
100
- ")": 71,
101
- "*": 72,
102
- "+": 73,
103
- ",": 74,
104
- "-": 75,
105
- ".": 76,
106
- "/": 77,
107
- ":": 78,
108
- ";": 79,
109
- "<": 80,
110
- "=": 81,
111
- ">": 82,
112
- "?": 83,
113
- "@": 84,
114
- "[": 85,
115
- "\\": 86,
116
- "]": 87,
117
- "^": 88,
118
- "_": 89,
119
- "`": 90,
120
- "{": 91,
121
- "|": 92,
122
- "}": 93,
123
- "~": 94,
124
- " ": 95
 
125
  },
126
  "merges": []
127
  }
 
27
  "byte_fallback": false,
28
  "vocab": {
29
  "[UNK]": 0,
30
+ "[MASK]": 1,
31
+ "0": 2,
32
+ "1": 3,
33
+ "2": 4,
34
+ "3": 5,
35
+ "4": 6,
36
+ "5": 7,
37
+ "6": 8,
38
+ "7": 9,
39
+ "8": 10,
40
+ "9": 11,
41
+ "a": 12,
42
+ "b": 13,
43
+ "c": 14,
44
+ "d": 15,
45
+ "e": 16,
46
+ "f": 17,
47
+ "g": 18,
48
+ "h": 19,
49
+ "i": 20,
50
+ "j": 21,
51
+ "k": 22,
52
+ "l": 23,
53
+ "m": 24,
54
+ "n": 25,
55
+ "o": 26,
56
+ "p": 27,
57
+ "q": 28,
58
+ "r": 29,
59
+ "s": 30,
60
+ "t": 31,
61
+ "u": 32,
62
+ "v": 33,
63
+ "w": 34,
64
+ "x": 35,
65
+ "y": 36,
66
+ "z": 37,
67
+ "A": 38,
68
+ "B": 39,
69
+ "C": 40,
70
+ "D": 41,
71
+ "E": 42,
72
+ "F": 43,
73
+ "G": 44,
74
+ "H": 45,
75
+ "I": 46,
76
+ "J": 47,
77
+ "K": 48,
78
+ "L": 49,
79
+ "M": 50,
80
+ "N": 51,
81
+ "O": 52,
82
+ "P": 53,
83
+ "Q": 54,
84
+ "R": 55,
85
+ "S": 56,
86
+ "T": 57,
87
+ "U": 58,
88
+ "V": 59,
89
+ "W": 60,
90
+ "X": 61,
91
+ "Y": 62,
92
+ "Z": 63,
93
+ "!": 64,
94
+ "\"": 65,
95
+ "#": 66,
96
+ "$": 67,
97
+ "%": 68,
98
+ "&": 69,
99
+ "'": 70,
100
+ "(": 71,
101
+ ")": 72,
102
+ "*": 73,
103
+ "+": 74,
104
+ ",": 75,
105
+ "-": 76,
106
+ ".": 77,
107
+ "/": 78,
108
+ ":": 79,
109
+ ";": 80,
110
+ "<": 81,
111
+ "=": 82,
112
+ ">": 83,
113
+ "?": 84,
114
+ "@": 85,
115
+ "[": 86,
116
+ "\\": 87,
117
+ "]": 88,
118
+ "^": 89,
119
+ "_": 90,
120
+ "`": 91,
121
+ "{": 92,
122
+ "|": 93,
123
+ "}": 94,
124
+ "~": 95,
125
+ " ": 96
126
  },
127
  "merges": []
128
  }