Hugging Face's logo

zaydzuhri
/
tasklets_tokenizer_75

Model card Files
xet
 Community
zaydzuhri commited on
Commit
784e11b
·
verified ·
1 Parent(s): 4652187

Add tokenizer files

Browse files
Files changed (1) hide show
  1. tokenizer.json +4 -61
tokenizer.json CHANGED
@@ -119,67 +119,10 @@
119
  "14": 45,
120
  "15": 46,
121
  "16": 47,
122
- "17": 48,
123
- "18": 49,
124
- "19": 50,
125
- "20": 51,
126
- "21": 52,
127
- "22": 53,
128
- "23": 54,
129
- "24": 55,
130
- "25": 56,
131
- "26": 57,
132
- "27": 58,
133
- "28": 59,
134
- "29": 60,
135
- "30": 61,
136
- "31": 62,
137
- "32": 63,
138
- "33": 64,
139
- "34": 65,
140
- "35": 66,
141
- "36": 67,
142
- "37": 68,
143
- "38": 69,
144
- "39": 70,
145
- "40": 71,
146
- "41": 72,
147
- "42": 73,
148
- "43": 74,
149
- "44": 75,
150
- "45": 76,
151
- "46": 77,
152
- "47": 78,
153
- "48": 79,
154
- "49": 80,
155
- "50": 81,
156
- "51": 82,
157
- "52": 83,
158
- "53": 84,
159
- "54": 85,
160
- "55": 86,
161
- "56": 87,
162
- "57": 88,
163
- "58": 89,
164
- "59": 90,
165
- "60": 91,
166
- "61": 92,
167
- "62": 93,
168
- "63": 94,
169
- "64": 95,
170
- "65": 96,
171
- "66": 97,
172
- "67": 98,
173
- "68": 99,
174
- "69": 100,
175
- "70": 101,
176
- "71": 102,
177
- "72": 103,
178
- "73": 104,
179
- "74": 105,
180
- "75": 106,
181
- "|": 107,
182
- "?": 108
183
  },
184
  "unk_token": "[UNK]"
185
  }
 
119
  "14": 45,
120
  "15": 46,
121
  "16": 47,
122
+ "|": 48,
123
+ "?": 49,
124
+ "(": 50,
125
+ ")": 51
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
126
  },
127
  "unk_token": "[UNK]"
128
  }