HARP-1B / tokenizer.json
frontterrain's picture
Upload HARP-1B Pre-trained weights (Step 52,000, 1,400,832,000 tokens trained)
88086d4 verified
Raw History Blame Contribute Delete
7.7 kB
{
"special_tokens": {
"<|pad|>": 0,
"<|unk|>": 1,
"<|im_start|>": 2,
"<|im_end|>": 3,
"<|system|>": 4,
"<|user|>": 5,
"<|assistant|>": 6
},
"byte_offset": 7,
"merges": [],
"vocab": {
"7": [
0
],
"8": [
1
],
"9": [
2
],
"10": [
3
],
"11": [
4
],
"12": [
5
],
"13": [
6
],
"14": [
7
],
"15": [
8
],
"16": [
9
],
"17": [
10
],
"18": [
11
],
"19": [
12
],
"20": [
13
],
"21": [
14
],
"22": [
15
],
"23": [
16
],
"24": [
17
],
"25": [
18
],
"26": [
19
],
"27": [
20
],
"28": [
21
],
"29": [
22
],
"30": [
23
],
"31": [
24
],
"32": [
25
],
"33": [
26
],
"34": [
27
],
"35": [
28
],
"36": [
29
],
"37": [
30
],
"38": [
31
],
"39": [
32
],
"40": [
33
],
"41": [
34
],
"42": [
35
],
"43": [
36
],
"44": [
37
],
"45": [
38
],
"46": [
39
],
"47": [
40
],
"48": [
41
],
"49": [
42
],
"50": [
43
],
"51": [
44
],
"52": [
45
],
"53": [
46
],
"54": [
47
],
"55": [
48
],
"56": [
49
],
"57": [
50
],
"58": [
51
],
"59": [
52
],
"60": [
53
],
"61": [
54
],
"62": [
55
],
"63": [
56
],
"64": [
57
],
"65": [
58
],
"66": [
59
],
"67": [
60
],
"68": [
61
],
"69": [
62
],
"70": [
63
],
"71": [
64
],
"72": [
65
],
"73": [
66
],
"74": [
67
],
"75": [
68
],
"76": [
69
],
"77": [
70
],
"78": [
71
],
"79": [
72
],
"80": [
73
],
"81": [
74
],
"82": [
75
],
"83": [
76
],
"84": [
77
],
"85": [
78
],
"86": [
79
],
"87": [
80
],
"88": [
81
],
"89": [
82
],
"90": [
83
],
"91": [
84
],
"92": [
85
],
"93": [
86
],
"94": [
87
],
"95": [
88
],
"96": [
89
],
"97": [
90
],
"98": [
91
],
"99": [
92
],
"100": [
93
],
"101": [
94
],
"102": [
95
],
"103": [
96
],
"104": [
97
],
"105": [
98
],
"106": [
99
],
"107": [
100
],
"108": [
101
],
"109": [
102
],
"110": [
103
],
"111": [
104
],
"112": [
105
],
"113": [
106
],
"114": [
107
],
"115": [
108
],
"116": [
109
],
"117": [
110
],
"118": [
111
],
"119": [
112
],
"120": [
113
],
"121": [
114
],
"122": [
115
],
"123": [
116
],
"124": [
117
],
"125": [
118
],
"126": [
119
],
"127": [
120
],
"128": [
121
],
"129": [
122
],
"130": [
123
],
"131": [
124
],
"132": [
125
],
"133": [
126
],
"134": [
127
],
"135": [
128
],
"136": [
129
],
"137": [
130
],
"138": [
131
],
"139": [
132
],
"140": [
133
],
"141": [
134
],
"142": [
135
],
"143": [
136
],
"144": [
137
],
"145": [
138
],
"146": [
139
],
"147": [
140
],
"148": [
141
],
"149": [
142
],
"150": [
143
],
"151": [
144
],
"152": [
145
],
"153": [
146
],
"154": [
147
],
"155": [
148
],
"156": [
149
],
"157": [
150
],
"158": [
151
],
"159": [
152
],
"160": [
153
],
"161": [
154
],
"162": [
155
],
"163": [
156
],
"164": [
157
],
"165": [
158
],
"166": [
159
],
"167": [
160
],
"168": [
161
],
"169": [
162
],
"170": [
163
],
"171": [
164
],
"172": [
165
],
"173": [
166
],
"174": [
167
],
"175": [
168
],
"176": [
169
],
"177": [
170
],
"178": [
171
],
"179": [
172
],
"180": [
173
],
"181": [
174
],
"182": [
175
],
"183": [
176
],
"184": [
177
],
"185": [
178
],
"186": [
179
],
"187": [
180
],
"188": [
181
],
"189": [
182
],
"190": [
183
],
"191": [
184
],
"192": [
185
],
"193": [
186
],
"194": [
187
],
"195": [
188
],
"196": [
189
],
"197": [
190
],
"198": [
191
],
"199": [
192
],
"200": [
193
],
"201": [
194
],
"202": [
195
],
"203": [
196
],
"204": [
197
],
"205": [
198
],
"206": [
199
],
"207": [
200
],
"208": [
201
],
"209": [
202
],
"210": [
203
],
"211": [
204
],
"212": [
205
],
"213": [
206
],
"214": [
207
],
"215": [
208
],
"216": [
209
],
"217": [
210
],
"218": [
211
],
"219": [
212
],
"220": [
213
],
"221": [
214
],
"222": [
215
],
"223": [
216
],
"224": [
217
],
"225": [
218
],
"226": [
219
],
"227": [
220
],
"228": [
221
],
"229": [
222
],
"230": [
223
],
"231": [
224
],
"232": [
225
],
"233": [
226
],
"234": [
227
],
"235": [
228
],
"236": [
229
],
"237": [
230
],
"238": [
231
],
"239": [
232
],
"240": [
233
],
"241": [
234
],
"242": [
235
],
"243": [
236
],
"244": [
237
],
"245": [
238
],
"246": [
239
],
"247": [
240
],
"248": [
241
],
"249": [
242
],
"250": [
243
],
"251": [
244
],
"252": [
245
],
"253": [
246
],
"254": [
247
],
"255": [
248
],
"256": [
249
],
"257": [
250
],
"258": [
251
],
"259": [
252
],
"260": [
253
],
"261": [
254
],
"262": [
255
]
}
}