galakenya commited on
Commit
c23c4c1
·
verified ·
1 Parent(s): 88e077c

Delete unwanted checkpoint file: checkpoint-7200/tokenizer_config.json

Browse files
Files changed (1) hide show
  1. checkpoint-7200/tokenizer_config.json +0 -290
checkpoint-7200/tokenizer_config.json DELETED
@@ -1,290 +0,0 @@
1
- {
2
- "audio_token": "<|audio|>",
3
- "backend": "tokenizers",
4
- "boa_token": "<|audio>",
5
- "boi_token": "<|image>",
6
- "bos_token": "<bos>",
7
- "eoa_token": "<audio|>",
8
- "eoc_token": "<channel|>",
9
- "eoi_token": "<image|>",
10
- "eos_token": "<eos>",
11
- "eot_token": "<turn|>",
12
- "escape_token": "<|\"|>",
13
- "etc_token": "<tool_call|>",
14
- "etd_token": "<tool|>",
15
- "etr_token": "<tool_response|>",
16
- "extra_special_tokens": [
17
- "<|video|>"
18
- ],
19
- "image_token": "<|image|>",
20
- "is_local": false,
21
- "local_files_only": false,
22
- "mask_token": "<mask>",
23
- "model_max_length": 131072,
24
- "model_specific_special_tokens": {
25
- "audio_token": "<|audio|>",
26
- "boa_token": "<|audio>",
27
- "boi_token": "<|image>",
28
- "eoa_token": "<audio|>",
29
- "eoc_token": "<channel|>",
30
- "eoi_token": "<image|>",
31
- "eot_token": "<turn|>",
32
- "escape_token": "<|\"|>",
33
- "etc_token": "<tool_call|>",
34
- "etd_token": "<tool|>",
35
- "etr_token": "<tool_response|>",
36
- "image_token": "<|image|>",
37
- "soc_token": "<|channel>",
38
- "sot_token": "<|turn>",
39
- "stc_token": "<|tool_call>",
40
- "std_token": "<|tool>",
41
- "str_token": "<|tool_response>",
42
- "think_token": "<|think|>"
43
- },
44
- "pad_token": "<pad>",
45
- "padding_side": "right",
46
- "processor_class": "Gemma4Processor",
47
- "response_schema": {
48
- "properties": {
49
- "content": {
50
- "type": "string"
51
- },
52
- "role": {
53
- "const": "assistant"
54
- },
55
- "thinking": {
56
- "type": "string"
57
- },
58
- "tool_calls": {
59
- "items": {
60
- "properties": {
61
- "function": {
62
- "properties": {
63
- "arguments": {
64
- "additionalProperties": {},
65
- "type": "object",
66
- "x-parser": "gemma4-tool-call"
67
- },
68
- "name": {
69
- "type": "string"
70
- }
71
- },
72
- "type": "object",
73
- "x-regex": "call\\:(?P<name>\\w+)(?P<arguments>\\{.*\\})"
74
- },
75
- "type": {
76
- "const": "function"
77
- }
78
- },
79
- "type": "object"
80
- },
81
- "type": "array",
82
- "x-regex-iterator": "<\\|tool_call>(.*?)<tool_call\\|>"
83
- }
84
- },
85
- "type": "object",
86
- "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?P<content>(?:(?!\\<turn\\|\\>)(?!\\<\\|tool_response\\>).)+)?(?:\\<turn\\|\\>|\\<\\|tool_response\\>)?"
87
- },
88
- "soc_token": "<|channel>",
89
- "sot_token": "<|turn>",
90
- "stc_token": "<|tool_call>",
91
- "std_token": "<|tool>",
92
- "str_token": "<|tool_response>",
93
- "think_token": "<|think|>",
94
- "tokenizer_class": "GemmaTokenizer",
95
- "unk_token": "<unk>",
96
- "added_tokens_decoder": {
97
- "0": {
98
- "content": "<pad>",
99
- "single_word": false,
100
- "lstrip": false,
101
- "rstrip": false,
102
- "normalized": false,
103
- "special": true
104
- },
105
- "1": {
106
- "content": "<eos>",
107
- "single_word": false,
108
- "lstrip": false,
109
- "rstrip": false,
110
- "normalized": false,
111
- "special": true
112
- },
113
- "2": {
114
- "content": "<bos>",
115
- "single_word": false,
116
- "lstrip": false,
117
- "rstrip": false,
118
- "normalized": false,
119
- "special": true
120
- },
121
- "3": {
122
- "content": "<unk>",
123
- "single_word": false,
124
- "lstrip": false,
125
- "rstrip": false,
126
- "normalized": false,
127
- "special": true
128
- },
129
- "4": {
130
- "content": "<mask>",
131
- "single_word": false,
132
- "lstrip": false,
133
- "rstrip": false,
134
- "normalized": false,
135
- "special": true
136
- },
137
- "46": {
138
- "content": "<|tool>",
139
- "single_word": false,
140
- "lstrip": false,
141
- "rstrip": false,
142
- "normalized": false,
143
- "special": true
144
- },
145
- "47": {
146
- "content": "<tool|>",
147
- "single_word": false,
148
- "lstrip": false,
149
- "rstrip": false,
150
- "normalized": false,
151
- "special": true
152
- },
153
- "48": {
154
- "content": "<|tool_call>",
155
- "single_word": false,
156
- "lstrip": false,
157
- "rstrip": false,
158
- "normalized": false,
159
- "special": true
160
- },
161
- "49": {
162
- "content": "<tool_call|>",
163
- "single_word": false,
164
- "lstrip": false,
165
- "rstrip": false,
166
- "normalized": false,
167
- "special": true
168
- },
169
- "50": {
170
- "content": "<|tool_response>",
171
- "single_word": false,
172
- "lstrip": false,
173
- "rstrip": false,
174
- "normalized": false,
175
- "special": true
176
- },
177
- "51": {
178
- "content": "<tool_response|>",
179
- "single_word": false,
180
- "lstrip": false,
181
- "rstrip": false,
182
- "normalized": false,
183
- "special": true
184
- },
185
- "52": {
186
- "content": "<|\"|>",
187
- "single_word": false,
188
- "lstrip": false,
189
- "rstrip": false,
190
- "normalized": false,
191
- "special": true
192
- },
193
- "98": {
194
- "content": "<|think|>",
195
- "single_word": false,
196
- "lstrip": false,
197
- "rstrip": false,
198
- "normalized": false,
199
- "special": true
200
- },
201
- "100": {
202
- "content": "<|channel>",
203
- "single_word": false,
204
- "lstrip": false,
205
- "rstrip": false,
206
- "normalized": false,
207
- "special": true
208
- },
209
- "101": {
210
- "content": "<channel|>",
211
- "single_word": false,
212
- "lstrip": false,
213
- "rstrip": false,
214
- "normalized": false,
215
- "special": true
216
- },
217
- "105": {
218
- "content": "<|turn>",
219
- "single_word": false,
220
- "lstrip": false,
221
- "rstrip": false,
222
- "normalized": false,
223
- "special": true
224
- },
225
- "106": {
226
- "content": "<turn|>",
227
- "single_word": false,
228
- "lstrip": false,
229
- "rstrip": false,
230
- "normalized": false,
231
- "special": true
232
- },
233
- "255999": {
234
- "content": "<|image>",
235
- "single_word": false,
236
- "lstrip": false,
237
- "rstrip": false,
238
- "normalized": false,
239
- "special": true
240
- },
241
- "256000": {
242
- "content": "<|audio>",
243
- "single_word": false,
244
- "lstrip": false,
245
- "rstrip": false,
246
- "normalized": false,
247
- "special": true
248
- },
249
- "258880": {
250
- "content": "<|image|>",
251
- "single_word": false,
252
- "lstrip": false,
253
- "rstrip": false,
254
- "normalized": false,
255
- "special": true
256
- },
257
- "258881": {
258
- "content": "<|audio|>",
259
- "single_word": false,
260
- "lstrip": false,
261
- "rstrip": false,
262
- "normalized": false,
263
- "special": true
264
- },
265
- "258882": {
266
- "content": "<image|>",
267
- "single_word": false,
268
- "lstrip": false,
269
- "rstrip": false,
270
- "normalized": false,
271
- "special": true
272
- },
273
- "258883": {
274
- "content": "<audio|>",
275
- "single_word": false,
276
- "lstrip": false,
277
- "rstrip": false,
278
- "normalized": false,
279
- "special": true
280
- },
281
- "258884": {
282
- "content": "<|video|>",
283
- "single_word": false,
284
- "lstrip": false,
285
- "rstrip": false,
286
- "normalized": false,
287
- "special": true
288
- }
289
- }
290
- }