Hugging Face's logo

brainworkup
/
LFM2.5-8B-A1B-oQ4e

MLX
Safetensors
lfm2_moe
oq
quantized
4-bit precision
Model card Files
xet
 Community
brainworkup commited on
Commit
88977e4
·
verified ·
1 Parent(s): 8812482

Upload LFM2.5-8B-A1B-oQ4e via oMLX

Browse files
Files changed (9) hide show
  1. .gitattributes +1 -0
  2. README.md +18 -0
  3. chat_template.jinja +115 -0
  4. config.json +451 -0
  5. generation_config.json +14 -0
  6. model.safetensors +3 -0
  7. oq_imatrix_report.json +237 -0
  8. tokenizer.json +3 -0
  9. tokenizer_config.json +11 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: mlx
3
+ tags:
4
+ - mlx
5
+ - oq
6
+ - quantized
7
+ ---
8
+
9
+ # LFM2.5-8B-A1B-oQ4e
10
+
11
+ This model was quantized using [oQ](https://github.com/jundot/omlx) (oMLX v0.5.7) mixed-precision quantization.
12
+
13
+ ## Quantization details
14
+
15
+ - **Model type**: lfm2_moe
16
+ - **Bits**: 4
17
+ - **Group size**: 64
18
+ - **Format**: MLX safetensors
chat_template.jinja ADDED
@@ -0,0 +1,115 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {{- bos_token -}}
2
+ {%- set preserve_thinking = preserve_thinking | default(false) -%}
3
+
4
+ {%- macro format_arg_value(arg_value) -%}
5
+ {%- if arg_value is string -%}
6
+ {{- "'" + arg_value + "'" -}}
7
+ {%- elif arg_value is mapping -%}
8
+ {{- arg_value | tojson -}}
9
+ {%- else -%}
10
+ {{- arg_value | string -}}
11
+ {%- endif -%}
12
+ {%- endmacro -%}
13
+
14
+ {%- macro parse_content(content) -%}
15
+ {%- if content is string -%}
16
+ {{- content -}}
17
+ {%- else -%}
18
+ {%- set _ns = namespace(result="") -%}
19
+ {%- for item in content -%}
20
+ {%- if item["type"] == "image" -%}
21
+ {%- set _ns.result = _ns.result + "<image>" -%}
22
+ {%- elif item["type"] == "text" -%}
23
+ {%- set _ns.result = _ns.result + item["text"] -%}
24
+ {%- else -%}
25
+ {%- set _ns.result = _ns.result + item | tojson -%}
26
+ {%- endif -%}
27
+ {%- endfor -%}
28
+ {{- _ns.result -}}
29
+ {%- endif -%}
30
+ {%- endmacro -%}
31
+
32
+ {%- macro render_tool_calls(tool_calls) -%}
33
+ {%- set tool_calls_ns = namespace(tool_calls=[]) -%}
34
+ {%- for tool_call in tool_calls -%}
35
+ {%- set func_name = tool_call["function"]["name"] -%}
36
+ {%- set func_args = tool_call["function"]["arguments"] -%}
37
+ {%- set args_ns = namespace(arg_strings=[]) -%}
38
+ {%- for arg_name, arg_value in func_args.items() -%}
39
+ {%- set args_ns.arg_strings = args_ns.arg_strings + [arg_name + "=" + format_arg_value(arg_value)] -%}
40
+ {%- endfor -%}
41
+ {%- set tool_calls_ns.tool_calls = tool_calls_ns.tool_calls + [func_name + "(" + (args_ns.arg_strings | join(", ")) + ")"] -%}
42
+ {%- endfor -%}
43
+ {{- "<|tool_call_start|>[" + (tool_calls_ns.tool_calls | join(", ")) + "]<|tool_call_end|>" -}}
44
+ {%- endmacro -%}
45
+
46
+ {%- set ns = namespace(system_prompt="", last_user_index=-1) -%}
47
+ {%- if messages[0]["role"] == "system" -%}
48
+ {%- if messages[0].get("content") -%}
49
+ {%- set ns.system_prompt = parse_content(messages[0]["content"]) -%}
50
+ {%- endif -%}
51
+ {%- set messages = messages[1:] -%}
52
+ {%- endif -%}
53
+ {%- if tools -%}
54
+ {%- set ns.system_prompt = ns.system_prompt + ("\n" if ns.system_prompt else "") + "List of tools: [" -%}
55
+ {%- for tool in tools -%}
56
+ {%- if tool is not string -%}
57
+ {%- set tool = tool | tojson -%}
58
+ {%- endif -%}
59
+ {%- set ns.system_prompt = ns.system_prompt + tool -%}
60
+ {%- if not loop.last -%}
61
+ {%- set ns.system_prompt = ns.system_prompt + ", " -%}
62
+ {%- endif -%}
63
+ {%- endfor -%}
64
+ {%- set ns.system_prompt = ns.system_prompt + "]" -%}
65
+ {%- endif -%}
66
+ {%- if ns.system_prompt -%}
67
+ {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
68
+ {%- endif -%}
69
+ {%- for message in messages -%}
70
+ {%- if message["role"] == "user" -%}
71
+ {%- set ns.last_user_index = loop.index0 -%}
72
+ {%- endif -%}
73
+ {%- endfor -%}
74
+ {%- for message in messages -%}
75
+ {{- "<|im_start|>" + message.role + "\n" -}}
76
+ {%- if message.role == "assistant" -%}
77
+ {%- generation -%}
78
+ {%- if message.thinking is defined and (preserve_thinking or loop.index0 > ns.last_user_index) -%}
79
+ {{- "<think>" + message.thinking + "</think>" -}}
80
+ {%- endif -%}
81
+ {%- set _cfm_tag = "CONTINUE_FINAL_MESSAGE_TAG " -%}
82
+ {%- set _has_cfm = false -%}
83
+ {%- if message.content is defined -%}
84
+ {%- set content = parse_content(message.content) -%}
85
+ {%- if not (preserve_thinking or loop.index0 > ns.last_user_index) -%}
86
+ {%- if "</think>" in content -%}
87
+ {%- set content = content.split("</think>")[-1] | trim -%}
88
+ {%- endif -%}
89
+ {%- endif -%}
90
+ {%- if message.tool_calls is defined and content.endswith(_cfm_tag) -%}
91
+ {%- set _has_cfm = true -%}
92
+ {%- set _trunc_len = (content | length) - (_cfm_tag | length) -%}
93
+ {{- content[:_trunc_len] -}}
94
+ {%- else -%}
95
+ {{- content -}}
96
+ {%- endif -%}
97
+ {%- endif -%}
98
+ {%- if message.tool_calls is defined -%}
99
+ {{- render_tool_calls(message.tool_calls) -}}
100
+ {%- endif -%}
101
+ {%- if _has_cfm -%}
102
+ {{- _cfm_tag -}}
103
+ {%- endif -%}
104
+ {{- "<|im_end|>\n" -}}
105
+ {%- endgeneration -%}
106
+ {%- else %}
107
+ {%- if message.get("content") -%}
108
+ {{- parse_content(message["content"]) -}}
109
+ {%- endif -%}
110
+ {{- "<|im_end|>\n" -}}
111
+ {%- endif %}
112
+ {%- endfor -%}
113
+ {%- if add_generation_prompt -%}
114
+ {{- "<|im_start|>assistant\n" -}}
115
+ {%- endif -%}
config.json ADDED
@@ -0,0 +1,451 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Lfm2MoeForCausalLM"
4
+ ],
5
+ "bos_token_id": 124894,
6
+ "conv_L_cache": 3,
7
+ "conv_bias": false,
8
+ "dtype": "bfloat16",
9
+ "eos_token_id": 124900,
10
+ "hidden_size": 2048,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 7168,
13
+ "layer_types": [
14
+ "conv",
15
+ "conv",
16
+ "full_attention",
17
+ "conv",
18
+ "conv",
19
+ "conv",
20
+ "full_attention",
21
+ "conv",
22
+ "conv",
23
+ "conv",
24
+ "full_attention",
25
+ "conv",
26
+ "conv",
27
+ "conv",
28
+ "full_attention",
29
+ "conv",
30
+ "conv",
31
+ "conv",
32
+ "full_attention",
33
+ "conv",
34
+ "conv",
35
+ "full_attention",
36
+ "conv",
37
+ "conv"
38
+ ],
39
+ "max_position_embeddings": 128000,
40
+ "model_type": "lfm2_moe",
41
+ "moe_intermediate_size": 1792,
42
+ "norm_eps": 1e-05,
43
+ "norm_topk_prob": true,
44
+ "num_attention_heads": 32,
45
+ "num_dense_layers": 2,
46
+ "num_experts": 32,
47
+ "num_experts_per_tok": 4,
48
+ "num_hidden_layers": 24,
49
+ "num_key_value_heads": 8,
50
+ "pad_token_id": 124893,
51
+ "rope_parameters": {
52
+ "rope_theta": 5000000,
53
+ "rope_type": "default"
54
+ },
55
+ "routed_scaling_factor": 1.0,
56
+ "tie_word_embeddings": true,
57
+ "transformers_version": "5.9.0",
58
+ "use_cache": true,
59
+ "use_expert_bias": true,
60
+ "vocab_size": 128000,
61
+ "quantization": {
62
+ "group_size": 64,
63
+ "bits": 4,
64
+ "mode": "affine",
65
+ "model.embed_tokens": {
66
+ "bits": 8,
67
+ "group_size": 64,
68
+ "mode": "affine"
69
+ },
70
+ "model.layers.0.conv.in_proj": {
71
+ "bits": 8,
72
+ "group_size": 64,
73
+ "mode": "affine"
74
+ },
75
+ "model.layers.0.conv.out_proj": {
76
+ "bits": 8,
77
+ "group_size": 64,
78
+ "mode": "affine"
79
+ },
80
+ "model.layers.0.feed_forward.gate_proj": {
81
+ "bits": 8,
82
+ "group_size": 64,
83
+ "mode": "affine"
84
+ },
85
+ "model.layers.0.feed_forward.down_proj": {
86
+ "bits": 8,
87
+ "group_size": 64,
88
+ "mode": "affine"
89
+ },
90
+ "model.layers.0.feed_forward.up_proj": {
91
+ "bits": 8,
92
+ "group_size": 64,
93
+ "mode": "affine"
94
+ },
95
+ "model.layers.1.conv.in_proj": {
96
+ "bits": 6,
97
+ "group_size": 64,
98
+ "mode": "affine"
99
+ },
100
+ "model.layers.1.conv.out_proj": {
101
+ "bits": 6,
102
+ "group_size": 64,
103
+ "mode": "affine"
104
+ },
105
+ "model.layers.1.feed_forward.gate_proj": {
106
+ "bits": 6,
107
+ "group_size": 64,
108
+ "mode": "affine"
109
+ },
110
+ "model.layers.1.feed_forward.down_proj": {
111
+ "bits": 6,
112
+ "group_size": 64,
113
+ "mode": "affine"
114
+ },
115
+ "model.layers.1.feed_forward.up_proj": {
116
+ "bits": 6,
117
+ "group_size": 64,
118
+ "mode": "affine"
119
+ },
120
+ "model.layers.10.self_attn.k_proj": {
121
+ "bits": 8,
122
+ "group_size": 64,
123
+ "mode": "affine"
124
+ },
125
+ "model.layers.10.self_attn.out_proj": {
126
+ "bits": 8,
127
+ "group_size": 64,
128
+ "mode": "affine"
129
+ },
130
+ "model.layers.10.self_attn.q_proj": {
131
+ "bits": 8,
132
+ "group_size": 64,
133
+ "mode": "affine"
134
+ },
135
+ "model.layers.10.self_attn.v_proj": {
136
+ "bits": 8,
137
+ "group_size": 64,
138
+ "mode": "affine"
139
+ },
140
+ "model.layers.13.conv.in_proj": {
141
+ "bits": 5,
142
+ "group_size": 64,
143
+ "mode": "affine"
144
+ },
145
+ "model.layers.14.self_attn.k_proj": {
146
+ "bits": 8,
147
+ "group_size": 64,
148
+ "mode": "affine"
149
+ },
150
+ "model.layers.14.self_attn.out_proj": {
151
+ "bits": 8,
152
+ "group_size": 64,
153
+ "mode": "affine"
154
+ },
155
+ "model.layers.14.self_attn.q_proj": {
156
+ "bits": 8,
157
+ "group_size": 64,
158
+ "mode": "affine"
159
+ },
160
+ "model.layers.14.self_attn.v_proj": {
161
+ "bits": 8,
162
+ "group_size": 64,
163
+ "mode": "affine"
164
+ },
165
+ "model.layers.18.self_attn.k_proj": {
166
+ "bits": 8,
167
+ "group_size": 64,
168
+ "mode": "affine"
169
+ },
170
+ "model.layers.18.self_attn.out_proj": {
171
+ "bits": 8,
172
+ "group_size": 64,
173
+ "mode": "affine"
174
+ },
175
+ "model.layers.18.self_attn.q_proj": {
176
+ "bits": 8,
177
+ "group_size": 64,
178
+ "mode": "affine"
179
+ },
180
+ "model.layers.18.self_attn.v_proj": {
181
+ "bits": 8,
182
+ "group_size": 64,
183
+ "mode": "affine"
184
+ },
185
+ "model.layers.2.self_attn.k_proj": {
186
+ "bits": 6,
187
+ "group_size": 64,
188
+ "mode": "affine"
189
+ },
190
+ "model.layers.2.self_attn.out_proj": {
191
+ "bits": 6,
192
+ "group_size": 64,
193
+ "mode": "affine"
194
+ },
195
+ "model.layers.2.self_attn.q_proj": {
196
+ "bits": 6,
197
+ "group_size": 64,
198
+ "mode": "affine"
199
+ },
200
+ "model.layers.2.self_attn.v_proj": {
201
+ "bits": 6,
202
+ "group_size": 64,
203
+ "mode": "affine"
204
+ },
205
+ "model.layers.21.self_attn.k_proj": {
206
+ "bits": 8,
207
+ "group_size": 64,
208
+ "mode": "affine"
209
+ },
210
+ "model.layers.21.self_attn.out_proj": {
211
+ "bits": 8,
212
+ "group_size": 64,
213
+ "mode": "affine"
214
+ },
215
+ "model.layers.21.self_attn.q_proj": {
216
+ "bits": 8,
217
+ "group_size": 64,
218
+ "mode": "affine"
219
+ },
220
+ "model.layers.21.self_attn.v_proj": {
221
+ "bits": 8,
222
+ "group_size": 64,
223
+ "mode": "affine"
224
+ },
225
+ "model.layers.3.conv.in_proj": {
226
+ "bits": 6,
227
+ "group_size": 64,
228
+ "mode": "affine"
229
+ },
230
+ "model.layers.3.conv.out_proj": {
231
+ "bits": 6,
232
+ "group_size": 64,
233
+ "mode": "affine"
234
+ },
235
+ "model.layers.6.self_attn.k_proj": {
236
+ "bits": 8,
237
+ "group_size": 64,
238
+ "mode": "affine"
239
+ },
240
+ "model.layers.6.self_attn.out_proj": {
241
+ "bits": 8,
242
+ "group_size": 64,
243
+ "mode": "affine"
244
+ },
245
+ "model.layers.6.self_attn.q_proj": {
246
+ "bits": 8,
247
+ "group_size": 64,
248
+ "mode": "affine"
249
+ },
250
+ "model.layers.6.self_attn.v_proj": {
251
+ "bits": 8,
252
+ "group_size": 64,
253
+ "mode": "affine"
254
+ }
255
+ },
256
+ "quantization_config": {
257
+ "group_size": 64,
258
+ "bits": 4,
259
+ "mode": "affine",
260
+ "model.embed_tokens": {
261
+ "bits": 8,
262
+ "group_size": 64,
263
+ "mode": "affine"
264
+ },
265
+ "model.layers.0.conv.in_proj": {
266
+ "bits": 8,
267
+ "group_size": 64,
268
+ "mode": "affine"
269
+ },
270
+ "model.layers.0.conv.out_proj": {
271
+ "bits": 8,
272
+ "group_size": 64,
273
+ "mode": "affine"
274
+ },
275
+ "model.layers.0.feed_forward.gate_proj": {
276
+ "bits": 8,
277
+ "group_size": 64,
278
+ "mode": "affine"
279
+ },
280
+ "model.layers.0.feed_forward.down_proj": {
281
+ "bits": 8,
282
+ "group_size": 64,
283
+ "mode": "affine"
284
+ },
285
+ "model.layers.0.feed_forward.up_proj": {
286
+ "bits": 8,
287
+ "group_size": 64,
288
+ "mode": "affine"
289
+ },
290
+ "model.layers.1.conv.in_proj": {
291
+ "bits": 6,
292
+ "group_size": 64,
293
+ "mode": "affine"
294
+ },
295
+ "model.layers.1.conv.out_proj": {
296
+ "bits": 6,
297
+ "group_size": 64,
298
+ "mode": "affine"
299
+ },
300
+ "model.layers.1.feed_forward.gate_proj": {
301
+ "bits": 6,
302
+ "group_size": 64,
303
+ "mode": "affine"
304
+ },
305
+ "model.layers.1.feed_forward.down_proj": {
306
+ "bits": 6,
307
+ "group_size": 64,
308
+ "mode": "affine"
309
+ },
310
+ "model.layers.1.feed_forward.up_proj": {
311
+ "bits": 6,
312
+ "group_size": 64,
313
+ "mode": "affine"
314
+ },
315
+ "model.layers.10.self_attn.k_proj": {
316
+ "bits": 8,
317
+ "group_size": 64,
318
+ "mode": "affine"
319
+ },
320
+ "model.layers.10.self_attn.out_proj": {
321
+ "bits": 8,
322
+ "group_size": 64,
323
+ "mode": "affine"
324
+ },
325
+ "model.layers.10.self_attn.q_proj": {
326
+ "bits": 8,
327
+ "group_size": 64,
328
+ "mode": "affine"
329
+ },
330
+ "model.layers.10.self_attn.v_proj": {
331
+ "bits": 8,
332
+ "group_size": 64,
333
+ "mode": "affine"
334
+ },
335
+ "model.layers.13.conv.in_proj": {
336
+ "bits": 5,
337
+ "group_size": 64,
338
+ "mode": "affine"
339
+ },
340
+ "model.layers.14.self_attn.k_proj": {
341
+ "bits": 8,
342
+ "group_size": 64,
343
+ "mode": "affine"
344
+ },
345
+ "model.layers.14.self_attn.out_proj": {
346
+ "bits": 8,
347
+ "group_size": 64,
348
+ "mode": "affine"
349
+ },
350
+ "model.layers.14.self_attn.q_proj": {
351
+ "bits": 8,
352
+ "group_size": 64,
353
+ "mode": "affine"
354
+ },
355
+ "model.layers.14.self_attn.v_proj": {
356
+ "bits": 8,
357
+ "group_size": 64,
358
+ "mode": "affine"
359
+ },
360
+ "model.layers.18.self_attn.k_proj": {
361
+ "bits": 8,
362
+ "group_size": 64,
363
+ "mode": "affine"
364
+ },
365
+ "model.layers.18.self_attn.out_proj": {
366
+ "bits": 8,
367
+ "group_size": 64,
368
+ "mode": "affine"
369
+ },
370
+ "model.layers.18.self_attn.q_proj": {
371
+ "bits": 8,
372
+ "group_size": 64,
373
+ "mode": "affine"
374
+ },
375
+ "model.layers.18.self_attn.v_proj": {
376
+ "bits": 8,
377
+ "group_size": 64,
378
+ "mode": "affine"
379
+ },
380
+ "model.layers.2.self_attn.k_proj": {
381
+ "bits": 6,
382
+ "group_size": 64,
383
+ "mode": "affine"
384
+ },
385
+ "model.layers.2.self_attn.out_proj": {
386
+ "bits": 6,
387
+ "group_size": 64,
388
+ "mode": "affine"
389
+ },
390
+ "model.layers.2.self_attn.q_proj": {
391
+ "bits": 6,
392
+ "group_size": 64,
393
+ "mode": "affine"
394
+ },
395
+ "model.layers.2.self_attn.v_proj": {
396
+ "bits": 6,
397
+ "group_size": 64,
398
+ "mode": "affine"
399
+ },
400
+ "model.layers.21.self_attn.k_proj": {
401
+ "bits": 8,
402
+ "group_size": 64,
403
+ "mode": "affine"
404
+ },
405
+ "model.layers.21.self_attn.out_proj": {
406
+ "bits": 8,
407
+ "group_size": 64,
408
+ "mode": "affine"
409
+ },
410
+ "model.layers.21.self_attn.q_proj": {
411
+ "bits": 8,
412
+ "group_size": 64,
413
+ "mode": "affine"
414
+ },
415
+ "model.layers.21.self_attn.v_proj": {
416
+ "bits": 8,
417
+ "group_size": 64,
418
+ "mode": "affine"
419
+ },
420
+ "model.layers.3.conv.in_proj": {
421
+ "bits": 6,
422
+ "group_size": 64,
423
+ "mode": "affine"
424
+ },
425
+ "model.layers.3.conv.out_proj": {
426
+ "bits": 6,
427
+ "group_size": 64,
428
+ "mode": "affine"
429
+ },
430
+ "model.layers.6.self_attn.k_proj": {
431
+ "bits": 8,
432
+ "group_size": 64,
433
+ "mode": "affine"
434
+ },
435
+ "model.layers.6.self_attn.out_proj": {
436
+ "bits": 8,
437
+ "group_size": 64,
438
+ "mode": "affine"
439
+ },
440
+ "model.layers.6.self_attn.q_proj": {
441
+ "bits": 8,
442
+ "group_size": 64,
443
+ "mode": "affine"
444
+ },
445
+ "model.layers.6.self_attn.v_proj": {
446
+ "bits": 8,
447
+ "group_size": 64,
448
+ "mode": "affine"
449
+ }
450
+ }
451
+ }
generation_config.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 124894,
4
+ "eos_token_id": 124900,
5
+ "output_attentions": false,
6
+ "output_hidden_states": false,
7
+ "pad_token_id": 124893,
8
+ "do_sample": true,
9
+ "temperature": 0.2,
10
+ "top_k": 80,
11
+ "repetition_penalty": 1.05,
12
+ "transformers_version": "5.9.0",
13
+ "use_cache": true
14
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fb8fc8e95dc2e912ceb1387f690ed69fb384cadff99f77bc64cc7b13dc2fc0e7
3
+ size 4976900037
oq_imatrix_report.json ADDED
@@ -0,0 +1,237 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "enabled": true,
3
+ "cache_path": "/Users/joey/.omlx/models/.oqe_imatrix/LFM2.5-8B-A1B-9e5e93b9f33b-s128-l512.npz",
4
+ "cache_reused": true,
5
+ "entry_count": 154,
6
+ "calib_dataset": "oqe_code_multilingual",
7
+ "collection": {
8
+ "dataset": "oqe_code_multilingual",
9
+ "requested_samples": 128,
10
+ "seq_length": 512,
11
+ "adaptive": true,
12
+ "adaptive_step_samples": 128,
13
+ "adaptive_max_samples": 1024,
14
+ "available_samples": 1024,
15
+ "micro_batch_size": 6,
16
+ "micro_batches": 22,
17
+ "batch_plan": {
18
+ "micro_batch_size": 6,
19
+ "estimated_sample_bytes": 16777216,
20
+ "capture_budget_bytes": 114543193,
21
+ "system_available_bytes": 28390326272,
22
+ "metal_available_bytes": 49387627408,
23
+ "live_available_bytes": 28390326272,
24
+ "model_bytes": 16936006912,
25
+ "remaining_available_bytes": 11454319360,
26
+ "fits_one_sample": true,
27
+ "hidden_size": 2048,
28
+ "num_experts": 32,
29
+ "top_k": 4,
30
+ "gemma4_state_bytes": 0,
31
+ "num_hidden_layers": 24,
32
+ "num_kv_shared_layers": 0,
33
+ "per_layer_input_size": 0
34
+ },
35
+ "processed_samples": 128,
36
+ "installed_modules": 154,
37
+ "capture_module_classes": {
38
+ "Linear": 88,
39
+ "SwitchLinear": 66
40
+ },
41
+ "switch_capture_modules": 66,
42
+ "requires_expert_counts": true,
43
+ "coverage_sufficient": true,
44
+ "collection_sufficient": true,
45
+ "coverage": {
46
+ "has_expert_counts": true,
47
+ "expert_modules": 66,
48
+ "total_experts": 2112,
49
+ "active_experts": 2112,
50
+ "zero_count_experts": 0,
51
+ "active_ratio": 1.0,
52
+ "min_count": 380,
53
+ "p05_count": 2275.0,
54
+ "p10_count": 3160.0,
55
+ "median_count": 7236.5,
56
+ "max_count": 49134,
57
+ "min_required_count": 16,
58
+ "required_percentile": 5
59
+ },
60
+ "rounds": [
61
+ {
62
+ "processed_samples": 128,
63
+ "coverage_sufficient": true,
64
+ "collection_sufficient": true,
65
+ "coverage": {
66
+ "has_expert_counts": true,
67
+ "expert_modules": 66,
68
+ "total_experts": 2112,
69
+ "active_experts": 2112,
70
+ "zero_count_experts": 0,
71
+ "active_ratio": 1.0,
72
+ "min_count": 380,
73
+ "p05_count": 2275.0,
74
+ "p10_count": 3160.0,
75
+ "median_count": 7236.5,
76
+ "max_count": 49134,
77
+ "min_required_count": 16,
78
+ "required_percentile": 5
79
+ }
80
+ }
81
+ ]
82
+ },
83
+ "expert_coverage": {
84
+ "has_expert_counts": true,
85
+ "expert_modules": 66,
86
+ "total_experts": 2112,
87
+ "active_experts": 2112,
88
+ "zero_count_experts": 0,
89
+ "active_ratio": 1.0,
90
+ "min_count": 380,
91
+ "p05_count": 2275.0,
92
+ "p10_count": 3160.0,
93
+ "median_count": 7236.5,
94
+ "max_count": 49134,
95
+ "min_required_count": 16,
96
+ "required_percentile": 5
97
+ },
98
+ "applied": [
99
+ "model.layers.0.conv.in_proj",
100
+ "model.layers.0.conv.out_proj",
101
+ "model.layers.0.feed_forward.down_proj",
102
+ "model.layers.0.feed_forward.gate_proj",
103
+ "model.layers.0.feed_forward.up_proj",
104
+ "model.layers.1.conv.in_proj",
105
+ "model.layers.1.conv.out_proj",
106
+ "model.layers.1.feed_forward.down_proj",
107
+ "model.layers.1.feed_forward.gate_proj",
108
+ "model.layers.1.feed_forward.up_proj",
109
+ "model.layers.10.feed_forward.switch_mlp.down_proj",
110
+ "model.layers.10.feed_forward.switch_mlp.gate_proj",
111
+ "model.layers.10.feed_forward.switch_mlp.up_proj",
112
+ "model.layers.10.self_attn.k_proj",
113
+ "model.layers.10.self_attn.out_proj",
114
+ "model.layers.10.self_attn.q_proj",
115
+ "model.layers.10.self_attn.v_proj",
116
+ "model.layers.11.conv.in_proj",
117
+ "model.layers.11.conv.out_proj",
118
+ "model.layers.11.feed_forward.switch_mlp.down_proj",
119
+ "model.layers.11.feed_forward.switch_mlp.gate_proj",
120
+ "model.layers.11.feed_forward.switch_mlp.up_proj",
121
+ "model.layers.12.conv.in_proj",
122
+ "model.layers.12.conv.out_proj",
123
+ "model.layers.12.feed_forward.switch_mlp.down_proj",
124
+ "model.layers.12.feed_forward.switch_mlp.gate_proj",
125
+ "model.layers.12.feed_forward.switch_mlp.up_proj",
126
+ "model.layers.13.conv.in_proj",
127
+ "model.layers.13.conv.out_proj",
128
+ "model.layers.13.feed_forward.switch_mlp.down_proj",
129
+ "model.layers.13.feed_forward.switch_mlp.gate_proj",
130
+ "model.layers.13.feed_forward.switch_mlp.up_proj",
131
+ "model.layers.14.feed_forward.switch_mlp.down_proj",
132
+ "model.layers.14.feed_forward.switch_mlp.gate_proj",
133
+ "model.layers.14.feed_forward.switch_mlp.up_proj",
134
+ "model.layers.14.self_attn.k_proj",
135
+ "model.layers.14.self_attn.out_proj",
136
+ "model.layers.14.self_attn.q_proj",
137
+ "model.layers.14.self_attn.v_proj",
138
+ "model.layers.15.conv.in_proj",
139
+ "model.layers.15.conv.out_proj",
140
+ "model.layers.15.feed_forward.switch_mlp.down_proj",
141
+ "model.layers.15.feed_forward.switch_mlp.gate_proj",
142
+ "model.layers.15.feed_forward.switch_mlp.up_proj",
143
+ "model.layers.16.conv.in_proj",
144
+ "model.layers.16.conv.out_proj",
145
+ "model.layers.16.feed_forward.switch_mlp.down_proj",
146
+ "model.layers.16.feed_forward.switch_mlp.gate_proj",
147
+ "model.layers.16.feed_forward.switch_mlp.up_proj",
148
+ "model.layers.17.conv.in_proj",
149
+ "model.layers.17.conv.out_proj",
150
+ "model.layers.17.feed_forward.switch_mlp.down_proj",
151
+ "model.layers.17.feed_forward.switch_mlp.gate_proj",
152
+ "model.layers.17.feed_forward.switch_mlp.up_proj",
153
+ "model.layers.18.feed_forward.switch_mlp.down_proj",
154
+ "model.layers.18.feed_forward.switch_mlp.gate_proj",
155
+ "model.layers.18.feed_forward.switch_mlp.up_proj",
156
+ "model.layers.18.self_attn.k_proj",
157
+ "model.layers.18.self_attn.out_proj",
158
+ "model.layers.18.self_attn.q_proj",
159
+ "model.layers.18.self_attn.v_proj",
160
+ "model.layers.19.conv.in_proj",
161
+ "model.layers.19.conv.out_proj",
162
+ "model.layers.19.feed_forward.switch_mlp.down_proj",
163
+ "model.layers.19.feed_forward.switch_mlp.gate_proj",
164
+ "model.layers.19.feed_forward.switch_mlp.up_proj",
165
+ "model.layers.2.feed_forward.switch_mlp.down_proj",
166
+ "model.layers.2.feed_forward.switch_mlp.gate_proj",
167
+ "model.layers.2.feed_forward.switch_mlp.up_proj",
168
+ "model.layers.2.self_attn.k_proj",
169
+ "model.layers.2.self_attn.out_proj",
170
+ "model.layers.2.self_attn.q_proj",
171
+ "model.layers.2.self_attn.v_proj",
172
+ "model.layers.20.conv.in_proj",
173
+ "model.layers.20.conv.out_proj",
174
+ "model.layers.20.feed_forward.switch_mlp.down_proj",
175
+ "model.layers.20.feed_forward.switch_mlp.gate_proj",
176
+ "model.layers.20.feed_forward.switch_mlp.up_proj",
177
+ "model.layers.21.feed_forward.switch_mlp.down_proj",
178
+ "model.layers.21.feed_forward.switch_mlp.gate_proj",
179
+ "model.layers.21.feed_forward.switch_mlp.up_proj",
180
+ "model.layers.21.self_attn.k_proj",
181
+ "model.layers.21.self_attn.out_proj",
182
+ "model.layers.21.self_attn.q_proj",
183
+ "model.layers.21.self_attn.v_proj",
184
+ "model.layers.22.conv.in_proj",
185
+ "model.layers.22.conv.out_proj",
186
+ "model.layers.22.feed_forward.switch_mlp.down_proj",
187
+ "model.layers.22.feed_forward.switch_mlp.gate_proj",
188
+ "model.layers.22.feed_forward.switch_mlp.up_proj",
189
+ "model.layers.23.conv.in_proj",
190
+ "model.layers.23.conv.out_proj",
191
+ "model.layers.23.feed_forward.switch_mlp.down_proj",
192
+ "model.layers.23.feed_forward.switch_mlp.gate_proj",
193
+ "model.layers.23.feed_forward.switch_mlp.up_proj",
194
+ "model.layers.3.conv.in_proj",
195
+ "model.layers.3.conv.out_proj",
196
+ "model.layers.3.feed_forward.switch_mlp.down_proj",
197
+ "model.layers.3.feed_forward.switch_mlp.gate_proj",
198
+ "model.layers.3.feed_forward.switch_mlp.up_proj",
199
+ "model.layers.4.conv.in_proj",
200
+ "model.layers.4.conv.out_proj",
201
+ "model.layers.4.feed_forward.switch_mlp.down_proj",
202
+ "model.layers.4.feed_forward.switch_mlp.gate_proj",
203
+ "model.layers.4.feed_forward.switch_mlp.up_proj",
204
+ "model.layers.5.conv.in_proj",
205
+ "model.layers.5.conv.out_proj",
206
+ "model.layers.5.feed_forward.switch_mlp.down_proj",
207
+ "model.layers.5.feed_forward.switch_mlp.gate_proj",
208
+ "model.layers.5.feed_forward.switch_mlp.up_proj",
209
+ "model.layers.6.feed_forward.switch_mlp.down_proj",
210
+ "model.layers.6.feed_forward.switch_mlp.gate_proj",
211
+ "model.layers.6.feed_forward.switch_mlp.up_proj",
212
+ "model.layers.6.self_attn.k_proj",
213
+ "model.layers.6.self_attn.out_proj",
214
+ "model.layers.6.self_attn.q_proj",
215
+ "model.layers.6.self_attn.v_proj",
216
+ "model.layers.7.conv.in_proj",
217
+ "model.layers.7.conv.out_proj",
218
+ "model.layers.7.feed_forward.switch_mlp.down_proj",
219
+ "model.layers.7.feed_forward.switch_mlp.gate_proj",
220
+ "model.layers.7.feed_forward.switch_mlp.up_proj",
221
+ "model.layers.8.conv.in_proj",
222
+ "model.layers.8.conv.out_proj",
223
+ "model.layers.8.feed_forward.switch_mlp.down_proj",
224
+ "model.layers.8.feed_forward.switch_mlp.gate_proj",
225
+ "model.layers.8.feed_forward.switch_mlp.up_proj",
226
+ "model.layers.9.conv.in_proj",
227
+ "model.layers.9.conv.out_proj",
228
+ "model.layers.9.feed_forward.switch_mlp.down_proj",
229
+ "model.layers.9.feed_forward.switch_mlp.gate_proj",
230
+ "model.layers.9.feed_forward.switch_mlp.up_proj"
231
+ ],
232
+ "missing": [
233
+ "model.embed_tokens"
234
+ ],
235
+ "mismatched": [],
236
+ "zero_count_experts": 0
237
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:695be7802a0e4b8a81048f0ff5ebb7fc811a0ba5a6be63dbb24deb5a81096f41
3
+ size 17905598
tokenizer_config.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "bos_token": "<|startoftext|>",
4
+ "clean_up_tokenization_spaces": false,
5
+ "eos_token": "<|im_end|>",
6
+ "is_local": true,
7
+ "model_max_length": 1000000000000000019884624838656,
8
+ "pad_token": "<|pad|>",
9
+ "tokenizer_class": "TokenizersBackend",
10
+ "use_default_system_prompt": false
11
+ }