Tech2wild commited on
Commit
d7c549e
·
verified ·
1 Parent(s): a382da8

config: prefix-agnostic compressed-tensors targets (vLLM matches its own module names)

Browse files
Files changed (1) hide show
  1. config.json +10 -9
config.json CHANGED
@@ -125,19 +125,20 @@
125
  "format": "pack-quantized",
126
  "quantization_status": "compressed",
127
  "ignore": [
128
- "re:model[.]language_model[.]layers[.]0[.](?:self_attn|linear_attn|mlp[.]shared_expert)[.].*",
129
- "re:model[.]language_model[.]layers[.][0-9]+[.]mlp[.]gate$",
130
- "re:model[.]language_model[.]layers[.][0-9]+[.]mlp[.]shared_expert_gate$",
131
- "re:model[.]language_model[.]layers[.][0-9]+[.]linear_attn[.](?:in_proj_a|in_proj_b|in_proj_ba|conv1d)$",
132
- "re:model[.]language_model[.]layers[.][0-9]+[.]self_attn[.](?:q_norm|k_norm)$",
133
- "re:model[.]language_model[.]embed_tokens$",
134
  "lm_head",
135
- "re:model[.]visual[.].*"
 
136
  ],
137
  "config_groups": {
138
  "w4a16_experts": {
139
  "targets": [
140
- "re:model[.]language_model[.]layers[.][0-9]+[.]mlp[.]experts[.][0-9]+[.](?:gate_proj|up_proj|down_proj)$"
141
  ],
142
  "input_activations": null,
143
  "output_activations": null,
@@ -154,7 +155,7 @@
154
  },
155
  "w8a16_linears": {
156
  "targets": [
157
- "re:model[.]language_model[.]layers[.](?:[1-9][0-9]*)[.](?:self_attn[.](?:q_proj|k_proj|v_proj|o_proj)|linear_attn[.](?:in_proj_qkv|in_proj_z|out_proj)|mlp[.]shared_expert[.](?:gate_proj|up_proj|down_proj))$"
158
  ],
159
  "input_activations": null,
160
  "output_activations": null,
 
125
  "format": "pack-quantized",
126
  "quantization_status": "compressed",
127
  "ignore": [
128
+ "re:.*layers[.]0[.](?:self_attn|linear_attn|mlp[.]shared_expert)[.].*",
129
+ "re:.*layers[.][0-9]+[.]mlp[.]gate$",
130
+ "re:.*layers[.][0-9]+[.]mlp[.]shared_expert_gate$",
131
+ "re:.*layers[.][0-9]+[.]linear_attn[.](?:in_proj_a|in_proj_b|in_proj_ba|conv1d)$",
132
+ "re:.*layers[.][0-9]+[.]self_attn[.](?:q_norm|k_norm)$",
133
+ "re:.*embed_tokens$",
134
  "lm_head",
135
+ "re:.*lm_head$",
136
+ "re:.*visual[.].*"
137
  ],
138
  "config_groups": {
139
  "w4a16_experts": {
140
  "targets": [
141
+ "re:.*layers[.][0-9]+[.]mlp[.]experts[.][0-9]+[.](?:gate_proj|up_proj|down_proj)$"
142
  ],
143
  "input_activations": null,
144
  "output_activations": null,
 
155
  },
156
  "w8a16_linears": {
157
  "targets": [
158
+ "re:.*layers[.](?:[1-9][0-9]*)[.](?:self_attn[.](?:q_proj|k_proj|v_proj|o_proj)|linear_attn[.](?:in_proj_qkv|in_proj_z|out_proj)|mlp[.]shared_expert[.](?:gate_proj|up_proj|down_proj))$"
159
  ],
160
  "input_activations": null,
161
  "output_activations": null,