config: prefix-agnostic compressed-tensors targets (vLLM matches its own module names)
Browse files- config.json +10 -9
config.json
CHANGED
|
@@ -125,19 +125,20 @@
|
|
| 125 |
"format": "pack-quantized",
|
| 126 |
"quantization_status": "compressed",
|
| 127 |
"ignore": [
|
| 128 |
-
"re:
|
| 129 |
-
"re:
|
| 130 |
-
"re:
|
| 131 |
-
"re:
|
| 132 |
-
"re:
|
| 133 |
-
"re:
|
| 134 |
"lm_head",
|
| 135 |
-
"re:
|
|
|
|
| 136 |
],
|
| 137 |
"config_groups": {
|
| 138 |
"w4a16_experts": {
|
| 139 |
"targets": [
|
| 140 |
-
"re:
|
| 141 |
],
|
| 142 |
"input_activations": null,
|
| 143 |
"output_activations": null,
|
|
@@ -154,7 +155,7 @@
|
|
| 154 |
},
|
| 155 |
"w8a16_linears": {
|
| 156 |
"targets": [
|
| 157 |
-
"re:
|
| 158 |
],
|
| 159 |
"input_activations": null,
|
| 160 |
"output_activations": null,
|
|
|
|
| 125 |
"format": "pack-quantized",
|
| 126 |
"quantization_status": "compressed",
|
| 127 |
"ignore": [
|
| 128 |
+
"re:.*layers[.]0[.](?:self_attn|linear_attn|mlp[.]shared_expert)[.].*",
|
| 129 |
+
"re:.*layers[.][0-9]+[.]mlp[.]gate$",
|
| 130 |
+
"re:.*layers[.][0-9]+[.]mlp[.]shared_expert_gate$",
|
| 131 |
+
"re:.*layers[.][0-9]+[.]linear_attn[.](?:in_proj_a|in_proj_b|in_proj_ba|conv1d)$",
|
| 132 |
+
"re:.*layers[.][0-9]+[.]self_attn[.](?:q_norm|k_norm)$",
|
| 133 |
+
"re:.*embed_tokens$",
|
| 134 |
"lm_head",
|
| 135 |
+
"re:.*lm_head$",
|
| 136 |
+
"re:.*visual[.].*"
|
| 137 |
],
|
| 138 |
"config_groups": {
|
| 139 |
"w4a16_experts": {
|
| 140 |
"targets": [
|
| 141 |
+
"re:.*layers[.][0-9]+[.]mlp[.]experts[.][0-9]+[.](?:gate_proj|up_proj|down_proj)$"
|
| 142 |
],
|
| 143 |
"input_activations": null,
|
| 144 |
"output_activations": null,
|
|
|
|
| 155 |
},
|
| 156 |
"w8a16_linears": {
|
| 157 |
"targets": [
|
| 158 |
+
"re:.*layers[.](?:[1-9][0-9]*)[.](?:self_attn[.](?:q_proj|k_proj|v_proj|o_proj)|linear_attn[.](?:in_proj_qkv|in_proj_z|out_proj)|mlp[.]shared_expert[.](?:gate_proj|up_proj|down_proj))$"
|
| 159 |
],
|
| 160 |
"input_activations": null,
|
| 161 |
"output_activations": null,
|