| { |
| "bits": 4, |
| "act_bits": 4, |
| "data_type": "mx_fp", |
| "act_data_type": "mx_fp_rceil", |
| "group_size": 32, |
| "act_group_size": 32, |
| "sym": true, |
| "act_sym": true, |
| "act_dynamic": true, |
| "low_gpu_mem_usage": true, |
| "autoround_version": "0.13.0", |
| "block_name_to_quantize": "model.language_model.layers", |
| "quant_method": "auto-round", |
| "packing_format": "auto_round:llm_compressor", |
| "extra_config": { |
| "model.language_model.layers.3.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.3.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.3.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.3.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.7.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.7.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.7.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.7.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.11.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.11.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.11.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.11.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.15.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.15.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.15.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.15.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.19.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.19.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.19.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.19.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.23.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.23.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.23.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.23.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.27.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.27.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.27.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.27.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.31.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.31.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.31.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.31.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.35.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.35.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.35.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.35.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.39.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.39.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.39.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.39.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.43.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.43.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.43.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.43.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.47.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.47.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.47.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.47.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.51.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.51.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.51.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.51.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.55.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.55.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.55.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.55.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.59.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.59.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.59.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.59.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.63.self_attn.q_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.63.self_attn.k_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.63.self_attn.v_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| "model.language_model.layers.63.self_attn.o_proj": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| }, |
| ".*self_attn.*": { |
| "bits": 16, |
| "data_type": "float", |
| "act_bits": 16, |
| "act_data_type": "float" |
| } |
| } |
| } |
