| { |
| "config_id": "loftq2", |
| "method": "loftq", |
| "source_model": "meta-llama/Llama-3.2-1B-Instruct", |
| "scheme": "W2A16-LoftQ-NF2", |
| "bits_weight": 2, |
| "bits_activation": 16, |
| "quantizer": "normal-float NF-k, block-wise absmax", |
| "block_size": 64, |
| "loftq_iters": 5, |
| "lora_rank": 16, |
| "lora_alpha": 32, |
| "lora_scaling": 2.0, |
| "factors_prescaled_by": "1/sqrt(alpha/r)", |
| "simulated": true, |
| "simulated_reason": "vLLM 0.11 compressed-tensors supports only 4/8-bit; 2/3-bit have no kernel", |
| "n_modules": 112, |
| "mean_rel_err_quant_only": 0.56728, |
| "mean_rel_err_loftq": 0.498203, |
| "per_module": [ |
| { |
| "module": "model.layers.0.self_attn.q_proj", |
| "rel_err_quant_only": 0.566828, |
| "rel_err_loftq": 0.47505 |
| }, |
| { |
| "module": "model.layers.0.self_attn.k_proj", |
| "rel_err_quant_only": 0.571923, |
| "rel_err_loftq": 0.470079 |
| }, |
| { |
| "module": "model.layers.0.self_attn.v_proj", |
| "rel_err_quant_only": 0.565481, |
| "rel_err_loftq": 0.48724 |
| }, |
| { |
| "module": "model.layers.0.self_attn.o_proj", |
| "rel_err_quant_only": 0.568331, |
| "rel_err_loftq": 0.498975 |
| }, |
| { |
| "module": "model.layers.0.mlp.gate_proj", |
| "rel_err_quant_only": 0.564853, |
| "rel_err_loftq": 0.506129 |
| }, |
| { |
| "module": "model.layers.0.mlp.up_proj", |
| "rel_err_quant_only": 0.564261, |
| "rel_err_loftq": 0.508477 |
| }, |
| { |
| "module": "model.layers.0.mlp.down_proj", |
| "rel_err_quant_only": 0.56682, |
| "rel_err_loftq": 0.50729 |
| }, |
| { |
| "module": "model.layers.1.self_attn.q_proj", |
| "rel_err_quant_only": 0.567312, |
| "rel_err_loftq": 0.494297 |
| }, |
| { |
| "module": "model.layers.1.self_attn.k_proj", |
| "rel_err_quant_only": 0.567312, |
| "rel_err_loftq": 0.48416 |
| }, |
| { |
| "module": "model.layers.1.self_attn.v_proj", |
| "rel_err_quant_only": 0.569565, |
| "rel_err_loftq": 0.492087 |
| }, |
| { |
| "module": "model.layers.1.self_attn.o_proj", |
| "rel_err_quant_only": 0.565782, |
| "rel_err_loftq": 0.497265 |
| }, |
| { |
| "module": "model.layers.1.mlp.gate_proj", |
| "rel_err_quant_only": 0.564656, |
| "rel_err_loftq": 0.503957 |
| }, |
| { |
| "module": "model.layers.1.mlp.up_proj", |
| "rel_err_quant_only": 0.564698, |
| "rel_err_loftq": 0.505917 |
| }, |
| { |
| "module": "model.layers.1.mlp.down_proj", |
| "rel_err_quant_only": 0.565635, |
| "rel_err_loftq": 0.508224 |
| }, |
| { |
| "module": "model.layers.2.self_attn.q_proj", |
| "rel_err_quant_only": 0.569393, |
| "rel_err_loftq": 0.495053 |
| }, |
| { |
| "module": "model.layers.2.self_attn.k_proj", |
| "rel_err_quant_only": 0.57035, |
| "rel_err_loftq": 0.488921 |
| }, |
| { |
| "module": "model.layers.2.self_attn.v_proj", |
| "rel_err_quant_only": 0.565248, |
| "rel_err_loftq": 0.494515 |
| }, |
| { |
| "module": "model.layers.2.self_attn.o_proj", |
| "rel_err_quant_only": 0.565595, |
| "rel_err_loftq": 0.500933 |
| }, |
| { |
| "module": "model.layers.2.mlp.gate_proj", |
| "rel_err_quant_only": 0.564786, |
| "rel_err_loftq": 0.506062 |
| }, |
| { |
| "module": "model.layers.2.mlp.up_proj", |
| "rel_err_quant_only": 0.564899, |
| "rel_err_loftq": 0.508195 |
| }, |
| { |
| "module": "model.layers.2.mlp.down_proj", |
| "rel_err_quant_only": 0.566565, |
| "rel_err_loftq": 0.507794 |
| }, |
| { |
| "module": "model.layers.3.self_attn.q_proj", |
| "rel_err_quant_only": 0.565959, |
| "rel_err_loftq": 0.496884 |
| }, |
| { |
| "module": "model.layers.3.self_attn.k_proj", |
| "rel_err_quant_only": 0.566143, |
| "rel_err_loftq": 0.486739 |
| }, |
| { |
| "module": "model.layers.3.self_attn.v_proj", |
| "rel_err_quant_only": 0.567733, |
| "rel_err_loftq": 0.491771 |
| }, |
| { |
| "module": "model.layers.3.self_attn.o_proj", |
| "rel_err_quant_only": 0.564079, |
| "rel_err_loftq": 0.503849 |
| }, |
| { |
| "module": "model.layers.3.mlp.gate_proj", |
| "rel_err_quant_only": 0.564819, |
| "rel_err_loftq": 0.504443 |
| }, |
| { |
| "module": "model.layers.3.mlp.up_proj", |
| "rel_err_quant_only": 0.565127, |
| "rel_err_loftq": 0.508046 |
| }, |
| { |
| "module": "model.layers.3.mlp.down_proj", |
| "rel_err_quant_only": 0.565937, |
| "rel_err_loftq": 0.507991 |
| }, |
| { |
| "module": "model.layers.4.self_attn.q_proj", |
| "rel_err_quant_only": 0.565954, |
| "rel_err_loftq": 0.494924 |
| }, |
| { |
| "module": "model.layers.4.self_attn.k_proj", |
| "rel_err_quant_only": 0.564318, |
| "rel_err_loftq": 0.487727 |
| }, |
| { |
| "module": "model.layers.4.self_attn.v_proj", |
| "rel_err_quant_only": 0.566377, |
| "rel_err_loftq": 0.493132 |
| }, |
| { |
| "module": "model.layers.4.self_attn.o_proj", |
| "rel_err_quant_only": 0.564536, |
| "rel_err_loftq": 0.502254 |
| }, |
| { |
| "module": "model.layers.4.mlp.gate_proj", |
| "rel_err_quant_only": 0.565789, |
| "rel_err_loftq": 0.503704 |
| }, |
| { |
| "module": "model.layers.4.mlp.up_proj", |
| "rel_err_quant_only": 0.565717, |
| "rel_err_loftq": 0.507872 |
| }, |
| { |
| "module": "model.layers.4.mlp.down_proj", |
| "rel_err_quant_only": 0.56752, |
| "rel_err_loftq": 0.506855 |
| }, |
| { |
| "module": "model.layers.5.self_attn.q_proj", |
| "rel_err_quant_only": 0.566803, |
| "rel_err_loftq": 0.493599 |
| }, |
| { |
| "module": "model.layers.5.self_attn.k_proj", |
| "rel_err_quant_only": 0.567013, |
| "rel_err_loftq": 0.485892 |
| }, |
| { |
| "module": "model.layers.5.self_attn.v_proj", |
| "rel_err_quant_only": 0.565679, |
| "rel_err_loftq": 0.495578 |
| }, |
| { |
| "module": "model.layers.5.self_attn.o_proj", |
| "rel_err_quant_only": 0.564878, |
| "rel_err_loftq": 0.503081 |
| }, |
| { |
| "module": "model.layers.5.mlp.gate_proj", |
| "rel_err_quant_only": 0.566092, |
| "rel_err_loftq": 0.503258 |
| }, |
| { |
| "module": "model.layers.5.mlp.up_proj", |
| "rel_err_quant_only": 0.56573, |
| "rel_err_loftq": 0.50792 |
| }, |
| { |
| "module": "model.layers.5.mlp.down_proj", |
| "rel_err_quant_only": 0.568557, |
| "rel_err_loftq": 0.506024 |
| }, |
| { |
| "module": "model.layers.6.self_attn.q_proj", |
| "rel_err_quant_only": 0.565834, |
| "rel_err_loftq": 0.494857 |
| }, |
| { |
| "module": "model.layers.6.self_attn.k_proj", |
| "rel_err_quant_only": 0.569683, |
| "rel_err_loftq": 0.475678 |
| }, |
| { |
| "module": "model.layers.6.self_attn.v_proj", |
| "rel_err_quant_only": 0.566625, |
| "rel_err_loftq": 0.492902 |
| }, |
| { |
| "module": "model.layers.6.self_attn.o_proj", |
| "rel_err_quant_only": 0.563964, |
| "rel_err_loftq": 0.503291 |
| }, |
| { |
| "module": "model.layers.6.mlp.gate_proj", |
| "rel_err_quant_only": 0.566577, |
| "rel_err_loftq": 0.502927 |
| }, |
| { |
| "module": "model.layers.6.mlp.up_proj", |
| "rel_err_quant_only": 0.565666, |
| "rel_err_loftq": 0.507989 |
| }, |
| { |
| "module": "model.layers.6.mlp.down_proj", |
| "rel_err_quant_only": 0.570344, |
| "rel_err_loftq": 0.504929 |
| }, |
| { |
| "module": "model.layers.7.self_attn.q_proj", |
| "rel_err_quant_only": 0.565657, |
| "rel_err_loftq": 0.494279 |
| }, |
| { |
| "module": "model.layers.7.self_attn.k_proj", |
| "rel_err_quant_only": 0.567128, |
| "rel_err_loftq": 0.487261 |
| }, |
| { |
| "module": "model.layers.7.self_attn.v_proj", |
| "rel_err_quant_only": 0.565889, |
| "rel_err_loftq": 0.492771 |
| }, |
| { |
| "module": "model.layers.7.self_attn.o_proj", |
| "rel_err_quant_only": 0.564939, |
| "rel_err_loftq": 0.502272 |
| }, |
| { |
| "module": "model.layers.7.mlp.gate_proj", |
| "rel_err_quant_only": 0.566911, |
| "rel_err_loftq": 0.501205 |
| }, |
| { |
| "module": "model.layers.7.mlp.up_proj", |
| "rel_err_quant_only": 0.565927, |
| "rel_err_loftq": 0.507679 |
| }, |
| { |
| "module": "model.layers.7.mlp.down_proj", |
| "rel_err_quant_only": 0.571952, |
| "rel_err_loftq": 0.504459 |
| }, |
| { |
| "module": "model.layers.8.self_attn.q_proj", |
| "rel_err_quant_only": 0.56607, |
| "rel_err_loftq": 0.494827 |
| }, |
| { |
| "module": "model.layers.8.self_attn.k_proj", |
| "rel_err_quant_only": 0.566126, |
| "rel_err_loftq": 0.485425 |
| }, |
| { |
| "module": "model.layers.8.self_attn.v_proj", |
| "rel_err_quant_only": 0.566598, |
| "rel_err_loftq": 0.493289 |
| }, |
| { |
| "module": "model.layers.8.self_attn.o_proj", |
| "rel_err_quant_only": 0.565372, |
| "rel_err_loftq": 0.502416 |
| }, |
| { |
| "module": "model.layers.8.mlp.gate_proj", |
| "rel_err_quant_only": 0.5679, |
| "rel_err_loftq": 0.501243 |
| }, |
| { |
| "module": "model.layers.8.mlp.up_proj", |
| "rel_err_quant_only": 0.56637, |
| "rel_err_loftq": 0.507963 |
| }, |
| { |
| "module": "model.layers.8.mlp.down_proj", |
| "rel_err_quant_only": 0.571638, |
| "rel_err_loftq": 0.506416 |
| }, |
| { |
| "module": "model.layers.9.self_attn.q_proj", |
| "rel_err_quant_only": 0.566455, |
| "rel_err_loftq": 0.493246 |
| }, |
| { |
| "module": "model.layers.9.self_attn.k_proj", |
| "rel_err_quant_only": 0.568509, |
| "rel_err_loftq": 0.486252 |
| }, |
| { |
| "module": "model.layers.9.self_attn.v_proj", |
| "rel_err_quant_only": 0.572749, |
| "rel_err_loftq": 0.495191 |
| }, |
| { |
| "module": "model.layers.9.self_attn.o_proj", |
| "rel_err_quant_only": 0.564103, |
| "rel_err_loftq": 0.501349 |
| }, |
| { |
| "module": "model.layers.9.mlp.gate_proj", |
| "rel_err_quant_only": 0.568555, |
| "rel_err_loftq": 0.501315 |
| }, |
| { |
| "module": "model.layers.9.mlp.up_proj", |
| "rel_err_quant_only": 0.566175, |
| "rel_err_loftq": 0.508074 |
| }, |
| { |
| "module": "model.layers.9.mlp.down_proj", |
| "rel_err_quant_only": 0.569898, |
| "rel_err_loftq": 0.506561 |
| }, |
| { |
| "module": "model.layers.10.self_attn.q_proj", |
| "rel_err_quant_only": 0.566355, |
| "rel_err_loftq": 0.49514 |
| }, |
| { |
| "module": "model.layers.10.self_attn.k_proj", |
| "rel_err_quant_only": 0.569432, |
| "rel_err_loftq": 0.485701 |
| }, |
| { |
| "module": "model.layers.10.self_attn.v_proj", |
| "rel_err_quant_only": 0.568946, |
| "rel_err_loftq": 0.495746 |
| }, |
| { |
| "module": "model.layers.10.self_attn.o_proj", |
| "rel_err_quant_only": 0.564751, |
| "rel_err_loftq": 0.5017 |
| }, |
| { |
| "module": "model.layers.10.mlp.gate_proj", |
| "rel_err_quant_only": 0.565964, |
| "rel_err_loftq": 0.50189 |
| }, |
| { |
| "module": "model.layers.10.mlp.up_proj", |
| "rel_err_quant_only": 0.565434, |
| "rel_err_loftq": 0.507811 |
| }, |
| { |
| "module": "model.layers.10.mlp.down_proj", |
| "rel_err_quant_only": 0.568857, |
| "rel_err_loftq": 0.50759 |
| }, |
| { |
| "module": "model.layers.11.self_attn.q_proj", |
| "rel_err_quant_only": 0.566239, |
| "rel_err_loftq": 0.49807 |
| }, |
| { |
| "module": "model.layers.11.self_attn.k_proj", |
| "rel_err_quant_only": 0.570407, |
| "rel_err_loftq": 0.485429 |
| }, |
| { |
| "module": "model.layers.11.self_attn.v_proj", |
| "rel_err_quant_only": 0.573632, |
| "rel_err_loftq": 0.494424 |
| }, |
| { |
| "module": "model.layers.11.self_attn.o_proj", |
| "rel_err_quant_only": 0.564072, |
| "rel_err_loftq": 0.501283 |
| }, |
| { |
| "module": "model.layers.11.mlp.gate_proj", |
| "rel_err_quant_only": 0.564423, |
| "rel_err_loftq": 0.503917 |
| }, |
| { |
| "module": "model.layers.11.mlp.up_proj", |
| "rel_err_quant_only": 0.564232, |
| "rel_err_loftq": 0.508039 |
| }, |
| { |
| "module": "model.layers.11.mlp.down_proj", |
| "rel_err_quant_only": 0.566263, |
| "rel_err_loftq": 0.507737 |
| }, |
| { |
| "module": "model.layers.12.self_attn.q_proj", |
| "rel_err_quant_only": 0.569252, |
| "rel_err_loftq": 0.494601 |
| }, |
| { |
| "module": "model.layers.12.self_attn.k_proj", |
| "rel_err_quant_only": 0.568985, |
| "rel_err_loftq": 0.483184 |
| }, |
| { |
| "module": "model.layers.12.self_attn.v_proj", |
| "rel_err_quant_only": 0.572483, |
| "rel_err_loftq": 0.493378 |
| }, |
| { |
| "module": "model.layers.12.self_attn.o_proj", |
| "rel_err_quant_only": 0.565505, |
| "rel_err_loftq": 0.503031 |
| }, |
| { |
| "module": "model.layers.12.mlp.gate_proj", |
| "rel_err_quant_only": 0.564732, |
| "rel_err_loftq": 0.504802 |
| }, |
| { |
| "module": "model.layers.12.mlp.up_proj", |
| "rel_err_quant_only": 0.564207, |
| "rel_err_loftq": 0.508302 |
| }, |
| { |
| "module": "model.layers.12.mlp.down_proj", |
| "rel_err_quant_only": 0.565013, |
| "rel_err_loftq": 0.508053 |
| }, |
| { |
| "module": "model.layers.13.self_attn.q_proj", |
| "rel_err_quant_only": 0.569869, |
| "rel_err_loftq": 0.496607 |
| }, |
| { |
| "module": "model.layers.13.self_attn.k_proj", |
| "rel_err_quant_only": 0.570524, |
| "rel_err_loftq": 0.48619 |
| }, |
| { |
| "module": "model.layers.13.self_attn.v_proj", |
| "rel_err_quant_only": 0.571265, |
| "rel_err_loftq": 0.492012 |
| }, |
| { |
| "module": "model.layers.13.self_attn.o_proj", |
| "rel_err_quant_only": 0.56362, |
| "rel_err_loftq": 0.503439 |
| }, |
| { |
| "module": "model.layers.13.mlp.gate_proj", |
| "rel_err_quant_only": 0.565708, |
| "rel_err_loftq": 0.504049 |
| }, |
| { |
| "module": "model.layers.13.mlp.up_proj", |
| "rel_err_quant_only": 0.564373, |
| "rel_err_loftq": 0.508295 |
| }, |
| { |
| "module": "model.layers.13.mlp.down_proj", |
| "rel_err_quant_only": 0.565345, |
| "rel_err_loftq": 0.507401 |
| }, |
| { |
| "module": "model.layers.14.self_attn.q_proj", |
| "rel_err_quant_only": 0.57378, |
| "rel_err_loftq": 0.490962 |
| }, |
| { |
| "module": "model.layers.14.self_attn.k_proj", |
| "rel_err_quant_only": 0.571156, |
| "rel_err_loftq": 0.480052 |
| }, |
| { |
| "module": "model.layers.14.self_attn.v_proj", |
| "rel_err_quant_only": 0.585876, |
| "rel_err_loftq": 0.482413 |
| }, |
| { |
| "module": "model.layers.14.self_attn.o_proj", |
| "rel_err_quant_only": 0.562324, |
| "rel_err_loftq": 0.502705 |
| }, |
| { |
| "module": "model.layers.14.mlp.gate_proj", |
| "rel_err_quant_only": 0.566064, |
| "rel_err_loftq": 0.504339 |
| }, |
| { |
| "module": "model.layers.14.mlp.up_proj", |
| "rel_err_quant_only": 0.564155, |
| "rel_err_loftq": 0.50714 |
| }, |
| { |
| "module": "model.layers.14.mlp.down_proj", |
| "rel_err_quant_only": 0.566368, |
| "rel_err_loftq": 0.505934 |
| }, |
| { |
| "module": "model.layers.15.self_attn.q_proj", |
| "rel_err_quant_only": 0.573405, |
| "rel_err_loftq": 0.48807 |
| }, |
| { |
| "module": "model.layers.15.self_attn.k_proj", |
| "rel_err_quant_only": 0.571472, |
| "rel_err_loftq": 0.475571 |
| }, |
| { |
| "module": "model.layers.15.self_attn.v_proj", |
| "rel_err_quant_only": 0.583362, |
| "rel_err_loftq": 0.487769 |
| }, |
| { |
| "module": "model.layers.15.self_attn.o_proj", |
| "rel_err_quant_only": 0.564932, |
| "rel_err_loftq": 0.497493 |
| }, |
| { |
| "module": "model.layers.15.mlp.gate_proj", |
| "rel_err_quant_only": 0.568194, |
| "rel_err_loftq": 0.497835 |
| }, |
| { |
| "module": "model.layers.15.mlp.up_proj", |
| "rel_err_quant_only": 0.56732, |
| "rel_err_loftq": 0.49712 |
| }, |
| { |
| "module": "model.layers.15.mlp.down_proj", |
| "rel_err_quant_only": 0.568418, |
| "rel_err_loftq": 0.505244 |
| } |
| ], |
| "backbone_dir": "/scratch/jeesup0103/safety_quant/models/Llama-3.2-1B-Instruct-loftq2", |
| "init_adapter_dir": "/scratch/jeesup0103/safety_quant/models/Llama-3.2-1B-Instruct-loftq2/loftq_init", |
| "reference": "Li et al., LoftQ, ICLR 2024, arXiv:2310.08659" |
| } |
