Upload folder using huggingface_hub

log_rank0_1767135176.txt

@@ -0,0 +1,71 @@
+[2025-12-30 23:52:56 root] (main.py 611): INFO Namespace(net=None, model='/sc/home/nianhui.guo/models/qwen_vl/models--Qwen--Qwen3-VL-32B-Instruct/snapshots/0cfaf48183f594c314753d30a4c4974bc75f3ccb', cache_dir='./cache', output_dir='./log/qwen3-vl-32b-instruct-w3.5a16g128-sample128-seqlen4096-search-layer-mix-gpu-batch4-hadmad-stage1/', save_dir='./log/qwen3-vl-32b-instruct-w3.5a16g128-sample128-seqlen4096-search-layer-mix-gpu-batch4-hadmad-stage1/', resume=None, real_quant=True, calib_dataset='tulu', nsamples=256, batch_size=4, mini_batch_size=4, seed=2, epochs=3, reverse_epochs=3, limit=-1, deactive_amp=False, clip_grad=1.0, seq_length=4096, gradient_checkpoint=False, blocks=0, reverse_block_qat=False, top_calibration=True, top_calibration_blocks=1, recurrent_calibration_iterations=1, bidirection_qat=True, tasks='', eval_ppl=True, num_fewshot=0, eval_seq_length=2048, wbits=3.5, abits=16, symmetric=False, group_size=128, act_group_size=-1, qat=False, shift=True, optimizer='adamw', wd=0, norm_lr=1e-05, scale_lr=1e-05, one_bit_weight_lr=1e-05, two_bit_weight_lr=2e-05, four_bit_weight_lr=5e-06, shift_lr=5e-06, post_lr=5e-06, channel_scaler_lr=5e-06, prepare_lr=5e-06, fp_lr=1e-06, aug_loss=True, layerwise_loss=False, attention_loss=False, kl_loss=True, lm_head_loss=False, search=True, measurement=True, strategy=False, measure_dir='./measure/qwen3-vl-32b-sample128-gpu/', strategy_dir='./strategy/qwen3-vl-32b-sample128-gpu/', nonuniform_layout=False, residual_ratio=0.1, base_rank_ratio=0.01, exl_v2_pack=False, mix_precision_layout='layer_mix', rotation=False, online_rotation=False, rotate_mode='hadamard', lora_adapter=False, attn_implementation='sdpa', local_rank=-1, master_port=25901)
+[2025-12-30 23:53:31 root] (main.py 642): INFO ====================================================================================================
+[2025-12-30 23:53:31 root] (main.py 643): INFO === START QUANTIZATION ===
+[2025-12-30 23:53:31 root] (main.py 644): INFO ====================================================================================================
+[2025-12-30 23:53:59 root] (omniquant.py 155): INFO ====================================================================================================
+[2025-12-30 23:53:59 root] (omniquant.py 156): INFO STARTING OMNIQUANT WITH MIXED-PRECISION SEARCH FOR QWEN3VL
+[2025-12-30 23:53:59 root] (omniquant.py 157): INFO ====================================================================================================
+[2025-12-30 23:53:59 root] (omniquant.py 168): INFO [CONFIG] Disabled use_cache in text_config (was: True)
+[2025-12-30 23:53:59 root] (omniquant.py 186): INFO ====================================================================================================
+[2025-12-30 23:53:59 root] (omniquant.py 187): INFO [MODEL] Detected Qwen3VL - Quantizing TEXT BRANCH ONLY
+[2025-12-30 23:53:59 root] (omniquant.py 188): INFO [MODEL] Vision encoder will remain in FP16
+[2025-12-30 23:53:59 root] (omniquant.py 189): INFO ====================================================================================================
+[2025-12-30 23:54:00 root] (omniquant.py 209): INFO [MODEL] Language model: 64 layers
+[2025-12-30 23:54:00 root] (omniquant.py 210): INFO [MODEL] Hidden size: 5120
+[2025-12-30 23:54:00 root] (omniquant.py 211): INFO [MODEL] Num attention heads: 64
+[2025-12-30 23:54:00 root] (omniquant.py 212): INFO [MODEL] Attention implementation: sdpa
+[2025-12-30 23:54:00 root] (omniquant.py 218): INFO [DEVICE] Using: cuda
+[2025-12-30 23:54:01 root] (omniquant.py 229): INFO [TRAINING] Using FP16 with AMP
+[2025-12-30 23:54:01 root] (omniquant.py 234): INFO ====================================================================================================
+[2025-12-30 23:54:01 root] (omniquant.py 235): INFO [INPUT CAPTURE] Starting...
+[2025-12-30 23:54:01 root] (omniquant.py 236): INFO ====================================================================================================
+[2025-12-30 23:54:02 root] (omniquant.py 239): INFO [INPUT CAPTURE] Allocated inps: torch.Size([256, 4096, 5120])
+[2025-12-30 23:55:51 root] (omniquant.py 270): INFO [INPUT CAPTURE] Captured 256 samples
+[2025-12-30 23:55:51 root] (omniquant.py 271): INFO [INPUT CAPTURE] attention_mask: torch.Size([1, 1, 4096, 4096])
+[2025-12-30 23:55:51 root] (omniquant.py 272): INFO [INPUT CAPTURE] position_ids: torch.Size([1, 4096])
+[2025-12-30 23:55:57 root] (omniquant.py 287): INFO ====================================================================================================
+[2025-12-30 23:55:57 root] (omniquant.py 288): INFO [CALIBRATION DATA] Preparing...
+[2025-12-30 23:55:57 root] (omniquant.py 289): INFO ====================================================================================================
+[2025-12-30 23:56:01 root] (omniquant.py 295): INFO [CALIBRATION DATA] quant_inps: torch.Size([256, 4096, 5120])
+[2025-12-30 23:56:01 root] (omniquant.py 296): INFO [CALIBRATION DATA] fp_inps: torch.Size([256, 4096, 5120])
+[2025-12-30 23:56:01 root] (omniquant.py 298): INFO [CALIBRATION DATA] fp_inps_2: torch.Size([256, 4096, 5120])
+[2025-12-30 23:56:01 root] (omniquant.py 320): INFO [ATTENTION MASK] Single sample: torch.Size([1, 1, 4096, 4096])
+[2025-12-30 23:56:01 root] (omniquant.py 321): INFO [ATTENTION MASK] Batch (mini_batch_size=4): torch.Size([4, 1, 4096, 4096])
+[2025-12-30 23:56:01 root] (omniquant.py 332): INFO ====================================================================================================
+[2025-12-30 23:56:01 root] (omniquant.py 333): INFO [POSITION IDS] Processing...
+[2025-12-30 23:56:01 root] (omniquant.py 334): INFO ====================================================================================================
+[2025-12-30 23:56:01 root] (omniquant.py 337): INFO [POSITION IDS] Captured: torch.Size([1, 4096])
+[2025-12-30 23:56:01 root] (omniquant.py 346): INFO [POSITION IDS] Base shape (will expand per batch): torch.Size([1, 4096])
+[2025-12-30 23:56:01 root] (omniquant.py 350): INFO [ROTARY EMB] Module type: Qwen3VLTextRotaryEmbedding
+[2025-12-30 23:56:01 root] (omniquant.py 356): INFO [POSITION EMBEDDINGS] cos shape: torch.Size([1, 4096, 128])
+[2025-12-30 23:56:01 root] (omniquant.py 357): INFO [POSITION EMBEDDINGS] sin shape: torch.Size([1, 4096, 128])
+[2025-12-30 23:56:02 root] (omniquant.py 370): INFO ====================================================================================================
+[2025-12-30 23:56:02 root] (omniquant.py 371): INFO [SEARCH] Starting mixed-precision search...
+[2025-12-30 23:56:02 root] (omniquant.py 372): INFO ====================================================================================================
+[2025-12-30 23:56:02 root] (omniquant.py 387): INFO [SEARCH] Loading measurements from file...
+[2025-12-30 23:56:02 root] (omniquant.py 391): INFO [SEARCH] Loaded measurements from ./measure/qwen3-vl-32b-sample128-gpu/
+[2025-12-30 23:56:02 root] (omniquant.py 411): INFO [SEARCH] Running optimization phase...
+[2025-12-30 23:56:08 root] (omniquant.py 422): INFO [SEARCH] Optimization complete
+[2025-12-30 23:56:08 root] (omniquant.py 461): INFO ====================================================================================================
+[2025-12-30 23:56:08 root] (omniquant.py 462): INFO [QUANTIZATION] Starting layer-wise quantization for 64 layers
+[2025-12-30 23:56:08 root] (omniquant.py 463): INFO ====================================================================================================
+[2025-12-30 23:56:08 root] (omniquant.py 473): INFO 
+====================================================================================================
+[2025-12-30 23:56:08 root] (omniquant.py 474): INFO [LAYER 0/63] Starting quantization
+[2025-12-30 23:56:08 root] (omniquant.py 475): INFO ====================================================================================================
+[2025-12-30 23:56:10 root] (omniquant.py 488): INFO [LAYER 0] Applying mixed-precision strategy...
+[2025-12-30 23:56:10 root] (omniquant.py 517): INFO   model.language_model.layers.0.self_attn.q_proj: QParams(32, [4], [1], 4, 32)
+[2025-12-30 23:56:10 root] (omniquant.py 517): INFO   model.language_model.layers.0.self_attn.k_proj: QParams(32, [4], [1], 4, 32)
+[2025-12-30 23:56:10 root] (omniquant.py 517): INFO   model.language_model.layers.0.self_attn.v_proj: QParams(32, [4], [1], 4, 32)
+[2025-12-30 23:56:10 root] (omniquant.py 517): INFO   model.language_model.layers.0.self_attn.o_proj: QParams(32, [4], [1], 4, 32)
+[2025-12-30 23:56:11 root] (omniquant.py 517): INFO   model.language_model.layers.0.mlp.gate_proj: QParams(32, [4], [1], 4, 32)
+[2025-12-30 23:56:11 root] (omniquant.py 517): INFO   model.language_model.layers.0.mlp.up_proj: QParams(32, [4], [1], 4, 32)
+[2025-12-30 23:56:11 root] (omniquant.py 517): INFO   model.language_model.layers.0.mlp.down_proj: QParams(32, [4], [1], 4, 32)
+[2025-12-30 23:56:12 root] (omniquant.py 534): INFO [LAYER 0] BPW: 5.00
+[2025-12-30 23:56:12 root] (omniquant.py 539): INFO [LAYER 0] Computing FP reference outputs...
+[2025-12-30 23:56:12 root] (omniquant.py 559): INFO [LAYER 0] FP reference sample 0:
+[2025-12-30 23:56:12 root] (omniquant.py 560): INFO   fp_inp_batch: torch.Size([1, 4096, 5120])
+[2025-12-30 23:56:12 root] (omniquant.py 561): INFO   batch_position_ids: torch.Size([1, 4096])
+[2025-12-30 23:56:12 root] (omniquant.py 567): INFO   position_embeddings[0] (cos): torch.Size([1, 4096, 128])
+[2025-12-30 23:56:12 root] (omniquant.py 568): INFO   position_embeddings[1] (sin): torch.Size([1, 4096, 128])
+[2025-12-30 23:56:12 root] (omniquant.py 587): INFO   fp_hidden_states: torch.Size([1, 4096, 5120])

model-00002-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0f97072b285aa566e62ce3874551553a21a9c3b33bf1d49422619d523fbb3a03
-size 4993559464
+oid sha256:6b5484e90778f0393ec54c0c5ea0b30c8ce8d240b91c90c2f7b9655645da0013
+size 4989486632

model-00003-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:77a3dd0d92f63898f9e935e6a361c6123ceb2785cea4d20b832e74c9b2f001c0
-size 4992961184
+oid sha256:ab8a3b79c0f9a632967d5e1e3dcb001e603838225789658bdc4e986b8752a330
+size 4989825872

model-00004-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:806c6049fb0d6283f97643cff70f63f2db4126c9bfa48579ca53feb0eb71249f
-size 4947587080
+oid sha256:c0c48c5c661d230ab351e18ea84272492dbcf3dfd8df7c2b2a8936fb0623f340
+size 4984013472

model-00005-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab8f65c27b6ffcdf79abf20dda07d2539439c815c3656fbb7e6b74be7af23866
-size 3272528192
+oid sha256:f01c987898ac5ec7898287607f3457efdaee8098c022d447ad18235baf7ffc7a
+size 3192028080

model.safetensors.index.json

@@ -1,7 +1,7 @@
 {
   "metadata": {
     "total_parameters": 2151768304,
-    "total_size": 23699030976
+    "total_size": 23647749056
   },
   "weight_map": {
     "lm_head.weight": "model-00006-of-00006.safetensors",
@@ -864,12 +864,12 @@
     "model.language_model.layers.28.self_attn.v_proj.qweight": "model-00003-of-00006.safetensors",
     "model.language_model.layers.28.self_attn.v_proj.scales": "model-00003-of-00006.safetensors",
     "model.language_model.layers.28.self_attn.v_proj.zeros": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.29.input_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.29.input_layernorm.weight": "model-00003-of-00006.safetensors",
     "model.language_model.layers.29.mlp.down_proj.channel_scale": "model-00003-of-00006.safetensors",
     "model.language_model.layers.29.mlp.down_proj.q_perm": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.29.mlp.down_proj.qweight": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.29.mlp.down_proj.scales": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.29.mlp.down_proj.zeros": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.29.mlp.down_proj.qweight": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.29.mlp.down_proj.scales": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.29.mlp.down_proj.zeros": "model-00003-of-00006.safetensors",
     "model.language_model.layers.29.mlp.gate_proj.channel_scale": "model-00003-of-00006.safetensors",
     "model.language_model.layers.29.mlp.gate_proj.q_perm": "model-00003-of-00006.safetensors",
     "model.language_model.layers.29.mlp.gate_proj.qweight": "model-00003-of-00006.safetensors",
@@ -880,7 +880,7 @@
     "model.language_model.layers.29.mlp.up_proj.qweight": "model-00003-of-00006.safetensors",
     "model.language_model.layers.29.mlp.up_proj.scales": "model-00003-of-00006.safetensors",
     "model.language_model.layers.29.mlp.up_proj.zeros": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.29.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.29.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
     "model.language_model.layers.29.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
     "model.language_model.layers.29.self_attn.k_proj.channel_scale": "model-00003-of-00006.safetensors",
     "model.language_model.layers.29.self_attn.k_proj.q_perm": "model-00003-of-00006.safetensors",
@@ -960,27 +960,27 @@
     "model.language_model.layers.30.mlp.up_proj.zeros": "model-00004-of-00006.safetensors",
     "model.language_model.layers.30.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
     "model.language_model.layers.30.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.30.self_attn.k_proj.channel_scale": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.30.self_attn.k_proj.q_perm": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.30.self_attn.k_proj.qweight": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.30.self_attn.k_proj.scales": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.30.self_attn.k_proj.zeros": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.30.self_attn.o_proj.channel_scale": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.30.self_attn.o_proj.q_perm": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.30.self_attn.k_proj.channel_scale": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.30.self_attn.k_proj.q_perm": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.30.self_attn.k_proj.qweight": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.30.self_attn.k_proj.scales": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.30.self_attn.k_proj.zeros": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.30.self_attn.o_proj.channel_scale": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.30.self_attn.o_proj.q_perm": "model-00003-of-00006.safetensors",
     "model.language_model.layers.30.self_attn.o_proj.qweight": "model-00004-of-00006.safetensors",
     "model.language_model.layers.30.self_attn.o_proj.scales": "model-00004-of-00006.safetensors",
     "model.language_model.layers.30.self_attn.o_proj.zeros": "model-00004-of-00006.safetensors",
     "model.language_model.layers.30.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.30.self_attn.q_proj.channel_scale": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.30.self_attn.q_proj.q_perm": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.30.self_attn.q_proj.qweight": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.30.self_attn.q_proj.scales": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.30.self_attn.q_proj.zeros": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.30.self_attn.v_proj.channel_scale": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.30.self_attn.v_proj.q_perm": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.30.self_attn.v_proj.qweight": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.30.self_attn.v_proj.scales": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.30.self_attn.v_proj.zeros": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.30.self_attn.q_proj.channel_scale": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.30.self_attn.q_proj.q_perm": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.30.self_attn.q_proj.qweight": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.30.self_attn.q_proj.scales": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.30.self_attn.q_proj.zeros": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.30.self_attn.v_proj.channel_scale": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.30.self_attn.v_proj.q_perm": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.30.self_attn.v_proj.qweight": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.30.self_attn.v_proj.scales": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.30.self_attn.v_proj.zeros": "model-00003-of-00006.safetensors",
     "model.language_model.layers.31.input_layernorm.weight": "model-00004-of-00006.safetensors",
     "model.language_model.layers.31.mlp.down_proj.channel_scale": "model-00004-of-00006.safetensors",
     "model.language_model.layers.31.mlp.down_proj.q_perm": "model-00004-of-00006.safetensors",
@@ -1722,12 +1722,12 @@
     "model.language_model.layers.48.self_attn.v_proj.qweight": "model-00004-of-00006.safetensors",
     "model.language_model.layers.48.self_attn.v_proj.scales": "model-00004-of-00006.safetensors",
     "model.language_model.layers.48.self_attn.v_proj.zeros": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.49.input_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.language_model.layers.49.input_layernorm.weight": "model-00004-of-00006.safetensors",
     "model.language_model.layers.49.mlp.down_proj.channel_scale": "model-00004-of-00006.safetensors",
     "model.language_model.layers.49.mlp.down_proj.q_perm": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.49.mlp.down_proj.qweight": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.49.mlp.down_proj.scales": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.49.mlp.down_proj.zeros": "model-00005-of-00006.safetensors",
+    "model.language_model.layers.49.mlp.down_proj.qweight": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.49.mlp.down_proj.scales": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.49.mlp.down_proj.zeros": "model-00004-of-00006.safetensors",
     "model.language_model.layers.49.mlp.gate_proj.channel_scale": "model-00004-of-00006.safetensors",
     "model.language_model.layers.49.mlp.gate_proj.q_perm": "model-00004-of-00006.safetensors",
     "model.language_model.layers.49.mlp.gate_proj.qweight": "model-00004-of-00006.safetensors",
@@ -1738,7 +1738,7 @@
     "model.language_model.layers.49.mlp.up_proj.qweight": "model-00004-of-00006.safetensors",
     "model.language_model.layers.49.mlp.up_proj.scales": "model-00004-of-00006.safetensors",
     "model.language_model.layers.49.mlp.up_proj.zeros": "model-00004-of-00006.safetensors",
-    "model.language_model.layers.49.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
+    "model.language_model.layers.49.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
     "model.language_model.layers.49.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
     "model.language_model.layers.49.self_attn.k_proj.channel_scale": "model-00004-of-00006.safetensors",
     "model.language_model.layers.49.self_attn.k_proj.q_perm": "model-00004-of-00006.safetensors",
@@ -1801,44 +1801,44 @@
     "model.language_model.layers.5.self_attn.v_proj.scales": "model-00002-of-00006.safetensors",
     "model.language_model.layers.5.self_attn.v_proj.zeros": "model-00002-of-00006.safetensors",
     "model.language_model.layers.50.input_layernorm.weight": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.mlp.down_proj.channel_scale": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.mlp.down_proj.q_perm": "model-00005-of-00006.safetensors",
+    "model.language_model.layers.50.mlp.down_proj.channel_scale": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.mlp.down_proj.q_perm": "model-00004-of-00006.safetensors",
     "model.language_model.layers.50.mlp.down_proj.qweight": "model-00005-of-00006.safetensors",
     "model.language_model.layers.50.mlp.down_proj.scales": "model-00005-of-00006.safetensors",
     "model.language_model.layers.50.mlp.down_proj.zeros": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.mlp.gate_proj.channel_scale": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.mlp.gate_proj.q_perm": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.mlp.gate_proj.qweight": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.mlp.gate_proj.scales": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.mlp.gate_proj.zeros": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.mlp.up_proj.channel_scale": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.mlp.up_proj.q_perm": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.mlp.up_proj.qweight": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.mlp.up_proj.scales": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.mlp.up_proj.zeros": "model-00005-of-00006.safetensors",
+    "model.language_model.layers.50.mlp.gate_proj.channel_scale": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.mlp.gate_proj.q_perm": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.mlp.gate_proj.qweight": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.mlp.gate_proj.scales": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.mlp.gate_proj.zeros": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.mlp.up_proj.channel_scale": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.mlp.up_proj.q_perm": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.mlp.up_proj.qweight": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.mlp.up_proj.scales": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.mlp.up_proj.zeros": "model-00004-of-00006.safetensors",
     "model.language_model.layers.50.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.k_proj.channel_scale": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.k_proj.q_perm": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.k_proj.qweight": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.k_proj.scales": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.k_proj.zeros": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.o_proj.channel_scale": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.o_proj.q_perm": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.o_proj.qweight": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.o_proj.scales": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.o_proj.zeros": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.q_proj.channel_scale": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.q_proj.q_perm": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.q_proj.qweight": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.q_proj.scales": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.q_proj.zeros": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.v_proj.channel_scale": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.v_proj.q_perm": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.v_proj.qweight": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.v_proj.scales": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.50.self_attn.v_proj.zeros": "model-00005-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.k_proj.channel_scale": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.k_proj.q_perm": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.k_proj.qweight": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.k_proj.scales": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.k_proj.zeros": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.o_proj.channel_scale": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.o_proj.q_perm": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.o_proj.qweight": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.o_proj.scales": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.o_proj.zeros": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.q_proj.channel_scale": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.q_proj.q_perm": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.q_proj.qweight": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.q_proj.scales": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.q_proj.zeros": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.v_proj.channel_scale": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.v_proj.q_perm": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.v_proj.qweight": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.v_proj.scales": "model-00004-of-00006.safetensors",
+    "model.language_model.layers.50.self_attn.v_proj.zeros": "model-00004-of-00006.safetensors",
     "model.language_model.layers.51.input_layernorm.weight": "model-00005-of-00006.safetensors",
     "model.language_model.layers.51.mlp.down_proj.channel_scale": "model-00005-of-00006.safetensors",
     "model.language_model.layers.51.mlp.down_proj.q_perm": "model-00005-of-00006.safetensors",
@@ -2190,12 +2190,12 @@
     "model.language_model.layers.59.self_attn.v_proj.qweight": "model-00005-of-00006.safetensors",
     "model.language_model.layers.59.self_attn.v_proj.scales": "model-00005-of-00006.safetensors",
     "model.language_model.layers.59.self_attn.v_proj.zeros": "model-00005-of-00006.safetensors",
-    "model.language_model.layers.6.input_layernorm.weight": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.6.mlp.down_proj.channel_scale": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.6.mlp.down_proj.q_perm": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.6.mlp.down_proj.qweight": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.6.mlp.down_proj.scales": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.6.mlp.down_proj.zeros": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.6.input_layernorm.weight": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.6.mlp.down_proj.channel_scale": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.6.mlp.down_proj.q_perm": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.6.mlp.down_proj.qweight": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.6.mlp.down_proj.scales": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.6.mlp.down_proj.zeros": "model-00002-of-00006.safetensors",
     "model.language_model.layers.6.mlp.gate_proj.channel_scale": "model-00002-of-00006.safetensors",
     "model.language_model.layers.6.mlp.gate_proj.q_perm": "model-00002-of-00006.safetensors",
     "model.language_model.layers.6.mlp.gate_proj.qweight": "model-00002-of-00006.safetensors",
@@ -2204,9 +2204,9 @@
     "model.language_model.layers.6.mlp.up_proj.channel_scale": "model-00002-of-00006.safetensors",
     "model.language_model.layers.6.mlp.up_proj.q_perm": "model-00002-of-00006.safetensors",
     "model.language_model.layers.6.mlp.up_proj.qweight": "model-00002-of-00006.safetensors",
-    "model.language_model.layers.6.mlp.up_proj.scales": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.6.mlp.up_proj.zeros": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.6.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.6.mlp.up_proj.scales": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.6.mlp.up_proj.zeros": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
     "model.language_model.layers.6.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
     "model.language_model.layers.6.self_attn.k_proj.channel_scale": "model-00002-of-00006.safetensors",
     "model.language_model.layers.6.self_attn.k_proj.q_perm": "model-00002-of-00006.safetensors",
@@ -2391,8 +2391,8 @@
     "model.language_model.layers.7.mlp.down_proj.qweight": "model-00003-of-00006.safetensors",
     "model.language_model.layers.7.mlp.down_proj.scales": "model-00003-of-00006.safetensors",
     "model.language_model.layers.7.mlp.down_proj.zeros": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.mlp.gate_proj.channel_scale": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.mlp.gate_proj.q_perm": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.7.mlp.gate_proj.channel_scale": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.mlp.gate_proj.q_perm": "model-00002-of-00006.safetensors",
     "model.language_model.layers.7.mlp.gate_proj.qweight": "model-00003-of-00006.safetensors",
     "model.language_model.layers.7.mlp.gate_proj.scales": "model-00003-of-00006.safetensors",
     "model.language_model.layers.7.mlp.gate_proj.zeros": "model-00003-of-00006.safetensors",
@@ -2402,28 +2402,28 @@
     "model.language_model.layers.7.mlp.up_proj.scales": "model-00003-of-00006.safetensors",
     "model.language_model.layers.7.mlp.up_proj.zeros": "model-00003-of-00006.safetensors",
     "model.language_model.layers.7.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.k_proj.channel_scale": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.k_proj.q_perm": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.k_proj.qweight": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.k_proj.scales": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.k_proj.zeros": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.o_proj.channel_scale": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.o_proj.q_perm": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.o_proj.qweight": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.o_proj.scales": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.o_proj.zeros": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.q_proj.channel_scale": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.q_proj.q_perm": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.q_proj.qweight": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.q_proj.scales": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.q_proj.zeros": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.v_proj.channel_scale": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.v_proj.q_perm": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.v_proj.qweight": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.v_proj.scales": "model-00003-of-00006.safetensors",
-    "model.language_model.layers.7.self_attn.v_proj.zeros": "model-00003-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.k_proj.channel_scale": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.k_proj.q_perm": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.k_proj.qweight": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.k_proj.scales": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.k_proj.zeros": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.o_proj.channel_scale": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.o_proj.q_perm": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.o_proj.qweight": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.o_proj.scales": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.o_proj.zeros": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.q_proj.channel_scale": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.q_proj.q_perm": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.q_proj.qweight": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.q_proj.scales": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.q_proj.zeros": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.v_proj.channel_scale": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.v_proj.q_perm": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.v_proj.qweight": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.v_proj.scales": "model-00002-of-00006.safetensors",
+    "model.language_model.layers.7.self_attn.v_proj.zeros": "model-00002-of-00006.safetensors",
     "model.language_model.layers.8.input_layernorm.weight": "model-00003-of-00006.safetensors",
     "model.language_model.layers.8.mlp.down_proj.channel_scale": "model-00003-of-00006.safetensors",
     "model.language_model.layers.8.mlp.down_proj.q_perm": "model-00003-of-00006.safetensors",

quant_strategy.json

@@ -1,11 +1,11 @@
 {
     "measurement": {
         "model.layers.0": {
-            "accuracy": 0.9828830574697349,
-            "total_bits": 2102722560.0,
+            "accuracy": 0.9774280267301947,
+            "total_bits": 1988444160.0,
             "o_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -18,7 +18,7 @@
             },
             "down_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -31,7 +31,7 @@
             },
             "q_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -44,7 +44,7 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -57,7 +57,7 @@
             },
             "v_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -70,7 +70,7 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -83,7 +83,7 @@
             },
             "up_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -96,11 +96,11 @@
             }
         },
         "model.layers.1": {
-            "accuracy": 0.9855277572060004,
-            "total_bits": 2102722560.0,
+            "accuracy": 0.979808229021728,
+            "total_bits": 1988444160.0,
             "o_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -113,7 +113,7 @@
             },
             "down_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -126,7 +126,7 @@
             },
             "q_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -139,7 +139,7 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -152,7 +152,7 @@
             },
             "v_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -165,7 +165,7 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -178,7 +178,7 @@
             },
             "up_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -286,8 +286,8 @@
             }
         },
         "model.layers.3": {
-            "accuracy": 0.9826611040625721,
-            "total_bits": 1988444160.0,
+            "accuracy": 0.9813712932809722,
+            "total_bits": 1907834880.0,
             "o_proj": {
                 "group_size": {
                     "4": 128
@@ -316,10 +316,10 @@
             },
             "q_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -476,11 +476,11 @@
             }
         },
         "model.layers.5": {
-            "accuracy": 0.9851978906372096,
-            "total_bits": 2102722560.0,
+            "accuracy": 0.9806460069958121,
+            "total_bits": 1988444160.0,
             "o_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -493,7 +493,7 @@
             },
             "down_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -506,7 +506,7 @@
             },
             "q_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -519,7 +519,7 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -532,7 +532,7 @@
             },
             "v_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -545,7 +545,7 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -558,7 +558,7 @@
             },
             "up_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -856,14 +856,14 @@
             }
         },
         "model.layers.9": {
-            "accuracy": 0.9856001285370439,
-            "total_bits": 1368227840.0,
+            "accuracy": 0.9862758388189832,
+            "total_bits": 1442283520.0,
             "o_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -951,8 +951,8 @@
             }
         },
         "model.layers.10": {
-            "accuracy": 0.9813145939260721,
-            "total_bits": 1210859520.0,
+            "accuracy": 0.9866171181201935,
+            "total_bits": 1442283520.0,
             "o_proj": {
                 "group_size": {
                     "4": 128
@@ -968,10 +968,10 @@
             },
             "down_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1046,14 +1046,14 @@
             }
         },
         "model.layers.11": {
-            "accuracy": 0.9843400969984941,
-            "total_bits": 1368227840.0,
+            "accuracy": 0.9859971911937464,
+            "total_bits": 1451540480.0,
             "o_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1089,10 +1089,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1141,14 +1141,14 @@
             }
         },
         "model.layers.12": {
-            "accuracy": 0.9840793606999796,
-            "total_bits": 1368227840.0,
+            "accuracy": 0.985051059658872,
+            "total_bits": 1442283520.0,
             "o_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1236,14 +1236,14 @@
             }
         },
         "model.layers.13": {
-            "accuracy": 0.9839283300098032,
-            "total_bits": 1368227840.0,
+            "accuracy": 0.9854843073990196,
+            "total_bits": 1451540480.0,
             "o_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1279,10 +1279,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1331,14 +1331,14 @@
             }
         },
         "model.layers.14": {
-            "accuracy": 0.9848056569171604,
-            "total_bits": 1368227840.0,
+            "accuracy": 0.9862457206763793,
+            "total_bits": 1451540480.0,
             "o_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1374,10 +1374,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1426,14 +1426,14 @@
             }
         },
         "model.layers.15": {
-            "accuracy": 0.9847434815601446,
-            "total_bits": 1368227840.0,
+            "accuracy": 0.9865853489609435,
+            "total_bits": 1451540480.0,
             "o_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1469,10 +1469,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1521,8 +1521,8 @@
             }
         },
         "model.layers.16": {
-            "accuracy": 0.9818843859247863,
-            "total_bits": 1210859520.0,
+            "accuracy": 0.987276211992139,
+            "total_bits": 1451540480.0,
             "o_proj": {
                 "group_size": {
                     "4": 128
@@ -1538,10 +1538,10 @@
             },
             "down_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1564,10 +1564,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1616,8 +1616,8 @@
             }
         },
         "model.layers.17": {
-            "accuracy": 0.9818928238819353,
-            "total_bits": 1210859520.0,
+            "accuracy": 0.987157837691484,
+            "total_bits": 1451540480.0,
             "o_proj": {
                 "group_size": {
                     "4": 128
@@ -1633,10 +1633,10 @@
             },
             "down_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1659,10 +1659,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1711,14 +1711,14 @@
             }
         },
         "model.layers.18": {
-            "accuracy": 0.9846934152592439,
-            "total_bits": 1368227840.0,
+            "accuracy": 0.98646844382165,
+            "total_bits": 1451540480.0,
             "o_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1754,10 +1754,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1901,14 +1901,14 @@
             }
         },
         "model.layers.20": {
-            "accuracy": 0.9828240241040476,
-            "total_bits": 1368227840.0,
+            "accuracy": 0.9845815218286589,
+            "total_bits": 1451540480.0,
             "o_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -1944,10 +1944,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -2091,8 +2091,8 @@
             }
         },
         "model.layers.22": {
-            "accuracy": 0.9825039394199848,
-            "total_bits": 1525596160.0,
+            "accuracy": 0.9820500311907381,
+            "total_bits": 1516339200.0,
             "o_proj": {
                 "group_size": {
                     "4": 128
@@ -2134,10 +2134,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -2186,8 +2186,8 @@
             }
         },
         "model.layers.23": {
-            "accuracy": 0.9812842943647411,
-            "total_bits": 1525596160.0,
+            "accuracy": 0.9808617235685233,
+            "total_bits": 1451540480.0,
             "o_proj": {
                 "group_size": {
                     "4": 128
@@ -2216,10 +2216,10 @@
             },
             "q_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -2281,8 +2281,8 @@
             }
         },
         "model.layers.24": {
-            "accuracy": 0.9836647573392838,
-            "total_bits": 1673707520.0,
+            "accuracy": 0.9812766579561867,
+            "total_bits": 1653227520.0,
             "o_proj": {
                 "group_size": {
                     "4": 128
@@ -2350,10 +2350,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -2363,10 +2363,10 @@
             },
             "up_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -2566,8 +2566,8 @@
             }
         },
         "model.layers.27": {
-            "accuracy": 0.9891230833891314,
-            "total_bits": 1907834880.0,
+            "accuracy": 0.9812095816305373,
+            "total_bits": 1673707520.0,
             "o_proj": {
                 "group_size": {
                     "4": 128
@@ -2596,7 +2596,7 @@
             },
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -2609,10 +2609,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -2635,10 +2635,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -2661,8 +2661,8 @@
             }
         },
         "model.layers.28": {
-            "accuracy": 0.9891000912321033,
-            "total_bits": 1907834880.0,
+            "accuracy": 0.9807323368440848,
+            "total_bits": 1673707520.0,
             "o_proj": {
                 "group_size": {
                     "4": 128
@@ -2691,7 +2691,7 @@
             },
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -2704,10 +2704,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -2730,10 +2730,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -2756,8 +2756,8 @@
             }
         },
         "model.layers.29": {
-            "accuracy": 0.9894378067547223,
-            "total_bits": 1907834880.0,
+            "accuracy": 0.980840009462554,
+            "total_bits": 1673707520.0,
             "o_proj": {
                 "group_size": {
                     "4": 128
@@ -2786,7 +2786,7 @@
             },
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -2799,10 +2799,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -2825,10 +2825,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -3421,8 +3421,8 @@
             }
         },
         "model.layers.36": {
-            "accuracy": 0.9883409765607212,
-            "total_bits": 1907834880.0,
+            "accuracy": 0.9810442902962677,
+            "total_bits": 1673707520.0,
             "o_proj": {
                 "group_size": {
                     "4": 128
@@ -3451,7 +3451,7 @@
             },
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -3464,10 +3464,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -3490,10 +3490,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -3516,8 +3516,8 @@
             }
         },
         "model.layers.37": {
-            "accuracy": 0.9876541621342767,
-            "total_bits": 1907834880.0,
+            "accuracy": 0.9805779054295272,
+            "total_bits": 1673707520.0,
             "o_proj": {
                 "group_size": {
                     "4": 128
@@ -3546,7 +3546,7 @@
             },
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -3559,10 +3559,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -3585,10 +3585,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -3611,8 +3611,8 @@
             }
         },
         "model.layers.38": {
-            "accuracy": 0.9870018090005033,
-            "total_bits": 1907834880.0,
+            "accuracy": 0.9795972040155903,
+            "total_bits": 1673707520.0,
             "o_proj": {
                 "group_size": {
                     "4": 128
@@ -3641,7 +3641,7 @@
             },
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -3654,10 +3654,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -3680,10 +3680,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -3706,8 +3706,8 @@
             }
         },
         "model.layers.39": {
-            "accuracy": 0.986489765055012,
-            "total_bits": 1907834880.0,
+            "accuracy": 0.979535614955239,
+            "total_bits": 1673707520.0,
             "o_proj": {
                 "group_size": {
                     "4": 128
@@ -3736,7 +3736,7 @@
             },
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -3749,10 +3749,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -3775,10 +3775,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -4181,8 +4181,8 @@
             }
         },
         "model.layers.44": {
-            "accuracy": 0.9865467178460676,
-            "total_bits": 1907834880.0,
+            "accuracy": 0.9810187924886122,
+            "total_bits": 1673707520.0,
             "o_proj": {
                 "group_size": {
                     "4": 128
@@ -4211,7 +4211,7 @@
             },
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -4224,10 +4224,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -4250,10 +4250,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -4276,8 +4276,8 @@
             }
         },
         "model.layers.45": {
-            "accuracy": 0.9846922820433974,
-            "total_bits": 1907834880.0,
+            "accuracy": 0.979574806347955,
+            "total_bits": 1673707520.0,
             "o_proj": {
                 "group_size": {
                     "4": 128
@@ -4306,7 +4306,7 @@
             },
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -4319,10 +4319,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -4345,10 +4345,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 32
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -4371,8 +4371,8 @@
             }
         },
         "model.layers.46": {
-            "accuracy": 0.9846796012716368,
-            "total_bits": 1907834880.0,
+            "accuracy": 0.9794538663118146,
+            "total_bits": 1726464000.0,
             "o_proj": {
                 "group_size": {
                     "4": 128
@@ -4401,10 +4401,10 @@
             },
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -4414,10 +4414,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -4440,10 +4440,10 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "4": 128
+                    "2": 64
                 },
                 "bits": [
-                    4
+                    2
                 ],
                 "bits_prop": [
                     1
@@ -5036,11 +5036,11 @@
             }
         },
         "model.layers.53": {
-            "accuracy": 0.985947548673721,
-            "total_bits": 2102722560.0,
+            "accuracy": 0.9816716767963953,
+            "total_bits": 1988444160.0,
             "o_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -5053,7 +5053,7 @@
             },
             "down_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -5066,7 +5066,7 @@
             },
             "q_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -5079,7 +5079,7 @@
             },
             "k_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -5092,7 +5092,7 @@
             },
             "v_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -5105,7 +5105,7 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -5118,7 +5118,7 @@
             },
             "up_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -5606,14 +5606,14 @@
             }
         },
         "model.layers.59": {
-            "accuracy": 0.989684437867254,
-            "total_bits": 1061437440.0,
+            "accuracy": 0.9957845042226836,
+            "total_bits": 1442283520.0,
             "o_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -5623,10 +5623,10 @@
             },
             "down_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -5636,7 +5636,7 @@
             },
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -5649,7 +5649,7 @@
             },
             "k_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -5675,7 +5675,7 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -5688,7 +5688,7 @@
             },
             "up_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -5701,14 +5701,14 @@
             }
         },
         "model.layers.60": {
-            "accuracy": 0.9913747301325202,
-            "total_bits": 1061437440.0,
+            "accuracy": 0.9971412243321538,
+            "total_bits": 1451540480.0,
             "o_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -5718,10 +5718,10 @@
             },
             "down_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -5731,7 +5731,7 @@
             },
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -5744,10 +5744,10 @@
             },
             "k_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -5770,7 +5770,7 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -5783,7 +5783,7 @@
             },
             "up_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -5796,14 +5796,14 @@
             }
         },
         "model.layers.61": {
-            "accuracy": 0.9891216587275267,
-            "total_bits": 1061437440.0,
+            "accuracy": 0.9959207735955715,
+            "total_bits": 1442283520.0,
             "o_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -5813,10 +5813,10 @@
             },
             "down_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -5826,7 +5826,7 @@
             },
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -5839,7 +5839,7 @@
             },
             "k_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -5865,7 +5865,7 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -5878,7 +5878,7 @@
             },
             "up_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -5891,14 +5891,14 @@
             }
         },
         "model.layers.62": {
-            "accuracy": 0.9859952349215746,
-            "total_bits": 1061437440.0,
+            "accuracy": 0.9954170882701874,
+            "total_bits": 1442283520.0,
             "o_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -5908,10 +5908,10 @@
             },
             "down_proj": {
                 "group_size": {
-                    "2": 64
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -5921,7 +5921,7 @@
             },
             "q_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -5934,7 +5934,7 @@
             },
             "k_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -5960,7 +5960,7 @@
             },
             "gate_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -5973,7 +5973,7 @@
             },
             "up_proj": {
                 "group_size": {
-                    "2": 64
+                    "2": 32
                 },
                 "bits": [
                     2
@@ -5986,14 +5986,14 @@
             }
         },
         "model.layers.63": {
-            "accuracy": 0.987260795198381,
-            "total_bits": 1389690880.0,
+            "accuracy": 0.9917106227949262,
+            "total_bits": 1442283520.0,
             "o_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1
@@ -6003,7 +6003,7 @@
             },
             "down_proj": {
                 "group_size": {
-                    "4": 32
+                    "4": 128
                 },
                 "bits": [
                     4
@@ -6042,10 +6042,10 @@
             },
             "v_proj": {
                 "group_size": {
-                    "2": 32
+                    "4": 128
                 },
                 "bits": [
-                    2
+                    4
                 ],
                 "bits_prop": [
                     1