Hugging Face's logo

GreenBitAI
/
Mistral-7B-Instruct-v0.2-layer-mix-bpw-2.5

Text Generation
Transformers
Safetensors
mistral
conversational
text-generation-inference
Model card Files
xet
Community
Mistral-7B-Instruct-v0.2-layer-mix-bpw-2.5 / quant_strategy.json
NicoNico6
update
8ea26c9
raw
history blame contribute delete
68.9 kB
 {
"measurement": {
"model.layers.0": {
"accuracy": 0.7869586944580078,
"total_bits": 466380288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.1": {
"accuracy": 0.8954000473022461,
"total_bits": 573724416,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.2": {
"accuracy": 0.9489546418190002,
"total_bits": 458124288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.3": {
"accuracy": 0.9539257287979126,
"total_bits": 458124288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.4": {
"accuracy": 0.948026180267334,
"total_bits": 458124288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.5": {
"accuracy": 0.938827395439148,
"total_bits": 458124288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.6": {
"accuracy": 0.9337625503540039,
"total_bits": 458124288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.7": {
"accuracy": 0.9263930320739746,
"total_bits": 466380288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.8": {
"accuracy": 0.9275798797607422,
"total_bits": 466380288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.9": {
"accuracy": 0.9196293354034424,
"total_bits": 458124288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.10": {
"accuracy": 0.9175989627838135,
"total_bits": 466380288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.11": {
"accuracy": 0.9152019023895264,
"total_bits": 466380288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.12": {
"accuracy": 0.9095911979675293,
"total_bits": 466380288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.13": {
"accuracy": 0.9048597812652588,
"total_bits": 458124288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.14": {
"accuracy": 0.8939895629882812,
"total_bits": 458124288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.15": {
"accuracy": 0.88232421875,
"total_bits": 458124288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.16": {
"accuracy": 0.8802495002746582,
"total_bits": 458124288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.17": {
"accuracy": 0.8810954093933105,
"total_bits": 466380288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.18": {
"accuracy": 0.8681793212890625,
"total_bits": 466380288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.19": {
"accuracy": 0.8697834014892578,
"total_bits": 466380288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.20": {
"accuracy": 0.8790159225463867,
"total_bits": 466380288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.21": {
"accuracy": 0.8862137794494629,
"total_bits": 466380288,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.22": {
"accuracy": 0.9187626838684082,
"total_bits": 573724416,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.23": {
"accuracy": 0.9179215431213379,
"total_bits": 573724416,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.24": {
"accuracy": 0.9173269271850586,
"total_bits": 573724416,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.25": {
"accuracy": 0.9187664985656738,
"total_bits": 573724416,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.26": {
"accuracy": 0.9419045448303223,
"total_bits": 722356992,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.27": {
"accuracy": 0.9398770332336426,
"total_bits": 722356992,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.28": {
"accuracy": 0.9681390523910522,
"total_bits": 846216960,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.29": {
"accuracy": 0.9653351306915283,
"total_bits": 846216960,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.30": {
"accuracy": 0.9626724720001221,
"total_bits": 846216960,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.31": {
"accuracy": 0.9576601982116699,
"total_bits": 879245568,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
}
}