Hugging Face's logo

GreenBitAI
/
01-Yi-9B-layer-mix-bpw-2.5

Text Generation
Transformers
Safetensors
llama
text-generation-inference
Model card Files
xet
Community
01-Yi-9B-layer-mix-bpw-2.5 / quant_strategy.json
NicoNico6
update
24a6a2c
raw
history blame contribute delete
103 kB
 {
"measurement": {
"model.layers.0": {
"accuracy": 0.9235353469848633,
"total_bits": 365124672,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.1": {
"accuracy": 0.7686681747436523,
"total_bits": 360997440,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.2": {
"accuracy": 0.8663358688354492,
"total_bits": 360997440,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.3": {
"accuracy": 0.8417501449584961,
"total_bits": 365124672,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.4": {
"accuracy": 0.8675830364227295,
"total_bits": 360997440,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.5": {
"accuracy": 0.8590667247772217,
"total_bits": 360997440,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.6": {
"accuracy": 0.8517341613769531,
"total_bits": 360997440,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.7": {
"accuracy": 0.8489260673522949,
"total_bits": 360997440,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.8": {
"accuracy": 0.9279017448425293,
"total_bits": 449761824,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.9": {
"accuracy": 0.9575139284133911,
"total_bits": 360997440,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.10": {
"accuracy": 0.95622718334198,
"total_bits": 360997440,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.11": {
"accuracy": 0.949416995048523,
"total_bits": 365124672,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.12": {
"accuracy": 0.9479968547821045,
"total_bits": 365124672,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.13": {
"accuracy": 0.9440830945968628,
"total_bits": 360997440,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.14": {
"accuracy": 0.9452991485595703,
"total_bits": 365124672,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.15": {
"accuracy": 0.9413189888000488,
"total_bits": 360997440,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.16": {
"accuracy": 0.944807767868042,
"total_bits": 365124672,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.17": {
"accuracy": 0.9402191638946533,
"total_bits": 365124672,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.18": {
"accuracy": 0.9394395351409912,
"total_bits": 360997440,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.19": {
"accuracy": 0.9472098350524902,
"total_bits": 365124672,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.20": {
"accuracy": 0.9527584314346313,
"total_bits": 365124672,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.21": {
"accuracy": 0.9584027528762817,
"total_bits": 365124672,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.22": {
"accuracy": 0.9615957736968994,
"total_bits": 365124672,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.23": {
"accuracy": 0.9349331855773926,
"total_bits": 365124672,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.24": {
"accuracy": 0.963486909866333,
"total_bits": 365124672,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.25": {
"accuracy": 0.9610995054244995,
"total_bits": 360997440,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.26": {
"accuracy": 0.9591997861862183,
"total_bits": 365124672,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.27": {
"accuracy": 0.9536696672439575,
"total_bits": 365124672,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.28": {
"accuracy": 0.9514199495315552,
"total_bits": 365124672,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.29": {
"accuracy": 0.944045901298523,
"total_bits": 360997440,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.30": {
"accuracy": 0.9456077814102173,
"total_bits": 360997440,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.31": {
"accuracy": 0.9442343711853027,
"total_bits": 360997440,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.32": {
"accuracy": 0.9421529769897461,
"total_bits": 365124672,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.33": {
"accuracy": 0.9256293773651123,
"total_bits": 360997440,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.34": {
"accuracy": 0.9283227920532227,
"total_bits": 365124672,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.35": {
"accuracy": 0.9434576034545898,
"total_bits": 449761824,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.36": {
"accuracy": 0.9467594623565674,
"total_bits": 486917664,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.37": {
"accuracy": 0.9436817169189453,
"total_bits": 486917664,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.38": {
"accuracy": 0.9477231502532959,
"total_bits": 571557408,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.39": {
"accuracy": 0.9525341987609863,
"total_bits": 604586016,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.40": {
"accuracy": 0.9732036590576172,
"total_bits": 664451616,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.41": {
"accuracy": 0.9750280380249023,
"total_bits": 664451616,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.42": {
"accuracy": 0.9755113124847412,
"total_bits": 664451616,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.43": {
"accuracy": 0.9758803844451904,
"total_bits": 664451616,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.44": {
"accuracy": 0.9759896993637085,
"total_bits": 664451616,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.45": {
"accuracy": 0.9787205457687378,
"total_bits": 664451616,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.46": {
"accuracy": 0.9827241897583008,
"total_bits": 697480224,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.47": {
"accuracy": 0.9262123107910156,
"total_bits": 664451616,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
}
}