| { |
| "summary": { |
| "group_size": 64, |
| "tensors_ternary": 0, |
| "tensors_int8": 99, |
| "tensors_total": 156, |
| "tensors_quantized": 99, |
| "float_mb": 100.72, |
| "result_mb": 26.79, |
| "ratio": 3.76, |
| "worst_relative_error": 0.0063, |
| "mean_relative_error": 0.006, |
| "mean_zero_fraction": 0.0084, |
| "method": "int8", |
| "quality": { |
| "tokens_compared": 95, |
| "top1_agreement": 0.9684, |
| "top5_agreement": 1.0, |
| "mean_kl_divergence": 0.0017, |
| "reference_perplexity": 49.146, |
| "candidate_perplexity": 49.537, |
| "perplexity_ratio": 1.008, |
| "greedy_identical_prefix": [ |
| 12, |
| 24, |
| 24, |
| 24, |
| 24 |
| ], |
| "greedy_identical_answers": "7/8" |
| } |
| }, |
| "tensors": [ |
| { |
| "name": "transformer.h.0.attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.0.attn.k_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00583, |
| "zero_fraction": 0.00865 |
| }, |
| { |
| "name": "transformer.h.0.attn.o_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00602, |
| "zero_fraction": 0.00944 |
| }, |
| { |
| "name": "transformer.h.0.attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.0.attn.q_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00591, |
| "zero_fraction": 0.00816 |
| }, |
| { |
| "name": "transformer.h.0.attn.v_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00605, |
| "zero_fraction": 0.00846 |
| }, |
| { |
| "name": "transformer.h.0.ln_1.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.0.ln_2.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.0.mlp.w_down.weight", |
| "shape": [ |
| 384, |
| 1024 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00603, |
| "zero_fraction": 0.00841 |
| }, |
| { |
| "name": "transformer.h.0.mlp.w_gate.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00597, |
| "zero_fraction": 0.00816 |
| }, |
| { |
| "name": "transformer.h.0.mlp.w_up.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00596, |
| "zero_fraction": 0.00822 |
| }, |
| { |
| "name": "transformer.h.1.attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.1.attn.k_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00599, |
| "zero_fraction": 0.00838 |
| }, |
| { |
| "name": "transformer.h.1.attn.o_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00598, |
| "zero_fraction": 0.00844 |
| }, |
| { |
| "name": "transformer.h.1.attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.1.attn.q_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00604, |
| "zero_fraction": 0.00813 |
| }, |
| { |
| "name": "transformer.h.1.attn.v_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00597, |
| "zero_fraction": 0.00897 |
| }, |
| { |
| "name": "transformer.h.1.ln_1.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.1.ln_2.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.1.mlp.w_down.weight", |
| "shape": [ |
| 384, |
| 1024 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.006, |
| "zero_fraction": 0.00813 |
| }, |
| { |
| "name": "transformer.h.1.mlp.w_gate.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00595, |
| "zero_fraction": 0.00814 |
| }, |
| { |
| "name": "transformer.h.1.mlp.w_up.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00597, |
| "zero_fraction": 0.00805 |
| }, |
| { |
| "name": "transformer.h.10.attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.10.attn.k_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00607, |
| "zero_fraction": 0.00848 |
| }, |
| { |
| "name": "transformer.h.10.attn.o_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00592, |
| "zero_fraction": 0.00844 |
| }, |
| { |
| "name": "transformer.h.10.attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.10.attn.q_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00594, |
| "zero_fraction": 0.00785 |
| }, |
| { |
| "name": "transformer.h.10.attn.v_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00605, |
| "zero_fraction": 0.00832 |
| }, |
| { |
| "name": "transformer.h.10.ln_1.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.10.ln_2.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.10.mlp.w_down.weight", |
| "shape": [ |
| 384, |
| 1024 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.006, |
| "zero_fraction": 0.00855 |
| }, |
| { |
| "name": "transformer.h.10.mlp.w_gate.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00592, |
| "zero_fraction": 0.00817 |
| }, |
| { |
| "name": "transformer.h.10.mlp.w_up.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00594, |
| "zero_fraction": 0.00822 |
| }, |
| { |
| "name": "transformer.h.11.attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.11.attn.k_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00626, |
| "zero_fraction": 0.00885 |
| }, |
| { |
| "name": "transformer.h.11.attn.o_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00597, |
| "zero_fraction": 0.00777 |
| }, |
| { |
| "name": "transformer.h.11.attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.11.attn.q_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00595, |
| "zero_fraction": 0.0084 |
| }, |
| { |
| "name": "transformer.h.11.attn.v_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00596, |
| "zero_fraction": 0.00791 |
| }, |
| { |
| "name": "transformer.h.11.ln_1.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.11.ln_2.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.11.mlp.w_down.weight", |
| "shape": [ |
| 384, |
| 1024 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00601, |
| "zero_fraction": 0.0082 |
| }, |
| { |
| "name": "transformer.h.11.mlp.w_gate.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00593, |
| "zero_fraction": 0.00824 |
| }, |
| { |
| "name": "transformer.h.11.mlp.w_up.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00595, |
| "zero_fraction": 0.00834 |
| }, |
| { |
| "name": "transformer.h.12.attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.12.attn.k_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00615, |
| "zero_fraction": 0.0084 |
| }, |
| { |
| "name": "transformer.h.12.attn.o_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00601, |
| "zero_fraction": 0.00823 |
| }, |
| { |
| "name": "transformer.h.12.attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.12.attn.q_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00596, |
| "zero_fraction": 0.00864 |
| }, |
| { |
| "name": "transformer.h.12.attn.v_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.006, |
| "zero_fraction": 0.00781 |
| }, |
| { |
| "name": "transformer.h.12.ln_1.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.12.ln_2.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.12.mlp.w_down.weight", |
| "shape": [ |
| 384, |
| 1024 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00601, |
| "zero_fraction": 0.00821 |
| }, |
| { |
| "name": "transformer.h.12.mlp.w_gate.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00596, |
| "zero_fraction": 0.00817 |
| }, |
| { |
| "name": "transformer.h.12.mlp.w_up.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00598, |
| "zero_fraction": 0.00801 |
| }, |
| { |
| "name": "transformer.h.13.attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.13.attn.k_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00599, |
| "zero_fraction": 0.00773 |
| }, |
| { |
| "name": "transformer.h.13.attn.o_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00606, |
| "zero_fraction": 0.00863 |
| }, |
| { |
| "name": "transformer.h.13.attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.13.attn.q_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00598, |
| "zero_fraction": 0.00827 |
| }, |
| { |
| "name": "transformer.h.13.attn.v_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00606, |
| "zero_fraction": 0.00928 |
| }, |
| { |
| "name": "transformer.h.13.ln_1.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.13.ln_2.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.13.mlp.w_down.weight", |
| "shape": [ |
| 384, |
| 1024 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00614, |
| "zero_fraction": 0.00856 |
| }, |
| { |
| "name": "transformer.h.13.mlp.w_gate.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00592, |
| "zero_fraction": 0.00835 |
| }, |
| { |
| "name": "transformer.h.13.mlp.w_up.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00596, |
| "zero_fraction": 0.00825 |
| }, |
| { |
| "name": "transformer.h.2.attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.2.attn.k_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00601, |
| "zero_fraction": 0.00887 |
| }, |
| { |
| "name": "transformer.h.2.attn.o_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00598, |
| "zero_fraction": 0.00812 |
| }, |
| { |
| "name": "transformer.h.2.attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.2.attn.q_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00595, |
| "zero_fraction": 0.00829 |
| }, |
| { |
| "name": "transformer.h.2.attn.v_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00597, |
| "zero_fraction": 0.00893 |
| }, |
| { |
| "name": "transformer.h.2.ln_1.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.2.ln_2.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.2.mlp.w_down.weight", |
| "shape": [ |
| 384, |
| 1024 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00598, |
| "zero_fraction": 0.00798 |
| }, |
| { |
| "name": "transformer.h.2.mlp.w_gate.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00595, |
| "zero_fraction": 0.00834 |
| }, |
| { |
| "name": "transformer.h.2.mlp.w_up.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00596, |
| "zero_fraction": 0.00831 |
| }, |
| { |
| "name": "transformer.h.3.attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.3.attn.k_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00608, |
| "zero_fraction": 0.00834 |
| }, |
| { |
| "name": "transformer.h.3.attn.o_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00592, |
| "zero_fraction": 0.00808 |
| }, |
| { |
| "name": "transformer.h.3.attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.3.attn.q_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00604, |
| "zero_fraction": 0.00793 |
| }, |
| { |
| "name": "transformer.h.3.attn.v_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00602, |
| "zero_fraction": 0.00958 |
| }, |
| { |
| "name": "transformer.h.3.ln_1.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.3.ln_2.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.3.mlp.w_down.weight", |
| "shape": [ |
| 384, |
| 1024 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00597, |
| "zero_fraction": 0.00831 |
| }, |
| { |
| "name": "transformer.h.3.mlp.w_gate.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00595, |
| "zero_fraction": 0.00805 |
| }, |
| { |
| "name": "transformer.h.3.mlp.w_up.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00596, |
| "zero_fraction": 0.00792 |
| }, |
| { |
| "name": "transformer.h.4.attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.4.attn.k_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00593, |
| "zero_fraction": 0.00832 |
| }, |
| { |
| "name": "transformer.h.4.attn.o_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00596, |
| "zero_fraction": 0.00853 |
| }, |
| { |
| "name": "transformer.h.4.attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.4.attn.q_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00596, |
| "zero_fraction": 0.00835 |
| }, |
| { |
| "name": "transformer.h.4.attn.v_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00594, |
| "zero_fraction": 0.0084 |
| }, |
| { |
| "name": "transformer.h.4.ln_1.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.4.ln_2.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.4.mlp.w_down.weight", |
| "shape": [ |
| 384, |
| 1024 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00598, |
| "zero_fraction": 0.00822 |
| }, |
| { |
| "name": "transformer.h.4.mlp.w_gate.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00596, |
| "zero_fraction": 0.00856 |
| }, |
| { |
| "name": "transformer.h.4.mlp.w_up.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00598, |
| "zero_fraction": 0.00826 |
| }, |
| { |
| "name": "transformer.h.5.attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.5.attn.k_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00608, |
| "zero_fraction": 0.0083 |
| }, |
| { |
| "name": "transformer.h.5.attn.o_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00593, |
| "zero_fraction": 0.0081 |
| }, |
| { |
| "name": "transformer.h.5.attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.5.attn.q_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00593, |
| "zero_fraction": 0.00823 |
| }, |
| { |
| "name": "transformer.h.5.attn.v_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00599, |
| "zero_fraction": 0.00869 |
| }, |
| { |
| "name": "transformer.h.5.ln_1.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.5.ln_2.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.5.mlp.w_down.weight", |
| "shape": [ |
| 384, |
| 1024 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00602, |
| "zero_fraction": 0.00835 |
| }, |
| { |
| "name": "transformer.h.5.mlp.w_gate.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00602, |
| "zero_fraction": 0.00861 |
| }, |
| { |
| "name": "transformer.h.5.mlp.w_up.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00599, |
| "zero_fraction": 0.00843 |
| }, |
| { |
| "name": "transformer.h.6.attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.6.attn.k_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00607, |
| "zero_fraction": 0.00883 |
| }, |
| { |
| "name": "transformer.h.6.attn.o_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00602, |
| "zero_fraction": 0.00898 |
| }, |
| { |
| "name": "transformer.h.6.attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.6.attn.q_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00596, |
| "zero_fraction": 0.00821 |
| }, |
| { |
| "name": "transformer.h.6.attn.v_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00603, |
| "zero_fraction": 0.00903 |
| }, |
| { |
| "name": "transformer.h.6.ln_1.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.6.ln_2.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.6.mlp.w_down.weight", |
| "shape": [ |
| 384, |
| 1024 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00608, |
| "zero_fraction": 0.00835 |
| }, |
| { |
| "name": "transformer.h.6.mlp.w_gate.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00597, |
| "zero_fraction": 0.00808 |
| }, |
| { |
| "name": "transformer.h.6.mlp.w_up.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00598, |
| "zero_fraction": 0.00831 |
| }, |
| { |
| "name": "transformer.h.7.attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.7.attn.k_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00628, |
| "zero_fraction": 0.00859 |
| }, |
| { |
| "name": "transformer.h.7.attn.o_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00591, |
| "zero_fraction": 0.00831 |
| }, |
| { |
| "name": "transformer.h.7.attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.7.attn.q_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00603, |
| "zero_fraction": 0.00844 |
| }, |
| { |
| "name": "transformer.h.7.attn.v_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00617, |
| "zero_fraction": 0.00897 |
| }, |
| { |
| "name": "transformer.h.7.ln_1.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.7.ln_2.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.7.mlp.w_down.weight", |
| "shape": [ |
| 384, |
| 1024 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00604, |
| "zero_fraction": 0.00819 |
| }, |
| { |
| "name": "transformer.h.7.mlp.w_gate.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00598, |
| "zero_fraction": 0.00854 |
| }, |
| { |
| "name": "transformer.h.7.mlp.w_up.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00597, |
| "zero_fraction": 0.00831 |
| }, |
| { |
| "name": "transformer.h.8.attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.8.attn.k_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00603, |
| "zero_fraction": 0.00773 |
| }, |
| { |
| "name": "transformer.h.8.attn.o_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00599, |
| "zero_fraction": 0.00867 |
| }, |
| { |
| "name": "transformer.h.8.attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.8.attn.q_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00593, |
| "zero_fraction": 0.00841 |
| }, |
| { |
| "name": "transformer.h.8.attn.v_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00609, |
| "zero_fraction": 0.0095 |
| }, |
| { |
| "name": "transformer.h.8.ln_1.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.8.ln_2.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.8.mlp.w_down.weight", |
| "shape": [ |
| 384, |
| 1024 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.006, |
| "zero_fraction": 0.00848 |
| }, |
| { |
| "name": "transformer.h.8.mlp.w_gate.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00595, |
| "zero_fraction": 0.00816 |
| }, |
| { |
| "name": "transformer.h.8.mlp.w_up.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00594, |
| "zero_fraction": 0.0081 |
| }, |
| { |
| "name": "transformer.h.9.attn.k_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.9.attn.k_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00618, |
| "zero_fraction": 0.0084 |
| }, |
| { |
| "name": "transformer.h.9.attn.o_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00597, |
| "zero_fraction": 0.00798 |
| }, |
| { |
| "name": "transformer.h.9.attn.q_norm.weight", |
| "shape": [ |
| 64 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.9.attn.q_proj.weight", |
| "shape": [ |
| 384, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00596, |
| "zero_fraction": 0.00831 |
| }, |
| { |
| "name": "transformer.h.9.attn.v_proj.weight", |
| "shape": [ |
| 128, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00606, |
| "zero_fraction": 0.0096 |
| }, |
| { |
| "name": "transformer.h.9.ln_1.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.9.ln_2.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.h.9.mlp.w_down.weight", |
| "shape": [ |
| 384, |
| 1024 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00602, |
| "zero_fraction": 0.00851 |
| }, |
| { |
| "name": "transformer.h.9.mlp.w_gate.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00594, |
| "zero_fraction": 0.00808 |
| }, |
| { |
| "name": "transformer.h.9.mlp.w_up.weight", |
| "shape": [ |
| 1024, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00594, |
| "zero_fraction": 0.00827 |
| }, |
| { |
| "name": "transformer.ln_f.weight", |
| "shape": [ |
| 384 |
| ], |
| "quantized": false, |
| "reason": "not a matrix", |
| "relative_error": 0.0, |
| "zero_fraction": 0.0 |
| }, |
| { |
| "name": "transformer.wte.weight", |
| "shape": [ |
| 8192, |
| 384 |
| ], |
| "quantized": true, |
| "reason": "8-bit", |
| "relative_error": 0.00606, |
| "zero_fraction": 0.00844 |
| } |
| ] |
| } |
|
|
